| author | |
| committer | |
| log | f82b44aebb7048af93617305487961e7659f112c |
| tree | e5f17048898f7c93453dc1c5a93eca69a1547a10 |
| parent | a235692734ea58ecdc64a0afc94800d595be91e2 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Share the relay runtime across Fedora and Snow Globe, with certificate-backed UDP discovery alongside Caddy HTTPS. Use private systemd credentials and Certbot renewal on r1, and a DNS certificate sidecar with live certificate reload on r2. Keep previews on dynamic UDP ports and extend the persistent public-port mappings for production.
Verified Iroh 1.3 relay-only transfers on both hosts, trusted-TLS UDP address discovery on r1 and the isolated r2 preview, r1 certificate-renewal rehearsal, Nix firewall/NAT evaluation, and seven deployment tests.
Assisted-by: gpt-6-astra10 files changed, 188 insertions(+), 22 deletions(-)
nixos/snowglobe.nix+5-4| ... | @@ -5,6 +5,7 @@ let | ... | @@ -5,6 +5,7 @@ let |
| 5 | xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json); | 5 | xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json); |
| 6 | xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; | 6 | xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; |
| 7 | xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; | 7 | xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; |
| 8 | irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); | ||
| 8 | encryptionRoots = { | 9 | encryptionRoots = { |
| 9 | apps = "apps"; | 10 | apps = "apps"; |
| 10 | clover = "clover"; | 11 | clover = "clover"; |
| ... | @@ -24,7 +25,7 @@ in | ... | @@ -24,7 +25,7 @@ in |
| 24 | networking.hostId = "4fa19ccb"; | 25 | networking.hostId = "4fa19ccb"; |
| 25 | services.nomad.settings.name = config.networking.hostName; | 26 | services.nomad.settings.name = config.networking.hostName; |
| 26 | networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; | 27 | networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; |
| 27 | networking.firewall.allowedUDPPorts = xmppUDPPorts; | 28 | networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ]; |
| 28 | networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; | 29 | networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; |
| 29 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' | 30 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' |
| 30 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept | 31 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept |
| ... | @@ -34,7 +35,7 @@ in | ... | @@ -34,7 +35,7 @@ in |
| 34 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept | 35 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept |
| 35 | ''; | 36 | ''; |
| 36 | 37 | ||
| 37 | systemd.services.xmpp-nat = { | 38 | systemd.services.public-ports = { |
| 38 | wantedBy = [ "multi-user.target" ]; | 39 | wantedBy = [ "multi-user.target" ]; |
| 39 | wants = [ "network-online.target" ]; | 40 | wants = [ "network-online.target" ]; |
| 40 | after = [ "network-online.target" ]; | 41 | after = [ "network-online.target" ]; |
| ... | @@ -42,9 +43,9 @@ in | ... | @@ -42,9 +43,9 @@ in |
| 42 | serviceConfig = { Type = "oneshot"; DynamicUser = true; }; | 43 | serviceConfig = { Type = "oneshot"; DynamicUser = true; }; |
| 43 | script = '' | 44 | script = '' |
| 44 | description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" | 45 | description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" |
| 45 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe XMPP" -r \ | 46 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ |
| 46 | ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ | 47 | ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ |
| 47 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} | 48 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} |
| 48 | ''; | 49 | ''; |
| 49 | }; | 50 | }; |
| 50 | 51 |
service/iroh-relay/Dockerfile+5-2| ... | @@ -1,6 +1,9 @@ | ... | @@ -1,6 +1,9 @@ |
| 1 | FROM docker.io/library/alpine:3.22@sha256:3e9b4b680bfc9fb5269227cffbd6d42be39fbf7c0b908123913864aa4447e764 | 1 | FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332 |
| 2 | RUN apt-get update && apt-get install -y --no-install-recommends certbot python3-certbot-dns-cloudflare \ | ||
| 3 | && rm -rf /var/lib/apt/lists/* | ||
| 2 | ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz | 4 | ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz |
| 3 | RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \ | 5 | RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \ |
| 4 | && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \ | 6 | && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \ |
| 5 | && rm /tmp/iroh-relay.tar.gz | 7 | && rm /tmp/iroh-relay.tar.gz |
| 6 | ENTRYPOINT ["/usr/local/bin/iroh-relay"] | 8 | COPY run.py network.json /app/ |
| 9 | ENTRYPOINT ["python3", "/app/run.py"] |
service/iroh-relay/iroh-relay.service+4-1| ... | @@ -5,7 +5,10 @@ After=network-online.target | ... | @@ -5,7 +5,10 @@ After=network-online.target |
| 5 | 5 | ||
| 6 | [Service] | 6 | [Service] |
| 7 | DynamicUser=true | 7 | DynamicUser=true |
| 8 | ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml | 8 | Environment=HTTP_PORT=3340 |
| 9 | LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem | ||
| 10 | LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem | ||
| 11 | ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay | ||
| 9 | Restart=on-failure | 12 | Restart=on-failure |
| 10 | RestartSec=5 | 13 | RestartSec=5 |
| 11 | NoNewPrivileges=true | 14 | NoNewPrivileges=true |
service/iroh-relay/network.json created+1| ... | @@ -0,0 +1 @@ | ||
| 1 | {"quic":7842} | ||
service/iroh-relay/r1.caddy+7-1| ... | @@ -1,3 +1,9 @@ | ... | @@ -1,3 +1,9 @@ |
| 1 | r1.iroh.paperclover.net { | 1 | r1.iroh.paperclover.net { |
| 2 | reverse_proxy 127.0.0.1:3340 | 2 | handle /.well-known/acme-challenge/* { |
| 3 | root * /var/lib/iroh-acme | ||
| 4 | file_server | ||
| 5 | } | ||
| 6 | handle { | ||
| 7 | reverse_proxy 127.0.0.1:3340 | ||
| 8 | } | ||
| 3 | } | 9 | } |
service/iroh-relay/relay.toml deleted-2| ... | @@ -1,2 +0,0 @@ | ||
| 1 | http_bind_addr = "0.0.0.0:3340" | ||
| 2 | enable_metrics = false | ||
service/iroh-relay/run.py created+50| ... | @@ -0,0 +1,50 @@ | ||
| 1 | #!/usr/bin/env python3 | ||
| 2 | import json | ||
| 3 | import os | ||
| 4 | from pathlib import Path | ||
| 5 | import subprocess | ||
| 6 | import sys | ||
| 7 | import time | ||
| 8 | |||
| 9 | |||
| 10 | os.umask(0o077) | ||
| 11 | mode = sys.argv[1] | ||
| 12 | tls = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/tls/live/server")) | ||
| 13 | |||
| 14 | if mode == "certificates": | ||
| 15 | credentials = Path("/tmp/cloudflare.ini") | ||
| 16 | credentials.write_text("dns_cloudflare_api_token = " + os.environ.pop("DNS_TOKEN") + "\n") | ||
| 17 | command = [ | ||
| 18 | "certbot", "certonly", "--non-interactive", "--agree-tos", | ||
| 19 | "--email", os.environ["OWNER_EMAIL"], "--dns-cloudflare", | ||
| 20 | "--dns-cloudflare-credentials", str(credentials), | ||
| 21 | "--dns-cloudflare-propagation-seconds", "30", "--cert-name", "server", | ||
| 22 | "--config-dir", "/tls", "--work-dir", "/tmp/certbot-work", "--logs-dir", "/tmp/certbot-logs", | ||
| 23 | "--keep-until-expiring", "-d", os.environ["RELAY_HOST"], | ||
| 24 | ] | ||
| 25 | while True: | ||
| 26 | subprocess.run(command, check=not (tls / "fullchain.pem").exists(), timeout=300) | ||
| 27 | time.sleep(12 * 60 * 60) | ||
| 28 | |||
| 29 | config = Path("/tmp/iroh-relay.toml") | ||
| 30 | if mode == "relay": | ||
| 31 | for _ in range(180): | ||
| 32 | if (tls / "fullchain.pem").exists() and (tls / "privkey.pem").exists(): | ||
| 33 | break | ||
| 34 | time.sleep(1) | ||
| 35 | else: | ||
| 36 | raise RuntimeError("Relay certificate was not issued within three minutes") | ||
| 37 | quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( | ||
| 38 | Path(__file__).with_name("network.json").read_text())["quic"] | ||
| 39 | config.write_text( | ||
| 40 | 'enable_metrics = false\nenable_quic_addr_discovery = true\n' | ||
| 41 | f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' | ||
| 42 | '[tls]\ncert_mode = "Reloading"\n' | ||
| 43 | f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' | ||
| 44 | f'manual_cert_path = {json.dumps(str(tls / "fullchain.pem"))}\n' | ||
| 45 | f'manual_key_path = {json.dumps(str(tls / "privkey.pem"))}\n' | ||
| 46 | ) | ||
| 47 | else: | ||
| 48 | raise ValueError("Unknown relay task") | ||
| 49 | # --dev leaves HTTPS to Caddy while retaining certificate-backed QUIC discovery. | ||
| 50 | os.execv("/usr/local/bin/iroh-relay", ["iroh-relay", "--dev", "--config-path", str(config)]) | ||
service/iroh-relay/service.pkl+47-11| ... | @@ -1,19 +1,55 @@ | ... | @@ -1,19 +1,55 @@ |
| 1 | amends "../../config/Service.pkl" | 1 | amends "../../config/Service.pkl" |
| 2 | 2 | ||
| 3 | import "../../config/site.pkl" as site | ||
| 4 | import "../ddns-updater/service.pkl" as dns | ||
| 5 | import "pkl:json" | ||
| 6 | |||
| 7 | local network = new json.Parser {}.parse(read("network.json").text) | ||
| 8 | local host = read?("prop:hostname") ?? "r2.iroh.\(site.domain)" | ||
| 9 | |||
| 3 | name = "Iroh relay" | 10 | name = "Iroh relay" |
| 4 | launcher = false | 11 | launcher = false |
| 5 | stageIsolation = "fresh" | 12 | stageIsolation = "fresh" |
| 13 | requirements { new dns.DnsCredentials {} } | ||
| 6 | 14 | ||
| 7 | container { | 15 | containers { |
| 8 | build = "." | 16 | ["app"] { |
| 9 | readOnlyRoot = true | 17 | build = "." |
| 10 | http { | 18 | hostNetwork = true |
| 11 | containerPort = 3340 | 19 | readOnlyRoot = true |
| 12 | subdomain = "r2.iroh" | 20 | tmpfs { "/tmp" } |
| 13 | checkPath = "/healthz" | 21 | args { "relay" } |
| 22 | http { containerPort = 3340; hostname = host; loopback = true; checkPath = "/healthz" } | ||
| 23 | tcp { | ||
| 24 | new { | ||
| 25 | name = "quic" | ||
| 26 | containerPort = network.quic | ||
| 27 | hostPort = if (site.preview) null else network.quic | ||
| 28 | loopback = false | ||
| 29 | } | ||
| 30 | } | ||
| 31 | volumes { ["/tls"] { readOnly = true } } | ||
| 32 | envTemplate = """ | ||
| 33 | HTTP_PORT={{ env "NOMAD_PORT_app_http" }} | ||
| 34 | QUIC_PORT={{ env "NOMAD_PORT_app_quic" }} | ||
| 35 | """ | ||
| 36 | cpu = 200 | ||
| 37 | memory = 256 | ||
| 38 | } | ||
| 39 | ["certificates"] { | ||
| 40 | build = "." | ||
| 41 | hostNetwork = true | ||
| 42 | readOnlyRoot = true | ||
| 43 | tmpfs { "/tmp" } | ||
| 44 | lifecycle = "prestartSidecar" | ||
| 45 | args { "certificates" } | ||
| 46 | volumes { ["/tls"] {} } | ||
| 47 | env { | ||
| 48 | ["RELAY_HOST"] = host | ||
| 49 | ["OWNER_EMAIL"] = site.ownerEmail | ||
| 50 | ["DNS_TOKEN"] = "${secret.dns.api_token}" | ||
| 51 | } | ||
| 52 | cpu = 100 | ||
| 53 | memory = 256 | ||
| 14 | } | 54 | } |
| 15 | args { "--config-path"; "/etc/iroh-relay.toml" } | ||
| 16 | volumes { ["/etc/iroh-relay.toml"] { config = "relay.toml" } } | ||
| 17 | cpu = 200 | ||
| 18 | memory = 256 | ||
| 19 | } | 55 | } |
tools/iroh-relay.md created+68| ... | @@ -0,0 +1,68 @@ | ||
| 1 | # Iroh relays | ||
| 2 | |||
| 3 | Clients use `https://r1.iroh.paperclover.net` and | ||
| 4 | `https://r2.iroh.paperclover.net`. Caddy terminates HTTPS on TCP 443; Iroh serves | ||
| 5 | QUIC address discovery on UDP 7842. TCP 80 remains available for ACME. | ||
| 6 | These are public relays without an application login. | ||
| 7 | |||
| 8 | The upstream binary version and checksum live in | ||
| 9 | [`Dockerfile`](../service/iroh-relay/Dockerfile). Both machines use | ||
| 10 | [`run.py`](../service/iroh-relay/run.py) and | ||
| 11 | [`network.json`](../service/iroh-relay/network.json). | ||
| 12 | Iroh's `--dev` flag disables its HTTPS listener behind Caddy, while its | ||
| 13 | `Reloading` certificate resolver still supplies valid TLS for UDP discovery. | ||
| 14 | |||
| 15 | ## Snow Globe | ||
| 16 | |||
| 17 | [`service.pkl`](../service/iroh-relay/service.pkl) declares the relay and its | ||
| 18 | certificate sidecar. The sidecar obtains the Cloudflare credential through the | ||
| 19 | DDNS provider, keeps it in a private tmpfs file, and checks certificate renewal | ||
| 20 | every twelve hours. Only the certificate volume is shared with the relay. | ||
| 21 | The relay reloads renewed certificates without restarting connections. | ||
| 22 | |||
| 23 | Stage and deploy with the normal `tools/deploy.py` workflow. Fresh previews have | ||
| 24 | their own certificate storage and dynamically allocated UDP port; they do not | ||
| 25 | bind production's port. The host's `public-ports.service` maintains the production | ||
| 26 | router mapping; the firewall declaration is in `nixos/snowglobe.nix`. | ||
| 27 | |||
| 28 | ## paperclover.net | ||
| 29 | |||
| 30 | Connect with `ssh root@paperclover.net`; the hostname is `usw-server-1`. | ||
| 31 | The checksum-verified upstream binary is installed at `/usr/local/bin/iroh-relay`. | ||
| 32 | Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and | ||
| 33 | [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into | ||
| 34 | `/etc/systemd/system/`. | ||
| 35 | |||
| 36 | [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as | ||
| 37 | `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main | ||
| 38 | Caddyfile. Validate before reloading Caddy. The ACME webroot is | ||
| 39 | `/var/lib/iroh-acme`; Caddy must be able to read it. | ||
| 40 | |||
| 41 | After Caddy's HTTPS certificate is ready, issue the discovery certificate: | ||
| 42 | |||
| 43 | ```sh | ||
| 44 | certbot certonly --non-interactive --agree-tos --email account@paperclover.net \ | ||
| 45 | --webroot -w /var/lib/iroh-acme --cert-name iroh-relay \ | ||
| 46 | -d r1.iroh.paperclover.net \ | ||
| 47 | --deploy-hook 'systemctl try-restart iroh-relay.service' | ||
| 48 | systemctl daemon-reload | ||
| 49 | systemctl enable --now iroh-relay.service certbot-renew.timer | ||
| 50 | firewall-cmd --add-port=7842/udp | ||
| 51 | firewall-cmd --permanent --add-port=7842/udp | ||
| 52 | ``` | ||
| 53 | |||
| 54 | Systemd loads the certificate and key as private service credentials. Certbot's | ||
| 55 | renewal hook restarts the service to refresh those credentials. The key stays | ||
| 56 | root-only under `/etc/letsencrypt`. | ||
| 57 | |||
| 58 | ## Verification | ||
| 59 | |||
| 60 | `GET /healthz` reports the relay version, but does not test packet forwarding. | ||
| 61 | Use two Iroh 1.3 endpoints with `clear_ip_transports()` and a custom relay map; | ||
| 62 | transfer and verify a payload in both directions. For UDP, connect a | ||
| 63 | `iroh_relay::quic::QuicClient` with normal CA validation and await its | ||
| 64 | `observed_external_addr()` stream. Test from outside the server's network. | ||
| 65 | |||
| 66 | Renewal can be exercised on r1 with | ||
| 67 | `certbot renew --cert-name iroh-relay --dry-run --run-deploy-hooks`. | ||
| 68 | Inspect the relay's restart count and health afterward. | ||
tools/xmpp.md+1-1| ... | @@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide | ... | @@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide |
| 24 | provisions client, direct-TLS client, and federation SRV records pointing at | 24 | provisions client, direct-TLS client, and federation SRV records pointing at |
| 25 | `xmpp.paperclover.net`; the website keeps its existing apex address. Router | 25 | `xmpp.paperclover.net`; the website keeps its existing apex address. Router |
| 26 | forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range | 26 | forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range |
| 27 | to Snow Globe. `xmpp-nat.service` refreshes UPnP mappings on startup and every five | 27 | to Snow Globe. `public-ports.service` refreshes UPnP mappings on startup and every five |
| 28 | minutes, discovering the current description URL with a bounded unicast SSDP | 28 | minutes, discovering the current description URL with a bounded unicast SSDP |
| 29 | request to the configured gateway on port 1900. Replies and HTTP description | 29 | request to the configured gateway on port 1900. Replies and HTTP description |
| 30 | hosts must match that gateway; discovery failure fails the refresh. Multicast | 30 | hosts must match that gateway; discovery failure fails the refresh. Multicast |