authoravatarclover caruso<git@paperclover.net>2026-10-20 01:41:33-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logf82b44aebb7048af93617305487961e7659f112c
treee5f17048898f7c93453dc1c5a93eca69a1547a10
parenta235692734ea58ecdc64a0afc94800d595be91e2
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Enable QUIC address discovery on both Iroh relays

Share the relay runtime across Fedora and Snow Globe, with certificate-backed UDP discovery alongside Caddy HTTPS. Use private systemd credentials and Certbot renewal on r1, and a DNS certificate sidecar with live certificate reload on r2. Keep previews on dynamic UDP ports and extend the persistent public-port mappings for production. Verified Iroh 1.3 relay-only transfers on both hosts, trusted-TLS UDP address discovery on r1 and the isolated r2 preview, r1 certificate-renewal rehearsal, Nix firewall/NAT evaluation, and seven deployment tests. Assisted-by: gpt-6-astra

10 files changed, 188 insertions(+), 22 deletions(-)

nixos/snowglobe.nix+5-4
...@@ -5,6 +5,7 @@ let...@@ -5,6 +5,7 @@ let
5 xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json);5 xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json);
6 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];6 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];
7 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];7 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];
8 irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json);
8 encryptionRoots = {9 encryptionRoots = {
9 apps = "apps";10 apps = "apps";
10 clover = "clover";11 clover = "clover";
...@@ -24,7 +25,7 @@ in...@@ -24,7 +25,7 @@ in
24 networking.hostId = "4fa19ccb";25 networking.hostId = "4fa19ccb";
25 services.nomad.settings.name = config.networking.hostName;26 services.nomad.settings.name = config.networking.hostName;
26 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;27 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;
27 networking.firewall.allowedUDPPorts = xmppUDPPorts;28 networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ];
28 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];29 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];
29 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''30 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
30 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept31 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
...@@ -34,7 +35,7 @@ in...@@ -34,7 +35,7 @@ in
34 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept35 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
35 '';36 '';
3637
37 systemd.services.xmpp-nat = {38 systemd.services.public-ports = {
38 wantedBy = [ "multi-user.target" ];39 wantedBy = [ "multi-user.target" ];
39 wants = [ "network-online.target" ];40 wants = [ "network-online.target" ];
40 after = [ "network-online.target" ];41 after = [ "network-online.target" ];
...@@ -42,9 +43,9 @@ in...@@ -42,9 +43,9 @@ in
42 serviceConfig = { Type = "oneshot"; DynamicUser = true; };43 serviceConfig = { Type = "oneshot"; DynamicUser = true; };
43 script = ''44 script = ''
44 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"45 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"
45 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe XMPP" -r \46 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \
46 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \47 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \
47 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}48 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
48 '';49 '';
49 };50 };
5051
service/iroh-relay/Dockerfile+5-2
...@@ -1,6 +1,9 @@...@@ -1,6 +1,9 @@
1FROM docker.io/library/alpine:3.22@sha256:3e9b4b680bfc9fb5269227cffbd6d42be39fbf7c0b908123913864aa4447e7641FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332
2RUN apt-get update && apt-get install -y --no-install-recommends certbot python3-certbot-dns-cloudflare \
3 && rm -rf /var/lib/apt/lists/*
2ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz4ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz
3RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \5RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \
4 && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \6 && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \
5 && rm /tmp/iroh-relay.tar.gz7 && rm /tmp/iroh-relay.tar.gz
6ENTRYPOINT ["/usr/local/bin/iroh-relay"]8COPY run.py network.json /app/
9ENTRYPOINT ["python3", "/app/run.py"]
service/iroh-relay/iroh-relay.service+4-1
...@@ -5,7 +5,10 @@ After=network-online.target...@@ -5,7 +5,10 @@ After=network-online.target
55
6[Service]6[Service]
7DynamicUser=true7DynamicUser=true
8ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml8Environment=HTTP_PORT=3340
9LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem
10LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem
11ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay
9Restart=on-failure12Restart=on-failure
10RestartSec=513RestartSec=5
11NoNewPrivileges=true14NoNewPrivileges=true
service/iroh-relay/network.json created+1
...@@ -0,0 +1 @@
1{"quic":7842}
service/iroh-relay/r1.caddy+7-1
...@@ -1,3 +1,9 @@...@@ -1,3 +1,9 @@
1r1.iroh.paperclover.net {1r1.iroh.paperclover.net {
2 reverse_proxy 127.0.0.1:33402 handle /.well-known/acme-challenge/* {
3 root * /var/lib/iroh-acme
4 file_server
5 }
6 handle {
7 reverse_proxy 127.0.0.1:3340
8 }
3}9}
service/iroh-relay/relay.toml deleted-2
...@@ -1,2 +0,0 @@
1http_bind_addr = "0.0.0.0:3340"
2enable_metrics = false
service/iroh-relay/run.py created+50
...@@ -0,0 +1,50 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import subprocess
6import sys
7import time
8
9
10os.umask(0o077)
11mode = sys.argv[1]
12tls = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/tls/live/server"))
13
14if mode == "certificates":
15 credentials = Path("/tmp/cloudflare.ini")
16 credentials.write_text("dns_cloudflare_api_token = " + os.environ.pop("DNS_TOKEN") + "\n")
17 command = [
18 "certbot", "certonly", "--non-interactive", "--agree-tos",
19 "--email", os.environ["OWNER_EMAIL"], "--dns-cloudflare",
20 "--dns-cloudflare-credentials", str(credentials),
21 "--dns-cloudflare-propagation-seconds", "30", "--cert-name", "server",
22 "--config-dir", "/tls", "--work-dir", "/tmp/certbot-work", "--logs-dir", "/tmp/certbot-logs",
23 "--keep-until-expiring", "-d", os.environ["RELAY_HOST"],
24 ]
25 while True:
26 subprocess.run(command, check=not (tls / "fullchain.pem").exists(), timeout=300)
27 time.sleep(12 * 60 * 60)
28
29config = Path("/tmp/iroh-relay.toml")
30if mode == "relay":
31 for _ in range(180):
32 if (tls / "fullchain.pem").exists() and (tls / "privkey.pem").exists():
33 break
34 time.sleep(1)
35 else:
36 raise RuntimeError("Relay certificate was not issued within three minutes")
37 quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads(
38 Path(__file__).with_name("network.json").read_text())["quic"]
39 config.write_text(
40 'enable_metrics = false\nenable_quic_addr_discovery = true\n'
41 f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n'
42 '[tls]\ncert_mode = "Reloading"\n'
43 f'quic_bind_addr = "0.0.0.0:{quic_port}"\n'
44 f'manual_cert_path = {json.dumps(str(tls / "fullchain.pem"))}\n'
45 f'manual_key_path = {json.dumps(str(tls / "privkey.pem"))}\n'
46 )
47else:
48 raise ValueError("Unknown relay task")
49# --dev leaves HTTPS to Caddy while retaining certificate-backed QUIC discovery.
50os.execv("/usr/local/bin/iroh-relay", ["iroh-relay", "--dev", "--config-path", str(config)])
service/iroh-relay/service.pkl+47-11
...@@ -1,19 +1,55 @@...@@ -1,19 +1,55 @@
1amends "../../config/Service.pkl"1amends "../../config/Service.pkl"
22
3import "../../config/site.pkl" as site
4import "../ddns-updater/service.pkl" as dns
5import "pkl:json"
6
7local network = new json.Parser {}.parse(read("network.json").text)
8local host = read?("prop:hostname") ?? "r2.iroh.\(site.domain)"
9
3name = "Iroh relay"10name = "Iroh relay"
4launcher = false11launcher = false
5stageIsolation = "fresh"12stageIsolation = "fresh"
13requirements { new dns.DnsCredentials {} }
614
7container {15containers {
8 build = "."16 ["app"] {
9 readOnlyRoot = true17 build = "."
10 http {18 hostNetwork = true
11 containerPort = 334019 readOnlyRoot = true
12 subdomain = "r2.iroh"20 tmpfs { "/tmp" }
13 checkPath = "/healthz"21 args { "relay" }
22 http { containerPort = 3340; hostname = host; loopback = true; checkPath = "/healthz" }
23 tcp {
24 new {
25 name = "quic"
26 containerPort = network.quic
27 hostPort = if (site.preview) null else network.quic
28 loopback = false
29 }
30 }
31 volumes { ["/tls"] { readOnly = true } }
32 envTemplate = """
33 HTTP_PORT={{ env "NOMAD_PORT_app_http" }}
34 QUIC_PORT={{ env "NOMAD_PORT_app_quic" }}
35 """
36 cpu = 200
37 memory = 256
38 }
39 ["certificates"] {
40 build = "."
41 hostNetwork = true
42 readOnlyRoot = true
43 tmpfs { "/tmp" }
44 lifecycle = "prestartSidecar"
45 args { "certificates" }
46 volumes { ["/tls"] {} }
47 env {
48 ["RELAY_HOST"] = host
49 ["OWNER_EMAIL"] = site.ownerEmail
50 ["DNS_TOKEN"] = "${secret.dns.api_token}"
51 }
52 cpu = 100
53 memory = 256
14 }54 }
15 args { "--config-path"; "/etc/iroh-relay.toml" }
16 volumes { ["/etc/iroh-relay.toml"] { config = "relay.toml" } }
17 cpu = 200
18 memory = 256
19}55}
tools/iroh-relay.md created+68
...@@ -0,0 +1,68 @@
1# Iroh relays
2
3Clients use `https://r1.iroh.paperclover.net` and
4`https://r2.iroh.paperclover.net`. Caddy terminates HTTPS on TCP 443; Iroh serves
5QUIC address discovery on UDP 7842. TCP 80 remains available for ACME.
6These are public relays without an application login.
7
8The upstream binary version and checksum live in
9[`Dockerfile`](../service/iroh-relay/Dockerfile). Both machines use
10[`run.py`](../service/iroh-relay/run.py) and
11[`network.json`](../service/iroh-relay/network.json).
12Iroh's `--dev` flag disables its HTTPS listener behind Caddy, while its
13`Reloading` certificate resolver still supplies valid TLS for UDP discovery.
14
15## Snow Globe
16
17[`service.pkl`](../service/iroh-relay/service.pkl) declares the relay and its
18certificate sidecar. The sidecar obtains the Cloudflare credential through the
19DDNS provider, keeps it in a private tmpfs file, and checks certificate renewal
20every twelve hours. Only the certificate volume is shared with the relay.
21The relay reloads renewed certificates without restarting connections.
22
23Stage and deploy with the normal `tools/deploy.py` workflow. Fresh previews have
24their own certificate storage and dynamically allocated UDP port; they do not
25bind production's port. The host's `public-ports.service` maintains the production
26router mapping; the firewall declaration is in `nixos/snowglobe.nix`.
27
28## paperclover.net
29
30Connect with `ssh root@paperclover.net`; the hostname is `usw-server-1`.
31The checksum-verified upstream binary is installed at `/usr/local/bin/iroh-relay`.
32Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and
33[`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into
34`/etc/systemd/system/`.
35
36[`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as
37`/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main
38Caddyfile. Validate before reloading Caddy. The ACME webroot is
39`/var/lib/iroh-acme`; Caddy must be able to read it.
40
41After Caddy's HTTPS certificate is ready, issue the discovery certificate:
42
43```sh
44certbot certonly --non-interactive --agree-tos --email account@paperclover.net \
45 --webroot -w /var/lib/iroh-acme --cert-name iroh-relay \
46 -d r1.iroh.paperclover.net \
47 --deploy-hook 'systemctl try-restart iroh-relay.service'
48systemctl daemon-reload
49systemctl enable --now iroh-relay.service certbot-renew.timer
50firewall-cmd --add-port=7842/udp
51firewall-cmd --permanent --add-port=7842/udp
52```
53
54Systemd loads the certificate and key as private service credentials. Certbot's
55renewal hook restarts the service to refresh those credentials. The key stays
56root-only under `/etc/letsencrypt`.
57
58## Verification
59
60`GET /healthz` reports the relay version, but does not test packet forwarding.
61Use two Iroh 1.3 endpoints with `clear_ip_transports()` and a custom relay map;
62transfer and verify a payload in both directions. For UDP, connect a
63`iroh_relay::quic::QuicClient` with normal CA validation and await its
64`observed_external_addr()` stream. Test from outside the server's network.
65
66Renewal can be exercised on r1 with
67`certbot renew --cert-name iroh-relay --dry-run --run-deploy-hooks`.
68Inspect the relay's restart count and health afterward.
tools/xmpp.md+1-1
...@@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide...@@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide
24provisions client, direct-TLS client, and federation SRV records pointing at24provisions client, direct-TLS client, and federation SRV records pointing at
25`xmpp.paperclover.net`; the website keeps its existing apex address. Router25`xmpp.paperclover.net`; the website keeps its existing apex address. Router
26forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range26forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range
27to Snow Globe. `xmpp-nat.service` refreshes UPnP mappings on startup and every five27to Snow Globe. `public-ports.service` refreshes UPnP mappings on startup and every five
28minutes, discovering the current description URL with a bounded unicast SSDP28minutes, discovering the current description URL with a bounded unicast SSDP
29request to the configured gateway on port 1900. Replies and HTTP description29request to the configured gateway on port 1900. Replies and HTTP description
30hosts must match that gateway; discovery failure fails the refresh. Multicast30hosts must match that gateway; discovery failure fails the refresh. Multicast