authoravatarclover caruso<git@paperclover.net>2026-10-20 01:41:33-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logf82b44aebb7048af93617305487961e7659f112c
treee5f17048898f7c93453dc1c5a93eca69a1547a10
parenta235692734ea58ecdc64a0afc94800d595be91e2
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Enable QUIC address discovery on both Iroh relays

Share the relay runtime across Fedora and Snow Globe, with certificate-backed UDP discovery alongside Caddy HTTPS. Use private systemd credentials and Certbot renewal on r1, and a DNS certificate sidecar with live certificate reload on r2. Keep previews on dynamic UDP ports and extend the persistent public-port mappings for production. Verified Iroh 1.3 relay-only transfers on both hosts, trusted-TLS UDP address discovery on r1 and the isolated r2 preview, r1 certificate-renewal rehearsal, Nix firewall/NAT evaluation, and seven deployment tests. Assisted-by: gpt-6-astra

10 files changed, 188 insertions(+), 22 deletions(-)

nixos/snowglobe.nix+5-4
......@@ -5,6 +5,7 @@ let
55 xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json);
66 xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ];
77 xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ];
8 irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json);
89 encryptionRoots = {
910 apps = "apps";
1011 clover = "clover";
......@@ -24,7 +25,7 @@ in
2425 networking.hostId = "4fa19ccb";
2526 services.nomad.settings.name = config.networking.hostName;
2627 networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts;
27 networking.firewall.allowedUDPPorts = xmppUDPPorts;
28 networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ];
2829 networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }];
2930 networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: ''
3031 iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept
......@@ -34,7 +35,7 @@ in
3435 ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept
3536 '';
3637
37 systemd.services.xmpp-nat = {
38 systemd.services.public-ports = {
3839 wantedBy = [ "multi-user.target" ];
3940 wants = [ "network-online.target" ];
4041 after = [ "network-online.target" ];
......@@ -42,9 +43,9 @@ in
4243 serviceConfig = { Type = "oneshot"; DynamicUser = true; };
4344 script = ''
4445 description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})"
45 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe XMPP" -r \
46 exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \
4647 ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \
47 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
48 ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)}
4849 '';
4950 };
5051
service/iroh-relay/Dockerfile+5-2
......@@ -1,6 +1,9 @@
1FROM docker.io/library/alpine:3.22@sha256:3e9b4b680bfc9fb5269227cffbd6d42be39fbf7c0b908123913864aa4447e764
1FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332
2RUN apt-get update && apt-get install -y --no-install-recommends certbot python3-certbot-dns-cloudflare \
3 && rm -rf /var/lib/apt/lists/*
24ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz
35RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \
46 && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \
57 && rm /tmp/iroh-relay.tar.gz
6ENTRYPOINT ["/usr/local/bin/iroh-relay"]
8COPY run.py network.json /app/
9ENTRYPOINT ["python3", "/app/run.py"]
service/iroh-relay/iroh-relay.service+4-1
......@@ -5,7 +5,10 @@ After=network-online.target
55
66[Service]
77DynamicUser=true
8ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml
8Environment=HTTP_PORT=3340
9LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem
10LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem
11ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay
912Restart=on-failure
1013RestartSec=5
1114NoNewPrivileges=true
service/iroh-relay/network.json created+1
......@@ -0,0 +1 @@
1{"quic":7842}
service/iroh-relay/r1.caddy+7-1
......@@ -1,3 +1,9 @@
11r1.iroh.paperclover.net {
2 reverse_proxy 127.0.0.1:3340
2 handle /.well-known/acme-challenge/* {
3 root * /var/lib/iroh-acme
4 file_server
5 }
6 handle {
7 reverse_proxy 127.0.0.1:3340
8 }
39}
service/iroh-relay/relay.toml deleted-2
......@@ -1,2 +0,0 @@
1http_bind_addr = "0.0.0.0:3340"
2enable_metrics = false
service/iroh-relay/run.py created+50
......@@ -0,0 +1,50 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import subprocess
6import sys
7import time
8
9
10os.umask(0o077)
11mode = sys.argv[1]
12tls = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/tls/live/server"))
13
14if mode == "certificates":
15 credentials = Path("/tmp/cloudflare.ini")
16 credentials.write_text("dns_cloudflare_api_token = " + os.environ.pop("DNS_TOKEN") + "\n")
17 command = [
18 "certbot", "certonly", "--non-interactive", "--agree-tos",
19 "--email", os.environ["OWNER_EMAIL"], "--dns-cloudflare",
20 "--dns-cloudflare-credentials", str(credentials),
21 "--dns-cloudflare-propagation-seconds", "30", "--cert-name", "server",
22 "--config-dir", "/tls", "--work-dir", "/tmp/certbot-work", "--logs-dir", "/tmp/certbot-logs",
23 "--keep-until-expiring", "-d", os.environ["RELAY_HOST"],
24 ]
25 while True:
26 subprocess.run(command, check=not (tls / "fullchain.pem").exists(), timeout=300)
27 time.sleep(12 * 60 * 60)
28
29config = Path("/tmp/iroh-relay.toml")
30if mode == "relay":
31 for _ in range(180):
32 if (tls / "fullchain.pem").exists() and (tls / "privkey.pem").exists():
33 break
34 time.sleep(1)
35 else:
36 raise RuntimeError("Relay certificate was not issued within three minutes")
37 quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads(
38 Path(__file__).with_name("network.json").read_text())["quic"]
39 config.write_text(
40 'enable_metrics = false\nenable_quic_addr_discovery = true\n'
41 f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n'
42 '[tls]\ncert_mode = "Reloading"\n'
43 f'quic_bind_addr = "0.0.0.0:{quic_port}"\n'
44 f'manual_cert_path = {json.dumps(str(tls / "fullchain.pem"))}\n'
45 f'manual_key_path = {json.dumps(str(tls / "privkey.pem"))}\n'
46 )
47else:
48 raise ValueError("Unknown relay task")
49# --dev leaves HTTPS to Caddy while retaining certificate-backed QUIC discovery.
50os.execv("/usr/local/bin/iroh-relay", ["iroh-relay", "--dev", "--config-path", str(config)])
service/iroh-relay/service.pkl+47-11
......@@ -1,19 +1,55 @@
11amends "../../config/Service.pkl"
22
3import "../../config/site.pkl" as site
4import "../ddns-updater/service.pkl" as dns
5import "pkl:json"
6
7local network = new json.Parser {}.parse(read("network.json").text)
8local host = read?("prop:hostname") ?? "r2.iroh.\(site.domain)"
9
310name = "Iroh relay"
411launcher = false
512stageIsolation = "fresh"
13requirements { new dns.DnsCredentials {} }
614
7container {
8 build = "."
9 readOnlyRoot = true
10 http {
11 containerPort = 3340
12 subdomain = "r2.iroh"
13 checkPath = "/healthz"
15containers {
16 ["app"] {
17 build = "."
18 hostNetwork = true
19 readOnlyRoot = true
20 tmpfs { "/tmp" }
21 args { "relay" }
22 http { containerPort = 3340; hostname = host; loopback = true; checkPath = "/healthz" }
23 tcp {
24 new {
25 name = "quic"
26 containerPort = network.quic
27 hostPort = if (site.preview) null else network.quic
28 loopback = false
29 }
30 }
31 volumes { ["/tls"] { readOnly = true } }
32 envTemplate = """
33 HTTP_PORT={{ env "NOMAD_PORT_app_http" }}
34 QUIC_PORT={{ env "NOMAD_PORT_app_quic" }}
35 """
36 cpu = 200
37 memory = 256
38 }
39 ["certificates"] {
40 build = "."
41 hostNetwork = true
42 readOnlyRoot = true
43 tmpfs { "/tmp" }
44 lifecycle = "prestartSidecar"
45 args { "certificates" }
46 volumes { ["/tls"] {} }
47 env {
48 ["RELAY_HOST"] = host
49 ["OWNER_EMAIL"] = site.ownerEmail
50 ["DNS_TOKEN"] = "${secret.dns.api_token}"
51 }
52 cpu = 100
53 memory = 256
1454 }
15 args { "--config-path"; "/etc/iroh-relay.toml" }
16 volumes { ["/etc/iroh-relay.toml"] { config = "relay.toml" } }
17 cpu = 200
18 memory = 256
1955}
tools/iroh-relay.md created+68
......@@ -0,0 +1,68 @@
1# Iroh relays
2
3Clients use `https://r1.iroh.paperclover.net` and
4`https://r2.iroh.paperclover.net`. Caddy terminates HTTPS on TCP 443; Iroh serves
5QUIC address discovery on UDP 7842. TCP 80 remains available for ACME.
6These are public relays without an application login.
7
8The upstream binary version and checksum live in
9[`Dockerfile`](../service/iroh-relay/Dockerfile). Both machines use
10[`run.py`](../service/iroh-relay/run.py) and
11[`network.json`](../service/iroh-relay/network.json).
12Iroh's `--dev` flag disables its HTTPS listener behind Caddy, while its
13`Reloading` certificate resolver still supplies valid TLS for UDP discovery.
14
15## Snow Globe
16
17[`service.pkl`](../service/iroh-relay/service.pkl) declares the relay and its
18certificate sidecar. The sidecar obtains the Cloudflare credential through the
19DDNS provider, keeps it in a private tmpfs file, and checks certificate renewal
20every twelve hours. Only the certificate volume is shared with the relay.
21The relay reloads renewed certificates without restarting connections.
22
23Stage and deploy with the normal `tools/deploy.py` workflow. Fresh previews have
24their own certificate storage and dynamically allocated UDP port; they do not
25bind production's port. The host's `public-ports.service` maintains the production
26router mapping; the firewall declaration is in `nixos/snowglobe.nix`.
27
28## paperclover.net
29
30Connect with `ssh root@paperclover.net`; the hostname is `usw-server-1`.
31The checksum-verified upstream binary is installed at `/usr/local/bin/iroh-relay`.
32Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and
33[`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into
34`/etc/systemd/system/`.
35
36[`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as
37`/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main
38Caddyfile. Validate before reloading Caddy. The ACME webroot is
39`/var/lib/iroh-acme`; Caddy must be able to read it.
40
41After Caddy's HTTPS certificate is ready, issue the discovery certificate:
42
43```sh
44certbot certonly --non-interactive --agree-tos --email account@paperclover.net \
45 --webroot -w /var/lib/iroh-acme --cert-name iroh-relay \
46 -d r1.iroh.paperclover.net \
47 --deploy-hook 'systemctl try-restart iroh-relay.service'
48systemctl daemon-reload
49systemctl enable --now iroh-relay.service certbot-renew.timer
50firewall-cmd --add-port=7842/udp
51firewall-cmd --permanent --add-port=7842/udp
52```
53
54Systemd loads the certificate and key as private service credentials. Certbot's
55renewal hook restarts the service to refresh those credentials. The key stays
56root-only under `/etc/letsencrypt`.
57
58## Verification
59
60`GET /healthz` reports the relay version, but does not test packet forwarding.
61Use two Iroh 1.3 endpoints with `clear_ip_transports()` and a custom relay map;
62transfer and verify a payload in both directions. For UDP, connect a
63`iroh_relay::quic::QuicClient` with normal CA validation and await its
64`observed_external_addr()` stream. Test from outside the server's network.
65
66Renewal can be exercised on r1 with
67`certbot renew --cert-name iroh-relay --dry-run --run-deploy-hooks`.
68Inspect the relay's restart count and health afterward.
tools/xmpp.md+1-1
......@@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide
2424provisions client, direct-TLS client, and federation SRV records pointing at
2525`xmpp.paperclover.net`; the website keeps its existing apex address. Router
2626forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range
27to Snow Globe. `xmpp-nat.service` refreshes UPnP mappings on startup and every five
27to Snow Globe. `public-ports.service` refreshes UPnP mappings on startup and every five
2828minutes, discovering the current description URL with a bounded unicast SSDP
2929request to the configured gateway on port 1900. Replies and HTTP description
3030hosts must match that gateway; discovery failure fails the refresh. Multicast