| 1 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 2 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 3 | import * as Fly from "@/Fly"; |
| 4 | import { ensureStarted } from "@/Fly/replicas"; |
| 5 | import * as Provider from "@/Provider"; |
| 6 | import * as Test from "@/Test/Alchemy"; |
| 7 | import { expect } from "alchemy-test"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import { MinimumLogLevel } from "effect/References"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Result from "effect/Result"; |
| 12 | import type { MachineContainer } from "@/Fly/Machine"; |
| 13 | |
| 14 | const { test } = Test.make({ providers: Fly.providers() }); |
| 15 | |
| 16 | const logLevel = Effect.provideService( |
| 17 | MinimumLogLevel, |
| 18 | process.env.DEBUG ? "Debug" : "Info", |
| 19 | ); |
| 20 | |
| 21 | const image = |
| 22 | "docker-hub-mirror.fly.io/library/nginx@sha256:7396be67b6f53012a5cf955fa9040619294c25ccacf11e22af5de1b572fc756e"; |
| 23 | const nextImage = |
| 24 | "docker-hub-mirror.fly.io/library/nginx@sha256:ef8676b33d681f272ba429b27658bdd7e640963279714c96bddf1dc76307f7b6"; |
| 25 | |
| 26 | const workload = ( |
| 27 | sidecarVersion: string, |
| 28 | includeObsolete = true, |
| 29 | ): MachineContainer[] => [ |
| 30 | { |
| 31 | name: "web", |
| 32 | image, |
| 33 | env: { ROLE: "web" }, |
| 34 | healthChecks: [ |
| 35 | { |
| 36 | name: "web-tcp", |
| 37 | kind: "readiness" as const, |
| 38 | tcp: { port: 80 }, |
| 39 | interval: 5, |
| 40 | timeout: 2, |
| 41 | }, |
| 42 | ], |
| 43 | }, |
| 44 | { |
| 45 | name: "sidecar", |
| 46 | image: sidecarVersion === "two" ? nextImage : image, |
| 47 | cmd: ["sh", "-c", "sleep 300"], |
| 48 | env: { ROLE: "sidecar", VERSION: sidecarVersion }, |
| 49 | dependsOn: [{ name: "web", condition: "healthy" as const }], |
| 50 | }, |
| 51 | ...(includeObsolete |
| 52 | ? [{ name: "obsolete", image, cmd: ["sh", "-c", "sleep 300"] }] |
| 53 | : []), |
| 54 | ]; |
| 55 | |
| 56 | const waitUntilGone = (appName: string, machineId: string) => |
| 57 | machines |
| 58 | .getMachine({ |
| 59 | app_name: appName, |
| 60 | machine_id: machineId, |
| 61 | }) |
| 62 | .pipe( |
| 63 | Effect.map((machine) => |
| 64 | machine.state === "destroyed" ? ("gone" as const) : ("found" as const), |
| 65 | ), |
| 66 | Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)), |
| 67 | Effect.repeat({ |
| 68 | schedule: Schedule.spaced("2 seconds"), |
| 69 | until: (status) => status === "gone", |
| 70 | times: 10, |
| 71 | }), |
| 72 | ); |
| 73 | |
| 74 | test.provider( |
| 75 | "create, update, and delete a machine", |
| 76 | (stack) => |
| 77 | Effect.gen(function* () { |
| 78 | yield* stack.destroy(); |
| 79 | |
| 80 | const created = yield* stack.deploy( |
| 81 | Effect.gen(function* () { |
| 82 | const app = yield* Fly.App("Site"); |
| 83 | return yield* Fly.Machine("Web", { |
| 84 | app, |
| 85 | region: "iad", |
| 86 | image: "nginx:alpine", |
| 87 | guest: { cpus: 1, memoryMb: 256 }, |
| 88 | }); |
| 89 | }), |
| 90 | ); |
| 91 | |
| 92 | expect(created.machineId).toEqual(expect.any(String)); |
| 93 | expect(created.machineId.length).toBeGreaterThan(0); |
| 94 | expect(created.machineIds).toEqual([created.machineId]); |
| 95 | expect(created.count).toEqual(1); |
| 96 | expect(created.appName).toEqual(expect.any(String)); |
| 97 | expect(created.name).toEqual(expect.any(String)); |
| 98 | expect(created.region).toEqual("iad"); |
| 99 | expect(created.state).toEqual("started"); |
| 100 | expect(created.privateIp).toEqual(expect.any(String)); |
| 101 | expect(created.guest?.cpus).toEqual(1); |
| 102 | expect(created.guest?.memoryMb).toEqual(256); |
| 103 | |
| 104 | const fetched = yield* machines.getMachine({ |
| 105 | app_name: created.appName, |
| 106 | machine_id: created.machineId, |
| 107 | }); |
| 108 | expect(fetched.id).toEqual(created.machineId); |
| 109 | expect(fetched.name).toEqual(created.name); |
| 110 | expect(fetched.region).toEqual("iad"); |
| 111 | expect(fetched.state).toEqual("started"); |
| 112 | expect(fetched.config?.image).toEqual(expect.stringContaining("nginx")); |
| 113 | expect(fetched.config?.guest?.cpus).toEqual(1); |
| 114 | expect(fetched.config?.guest?.memory_mb).toEqual(256); |
| 115 | expect(fetched.config?.metadata?.["alchemy.type"]).toEqual("Fly.Machine"); |
| 116 | expect(fetched.config?.metadata?.["alchemy.replica"]).toEqual("0"); |
| 117 | expect(fetched.config?.metadata?.["alchemy.stack"]).toEqual( |
| 118 | expect.any(String), |
| 119 | ); |
| 120 | |
| 121 | const updated = yield* stack.deploy( |
| 122 | Effect.gen(function* () { |
| 123 | const app = yield* Fly.App("Site"); |
| 124 | return yield* Fly.Machine("Web", { |
| 125 | app, |
| 126 | region: "iad", |
| 127 | image: "nginx:alpine", |
| 128 | guest: { cpus: 1, memoryMb: 256 }, |
| 129 | env: { ALCHEMY_MARK: "updated" }, |
| 130 | metadata: { role: "web" }, |
| 131 | restart: { policy: "on-failure", maxRetries: 3 }, |
| 132 | services: [ |
| 133 | { |
| 134 | protocol: "tcp", |
| 135 | internalPort: 80, |
| 136 | ports: [{ port: 80, handlers: ["http"] }], |
| 137 | checks: [ |
| 138 | { |
| 139 | type: "http", |
| 140 | port: 80, |
| 141 | interval: "20s", |
| 142 | timeout: "3s", |
| 143 | gracePeriod: "6s", |
| 144 | method: "HEAD", |
| 145 | path: "/", |
| 146 | protocol: "http", |
| 147 | headers: [ |
| 148 | { |
| 149 | name: "X-Alchemy-Check", |
| 150 | values: ["ready", "routing"], |
| 151 | }, |
| 152 | ], |
| 153 | tlsServerName: "example.com", |
| 154 | tlsSkipVerify: true, |
| 155 | }, |
| 156 | ], |
| 157 | }, |
| 158 | ], |
| 159 | }); |
| 160 | }), |
| 161 | ); |
| 162 | |
| 163 | expect(updated.machineId).toEqual(created.machineId); |
| 164 | expect(updated.appName).toEqual(created.appName); |
| 165 | expect(updated.name).toEqual(created.name); |
| 166 | expect(updated.region).toEqual("iad"); |
| 167 | expect(updated.state).toEqual("started"); |
| 168 | |
| 169 | const refetched = yield* machines.getMachine({ |
| 170 | app_name: updated.appName, |
| 171 | machine_id: updated.machineId, |
| 172 | }); |
| 173 | expect(refetched.id).toEqual(created.machineId); |
| 174 | expect(refetched.config?.env?.ALCHEMY_MARK).toEqual("updated"); |
| 175 | expect(refetched.config?.metadata?.role).toEqual("web"); |
| 176 | expect(refetched.config?.metadata?.["alchemy.type"]).toEqual( |
| 177 | "Fly.Machine", |
| 178 | ); |
| 179 | expect(refetched.config?.restart?.policy).toEqual("on-failure"); |
| 180 | expect(refetched.config?.restart?.max_retries).toEqual(3); |
| 181 | expect(refetched.config?.services?.[0]?.internal_port).toEqual(80); |
| 182 | expect(refetched.config?.services?.[0]?.ports?.[0]?.port).toEqual(80); |
| 183 | const check = refetched.config?.services?.[0]?.checks?.[0]; |
| 184 | expect(check?.type).toEqual("http"); |
| 185 | expect(check?.port).toEqual(80); |
| 186 | expect(check?.interval).toEqual("20s"); |
| 187 | expect(check?.timeout).toEqual("3s"); |
| 188 | expect(check?.grace_period).toEqual("6s"); |
| 189 | expect(check?.method).toEqual("HEAD"); |
| 190 | expect(check?.path).toEqual("/"); |
| 191 | expect(check?.protocol).toEqual("http"); |
| 192 | expect(check?.headers?.[0]?.name).toEqual("X-Alchemy-Check"); |
| 193 | expect(check?.headers?.[0]?.values).toEqual(["ready", "routing"]); |
| 194 | expect(check?.tls_server_name).toEqual("example.com"); |
| 195 | expect(check?.tls_skip_verify).toEqual(true); |
| 196 | |
| 197 | yield* stack.destroy(); |
| 198 | |
| 199 | const gone = yield* waitUntilGone(created.appName, created.machineId); |
| 200 | expect(gone).toEqual("gone"); |
| 201 | }).pipe(logLevel), |
| 202 | { |
| 203 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 204 | timeout: 180_000, |
| 205 | }, |
| 206 | ); |
| 207 | |
| 208 | test.provider( |
| 209 | "starts an unlaunched machine and recovers from stale state", |
| 210 | (stack) => |
| 211 | Effect.gen(function* () { |
| 212 | yield* stack.destroy(); |
| 213 | |
| 214 | const deploy = (skipLaunch: boolean) => |
| 215 | stack.deploy( |
| 216 | Effect.gen(function* () { |
| 217 | const app = yield* Fly.App("LaunchSite"); |
| 218 | return yield* Fly.Machine("LaunchWeb", { |
| 219 | app, |
| 220 | region: "iad", |
| 221 | image: "nginx:alpine", |
| 222 | guest: { cpus: 1, memoryMb: 256 }, |
| 223 | skipLaunch, |
| 224 | services: [ |
| 225 | { |
| 226 | protocol: "tcp", |
| 227 | internalPort: 80, |
| 228 | checks: [ |
| 229 | { |
| 230 | type: "http", |
| 231 | port: 80, |
| 232 | path: skipLaunch ? "/missing" : "/", |
| 233 | interval: "5s", |
| 234 | timeout: "2s", |
| 235 | gracePeriod: "1s", |
| 236 | }, |
| 237 | ], |
| 238 | }, |
| 239 | ], |
| 240 | }); |
| 241 | }), |
| 242 | ); |
| 243 | const created = yield* deploy(true); |
| 244 | const target = { |
| 245 | app_name: created.appName, |
| 246 | machine_id: created.machineId, |
| 247 | }; |
| 248 | const unlaunched = yield* machines.getMachine(target); |
| 249 | expect(["created", "stopped"]).toContain(unlaunched.state); |
| 250 | yield* ensureStarted(created.appName, unlaunched, true); |
| 251 | expect(["created", "stopped"]).toContain( |
| 252 | (yield* machines.getMachine(target)).state, |
| 253 | ); |
| 254 | |
| 255 | const waitError = yield* machines |
| 256 | .waitMachine({ ...target, state: "started", timeout: 1 }) |
| 257 | .pipe(Retry.none, Effect.flip); |
| 258 | expect(["MachineWaitTimeout", "GatewayTimeout"]).toContain( |
| 259 | waitError._tag, |
| 260 | ); |
| 261 | |
| 262 | const launched = yield* deploy(false); |
| 263 | expect(launched.machineId).toBe(created.machineId); |
| 264 | expect(launched.state).toBe("started"); |
| 265 | const running = yield* machines.getMachine(target); |
| 266 | |
| 267 | const alreadyStarted = yield* ensureStarted( |
| 268 | created.appName, |
| 269 | unlaunched, |
| 270 | false, |
| 271 | ); |
| 272 | expect(alreadyStarted.state).toBe("started"); |
| 273 | expect(alreadyStarted.instance_id).toBe(running.instance_id); |
| 274 | |
| 275 | yield* machines.stopMachine({ ...target, signal: "SIGTERM" }); |
| 276 | yield* machines |
| 277 | .waitMachine({ |
| 278 | ...target, |
| 279 | instance_id: running.instance_id, |
| 280 | state: "stopped", |
| 281 | timeout: 8, |
| 282 | }) |
| 283 | .pipe( |
| 284 | Effect.retry({ |
| 285 | while: (error) => |
| 286 | error._tag === "GatewayTimeout" || |
| 287 | error._tag === "MachineWaitTimeout", |
| 288 | schedule: Schedule.spaced("1 second"), |
| 289 | times: 3, |
| 290 | }), |
| 291 | ); |
| 292 | expect((yield* machines.getMachine(target)).state).toBe("stopped"); |
| 293 | |
| 294 | const restarted = yield* ensureStarted(created.appName, running, false); |
| 295 | expect(restarted.state).toBe("started"); |
| 296 | expect((yield* machines.getMachine(target)).state).toBe("started"); |
| 297 | |
| 298 | yield* stack.destroy(); |
| 299 | expect(yield* waitUntilGone(created.appName, created.machineId)).toBe( |
| 300 | "gone", |
| 301 | ); |
| 302 | }).pipe(logLevel), |
| 303 | { |
| 304 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 305 | timeout: 120_000, |
| 306 | }, |
| 307 | ); |
| 308 | |
| 309 | test.provider( |
| 310 | "replace when name changes", |
| 311 | (stack) => |
| 312 | Effect.gen(function* () { |
| 313 | yield* stack.destroy(); |
| 314 | |
| 315 | const created = yield* stack.deploy( |
| 316 | Effect.gen(function* () { |
| 317 | const app = yield* Fly.App("ReplaceSite"); |
| 318 | return yield* Fly.Machine("ReplaceWeb", { |
| 319 | app, |
| 320 | region: "iad", |
| 321 | image: "nginx:alpine", |
| 322 | guest: { cpus: 1, memoryMb: 256 }, |
| 323 | }); |
| 324 | }), |
| 325 | ); |
| 326 | |
| 327 | expect(created.region).toEqual("iad"); |
| 328 | |
| 329 | const nextName = |
| 330 | created.name.slice(0, -1) + (created.name.endsWith("z") ? "y" : "z"); |
| 331 | |
| 332 | const replaced = yield* stack.deploy( |
| 333 | Effect.gen(function* () { |
| 334 | const app = yield* Fly.App("ReplaceSite"); |
| 335 | return yield* Fly.Machine("ReplaceWeb", { |
| 336 | app, |
| 337 | name: nextName, |
| 338 | region: "iad", |
| 339 | image: "nginx:alpine", |
| 340 | guest: { cpus: 1, memoryMb: 256 }, |
| 341 | }); |
| 342 | }), |
| 343 | ); |
| 344 | |
| 345 | expect(replaced.machineId).not.toEqual(created.machineId); |
| 346 | expect(replaced.name).toEqual(nextName); |
| 347 | expect(replaced.appName).toEqual(created.appName); |
| 348 | expect(replaced.region).toEqual("iad"); |
| 349 | expect(replaced.state).toEqual("started"); |
| 350 | |
| 351 | const fetched = yield* machines.getMachine({ |
| 352 | app_name: replaced.appName, |
| 353 | machine_id: replaced.machineId, |
| 354 | }); |
| 355 | expect(fetched.id).toEqual(replaced.machineId); |
| 356 | expect(fetched.name).toEqual(nextName); |
| 357 | |
| 358 | const oldGone = yield* waitUntilGone(created.appName, created.machineId); |
| 359 | expect(oldGone).toEqual("gone"); |
| 360 | |
| 361 | yield* stack.destroy(); |
| 362 | |
| 363 | const gone = yield* waitUntilGone(replaced.appName, replaced.machineId); |
| 364 | expect(gone).toEqual("gone"); |
| 365 | }).pipe(logLevel), |
| 366 | { |
| 367 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 368 | timeout: 120_000, |
| 369 | }, |
| 370 | ); |
| 371 | |
| 372 | test.provider( |
| 373 | "list enumerates the deployed machine", |
| 374 | (stack) => |
| 375 | Effect.gen(function* () { |
| 376 | yield* stack.destroy(); |
| 377 | |
| 378 | const deployed = yield* stack.deploy( |
| 379 | Effect.gen(function* () { |
| 380 | const app = yield* Fly.App("ListSite"); |
| 381 | return yield* Fly.Machine("ListWeb", { |
| 382 | app, |
| 383 | region: "iad", |
| 384 | image: "nginx:alpine", |
| 385 | guest: { cpus: 1, memoryMb: 256 }, |
| 386 | }); |
| 387 | }), |
| 388 | ); |
| 389 | |
| 390 | const provider = yield* Provider.findProvider(Fly.Machine); |
| 391 | const all = yield* provider.list(); |
| 392 | const found = all.find( |
| 393 | (machine) => machine.machineId === deployed.machineId, |
| 394 | ); |
| 395 | expect(found).toBeDefined(); |
| 396 | expect(found?.appName).toEqual(deployed.appName); |
| 397 | expect(found?.name).toEqual(deployed.name); |
| 398 | expect(found?.region).toEqual("iad"); |
| 399 | expect(found?.state).toEqual("started"); |
| 400 | |
| 401 | yield* stack.destroy(); |
| 402 | |
| 403 | const gone = yield* waitUntilGone(deployed.appName, deployed.machineId); |
| 404 | expect(gone).toEqual("gone"); |
| 405 | }).pipe(logLevel), |
| 406 | { |
| 407 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 408 | timeout: 120_000, |
| 409 | }, |
| 410 | ); |
| 411 | |
| 412 | test.provider( |
| 413 | "destroy recovers a machine whose initial service checks failed", |
| 414 | (stack) => |
| 415 | Effect.gen(function* () { |
| 416 | yield* stack.destroy(); |
| 417 | const app = yield* stack.deploy(Fly.App("UnhealthySite")); |
| 418 | const result = yield* stack |
| 419 | .deploy( |
| 420 | Effect.gen(function* () { |
| 421 | const app = yield* Fly.App("UnhealthySite"); |
| 422 | return yield* Fly.Machine("UnhealthyWeb", { |
| 423 | app, |
| 424 | region: "iad", |
| 425 | image: "nginx:alpine", |
| 426 | guest: { cpus: 1, memoryMb: 256 }, |
| 427 | count: 2, |
| 428 | services: [ |
| 429 | { |
| 430 | protocol: "tcp", |
| 431 | internalPort: 80, |
| 432 | checks: [ |
| 433 | { |
| 434 | type: "http", |
| 435 | port: 80, |
| 436 | path: "/missing", |
| 437 | interval: "5s", |
| 438 | timeout: "2s", |
| 439 | gracePeriod: "1s", |
| 440 | }, |
| 441 | ], |
| 442 | }, |
| 443 | ], |
| 444 | }); |
| 445 | }), |
| 446 | ) |
| 447 | .pipe(Effect.result); |
| 448 | expect(Result.isFailure(result)).toBe(true); |
| 449 | if (Result.isFailure(result)) { |
| 450 | expect(result.failure).toMatchObject({ |
| 451 | _tag: "Fly.ReplicaChecksNotPassing", |
| 452 | }); |
| 453 | } |
| 454 | const live = yield* machines.listMachines({ app_name: app.appName }); |
| 455 | expect( |
| 456 | live.filter((machine) => machine.state !== "destroyed"), |
| 457 | ).toHaveLength(1); |
| 458 | yield* stack.destroy(); |
| 459 | for (const machine of live) { |
| 460 | expect(yield* waitUntilGone(app.appName, machine.id!)).toBe("gone"); |
| 461 | } |
| 462 | }).pipe(logLevel), |
| 463 | { |
| 464 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 465 | timeout: 120_000, |
| 466 | }, |
| 467 | ); |
| 468 | |
| 469 | test.provider( |
| 470 | "count 2 waits for checks and leaves the next replica unchanged on failure", |
| 471 | (stack) => |
| 472 | Effect.gen(function* () { |
| 473 | yield* stack.destroy(); |
| 474 | |
| 475 | const deploy = (path: string) => |
| 476 | stack.deploy( |
| 477 | Effect.gen(function* () { |
| 478 | const app = yield* Fly.App("CheckSite"); |
| 479 | return yield* Fly.Machine("CheckWeb", { |
| 480 | app, |
| 481 | region: "iad", |
| 482 | image: "nginx:alpine", |
| 483 | guest: { cpus: 1, memoryMb: 256 }, |
| 484 | count: 2, |
| 485 | services: [ |
| 486 | { |
| 487 | protocol: "tcp", |
| 488 | internalPort: 80, |
| 489 | ports: [{ port: 80, handlers: ["http"] }], |
| 490 | checks: [ |
| 491 | { |
| 492 | type: "http", |
| 493 | port: 80, |
| 494 | method: "GET", |
| 495 | path, |
| 496 | protocol: "http", |
| 497 | interval: "5s", |
| 498 | timeout: "2s", |
| 499 | gracePeriod: "5s", |
| 500 | }, |
| 501 | { |
| 502 | type: "tcp", |
| 503 | port: 80, |
| 504 | interval: "5s", |
| 505 | timeout: "2s", |
| 506 | gracePeriod: "5s", |
| 507 | }, |
| 508 | ], |
| 509 | }, |
| 510 | ], |
| 511 | }); |
| 512 | }), |
| 513 | ); |
| 514 | |
| 515 | const deployed = yield* deploy("/"); |
| 516 | expect(deployed.count).toEqual(2); |
| 517 | expect(deployed.machineIds).toHaveLength(2); |
| 518 | expect(deployed.replicas).toHaveLength(2); |
| 519 | expect(deployed.state).toEqual("started"); |
| 520 | expect(deployed.replicas[0]?.state).toEqual("started"); |
| 521 | expect(deployed.replicas[1]?.state).toEqual("started"); |
| 522 | |
| 523 | for (const machineId of deployed.machineIds) { |
| 524 | const live = yield* machines.getMachine({ |
| 525 | app_name: deployed.appName, |
| 526 | machine_id: machineId, |
| 527 | }); |
| 528 | expect(live.state).toEqual("started"); |
| 529 | const serviceChecks = |
| 530 | live.checks?.filter((check) => |
| 531 | check.name?.startsWith("servicecheck-"), |
| 532 | ) ?? []; |
| 533 | expect(serviceChecks).toHaveLength(2); |
| 534 | expect( |
| 535 | serviceChecks.every((check) => check.status === "passing"), |
| 536 | ).toEqual(true); |
| 537 | } |
| 538 | |
| 539 | const secondBefore = yield* machines.getMachine({ |
| 540 | app_name: deployed.appName, |
| 541 | machine_id: deployed.machineIds[1]!, |
| 542 | }); |
| 543 | const failed = yield* deploy("/missing").pipe(Effect.result); |
| 544 | expect(Result.isFailure(failed)).toBe(true); |
| 545 | if (Result.isFailure(failed)) { |
| 546 | expect(failed.failure).toMatchObject({ |
| 547 | _tag: "Fly.ReplicaChecksNotPassing", |
| 548 | }); |
| 549 | } |
| 550 | const first = yield* machines.getMachine({ |
| 551 | app_name: deployed.appName, |
| 552 | machine_id: deployed.machineIds[0]!, |
| 553 | }); |
| 554 | const second = yield* machines.getMachine({ |
| 555 | app_name: deployed.appName, |
| 556 | machine_id: deployed.machineIds[1]!, |
| 557 | }); |
| 558 | expect(first.config?.services?.[0]?.checks?.[0]?.path).toBe("/missing"); |
| 559 | expect(first.checks?.some((check) => check.status !== "passing")).toBe( |
| 560 | true, |
| 561 | ); |
| 562 | expect(second.config?.services?.[0]?.checks?.[0]?.path).toBe("/"); |
| 563 | expect(second.instance_id).toBe(secondBefore.instance_id); |
| 564 | |
| 565 | const recovered = yield* deploy("/"); |
| 566 | expect(recovered.machineIds).toEqual(deployed.machineIds); |
| 567 | for (const machineId of recovered.machineIds) { |
| 568 | const live = yield* machines.getMachine({ |
| 569 | app_name: recovered.appName, |
| 570 | machine_id: machineId, |
| 571 | }); |
| 572 | expect( |
| 573 | live.checks?.filter((check) => |
| 574 | check.name?.startsWith("servicecheck-"), |
| 575 | ), |
| 576 | ).toEqual([ |
| 577 | expect.objectContaining({ status: "passing" }), |
| 578 | expect.objectContaining({ status: "passing" }), |
| 579 | ]); |
| 580 | } |
| 581 | yield* stack.destroy(); |
| 582 | |
| 583 | for (const machineId of deployed.machineIds) { |
| 584 | const gone = yield* waitUntilGone(deployed.appName, machineId); |
| 585 | expect(gone).toEqual("gone"); |
| 586 | } |
| 587 | }).pipe(logLevel), |
| 588 | { |
| 589 | tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"], |
| 590 | timeout: 180_000, |
| 591 | }, |
| 592 | ); |
| 593 | |
| 594 | test.provider( |
| 595 | "rolling named containers preserve group, digest pins, and mounted data", |
| 596 | (stack) => |
| 597 | Effect.gen(function* () { |
| 598 | yield* stack.destroy(); |
| 599 | const deploy = (version: string, includeObsolete = true) => |
| 600 | stack.deploy( |
| 601 | Effect.gen(function* () { |
| 602 | const app = yield* Fly.App("MultiContainerSite"); |
| 603 | return yield* Fly.Machine("Group", { |
| 604 | app, |
| 605 | region: "fra", |
| 606 | guest: { cpus: 1, memoryMb: 256 }, |
| 607 | env: { COMMON: "machine", ROLE: "machine" }, |
| 608 | containers: workload(version, includeObsolete), |
| 609 | mounts: [{ path: "/data", sizeGb: 1, encrypted: true }], |
| 610 | }); |
| 611 | }), |
| 612 | ); |
| 613 | const first = yield* deploy("one"); |
| 614 | expect(first.state).toEqual("started"); |
| 615 | const target = { app_name: first.appName, machine_id: first.machineId }; |
| 616 | const observed = yield* machines.getMachine(target); |
| 617 | expect( |
| 618 | observed.config?.containers?.map((container) => container.name), |
| 619 | ).toEqual(["web", "sidecar", "obsolete"]); |
| 620 | expect( |
| 621 | observed.config?.containers?.map((container) => container.image), |
| 622 | ).toEqual([image, image, image]); |
| 623 | expect(observed.config?.env?.COMMON).toEqual("machine"); |
| 624 | expect(observed.config?.containers?.[0]?.env?.ROLE).toEqual("web"); |
| 625 | expect(observed.config?.containers?.[1]?.env?.ROLE).toEqual("sidecar"); |
| 626 | expect(observed.config?.containers?.[1]?.depends_on).toEqual([ |
| 627 | { name: "web", condition: "healthy" }, |
| 628 | ]); |
| 629 | |
| 630 | const connected = yield* machines |
| 631 | .execMachine({ |
| 632 | ...target, |
| 633 | container: "sidecar", |
| 634 | command: [ |
| 635 | "sh", |
| 636 | "-c", |
| 637 | "wget -qO- http://127.0.0.1:80/ | grep -q 'Welcome to nginx' && printf connected", |
| 638 | ], |
| 639 | timeout: 10, |
| 640 | }) |
| 641 | .pipe( |
| 642 | Effect.repeat({ |
| 643 | schedule: Schedule.spaced("2 seconds"), |
| 644 | times: 8, |
| 645 | until: (result) => result.stdout?.includes("connected") === true, |
| 646 | }), |
| 647 | ); |
| 648 | expect(connected.stdout).toContain("connected"); |
| 649 | expect(connected.exit_code).toBe(0); |
| 650 | const environment = yield* machines.execMachine({ |
| 651 | ...target, |
| 652 | container: "sidecar", |
| 653 | command: ["sh", "-c", 'printf "%s|%s" "$COMMON" "$ROLE"'], |
| 654 | timeout: 10, |
| 655 | }); |
| 656 | expect(environment.stdout).toContain("machine|sidecar"); |
| 657 | expect(environment.exit_code).toBe(0); |
| 658 | const written = yield* machines.execMachine({ |
| 659 | ...target, |
| 660 | container: "web", |
| 661 | command: [ |
| 662 | "sh", |
| 663 | "-c", |
| 664 | "printf persistent > /data/ogm-1764-marker && cat /data/ogm-1764-marker", |
| 665 | ], |
| 666 | timeout: 10, |
| 667 | }); |
| 668 | expect(written.stdout).toContain("persistent"); |
| 669 | |
| 670 | const unchanged = yield* deploy("one"); |
| 671 | expect(unchanged.machineId).toEqual(first.machineId); |
| 672 | const same = yield* machines.getMachine(target); |
| 673 | expect(same.instance_id).toEqual(observed.instance_id); |
| 674 | |
| 675 | const updated = yield* deploy("two", false); |
| 676 | expect(updated.machineId).toEqual(first.machineId); |
| 677 | expect(updated.mounts[0]?.volumeId).toEqual(first.mounts[0]?.volumeId); |
| 678 | const after = yield* machines.getMachine(target); |
| 679 | expect( |
| 680 | after.config?.containers?.map((container) => container.name), |
| 681 | ).toEqual(["web", "sidecar"]); |
| 682 | expect( |
| 683 | after.config?.containers?.map((container) => container.image), |
| 684 | ).toEqual([image, nextImage]); |
| 685 | expect(after.config?.containers?.[1]?.env?.VERSION).toEqual("two"); |
| 686 | const persisted = yield* machines.execMachine({ |
| 687 | ...target, |
| 688 | container: "web", |
| 689 | command: ["cat", "/data/ogm-1764-marker"], |
| 690 | timeout: 10, |
| 691 | }); |
| 692 | expect(persisted.stdout).toContain("persistent"); |
| 693 | yield* stack.destroy(); |
| 694 | const gone = yield* machines.getApp({ app_name: first.appName }).pipe( |
| 695 | Effect.as(false), |
| 696 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 697 | ); |
| 698 | expect(gone).toBe(true); |
| 699 | }).pipe(logLevel), |
| 700 | { timeout: 120_000 }, |
| 701 | ); |
| 702 | |
| 703 | test.provider( |
| 704 | "rerun converges on a named group already updated outside Alchemy", |
| 705 | (stack) => |
| 706 | Effect.gen(function* () { |
| 707 | yield* stack.destroy(); |
| 708 | const deploy = (sidecarImage: string) => |
| 709 | stack.deploy( |
| 710 | Effect.gen(function* () { |
| 711 | const app = yield* Fly.App("InterruptedMultiContainerSite"); |
| 712 | return yield* Fly.Machine("Group", { |
| 713 | app, |
| 714 | region: "fra", |
| 715 | guest: { cpus: 1, memoryMb: 256 }, |
| 716 | mounts: [{ path: "/data", sizeGb: 1, encrypted: true }], |
| 717 | containers: [ |
| 718 | { name: "web", image }, |
| 719 | { |
| 720 | name: "sidecar", |
| 721 | image: sidecarImage, |
| 722 | cmd: ["sh", "-c", "sleep 300"], |
| 723 | }, |
| 724 | ], |
| 725 | }); |
| 726 | }), |
| 727 | ); |
| 728 | const first = yield* deploy(image); |
| 729 | const target = { app_name: first.appName, machine_id: first.machineId }; |
| 730 | |
| 731 | // Apply the next deployment directly, as if Fly accepted Alchemy's |
| 732 | // update but the response never arrived. |
| 733 | const before = yield* machines.getMachine(target); |
| 734 | const accepted = yield* machines.updateMachine({ |
| 735 | ...target, |
| 736 | config: { |
| 737 | ...before.config, |
| 738 | containers: before.config?.containers?.map((container) => |
| 739 | container.name === "sidecar" |
| 740 | ? { ...container, image: nextImage } |
| 741 | : container, |
| 742 | ), |
| 743 | }, |
| 744 | }); |
| 745 | yield* machines.waitMachine({ |
| 746 | ...target, |
| 747 | state: "started", |
| 748 | instance_id: accepted.instance_id, |
| 749 | timeout: 30, |
| 750 | }); |
| 751 | |
| 752 | const converged = yield* deploy(nextImage); |
| 753 | expect(converged.machineId).toBe(first.machineId); |
| 754 | expect(converged.mounts[0]?.volumeId).toBe(first.mounts[0]?.volumeId); |
| 755 | const observed = yield* machines.getMachine(target); |
| 756 | expect(observed.instance_id).toBe(accepted.instance_id); |
| 757 | expect( |
| 758 | observed.config?.containers?.map(({ name, image }) => ({ |
| 759 | name, |
| 760 | image, |
| 761 | })), |
| 762 | ).toEqual([ |
| 763 | { name: "web", image }, |
| 764 | { name: "sidecar", image: nextImage }, |
| 765 | ]); |
| 766 | const inventory = (yield* machines.listMachines({ |
| 767 | app_name: first.appName, |
| 768 | })).filter((machine) => machine.state !== "destroyed"); |
| 769 | expect(inventory.map(({ id }) => id)).toEqual([first.machineId]); |
| 770 | const volumes = yield* machines.listVolumes({ app_name: first.appName }); |
| 771 | expect(volumes.map(({ id }) => id)).toEqual([first.mounts[0]?.volumeId]); |
| 772 | yield* stack.destroy(); |
| 773 | const gone = yield* machines.getApp({ app_name: first.appName }).pipe( |
| 774 | Effect.as(false), |
| 775 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 776 | ); |
| 777 | expect(gone).toBe(true); |
| 778 | }).pipe(logLevel), |
| 779 | { timeout: 120_000 }, |
| 780 | ); |
| 781 | |
| 782 | test.provider( |
| 783 | "invalid container groups fail before any Machine is created", |
| 784 | (stack) => |
| 785 | Effect.gen(function* () { |
| 786 | yield* stack.destroy(); |
| 787 | const app = yield* stack.deploy(Fly.App("InvalidContainerSite")); |
| 788 | const invalid: Array<[string, MachineContainer[]]> = [ |
| 789 | ["empty group", []], |
| 790 | ["empty image", [{ name: "web", image: "" }]], |
| 791 | [ |
| 792 | "duplicate names", |
| 793 | [ |
| 794 | { name: "web", image }, |
| 795 | { name: "web", image: nextImage }, |
| 796 | ], |
| 797 | ], |
| 798 | [ |
| 799 | "undeclared dependency", |
| 800 | [{ name: "web", image, dependsOn: [{ name: "absent" }] }], |
| 801 | ], |
| 802 | [ |
| 803 | "exec override", |
| 804 | // @ts-expect-error Fly accepts container exec without running it |
| 805 | [{ name: "web", image, exec: ["nginx", "-g", "daemon off;"] }], |
| 806 | ], |
| 807 | ]; |
| 808 | for (const [label, containers] of invalid) { |
| 809 | const failed = yield* stack |
| 810 | .deploy( |
| 811 | Effect.gen(function* () { |
| 812 | const app = yield* Fly.App("InvalidContainerSite"); |
| 813 | return yield* Fly.Machine("Group", { |
| 814 | app, |
| 815 | region: "iad", |
| 816 | guest: { cpus: 1, memoryMb: 256 }, |
| 817 | containers, |
| 818 | }); |
| 819 | }), |
| 820 | ) |
| 821 | .pipe(Effect.flip); |
| 822 | expect(failed, label).toMatchObject({ |
| 823 | _tag: "Fly.InvalidMachineContainers", |
| 824 | }); |
| 825 | } |
| 826 | const failed = yield* stack |
| 827 | .deploy( |
| 828 | Effect.gen(function* () { |
| 829 | const app = yield* Fly.App("InvalidContainerSite"); |
| 830 | return yield* Fly.Machine("Group", { |
| 831 | app, |
| 832 | region: "iad", |
| 833 | guest: { cpus: 1, memoryMb: 256 }, |
| 834 | containers: [ |
| 835 | { |
| 836 | name: "web", |
| 837 | image: "docker-hub-mirror.fly.io/library/nginx:alpine", |
| 838 | }, |
| 839 | ], |
| 840 | checks: { ready: { type: "tcp", port: 80 } }, |
| 841 | deploy: { strategy: "bluegreen" }, |
| 842 | }); |
| 843 | }), |
| 844 | ) |
| 845 | .pipe(Effect.flip); |
| 846 | expect(failed).toMatchObject({ _tag: "Fly.InvalidDeployment" }); |
| 847 | const live = yield* machines.listMachines({ app_name: app.appName }); |
| 848 | expect(live.filter((machine) => machine.state !== "destroyed")).toEqual( |
| 849 | [], |
| 850 | ); |
| 851 | yield* stack.destroy(); |
| 852 | }).pipe(logLevel), |
| 853 | { timeout: 120_000 }, |
| 854 | ); |