feat: add test framework (#48)

* Add test framework

* Add aws-region tests

* Add azure-region tests

* Update CONTRIBUTING.md

* Add formatting

* Improve fmt

* Format Terraform

.github/workflows/ci.yaml

@@ -0,0 +1,31 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+      - run: bun test
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+      - run: bun fmt:ci

.gitignore

@@ -1 +1,4 @@
 .terraform*
+node_modules
+*.tfstate
+*.tfstate.lock.info

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "files.exclude": {
+    "**/terraform.tfstate": true,
+    "**/.terraform": true
+  }
+}

CONTRIBUTING.md

@@ -6,37 +6,13 @@ To create a new module, clone this repository and run:
 ./new.sh MOUDLE_NAME
 ```
 
-Test a module by running an instance of Coder on your local machine:
-
-```shell
-coder server --in-memory
-```
-
-This will create a new module in the modules directory with the given name and scaffolding.
-Edit the files, adding your module's implementation, documentation and screenshots.
-
 ## Testing a Module
 
-Create a template and edit it to include your development module:
+A suite of test-helpers exists to run `terraform apply` on modules with variables, and test script output against containers.
 
-> [!NOTE]
+Reference existing `*.test.ts` files for implementation.
-> The Docker starter template is recommended for quick-iteration!
 
-```hcl
+```sh
-module "MOUDLE_NAME" {
+# Run tests for a specific module!
-    source = "/home/user/coder/modules/MOUDLE_NAME"
+$ bun test -t '<module>'
-}
 ```
-
-You can also test your module by specifying the source as a git repository:
-
-```hcl
-module "MOUDLE_NAME" {
-    source = "git::https://github.com/<USERNAME>/<REPO>.git//<FOLDER>?ref=<BRANCH>"
-}
-```
-
-Build a workspace and your module will be consumed! 🥳
-
-Open a pull-request with your module, a member of the Coder team will
-manually test it, and after-merge it will appear on the Registry.

aws-region/main.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "bun:test";
+import {
+  executeScriptInContainer,
+  runTerraformApply,
+  runTerraformInit,
+  testRequiredVariables,
+} from "../test";
+
+describe("aws-region", async () => {
+  await runTerraformInit(import.meta.dir);
+
+  testRequiredVariables(import.meta.dir, {});
+
+  it("default output", async () => {
+    const state = await runTerraformApply(import.meta.dir, {});
+    expect(state.outputs.value.value).toBe("us-east-1");
+  });
+
+  it("customized default", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      default: "us-west-2",
+    });
+    expect(state.outputs.value.value).toBe("us-west-2");
+  });
+});

azure-region/main.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "bun:test";
+import {
+  executeScriptInContainer,
+  runTerraformApply,
+  runTerraformInit,
+  testRequiredVariables,
+} from "../test";
+
+describe("azure-region", async () => {
+  await runTerraformInit(import.meta.dir);
+
+  testRequiredVariables(import.meta.dir, {});
+
+  it("default output", async () => {
+    const state = await runTerraformApply(import.meta.dir, {});
+    expect(state.outputs.value.value).toBe("eastus");
+  });
+
+  it("customized default", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      default: "westus",
+    });
+    expect(state.outputs.value.value).toBe("westus");
+  });
+});

bunfig.toml

@@ -0,0 +1,2 @@
+[test]
+preload = ["./setup.ts"]

gcp-region/README.md

@@ -6,6 +6,7 @@ maintainer_github: coder
 verified: true
 tags: [gcp, regions, parameter, helper]
 ---
+
 # Google Cloud Platform Regions
 
 This module adds Google Cloud Platform regions to your Coder template.

git-clone/README.md

@@ -6,6 +6,7 @@ maintainer_github: coder
 verified: true
 tags: [git, helper]
 ---
+
 # Git Clone
 
 This module allows you to automatically clone a repository by URL and skip if it exists in the path provided.

git-clone/main.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "bun:test";
+import {
+  executeScriptInContainer,
+  runTerraformApply,
+  runTerraformInit,
+  testRequiredVariables,
+} from "../test";
+
+describe("git-clone", async () => {
+  await runTerraformInit(import.meta.dir);
+
+  testRequiredVariables(import.meta.dir, {
+    agent_id: "foo",
+    url: "foo",
+  });
+
+  it("fails without git", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      agent_id: "foo",
+      url: "some-url",
+    });
+    const output = await executeScriptInContainer(state, "alpine");
+    expect(output.exitCode).toBe(1);
+    expect(output.stdout).toEqual(["Git is not installed!"]);
+  });
+
+  it("runs with git", async () => {
+    const state = await runTerraformApply(import.meta.dir, {
+      agent_id: "foo",
+      url: "fake-url",
+    });
+    const output = await executeScriptInContainer(state, "alpine/git");
+    expect(output.exitCode).toBe(128);
+    expect(output.stdout).toEqual([
+      "Creating directory ~/fake-url...",
+      "Cloning fake-url to ~/fake-url...",
+    ]);
+  });
+});

jetbrains-gateway/README.md

@@ -6,6 +6,7 @@ maintainer_github: coder
 verified: true
 tags: [ide, jetbrains, helper, parameter]
 ---
+
 # JetBrains Gateway
 
 This module adds a JetBrains Gateway Button to open any workspace with a single click.

package.json

@@ -0,0 +1,14 @@
+{
+  "name": "modules",
+  "scripts": {
+    "test": "bun test",
+    "fmt": "bun x prettier -w **/*.ts **/*.md *.md && terraform fmt **/*.tf",
+    "fmt:ci": "bun x prettier --check **/*.ts **/*.md *.md && terraform fmt -check **/*.tf"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.3"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  }
+}

setup.ts

@@ -0,0 +1,47 @@
+import { readableStreamToText, spawn } from "bun";
+import { afterAll, beforeAll } from "bun:test";
+
+const removeStatefiles = async () => {
+  const proc = spawn([
+    "find",
+    ".",
+    "-type",
+    "f",
+    "-name",
+    "*.tfstate",
+    "-name",
+    "*.tfstate.lock.info",
+    "-delete",
+  ]);
+  await proc.exited;
+};
+
+const removeOldContainers = async () => {
+  let proc = spawn([
+    "docker",
+    "ps",
+    "-a",
+    "-q",
+    "--filter",
+    `label=modules-test`,
+  ]);
+  let containerIDsRaw = await readableStreamToText(proc.stdout);
+  let exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(containerIDsRaw);
+  }
+  containerIDsRaw = containerIDsRaw.trim();
+  if (containerIDsRaw === "") {
+    return;
+  }
+  proc = spawn(["docker", "rm", "-f", ...containerIDsRaw.split("\n")]);
+  const stdout = await readableStreamToText(proc.stdout);
+  exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(stdout);
+  }
+};
+
+afterAll(async () => {
+  await Promise.all([removeStatefiles(), removeOldContainers()]);
+});

test.ts

@@ -0,0 +1,212 @@
+import { readableStreamToText, spawn } from "bun";
+import { afterEach, expect, it } from "bun:test";
+import { readFile, unlink } from "fs/promises";
+
+export const runContainer = async (
+  image: string,
+  init = "sleep infinity",
+): Promise<string> => {
+  const proc = spawn([
+    "docker",
+    "run",
+    "--rm",
+    "-d",
+    "--label",
+    "modules-test=true",
+    "--entrypoint",
+    "sh",
+    image,
+    "-c",
+    init,
+  ]);
+  let containerID = await readableStreamToText(proc.stdout);
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(containerID);
+  }
+  return containerID.trim();
+};
+
+// executeScriptInContainer finds the only "coder_script"
+// resource in the given state and runs it in a container.
+export const executeScriptInContainer = async (
+  state: TerraformState,
+  image: string,
+): Promise<{
+  exitCode: number;
+  stdout: string[];
+  stderr: string[];
+}> => {
+  const instance = findResourceInstance(state, "coder_script");
+  const id = await runContainer(image);
+  const resp = await execContainer(id, ["sh", "-c", instance.script]);
+  const stdout = resp.stdout.trim().split("\n");
+  const stderr = resp.stderr.trim().split("\n");
+  return {
+    exitCode: resp.exitCode,
+    stdout,
+    stderr,
+  };
+};
+
+export const execContainer = async (
+  id: string,
+  cmd: string[],
+): Promise<{
+  exitCode: number;
+  stderr: string;
+  stdout: string;
+}> => {
+  const proc = spawn(["docker", "exec", id, ...cmd], {
+    stderr: "pipe",
+    stdout: "pipe",
+  });
+  const [stderr, stdout] = await Promise.all([
+    readableStreamToText(proc.stderr),
+    readableStreamToText(proc.stdout),
+  ]);
+  const exitCode = await proc.exited;
+  return {
+    exitCode,
+    stderr,
+    stdout,
+  };
+};
+
+export interface TerraformState {
+  outputs: {
+    [key: string]: {
+      type: string;
+      value: any;
+    };
+  }
+  resources: [
+    {
+      type: string;
+      name: string;
+      provider: string;
+      instances: [
+        {
+          attributes: {
+            [key: string]: any;
+          };
+        },
+      ];
+    },
+  ];
+}
+
+export interface CoderScriptAttributes {
+  script: string;
+  agent_id: string;
+  url: string;
+}
+
+// findResourceInstance finds the first instance of the given resource
+// type in the given state. If name is specified, it will only find
+// the instance with the given name.
+export const findResourceInstance = <T extends "coder_script" | string>(
+  state: TerraformState,
+  type: T,
+  name?: string,
+  // if type is "coder_script" return CoderScriptAttributes
+): T extends "coder_script"
+  ? CoderScriptAttributes
+  : Record<string, string> => {
+  const resource = state.resources.find(
+    (resource) =>
+      resource.type === type && (name ? resource.name === name : true),
+  );
+  if (!resource) {
+    throw new Error(`Resource ${type} not found`);
+  }
+  if (resource.instances.length !== 1) {
+    throw new Error(
+      `Resource ${type} has ${resource.instances.length} instances`,
+    );
+  }
+  return resource.instances[0].attributes as any;
+};
+
+// assertRequiredVariables creates a test-case
+// for each variable provided and ensures that
+// the apply fails without it.
+export const testRequiredVariables = (
+  dir: string,
+  vars: Record<string, string>,
+) => {
+  // Ensures that all required variables are provided.
+  it("required variables", async () => {
+    await runTerraformApply(dir, vars);
+  });
+  const varNames = Object.keys(vars);
+  varNames.forEach((varName) => {
+    // Ensures that every variable provided is required!
+    it("missing variable " + varName, async () => {
+      const localVars = {};
+      varNames.forEach((otherVarName) => {
+        if (otherVarName !== varName) {
+          localVars[otherVarName] = vars[otherVarName];
+        }
+      });
+      try {
+        await runTerraformApply(dir, localVars);
+      } catch (ex) {
+        expect(ex.message).toContain(
+          `input variable \"${varName}\" is not set, and has no default`,
+        );
+        return;
+      }
+      throw new Error(`${varName} is not a required variable!`);
+    });
+  });
+};
+
+// runTerraformApply runs terraform apply in the given directory
+// with the given variables. It is fine to run in parallel with
+// other instances of this function, as it uses a random state file.
+export const runTerraformApply = async (
+  dir: string,
+  vars: Record<string, string>,
+): Promise<TerraformState> => {
+  const stateFile = `${dir}/${crypto.randomUUID()}.tfstate`;
+  const env = {};
+  Object.keys(vars).forEach((key) => (env[`TF_VAR_${key}`] = vars[key]));
+  const proc = spawn(
+    [
+      "terraform",
+      "apply",
+      "-compact-warnings",
+      "-input=false",
+      "-auto-approve",
+      "-state",
+      stateFile,
+    ],
+    {
+      cwd: dir,
+      env,
+      stderr: "pipe",
+      stdout: "pipe",
+    },
+  );
+  const text = await readableStreamToText(proc.stderr);
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(text);
+  }
+  const content = await readFile(stateFile, "utf8");
+  await unlink(stateFile);
+  return JSON.parse(content);
+};
+
+// runTerraformInit runs terraform init in the given directory.
+export const runTerraformInit = async (dir: string) => {
+  const proc = spawn(["terraform", "init"], {
+    cwd: dir,
+  });
+  const text = await readableStreamToText(proc.stdout);
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(text);
+  }
+};

tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "types": ["bun-types"]
+  }
+}