[WIP] Move Integration Tests to NodeJS

TESTING.md

@@ -0,0 +1,8 @@
+# Testing
+
+## Unit Tests
+We are working on an npm-based runner but for now we use the existing flow.  Run `./test.sh` as described in the main README.
+
+## Integration Tests
+1. Start the Minio and Gateway containers with `docker compose up --abort-on-container-exit`
+2. `npm test`

__tests__/TESTING.md

@@ -0,0 +1,24 @@
+# Testing
+
+
+## Integration Tests
+
+### Troubleshooting Integration Tests
+
+#### `AggregateError` doesn't tell me what went wrong
+Wrap the "expect" statements of the offending test in a `try/catch` block and print the error like so:
+```javascript
+try {
+  expect(foo).toEqual(bar);
+} catch (e) {
+  console.log("Aggregate Error: ", e);
+}
+```
+
+TODO: Maybe write a custom reporter
+
+#### I want to know what request the container actually received
+There are two options here:
+1. In the `afterAll` block, comment out the code that runs the container teardown. After the test fails, you can run `docker logs -f nginx-s3-gateway-test-base`
+
+2. Before the code that is erroring, add `timeout(3000)` then quickly run `docker logs -f nginx-s3-gateway-test-base` in another tab after the container starts.

__tests__/basic.test.ts

@@ -0,0 +1,156 @@
+import request from "supertest";
+import container from "./support/container";
+import s3Mock from "./support/s3Mock";
+import { describe, expect, test, beforeAll, afterAll } from "@jest/globals";
+import { TestConfig, DummyFileList } from "./support/configuration";
+
+const BUCKET_NAME = "bucket-2";
+
+// Config for the running container per test
+const testConfig: TestConfig = {
+  name: "base_test",
+  image: {
+    dockerfile: "Dockerfile.oss",
+  },
+  container: {
+    env: {
+      S3_BUCKET_NAME: BUCKET_NAME,
+      AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE",
+      AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      S3_SERVER: "minio",
+      S3_SERVER_PORT: "9000",
+      S3_SERVER_PROTO: "http",
+      S3_REGION: "us-east-1",
+      DEBUG: "true",
+      S3_STYLE: "virtual",
+      ALLOW_DIRECTORY_LIST: "false",
+      PROVIDE_INDEX_PAGE: "",
+      APPEND_SLASH_FOR_POSSIBLE_DIRECTORY: "",
+      STRIP_LEADING_DIRECTORY_PATH: "",
+      PREFIX_LEADING_DIRECTORY_PATH: "",
+      AWS_SIGS_VERSION: "4",
+      STATIC_SITE_HOSTING: "",
+      PROXY_CACHE_MAX_SIZE: "10g",
+      PROXY_CACHE_INACTIVE: "60m",
+      PROXY_CACHE_VALID_OK: "1h",
+      PROXY_CACHE_VALID_NOTFOUND: "1m",
+      PROXY_CACHE_VALID_FORBIDDEN: "30s",
+    },
+  },
+};
+
+const CONFIG = container.Config(testConfig);
+
+const minioClient = s3Mock.Client(
+  "localhost",
+  9090,
+  "AKIAIOSFODNN7EXAMPLE",
+  "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+);
+
+beforeAll(async () => {
+  try {
+    await container.stop(CONFIG);
+  } catch (e) {
+    console.log("no container to stop");
+  }
+
+  await s3Mock.ensureBucketWithObjects(minioClient, BUCKET_NAME, files());
+  await container.build(CONFIG);
+  await container.start(CONFIG);
+});
+
+afterAll(async () => {
+  await container.stop(CONFIG);
+  await s3Mock.deleteBucket(minioClient, BUCKET_NAME);
+});
+
+describe("Ordinary filenames", () => {
+  test("simple url", async () => {
+    const objectPath = "a.txt";
+    const res = await request(CONFIG.testContainer.baseUrl).get(
+      `/${objectPath}`,
+    );
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+
+  test("many params that should be stripped", async () => {
+    const objectPath = "a.txt";
+    const res = await request(CONFIG.testContainer.baseUrl)
+      .get("/a.txt?some=param&that=should&be=stripped#aaah")
+      .set("accept", "binary/octet-stream");
+
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+
+  test("with a more complex path", async () => {
+    const objectPath = "b/c/d.txt";
+    const res = await request(CONFIG.testContainer.baseUrl)
+      .get("/b/c/d.txt")
+      .set("accept", "binary/octet-stream");
+
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+
+  test("another simple path", async () => {
+    const objectPath = "b/e.txt";
+    const res = await request(CONFIG.testContainer.baseUrl)
+      .get("/b/e.txt")
+      .set("accept", "binary/octet-stream");
+
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+
+  test("too many forward slashes", async () => {
+    const objectPath = "b/e.txt";
+    const res = await request(CONFIG.testContainer.baseUrl)
+      .get("/b//e.txt")
+      .set("accept", "binary/octet-stream");
+
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+
+  test("very long file name", async () => {
+    const objectPath =
+      "a/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.txt";
+    const res = await request(CONFIG.testContainer.baseUrl)
+      .get(
+        "/a/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.txt",
+      )
+      .set("accept", "binary/octet-stream");
+
+    expect(res.statusCode).toBe(200);
+    expect(res.text).toBe(fileContent(objectPath));
+  });
+});
+
+function fileContent(key: string): string | undefined {
+  return files()[key]?.content;
+}
+
+function files(): DummyFileList {
+  return {
+    "a.txt": {
+      content: "Let go, or be dragged.",
+    },
+    "b/c/d.txt": {
+      content: `When thoughts arise, then do all things arise. When thoughts vanish, then do all things vanish.`,
+    },
+    "b/e.txt": {
+      content: "If only you could hear the sound of snow.",
+    },
+    "a/abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.txt": {
+      content: `
+      "Where the is not one thing, what then?"
+      "Throw it away!"
+      "With not one thing, what there is to throw away?"
+      "Then carry it off!"
+      `,
+    },
+  };
+}

__tests__/support/configuration.ts

@@ -0,0 +1,28 @@
+/**
+ * Configuration for a single test
+ */
+export type TestConfig = {
+  name: string;
+  image: {
+    dockerfile: string;
+    prebuiltName?: string;
+  };
+  container: {
+    env: ContainerEnvironment;
+  };
+};
+
+/**
+ * The environment variables to be given to the container
+ */
+type ContainerEnvironment = {
+  [key: string]: string;
+};
+
+export type DummyFileList = {
+  [key: string]: DummyFile;
+};
+
+type DummyFile = {
+  content: string;
+};

__tests__/support/container.ts

@@ -0,0 +1,148 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+const asyncExec = promisify(exec);
+import http from "http";
+import { TestConfig } from "./configuration";
+
+const GATEWAY_HOST = "localhost";
+const GATEWAY_PORT = 8989;
+const GATEWAY_BASE_URL = `http://${GATEWAY_HOST}:${GATEWAY_PORT}`;
+const START_TIMEOUT_SECONDS = 10;
+const STOP_TIMEOUT_SECONDS = 5;
+
+// Must match network name in docker-compose.yaml
+const DOCKER_NETWORK_NAME = "s3-gateway-test";
+
+type ContainerEnvironment = {
+  [key: string]: string;
+};
+
+type TestContainer = {
+  host: string;
+  port: number;
+  baseUrl: string;
+};
+
+type ContainerConfig = {
+  imageName: string;
+  containerName: string;
+  env: ContainerEnvironment;
+  dockerfileName?: string;
+  usePrebuiltImage: boolean;
+  networkName: string;
+  testContainer: TestContainer;
+};
+
+export function Config(testConfig: TestConfig): ContainerConfig {
+  const imageName =
+    testConfig.image.prebuiltName ||
+    (testConfig.name && buildImageName(testConfig.name));
+
+  return {
+    imageName: imageName,
+    containerName: imageNameToContainerName(imageName),
+    env: testConfig.container.env,
+    dockerfileName: testConfig.image.dockerfile,
+    usePrebuiltImage: !!testConfig.image.prebuiltName,
+    networkName: DOCKER_NETWORK_NAME,
+    testContainer: {
+      host: GATEWAY_HOST,
+      port: GATEWAY_PORT,
+      baseUrl: GATEWAY_BASE_URL,
+    },
+  };
+}
+
+export async function build(config: ContainerConfig) {
+  if (config.usePrebuiltImage) return config.imageName;
+  await asyncExec(
+    `docker build -t ${config.imageName} -f ${config.dockerfileName} .`,
+  );
+}
+
+export async function start(config: ContainerConfig) {
+  console.log("Waiting for test container to be ready...");
+  async function waitForContainerStart(timeoutAt: number) {
+    if (new Date().getTime() > timeoutAt)
+      throw new Error(
+        `Failed to start S3 Gateway test container ${config.imageName} with name ${config.containerName}. Check container logs for details`,
+      );
+
+    try {
+      const statusCode = await getStatusCode(
+        `${config.testContainer.baseUrl}/health`,
+      );
+      console.log(statusCode);
+
+      if (statusCode === 200) {
+        console.log("Verified test container is running!");
+      } else {
+        await timeout(1500);
+        await waitForContainerStart(timeoutAt);
+      }
+    } catch (e) {
+      await timeout(1500);
+      await waitForContainerStart(timeoutAt);
+    }
+  }
+
+  const dockerRunCmd = [
+    "docker run -d --rm",
+    `--name ${config.containerName}`,
+    `--network ${config.networkName}`,
+    `-p ${config.testContainer.port}:80`,
+    envToDockerRunArgs(config.env),
+    config.imageName,
+  ].join(" ");
+
+  await asyncExec(dockerRunCmd);
+
+  await waitForContainerStart(
+    new Date().getTime() + START_TIMEOUT_SECONDS * 1000,
+  );
+}
+
+export async function stop(config: ContainerConfig) {
+  await asyncExec(
+    `docker stop -t ${STOP_TIMEOUT_SECONDS} ${config.containerName}`,
+  );
+}
+
+function timeout(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function getStatusCode(url: string): Promise<number> {
+  return new Promise((resolve, reject) => {
+    http
+      .get(url, (response) => {
+        // Check the status code.
+        const statusCode = response.statusCode;
+        if (statusCode) {
+          resolve(statusCode);
+        } else {
+          reject(new Error(`No status code from request to ${url}`));
+        }
+      })
+      .on("error", (error) => {
+        reject(error);
+      });
+  });
+}
+
+function buildImageName(testName: string): string {
+  return `nginx-s3-gateway:test-${testName}`;
+}
+
+function imageNameToContainerName(name: string): string {
+  return name.replace(":", "-");
+}
+
+function envToDockerRunArgs(env: ContainerEnvironment) {
+  return Object.keys(env).reduce(
+    (acc, key) => `${acc} -e ${key}=${env[key]}`,
+    "",
+  );
+}
+
+export default { build, start, stop, Config };