DOP-3202: Persistence Module (#697)

* DOP-3202: Scaffold new module

* lint

* DOP-3202: Add upload w/ buildId logic

* DOP-3202: Add basic error handling

* fixup

* various productionization changes

* lint fixup

* fixup

* fixup

* fixup

* fixup

* fixup

* Incorporate feedback: Round 1

* lint

* fixup

* update readme, add try-catch, stronger typing

* fixup

* fixup

* Update README.md

* Derive created_at from objectID

* update comments

* rename entries -> pages

Author: casthewiz

.gitignore

@@ -1,5 +1,8 @@
 node_modules
 build
+modules/**/build
+modules/**/dist
+modules/**/.env
 npm-debug.log
 .env
 .DS_Store

jest.config.js

@@ -1,8 +1,8 @@
 module.exports = {
   collectCoverage: true,
   collectCoverageFrom: ['src/**/*.ts'],
-  coveragePathIgnorePatterns: ['node_modules', 'tests', 'errors', 'app', 'app_test'],
-  modulePathIgnorePatterns: ['<rootDir>/infrastructure/'],
+  coveragePathIgnorePatterns: ['node_modules', 'modules', 'tests', 'errors', 'app', 'app_test'],
+  modulePathIgnorePatterns: ['<rootDir>/infrastructure/', '<rootDir>/modules'],
   coverageDirectory: '<rootDir>/coverage/',
   verbose: false,
   // coverageThreshold: {

modules/README.md

@@ -0,0 +1,3 @@
+# Overview
+
+This directory contains various build-time modules for use in makefiles or for use as published modules utilized in other CI/CD routines.

modules/persistence/.babelrc

@@ -0,0 +1,9 @@
+{
+    "presets": [
+      "@babel/preset-env",
+      "@babel/preset-typescript"
+    ],
+    "plugins": [
+      "@babel/plugin-transform-runtime"
+    ]
+}

modules/persistence/.eslintrc.js

@@ -0,0 +1,19 @@
+module.exports = {
+  env: {
+    browser: true,
+    es6: true,
+  },
+  extends: 'plugin:@typescript-eslint/recommended',
+  globals: {
+    Atomics: 'readonly',
+    SharedArrayBuffer: 'readonly',
+  },
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ecmaVersion: 2018,
+    sourceType: 'module',
+  },
+  rules: {
+    'no-console': 'off',
+  },
+};

modules/persistence/.nvmrc

@@ -0,0 +1 @@
+v14.17.6

modules/persistence/README.md

@@ -0,0 +1,57 @@
+# Snooty Transport Module
+
+As part of integrating Snooty Parser with the unified Snooty toolchain, this module holds exclusive responsibility over transporting and persisting AST and metadata output artifacts from the parser into datastores.
+
+Currently, this module supports persistence to Mongodb Atlas instances, and is also responsible for mutation of entries at persist time.
+
+## Installation
+
+By default, this module requires Node v14.17.6 and uses a `.nvmrc` file if using nvm.
+Running `npm ci` to install all dependencies is required for usage.
+
+An example environment file is available at `sample.env`.
+Copy this file to `.env` in order to use a local environment file.
+
+## Building and Running
+
+### `npm run build`
+
+Compiles the module using `tsc`. By default, compiles to the `./dist` directory.
+
+### `npm run start`
+
+Runs the contents of the `./dist` directory.
+Requires usage of `-- -path` argument, eg `npm run start -- --path ./build/artifacts.zip`.
+Recommended command for running this module in higher than local environments.
+Requires parser output artifacts to be present in specified directory and zip file at `--path` value specified.
+
+### `npm run dev`
+
+Cleans dist, compiles, and runs with arguments `-path ./build/artifacts.zip`.
+Requires parser output artifacts to be present in specified directory and zip file at `./build/artifacts.zip`.
+
+## Available Arguments
+
+This module utilizes [minimist](https://www.npmjs.com/package/minimist) for argument parsing.
+For more information on accepted argument formats, please consult [its documentation](https://www.npmjs.com/package/minimist).
+
+### `-path`
+
+| values | any string |
+| ------ | ---------- |
+
+Required string formatted filepath where build artifacts are housed.
+
+### `-strict`
+
+| values | 'y', 'yes', 'true' |
+| ------ | ------------------ |
+
+Optional string formatted argument for whether module should exit with non-zero status on failure.
+Highly recommended for use in production environments.
+
+## Using/Developing This Module
+
+Usage and development of this module requires specifying the following environment variables. Use of a `.env` file is supported, but only recommended for local development.
+
+If adding a new environment variable, please update the `sample.env`. The `sample.env` should be considered the primary documentation for supported environment variables within this module.

modules/persistence/index.ts

@@ -0,0 +1,51 @@
+import * as dotenv from 'dotenv';
+// dotenv.config() should be invoked immediately, before any other imports, to ensure config is present
+dotenv.config();
+
+import AdmZip from 'adm-zip';
+import minimist from 'minimist';
+import * as mongodb from 'mongodb';
+import { insertPages } from './src/services/pages';
+import { insertMetadata } from './src/services/metadata';
+
+interface ModuleArgs {
+  path: string;
+  strict: string;
+  [props: string | number | symbol]: unknown;
+}
+
+const missingPathMessage = 'No path specified in arguments - please specify a build directory at arg "path"';
+
+// Callable via npm run start, with -- --path='' --strict=''
+// being accepted args
+// Also callable w/ default args via npm run dev
+// Load command line args into a parameterized argv
+const argv: ModuleArgs = minimist(process.argv.slice(2));
+
+const app = async (path: string) => {
+  try {
+    if (!path) throw missingPathMessage;
+    const zip = new AdmZip(path);
+    // atomic buildId for all artifacts read by this module - fundamental assumption
+    // that only one build will be used per run of this module.
+    const buildId = new mongodb.ObjectId();
+    await Promise.all([insertPages(buildId, zip), insertMetadata(buildId, zip)]);
+    process.exit(0);
+  } catch (error) {
+    console.error(`Persistence Module encountered a terminal error: ${error}`);
+    throw error;
+  }
+};
+
+try {
+  console.log(argv);
+  app(argv['path']);
+} catch (error) {
+  console.log('caught in terminal handling');
+  // only exit with non zero error code if running with strict mode on
+  if (['y', 'yes', 'true'].includes(argv['strict'].toLowerCase())) {
+    process.exit(1);
+  } else {
+    process.exit(0);
+  }
+}