implement writer pushing content to remote server

default-config.json

@@ -1,7 +1,7 @@
 {
   "parsers": [
     { "parser": "LinkedInJobCollections", "logger": "ConsoleWriter" },
-    { "parser": "TwitterFeed", "logger": "MongoDbWriter" }
+    { "parser": "TwitterFeed", "logger": "RemoteWriter" }
   ],
   "components": [
     {
@@ -12,6 +12,19 @@
           "value": "mongodb://127.0.0.1:27017/bopilot"
         }
       ]
+    },
+    {
+      "name": "RemoteWriter",
+      "config": [
+        {
+          "param": "server_content_endpoint",
+          "value": "http://127.0.0.1:8080/api/content"
+        },
+        {
+          "param": "api_secret",
+          "value": "<secret>"
+        }
+      ]
     }
   ]
 }

package.json

@@ -5,6 +5,7 @@
   "author": "Niko G.",
   "license": "MIT",
   "dependencies": {
+    "axios": "^1.6.5",
     "class-transformer": "^0.5.1",
     "mongodb": "^6.3.0",
     "pino": "^8.17.2",

src/engine/ComponentLoader.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import IWebParser from '../base/IWebParser';
 import { config } from './JsonConfig';
 import IContentWriter from '../base/IContentWriter';
-import { logger } from './logger'; 
+import { logger } from './logger';
 
 let instance = null;
 
@@ -38,12 +38,16 @@ export default class ComponentLoader {
     }
 
     protected getSearchDirs(fileName: string) {
-        return [
+        const dirs = [
             '.',
             path.resolve(__dirname, '../parsers'),
             path.resolve(__dirname, '../writers'),
-            ...(config.sourceDirs.map(v => path.resolve(__dirname, '..', '..', v)) ?? []),
-        ].map(v => `${v}/${fileName}`);
+            ...[...(config.sourceDirs ?? [])].map(v =>
+                path.resolve(__dirname, '..', '..', v),
+            ),
+        ];
+        const res = dirs.map(v => `${v}/${fileName}`);
+        return res;
     }
 
     async loadParser<T>(parserClassName: string): Promise<T | undefined> {

src/engine/logger.ts

@@ -1,11 +1,11 @@
 import pino from 'pino';
 
 export const logger = pino({
-  transport: {
-    target: 'pino-pretty',
-    options: {
-      colorize: true
-    }
-  },
-  level: 'info'
-})
+    transport: {
+        target: 'pino-pretty',
+        options: {
+            colorize: true,
+        },
+    },
+    level: 'info',
+});

src/writers/RemoteWriter.ts

@@ -0,0 +1,44 @@
+import BaseEntity from '../base/BaseEntity';
+import IContentWriter from '../base/IContentWriter';
+import ComponentLoader from '../engine/ComponentLoader';
+import axios from 'axios';
+import { logger } from '../engine/logger';
+
+export class RemoteWriter<T extends BaseEntity> implements IContentWriter<T> {
+    private apiEndpoint: string;
+    private apiKeySecret: string;
+    private getClient() {
+        if (!this.apiEndpoint) {
+            this.apiEndpoint =
+                ComponentLoader.getInstance().getComponentConfigString(
+                    'RemoteWriter',
+                    'server_content_endpoint',
+                    'http://127.0.0.1/api/content',
+                );
+            this.apiKeySecret =
+                ComponentLoader.getInstance().getComponentConfigString(
+                    'RemoteWriter',
+                    'api_secret',
+                    'test1234',
+                );
+        }
+    }
+
+    async log(record: T): Promise<void> {
+        try {
+            this.getClient(); //TODO: need to decide on what stage config is loaded, should all components have contructors?
+            const response = await axios.post(
+                this.apiEndpoint,
+                JSON.stringify(record),
+                {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-API-KEY': this.apiKeySecret,
+                    },
+                },
+            );
+        } catch (error) {
+            logger.error(error, 'Failed to POST to remote server');
+        }
+    }
+}

yarn.lock

@@ -1770,11 +1770,25 @@ async-wait-until@^2.0.12:
   resolved "https://registry.yarnpkg.com/async-wait-until/-/async-wait-until-2.0.12.tgz#8a94683bf29e74642a8bcbb9385f6ea330d4383f"
   integrity sha512-SXy/vDs6UPJMG6YeEYOQ4ilA/JnGxk187KPGqFx9O+qVxsjkSl+jH+3P50qSNyMpEmDgr8qOFGOKCJckWb1i7A==
 
+asynckit@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+  integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
+
 atomic-sleep@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
   integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==
 
+axios@^1.6.5:
+  version "1.6.5"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.5.tgz#2c090da14aeeab3770ad30c3a1461bc970fb0cd8"
+  integrity sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==
+  dependencies:
+    follow-redirects "^1.15.4"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 babel-jest@^29.7.0:
   version "29.7.0"
   resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz"
@@ -2063,6 +2077,13 @@ colorette@^2.0.7:
   resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a"
   integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==
 
+combined-stream@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
+  integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
+  dependencies:
+    delayed-stream "~1.0.0"
+
 [email protected]:
   version "0.0.1"
   resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
@@ -2172,6 +2193,11 @@ define-data-property@^1.1.1:
     gopd "^1.0.1"
     has-property-descriptors "^1.0.0"
 
+delayed-stream@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
+  integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
+
 [email protected]:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df"
@@ -2565,6 +2591,20 @@ flatted@^3.2.9:
   resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz"
   integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==
 
+follow-redirects@^1.15.4:
+  version "1.15.4"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf"
+  integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==
+
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 [email protected]:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
@@ -3463,7 +3503,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@~2.1.24, mime-types@~2.1.34:
+mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
@@ -3822,6 +3862,11 @@ proxy-addr@~2.0.7:
     forwarded "0.2.0"
     ipaddr.js "1.9.1"
 
+proxy-from-env@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
+  integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
+
 pump@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"