Merge pull request #101 from medic/arch-v3

Arch v3

.dockerignore

@@ -1 +1,2 @@
 node_modules
+Dockerfile

.github/workflows/ci.yml

@@ -1,11 +1,12 @@
 name: Build cht-couch2pg and run unit test cases
 
-on: [ push, pull_request ]
+on: [ push]
 
 jobs:
   build-and-test:
     name: Build cht-couch2pg
     runs-on: ubuntu-18.04
+    timeout-minutes: 15
 
     strategy:
       matrix:
@@ -18,14 +19,41 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
-          username: ${{ secrets.CI_DOCKER_USERNAME }}
-          password: ${{ secrets.CI_DOCKER_PASSWORD }}
+          username: dockermedic
+          password: ${{ secrets.DOCKER_WRITE_PASSWORD }}
         if: steps.get-docker-hub-username.outputs.dockerhub_username
       - name: Checkout branch
         uses: actions/checkout@v2
 
-      - name: Build
-        run: docker-compose build --build-arg node_version=${{ matrix.node-version }} test
 
-      - name: Run tests
-        run: docker-compose run test grunt test
+      - name: Build test image
+        run: docker-compose -f docker-compose.test.yml build --build-arg node_version=${{ matrix.node-version }} cht-couch2pg
+
+      - name: Run grunt Tests
+        run: docker-compose -f docker-compose.test.yml run cht-couch2pg grunt test
+
+      - name: Run entry point script Tests
+        run: docker-compose -f docker-compose.test.yml run cht-couch2pg ./tests/bash/bats/bin/bats  /app/tests/bash/test.bats
+
+      - name:  Get tags to apply to image
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: medicmobile/cht-couch2pg
+          labels: |
+            org.opencontainers.image.title=cht-couch2pg
+            org.opencontainers.image.description= Software for creating read-only replicas of CouchDB data inside PostgreSQL
+            org.opencontainers.image.vendor=Medic
+          flavor: |
+            latest=auto
+            suffix=-node-${{matrix.node-version}}
+
+
+      - name: Build and Publish to Docker Hub
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: ${{github.ref == 'refs/heads/master' || github.ref == 'refs/heads/arch-v3' || github.event.ref_type == 'tag' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          target: cht-couch2pg

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "tests/bash/bats"]
+	path = tests/bash/bats
+	url = https://github.com/bats-core/bats-core.git
+[submodule "tests/bash/test_helper/bats-support"]
+	path = tests/bash/test_helper/bats-support
+	url = https://github.com/bats-core/bats-support.git
+[submodule "tests/bash/test_helper/bats-assert"]
+	path = tests/bash/test_helper/bats-assert
+	url = https://github.com/bats-core/bats-assert.git
+[submodule "tests/bash/test_helper/bats-shell-mock"]
+	path = tests/bash/test_helper/bats-shell-mock
+	url = https://github.com/capitalone/bash_shell_mock

Dockerfile

@@ -1,12 +1,23 @@
-ARG node_version
-FROM node:$node_version
-
-WORKDIR /usr/app
-
-COPY package.json .
-COPY package-lock.json .
-
+#base Build
+ARG node_version=10
+FROM node:$node_version as base_couch2pg_build
+RUN apt-get update
+RUN apt-get -y install postgresql-client curl
+WORKDIR /app
+COPY . .
 RUN npm ci
+
+#Test build
+FROM base_couch2pg_build AS test-couch2pg
+WORKDIR /app
+RUN git submodule update --init
 RUN npm i -g grunt-cli
 
-COPY . .
+
+# Final
+FROM base_couch2pg_build AS cht-couch2pg
+LABEL Authors="MEDIC SRE TEAM<[email protected]>"
+WORKDIR /app
+RUN rm -rf tests
+ENTRYPOINT [ "/app/couch2pg-entrypoint.sh" ]
+CMD ["main"]

README.md

@@ -1,6 +1,6 @@
 # CHT couch2pg
 
-Software for creating read-only replicas of CouchDB data inside PostgreSQL v9.4.
+Software for creating read-only replicas of CouchDB data inside PostgreSQL.
 
 The focus is specifically on CHT application data currently stored in CouchDB. If you are looking to have a read-only replica of CouchDB data for your application, consider [couch2pg](https://www.npmjs.com/package/couch2pg).
 
@@ -34,122 +34,138 @@ export COUCH2PG_DOC_LIMIT=1000
 export COUCH2PG_RETRY_COUNT=5
 ```
 
-Run it locally with environment variables: `npm ci && node .`
+*NOTE:* Currently, cht-couch2pg only runs in node versions 8 , 10 and 12. Later versions of node have been known to fail.+
 
-### Running locally in interactive mode (no environment variables needed)
+To run the contents of this repository locally, first, export the above environment variables and then run:  `npm ci && node .`
 
-Run it locally in interactive mode: `npm ci && node . -i`
+### Running locally in interactive mode without environment variables
 
+Run it locally in interactive mode: `npm ci && node . -i`
 
 ## Running tests through docker-compose
 
 Run tests with:
 
-```
-docker-compose build --build-arg node_version=[node version] test
-docker-compose up
+```bash
+docker-compose  -f docker-compose.test.yml build cht-couch2pg
+docker-compose  -f docker-compose.test.yml run cht-couch2pg grunt test
 ```
 
-Then in another terminal:
+Run tests in interactive watch mode with: `docker-compose -f docker-compose.test.yml run cht-couch2pg npm run watch`
 
-```
-docker-compose run test grunt test
+Run entrypoint script tests with
+
+```bash
+docker-compose -f docker-compose.test.yml run cht-couch2pg ./tests/bash/bats/bin/bats  /app/tests/bash/test.bats
 ```
 
-Run tests in interactive watch mode with: `docker-compose run test npm run watch`.
+## Running  the docker image
 
+You can run the [docker image](https://hub.docker.com/r/medicmobile/cht-couch2pg) available on docker hub.  You will need to provide the required environment variables.
 
-## Running tests against local couch and postgres databases
+### Required Environment Variables
 
-Run tests with: `grunt test`.
-Run tests in interactive watch mode with: `npm run watch`.
+#### 1. COUCHDB_URL
 
-Environment variables required for the integration tests to run correctly:
- * `TEST_PG_URL`: postgres url. ie: `postgres://localhost:5432`
- * `TEST_COUCH_URL`: couch url. ie: `http://admin:pass@localhost:5984`
+This sets the CouchDB instance URL, format: `https://[user]:[password]@[instance-name]:[port]/
 
-NB: The integration tests destroy and re-create the given databases each time they are run.
+#### 2.COUCH2PG_SLEEP_MINS
 
+This sets the number of minutes between synchronization runs.
 
-## Required database setup
+#### 3. COUCH2PG_DOC_LIMIT
 
-We support PostgreSQL 9.4 and greater. The user passed in the postgres url needs to have full creation rights on the given database.
+This sets the document batch size cht-couch2pg fetches from CouchDB every time it querries CouchDB.
 
+#### 4. COUCH2PG_RETRY_COUNT
 
-## Example usage
+This sets the number of times cht-couch2pg will retry synchronizing documents from CouchDB after experiencing an error
 
-You should probably install cht-couch2pg as a service and leave it to do its thing, as it should be able to run independently without any user input.
+#### 5. COUCH2PG_CHANGES_LIMIT
 
-### Installing as a service using Upstart (Ubuntu 14.4)
+The number of document ids to fetch per change limit request
 
-To setup a really simple service with upstart, all you need is sudo rights on the server. You want to do something like this:
- - For now, you should still just clone this repo onto your server, check out the relevant tag, and run `npm install`. In the future this will be better!
- - `sudo` create a `/etc/init/couch2pg-example-client.conf`
- - As we are going to put passwords in this file, you want to `chown o-r /etc/init/couch2pg-example-client.conf` so that only root can read it
- - Edit this file and put something like this in it:
+#### 6. COUCH2PG_USERS_META_DOC_LIMIT
 
-```
-description "Service for running Example Client's couch2pg integration"
-author "Your name"
-script
-    export POSTGRESQL_URL="..."
-    export COUCHDB_URL="..."
-    exec nodejs /path/to/the/repo/index
-end script
-```
- - The service is then a standard service, e.g. `service couch2pg-example-client start`
-
-### Installing as a service using Systemd (18.04.3 LTS [Bionic Beaver])
-
-To setup couch2pg using systemd is also pretty simple. You will need to have sudo rights to the server and then follow the steps listed below:
-
- - Install git and clone this repo onto your server, check out the relevant tag `git checkout tag_id`, and run `npm ci`. 
- - Create a systemd unit file for your project `sudo` create `/etc/systemd/system/couch2pg-sample-client.service`
- - As we are going to put passwords in this file, you want to `sudo chmod o-r /etc/systemd/system/couch2pg-sample-client.service` so that only root can read it.
- - Edit this file and configure the couch2pg system unit. It could be something simillar to this;
- ```[Unit]
-Description=Service for running ACME couch2pg integration
-
-[Service]
-Environment='POSTGRESQL_URL=postgres://couch2pg:secret=@localhost:5432/db'
-Environment='COUCHDB_URL=https://username:pass@couchdburl/medic'
-Environment='COUCH2PG_SLEEP_MINS=720'
-Environment='COUCH2PG_DOC_LIMIT=1000'
-Environment='COUCH2PG_RETRY_COUNT=5'
-Environment='COUCH2PG_CHANGES_LIMIT=1000'
-
-ExecStart=/usr/bin/npm run cht-couch2pg --prefix /path/to/cht-couch2pg/index.js
-
-ExecStartPost= add monitoring script command to run after service starts.
-ExecStopPost= add monitoring script to run if service stops 
-# Required on some systems
-WorkingDirectory=/path/to/cht-couch2pg/source
-Restart=always
-# Restart service after 10 seconds if couch2pg service crashes
-RestartSec=10
-# Output to syslog
-StandardOutput=syslog
-StandardError=syslog
-SyslogIdentifier=couch2pg-acme
-
-[Install]
-WantedBy=multi-user.target
-```
+The number of documents to fetch concurrently from the users-meta database
+
+#### 7. POSTGRES_DB
+
+The name of the PostgreSQL instance running PostgreSQL.
+
+#### 8. POSTGRES_USER_NAME
 
- - Reload systemd settings `systemctl daemon-reload`
- - Start the service `sudo service couch2pg-sample-client start`
- - If all goes well the service should start smoothly.
- - You can check the service logs using `journalctl` like this `journalctl -u couch2pg-sample-client --since today`
+The user name cht-couch2pg will use to access PostgreSQL.
 
+#### 9. POSTGRES_DB_NAME
+
+The name of the PostgreSQL database to sync to.
+
+#### 10. POSTGRES_PASSWORD
+
+The password the cht-couch2pg uses to authenticate to PostgreSQL.
+
+A sample docker-compose snippet is shown below. This image can work with the [cht-postgres](https://hub.docker.com/r/medicmobile/cht-postgres) docker image also available on docker hub.
+
+```yaml
+version: '3.7'
+services:
+  couch:
+        image: couchdb:2.3.1
+        environment:
+          COUCHDB_USER: cht
+          COUCHDB_PASSWORD: cht-password
+        networks:
+        - cht-net
+  cht-couch2pg:
+        container_name: cht-couch2pg
+        image: medicmobile/cht-couch2pg:v3.2.0-node-10
+        environment:
+           COUCHDB_URL: "http://cht:cht_password@couch:5984"
+           COUCH2PG_SLEEP_MINS: '720'
+           COUCH2PG_DOC_LIMIT: '1000'
+           COUCH2PG_RETRY_COUNT: '5'
+           COUCH2PG_CHANGES_LIMIT: '100'
+           POSTGRES_DB: cht-postgres
+           POSTGRES_USER_NAME: cht_couch2pg
+           POSTGRES_DB_NAME: cht
+           POSTGRES_PASSWORD: couch2pg_password
+        depends_on:
+          - cht-postgres
+
+  cht-postgres:
+        container_name: cht-postgres
+        image: medicmobile/cht-postgres:release-postgres13-rc.1
+        environment:
+            POSTGRES_DB: cht
+            POSTGRES_USER: cht
+            POSTGRES_PASSWORD: cht_password
+            COUCH2PG_USER: cht_couch2pg
+            COUCH2PG_USER_PASSWORD: couch2pg_password
+            DB_OWNER_GROUP: cht_analytics
+        volumes:
+            - cht-postgres-data:/var/lib/postgresql/data
+        networks:
+          - cht-net
+
+volumes:
+  cht-postgres-data:
+    name: cht-postgres-data
+
+networks:
+  cht-net:
+   name: cht-net
+
+```
 
 ## Known issues
 
 ### Error "Checksum failed for migration ..." when upgrading from 3.2.0 to latest
 
 An SQL migration file was changed in version 3.2.0. This made upgrades from 3.1.x impossible, with the process crashing upon startup after the upgrade. See more [details about the error](https://github.com/medic/cht-couch2pg/issues/78).
 
-This was fixed in version 3.2.1, by reverting the changes made to the migration file. 
-Fresh installations of 3.2.0 should execute this SQL before upgrading: 
+This was fixed in version 3.2.1, by reverting the changes made to the migration file.
+Fresh installations of 3.2.0 should execute this SQL before upgrading:
 
 ```sql
 UPDATE xmlforms_migrations