added postgresql functions for graph ql api

README.md

@@ -36,7 +36,7 @@ yarn cli repo create default
 # add datasource giving a config in json format
 yarn cli ds add -r <repo> <plugin-name> <config>
 # for example the cba plugin - need to define the api key for cba in .env file
-yarn cli ds add -r default repco:datasource:cba '{"endpoint": "https://cba.fro.at/wp-json/wp/v2"}'
+yarn cli ds add -r default urn:repco:datasource:cba https://cba.media/wp-json/wp/v2
 # ingest updates from all datasources
 yarn cli ds ingest
 # print all revisions in a repo
@@ -49,6 +49,25 @@ http://localhost:3000
 
 ## Development notes
 
+## Prod Deployment
+
+```sh
+# fetch changes
+git pull
+# check container status
+docker compose -f "docker/docker-compose.build.yml" ps
+# build new docker image
+docker compose -f "docker/docker-compose.build.yml" build
+# deploy docker image
+docker compose -f "docker/docker-compose.build.yml" up
+# create default repo
+docker compose -f "docker/docker-compose.build.yml" exec app yarn repco repo create default
+# add cba datasource
+docker compose -f "docker/docker-compose.build.yml" exec app yarn repco ds add -r default urn:repco:datasource:cba https://cba.media/wp-json/wp/v2
+# restart app container so it runs in a loop
+docker compose -f "docker/docker-compose.build.yml" restart app
+```
+
 ### Logging
 
 To enable debug output, set `LOG_LEVEL=debug` environment variable.

docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - "./data/postgres:/var/lib/postgresql/data"
     ports:
       - 5432:5432
+    command: [ "postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=4" ]
     environment:
       POSTGRES_PASSWORD: repco
       POSTGRES_USER: repco
@@ -23,3 +24,86 @@ services:
       - MEILI_MASTER_KEY=${MEILISEARCH_API_KEY}
     volumes:
       - ./data/meilisearch:/meili_data
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01:/usr/share/elasticsearch/data
+    ports:
+      - ${ELASTIC_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${ELASTIC_CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
+    mem_limit: ${ELASTIC_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --user elastic:${ELASTIC_PASSWORD} -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: redis
+    command: [ "redis-server", "--requirepass", "${REDIS_PASSWORD}" ]
+    volumes:
+      - ./data/redis:/data
+    ports:
+      - '6379:6379'
+
+  pgsync:
+    build:
+      context: ./pgsync
+    volumes:
+      - ./data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: "pgsync"
+      org.label-schema.description: "Postgres to Elasticsearch sync"
+      com.label-schema.service-type: "daemon"
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=${ELASTIC_PORT}
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=100
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=3242880
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=1
+      - ELASTICSEARCH_STREAMING_BULK=True
+      - ELASTICSEARCH_THREAD_COUNT=1
+      - ELASTICSEARCH_TIMEOUT=320
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=${REDIS_PASSWORD}
+      - REDIS_READ_CHUNK_SIZE=100
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data

package.json

@@ -37,6 +37,7 @@
     "migrate:dev": "yarn --cwd packages/repco-prisma prisma migrate dev && yarn --cwd packages/repco-graphql export-schema",
     "codegen": "yarn migrate:dev",
     "server": "yarn --cwd packages/repco-server start",
+    "graphql": "yarn --cwd packages/repco-graphql export-schema && yarn --cwd packages/repco-graphql start",
     "cli": "node packages/repco-cli/bin.js",
     "lint": "eslint packages",
     "lint:fix": "eslint packages --fix",

packages/repco-cli/bin.js

@@ -1,5 +1,4 @@
 #!/usr/bin/env node
-
 import 'source-map-support/register.js'
 // TODO: bundle build is broken because we now use multiple prisma clients
 // (one in repco-prisma, one in repco-activitypub)

packages/repco-cli/src/commands/debug.ts

@@ -82,6 +82,7 @@ function createItem() {
       contentFormat: 'text/plain',
       title: casual.catch_phrase,
       content: casual.sentences(3),
+      summary: '{}',
     },
   }
   return item