Translations

README.md

@@ -10,9 +10,9 @@ Note: These are priliminary docs for how to run Repco in a developer's setup. Do
 
 #### Requirements
 
-* Node.js v18+
-* yarn v1 (yarn classic)
-* Docker and Docker Compose
+- Node.js v18+
+- yarn v1 (yarn classic)
+- Docker and Docker Compose
 
 ### Development setup
 
@@ -36,10 +36,9 @@ yarn repco repo create default
 # add datasource giving a config in json format
 yarn repco ds add -r <repo> <plugin-name> <config>
 # for example the cba plugin - need to define the api key for cba in .env file
-yarn repco ds add -r default repco:datasource:cba '{"endpoint": "https://cba.fro.at/wp-json/wp/v2"}'
+yarn cli ds add -r default repco:datasource:cba https://cba.media/wp-json/wp/v2
 # or add a peertube channel
 yarn repco ds add -r default repco:datasource:activitypub '{"user":"root_channel","domain":"https://your-peertube-server.org"}'
-
 # ingest updates from all datasources
 yarn repco ds ingest
 # print all revisions in a repo

docker-compose.yml

@@ -1,25 +1,114 @@
-
 version: '3.1'
 
 services:
-
   db:
     image: postgres
+    container_name: repco-db
     # restart: always
     volumes:
-      - "./data/postgres:/var/lib/postgresql/data"
+      - './data/postgres:/var/lib/postgresql/data'
     ports:
       - 5432:5432
+    command:
+      ['postgres', '-c', 'wal_level=logical', '-c', 'max_replication_slots=4']
     environment:
       POSTGRES_PASSWORD: repco
       POSTGRES_USER: repco
       POSTGRES_DB: repco
 
   meilisearch:
     image: getmeili/meilisearch:v1.0
-    ports: 
+    ports:
       - 7700:7700
     environment:
       - MEILI_MASTER_KEY=${MEILISEARCH_API_KEY}
     volumes:
       - ./data/meilisearch:/meili_data
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    container_name: repco-es
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01:/var/lib/elasticsearch/data
+    ports:
+      - ${ELASTIC_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${ELASTIC_CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
+      - ES_JAVA_OPTS=-Xms750m -Xmx750m
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+    #mem_limit: 1073741824
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "curl -s --user elastic:${ELASTIC_PASSWORD} -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: repco-redis
+    command: ['redis-server', '--requirepass', '${REDIS_PASSWORD}']
+    volumes:
+      - ./data/redis:/data
+    ports:
+      - '6379:6379'
+
+  pgsync:
+    build:
+      context: ./pgsync
+    container_name: repco-pgsync
+    volumes:
+      - ./data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: 'pgsync'
+      org.label-schema.description: 'Postgres to Elasticsearch sync'
+      com.label-schema.service-type: 'daemon'
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=9200
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=100
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=3242880
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=1
+      - ELASTICSEARCH_STREAMING_BULK=True
+      - ELASTICSEARCH_THREAD_COUNT=1
+      - ELASTICSEARCH_TIMEOUT=320
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=${REDIS_PASSWORD}
+      - REDIS_READ_CHUNK_SIZE=100
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data

docker/docker-compose.arbeit-showcase.build.yml

@@ -0,0 +1,136 @@
+version: '3.1'
+
+services:
+  app:
+    build:
+      context: '..'
+      dockerfile: './docker/Dockerfile'
+    container_name: repco-showcase-app
+    restart: unless-stopped
+    ports:
+      - 8767:8765
+    # links:
+    #   - 'es01:repco-es'
+    environment:
+      - DATABASE_URL=postgresql://repco:repco@db:5432/repco
+      - REPCO_ADMIN_TOKEN=kUBY0zsPHC9ubj3T6DZJKUACi3M=
+      - REPCO_URL=http://localhost:8765
+      - CBA_API_KEY=k8WHfNbal0rjIs2f
+      - AP_BASE_URL=http://localhost:8765/ap
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres
+    container_name: repco-showcase-db
+    restart: unless-stopped
+    volumes:
+      - './data/postgres-showcase:/var/lib/postgresql/data'
+    expose:
+      - 5432
+    ports:
+      - 5435:5432
+    command:
+      ['postgres', '-c', 'wal_level=logical', '-c', 'max_replication_slots=4']
+    environment:
+      POSTGRES_PASSWORD: repco
+      POSTGRES_USER: repco
+      POSTGRES_DB: repco
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U repco']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.4
+    container_name: repco-showcase-es
+    restart: unless-stopped
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01-showcase:/var/lib/elasticsearch/data
+    ports:
+      - 9202:9200
+    environment:
+      - node.name=es01
+      - cluster.name=repco-showcase-es
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=repco
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=basic
+      - ES_JAVA_OPTS=-Xms750m -Xmx4g
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+      #mem_limit: 1073741824
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "curl -s --user elastic:repco -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: repco-showcase-redis
+    restart: unless-stopped
+    command: ['redis-server', '--requirepass', 'repco']
+    volumes:
+      - ./data/redis-showcase:/data
+    ports:
+      - '6380:6379'
+
+  pgsync:
+    build:
+      context: ../pgsync
+    container_name: repco-showcase-pgsync
+    restart: unless-stopped
+    volumes:
+      - /var/www/vhosts/arbeit.cba.media/httpdocs/repco-showcase/repco/docker/data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: 'pgsync'
+      org.label-schema.description: 'Postgres to Elasticsearch sync'
+      com.label-schema.service-type: 'daemon'
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - CONSOLE_LOGGING_HANDLER_MIN_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=9200
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=2000
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=104857600
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=4
+      - ELASTICSEARCH_STREAMING_BULK=False
+      - ELASTICSEARCH_THREAD_COUNT=4
+      - ELASTICSEARCH_TIMEOUT=10
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=repco
+      - REDIS_READ_CHUNK_SIZE=1000
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data