Translations

README.md

@@ -10,9 +10,9 @@ Note: These are priliminary docs for how to run Repco in a developer's setup. Do
 
 #### Requirements
 
-* Node.js v18+
-* yarn v1 (yarn classic)
-* Docker and Docker Compose
+- Node.js v18+
+- yarn v1 (yarn classic)
+- Docker and Docker Compose
 
 ### Development setup
 

docker-compose.yml

@@ -1,25 +1,114 @@
-
 version: '3.1'
 
 services:
-
   db:
     image: postgres
+    container_name: repco-db
     # restart: always
     volumes:
-      - "./data/postgres:/var/lib/postgresql/data"
+      - './data/postgres:/var/lib/postgresql/data'
     ports:
       - 5432:5432
+    command:
+      ['postgres', '-c', 'wal_level=logical', '-c', 'max_replication_slots=4']
     environment:
       POSTGRES_PASSWORD: repco
       POSTGRES_USER: repco
       POSTGRES_DB: repco
 
   meilisearch:
     image: getmeili/meilisearch:v1.0
-    ports: 
+    ports:
       - 7700:7700
     environment:
       - MEILI_MASTER_KEY=${MEILISEARCH_API_KEY}
     volumes:
       - ./data/meilisearch:/meili_data
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    container_name: repco-es
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01:/var/lib/elasticsearch/data
+    ports:
+      - ${ELASTIC_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${ELASTIC_CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
+      - ES_JAVA_OPTS=-Xms750m -Xmx750m
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+    #mem_limit: 1073741824
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "curl -s --user elastic:${ELASTIC_PASSWORD} -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: repco-redis
+    command: ['redis-server', '--requirepass', '${REDIS_PASSWORD}']
+    volumes:
+      - ./data/redis:/data
+    ports:
+      - '6379:6379'
+
+  pgsync:
+    build:
+      context: ./pgsync
+    container_name: repco-pgsync
+    volumes:
+      - ./data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: 'pgsync'
+      org.label-schema.description: 'Postgres to Elasticsearch sync'
+      com.label-schema.service-type: 'daemon'
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=9200
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=100
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=3242880
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=1
+      - ELASTICSEARCH_STREAMING_BULK=True
+      - ELASTICSEARCH_THREAD_COUNT=1
+      - ELASTICSEARCH_TIMEOUT=320
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=${REDIS_PASSWORD}
+      - REDIS_READ_CHUNK_SIZE=100
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data

docker/docker-compose.build.yml

@@ -1,31 +1,134 @@
-
 version: '3.1'
 
 services:
   app:
     build:
       context: '..'
       dockerfile: './docker/Dockerfile'
+    container_name: repco-app
+    restart: unless-stopped
     ports:
       - 8766:8765
+    # links:
+    #   - 'es01:repco-es'
     environment:
       - DATABASE_URL=postgresql://repco:repco@db:5432/repco
+      - REPCO_ADMIN_TOKEN=kUBY0zsPHC9ubj3T6DZJKUACi3M=
+      - REPCO_URL=http://localhost:8765
+      - CBA_API_KEY=k8WHfNbal0rjIs2f
+      - AP_BASE_URL=http://localhost:8765/ap
     depends_on:
       db:
         condition: service_healthy
 
   db:
     image: postgres
-    # volumes:
-    #     - "/tmp/repco/postgres:/var/lib/postgresql/data"
+    container_name: repco-db
+    restart: unless-stopped
+    volumes:
+      - './data/postgres:/var/lib/postgresql/data'
     expose:
       - 5432
+    command:
+      ['postgres', '-c', 'wal_level=logical', '-c', 'max_replication_slots=4']
     environment:
       POSTGRES_PASSWORD: repco
       POSTGRES_USER: repco
       POSTGRES_DB: repco
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U repco"]
+      test: ['CMD-SHELL', 'pg_isready -U repco']
       interval: 5s
       timeout: 5s
       retries: 5
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.4
+    container_name: repco-es
+    restart: unless-stopped
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01:/var/lib/elasticsearch/data
+    ports:
+      - 9201:9200
+    environment:
+      - node.name=es01
+      - cluster.name=repco-es
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=repco
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=basic
+      - ES_JAVA_OPTS=-Xms750m -Xmx4g
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+      #mem_limit: 1073741824
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "curl -s --user elastic:repco -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: repco-redis
+    restart: unless-stopped
+    command: ['redis-server', '--requirepass', 'repco']
+    volumes:
+      - ./data/redis:/data
+    ports:
+      - '6379:6379'
+
+  pgsync:
+    build:
+      context: ../pgsync
+    container_name: repco-pgsync
+    restart: unless-stopped
+    volumes:
+      - /var/www/repco.cba.media/repco/docker/data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: 'pgsync'
+      org.label-schema.description: 'Postgres to Elasticsearch sync'
+      com.label-schema.service-type: 'daemon'
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - CONSOLE_LOGGING_HANDLER_MIN_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=9200
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=2000
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=104857600
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=4
+      - ELASTICSEARCH_STREAMING_BULK=False
+      - ELASTICSEARCH_THREAD_COUNT=4
+      - ELASTICSEARCH_TIMEOUT=10
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=repco
+      - REDIS_READ_CHUNK_SIZE=1000
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data

docker/docker-compose.yml

@@ -3,6 +3,8 @@ version: '3.1'
 services:
   app:
     image: arsoxyz/repco:main
+    container_name: repco-app
+    restart: unless-stopped
     expose:
       - 8765
     ports:
@@ -15,16 +17,110 @@ services:
 
   db:
     image: postgres
+    container_name: repco-db
+    restart: unless-stopped
     volumes:
-        - ./data/repco-db:/var/lib/postgresql/data"
+      - ./data/repco-db:/var/lib/postgresql/data"
     expose:
       - 5432
+    command:
+      ['postgres', '-c', 'wal_level=logical', '-c', 'max_replication_slots=4']
     environment:
       POSTGRES_PASSWORD: repco
       POSTGRES_USER: repco
       POSTGRES_DB: repco
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U repco"]
+      test: ['CMD-SHELL', 'pg_isready -U repco']
       interval: 5s
       timeout: 5s
       retries: 5
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    container_name: repco-es
+    restart: unless-stopped
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - ./data/elastic/es01:/var/lib/elasticsearch/data
+    ports:
+      - 9201:${ELASTIC_PORT}
+    environment:
+      - node.name=es01
+      - cluster.name=${ELASTIC_CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
+      - ES_JAVA_OPTS=-Xms750m -Xmx4g
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "curl -s --user elastic:${ELASTIC_PASSWORD} -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  redis:
+    image: 'redis:alpine'
+    container_name: repco-redis
+    restart: unless-stopped
+    command: ['redis-server', '--requirepass', '${REDIS_PASSWORD}']
+    volumes:
+      - ./data/redis:/data
+    ports:
+      - '6379:6379'
+
+  pgsync:
+    build:
+      context: ../pgsync
+    container_name: repco-pgsync
+    volumes:
+      - /var/www/repco.cba.media/repco/docker/data/pgsync:/data
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=200
+      - net.ipv4.tcp_keepalive_intvl=200
+      - net.ipv4.tcp_keepalive_probes=5
+    labels:
+      org.label-schema.name: 'pgsync'
+      org.label-schema.description: 'Postgres to Elasticsearch sync'
+      com.label-schema.service-type: 'daemon'
+    depends_on:
+      - db
+      - es01
+      - redis
+    environment:
+      - PG_USER=repco
+      - PG_HOST=db
+      - PG_PORT=5432
+      - PG_PASSWORD=repco
+      - PG_DATABASE=repco
+      - LOG_LEVEL=DEBUG
+      - CONSOLE_LOGGING_HANDLER_MIN_LEVEL=DEBUG
+      - ELASTICSEARCH_PORT=9200
+      - ELASTICSEARCH_SCHEME=http
+      - ELASTICSEARCH_HOST=es01
+      - ELASTICSEARCH_CHUNK_SIZE=2000
+      - ELASTICSEARCH_MAX_CHUNK_BYTES=104857600
+      - ELASTICSEARCH_MAX_RETRIES=14
+      - ELASTICSEARCH_QUEUE_SIZE=4
+      - ELASTICSEARCH_STREAMING_BULK=False
+      - ELASTICSEARCH_THREAD_COUNT=4
+      - ELASTICSEARCH_TIMEOUT=10
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_AUTH=repco
+      - REDIS_READ_CHUNK_SIZE=1000
+      - ELASTICSEARCH=true
+      - OPENSEARCH=false
+      - SCHEMA=/data
+      - CHECKPOINT_PATH=/data