Mock data for E2E tests

.github/workflows/e2e_tests.yml

@@ -12,35 +12,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          repository: datalens-tech/datalens
-          path: datalens
-      - uses: actions/checkout@v4
-        with:
-          path: datalens-ui
       - uses: actions/setup-node@v3
         with:
           node-version: 18
           cache: 'npm'
-          cache-dependency-path: ./datalens-ui/package-lock.json
+          cache-dependency-path: ./package-lock.json
 
       - name: Run Docker Compose and Install dependencies for Datalens-UI
-        working-directory: ./datalens
-        run: docker compose -f ./docker-compose.yml -f ../datalens-ui/tests/docker-compose.override.yml up -d & cd ../datalens-ui && npm ci && npm run test:install:chromium & wait
+        working-directory: ./tests
+        run: docker compose -f ./docker-compose.e2e.yml up -d & cd .. && npm ci && npm run test:install:chromium & wait
 
       - name: Run Test 
-        working-directory: ./datalens-ui
         run: E2E_DOMAIN="http://localhost:8080" NO_AUTH=true npm run test:e2e:opensource
 
       - name: Run Docker Shutdown
-        working-directory: ./datalens
-        run: docker compose down
+        working-directory: ./tests
+        run: docker compose -f ./docker-compose.e2e.yml down
 
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-report
-          path: ./datalens-ui/tests/artifacts/
+          path: ./tests/artifacts/
           retention-days: 30
 
   publish_report:

.gitignore

@@ -22,4 +22,7 @@ secrets/*
 !secrets/.keep
 
 tests/dist/*
-src/i18n/keysets
+src/i18n/keysets
+
+tests/metadata
+tests/certs

tests/backend-configs/api.yaml

@@ -0,0 +1,37 @@
+---
+CONNECTORS:
+  CHYT:
+    CONN_CHYT_DEFAULT_CLIQUE: '*ch_public'
+    CONN_CHYT_FORBIDDEN_CLIQUES: []
+    CONN_CHYT_PUBLIC_CLIQUES: []
+CONNECTOR_AVAILABILITY:
+  sections: []
+  uncategorized:
+    - availability: free
+      conn_type: clickhouse
+    - availability: free
+      conn_type: postgres
+    - availability: free
+      conn_type: chyt
+  visible_connectors: []
+CORS_ALLOWED_HEADERS:
+- Content-Type
+- X-Request-ID
+- X-DL-TenantId
+- X-CSRF-Token
+- X-DL-Allow-Superuser
+- X-DL-Sudo
+CORS_ALLOWED_ORIGINS:
+- '*'
+CORS_EXPOSE_HEADERS:
+- X-Request-ID
+CSRF_HEADER_NAME: X-CSRF-Token
+CSRF_METHODS:
+- POST
+- PUT
+- DELETE
+CSRF_TIME_LIMIT: 43200
+DL_CRY_ACTUAL_KEY_ID: '0'
+DL_CRY_KEYMAP: {}
+SENTRY_DSN_DATASET_API: ''
+SENTRY_ENABLED: false

tests/data/superstore-sales.sql → tests/data/sales-db.sql

@@ -29,27 +29,7 @@ CREATE TABLE sales (
     sales TEXT NOT NULL 
 );
 
-COPY sales (
-    id,
-    order_id,
-    order_date,
-    ship_date,
-    ship_mode,
-    customer_id,
-    customer_name,
-    segment,
-    country,
-    city,
-    state,
-    postal_code,
-    region,
-    product_id,
-    category,
-    sub_category,
-    product_name,
-    sales
-)
-FROM './superstore_sales.csv' DELIMITER ',' CSV HEADER;
+COPY sales FROM '/docker-entrypoint-initdb.d/superstore-sales.csv' DELIMITER ',' CSV HEADER;
 
 ALTER TABLE ONLY sales ADD CONSTRAINT sales_pkey PRIMARY KEY (id);
 

tests/docker-compose.e2e.yml

@@ -0,0 +1,123 @@
+version: "3.7"
+
+services:
+  pg-compeng:
+    container_name: datalens-pg-compeng
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_PASSWORD: "postgres"
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+
+  control-api:
+    container_name: datalens-control-api
+    image: ghcr.io/datalens-tech/datalens-control-api:0.2037.0
+    environment:
+      BI_API_UWSGI_WORKERS_COUNT: 4
+      CONNECTOR_AVAILABILITY_VISIBLE: "clickhouse,postgres,chyt"
+      RQE_FORCE_OFF: 1
+      DL_CRY_ACTUAL_KEY_ID: key_1
+      DL_CRY_KEY_VAL_ID_key_1: "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk="
+      RQE_SECRET_KEY: ""
+      US_HOST: "http://us"
+      US_MASTER_TOKEN: "fake-us-master-token"
+      CONFIG_PATH: "/etc/backend-configs/api.yaml"
+    ports:
+      - 8081:8080
+    depends_on:
+      - us
+    volumes:
+      - ./backend-configs:/etc/backend-configs
+
+  data-api:
+    container_name: datalens-data-api
+    image: ghcr.io/datalens-tech/datalens-data-api:0.2037.0
+    environment:
+      GUNICORN_WORKERS_COUNT: 5
+      RQE_FORCE_OFF: 1
+      CACHES_ON: 0
+      MUTATIONS_CACHES_ON: 0
+      RQE_SECRET_KEY: ""
+      DL_CRY_ACTUAL_KEY_ID: key_1
+      DL_CRY_KEY_VAL_ID_key_1: "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk="
+      BI_COMPENG_PG_ON: 1
+      BI_COMPENG_PG_URL: "postgresql://postgres:postgres@pg-compeng:5432/postgres"
+      US_HOST: "http://us"
+      US_MASTER_TOKEN: "fake-us-master-token"
+      CONFIG_PATH: "/etc/backend-configs/api.yaml"
+    ports:
+      - 8082:8080
+    depends_on:
+      - us
+      - pg-compeng
+    volumes:
+      - ./backend-configs:/etc/backend-configs
+
+  pg-us:
+    container_name: datalens-pg-us
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_DB: us-db-ci_purgeable
+      POSTGRES_USER: us
+      POSTGRES_PASSWORD: us
+    volumes:
+      - ./metadata:/var/lib/postgresql/data
+
+  postgres-e2e:
+    container_name: postgres-e2e
+    image: postgres:13-alpine
+    environment:
+      POSTGRES_PASSWORD: "e2e"
+      POSTGRES_DB: sales
+      POSTGRES_USER: e2e
+    volumes:
+      - e2e-data-volume:/docker-entrypoint-initdb.d
+
+  us:
+    container_name: datalens-us
+    image: ghcr.io/datalens-tech/datalens-us:0.107.0
+    depends_on:
+      - pg-us
+    environment:
+      APP_INSTALLATION: "opensource"
+      APP_ENV: "prod"
+      MASTER_TOKEN: "fake-us-master-token"
+      POSTGRES_DSN_LIST: ${METADATA_POSTGRES_DSN_LIST:-postgres://us:us@pg-us:5432/us-db-ci_purgeable}
+      SKIP_INSTALL_DB_EXTENSIONS: ${METADATA_SKIP_INSTALL_DB_EXTENSIONS:-0}
+      USE_DEMO_DATA: 0
+      USE_E2E_MOCK_DATA: 1
+      HC: ${HC:-0}
+      NODE_EXTRA_CA_CERTS: /certs/root.crt
+    ports:
+      - "50500:80"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./certs:/certs
+      - e2e-data-volume:/opt/e2e-data
+
+  datalens:
+    container_name: datalens-ui
+    build:
+      dockerfile: Dockerfile
+      context: ../
+      network: host
+    ports:
+      - 8080:80
+    depends_on:
+      - us
+      - control-api
+      - data-api
+      - postgres-e2e
+    environment:
+      APP_MODE: "full"
+      APP_ENV: "production"
+      APP_INSTALLATION: "opensource"
+      AUTH_POLICY: "disabled"
+      NODE_EXTRA_CA_CERTS: "/usr/local/share/ca-certificates/cert.pem"
+      HC: ${HC:-0}
+    volumes:
+      - e2e-data-volume:/opt/app/tests/data
+
+volumes:
+  e2e-data-volume: