Main Pipeline

.github/workflows/pipeline-tests.yml

@@ -0,0 +1,128 @@
+name: Pipeline Tests
+
+on:
+  push:
+    branches: [ main, development ]
+  pull_request:
+    branches: [ main, development ]
+
+jobs:
+  test-minimal:
+    name: Minimal Pipeline Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: false  # avoid .gitmodules errors
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: pip install -r tests/requirements-minimal.txt
+
+      - name: Run minimal pipeline tests (no external services)
+        run: python -m pytest tests/pipeline/test_minimal_pipeline.py tests/pipeline/test_components.py -v --tb=short
+
+      - name: Run configuration validation
+        run: |
+          python -c "
+          import yaml, sys
+          configs = ['config.yml', 'pipelines/configs/retrieval/ci_google_gemini.yml']
+          for config in configs:
+              try:
+                  with open(config) as f:
+                      yaml.safe_load(f)
+                  print(f'{config} is valid')
+              except Exception as e:
+                  print(f'{config} failed: {e}')
+                  sys.exit(1)
+          "
+
+  test-integration:
+    name: Integration Tests with Qdrant
+    runs-on: ubuntu-latest
+
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333  # REST
+          - 6334:6334  # gRPC
+        env:
+          QDRANT__SERVICE__HTTP_PORT: 6333
+          QDRANT__SERVICE__GRPC_PORT: 6334
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: pip install -r tests/requirements-minimal.txt
+
+      - name: Wait for Qdrant to be ready (readiness + API check)
+        run: |
+          for i in {1..60}; do
+            if curl -fsS http://127.0.0.1:6333/readyz > /dev/null && \
+               curl -fsS http://127.0.0.1:6333/collections > /dev/null; then
+              echo "Qdrant is ready!"
+              exit 0
+            fi
+            echo "Waiting for Qdrant ($i/60)..."
+            sleep 2
+          done
+          echo "Qdrant did not become ready in time"
+          exit 1
+
+      - name: Test Qdrant connectivity
+        run: python -m pytest tests/pipeline/test_qdrant_connectivity.py -v --tb=short
+
+      - name: Run basic integration tests
+        run: python -m pytest tests/pipeline/ -v --tb=short -m "not requires_api"
+
+  test-security:
+    name: Security and Config Validation
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Check for hardcoded secrets
+        run: |
+          if grep -r "sk-" . --exclude-dir=.git --exclude="*.md" --exclude="*.yml"; then
+            echo "Found potential hardcoded API keys"
+            exit 1
+          fi
+          if grep -r "google_api_key.*=" . --exclude-dir=.git --exclude="*.md" --exclude="*.yml" | grep -v "getenv\|environ"; then
+            echo "Found potential hardcoded Google API keys"
+            exit 1
+          fi
+          echo "No hardcoded secrets found"
+
+      - name: Validate configuration structure
+        run: |
+          python -c "
+          import yaml
+          with open('pipelines/configs/retrieval/ci_google_gemini.yml') as f:
+              config = yaml.safe_load(f)
+          assert 'retrieval_pipeline' in config
+          assert 'retriever' in config['retrieval_pipeline']
+          assert 'embedding' in config['retrieval_pipeline']['retriever']
+          assert 'google' == config['retrieval_pipeline']['retriever']['embedding']['dense']['provider']
+          assert 'GOOGLE_API_KEY' == config['retrieval_pipeline']['retriever']['embedding']['dense']['api_key_env']
+          print('Configuration structure is valid')
+          "

.gitignore

@@ -19,4 +19,14 @@ climate-fever
 *.log
 __pycache__
 sandbox/*
-/__pycache__
+/__pycache__
+synthetic_dataset\text_dataset_template.json
+extraction_output/
+.idea/misc.xml
+.idea/modules.xml
+.idea/Thesis.iml
+.idea/vcs.xml
+.idea/inspectionProfiles/profiles_settings.xml
+*.json
+*.csv
+

Dockerfile

@@ -0,0 +1,18 @@
+# Use a slim Python base image
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system packages if needed
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the full source code
+COPY . .