feat: Redis to ES migration

.env.sample

@@ -1,3 +1,6 @@
 REDIS_PORT=6379
 REDIS_INSIGHT_PORT=8001
+ES_HOSTS=http://localhost:9200
+ES_PORT=9200
+ES_PASSOWRD=changeme
 PORT=8000

.github/workflows/test.yml

@@ -5,47 +5,35 @@ on: ["pull_request", "push"]
 jobs:
   test:
     runs-on: ubuntu-latest
-
-    services:
-      redis:
-        image: "redis/redis-stack:6.2.6-v7"
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
     strategy:
       matrix:
         python-version: ["3.11"]
-        poetry-version: ["1.6.1"]
-
+        poetry-version: ["1.8.2"]
     steps:
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+      - name: Runs Elasticsearch
+        uses: elastic/elastic-github-actions/elasticsearch@master
+        with:
+          stack-version: 8.3.0
+          elasticsearch_password: changeme
       - uses: actions/checkout@v4
-
-      - name: Install dependencies
-        run: sudo apt-get -yqq install build-essential
-
       - name: Setup poetry
-        uses: abatilo/actions-poetry@v2
+        uses: abatilo/actions-poetry@v3
         with:
           poetry-version: ${{ matrix.poetry-version }}
-
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "poetry"
-
-      - name: Install Python dependencies
-        env:
-          PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
+      - name: Poetry install
         run: poetry install
-
       - name: Run tests
         env:
-          REDIS_OM_URL: redis://localhost:6379
-          REDIS_CACHE_URL: redis://localhost:6379
-          ARQ_REDIS_URL: redis://localhost:6379
+          ES_HOSTS: http://localhost:9200
+          ES_PASSWORD: changeme
         run: poetry run pytest

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-toml
@@ -11,49 +11,28 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/python-poetry/poetry
-    rev: 1.6.1
+    rev: 1.8.0
     hooks:
       - id: poetry-check
 
   - repo: https://github.com/andrei-shabanski/poetry-plugin-sort
-    rev: v0.2.0
+    rev: v0.2.1
     hooks:
       - id: poetry-sort
 
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
-    hooks:
-      - id: pyupgrade
-        args: [--py311-plus]
-
-  - repo: https://github.com/pycqa/autoflake
-    rev: v2.2.1
-    hooks:
-      - id: autoflake
-        args:
-          [
-            "--in-place",
-            "--remove-all-unused-imports",
-            "--remove-unused-variable",
-          ]
-
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.292
+    rev: v0.3.5
     hooks:
       - id: ruff
         args:
           - --fix
-
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
-    hooks:
-      - id: black
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.6.0
+    rev: v1.9.0
     hooks:
       - id: mypy
         additional_dependencies:
           - types-all
           - returns==0.22.0
-          - pydantic==1.10.12
+          - pydantic

Dockerfile

@@ -0,0 +1,28 @@
+# base
+FROM python:3.11-slim-bookworm as base
+
+RUN apt-get update \
+	&& apt-get install --no-install-recommends -y build-essential
+
+COPY requirements.txt ./
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY pyproject.toml poetry.lock ./
+
+RUN poetry config virtualenvs.create false && \
+	poetry install --without dev
+
+# main
+FROM python:3.11-slim-bookworm
+
+WORKDIR /usr/src/app
+
+COPY --from=base /usr/local/bin /usr/local/bin
+COPY --from=base /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+
+RUN apt-get update \
+	&& apt-get install --no-install-recommends -y git
+
+COPY mihama ./mihama
+COPY pyproject.toml gunicorn.conf.py ./

README.md

@@ -1,26 +1,39 @@
 # mihama
 
-[osv.dev](https://osv.dev/) API clone for on-premise usage with extra features.
+An [osv.dev](https://osv.dev/) clone for on-premise usage.
 
 ## Why?
 
-osv.dev is an OSS project. ([google/osv.dev](https://github.com/google/osv.dev))
+[google/osv.dev](https://github.com/google/osv.dev) is an OSS project. But it's tightly coupled with GCP. This makes it difficult for a non-GCP user is difficult to deploy their own osv.dev in their premise.
 
-But it is highly coupled with GCP. So it's difficult to deploy osv.dev on premise. (ref. [Deploy osv.dev on-premise #546](https://github.com/google/osv.dev/issues/546))
+## Overview
 
-## Supported Sources
+```mermaid
+flowchart LR
+    FastAPI --> Elasticsearch[(Elasticsearch)]
+    ARQ --> Elasticsearch
+    ARQ --> Redis
+    ARQ --> OSV-data-sources[(OSV data sources)]
+```
 
-- [google/osv.dev](https://github.com/google/osv.dev)
-- [ossf/malicious-packages](https://github.com/ossf/malicious-packages)
+- Elasticsearch as a database
+- FastAPI as an backend API
+- ARQ as a job queue (for periodic OSV data updates)
+  - Redis is required to run ARQ.
+- OSV data sources:
+  - [google/osv.dev](https://github.com/google/osv.dev)
+  - [ossf/malicious-packages](https://github.com/ossf/malicious-packages)
 
-## Extra features
+## Extra Features
 
 - Query by [CycloneDX](https://cyclonedx.org/) SBOM
 - Query by [SPDX](https://spdx.dev/) SBOM
 
+## Known Limitations
+
+- Query by `commit` is not supported.
+
 ## Docs
 
 - [Installation](https://github.com/ninoseki/mihama/wiki/Installation)
-- [Configuration](https://github.com/ninoseki/mihama/wiki/Configuration)
-- [Known Limitations](https://github.com/ninoseki/mihama/wiki/Known-Limitations)
 - [CLI](https://github.com/ninoseki/mihama/wiki/CLI)

docker-compose.yml

@@ -1,44 +1,117 @@
 version: "3"
 services:
+  es:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
+    container_name: elasticsearch
+    restart: always
+    environment:
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+      - xpack.security.enabled=true
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-changeme}
+    volumes:
+      - es-data:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT:-9200}:9200
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl --silent --fail -u elastic:${ELASTIC_PASSWORD:-changeme} localhost:9200/_cluster/health || exit 1",
+        ]
+      interval: 30s
+      timeout: 30s
+      retries: 3
+
+  setup:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
+    environment:
+      - KIBANA_PASSWORD=${KIBANA_PASSWORD:-changeme}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-changeme}
+    command: >
+      bash -c '
+        echo "Waiting for Elasticsearch availability";j
+        until curl -s http://es:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+        echo "Setting kibana_system password";
+        until curl -s -X POST -u "elastic:${ELASTIC_PASSWORD:-changeme}" -H "Content-Type: application/json" http://es:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD:-changeme}\"}" | grep -q "^{}"; do sleep 10; done;
+        echo "All done!";
+      '
+    depends_on:
+      es:
+        condition: service_healthy
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.13.0
+    restart: always
+    ports:
+      - ${KIBANA_PORT:-5601}:5601
+    environment:
+      - ELASTICSEARCH_HOSTS=http://es:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD:-changeme}
+      - KIBANA_PASSWORD=${KIBANA_PASSWORD:-changeme}
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl --silent --fail -u elastic:${ELASTIC_PASSWORD:-changeme} http://localhost:5601/",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    depends_on:
+      es:
+        condition: service_healthy
+      setup:
+        condition: service_completed_successfully
+
   redis:
     image: "redis/redis-stack:6.2.6-v7"
     restart: always
     ports:
       - ${REDIS_PORT:-6379}:6379
       - ${REDIS_INSIGHT_PORT:-8001}:8001
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
     volumes:
       - redis-data:/data
 
   worker:
     build:
       context: ./
-      dockerfile: default.Dockerfile
-    command: poetry run arq mihama.arq.worker.ArqWorkerSettings
+      dockerfile: Dockerfile
+    command: arq mihama.arq.worker.ArqWorkerSettings
     environment:
-      - TESTING=${TESTING:-False}
-      - DEBUG=${DEBUG:-False}
-      - REDIS_OM_URL=redis://redis:${REDIS_PORT:-6379}
-      - ARQ_REDIS_URL=redis://redis:${REDIS_PORT:-6379}
+      - REDIS_URL=redis://redis:${REDIS_PORT:-6379}
+      - ES_HOSTS=http://es:9200
+      - ES_PASSOWRD=${ES_PASSOWRD:-changeme}
     restart: always
     depends_on:
-      - redis
+      es:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
 
   api:
     build:
       context: ./
-      dockerfile: default.Dockerfile
-    command: poetry run gunicorn -k uvicorn.workers.UvicornWorker mihama.main:app
+      dockerfile: Dockerfile
+    command: gunicorn -k uvicorn.workers.UvicornWorker mihama.main:app
     ports:
       - ${PORT:-8000}:${PORT:-8000}
     environment:
-      - TESTING=${TESTING:-False}
-      - DEBUG=${DEBUG:-False}
-      - PORT=${PORT:-8000}
-      - REDIS_OM_URL=redis://redis:${REDIS_PORT:-6379}
-      - REDIS_ARQ_URL=redis://redis:${REDIS_PORT:-6379}
+      - REDIS_URL=redis://redis:${REDIS_PORT:-6379}
+      - ES_HOSTS=http://es:9200
+      - ES_PASSOWRD=${ES_PASSOWRD:-changeme}
     restart: always
     depends_on:
-      - redis
+      es:
+        condition: service_healthy
 
 volumes:
   redis-data:
+  es-data:

mihama/api/v1/endpoints/cyclonedx.py

@@ -1,17 +1,16 @@
 from fastapi import APIRouter
 
-from mihama import schemas
-from mihama.api.v1.query import batch_query
+from mihama import crud, deps, schemas
 
 router = APIRouter()
 
 
 @router.post(
     "/querybatch",
-    response_model=schemas.BatchResponse,
     response_model_exclude_none=True,
     description="Query vulnerabilities by CycloneDX SBOM. (Components inside should have package URL to work)",
 )
-async def querybatch(bom: schemas.CycloneDX) -> schemas.BatchResponse:
-    queries = bom.to_batch_query()
-    return await batch_query(queries=queries)
+async def querybatch(
+    bom: schemas.CycloneDX, *, es: deps.Elasticsearch
+) -> schemas.BatchResponse:
+    return await crud.vulnerability.querybatch(es, queries=bom.to_queries())