diff --git a/README.md b/README.md index c0932eb..4c7e3bb 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,49 @@ # Gradle Build Cache Server -A high-performance, Kubernetes-native HTTP build cache server for Gradle builds. Built with Go and backed by MinIO for reliable, persistent storage. - +A high-performance, Kubernetes-native HTTP build cache server for Gradle builds. Built with Go and backed by Redis for fast, in-memory storage. ## Overview -This project provides a lightweight, self-hosted Gradle Build Cache server implementation that can be deployed in Kubernetes clusters to accelerate build times across teams and CI/CD pipelines. It implements the Gradle HTTP Build Cache API and uses MinIO as a persistent storage backend. +This project provides a lightweight, self-hosted Gradle Build Cache server that can be deployed in Kubernetes clusters to accelerate build times across teams and CI/CD pipelines. It implements the Gradle HTTP Build Cache API and uses Redis as an in-memory storage backend. ### Key Features - **Gradle HTTP Build Cache API** - Fully compatible with Gradle's remote cache protocol -- **Persistent Storage** - Uses MinIO (S3-compatible) for reliable, scalable artifact storage -- **Authentication** - HTTP Basic Authentication for access control +- **In-Memory Storage** - Uses Redis for fast cache lookups and storage +- **Role-Based Authentication** - HTTP Basic Authentication with separate reader/writer roles - **Kubernetes-Native** - Designed for containerized deployments with production-ready Helm charts -- **Observability** - Built-in Prometheus metrics and structured logging (JSON/text) +- **Observability** - Built-in Prometheus metrics, Grafana dashboard, and structured logging +- **Dependency Proxy** - Optional Reposilite integration for caching Maven/Gradle dependencies - **Lightweight** - Minimal resource footprint (~256Mi RAM, ~100m CPU) - **Health Checks** - Kubernetes-ready liveness and readiness probes -- **Easy Deployment** - Single Helm command to get started ### Use Cases - **Team Development** - Share build cache across development teams - **CI/CD Pipelines** - Accelerate build times in Jenkins, GitLab CI, GitHub Actions, etc. - **Monorepos** - Reduce build times for large multi-module projects -- **Multi-Environment Builds** - Share cache between dev, staging, and production builds - **Cost Optimization** - Reduce build server compute costs by avoiding redundant work ## Architecture ``` -┌─────────────────────────────────────────────────────────┐ -│ Kubernetes Cluster │ -│ │ -│ ┌─────────────────┐ ┌─────────────────────┐ │ -│ │ Cache Server │──────▶│ MinIO │ │ -│ │ (Deployment) │ │ (StatefulSet) │ │ -│ │ Port: 8080 │ │ Port: 9000 │ │ -│ └────────┬────────┘ └─────────────────────┘ │ -│ │ │ -│ ┌────────▼────────┐ │ -│ │ Service │◀──── Developer Workstations │ -│ │ Port: 8080 │◀──── CI/CD Pipelines │ -│ └─────────────────┘◀──── Build Agents │ -└─────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────┐ +│ Kubernetes Cluster │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────────────┐ │ +│ │ Cache Server │──────▶│ Redis (Deployment) │ │ +│ │ (Deployment) │ │ Port: 6379 │ │ +│ │ Port: 8080 │ │ + Redis Exporter sidecar │ │ +│ └────────┬─────────┘ └──────────────────────────────┘ │ +│ │ │ +│ ┌────────▼─────────┐ ┌──────────────────────────────┐ │ +│ │ Service │ │ Reposilite (optional) │ │ +│ │ Port: 8080 │ │ Maven/Gradle dependency │ │ +│ └──────────────────┘ │ proxy with caching │ │ +│ ▲ └──────────────────────────────┘ │ +│ │ │ +│ Developer Workstations / CI/CD Pipelines │ +└──────────────────────────────────────────────────────────────┘ ``` ### Components @@ -50,8 +51,9 @@ This project provides a lightweight, self-hosted Gradle Build Cache server imple | Component | Description | Technology | |-----------|-------------|------------| | **Cache Server** | HTTP API server implementing Gradle Build Cache protocol | Go, Gin Framework | -| **Storage Backend** | S3-compatible object storage for cache artifacts | MinIO | -| **Kubernetes Service** | ClusterIP service for internal access | Kubernetes | +| **Storage Backend** | In-memory key-value store for cache artifacts | Redis 7 | +| **Redis Exporter** | Sidecar that exposes Redis metrics to Prometheus | oliver006/redis_exporter | +| **Reposilite** | Maven/Gradle dependency proxy and cache (optional) | Reposilite 3.x | | **Helm Chart** | Declarative deployment configuration | Helm 3 | ## Quick Start @@ -61,7 +63,6 @@ This project provides a lightweight, self-hosted Gradle Build Cache server imple - Kubernetes cluster (v1.19+) - Helm 3.x - kubectl configured to access your cluster -- (Optional) Domain name and TLS certificates for production deployments ### Installation @@ -71,35 +72,24 @@ git clone https://github.com/kevingruber/theia-shared-cache.git cd theia-shared-cache ``` -2. Configure your deployment by editing `chart/values.yaml`: -```yaml -cacheServer: - auth: - username: "gradle" - password: "your-secure-password" # CHANGE THIS! - -minio: - auth: - accessKey: "your-access-key" # CHANGE THIS! - secretKey: "your-secret-key" # CHANGE THIS! -``` - -3. Deploy using Helm: +2. Deploy using Helm: ```bash -helm install theia-cache ./chart +helm install gradle-cache ./chart ``` -4. Verify the deployment: +The Redis password is auto-generated on first install and stored as a Kubernetes Secret. No manual credential configuration is needed for the storage backend. + +3. Verify the deployment: ```bash kubectl get pods -kubectl logs -f deployment/theia-cache +kubectl logs -f deployment/gradle-cache-cache-server ``` ### Testing the Cache Port-forward the service to your local machine: ```bash -kubectl port-forward svc/theia-cache 8080:8080 +kubectl port-forward svc/gradle-cache-cache 8080:8080 ``` Test the health endpoint: @@ -110,43 +100,56 @@ curl http://localhost:8080/ping Test cache operations: ```bash -# Store a cache entry -echo "test data" | curl -u gradle:your-password \ - -X PUT \ +# Store a cache entry (requires writer role) +curl -X PUT -u writer:changeme-writer \ -H "Content-Type: application/octet-stream" \ - --data-binary @- \ + -d "test data" \ http://localhost:8080/cache/test-key -# Retrieve the cache entry -curl -u gradle:your-password http://localhost:8080/cache/test-key +# Retrieve the cache entry (reader or writer role) +curl -u reader:changeme-reader http://localhost:8080/cache/test-key + +# Check if entry exists +curl -I -u reader:changeme-reader http://localhost:8080/cache/test-key ``` ## Configuration ### Gradle Configuration -Configure your Gradle builds to use the remote cache by adding to `settings.gradle`: +Configure your Gradle builds to use the remote cache: +**settings.gradle.kts (Kotlin DSL):** +```kotlin +buildCache { + remote { + url = uri("http://-cache:8080/cache/") + credentials { + username = "writer" + password = "changeme-writer" + } + isPush = true // set to false for read-only access + } +} +``` + +**settings.gradle (Groovy DSL):** ```groovy buildCache { remote(HttpBuildCache) { - url = 'http://theia-cache:8080/cache/' + url = 'http://-cache:8080/cache/' credentials { - username = 'gradle' - password = 'your-password' + username = 'writer' + password = 'changeme-writer' } push = true } } ``` -Or via `gradle.properties`: +**gradle.properties:** ```properties org.gradle.caching=true -org.gradle.caching.remote.url=http://theia-cache:8080/cache/ -org.gradle.caching.remote.username=gradle -org.gradle.caching.remote.password=your-password -org.gradle.caching.remote.push=true ``` ### Helm Chart Configuration @@ -155,85 +158,46 @@ Key configuration options in `chart/values.yaml`: | Parameter | Description | Default | |-----------|-------------|---------| -| `cacheServer.replicaCount` | Number of cache server replicas | `1` | -| `cacheServer.image.repository` | Container image repository | `ghcr.io/kevingruber/theia-shared-cache/gradle-cache` | -| `cacheServer.image.tag` | Container image tag | `latest` | -| `cacheServer.tls.enabled` | Enable TLS/HTTPS | `false` | -| `cacheServer.tls.secretName` | TLS certificate secret name | `""` | -| `cacheServer.tls.certManager.enabled` | Use cert-manager for certificates | `false` | -| `cacheServer.auth.username` | Authentication username | `gradle` | -| `cacheServer.auth.password` | Authentication password | `changeme` | -| `cacheServer.config.maxEntrySizeMB` | Maximum cache entry size | `100` | -| `minio.enabled` | Deploy MinIO with the chart | `true` | -| `minio.persistence.size` | MinIO storage size | `50Gi` | -| `minio.auth.accessKey` | MinIO access key | `minioadmin` | -| `minio.auth.secretKey` | MinIO secret key | `minioadmin` | +| `enabled` | Enable/disable the entire deployment | `true` | +| `image.repository` | Cache server image | `ghcr.io/ls1intum/theia-shared-cache/gradle-cache` | +| `image.tag` | Cache server image tag | `main` | +| `auth.enabled` | Enable authentication | `true` | +| `auth.username` | Cache username | `gradle` | +| `auth.password` | Cache password | `changeme` | +| `resources.cacheServer` | Cache server resource limits | See values.yaml | +| `resources.redis` | Redis resource limits | See values.yaml | +| `tls.enabled` | Enable TLS/HTTPS | `false` | +| `tls.secretName` | TLS certificate secret name | `""` | +| `metrics.serviceMonitor.enabled` | Create ServiceMonitor for Prometheus Operator | `false` | +| `reposilite.enabled` | Deploy Reposilite dependency proxy | `true` | For a complete list of configuration options, see the [Helm chart documentation](chart/README.md). -### Enabling TLS/HTTPS - -To secure communication with TLS, you can use cert-manager or provide your own certificates. Here's a quick example: - -**With cert-manager (recommended):** -```yaml -cacheServer: - tls: - enabled: true - secretName: gradle-cache-tls - certManager: - enabled: true - issuerName: "letsencrypt-prod" - issuerKind: "ClusterIssuer" -``` - -**With self-signed certificates:** -```bash -# Generate certificate -openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -keyout tls.key -out tls.crt \ - -subj "/CN=cache-server.default.svc.cluster.local" - -# Create Kubernetes secret -kubectl create secret tls gradle-cache-tls --cert=tls.crt --key=tls.key - -# Update values.yaml -cacheServer: - tls: - enabled: true - secretName: gradle-cache-tls -``` - -For detailed TLS setup instructions, see the [TLS Setup Guide](docs/tls-setup.md). - ### Environment Variables The cache server can be configured via environment variables: | Variable | Description | Default | |----------|-------------|---------| -| `SERVER_PORT` | HTTP server port | `8080` | -| `SERVER_READ_TIMEOUT` | Request read timeout | `30s` | -| `SERVER_WRITE_TIMEOUT` | Request write timeout | `120s` | -| `MINIO_ENDPOINT` | MinIO server endpoint | `minio:9000` | -| `MINIO_ACCESS_KEY` | MinIO access key | - | -| `MINIO_SECRET_KEY` | MinIO secret key | - | -| `CACHE_PASSWORD` | Cache authentication password | - | -| `LOG_LEVEL` | Logging level (debug, info, warn, error) | `info` | -| `LOG_FORMAT` | Log format (json, text) | `json` | +| `REDIS_PASSWORD` | Redis authentication password | Auto-generated | +| `CACHE_READER_USERNAME` | Reader role username | From values.yaml | +| `CACHE_READER_PASSWORD` | Reader role password | From values.yaml | +| `CACHE_WRITER_USERNAME` | Writer role username | From values.yaml | +| `CACHE_WRITER_PASSWORD` | Writer role password | From values.yaml | +| `SENTRY_DSN` | Sentry error tracking DSN | Disabled | ## API Reference ### Endpoints -| Endpoint | Method | Description | Authentication | -|----------|--------|-------------|----------------| -| `/ping` | GET | Health check (liveness probe) | No | -| `/health` | GET | Storage health check (readiness probe) | No | -| `/metrics` | GET | Prometheus metrics | No | -| `/cache/:key` | GET | Retrieve cache entry | Yes | -| `/cache/:key` | PUT | Store cache entry | Yes | -| `/cache/:key` | HEAD | Check if cache entry exists | Yes | +| Endpoint | Method | Auth | Description | +|----------|--------|------|-------------| +| `/ping` | GET | No | Health check (liveness probe) | +| `/health` | GET | No | Storage connectivity check (readiness probe) | +| `/metrics` | GET | No | Prometheus metrics | +| `/cache/:key` | GET | reader/writer | Retrieve cache entry | +| `/cache/:key` | HEAD | reader/writer | Check if cache entry exists | +| `/cache/:key` | PUT | writer only | Store cache entry | ### HTTP Status Codes @@ -241,120 +205,156 @@ The cache server can be configured via environment variables: |------|-------------| | `200 OK` | Cache hit (GET), entry exists (HEAD) | | `201 Created` | Cache entry stored successfully (PUT) | -| `204 No Content` | Entry does not exist (HEAD) | | `401 Unauthorized` | Authentication failed | -| `404 Not Found` | Cache miss (GET) | -| `413 Payload Too Large` | Entry exceeds maximum size | +| `403 Forbidden` | Insufficient role (e.g., reader trying to PUT) | +| `404 Not Found` | Cache miss (GET/HEAD) | +| `413 Payload Too Large` | Entry exceeds maximum size (default: 100MB) | | `500 Internal Server Error` | Server or storage error | ## Development -### Building from Source - -#### Prerequisites +### Prerequisites - Go 1.24+ -- Docker (optional, for containerization) +- Docker & Docker Compose -#### Build the binary -```bash -cd src -go build -o bin/cache-server ./cmd/server -``` +### Local Development with Docker Compose + +The fastest way to run the full stack locally: -#### Run locally ```bash -# Set required environment variables -export MINIO_ENDPOINT=localhost:9000 -export MINIO_ACCESS_KEY=minioadmin -export MINIO_SECRET_KEY=minioadmin -export CACHE_PASSWORD=changeme - -# Run the server -./bin/cache-server +cd src/deployments +docker compose up --build ``` -#### Build Docker image +This starts all services: + +| Service | URL | Description | +|---------|-----|-------------| +| Cache Server | http://localhost:8080 | Gradle build cache API | +| Redis | localhost:6379 | In-memory storage | +| Redis Exporter | http://localhost:9121 | Redis metrics for Prometheus | +| Prometheus | http://localhost:9090 | Metrics collection | +| Grafana | http://localhost:3000 | Dashboards (no login required) | +| Reposilite | http://localhost:8081 | Dependency proxy | + +A pre-built Grafana dashboard ("Gradle Build Cache") is automatically provisioned with panels for cache hit rate, request latency, Redis memory usage, and more. + +### Building from Source + ```bash cd src -docker build -t theia-cache:dev . +go build -o bin/cache-server ./cmd/server ``` -### Testing +### Running Tests -Run unit tests: ```bash cd src go test ./... ``` -Run with coverage: -```bash -go test -cover ./... -``` - ### Project Structure ``` . -├── chart/ # Helm chart for Kubernetes deployment -│ ├── templates/ # Kubernetes manifests -│ ├── values.yaml # Default configuration -│ └── Chart.yaml # Chart metadata -├── src/ # Go source code -│ ├── cmd/ -│ │ └── server/ # Main application entry point +├── chart/ # Helm chart for Kubernetes deployment +│ ├── templates/ +│ │ ├── _helpers.tpl # Shared label templates +│ │ ├── deployment.yaml # Cache server Deployment +│ │ ├── redis-deployment.yaml # Redis Deployment +│ │ ├── redis-service.yaml # Redis Service +│ │ ├── service.yaml # Cache server Service +│ │ ├── configmap.yaml # Server configuration +│ │ └── secrets.yaml # Auto-generated Redis password +│ ├── values.yaml # Default configuration +│ └── Chart.yaml # Chart metadata +├── src/ # Go source code +│ ├── cmd/server/ # Application entry point │ ├── internal/ -│ │ ├── config/ # Configuration management -│ │ ├── middleware/ # HTTP middleware (auth, logging) -│ │ ├── server/ # HTTP server and routes -│ │ └── storage/ # MinIO storage backend -│ ├── Dockerfile # Multi-stage Docker build -│ └── go.mod # Go module dependencies -└── .github/ - └── workflows/ # GitHub Actions CI/CD +│ │ ├── config/ # Configuration management +│ │ ├── handler/ # HTTP handlers (GET/PUT/HEAD) +│ │ ├── middleware/ # Auth, logging, metrics middleware +│ │ ├── server/ # HTTP server and routes +│ │ ├── storage/ # Redis storage backend +│ │ └── telemetry/ # OpenTelemetry setup +│ ├── deployments/ # Docker Compose + monitoring config +│ │ ├── docker-compose.yaml +│ │ ├── prometheus.yaml +│ │ └── grafana/ # Grafana provisioning & dashboards +│ ├── configs/config.yaml # Default app configuration +│ ├── Dockerfile # Multi-stage Docker build +│ └── go.mod # Go module dependencies +└── .github/workflows/ # GitHub Actions CI/CD ``` ## Monitoring ### Prometheus Metrics -The cache server exposes Prometheus metrics at `/metrics`: +The cache server exposes metrics at `/metrics`: | Metric | Type | Description | |--------|------|-------------| -| `cache_requests_total` | Counter | Total number of cache requests | -| `cache_hits_total` | Counter | Number of cache hits | -| `cache_misses_total` | Counter | Number of cache misses | -| `cache_errors_total` | Counter | Number of cache errors | -| `http_request_duration_seconds` | Histogram | HTTP request latency | - -### ServiceMonitor - -If using Prometheus Operator, enable the ServiceMonitor in `values.yaml`: -```yaml -metrics: - enabled: true - serviceMonitor: - enabled: true +| `gradle_cache_requests_total` | Counter | Total requests by method and status | +| `gradle_cache_cache_hits` | Counter | Cache hit count | +| `gradle_cache_cache_misses` | Counter | Cache miss count | +| `gradle_cache_request_duration_seconds` | Histogram | Request latency (p50/p95/p99) | +| `gradle_cache_entry_size` | Histogram | Cache entry sizes | + +Redis metrics are exposed via the redis-exporter sidecar: + +| Metric | Type | Description | +|--------|------|-------------| +| `redis_memory_used_bytes` | Gauge | Redis memory consumption | +| `redis_db_keys` | Gauge | Number of cached entries | +| `redis_keyspace_hits_total` | Counter | Redis-level cache hits | +| `redis_keyspace_misses_total` | Counter | Redis-level cache misses | +| `redis_commands_processed_total` | Counter | Total Redis operations | + +### Grafana Dashboard + +A pre-built dashboard is included at `src/deployments/grafana/dashboards/gradle-build-cache.json`. It can be imported into any Grafana instance and includes panels for: + +- Cache hit rate (with color-coded thresholds) +- Request rate by HTTP method +- Request latency percentiles (p50, p95, p99) +- Cache hits vs misses over time +- Server error rate (5xx) +- Redis memory usage and trends +- Redis keyspace hit/miss ratio +- Redis operations per second + +### ServiceMonitor (Prometheus Operator) + +If your cluster uses the Prometheus Operator, enable automatic scrape discovery: + +```bash +helm install gradle-cache ./chart --set metrics.serviceMonitor.enabled=true ``` -## Security Considerations +Otherwise, Prometheus pod annotations are included by default for annotation-based discovery. + +## Security + +### Authentication Model -### Current Limitations +The cache server uses role-based HTTP Basic Authentication: -- **TLS Optional** - TLS/HTTPS is disabled by default. Enable it for production deployments. -- **Basic Authentication** - Simple username/password authentication. -- **Single User** - Only one set of credentials supported. -- **Single Replica** - No built-in high availability (single point of failure). +| Role | Permissions | Use Case | +|------|-------------|----------| +| **reader** | GET, HEAD | CI/CD pipelines that only consume cache | +| **writer** | GET, HEAD, PUT | Build agents that produce and consume cache | + +### Redis Password + +The Redis password is auto-generated on first `helm install` and stored in a Kubernetes Secret. Both the cache server and Redis read it from the same Secret. No human ever needs to know this password. ### Recommendations for Production -1. **Enable TLS** - Encrypt all traffic (see [TLS Setup Guide](docs/tls-setup.md)) -2. **Change default credentials** - Use strong, randomly-generated passwords +1. **Enable TLS** - Encrypt traffic between Gradle clients and the cache server +2. **Change default credentials** - Override `auth.password` with strong passwords 3. **Network Policies** - Restrict access to authorized pods only -4. **Use Kubernetes Secrets** - Never commit credentials to version control -5. **Monitor certificate expiration** - Set up alerts for certificate renewal -6. **Regular updates** - Keep dependencies and base images up to date +4. **Set Redis memory limits** - Configure `maxmemory` and `maxmemory-policy allkeys-lru` to handle cache eviction gracefully ## Troubleshooting @@ -366,15 +366,36 @@ kubectl describe pod kubectl logs ``` -**MinIO connection errors** -- Verify MinIO is running: `kubectl get pods -l app=minio` -- Check MinIO logs: `kubectl logs -l app=minio` -- Verify credentials match in both cache and MinIO configurations +**Redis connection errors** +- Verify Redis is running: `kubectl get pods -l app.kubernetes.io/component=storage` +- Check Redis logs: `kubectl logs -l app.kubernetes.io/component=storage` +- Verify the Redis Secret exists: `kubectl get secret -redis-secret` **Authentication failures** -- Ensure credentials in Gradle match those in `values.yaml` -- Check for special characters that need URL encoding +- Ensure Gradle credentials match those configured in `values.yaml` +- Check user role — readers cannot PUT, only writers can + +**Cache not helping (low hit rate)** +- Verify Gradle has `org.gradle.caching=true` in `gradle.properties` +- Check that `isPush = true` is set for at least one build agent +- Different Gradle versions or JDKs produce different cache keys + +## Upgrading + +### From 0.2.x to 0.3.0 + +Version 0.3.0 replaces MinIO with Redis as the storage backend. + +Changes: +- Storage backend changed from MinIO (S3-compatible) to Redis (in-memory) +- Redis password is auto-generated (no manual credential configuration) +- MinIO StatefulSet replaced with Redis Deployment (no persistent volume needed) +- Redis exporter sidecar added for Prometheus metrics +- Grafana dashboard included for local development +- Helm labels updated to Kubernetes recommended labels (`app.kubernetes.io/*`) -**Out of storage** -- Increase MinIO PVC size in `values.yaml` -- Clean old cache entries manually via MinIO console \ No newline at end of file +Migration steps: +1. Cache data cannot be migrated (MinIO objects to Redis keys) — the cache will be cold after upgrade +2. Uninstall the old release: `helm uninstall ` +3. Install the new version: `helm install ./chart` +4. Gradle builds will repopulate the cache automatically on first run diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 5d79bf9..4460d00 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,11 +1,11 @@ apiVersion: v2 name: theia-shared-cache -description: A Gradle Build Cache server with MinIO backend for Theia IDE deployments in Kubernetes +description: A Gradle Build Cache server with Redis backend for Theia IDE deployments in Kubernetes type: application # Chart version - bump for breaking changes -version: 0.2.3 +version: 0.3.0 # Application version - matches the cache server version appVersion: "0.1.0" diff --git a/chart/README.md b/chart/README.md index 9116a4d..893dadf 100644 --- a/chart/README.md +++ b/chart/README.md @@ -1,39 +1,44 @@ # theia-shared-cache Helm Chart -Helm chart to deploy a custom Gradle Build Cache server with MinIO backend for Theia IDE deployments in Kubernetes. +Helm chart to deploy a Gradle Build Cache server with Redis backend for Kubernetes. ## Architecture -This chart deploys two components: +This chart deploys the following components: -1. **Cache Server** - A custom Go-based Gradle build cache server -2. **MinIO** - S3-compatible object storage for cache data +1. **Cache Server** - A Go-based Gradle build cache server +2. **Redis** - In-memory storage for cache artifacts (with optional redis-exporter sidecar) +3. **Reposilite** (optional) - Maven/Gradle dependency proxy ``` -┌─────────────────────────────────────────────────────┐ -│ Kubernetes Cluster │ -│ │ -│ ┌─────────────────┐ ┌─────────────────────┐ │ -│ │ Cache Server │──────▶│ MinIO │ │ -│ │ (Deployment) │ │ (StatefulSet) │ │ -│ │ Port: 8080 │ │ Port: 9000 │ │ -│ └────────┬────────┘ └─────────────────────┘ │ -│ │ │ -│ ┌────────▼────────┐ │ -│ │ Service │◀──── Gradle Builds │ -│ │ Port: 8080 │ │ -│ └─────────────────┘ │ -└─────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────┐ +│ Kubernetes Cluster │ +│ │ +│ ┌─────────────────┐ ┌─────────────────────────┐ │ +│ │ Cache Server │──────▶│ Redis (Deployment) │ │ +│ │ (Deployment) │ │ Port: 6379 │ │ +│ │ Port: 8080 │ │ + Exporter :9121 │ │ +│ └────────┬─────────┘ └─────────────────────────┘ │ +│ │ │ +│ ┌────────▼─────────┐ ┌─────────────────────────┐ │ +│ │ Service │ │ Reposilite (optional) │ │ +│ │ Port: 8080 │ │ Dependency proxy │ │ +│ └──────────────────┘ └─────────────────────────┘ │ +│ ▲ │ +│ Gradle Builds │ +└──────────────────────────────────────────────────────────┘ ``` ## Features -- Custom Go-based cache server with MinIO storage backend +- Go-based cache server with Redis storage backend +- Auto-generated Redis password (stored in Kubernetes Secret) - Prometheus metrics endpoint (`/metrics`) +- Redis metrics via redis-exporter sidecar +- Optional ServiceMonitor for Prometheus Operator - Health checks (`/ping`, `/health`) -- Basic authentication for cache operations -- Persistent storage for MinIO data -- Configurable resource limits +- Optional Reposilite dependency proxy +- Kubernetes recommended labels (`app.kubernetes.io/*`) ## Quick Start @@ -47,10 +52,8 @@ helm install gradle-cache ./chart ```bash helm install gradle-cache ./chart \ - --set cacheServer.auth.password=mysecretpassword \ - --set minio.auth.accessKey=myaccesskey \ - --set minio.auth.secretKey=mysecretkey \ - --set minio.persistence.size=100Gi + --set auth.password=mysecretpassword \ + --set metrics.serviceMonitor.enabled=true ``` ## Configuration @@ -59,47 +62,53 @@ helm install gradle-cache ./chart \ | Parameter | Description | Default | |-----------|-------------|---------| -| `cacheServer.replicaCount` | Number of cache server replicas | `1` | -| `cacheServer.image.repository` | Cache server image repository | `ghcr.io/kevingruber/theia-shared-cache/gradle-cache` | -| `cacheServer.image.tag` | Cache server image tag | `latest` | -| `cacheServer.port` | Cache server port | `8080` | -| `cacheServer.auth.enabled` | Enable authentication | `true` | -| `cacheServer.auth.username` | Cache username | `gradle` | -| `cacheServer.auth.password` | Cache password | `changeme` | -| `cacheServer.config.maxEntrySizeMB` | Max cache entry size in MB | `100` | -| `cacheServer.resources` | Resource limits/requests | See values.yaml | - -### MinIO +| `enabled` | Enable/disable the entire deployment | `true` | +| `image.repository` | Cache server image repository | `ghcr.io/ls1intum/theia-shared-cache/gradle-cache` | +| `image.tag` | Cache server image tag | `main` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `auth.enabled` | Enable authentication | `true` | +| `auth.username` | Cache username | `gradle` | +| `auth.password` | Cache password | `changeme` | +| `resources.cacheServer.requests.memory` | Memory request | `256Mi` | +| `resources.cacheServer.requests.cpu` | CPU request | `100m` | +| `resources.cacheServer.limits.memory` | Memory limit | `1Gi` | +| `resources.cacheServer.limits.cpu` | CPU limit | `500m` | + +### Redis | Parameter | Description | Default | |-----------|-------------|---------| -| `minio.enabled` | Deploy MinIO alongside cache server | `true` | -| `minio.image.repository` | MinIO image repository | `minio/minio` | -| `minio.image.tag` | MinIO image tag | `latest` | -| `minio.auth.accessKey` | MinIO access key | `minioadmin` | -| `minio.auth.secretKey` | MinIO secret key | `minioadmin` | -| `minio.persistence.enabled` | Enable persistent storage | `true` | -| `minio.persistence.size` | Storage size | `50Gi` | -| `minio.persistence.storageClass` | Storage class | `""` (default) | -| `minio.resources` | Resource limits/requests | See values.yaml | +| `resources.redis.requests.memory` | Memory request | `128Mi` | +| `resources.redis.requests.cpu` | CPU request | `100m` | +| `resources.redis.limits.memory` | Memory limit | `2Gi` | +| `resources.redis.limits.cpu` | CPU limit | `1000m` | -### Metrics +The Redis password is auto-generated on first install and persisted across `helm upgrade`. Both Redis and the cache server read it from the same Kubernetes Secret. + +### TLS | Parameter | Description | Default | |-----------|-------------|---------| -| `metrics.enabled` | Enable Prometheus metrics | `true` | -| `metrics.serviceMonitor.enabled` | Create ServiceMonitor (Prometheus Operator) | `false` | +| `tls.enabled` | Enable TLS for the cache server | `false` | +| `tls.secretName` | Kubernetes TLS Secret name | `""` | -### External Secret +### Metrics | Parameter | Description | Default | |-----------|-------------|---------| -| `existingSecret` | Use existing secret for credentials | `""` | +| `metrics.serviceMonitor.enabled` | Create ServiceMonitor for Prometheus Operator | `false` | +| `metrics.serviceMonitor.interval` | Scrape interval | `15s` | + +Prometheus pod annotations (`prometheus.io/scrape`, `prometheus.io/port`) are always included on pod templates for annotation-based discovery. -If using `existingSecret`, the secret must contain these keys: -- `minio-access-key` -- `minio-secret-key` -- `cache-password` +### Reposilite + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `reposilite.enabled` | Deploy Reposilite dependency proxy | `true` | +| `reposilite.persistence.enabled` | Enable persistent storage | `true` | +| `reposilite.persistence.size` | Storage size | `20Gi` | +| `reposilite.persistence.storageClass` | Storage class | `csi-rbd-sc` | ## Gradle Configuration @@ -111,20 +120,17 @@ buildCache { remote { url = uri("http://-cache:8080/cache/") credentials { - username = "gradle" - password = "your-password" + username = "writer" + password = "changeme-writer" } isPush = true } } ``` -Or via environment variables: - -```bash -export GRADLE_CACHE_URL=http://-cache:8080/cache/ -export GRADLE_CACHE_USERNAME=gradle -export GRADLE_CACHE_PASSWORD=your-password +Enable caching in `gradle.properties`: +```properties +org.gradle.caching=true ``` ## Port Forwarding (Development) @@ -141,33 +147,46 @@ curl http://localhost:8080/health ## Monitoring -If `metrics.enabled` is true, Prometheus metrics are available at `/metrics`: +### Prometheus Metrics -```bash -curl http://-cache:8080/metrics -``` +Cache server metrics at `/metrics`: + +| Metric | Type | Description | +|--------|------|-------------| +| `gradle_cache_requests_total` | Counter | Total requests by method and status | +| `gradle_cache_cache_hits_total` | Counter | Cache hit count | +| `gradle_cache_cache_misses_total` | Counter | Cache miss count | +| `gradle_cache_request_duration_seconds` | Histogram | Request latency | +| `gradle_cache_entry_size` | Histogram | Cache entry sizes | + +Redis metrics via redis-exporter sidecar at `:9121/metrics`: + +| Metric | Type | Description | +|--------|------|-------------| +| `redis_memory_used_bytes` | Gauge | Redis memory consumption | +| `redis_db_keys` | Gauge | Number of cached entries | +| `redis_keyspace_hits_total` | Counter | Redis-level cache hits | +| `redis_keyspace_misses_total` | Counter | Redis-level cache misses | + +### Grafana Dashboard -Available metrics: -- `cache_requests_total` - Total cache requests by method and status -- `cache_hits_total` - Cache hit count -- `cache_misses_total` - Cache miss count -- `cache_request_duration_seconds` - Request duration histogram -- `cache_entry_size_bytes` - Cache entry size histogram +A pre-built dashboard is available at `src/deployments/grafana/dashboards/gradle-build-cache.json`. Import it into your Grafana instance for cache hit rate, latency, Redis memory, and error monitoring. ## Upgrading -### From 0.1.x to 0.2.0 +### From 0.2.x to 0.3.0 -Version 0.2.0 is a breaking change that replaces the official Gradle cache node with a custom implementation. +Version 0.3.0 is a breaking change that replaces MinIO with Redis. Changes: -- New image: custom Go-based cache server instead of `gradle/build-cache-node` -- New storage: MinIO backend instead of local PersistentVolume -- New port: 8080 instead of 5071 -- New authentication: Basic auth built into cache server +- Storage backend: MinIO (S3-compatible) replaced with Redis (in-memory) +- Redis password auto-generated (no manual credential config) +- MinIO StatefulSet replaced with Redis Deployment +- Redis exporter sidecar for Prometheus metrics +- Helm labels updated to `app.kubernetes.io/*` standard Migration steps: -1. Back up any important cache data (usually safe to lose) +1. Cache data cannot be migrated — the cache will start cold 2. Uninstall the old release: `helm uninstall ` -3. Install the new version with updated values -4. Update Gradle configuration to use new port and credentials +3. Install the new version: `helm install ./chart` +4. Gradle builds repopulate the cache automatically diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl new file mode 100644 index 0000000..eb391c4 --- /dev/null +++ b/chart/templates/_helpers.tpl @@ -0,0 +1,15 @@ +{{/* +Standard labels for theia-shared-cache resources +*/}} +{{- define "theia-shared-cache.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +{{- end -}} + +{{- define "theia-shared-cache.selectorLabels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 0aee82d..18c19f2 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -4,8 +4,8 @@ kind: ConfigMap metadata: name: {{ .Release.Name }}-config labels: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: cache-server data: config.yaml: | server: @@ -14,15 +14,13 @@ data: write_timeout: 120s {{- if .Values.tls.enabled }} tls: - enabled: true + enabled: true cert_file: /etc/certs/tls.crt key_file: /etc/certs/tls.key {{- end}} storage: - endpoint: "{{ .Release.Name }}-minio:9000" - bucket: "gradle-cache" - use_ssl: false + addr: "{{ .Release.Name }}-redis:6379" cache: max_entry_size_mb: 100 diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 34e78d8..d796058 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -4,56 +4,36 @@ kind: Deployment metadata: name: {{ .Release.Name }}-cache-server labels: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: cache-server spec: replicas: 1 selector: matchLabels: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cache-server template: metadata: labels: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cache-server annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} checksum/secret: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} spec: initContainers: - - name: wait-for-minio + - name: wait-for-redis image: busybox:1.36 command: - sh - -c - | - echo "Waiting for MinIO to be ready..." - until nc -z {{ .Release.Name }}-minio 9000; do - echo "MinIO not ready, waiting..." + echo "Waiting for Redis to be ready..." + until nc -z {{ .Release.Name }}-redis 6379; do + echo "Redis not ready, waiting..." sleep 2 done - echo "MinIO is ready!" - - name: create-bucket - image: minio/mc:latest - command: - - sh - - -c - - | - mc alias set minio http://{{ .Release.Name }}-minio:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY - mc mb --ignore-existing minio/gradle-cache - echo "Bucket created or already exists" - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-secret-key + echo "Redis is ready!" containers: - name: cache-server image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -63,21 +43,16 @@ spec: containerPort: 8080 protocol: TCP env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-access-key - - name: MINIO_SECRET_KEY + - name: REDIS_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-secret-key + name: {{ .Release.Name }}-redis-secret + key: redis-password {{- if .Values.auth.enabled }} - name: CACHE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-secrets + name: {{ .Release.Name }}-redis-secret key: cache-password {{- end }} args: diff --git a/chart/templates/minio-service.yaml b/chart/templates/minio-service.yaml deleted file mode 100644 index c67cf6b..0000000 --- a/chart/templates/minio-service.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-minio - labels: - app: {{ .Release.Name }} - component: storage -spec: - type: ClusterIP - ports: - - name: api - port: 9000 - targetPort: api - protocol: TCP - - name: console - port: 9001 - targetPort: console - protocol: TCP - selector: - app: {{ .Release.Name }} - component: storage -{{- end }} diff --git a/chart/templates/minio-statefulset.yaml b/chart/templates/minio-statefulset.yaml deleted file mode 100644 index c889bda..0000000 --- a/chart/templates/minio-statefulset.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if .Values.enabled }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ .Release.Name }}-minio - labels: - app: {{ .Release.Name }} - component: storage -spec: - selector: - matchLabels: - app: {{ .Release.Name }} - component: storage - serviceName: {{ .Release.Name }}-minio - replicas: 1 - template: - metadata: - labels: - app: {{ .Release.Name }} - component: storage - spec: - containers: - - name: minio - image: "minio/minio:latest" - imagePullPolicy: IfNotPresent - args: - - server - - /data - - --console-address - - ":9001" - ports: - - name: api - containerPort: 9000 - - name: console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-access-key - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-secrets - key: minio-secret-key - resources: - {{- toYaml .Values.resources.minio | nindent 12 }} - livenessProbe: - httpGet: - path: /minio/health/live - port: api - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /minio/health/ready - port: api - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 - volumeMounts: - - name: data - mountPath: /data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - storageClassName: {{ .Values.storage.storageClass | default "default" }} - resources: - requests: - storage: {{ .Values.storage.size | quote }} -{{- end }} diff --git a/chart/templates/redis-deployment.yaml b/chart/templates/redis-deployment.yaml new file mode 100644 index 0000000..be8009a --- /dev/null +++ b/chart/templates/redis-deployment.yaml @@ -0,0 +1,84 @@ +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-redis + labels: + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + replicas: 1 + selector: + matchLabels: + {{- include "theia-shared-cache.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: storage + template: + metadata: + labels: + {{- include "theia-shared-cache.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: storage + spec: + containers: + - name: redis + image: "redis:7-alpine" + imagePullPolicy: IfNotPresent + command: + - redis-server + - --requirepass + - $(REDIS_PASSWORD) + - --save + - "" + - --loglevel + - warning + ports: + - name: redis + containerPort: 6379 + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-redis-secret + key: redis-password + resources: + {{- toYaml .Values.resources.redis | nindent 12 }} + livenessProbe: + exec: + command: + - sh + - -c + - redis-cli -a $REDIS_PASSWORD ping | grep -q PONG + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - -c + - redis-cli -a $REDIS_PASSWORD ping | grep -q PONG + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + - name: redis-exporter + image: "oliver006/redis_exporter:v1.40.0" + ports: + - name: metrics + containerPort: 9121 + env: + - name: REDIS_ADDR + value: "localhost:6379" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-redis-secret + key: redis-password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi +{{- end }} diff --git a/chart/templates/redis-service.yaml b/chart/templates/redis-service.yaml new file mode 100644 index 0000000..17545e0 --- /dev/null +++ b/chart/templates/redis-service.yaml @@ -0,0 +1,23 @@ +{{- if .Values.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-redis + labels: + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + type: ClusterIP + ports: + - name: redis + port: 6379 + targetPort: redis + protocol: TCP + - name: metrics + port: 9121 + targetPort: metrics + protocol: TCP + selector: + {{- include "theia-shared-cache.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: storage +{{- end }} diff --git a/chart/templates/reposilite-shared-config.yaml b/chart/templates/reposilite-shared-config.yaml index 09e76e4..7fcee37 100644 --- a/chart/templates/reposilite-shared-config.yaml +++ b/chart/templates/reposilite-shared-config.yaml @@ -4,8 +4,8 @@ kind: ConfigMap metadata: name: reposilite-shared-config labels: - app: {{ .Release.Name }} - component: reposilite + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: dependency-cache data: configuration.shared.json: | { @@ -34,4 +34,4 @@ data: ] } } -{{- end }} \ No newline at end of file +{{- end }} diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml index 3fbee47..6dfc22c 100644 --- a/chart/templates/secrets.yaml +++ b/chart/templates/secrets.yaml @@ -2,14 +2,18 @@ apiVersion: v1 kind: Secret metadata: - name: {{ .Release.Name }}-secrets + name: {{ .Release.Name }}-redis-secret labels: - app: {{ .Release.Name }} + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: storage type: Opaque data: - minio-access-key: {{ "minioadmin" | b64enc | quote }} - minio-secret-key: {{ "minioadmin" | b64enc | quote }} - {{- if .Values.auth.enabled }} - cache-password: {{ .Values.auth.password | b64enc | quote }} + {{- $secretName := printf "%s-redis-secret" .Release.Name }} + {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace $secretName }} + {{- if $existingSecret }} + redis-password: {{ index $existingSecret.data "redis-password" }} + {{- else }} + redis-password: {{ randAlphaNum 32 | b64enc}} {{- end }} + cache-password: {{ .Values.auth.password | b64enc | quote}} {{- end }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml index 7c09252..ca87d72 100644 --- a/chart/templates/service.yaml +++ b/chart/templates/service.yaml @@ -4,8 +4,8 @@ kind: Service metadata: name: {{ .Release.Name }}-cache labels: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.labels" . | nindent 4 }} + app.kubernetes.io/component: cache-server spec: type: ClusterIP ports: @@ -14,6 +14,6 @@ spec: targetPort: http protocol: TCP selector: - app: {{ .Release.Name }} - component: cache-server + {{- include "theia-shared-cache.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cache-server {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 977f21f..594b01a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -16,13 +16,6 @@ auth: # IMPORTANT: Change this password in production! password: "changeme" -# Storage configuration -storage: - # Persistent volume size for cache data - size: 50Gi - # Storage class for persistent volume - storageClass: "csi-rbd-sc" - # Resource limits resources: cacheServer: @@ -33,10 +26,10 @@ resources: memory: "1Gi" cpu: "500m" - minio: + redis: requests: - memory: "512Mi" - cpu: "250m" + memory: "128Mi" + cpu: "100m" limits: memory: "2Gi" cpu: "1000m" @@ -99,4 +92,4 @@ reposilite: subPath: configuration.shared.json readOnly: true - name: plugins - mountPath: /app/data/plugins \ No newline at end of file + mountPath: /app/data/plugins diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go index 4e7156c..d184970 100644 --- a/src/cmd/server/main.go +++ b/src/cmd/server/main.go @@ -3,11 +3,12 @@ package main import ( "context" "flag" - "github.com/kevingruber/gradle-cache/internal/telemetry" "os" "os/signal" "syscall" + "github.com/kevingruber/gradle-cache/internal/telemetry" + "github.com/kevingruber/gradle-cache/internal/config" "github.com/kevingruber/gradle-cache/internal/server" "github.com/kevingruber/gradle-cache/internal/storage" @@ -39,28 +40,19 @@ func main() { } defer cleanup() - // Create storage - store, err := storage.NewMinIOStorage(storage.MinIOConfig{ - Endpoint: cfg.Storage.Endpoint, - AccessKey: cfg.Storage.AccessKey, - SecretKey: cfg.Storage.SecretKey, - Bucket: cfg.Storage.Bucket, - UseSSL: cfg.Storage.UseSSL, + store, err := storage.NewRedisStorage(storage.RedisConfig{ + Addr: cfg.Storage.Addr, + Password: cfg.Storage.Password, }) + if err != nil { logger.Fatal().Err(err).Msg("failed to create storage") } - // Ensure bucket exists - ctx := context.Background() - if err := store.EnsureBucket(ctx); err != nil { - logger.Fatal().Err(err).Msg("failed to ensure bucket exists") - } - logger.Info().Str("bucket", cfg.Storage.Bucket).Msg("storage bucket ready") - // Create and run server srv := server.New(cfg, store, logger) + ctx := context.Background() // Setup graceful shutdown ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/src/configs/config.yaml b/src/configs/config.yaml index 46d5b45..aa5217e 100644 --- a/src/configs/config.yaml +++ b/src/configs/config.yaml @@ -4,12 +4,9 @@ server: write_timeout: 120s storage: - endpoint: "minio:9000" - # These are overridden by environment variables - access_key: "${MINIO_ACCESS_KEY}" - secret_key: "${MINIO_SECRET_KEY}" - bucket: "gradle-cache" - use_ssl: false + addr: "redis:6379" + password: "" + db: 0 cache: max_entry_size_mb: 100 diff --git a/src/deployments/docker-compose.yaml b/src/deployments/docker-compose.yaml index e5fe2d8..f321aa3 100644 --- a/src/deployments/docker-compose.yaml +++ b/src/deployments/docker-compose.yaml @@ -6,35 +6,47 @@ services: ports: - "8080:8080" environment: - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin + - REDIS_PASSWORD=changeme - CACHE_PASSWORD=changeme volumes: - ../configs/config.yaml:/app/config.yaml:ro command: ["-config", "/app/config.yaml"] depends_on: - minio: + redis: condition: service_healthy restart: unless-stopped - minio: - image: minio/minio:latest - command: server /data --console-address ":9001" + redis: + image: redis:7-alpine + command: + - redis-server + - --requirepass + - "changeme" + - --loglevel + - warning ports: - - "9000:9000" - - "9001:9001" - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin + - "6379:6379" volumes: - - minio-data:/data + - redis-data:/data healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "redis-cli", "-a", "changeme", "ping"] interval: 5s timeout: 5s retries: 5 restart: unless-stopped + redis-exporter: + image: oliver006/redis_exporter:latest + ports: + - "9121:9121" + environment: + - REDIS_ADDR=redis:6379 + - REDIS_PASSWORD=changeme + depends_on: + redis: + condition: service_healthy + restart: unless-stopped + prometheus: image: prom/prometheus:latest ports: @@ -43,10 +55,39 @@ services: - ./prometheus.yaml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + restart: unless-stopped + + reposilite: + image: dzikoysk/reposilite:3.5.26 + ports: + - "8081:8080" + environment: + - JAVA_OPTS=-Xmx128M + - REPOSILITE_OPTS=--token admin:changeme --shared-configuration=/etc/reposilite/configuration.shared.json + volumes: + - ./reposilite/reposilite.shared.json:/etc/reposilite/configuration.shared.json:ro + - reposilite-data:/app/data restart: unless-stopped volumes: - minio-data: + redis-data: prometheus-data: + reposilite-data: diff --git a/src/deployments/grafana/dashboards/gradle-build-cache.json b/src/deployments/grafana/dashboards/gradle-build-cache.json new file mode 100644 index 0000000..71d7b97 --- /dev/null +++ b/src/deployments/grafana/dashboards/gradle-build-cache.json @@ -0,0 +1,473 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Cache Server", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.5 }, + { "color": "yellow", "value": 0.7 }, + { "color": "green", "value": 0.85 } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Cache Hit Rate", + "description": "Percentage of cache lookups that found a cached result. Below 50% means the cache is barely helping. 70-90% is healthy. A sudden drop means build keys changed.", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(gradle_cache_cache_hits[5m]) / (rate(gradle_cache_cache_hits[5m]) + rate(gradle_cache_cache_misses[5m]))", + "legendFormat": "Hit Rate", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisLabel": "req/s", + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 9, "x": 6, "y": 1 }, + "id": 2, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Request Rate by Method", + "description": "Traffic pattern: GET=download, PUT=upload, HEAD=existence check. High PUT with few GETs means nobody reads the cache. Spikes correlate with CI runs.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (method) (rate(gradle_cache_requests_total[5m]))", + "legendFormat": "{{method}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 9, "x": 15, "y": 1 }, + "id": 3, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Request Latency", + "description": "Response time percentiles. Under 10ms is normal for Redis-backed cache. Spikes during PUTs indicate large artifacts. Gradual increase means Redis memory pressure.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(gradle_cache_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(gradle_cache_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(gradle_cache_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisLabel": "req/s", + "drawStyle": "line", + "fillOpacity": 30, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Hits" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "Misses" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 7 }, + "id": 4, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Cache Hits vs Misses", + "description": "Green=hits (served from cache), Red=misses (Gradle had to rebuild). The ratio between these is your hit rate. Ideally green dominates.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(gradle_cache_cache_hits_total[5m])", + "legendFormat": "Hits", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(gradle_cache_cache_misses_total[5m])", + "legendFormat": "Misses", + "refId": "B" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "axisBorderShow": false, + "drawStyle": "line", + "fillOpacity": 30, + "lineWidth": 2, + "stacking": { "mode": "none" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 7 }, + "id": 5, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "title": "Server Errors (5xx)", + "description": "Rate of internal server errors. Should be zero. Any non-zero means Redis is unreachable or there's a bug. Investigate immediately — errors slow down every Gradle build.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(rate(gradle_cache_requests_total{status=~\"5..\"}[5m]))", + "legendFormat": "5xx errors", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "id": 200, + "title": "Redis", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 536870912 }, + { "color": "red", "value": 858993459 } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 14 }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Redis Memory Used", + "description": "How much RAM Redis is consuming. Approaching your container limit (1-2Gi) means risk of OOM kill. Action: increase limit or add eviction policy (maxmemory-policy allkeys-lru).", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "redis_memory_used_bytes", + "legendFormat": "Memory", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 14 }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Cached Entries", + "description": "Total keys in Redis. Combined with memory, gives average entry size (memory / keys). Zero after restart is expected with no persistence.", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "redis_db_keys{db=\"db0\"}", + "legendFormat": "Keys", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 }, + "id": 8, + "options": { + "legend": { "calcs": ["mean", "lastNotNull", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "title": "Redis Memory Over Time", + "description": "Memory trend. Steady growth = cache filling up. Sawtooth (up then drop) = pod crashing and restarting. Flat = cache is stable or nobody is writing.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "redis_memory_used_bytes", + "legendFormat": "Used", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisLabel": "ops/s", + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Hits" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "Misses" }, + "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 20 }, + "id": 9, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Redis Keyspace Hits vs Misses", + "description": "Redis-level hit/miss. Confirms app-level metrics. High misses here + high app misses = entries truly don't exist (build config issue, not Redis issue).", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(redis_keyspace_hits_total[5m])", + "legendFormat": "Hits", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(redis_keyspace_misses_total[5m])", + "legendFormat": "Misses", + "refId": "B" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisLabel": "ops/s", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 20 }, + "id": 10, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "title": "Redis Commands/sec", + "description": "Total Redis operations per second. Shows overall load on Redis. Redis handles ~100k ops/sec easily — if this approaches that, you have a scaling problem (unlikely for a build cache).", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(redis_commands_processed_total[5m])", + "legendFormat": "Commands/s", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["gradle", "cache", "redis"], + "templating": { "list": [] }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Gradle Build Cache", + "uid": "gradle-build-cache", + "version": 1, + "refresh": "10s" +} diff --git a/src/deployments/grafana/provisioning/dashboards/dashboards.yaml b/src/deployments/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..da12b17 --- /dev/null +++ b/src/deployments/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/src/deployments/grafana/provisioning/datasources/prometheus.yaml b/src/deployments/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 0000000..fbd3541 --- /dev/null +++ b/src/deployments/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: PBFA97CFB590B2093 + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/src/deployments/prometheus.yaml b/src/deployments/prometheus.yaml index 2dbf733..afb1f5d 100644 --- a/src/deployments/prometheus.yaml +++ b/src/deployments/prometheus.yaml @@ -3,7 +3,14 @@ global: evaluation_interval: 15s scrape_configs: - - job_name: 'gradle-cache' + - job_name: "gradle-cache" static_configs: - - targets: ['cache-server:8080'] + - targets: ["cache-server:8080"] metrics_path: /metrics + - job_name: "redis" + static_configs: + - targets: ["redis-exporter:9121"] + - job_name: "reposilite" + static_configs: + - targets: ["reposilite:8081"] + metrics_path: /api/prometheus diff --git a/src/deployments/reposilite/reposilite.shared.json b/src/deployments/reposilite/reposilite.shared.json new file mode 100644 index 0000000..a518839 --- /dev/null +++ b/src/deployments/reposilite/reposilite.shared.json @@ -0,0 +1,48 @@ +{ + "maven": { + "repositories": [ + { + "id": "releases", + "visibility": "PUBLIC", + "redeployment": false, + "preserveSnapshots": false, + "proxied": [ + { + "reference": "https://repo1.maven.org/maven2", + "store": true, + "allowedGroups": [], + "allowedExtensions": [ + ".jar", + ".war", + ".xml", + ".pom", + ".module", + ".asc", + ".md5", + ".sha1", + ".sha256", + ".sha512" + ] + }, + { + "reference": "https://plugins.gradle.org/m2", + "store": true, + "allowedGroups": [], + "allowedExtensions": [ + ".jar", + ".war", + ".xml", + ".pom", + ".module", + ".asc", + ".md5", + ".sha1", + ".sha256", + ".sha512" + ] + } + ] + } + ] + } +} diff --git a/src/go.mod b/src/go.mod index 76a454f..312b502 100644 --- a/src/go.mod +++ b/src/go.mod @@ -3,13 +3,18 @@ module github.com/kevingruber/gradle-cache go 1.24.3 require ( + github.com/getsentry/sentry-go v0.42.0 + github.com/getsentry/sentry-go/otel v0.42.0 github.com/gin-gonic/gin v1.11.0 - github.com/minio/minio-go/v7 v7.0.97 github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.3 github.com/rs/zerolog v1.34.0 github.com/spf13/viper v1.21.0 go.opentelemetry.io/otel v1.40.0 go.opentelemetry.io/otel/exporters/prometheus v0.62.0 + go.opentelemetry.io/otel/metric v1.40.0 + go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/sdk/metric v1.40.0 ) require ( @@ -19,13 +24,10 @@ require ( github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect - github.com/getsentry/sentry-go v0.42.0 // indirect - github.com/getsentry/sentry-go/otel v0.42.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -36,39 +38,29 @@ require ( github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/klauspost/crc32 v1.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/minio/crc64nvme v1.1.0 // indirect - github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/philhofer/fwd v1.2.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect - github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tinylib/msgp v1.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect go.opentelemetry.io/otel/trace v1.40.0 // indirect go.uber.org/mock v0.6.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect @@ -79,5 +71,4 @@ require ( golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/src/go.sum b/src/go.sum index fae148c..38e0e66 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,5 +1,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -14,8 +18,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -30,8 +34,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -61,11 +65,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= -github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -80,12 +81,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= -github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= -github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -95,8 +90,9 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= -github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -114,9 +110,10 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1DQx4= +github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -145,8 +142,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= diff --git a/src/internal/config/config.go b/src/internal/config/config.go index cd4128c..ef37078 100644 --- a/src/internal/config/config.go +++ b/src/internal/config/config.go @@ -32,11 +32,9 @@ type TLSConfig struct { } type StorageConfig struct { - Endpoint string `mapstructure:"endpoint"` - AccessKey string `mapstructure:"access_key"` - SecretKey string `mapstructure:"secret_key"` - Bucket string `mapstructure:"bucket"` - UseSSL bool `mapstructure:"use_ssl"` + Addr string `mapstructure:"addr"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` } type CacheConfig struct { @@ -78,9 +76,9 @@ func Load(configPath string) (*Config, error) { v.SetDefault("server.tls.cert_file", "/etc/certs/tls.crt") v.SetDefault("server.tls.key_file", "/etc/certs/tls.key") - v.SetDefault("storage.endpoint", "localhost:9000") - v.SetDefault("storage.bucket", "gradle-cache") - v.SetDefault("storage.use_ssl", false) + v.SetDefault("storage.addr", "localhost:6379") + v.SetDefault("storage.password", "") + v.SetDefault("storage.db", 0) v.SetDefault("cache.max_entry_size_mb", 100) @@ -106,8 +104,8 @@ func Load(configPath string) (*Config, error) { v.AutomaticEnv() // Bind specific environment variables - v.BindEnv("storage.access_key", "MINIO_ACCESS_KEY") - v.BindEnv("storage.secret_key", "MINIO_SECRET_KEY") + v.BindEnv("storage.password", "REDIS_PASSWORD") + v.BindEnv("auth.users.0.password", "CACHE_PASSWORD") v.BindEnv("sentry.dsn", "SENTRY_DSN") @@ -127,17 +125,8 @@ func Load(configPath string) (*Config, error) { } func (c *Config) Validate() error { - if c.Storage.Endpoint == "" { - return fmt.Errorf("storage.endpoint is required") - } - if c.Storage.AccessKey == "" { - return fmt.Errorf("storage.access_key is required") - } - if c.Storage.SecretKey == "" { - return fmt.Errorf("storage.secret_key is required") - } - if c.Storage.Bucket == "" { - return fmt.Errorf("storage.bucket is required") + if c.Storage.Addr == "" { + return fmt.Errorf("storage.addr is required") } if c.Auth.Enabled && len(c.Auth.Users) == 0 { return fmt.Errorf("auth.users is required when auth is enabled") diff --git a/src/internal/middleware/metrics.go b/src/internal/middleware/metrics.go index b9bb2bc..485c5a1 100644 --- a/src/internal/middleware/metrics.go +++ b/src/internal/middleware/metrics.go @@ -1,11 +1,12 @@ package middleware import ( + "strconv" + "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "strconv" - "time" "github.com/gin-gonic/gin" ) @@ -31,6 +32,9 @@ func NewMetrics() (*Metrics, error) { "gradle_cache.request_duration_seconds", metric.WithDescription("HTTP request duration in seconds"), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + ), ) if err != nil { return nil, err diff --git a/src/internal/storage/minio.go b/src/internal/storage/minio.go deleted file mode 100644 index 590e325..0000000 --- a/src/internal/storage/minio.go +++ /dev/null @@ -1,150 +0,0 @@ -package storage - -import ( - "context" - "fmt" - "io" - "path" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -// MinIOStorage implements Storage using MinIO as the backend. -type MinIOStorage struct { - client *minio.Client - bucket string - namespace string -} - -// MinIOConfig holds the configuration for MinIO connection. -type MinIOConfig struct { - Endpoint string - AccessKey string - SecretKey string - Bucket string - UseSSL bool -} - -// NewMinIOStorage creates a new MinIO storage instance. -func NewMinIOStorage(cfg MinIOConfig) (*MinIOStorage, error) { - client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.UseSSL, - }) - if err != nil { - return nil, fmt.Errorf("failed to create MinIO client: %w", err) - } - - return &MinIOStorage{ - client: client, - bucket: cfg.Bucket, - }, nil -} - -// EnsureBucket creates the bucket if it doesn't exist. -func (s *MinIOStorage) EnsureBucket(ctx context.Context) error { - exists, err := s.client.BucketExists(ctx, s.bucket) - if err != nil { - return fmt.Errorf("failed to check bucket existence: %w", err) - } - - if !exists { - err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}) - if err != nil { - return fmt.Errorf("failed to create bucket: %w", err) - } - } - - return nil -} - -// objectKey returns the full object key including namespace prefix. -func (s *MinIOStorage) objectKey(key string) string { - if s.namespace == "" { - return key - } - return path.Join(s.namespace, key) -} - -// Get retrieves a cache entry from MinIO. -func (s *MinIOStorage) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { - objectKey := s.objectKey(key) - - obj, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{}) - if err != nil { - return nil, 0, fmt.Errorf("failed to get object: %w", err) - } - - stat, err := obj.Stat() - if err != nil { - obj.Close() - errResp := minio.ToErrorResponse(err) - if errResp.Code == "NoSuchKey" { - return nil, 0, ErrNotFound - } - return nil, 0, fmt.Errorf("failed to stat object: %w", err) - } - - return obj, stat.Size, nil -} - -// Put stores a cache entry in MinIO. -func (s *MinIOStorage) Put(ctx context.Context, key string, reader io.Reader, size int64) error { - objectKey := s.objectKey(key) - - _, err := s.client.PutObject(ctx, s.bucket, objectKey, reader, size, minio.PutObjectOptions{ - ContentType: "application/octet-stream", - }) - if err != nil { - return fmt.Errorf("failed to put object: %w", err) - } - - return nil -} - -// Exists checks if a cache entry exists in MinIO. -func (s *MinIOStorage) Exists(ctx context.Context, key string) (bool, error) { - objectKey := s.objectKey(key) - - _, err := s.client.StatObject(ctx, s.bucket, objectKey, minio.StatObjectOptions{}) - if err != nil { - errResp := minio.ToErrorResponse(err) - if errResp.Code == "NoSuchKey" { - return false, nil - } - return false, fmt.Errorf("failed to stat object: %w", err) - } - - return true, nil -} - -// Delete removes a cache entry from MinIO. -func (s *MinIOStorage) Delete(ctx context.Context, key string) error { - objectKey := s.objectKey(key) - - err := s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{}) - if err != nil { - return fmt.Errorf("failed to remove object: %w", err) - } - - return nil -} - -// Ping checks if MinIO is reachable. -func (s *MinIOStorage) Ping(ctx context.Context) error { - _, err := s.client.BucketExists(ctx, s.bucket) - if err != nil { - return fmt.Errorf("failed to ping MinIO: %w", err) - } - return nil -} - -// WithNamespace returns a new MinIOStorage instance scoped to the given namespace. -func (s *MinIOStorage) WithNamespace(namespace string) Storage { - return &MinIOStorage{ - client: s.client, - bucket: s.bucket, - namespace: namespace, - } -} diff --git a/src/internal/storage/redis.go b/src/internal/storage/redis.go new file mode 100644 index 0000000..1ef3aa3 --- /dev/null +++ b/src/internal/storage/redis.go @@ -0,0 +1,85 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/redis/go-redis/v9" +) + +type RedisStorage struct { + client *redis.Client + namespace string +} + +type RedisConfig struct { + Addr string + Password string +} + +func NewRedisStorage(cfg RedisConfig) (*RedisStorage, error) { + client := redis.NewClient(&redis.Options{ + Addr: cfg.Addr, + Password: cfg.Password, + DB: 0, + }) + storage := &RedisStorage{client: client} + + // Test connection + if err := storage.Ping(context.Background()); err != nil { + return nil, fmt.Errorf("failed to connect to Redis: %w", err) + } + return storage, nil +} + +func (s *RedisStorage) redisKey(key string) string { + if s.namespace == "" { + return key + } + return s.namespace + ":" + key +} + +func (s *RedisStorage) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { + data, err := s.client.Get(ctx, s.redisKey(key)).Bytes() + if err != nil { + if err == redis.Nil { + return nil, 0, ErrNotFound + } + return nil, 0, fmt.Errorf("failed to get key from Redis: %w", err) + } + return io.NopCloser(bytes.NewReader(data)), int64(len(data)), nil +} + +func (s *RedisStorage) Put(ctx context.Context, key string, reader io.Reader, size int64) error { + data, err := io.ReadAll(reader) + + if err != nil { + return fmt.Errorf("failed to read data: %w", err) + } + return s.client.Set(ctx, s.redisKey(key), data, 0).Err() +} + +func (s *RedisStorage) Exists(ctx context.Context, key string) (bool, error) { + n, err := s.client.Exists(ctx, s.redisKey(key)).Result() + if err != nil { + return false, fmt.Errorf("failed to check key existence in Redis: %w", err) + } + return n > 0, nil +} + +func (s *RedisStorage) Delete(ctx context.Context, key string) error { + return s.client.Del(ctx, s.redisKey(key)).Err() +} + +func (s *RedisStorage) Ping(ctx context.Context) error { + return s.client.Ping(ctx).Err() +} + +func (s *RedisStorage) WithNamespace(namespace string) Storage { + return &RedisStorage{ + client: s.client, + namespace: namespace, + } +}