diff --git a/.env.example b/.env.example index 401bbb66..068d3ef6 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,10 @@ WATCH_NAMESPACE="hades" DELETE_ON_COMPLETE="true" MAX_PARALLELISM="100" +### HadesLogManager Options ### +HADESLOGMANAGER_API_PORT=8081 # Port for the Log Manager to listen on (default: 8081) +ARTEMIS_ADAPTER_URL=http://localhost:8082/adapter/logs # URL for the Artemis Adapter to send logs to (default: http://localhost:8082/adapter/logs) + ### DockerScheduler Options ### DOCKER_HOST="unix:///var/run/docker.sock" # Docker host to connect to (default: unix:///var/run/docker.sock) DOCKER_SCRIPT_EXECUTOR="/bin/bash -c" # Path to the script executor to use (default: /bin/sh) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56646976..434221bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,8 +127,17 @@ jobs: image-tag: v3.6 secrets: inherit + build-log-manager: + needs: [lint, build, test] + if: github.actor != 'dependabot[bot]' + uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 + with: + image-name: ls1intum/hades/hades-log-manager + docker-file: ./HadesLogManager/Dockerfile + secrets: inherit + deploy: - needs: [build-api, build-scheduler] + needs: [build-api, build-scheduler, build-log-manager] runs-on: ubuntu-latest environment: hades-test steps: diff --git a/HadesAPI/go.sum b/HadesAPI/go.sum index 5af2341f..719d7875 100644 --- a/HadesAPI/go.sum +++ b/HadesAPI/go.sum @@ -39,6 +39,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= @@ -177,11 +178,17 @@ go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzyb go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= @@ -196,6 +203,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/HadesLogManager/Dockerfile b/HadesLogManager/Dockerfile new file mode 100644 index 00000000..4fb3f6d2 --- /dev/null +++ b/HadesLogManager/Dockerfile @@ -0,0 +1,30 @@ +# Use an official Go runtime as a parent image +FROM golang:1.25-alpine AS builder + +# Set the working directory inside the container +WORKDIR /app + +# Copy go.mod and go.sum files +COPY ./HadesLogManager/go.mod ./HadesLogManager/go.sum ./HadesLogManager/ +COPY ./shared/go.mod ./shared/go.sum ./shared/ +RUN cd HadesLogManager && go mod download + +# Copy the Go application source code into the container +COPY ./HadesLogManager ./HadesLogManager +COPY ./shared ./shared + +# Build the Go application +WORKDIR /app/HadesLogManager +RUN CGO_ENABLED=0 go build -o hades-log-manager . + +# Start a new stage for the minimal runtime container +FROM gcr.io/distroless/static-debian12 + +# Set the working directory inside the minimal runtime container +WORKDIR /app + +# Copy the built binary from the builder container into the minimal runtime container +COPY --from=builder /app/HadesLogManager/hades-log-manager . + +# Run your Go application +CMD ["/app/hades-log-manager"] \ No newline at end of file diff --git a/HadesLogManager/go.mod b/HadesLogManager/go.mod index 56c5cd6c..56f4d805 100644 --- a/HadesLogManager/go.mod +++ b/HadesLogManager/go.mod @@ -1,4 +1,5 @@ -module HadesAdapter +module github.com/ls1intum/hades/hadesLogManager +replace github.com/ls1intum/hades/shared => ../shared go 1.25.0 @@ -42,7 +43,7 @@ require ( require ( github.com/klauspost/compress v1.18.6 // indirect - github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2 + github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556 github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.52.0 // indirect diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index fae1cb5f..be6830f8 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -45,8 +45,8 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2 h1:vrqgmPmtTBRBBZEAEZMKqcxQK5OFmM4IE7+nDUOKd2w= -github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2/go.mod h1:qB8+4591QBf959Hd6DGmE0ATXsF0SsR1UDTGLKCbC1M= +github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556 h1:kfdGL7z/A4o7rr3Vi8fUkqAC+i6am/IUMkBjNnPX5Jg= +github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556/go.mod h1:rB2Xc7vj0gHoGp8Jq91fE9rCakAfBFqNqlJfWwcQ72A= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -88,6 +88,7 @@ github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8= go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= diff --git a/HadesLogManager/log_subscriber.go b/HadesLogManager/log_subscriber.go index c94022c5..4172c25d 100644 --- a/HadesLogManager/log_subscriber.go +++ b/HadesLogManager/log_subscriber.go @@ -32,6 +32,7 @@ type DynamicLogManager struct { type watcherState struct { ctx context.Context cancel context.CancelFunc + wg *sync.WaitGroup } // NewDynamicLogManager creates a new DynamicLogManager instance with the provided dependencies. @@ -164,6 +165,8 @@ func (dlm *DynamicLogManager) cleanupSubscriptions(subs []*nats.Subscription) { func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID string) { // Create new context for this job outside the lock jobCtx, cancel := context.WithCancel(ctx) + wg := &sync.WaitGroup{} + wg.Add(1) // Minimize critical section - only lock for map operations dlm.mu.Lock() @@ -171,16 +174,19 @@ func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID st dlm.watchers[jobID] = watcherState{ ctx: jobCtx, cancel: cancel, + wg: wg, } dlm.mu.Unlock() // Cancel old watcher outside the lock to avoid potential deadlock if exists { oldWatcher.cancel() + oldWatcher.wg.Wait() } // Start watching logs for this job go func() { + defer wg.Done() defer func() { // Use a more efficient cleanup check dlm.mu.Lock() @@ -218,13 +224,27 @@ func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID st // - jobID: Unique identifier of the job to stop watching logs for func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) { dlm.mu.Lock() - defer dlm.mu.Unlock() - - if watcher, exists := dlm.watchers[jobID]; exists { + watcher, exists := dlm.watchers[jobID] + if exists { delete(dlm.watchers, jobID) + } + dlm.mu.Unlock() + + if exists { slog.Info("Stopping log watch", "job_id", jobID) watcher.cancel() + watcher.wg.Wait() // Wait outside the lock dlm.logAggregator.MarkJobCompleted(jobID) + go func() { + defer func() { + if r := recover(); r != nil { + slog.Error("Panic while sending job logs", "job_id", jobID, "panic", r) + } + }() + if err := dlm.logAggregator.SendJobLogs(jobID); err != nil { + slog.Error("Failed to send job logs", "job_id", jobID, "error", err) + } + }() } } diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 2d96b017..b226a3ec 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -25,7 +25,7 @@ const ( // HadesLogManagerConfig holds the configuration for the log manager type HadesLogManagerConfig struct { NatsConfig hadesnats.ConnectionConfig - APIPort string `env:"API_PORT" envDefault:"8081"` + APIPort string `env:"HADESLOGMANAGER_API_PORT" envDefault:"8081"` } func main() { @@ -80,12 +80,7 @@ func connectNATS(config hadesnats.ConnectionConfig) (*nats.Conn, error) { } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown( - ctx context.Context, - cancel context.CancelFunc, - cfg HadesLogManagerConfig, - dynamicManager buildlogs.LogManager, - logAggregator buildlogs.LogAggregator, +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesLogManagerConfig, dynamicManager buildlogs.LogManager, logAggregator buildlogs.LogAggregator, ) error { var wg sync.WaitGroup errChan := make(chan error, 2) @@ -128,12 +123,7 @@ func runWithGracefulShutdown( } // waitForShutdown waits for OS signal or error and performs graceful shutdown -func waitForShutdown( - ctx context.Context, - cancel context.CancelFunc, - server *http.Server, - wg *sync.WaitGroup, - errChan chan error, +func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error, ) error { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) @@ -172,6 +162,13 @@ func waitForShutdown( return shutdownErr } +// setupAPIRoute creates and configures the Gin router with all log manager endpoints. +// It registers the following routes: +// +// - GET /jobs/:jobId/logs — returns all aggregated log entries for the given job ID. (Used for testing purposes) +// - GET /jobs/:jobId/status — returns the current build status for the given job ID, or 404 if the job is not found. +// - GET /jobs — returns a list of all known job IDs (active and completed). +// - GET /health — liveness probe returning a static OK response. func setupAPIRoute(aggregator buildlogs.LogAggregator) *gin.Engine { r := gin.Default() jobs := r.Group("/jobs") diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index cdfd2c1e..411acb2f 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "log/slog" + "net/http" "sync" "time" @@ -11,6 +14,10 @@ import ( "github.com/ls1intum/hades/shared/buildstatus" ) +const ( + httpClientTimeout = 10 * time.Second +) + // NATSLogAggregator implements LogAggregator using in-memory storage for fast log retrieval. // It provides thread-safe log aggregation with configurable batching, automatic log rotation, // and memory management. Thread-safety is provided by sync.Map for all operations. @@ -19,7 +26,7 @@ type NATSLogAggregator struct { logs sync.Map // jobID (string) -> logsVersion completed sync.Map // jobID (string) -> time.Time (completion time) status sync.Map // jobID (string) -> buildstatus.JobStatus - config AggregatorConfig + cfg AggregatorConfig } // wrapper stored in sync.Map: comparable (uint64 + pointer) @@ -30,9 +37,10 @@ type logsVersion struct { // AggregatorConfig defines the configuration parameters for log aggregation behavior. type AggregatorConfig struct { - BatchSize int `env:"LOG_BATCH_SIZE" envDefault:"100"` - Retention time.Duration `env:"LOG_RETENTION" envDefault:"1h"` - MaxJobLogs int `env:"MAX_JOB_LOGS" envDefault:"1000"` + BatchSize int `env:"LOG_BATCH_SIZE" envDefault:"100"` + Retention time.Duration `env:"LOG_RETENTION" envDefault:"1h"` + MaxJobLogs int `env:"MAX_JOB_LOGS" envDefault:"1000"` + APIendpoint string `env:"ARTEMIS_ADAPTER_URL"` } // NewLogAggregator creates a new NATS-based LogAggregator instance with the specified configuration. @@ -47,8 +55,8 @@ type AggregatorConfig struct { // - LogAggregator: A new instance ready to aggregate logs func NewLogAggregator(ctx context.Context, hlc *buildlogs.HadesLogConsumer, config AggregatorConfig) buildlogs.LogAggregator { la := &NATSLogAggregator{ - hlc: hlc, - config: config, + hlc: hlc, + cfg: config, } // Start background cleanup goroutine @@ -59,7 +67,7 @@ func NewLogAggregator(ctx context.Context, hlc *buildlogs.HadesLogConsumer, conf // cleanupLoop runs periodic cleanup of completed jobs func (la *NATSLogAggregator) cleanupLoop(ctx context.Context) { - ticker := time.NewTicker(time.Minute) + ticker := time.NewTicker(time.Hour) defer ticker.Stop() for { @@ -87,9 +95,7 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { return } - slog.Debug("Adding log to aggregator", - "job_id", log.JobID, - "entries", len(log.Logs)) + slog.Debug("Adding log to aggregator", "job_id", log.JobID, "entries", len(log.Logs)) // Use LoadOrStore and CompareAndSwap for thread-safe updates for { @@ -103,12 +109,10 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { newLogs = append(newLogs, log) // Trim if needed - if len(newLogs) > la.config.MaxJobLogs { - trimStart := len(newLogs) - la.config.MaxJobLogs + if len(newLogs) > la.cfg.MaxJobLogs { + trimStart := len(newLogs) - la.cfg.MaxJobLogs newLogs = newLogs[trimStart:] - slog.Debug("Trimmed old logs", - "job_id", log.JobID, - "trimmed_count", trimStart) + slog.Debug("Trimmed old logs", "job_id", log.JobID, "trimmed_count", trimStart) } newPtr := &newLogs @@ -116,9 +120,7 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { // Atomically swap if the value hasn't changed if la.logs.CompareAndSwap(log.JobID, old, newVal) { - slog.Debug("Added log to aggregator", - "job_id", log.JobID, - "total_batches", len(newLogs)) + slog.Debug("Added log to aggregator", "job_id", log.JobID, "total_batches", len(newLogs)) break } // If swap failed, another goroutine modified the value - retry @@ -138,9 +140,7 @@ func (la *NATSLogAggregator) FlushJob(jobID string) error { if logsExists { v := value.(logsVersion) logs := *v.ptr - slog.Info("Flushed job logs", - "job_id", jobID, - "batch_count", len(logs)) + slog.Info("Flushed job logs", "job_id", jobID, "batch_count", len(logs)) } else { slog.Debug("No logs to flush for job", "job_id", jobID) } @@ -157,9 +157,7 @@ func (la *NATSLogAggregator) FlushJob(jobID string) error { // MarkJobCompleted marks a job as completed and schedules it for cleanup after retention period. func (la *NATSLogAggregator) MarkJobCompleted(jobID string) { la.completed.Store(jobID, time.Now()) - slog.Info("Marked job as completed", - "job_id", jobID, - "retention", la.config.Retention) + slog.Info("Marked job as completed", "job_id", jobID, "retention", la.cfg.Retention) } // cleanupCompletedJobs removes logs for jobs that have exceeded the retention period. @@ -171,13 +169,11 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { jobID := key.(string) completedAt := value.(time.Time) - if now.Sub(completedAt) >= la.config.Retention { + if now.Sub(completedAt) >= la.cfg.Retention { slog.Debug("Retention expired, flushing job", "job_id", jobID) if err := la.FlushJob(jobID); err != nil { - slog.Error("Failed to flush job during cleanup", - "job_id", jobID, - "error", err) + slog.Error("Failed to flush job during cleanup", "job_id", jobID, "error", err) } else { cleanedCount++ } @@ -191,6 +187,36 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { } } +// SendJobLogs retrieves all stored logs for jobID, marshals them to JSON, and +// sends them via an HTTP POST to the configured APIendpoint. It returns an error +// if marshalling, request creation, or the HTTP call itself fails. +func (la *NATSLogAggregator) SendJobLogs(jobID string) error { + logs := la.GetJobLogs(jobID) + + jsonData, err := json.Marshal(logs) + if err != nil { + return fmt.Errorf("marshaling logs to JSON: %w", err) + } + slog.Debug("Marshaled logs to JSON", "job_id", jobID) + + req, err := http.NewRequest("POST", la.cfg.APIendpoint, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("creating HTTP request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + client := &http.Client{Timeout: httpClientTimeout} + resp, err := client.Do(req) + + if err != nil { + return fmt.Errorf("sending HTTP request: %w", err) + } + + defer resp.Body.Close() + slog.Info("Sent job logs to adapter", "job_id", jobID, "status", resp.Status) + return nil +} + // GetJobLogs retrieves all log entries for a specific job ID by flattening // the batched logs into a single slice. // @@ -231,14 +257,18 @@ func (la *NATSLogAggregator) GetAllJobs() []string { return jobs } +// UpdateJobStatus stores or overwrites the build status for jobID. +// This method is thread-safe via sync.Map.Store. func (la *NATSLogAggregator) UpdateJobStatus(jobID string, status buildstatus.JobStatus) { la.status.Store(jobID, status) } +// GetJobStatus returns the string representation of the current build status for jobID. +// It returns an error if no status has been recorded for the given job. func (la *NATSLogAggregator) GetJobStatus(jobID string) (buildstatus.JobStatus, error) { value, exists := la.status.Load(jobID) if !exists { - return "", fmt.Errorf("job not found: %s", jobID) + return buildstatus.JobStatus(""), fmt.Errorf("job not found: %s", jobID) } return value.(buildstatus.JobStatus), nil } diff --git a/HadesScheduler/HadesOperator/api/v1/buildjob_types.go b/HadesScheduler/HadesOperator/api/v1/buildjob_types.go index 658216b2..9d3964f8 100644 --- a/HadesScheduler/HadesOperator/api/v1/buildjob_types.go +++ b/HadesScheduler/HadesOperator/api/v1/buildjob_types.go @@ -89,6 +89,11 @@ type BuildStep struct { // Example: "set -e && cd /shared/example && ./gradlew clean test" Script string `json:"script,omitempty"` + // If true, the BuildJob pipeline continues to the next step even if this step's container + // exits with a non-zero status. Useful for steps that must always run regardless of prior + // failures, such as result parsing or cleanup. + ContinueOnError bool `json:"continueOnError,omitempty"` + // Step-specific key-value pairs injected as environment variables into this step's container. // Use for step-specific configuration like repository URLs, file paths, or credentials. // Variables can reference placeholders (e.g., "{{user}}") that get substituted at runtime. diff --git a/HadesScheduler/HadesOperator/go.sum b/HadesScheduler/HadesOperator/go.sum index 732b9e9e..e63bff97 100644 --- a/HadesScheduler/HadesOperator/go.sum +++ b/HadesScheduler/HadesOperator/go.sum @@ -1,6 +1,7 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -30,9 +31,11 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= @@ -70,6 +73,7 @@ github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTe github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA= github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y= github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= @@ -83,7 +87,9 @@ github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4Y github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE= github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4= +github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= @@ -116,9 +122,13 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -126,6 +136,7 @@ github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85 github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -154,6 +165,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -165,7 +177,9 @@ github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4Ul github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -175,18 +189,27 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= github.com/testcontainers/testcontainers-go/modules/k3s v0.42.0 h1:bTVmcnYaSHesN6HXXxV/k0+BMkyfo3VBy4w4yRqOIgE= +github.com/testcontainers/testcontainers-go/modules/k3s v0.42.0/go.mod h1:2O8+V4WzMb/bjg/Sez+aYci9LpGUbT5cSz7ildfTIb8= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -200,6 +223,7 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -215,6 +239,7 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go b/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go index cd11cc5b..bc63885a 100644 --- a/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go +++ b/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go @@ -343,13 +343,18 @@ func buildK8sJob(bj *buildv1.BuildJob, jobName string, deleteOnComplete bool, su Env: append( envFromMeta(s.Metadata), corev1.EnvVar{Name: "UUID", Value: bj.Name}, + corev1.EnvVar{Name: "JOB_NAME", Value: bj.Spec.Name}, ), // Convert metadata to environment variables VolumeMounts: []corev1.VolumeMount{sharedMount}, } if strings.TrimSpace(s.Script) != "" { c.Command = []string{"/bin/sh", "-c"} - c.Args = []string{s.Script} + if s.ContinueOnError { + c.Args = []string{s.Script + "; exit 0"} + } else { + c.Args = []string{s.Script} + } } // Set resource limits if specified diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 444e8094..9a038c00 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -2,6 +2,8 @@ package docker import ( "context" + "errors" + "fmt" "log/slog" "maps" @@ -21,6 +23,13 @@ type Job struct { type jobIDContextKey string func (d Job) execute(ctx context.Context) error { + // Failures from steps marked ContinueOnError are tolerated by design (e.g. a test step whose + // failure is a normal, expected outcome that a later step, such as result parsing, still needs + // to report). They're joined here purely for logging and must not fail the overall job - that + // would defeat the point of ContinueOnError and would be inconsistent with the Kubernetes + // executor, where such a step never fails the job at all. + stepErr := error(nil) + for _, step := range d.Steps { d.logger.Info("Executing step", slog.Any("step", step)) @@ -28,6 +37,8 @@ func (d Job) execute(ctx context.Context) error { var envs = make(map[string]string) maps.Copy(envs, d.Metadata) maps.Copy(envs, step.Metadata) + envs["UUID"] = d.ID.String() + envs["JOB_NAME"] = d.Name step.Metadata = envs dockerStep := Step{ @@ -42,8 +53,17 @@ func (d Job) execute(ctx context.Context) error { err := dockerStep.execute(stepCtx) if err != nil { d.logger.Error("Failed to execute step", slog.Any("error", err)) + if step.ContinueOnError { + d.logger.Info("Next step should be executed despite error due to ContinueOnError setting", slog.Any("step", step)) + stepErr = errors.Join(stepErr, fmt.Errorf("step %v failed with ContinueOnError set: %w", step.ID, err)) + continue + } return err } } + + if stepErr != nil { + d.logger.Warn("Job completed with tolerated step failures", slog.Any("error", stepErr)) + } return nil } diff --git a/HadesScheduler/docker/step.go b/HadesScheduler/docker/step.go index 9c530990..f7292274 100644 --- a/HadesScheduler/docker/step.go +++ b/HadesScheduler/docker/step.go @@ -87,6 +87,15 @@ func (s Step) execute(ctx context.Context) error { return err } + defer func() { + if s.Options.containerAutoremove { + return + } + if err := removeContainer(ctx, s.cli, resp.ID); err != nil { + s.logger.Error("Failed to cleanup container", slog.Any("error", err), slog.Any("container_id", resp.ID)) + } + }() + // Start the container err = s.cli.ContainerStart(ctx, resp.ID, container.StartOptions{}) if err != nil { @@ -95,7 +104,7 @@ func (s Step) execute(ctx context.Context) error { } // Wait for the container to finish - statusCh, errCh := s.cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + statusCh, errCh := s.cli.ContainerWait(ctx, resp.ID, container.WaitCondition(container.WaitConditionNotRunning)) select { case err := <-errCh: if err != nil { @@ -103,6 +112,15 @@ func (s Step) execute(ctx context.Context) error { return err } case status := <-statusCh: + // Write the container logs to NATS before checking status + // Logs should be written even if the container fails to start or crashes during execution + if err := processContainerLogs(ctx, s.cli, s.publisher, resp.ID, jobId); err != nil { + s.logger.Error("Failed to write container logs to NATS", slog.Any("error", err), slog.Any("container_id", resp.ID)) + return err + } else { + s.logger.Debug("Container logs written to NATS", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) + } + if status.StatusCode != 0 { s.logger.Error("Container exited with status", slog.Any("status", status.StatusCode), slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) return fmt.Errorf("container exited with status %d", status.StatusCode) @@ -110,20 +128,5 @@ func (s Step) execute(ctx context.Context) error { } s.logger.Debug("Container completed", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) - - // Write the container logs to NATS - err = processContainerLogs(ctx, s.cli, s.publisher, resp.ID, jobId) - - if err != nil { - s.logger.Error("Failed to write container logs to NATS", slog.Any("error", err), slog.Any("container_id", resp.ID)) - return err - } else { - s.logger.Debug("Container logs written to NATS", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) - } - - if err := removeContainer(ctx, s.cli, resp.ID); err != nil { - s.logger.Error("Failed to cleanup container", slog.Any("error", err), slog.Any("container_id", resp.ID)) - } - return nil } diff --git a/HadesScheduler/go.sum b/HadesScheduler/go.sum index 744a3524..7224a7af 100644 --- a/HadesScheduler/go.sum +++ b/HadesScheduler/go.sum @@ -70,6 +70,7 @@ github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTe github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA= github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y= github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= @@ -83,7 +84,9 @@ github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4Y github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE= github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4= +github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/HadesScheduler/k8s/k8s.go b/HadesScheduler/k8s/k8s.go index a04d3fef..33fd5bc9 100644 --- a/HadesScheduler/k8s/k8s.go +++ b/HadesScheduler/k8s/k8s.go @@ -212,9 +212,10 @@ func (k Scheduler) createBuildJobCR(ctx context.Context, job payload.QueuePayloa steps := make([]map[string]interface{}, 0, len(job.Steps)) for _, s := range job.Steps { sm := map[string]interface{}{ - "id": s.ID, - "name": s.Name, - "image": s.Image, + "id": s.ID, + "name": s.Name, + "image": s.Image, + "continueOnError": s.ContinueOnError, } if s.Script != "" { sm["script"] = s.Script diff --git a/compose.yml b/compose.yml index 711acc7b..76f7844c 100644 --- a/compose.yml +++ b/compose.yml @@ -37,6 +37,25 @@ services: environment: - NATS_URL=nats://nats:4222 - HADES_EXECUTOR=docker + + hadesLogManager: + image: ghcr.io/ls1intum/hades/hades-log-manager + build: + context: . + dockerfile: ./HadesLogManager/Dockerfile + ports: + - "8083:8081" # Map to 8083 to avoid conflicts + networks: + - hades + depends_on: + nats: + condition: service_healthy + hadesAPI: + condition: service_started + environment: + - HADESLOGMANAGER_API_PORT=8081 + - ARTEMIS_ADAPTER_URL=${ARTEMIS_ADAPTER_URL} + - NATS_URL=nats://nats:4222 nats: image: nats:2.11.4 diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index 3e6c807f..38a469e5 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -35,6 +35,19 @@ services: networks: - hades + hadesLogManager: + ports: [] + expose: + - "8081" + networks: + - hades + labels: + - "traefik.enable=true" + - "traefik.http.routers.hadeslogmanager.rule=Host(`${HADES_LOGMANAGER_HOST:-logs.${HADES_API_HOST}}`)" + - "traefik.http.routers.hadeslogmanager.entrypoints=web,websecure" + - "traefik.http.routers.hadeslogmanager.tls.certresolver=letsencrypt" + - "traefik.http.services.hadeslogmanager.loadbalancer.server.port=8081" + nats: ports: [] networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 69830f17..57f2fe99 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -15,4 +15,12 @@ services: - DOCKER_CONTAINER_AUTOREMOVE=false - DEBUG=true volumes: - - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file + - /var/run/docker.sock:/var/run/docker.sock + hadesLogManager: + ports: + - "8083:8081" # Expose for local access + environment: + - DEBUG=true + - HADESLOGMANAGER_API_PORT=8081 + - ARTEMIS_ADAPTER_URL=${ARTEMIS_ADAPTER_URL} + \ No newline at end of file diff --git a/docker-compose.k8s.yml b/docker-compose.k8s.yml index 43f00bef..d2378783 100644 --- a/docker-compose.k8s.yml +++ b/docker-compose.k8s.yml @@ -1,7 +1,6 @@ version: '3.7' services: - hadesScheduler: environment: - NATS_URL=nats://nats:4222 diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 0f5d0e5a..58e5f9b9 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -47,6 +47,20 @@ services: - DOCKER_MEMORY_LIMIT=${DOCKER_MEMORY_LIMIT:-2g} restart: unless-stopped + hadesLogManager: + image: ghcr.io/ls1intum/hades/hades-log-manager:${IMAGE_TAG:?IMAGE_TAG is required} + expose: + - "8081" + networks: + - hades + depends_on: + nats: + condition: service_healthy + environment: + - NATS_URL=nats://nats:4222 + - ARTEMIS_ADAPTER_URL=https://hades-artemis-connector.student.k8s.aet.cit.tum.de/adapter/logs + restart: unless-stopped + nats: image: nats:2.11.4 command: ["-js", "-m", "8222"] diff --git a/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru new file mode 100644 index 00000000..174c5682 --- /dev/null +++ b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru @@ -0,0 +1,89 @@ +meta { + name: Create Build Job (Test Fail) with Result Parse Step + type: http + seq: 6 +} + +post { + url: http://{{hostname}}/build + body: json + auth: none +} + +body:json { + { + "name": "Example Job", + "volumes": [ + { + "name": "shared", + "emptyDir": {} + } + ], + "metadata": { + "GLOBAL": "test" + }, + "timestamp": "2021-01-01T00:00:00.000Z", + "priority": 3, // optional, default 3 + "steps": [ + { + "id": 1, // mandatory to declare the order of execution + "name": "Clone", + "image": "ghcr.io/ls1intum/hades/hades-clone-container:latest", // mandatory + "volumeMounts": [ + { "name": "shared", "mountPath": "/shared" } + ], + "workingDir": "/shared", + "metadata": { + "REPOSITORY_DIR": "/shared", + "HADES_TEST_USERNAME": "{{user}}", + "HADES_TEST_PASSWORD": "{{password}}", + "HADES_TEST_URL": "{{test_repo}}", + "HADES_TEST_PATH": "./example", + "HADES_TEST_ORDER": "1", + "HADES_ASSIGNMENT_USERNAME": "{{user}}", + "HADES_ASSIGNMENT_PASSWORD": "{{password}}", + "HADES_ASSIGNMENT_URL": "{{assignment_repo}}", + "HADES_ASSIGNMENT_PATH": "./example/assignment", + "HADES_ASSIGNMENT_ORDER": "2" + } + }, + { + "id": 2, // mandatory to declare the order of execution + "name": "Execute", + "image": "ls1tum/artemis-maven-template:java17-18", // mandatory + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test", + "metadata": { + "CONTINUE_ON_ERROR": "true" + } + }, + { + "id": 3, + "name": "Parse Results", + "image": "ghcr.io/ls1intum/hades/junit-result-parser:latest", + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "metadata": { + "API_ENDPOINT": "http://host.docker.internal:8083/adapter/test-results", + "INGEST_DIR": "/shared/example/build/test-results/test" + } + } + ] + } +} + +vars:pre-request { + user: + password: + test_repo: https://github.com/Mtze/Artemis-Java-Test.git + assignment_repo: https://github.com/Mtze/Artemis-Java-Template.git +} diff --git a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru new file mode 100644 index 00000000..5250ff68 --- /dev/null +++ b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru @@ -0,0 +1,89 @@ +meta { + name: Create Build Job (Test Succeed) with Result Parse Step + type: http + seq: 5 +} + +post { + url: http://{{hostname}}/build + body: json + auth: none +} + +body:json { + { + "name": "Example Job", + "volumes": [ + { + "name": "shared", + "emptyDir": {} + } + ], + "metadata": { + "GLOBAL": "test" + }, + "timestamp": "2021-01-01T00:00:00.000Z", + "priority": 1, // optional, default 3 + "steps": [ + { + "id": 1, // mandatory to declare the order of execution + "name": "Clone", + "image": "ghcr.io/ls1intum/hades/hades-clone-container:latest", // mandatory + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "workingDir": "/shared", + "metadata": { + "REPOSITORY_DIR": "/shared", + "HADES_TEST_USERNAME": "{{user}}", + "HADES_TEST_PASSWORD": "{{password}}", + "HADES_TEST_URL": "{{test_repo}}", + "HADES_TEST_PATH": "./example", + "HADES_TEST_ORDER": "1", + "HADES_ASSIGNMENT_USERNAME": "{{user}}", + "HADES_ASSIGNMENT_PASSWORD": "{{password}}", + "HADES_ASSIGNMENT_URL": "{{assignment_repo}}", + "HADES_ASSIGNMENT_PATH": "./example/assignment", + "HADES_ASSIGNMENT_ORDER": "2" + } + }, + { + "id": 2, // mandatory to declare the order of execution + "name": "Execute", + "image": "ls1tum/artemis-maven-template:java17-18", // mandatory + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test" + }, + { + "id": 3, + "name": "Parse Results", + "image": "ghcr.io/ls1intum/hades/junit-result-parser:latest", + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "metadata": { + "API_ENDPOINT": "http://host.docker.internal:8083/adapter/test-results", + "INGEST_DIR": "/shared/example/build/test-results/test" + } + } + ] + } +} + +vars:pre-request { + user: + password: + test_repo: https://github.com/Mtze/Artemis-Java-Test.git + assignment_repo: https://github.com/Mtze/Artemis-Java-Solution.git +} diff --git a/helm/hades/crds/build.hades.tum.de_buildjobs.yaml b/helm/hades/crds/build.hades.tum.de_buildjobs.yaml index 7f8e006d..97df19f2 100644 --- a/helm/hades/crds/build.hades.tum.de_buildjobs.yaml +++ b/helm/hades/crds/build.hades.tum.de_buildjobs.yaml @@ -70,6 +70,12 @@ spec: Typical pipeline: Step 1 clones code, Step 2 builds it, Step 3 runs tests. items: properties: + continueOnError: + description: |- + If true, the BuildJob pipeline continues to the next step even if this step's container + exits with a non-zero status. Useful for steps that must always run regardless of prior + failures, such as result parsing or cleanup. + type: boolean cpuLimit: anyOf: - type: integer diff --git a/shared/buildlogs/interfaces.go b/shared/buildlogs/interfaces.go index 2e7095e4..e2b630d3 100644 --- a/shared/buildlogs/interfaces.go +++ b/shared/buildlogs/interfaces.go @@ -12,6 +12,7 @@ type LogAggregator interface { FlushJob(jobID string) error GetJobLogs(jobID string) []Log GetAllJobs() []string + SendJobLogs(jobID string) error MarkJobCompleted(jobID string) UpdateJobStatus(jobID string, status buildstatus.JobStatus) GetJobStatus(jobID string) (buildstatus.JobStatus, error) diff --git a/shared/payload/payload.go b/shared/payload/payload.go index 4c24296c..4d8d00fd 100644 --- a/shared/payload/payload.go +++ b/shared/payload/payload.go @@ -32,13 +32,14 @@ type QueuePayload struct { // Step represents a single execution step in a job. // Each step runs in its own container with the specified image and resources. type Step struct { - ID int `json:"id"` // Step execution order (starts at 1) - Name string `json:"name"` // Human-readable step name - Image string `json:"image"` // Container image to use (e.g., "alpine:latest") - Script string `json:"script"` // Shell script to execute in the container - Metadata map[string]string `json:"metadata"` // Step-specific environment variables and metadata - CPULimit uint `json:"cpu_limit"` // CPU limit in millicores (e.g., 1000 = 1 CPU core) - MemoryLimit string `json:"memory_limit"` // Memory limit (e.g., "512M", "2G") + ID int `json:"id"` // Step execution order (starts at 1) + Name string `json:"name"` // Human-readable step name + Image string `json:"image"` // Container image to use (e.g., "alpine:latest") + Script string `json:"script"` // Shell script to execute in the container + ContinueOnError bool `json:"continue_on_error"` // Whether to continue with the next step if this step fails + Metadata map[string]string `json:"metadata"` // Step-specific environment variables and metadata + CPULimit uint `json:"cpu_limit"` // CPU limit in millicores (e.g., 1000 = 1 CPU core) + MemoryLimit string `json:"memory_limit"` // Memory limit (e.g., "512M", "2G") } // IDString returns the step ID as a string.