From ed1af69a06d8d419f88185060dc1538aa6f84f4e Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 26 Jan 2026 14:10:56 +0100 Subject: [PATCH 01/36] HadesArtemisAdapter init --- HadesArtemisAdapter/go.mod | 50 +++++ HadesArtemisAdapter/go.sum | 110 +++++++++++ HadesArtemisAdapter/main.go | 183 ++++++++++++++++++ ... (Test Succeed) with Result Parse Step.bru | 81 ++++++++ go.work | 3 +- 5 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 HadesArtemisAdapter/go.mod create mode 100644 HadesArtemisAdapter/go.sum create mode 100644 HadesArtemisAdapter/main.go create mode 100644 docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod new file mode 100644 index 00000000..7ac3fa5c --- /dev/null +++ b/HadesArtemisAdapter/go.mod @@ -0,0 +1,50 @@ +module Hades-ArtemisAdapter + +go 1.25.3 + +require ( + github.com/gin-gonic/gin v1.11.0 + github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b +) + +require ( + github.com/bytedance/sonic v1.14.0 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/caarlos0/env/v11 v11.3.1 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/joshdk/go-junit v1.0.0 + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nats-io/nats.go v1.48.0 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.54.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + go.uber.org/mock v0.5.0 // indirect + golang.org/x/arch v0.20.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect + google.golang.org/protobuf v1.36.9 // indirect +) diff --git a/HadesArtemisAdapter/go.sum b/HadesArtemisAdapter/go.sum new file mode 100644 index 00000000..f720fbda --- /dev/null +++ b/HadesArtemisAdapter/go.sum @@ -0,0 +1,110 @@ +github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= +github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= +github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b h1:qr4I+vH9qvuUIDMNyhcovwSapdd1AAbVjr2D8M1rlbw= +github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b/go.mod h1:dtZoDaLxyhyNlMNGZ/zHJ2pEwvVcv62uPZD4f4y1jKY= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= +github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= +github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= +golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= +google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go new file mode 100644 index 00000000..d62825b9 --- /dev/null +++ b/HadesArtemisAdapter/main.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/gin-gonic/gin" + "github.com/joshdk/go-junit" + "github.com/ls1intum/hades/shared/buildlogs" + "github.com/ls1intum/hades/shared/utils" +) + +type HadesArtemisAdapterConfig struct { + APIPort string `env:"API_PORT" envDefault:"8082"` +} + +// ResultMetadata populated from environment variables +type ResultMetadata struct { + JobName string `json:"jobName" env:"JOB_NAME"` + UUID string `json:"uuid" env:"UUID"` + AssignmentRepoBranchName string `json:"assignmentRepoBranchName" env:"ASSIGNMENT_REPO_BRANCH_NAME" envDefault:"main"` + IsBuildSuccessful bool `json:"isBuildSuccessful" env:"IS_BUILD_SUCCESSFUL"` + AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash" env:"ASSIGNMENT_REPO_COMMIT_HASH"` + TestsRepoCommitHash string `json:"testsRepoCommitHash" env:"TESTS_REPO_COMMIT_HASH"` + BuildCompletionTime string `json:"buildCompletionTime" env:"BUILD_COMPLETION_TIME"` +} +type ResultDTO struct { + ResultMetadata + BuildJobs []junit.Suite `json:"buildJobs"` +} + +func main() { + // Setup logging + utils.SetupLogging() + + // Load configuration + var cfg HadesArtemisAdapterConfig + utils.LoadConfig(&cfg) + + // Run main application + if err := run(cfg); err != nil { + slog.Error("Application error", "error", err) + os.Exit(1) + } +} + +// run contains the main application logic with proper error handling +func run(cfg HadesArtemisAdapterConfig) error { + + // Create context for application lifecycle + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Set up graceful shutdown + return runWithGracefulShutdown(ctx, cancel, cfg) +} + +// runWithGracefulShutdown starts services and handles graceful shutdown +func runWithGracefulShutdown( + ctx context.Context, + cancel context.CancelFunc, + cfg HadesArtemisAdapterConfig, +) error { + var wg sync.WaitGroup + errChan := make(chan error, 2) + + // Start API server + router := setupAPIRoute() + server := &http.Server{ + Addr: ":" + cfg.APIPort, + Handler: router, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + ReadHeaderTimeout: 5 * time.Second, + } + + wg.Add(1) + go func() { + defer wg.Done() + slog.Info("Starting API server", "port", cfg.APIPort) + + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("API server failed", "error", err) + errChan <- err + } + }() + + // Wait for shutdown signal or error + return waitForShutdown(ctx, cancel, server, &wg, errChan) +} + +// waitForShutdown waits for OS signal or error and performs graceful shutdown +func waitForShutdown( + ctx context.Context, + cancel context.CancelFunc, + server *http.Server, + wg *sync.WaitGroup, + errChan chan error, +) error { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + var shutdownErr error + + select { + case sig := <-sigChan: + slog.Info("Received shutdown signal", "signal", sig.String()) + case err := <-errChan: + slog.Error("Error during operation", "error", err) + shutdownErr = err + } + + // Cancel context to stop background goroutines + slog.Info("Starting graceful shutdown...") + cancel() + + // Shutdown API server with timeout + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + slog.Error("API server shutdown error", "error", err) + if shutdownErr == nil { + shutdownErr = err + } + } else { + slog.Info("API server shutdown complete") + } + + // Wait for all goroutines to finish + wg.Wait() + slog.Info("Graceful shutdown complete") + + return shutdownErr +} + +func setupAPIRoute() *gin.Engine { + r := gin.Default() + jobs := r.Group("/adapter") + { + // post logs for specific job + jobs.POST("/logs", func(c *gin.Context) { + var newLogs buildlogs.Log + if err := c.BindJSON(&newLogs); err != nil { + return + } + + // logs = append(logs, newLogs) + c.IndentedJSON(http.StatusCreated, newLogs) + }) + + // post results for specific job + jobs.POST("/results", func(c *gin.Context) { + var newResults ResultDTO + if err := c.BindJSON(&newResults); err != nil { + return + } + + // Pretty print the whole thing + jsonBytes, _ := json.MarshalIndent(newResults, "", " ") + fmt.Println(string(jsonBytes)) + + // results = append(results, newResults) + c.IndentedJSON(http.StatusCreated, newResults) + }) + } + + // Health check endpoint + r.GET("/health", func(c *gin.Context) { + c.JSON(200, gin.H{"status": "ok"}) + }) + + return r +} diff --git a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru new file mode 100644 index 00000000..bb5872d8 --- /dev/null +++ b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru @@ -0,0 +1,81 @@ +meta { + name: Create Build Job (Test Succeed) with Result Parse Step + type: http + seq: 5 +} + +post { + url: http://{{hostname}}/build + body: json + auth: none +} + +body:json { + { + "name": "Example Job", + "volumes": [ + { + "name": "shared", + "emptyDir": {} + } + ], + "metadata": { + "GLOBAL": "test" + }, + "timestamp": "2021-01-01T00:00:00.000Z", + "priority": 1, // optional, default 3 + "steps": [ + { + "id": 1, // mandatory to declare the order of execution + "name": "Clone", + "image": "ghcr.io/ls1intum/hades/hades-clone-container:latest", // mandatory + "volumeMounts": [ + { "name": "shared", "mountPath": "/shared" } + ], + "workingDir": "/shared", + "metadata": { + "REPOSITORY_DIR": "/shared", + "HADES_TEST_USERNAME": "{{user}}", + "HADES_TEST_PASSWORD": "{{password}}", + "HADES_TEST_URL": "{{test_repo}}", + "HADES_TEST_PATH": "./example", + "HADES_TEST_ORDER": "1", + "HADES_ASSIGNMENT_USERNAME": "{{user}}", + "HADES_ASSIGNMENT_PASSWORD": "{{password}}", + "HADES_ASSIGNMENT_URL": "{{assignment_repo}}", + "HADES_ASSIGNMENT_PATH": "./example/assignment", + "HADES_ASSIGNMENT_ORDER": "2" + } + }, + { + "id": 2, // mandatory to declare the order of execution + "name": "Execute", + "image": "ls1tum/artemis-maven-template:java17-18", // mandatory + "volumeMounts": [ + { "name": "shared", "mountPath": "/shared" } + ], + "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test" + }, + { + "id": 3, + "name": "Parse Results", + "image": "ghcr.io/ls1intum/hades/junit-result-parser:latest", + "volumeMounts": [ + { "name": "shared", "mountPath": "/shared" } + ], + "metadata": { + "ADAPTER_ENDPOINT": "{ADAPTER_ENDPOINT}/jobs/{JOB_ID}/results", + "INGEST_DIR": "/shared/example/build/test-results/test", + "JOB_ID": "{jobId}", + } + } + ] + } +} + +vars:pre-request { + user: + password: + test_repo: https://github.com/Mtze/Artemis-Java-Test.git + assignment_repo: https://github.com/Mtze/Artemis-Java-Solution.git +} diff --git a/go.work b/go.work index d93335e4..daf80089 100644 --- a/go.work +++ b/go.work @@ -1,7 +1,8 @@ -go 1.25.0 +go 1.25.3 use ( ./HadesAPI + ./HadesArtemisAdapter ./HadesLogManager ./HadesScheduler ./HadesScheduler/HadesOperator From 123ecce99aeb6cd71474f67b50ce6633ab26dfe1 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 26 Jan 2026 15:51:06 +0100 Subject: [PATCH 02/36] modify HadesLogManager to post logs to HadesArtemisAdapter --- HadesArtemisAdapter/main.go | 12 +--- HadesLogManager/log_subscriber.go | 1 + HadesLogManager/processor.go | 70 ++++++++++++++----- ... (Test Succeed) with Result Parse Step.bru | 25 ++++--- 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index d62825b9..76335ae9 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -2,9 +2,7 @@ package main import ( "context" - "encoding/json" "errors" - "fmt" "log/slog" "net/http" "os" @@ -149,7 +147,7 @@ func setupAPIRoute() *gin.Engine { { // post logs for specific job jobs.POST("/logs", func(c *gin.Context) { - var newLogs buildlogs.Log + var newLogs []buildlogs.Log if err := c.BindJSON(&newLogs); err != nil { return } @@ -158,17 +156,13 @@ func setupAPIRoute() *gin.Engine { c.IndentedJSON(http.StatusCreated, newLogs) }) - // post results for specific job - jobs.POST("/results", func(c *gin.Context) { + // post test results for specific job + jobs.POST("/test-results", func(c *gin.Context) { var newResults ResultDTO if err := c.BindJSON(&newResults); err != nil { return } - // Pretty print the whole thing - jsonBytes, _ := json.MarshalIndent(newResults, "", " ") - fmt.Println(string(jsonBytes)) - // results = append(results, newResults) c.IndentedJSON(http.StatusCreated, newResults) }) diff --git a/HadesLogManager/log_subscriber.go b/HadesLogManager/log_subscriber.go index 1aeb6246..36f0752c 100644 --- a/HadesLogManager/log_subscriber.go +++ b/HadesLogManager/log_subscriber.go @@ -231,5 +231,6 @@ func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) { watcher.cancel() dlm.logAggregator.MarkJobCompleted(jobID) + dlm.logAggregator.SendJobLogs(jobID) } } diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index 86d9127f..f3b85cf0 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "log/slog" + "net/http" "sync" "time" @@ -11,15 +14,19 @@ import ( "github.com/ls1intum/hades/shared/buildstatus" ) +const APIendpoint = "http://localhost:8082/adapter/logs" +const APItoken = "" + // LogAggregator defines the interface for aggregating and managing job logs type LogAggregator interface { AddLog(log buildlogs.Log) FlushJob(jobID string) error GetJobLogs(jobID string) []buildlogs.Log + GetJobStatus(jobID string) (string, error) GetAllJobs() []string + SendJobLogs(jobID string) error MarkJobCompleted(jobID string) UpdateJobStatus(jobID string, status buildstatus.JobStatus) - GetJobStatus(jobID string) (string, error) } // NATSLogAggregator implements LogAggregator using in-memory storage for fast log retrieval. @@ -98,9 +105,7 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { return } - slog.Debug("Adding log to aggregator", - "job_id", log.JobID, - "entries", len(log.Logs)) + slog.Debug("Adding log to aggregator", "job_id", log.JobID, "entries", len(log.Logs)) // Use LoadOrStore and CompareAndSwap for thread-safe updates for { @@ -117,9 +122,7 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { if len(newLogs) > la.config.MaxJobLogs { trimStart := len(newLogs) - la.config.MaxJobLogs newLogs = newLogs[trimStart:] - slog.Debug("Trimmed old logs", - "job_id", log.JobID, - "trimmed_count", trimStart) + slog.Debug("Trimmed old logs", "job_id", log.JobID, "trimmed_count", trimStart) } newPtr := &newLogs @@ -127,9 +130,7 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { // Atomically swap if the value hasn't changed if la.logs.CompareAndSwap(log.JobID, old, newVal) { - slog.Debug("Added log to aggregator", - "job_id", log.JobID, - "total_batches", len(newLogs)) + slog.Debug("Added log to aggregator", "job_id", log.JobID, "total_batches", len(newLogs)) break } // If swap failed, another goroutine modified the value - retry @@ -149,9 +150,7 @@ func (la *NATSLogAggregator) FlushJob(jobID string) error { if logsExists { v := value.(logsVersion) logs := *v.ptr - slog.Info("Flushed job logs", - "job_id", jobID, - "batch_count", len(logs)) + slog.Info("Flushed job logs", "job_id", jobID, "batch_count", len(logs)) } else { slog.Debug("No logs to flush for job", "job_id", jobID) } @@ -168,9 +167,7 @@ func (la *NATSLogAggregator) FlushJob(jobID string) error { // MarkJobCompleted marks a job as completed and schedules it for cleanup after retention period. func (la *NATSLogAggregator) MarkJobCompleted(jobID string) { la.completed.Store(jobID, time.Now()) - slog.Info("Marked job as completed", - "job_id", jobID, - "retention", la.config.Retention) + slog.Info("Marked job as completed", "job_id", jobID, "retention", la.config.Retention) } // cleanupCompletedJobs removes logs for jobs that have exceeded the retention period. @@ -186,9 +183,7 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { slog.Debug("Retention expired, flushing job", "job_id", jobID) if err := la.FlushJob(jobID); err != nil { - slog.Error("Failed to flush job during cleanup", - "job_id", jobID, - "error", err) + slog.Error("Failed to flush job during cleanup", "job_id", jobID, "error", err) } else { cleanedCount++ } @@ -202,6 +197,43 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { } } +func (la *NATSLogAggregator) SendJobLogs(jobID string) error { + logs := la.GetJobLogs(jobID) + + // Convert the DTO to JSON + jsonData, err := json.Marshal(logs) + if err != nil { + slog.Error("Error parsing logs to JSON", "error", err) + } + slog.Debug("Parsed logs to JSON", "json", string(jsonData)) + + slog.Info("Sending job logs to the API", "job_id", jobID, "batch_count", len(logs)) + req, err := http.NewRequest("POST", APIendpoint, bytes.NewBuffer(jsonData)) + if err != nil { + slog.Error("Error creating the request", "error", err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", APItoken) + + // Create a Client + client := &http.Client{} + + // Send the request via a client + resp, err := client.Do(req) + if err != nil { + slog.Error("Error sending the request", "error", err) + } + + slog.Info("Request sent", "status", resp.Status) + + // Close response body + defer resp.Body.Close() + + return nil +} + // GetJobLogs retrieves all log entries for a specific job ID by flattening // the batched logs into a single slice. // diff --git a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru index bb5872d8..26588279 100644 --- a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru +++ b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru @@ -30,7 +30,10 @@ body:json { "name": "Clone", "image": "ghcr.io/ls1intum/hades/hades-clone-container:latest", // mandatory "volumeMounts": [ - { "name": "shared", "mountPath": "/shared" } + { + "name": "shared", + "mountPath": "/shared" + } ], "workingDir": "/shared", "metadata": { @@ -52,7 +55,10 @@ body:json { "name": "Execute", "image": "ls1tum/artemis-maven-template:java17-18", // mandatory "volumeMounts": [ - { "name": "shared", "mountPath": "/shared" } + { + "name": "shared", + "mountPath": "/shared" + } ], "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test" }, @@ -61,13 +67,16 @@ body:json { "name": "Parse Results", "image": "ghcr.io/ls1intum/hades/junit-result-parser:latest", "volumeMounts": [ - { "name": "shared", "mountPath": "/shared" } + { + "name": "shared", + "mountPath": "/shared" + } ], - "metadata": { - "ADAPTER_ENDPOINT": "{ADAPTER_ENDPOINT}/jobs/{JOB_ID}/results", - "INGEST_DIR": "/shared/example/build/test-results/test", - "JOB_ID": "{jobId}", - } + "metadata": { + "API_ENDPOINT": "http://host.docker.internal:8082/adapter/test-results", + "INGEST_DIR": "/shared/example/build/test-results/test", + "JOB_NAME": "{jobId}" + } } ] } From 6b76ced462b15db4974956e7cdd4ee20f4d00f19 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 2 Feb 2026 16:00:57 +0100 Subject: [PATCH 03/36] add jobID to global env, add adapter.go init --- HadesArtemisAdapter/artemis_adapter.go | 39 +++++++++++++++++++ HadesArtemisAdapter/main.go | 35 +++++------------ HadesLogManager/main.go | 14 +------ HadesScheduler/docker/job.go | 1 + ... (Test Succeed) with Result Parse Step.bru | 3 +- 5 files changed, 52 insertions(+), 40 deletions(-) create mode 100644 HadesArtemisAdapter/artemis_adapter.go diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go new file mode 100644 index 00000000..03ff6a0c --- /dev/null +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "sync" + + "github.com/joshdk/go-junit" + "github.com/ls1intum/hades/shared/buildlogs" +) + +type ArtemisAdapter struct { + logs sync.Map // jobID (string) -> logsVersion + results sync.Map // jobID (string) -> reults +} + +// ResultMetadata populated from environment variables +type ResultMetadata struct { + JobName string `json:"jobName" env:"JOB_NAME"` + UUID string `json:"uuid" env:"UUID"` + AssignmentRepoBranchName string `json:"assignmentRepoBranchName" env:"ASSIGNMENT_REPO_BRANCH_NAME" envDefault:"main"` + IsBuildSuccessful bool `json:"isBuildSuccessful" env:"IS_BUILD_SUCCESSFUL"` + AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash" env:"ASSIGNMENT_REPO_COMMIT_HASH"` + TestsRepoCommitHash string `json:"testsRepoCommitHash" env:"TESTS_REPO_COMMIT_HASH"` + BuildCompletionTime string `json:"buildCompletionTime" env:"BUILD_COMPLETION_TIME"` +} +type ResultDTO struct { + ResultMetadata + BuildJobs []junit.Suite `json:"buildJobs"` + BuildLogs buildlogs.Log `json:"buildLogs"` +} + +func NewAdapter(ctx context.Context) *ArtemisAdapter { + aa := ArtemisAdapter{} + + // Start background cleanup goroutine + //go aa.cleanupLoop(ctx) + + return &aa +} diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index 76335ae9..e77e7d95 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/joshdk/go-junit" "github.com/ls1intum/hades/shared/buildlogs" "github.com/ls1intum/hades/shared/utils" ) @@ -21,21 +20,6 @@ type HadesArtemisAdapterConfig struct { APIPort string `env:"API_PORT" envDefault:"8082"` } -// ResultMetadata populated from environment variables -type ResultMetadata struct { - JobName string `json:"jobName" env:"JOB_NAME"` - UUID string `json:"uuid" env:"UUID"` - AssignmentRepoBranchName string `json:"assignmentRepoBranchName" env:"ASSIGNMENT_REPO_BRANCH_NAME" envDefault:"main"` - IsBuildSuccessful bool `json:"isBuildSuccessful" env:"IS_BUILD_SUCCESSFUL"` - AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash" env:"ASSIGNMENT_REPO_COMMIT_HASH"` - TestsRepoCommitHash string `json:"testsRepoCommitHash" env:"TESTS_REPO_COMMIT_HASH"` - BuildCompletionTime string `json:"buildCompletionTime" env:"BUILD_COMPLETION_TIME"` -} -type ResultDTO struct { - ResultMetadata - BuildJobs []junit.Suite `json:"buildJobs"` -} - func main() { // Setup logging utils.SetupLogging() @@ -58,21 +42,19 @@ func run(cfg HadesArtemisAdapterConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + adapter := NewAdapter(ctx) + // Set up graceful shutdown - return runWithGracefulShutdown(ctx, cancel, cfg) + return runWithGracefulShutdown(ctx, cancel, cfg, adapter) } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown( - ctx context.Context, - cancel context.CancelFunc, - cfg HadesArtemisAdapterConfig, -) error { +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesArtemisAdapterConfig, adapter *ArtemisAdapter) error { var wg sync.WaitGroup errChan := make(chan error, 2) // Start API server - router := setupAPIRoute() + router := setupAPIRoute(adapter) server := &http.Server{ Addr: ":" + cfg.APIPort, Handler: router, @@ -141,7 +123,7 @@ func waitForShutdown( return shutdownErr } -func setupAPIRoute() *gin.Engine { +func setupAPIRoute(adapter *ArtemisAdapter) *gin.Engine { r := gin.Default() jobs := r.Group("/adapter") { @@ -152,7 +134,7 @@ func setupAPIRoute() *gin.Engine { return } - // logs = append(logs, newLogs) + //adapter.logs.Store(newLogs., newLogs) c.IndentedJSON(http.StatusCreated, newLogs) }) @@ -163,7 +145,8 @@ func setupAPIRoute() *gin.Engine { return } - // results = append(results, newResults) + adapter.results.Store(newResults.UUID, newResults) + slog.Debug("Stored new test results", "uuid", newResults.UUID) c.IndentedJSON(http.StatusCreated, newResults) }) } diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 45f4ec39..3d3c4516 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -86,12 +86,7 @@ func connectNATS(config utils.NatsConfig) (*nats.Conn, error) { } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown( - ctx context.Context, - cancel context.CancelFunc, - cfg HadesLogManagerConfig, - dynamicManager LogManager, - logAggregator LogAggregator, +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesLogManagerConfig, dynamicManager LogManager, logAggregator LogAggregator, ) error { var wg sync.WaitGroup errChan := make(chan error, 2) @@ -134,12 +129,7 @@ func runWithGracefulShutdown( } // waitForShutdown waits for OS signal or error and performs graceful shutdown -func waitForShutdown( - ctx context.Context, - cancel context.CancelFunc, - server *http.Server, - wg *sync.WaitGroup, - errChan chan error, +func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error, ) error { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 444e8094..45cb9631 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -28,6 +28,7 @@ func (d Job) execute(ctx context.Context) error { var envs = make(map[string]string) maps.Copy(envs, d.Metadata) maps.Copy(envs, step.Metadata) + envs["UUID"] = d.ID.String() step.Metadata = envs dockerStep := Step{ diff --git a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru index 26588279..292958b0 100644 --- a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru +++ b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru @@ -74,8 +74,7 @@ body:json { ], "metadata": { "API_ENDPOINT": "http://host.docker.internal:8082/adapter/test-results", - "INGEST_DIR": "/shared/example/build/test-results/test", - "JOB_NAME": "{jobId}" + "INGEST_DIR": "/shared/example/build/test-results/test" } } ] From d18671435efd2172f0b0452fd234ea9ad0c42a99 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 2 Feb 2026 21:12:16 +0100 Subject: [PATCH 04/36] add adapter logic + send to artemis endpoint --- HadesAPI/main.go | 2 +- HadesArtemisAdapter/artemis_adapter.go | 101 ++++++++++++++++++++++-- HadesArtemisAdapter/main.go | 25 +++--- HadesLogManager/main.go | 2 +- HadesLogManager/processor.go | 2 +- docs/environments/Local-Development.bru | 2 +- 6 files changed, 107 insertions(+), 27 deletions(-) diff --git a/HadesAPI/main.go b/HadesAPI/main.go index d507858c..97faa930 100644 --- a/HadesAPI/main.go +++ b/HadesAPI/main.go @@ -10,7 +10,7 @@ import ( ) type HadesAPIConfig struct { - APIPort uint `env:"API_PORT,notEmpty" envDefault:"8080"` + APIPort uint `env:"API_PORT,notEmpty" envDefault:"8081"` NatsConfig utils.NatsConfig AuthKey string `env:"AUTH_KEY"` PrometheusAddress string `env:"PROMETHEUS_ADDRESS" envDefault:""` diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 03ff6a0c..e009eb18 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -1,16 +1,23 @@ package main import ( + "bytes" "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" "sync" + "time" "github.com/joshdk/go-junit" "github.com/ls1intum/hades/shared/buildlogs" ) type ArtemisAdapter struct { - logs sync.Map // jobID (string) -> logsVersion - results sync.Map // jobID (string) -> reults + logs sync.Map // jobID (string) -> []buildlogs.Log + results sync.Map // jobID (string) -> results + httpClient *http.Client } // ResultMetadata populated from environment variables @@ -25,15 +32,93 @@ type ResultMetadata struct { } type ResultDTO struct { ResultMetadata - BuildJobs []junit.Suite `json:"buildJobs"` - BuildLogs buildlogs.Log `json:"buildLogs"` + BuildJobs []junit.Suite `json:"buildJobs"` + BuildLogs []buildlogs.Log `json:"buildLogs"` } -func NewAdapter(ctx context.Context) *ArtemisAdapter { - aa := ArtemisAdapter{} +const artemisBaseURL = "http://localhost:8080" // Replace with actual Artemis URL +const newResultEndpoint = "api/programming/public/programming-exercises/new-result" +const requestTimeout = 10 * time.Second +const artemisAuthToken = "superduperlongtestingsecrettoken" - // Start background cleanup goroutine - //go aa.cleanupLoop(ctx) +func NewAdapter(ctx context.Context) *ArtemisAdapter { + aa := ArtemisAdapter{ + httpClient: &http.Client{Timeout: requestTimeout}, + } return &aa } + +// StoreLogs stores logs for a job ID and checks if results are ready +func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { + aa.logs.Store(jobID, logs) + return aa.checkAndSendIfReady(jobID) +} + +// StoreResults stores results for a job ID and checks if logs are ready +func (aa *ArtemisAdapter) StoreResults(jobID string, results ResultDTO) error { + aa.results.Store(jobID, results) + return aa.checkAndSendIfReady(jobID) +} + +// checkAndSendIfReady checks if both logs and results exist for a jobID +// If both are present, combines them and sends to Artemis +func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { + logs, logsExist := aa.logs.Load(jobID) + results, resultsExist := aa.results.Load(jobID) + + // If both logs and results exist, combine and send + if logsExist && resultsExist { + logs := logs.([]buildlogs.Log) + results := results.(ResultDTO) + + results.BuildLogs = logs + slog.Debug("Combined logs and results", "jobID", jobID) + + // Send to Artemis + if err := aa.sendToArtemis(results); err != nil { + slog.Error("Failed to send results with logs to Artemis", "jobID", jobID, "error", err) + return err + } + + // Clean up after successful send + aa.logs.Delete(jobID) + aa.results.Delete(jobID) + + slog.Info("Successfully sent results with logs to Artemis", "jobID", jobID) + } else { + slog.Debug("Waiting for complete data", "jobID", jobID, "hasLogs", logsExist, "hasResults", resultsExist) + } + + return nil +} + +// sendToArtemis sends the combined result DTO to the Artemis endpoint +func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { + endpoint := fmt.Sprintf("%s/%s", artemisBaseURL, newResultEndpoint) + + jsonData, err := json.Marshal(dto) + if err != nil { + return fmt.Errorf("failed to marshal DTO: %w", err) + } + + req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", artemisAuthToken) + + resp, err := aa.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("Artemis returned error status: %d", resp.StatusCode) + } + + return nil +} diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index e77e7d95..1cade3a8 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -17,7 +17,7 @@ import ( ) type HadesArtemisAdapterConfig struct { - APIPort string `env:"API_PORT" envDefault:"8082"` + APIPort string `env:"API_PORT" envDefault:"8083"` } func main() { @@ -42,19 +42,19 @@ func run(cfg HadesArtemisAdapterConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - adapter := NewAdapter(ctx) + aa := NewAdapter(ctx) // Set up graceful shutdown - return runWithGracefulShutdown(ctx, cancel, cfg, adapter) + return runWithGracefulShutdown(ctx, cancel, cfg, aa) } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesArtemisAdapterConfig, adapter *ArtemisAdapter) error { +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesArtemisAdapterConfig, aa *ArtemisAdapter) error { var wg sync.WaitGroup errChan := make(chan error, 2) // Start API server - router := setupAPIRoute(adapter) + router := setupAPIRoute(aa) server := &http.Server{ Addr: ":" + cfg.APIPort, Handler: router, @@ -79,13 +79,7 @@ func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg } // waitForShutdown waits for OS signal or error and performs graceful shutdown -func waitForShutdown( - ctx context.Context, - cancel context.CancelFunc, - server *http.Server, - wg *sync.WaitGroup, - errChan chan error, -) error { +func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error) error { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) @@ -123,7 +117,7 @@ func waitForShutdown( return shutdownErr } -func setupAPIRoute(adapter *ArtemisAdapter) *gin.Engine { +func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { r := gin.Default() jobs := r.Group("/adapter") { @@ -134,7 +128,8 @@ func setupAPIRoute(adapter *ArtemisAdapter) *gin.Engine { return } - //adapter.logs.Store(newLogs., newLogs) + aa.StoreLogs(newLogs[0].JobID, newLogs) + slog.Debug("Stored new logs", "uuid", newLogs[0].JobID) c.IndentedJSON(http.StatusCreated, newLogs) }) @@ -145,7 +140,7 @@ func setupAPIRoute(adapter *ArtemisAdapter) *gin.Engine { return } - adapter.results.Store(newResults.UUID, newResults) + aa.StoreResults(newResults.UUID, newResults) slog.Debug("Stored new test results", "uuid", newResults.UUID) c.IndentedJSON(http.StatusCreated, newResults) }) diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 3d3c4516..d8ee1d2a 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -25,7 +25,7 @@ const ( // HadesLogManagerConfig holds the configuration for the log manager type HadesLogManagerConfig struct { NatsConfig utils.NatsConfig - APIPort string `env:"API_PORT" envDefault:"8081"` + APIPort string `env:"API_PORT" envDefault:"8082"` } func main() { diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index f3b85cf0..1f8a0fd8 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -14,7 +14,7 @@ import ( "github.com/ls1intum/hades/shared/buildstatus" ) -const APIendpoint = "http://localhost:8082/adapter/logs" +const APIendpoint = "http://localhost:8083/adapter/logs" const APItoken = "" // LogAggregator defines the interface for aggregating and managing job logs diff --git a/docs/environments/Local-Development.bru b/docs/environments/Local-Development.bru index ada7af64..60e07f99 100644 --- a/docs/environments/Local-Development.bru +++ b/docs/environments/Local-Development.bru @@ -1,3 +1,3 @@ vars { - hostname: localhost:8080 + hostname: localhost:8081 } From f8cee80c58f74b79747600a610789560d62a426b Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Wed, 4 Feb 2026 18:39:39 +0100 Subject: [PATCH 05/36] use slog.Error, change ports back --- HadesAPI/main.go | 2 +- HadesArtemisAdapter/artemis_adapter.go | 10 ++++------ HadesArtemisAdapter/main.go | 2 +- HadesLogManager/main.go | 2 +- HadesLogManager/processor.go | 4 +--- docs/environments/Local-Development.bru | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/HadesAPI/main.go b/HadesAPI/main.go index 97faa930..d507858c 100644 --- a/HadesAPI/main.go +++ b/HadesAPI/main.go @@ -10,7 +10,7 @@ import ( ) type HadesAPIConfig struct { - APIPort uint `env:"API_PORT,notEmpty" envDefault:"8081"` + APIPort uint `env:"API_PORT,notEmpty" envDefault:"8080"` NatsConfig utils.NatsConfig AuthKey string `env:"AUTH_KEY"` PrometheusAddress string `env:"PROMETHEUS_ADDRESS" envDefault:""` diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index e009eb18..0d664d15 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -99,12 +99,12 @@ func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { jsonData, err := json.Marshal(dto) if err != nil { - return fmt.Errorf("failed to marshal DTO: %w", err) + slog.Error("Error parsing logs to JSON", "error", err) } req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData)) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + slog.Error("Error creating the request", "error", err) } req.Header.Set("Content-Type", "application/json") @@ -112,13 +112,11 @@ func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { resp, err := aa.httpClient.Do(req) if err != nil { - return fmt.Errorf("failed to send request: %w", err) + slog.Error("Error sending the request", "error", err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("Artemis returned error status: %d", resp.StatusCode) - } + slog.Info("Request sent", "status", resp.Status) return nil } diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index 1cade3a8..ad7cb4bb 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -17,7 +17,7 @@ import ( ) type HadesArtemisAdapterConfig struct { - APIPort string `env:"API_PORT" envDefault:"8083"` + APIPort string `env:"API_PORT" envDefault:"8082"` } func main() { diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index d8ee1d2a..3d3c4516 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -25,7 +25,7 @@ const ( // HadesLogManagerConfig holds the configuration for the log manager type HadesLogManagerConfig struct { NatsConfig utils.NatsConfig - APIPort string `env:"API_PORT" envDefault:"8082"` + APIPort string `env:"API_PORT" envDefault:"8081"` } func main() { diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index 1f8a0fd8..f64d9035 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -14,8 +14,7 @@ import ( "github.com/ls1intum/hades/shared/buildstatus" ) -const APIendpoint = "http://localhost:8083/adapter/logs" -const APItoken = "" +const APIendpoint = "http://localhost:8082/adapter/logs" // LogAggregator defines the interface for aggregating and managing job logs type LogAggregator interface { @@ -215,7 +214,6 @@ func (la *NATSLogAggregator) SendJobLogs(jobID string) error { // Set headers req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", APItoken) // Create a Client client := &http.Client{} diff --git a/docs/environments/Local-Development.bru b/docs/environments/Local-Development.bru index 60e07f99..ada7af64 100644 --- a/docs/environments/Local-Development.bru +++ b/docs/environments/Local-Development.bru @@ -1,3 +1,3 @@ vars { - hostname: localhost:8081 + hostname: localhost:8080 } From fbe1c44ff53a2abb3cb965c8d4274cbc300d0d89 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Tue, 10 Feb 2026 01:24:04 +0100 Subject: [PATCH 06/36] edit adapter sendToArtemis endpoint --- HadesArtemisAdapter/artemis_adapter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 0d664d15..ef433be2 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -95,7 +95,7 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { // sendToArtemis sends the combined result DTO to the Artemis endpoint func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { - endpoint := fmt.Sprintf("%s/%s", artemisBaseURL, newResultEndpoint) + endpoint := fmt.Sprintf("%s/%s/%s", artemisBaseURL, newResultEndpoint, dto.JobName) jsonData, err := json.Marshal(dto) if err != nil { From 6af0e66eb5c1d49b5524ce3bdf3b28f0401f5266 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Thu, 12 Feb 2026 21:55:46 +0100 Subject: [PATCH 07/36] send only execution logs instead --- HadesArtemisAdapter/artemis_adapter.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index ef433be2..94319624 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -15,7 +15,7 @@ import ( ) type ArtemisAdapter struct { - logs sync.Map // jobID (string) -> []buildlogs.Log + logs sync.Map // jobID (string) -> []buildlogs.LogEntry (only Step 2:execution logs) results sync.Map // jobID (string) -> results httpClient *http.Client } @@ -32,8 +32,8 @@ type ResultMetadata struct { } type ResultDTO struct { ResultMetadata - BuildJobs []junit.Suite `json:"buildJobs"` - BuildLogs []buildlogs.Log `json:"buildLogs"` + BuildJobs []junit.Suite `json:"results"` + BuildLogs []buildlogs.LogEntry `json:"logs"` } const artemisBaseURL = "http://localhost:8080" // Replace with actual Artemis URL @@ -49,9 +49,11 @@ func NewAdapter(ctx context.Context) *ArtemisAdapter { return &aa } -// StoreLogs stores logs for a job ID and checks if results are ready +// StoreLogs only stores Step 2 (execution) logs for a job ID and checks if results are ready func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { - aa.logs.Store(jobID, logs) + execution_logs := logs[1].Logs + aa.logs.Store(jobID, execution_logs) + return aa.checkAndSendIfReady(jobID) } @@ -69,7 +71,7 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { // If both logs and results exist, combine and send if logsExist && resultsExist { - logs := logs.([]buildlogs.Log) + logs := logs.([]buildlogs.LogEntry) results := results.(ResultDTO) results.BuildLogs = logs @@ -80,12 +82,11 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { slog.Error("Failed to send results with logs to Artemis", "jobID", jobID, "error", err) return err } + slog.Info("Successfully sent results with logs to Artemis", "jobID", jobID) // Clean up after successful send aa.logs.Delete(jobID) aa.results.Delete(jobID) - - slog.Info("Successfully sent results with logs to Artemis", "jobID", jobID) } else { slog.Debug("Waiting for complete data", "jobID", jobID, "hasLogs", logsExist, "hasResults", resultsExist) } From 0a2120b840d296d48033bad764a6f5971c8b3129 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Sun, 15 Feb 2026 17:03:19 +0100 Subject: [PATCH 08/36] adapt adapter to new ResultDTO --- HadesArtemisAdapter/artemis_adapter.go | 31 ++++++++++++++++++++++++-- HadesArtemisAdapter/go.mod | 1 - HadesArtemisAdapter/go.sum | 4 ---- HadesArtemisAdapter/main.go | 2 ++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 94319624..27a60f9b 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/joshdk/go-junit" "github.com/ls1intum/hades/shared/buildlogs" ) @@ -29,10 +28,37 @@ type ResultMetadata struct { AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash" env:"ASSIGNMENT_REPO_COMMIT_HASH"` TestsRepoCommitHash string `json:"testsRepoCommitHash" env:"TESTS_REPO_COMMIT_HASH"` BuildCompletionTime string `json:"buildCompletionTime" env:"BUILD_COMPLETION_TIME"` + Passed int `json:"passed" env:"PASSED"` } + +type TestSuiteDTO struct { + Name string `json:"name"` + Time float64 `json:"time"` + Errors int `json:"errors"` + Skipped int `json:"skipped"` + Failures int `json:"failures"` + Tests int `json:"tests"` + TestCases []TestCaseDTO `json:"testCases"` +} + +type TestCaseDTO struct { + Name string `json:"name"` + Classname string `json:"classname"` + Time float64 `json:"time"` + Failures []TestCaseDetailMessageDTO `json:"failures"` // empty for passing tests + Errors []TestCaseDetailMessageDTO `json:"errors"` // empty for passing tests + Successes []TestCaseDetailMessageDTO `json:"successInfos"` // empty for failing tests +} + +type TestCaseDetailMessageDTO struct { + Message string `json:"message"` + Type string `json:"type"` + MessageWithStackTrace string `json:"messageWithStackTrace"` +} + type ResultDTO struct { ResultMetadata - BuildJobs []junit.Suite `json:"results"` + Results []TestSuiteDTO `json:"results"` BuildLogs []buildlogs.LogEntry `json:"logs"` } @@ -60,6 +86,7 @@ func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { // StoreResults stores results for a job ID and checks if logs are ready func (aa *ArtemisAdapter) StoreResults(jobID string, results ResultDTO) error { aa.results.Store(jobID, results) + slog.Debug("Stored results", "jobID", jobID, "results", results.Results) return aa.checkAndSendIfReady(jobID) } diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod index 7ac3fa5c..11779d8d 100644 --- a/HadesArtemisAdapter/go.mod +++ b/HadesArtemisAdapter/go.mod @@ -21,7 +21,6 @@ require ( github.com/goccy/go-yaml v1.18.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/joho/godotenv v1.5.1 // indirect - github.com/joshdk/go-junit v1.0.0 github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect diff --git a/HadesArtemisAdapter/go.sum b/HadesArtemisAdapter/go.sum index f720fbda..6436c0b7 100644 --- a/HadesArtemisAdapter/go.sum +++ b/HadesArtemisAdapter/go.sum @@ -35,8 +35,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= -github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= @@ -72,7 +70,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -104,7 +101,6 @@ golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index ad7cb4bb..eb055f18 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -125,6 +125,7 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { jobs.POST("/logs", func(c *gin.Context) { var newLogs []buildlogs.Log if err := c.BindJSON(&newLogs); err != nil { + slog.Error("Failed to bind test results JSON", "error", err) return } @@ -137,6 +138,7 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { jobs.POST("/test-results", func(c *gin.Context) { var newResults ResultDTO if err := c.BindJSON(&newResults); err != nil { + slog.Error("Failed to bind test results JSON", "error", err) return } From d41c8f4bc5cfe11e3d86cceeb4d638810618e074 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 18:45:25 +0100 Subject: [PATCH 09/36] update .env.example, rabbit comments --- .env.example | 9 +++++ HadesArtemisAdapter/artemis_adapter.go | 54 ++++++++++++++++---------- HadesArtemisAdapter/go.mod | 2 +- HadesArtemisAdapter/main.go | 44 +++++++++++++++------ HadesLogManager/processor.go | 48 ++++++++++------------- 5 files changed, 97 insertions(+), 60 deletions(-) diff --git a/.env.example b/.env.example index 401bbb66..b5424666 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,15 @@ WATCH_NAMESPACE="hades" DELETE_ON_COMPLETE="true" MAX_PARALLELISM="100" +### HadesArtemisAdapter Options ### +ARTEMIS_ADAPTER_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) +ARTEMIS_BASE_URL=http://localhost:8080 +ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result +ARTEMIS_AUTH_TOKEN=superduperlongtestingsecrettoken + +### HadesLogManager Options ### +API_ENDPOINT="http://localhost:8082/adapter/logs" + ### DockerScheduler Options ### DOCKER_HOST="unix:///var/run/docker.sock" # Docker host to connect to (default: unix:///var/run/docker.sock) DOCKER_SCRIPT_EXECUTOR="/bin/bash -c" # Path to the script executor to use (default: /bin/sh) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 27a60f9b..eb1e9915 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -16,19 +16,21 @@ import ( type ArtemisAdapter struct { logs sync.Map // jobID (string) -> []buildlogs.LogEntry (only Step 2:execution logs) results sync.Map // jobID (string) -> results + jobLocks sync.Map // jobID (string) -> *sync.Mutex httpClient *http.Client + cfg AdapterConfig } -// ResultMetadata populated from environment variables +// Result DTOs used by Artemis type ResultMetadata struct { - JobName string `json:"jobName" env:"JOB_NAME"` - UUID string `json:"uuid" env:"UUID"` - AssignmentRepoBranchName string `json:"assignmentRepoBranchName" env:"ASSIGNMENT_REPO_BRANCH_NAME" envDefault:"main"` - IsBuildSuccessful bool `json:"isBuildSuccessful" env:"IS_BUILD_SUCCESSFUL"` - AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash" env:"ASSIGNMENT_REPO_COMMIT_HASH"` - TestsRepoCommitHash string `json:"testsRepoCommitHash" env:"TESTS_REPO_COMMIT_HASH"` - BuildCompletionTime string `json:"buildCompletionTime" env:"BUILD_COMPLETION_TIME"` - Passed int `json:"passed" env:"PASSED"` + JobName string `json:"jobName"` + UUID string `json:"uuid"` + AssignmentRepoBranchName string `json:"assignmentRepoBranchName"` + IsBuildSuccessful bool `json:"isBuildSuccessful"` + AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash"` + TestsRepoCommitHash string `json:"testsRepoCommitHash"` + BuildCompletionTime string `json:"buildCompletionTime"` + Passed int `json:"passed"` } type TestSuiteDTO struct { @@ -62,14 +64,12 @@ type ResultDTO struct { BuildLogs []buildlogs.LogEntry `json:"logs"` } -const artemisBaseURL = "http://localhost:8080" // Replace with actual Artemis URL -const newResultEndpoint = "api/programming/public/programming-exercises/new-result" const requestTimeout = 10 * time.Second -const artemisAuthToken = "superduperlongtestingsecrettoken" -func NewAdapter(ctx context.Context) *ArtemisAdapter { +func NewAdapter(ctx context.Context, cfg AdapterConfig) *ArtemisAdapter { aa := ArtemisAdapter{ httpClient: &http.Client{Timeout: requestTimeout}, + cfg: cfg, } return &aa @@ -77,9 +77,15 @@ func NewAdapter(ctx context.Context) *ArtemisAdapter { // StoreLogs only stores Step 2 (execution) logs for a job ID and checks if results are ready func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { - execution_logs := logs[1].Logs - aa.logs.Store(jobID, execution_logs) + executionLogs := []buildlogs.LogEntry{} + if len(logs) < 2 { + slog.Error("Execution logs missing", "jobID", jobID) + executionLogs = []buildlogs.LogEntry{} + } else { + executionLogs = logs[1].Logs + } + aa.logs.Store(jobID, executionLogs) return aa.checkAndSendIfReady(jobID) } @@ -93,6 +99,13 @@ func (aa *ArtemisAdapter) StoreResults(jobID string, results ResultDTO) error { // checkAndSendIfReady checks if both logs and results exist for a jobID // If both are present, combines them and sends to Artemis func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { + // Get or create mutex for this specific job + lockVal, _ := aa.jobLocks.LoadOrStore(jobID, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + + lock.Lock() + defer lock.Unlock() + logs, logsExist := aa.logs.Load(jobID) results, resultsExist := aa.results.Load(jobID) @@ -114,6 +127,7 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { // Clean up after successful send aa.logs.Delete(jobID) aa.results.Delete(jobID) + aa.jobLocks.Delete(jobID) } else { slog.Debug("Waiting for complete data", "jobID", jobID, "hasLogs", logsExist, "hasResults", resultsExist) } @@ -123,24 +137,24 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { // sendToArtemis sends the combined result DTO to the Artemis endpoint func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { - endpoint := fmt.Sprintf("%s/%s/%s", artemisBaseURL, newResultEndpoint, dto.JobName) + endpoint := fmt.Sprintf("%s/%s/%s", aa.cfg.ArtemisBaseURL, aa.cfg.NewResultEndpoint, dto.JobName) jsonData, err := json.Marshal(dto) if err != nil { - slog.Error("Error parsing logs to JSON", "error", err) + return fmt.Errorf("marshaling DTO to JSON: %w", err) } req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData)) if err != nil { - slog.Error("Error creating the request", "error", err) + return fmt.Errorf("creating request: %w", err) } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", artemisAuthToken) + req.Header.Set("Authorization", aa.cfg.ArtemisAuthToken) resp, err := aa.httpClient.Do(req) if err != nil { - slog.Error("Error sending the request", "error", err) + return fmt.Errorf("sending request to Artemis: %w", err) } defer resp.Body.Close() diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod index 11779d8d..633c9daf 100644 --- a/HadesArtemisAdapter/go.mod +++ b/HadesArtemisAdapter/go.mod @@ -1,4 +1,4 @@ -module Hades-ArtemisAdapter +module HadesArtemisAdapter go 1.25.3 diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index eb055f18..bcfd582f 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -16,16 +16,22 @@ import ( "github.com/ls1intum/hades/shared/utils" ) -type HadesArtemisAdapterConfig struct { - APIPort string `env:"API_PORT" envDefault:"8082"` +const ( + shutdownTimeout = 30 * time.Second +) + +type AdapterConfig struct { + APIPort string `env:"API_PORT" envDefault:"8082"` + ArtemisBaseURL string `env:"ARTEMIS_BASE_URL"` + NewResultEndpoint string `env:"ARTEMIS_NEW_RESULT_ENDPOINT"` + ArtemisAuthToken string `env:"ARTEMIS_AUTH_TOKEN"` } func main() { // Setup logging utils.SetupLogging() - // Load configuration - var cfg HadesArtemisAdapterConfig + var cfg AdapterConfig utils.LoadConfig(&cfg) // Run main application @@ -36,20 +42,20 @@ func main() { } // run contains the main application logic with proper error handling -func run(cfg HadesArtemisAdapterConfig) error { +func run(cfg AdapterConfig) error { // Create context for application lifecycle ctx, cancel := context.WithCancel(context.Background()) defer cancel() - aa := NewAdapter(ctx) + aa := NewAdapter(ctx, cfg) // Set up graceful shutdown return runWithGracefulShutdown(ctx, cancel, cfg, aa) } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesArtemisAdapterConfig, aa *ArtemisAdapter) error { +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg AdapterConfig, aa *ArtemisAdapter) error { var wg sync.WaitGroup errChan := make(chan error, 2) @@ -98,7 +104,7 @@ func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *htt cancel() // Shutdown API server with timeout - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout) defer shutdownCancel() if err := server.Shutdown(shutdownCtx); err != nil { @@ -129,9 +135,18 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { return } - aa.StoreLogs(newLogs[0].JobID, newLogs) - slog.Debug("Stored new logs", "uuid", newLogs[0].JobID) - c.IndentedJSON(http.StatusCreated, newLogs) + if len(newLogs) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "empty logs array"}) + return + } + + jobID := newLogs[0].JobID + if err := aa.StoreLogs(jobID, newLogs); err != nil { + slog.Error("Failed to store logs", "job_id", jobID, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to store logs"}) + return + } + c.JSON(http.StatusCreated, gin.H{"status": "stored", "job_id": jobID}) }) // post test results for specific job @@ -142,7 +157,12 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { return } - aa.StoreResults(newResults.UUID, newResults) + if err := aa.StoreResults(newResults.UUID, newResults); err != nil { + slog.Error("Failed to store/send results", "uuid", newResults.UUID, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to process results"}) + return + } + slog.Debug("Stored new test results", "uuid", newResults.UUID) c.IndentedJSON(http.StatusCreated, newResults) }) diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index f64d9035..ff93415d 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -14,7 +14,9 @@ import ( "github.com/ls1intum/hades/shared/buildstatus" ) -const APIendpoint = "http://localhost:8082/adapter/logs" +const ( + httpClientTimeout = 10 * time.Second +) // LogAggregator defines the interface for aggregating and managing job logs type LogAggregator interface { @@ -36,7 +38,7 @@ type NATSLogAggregator struct { logs sync.Map // jobID (string) -> logsVersion completed sync.Map // jobID (string) -> time.Time (completion time) status sync.Map // jobID (string) -> buildstatus.JobStatus - config AggregatorConfig + cfg AggregatorConfig } // wrapper stored in sync.Map: comparable (uint64 + pointer) @@ -47,9 +49,10 @@ type logsVersion struct { // AggregatorConfig defines the configuration parameters for log aggregation behavior. type AggregatorConfig struct { - BatchSize int `env:"LOG_BATCH_SIZE" envDefault:"100"` - Retention time.Duration `env:"LOG_RETENTION" envDefault:"1h"` - MaxJobLogs int `env:"MAX_JOB_LOGS" envDefault:"1000"` + BatchSize int `env:"LOG_BATCH_SIZE" envDefault:"100"` + Retention time.Duration `env:"LOG_RETENTION" envDefault:"1h"` + MaxJobLogs int `env:"MAX_JOB_LOGS" envDefault:"1000"` + APIendpoint string `env:"API_ENDPOINT"` } // NewLogAggregator creates a new NATS-based LogAggregator instance with the specified configuration. @@ -64,8 +67,8 @@ type AggregatorConfig struct { // - LogAggregator: A new instance ready to aggregate logs func NewLogAggregator(ctx context.Context, hlc *buildlogs.HadesLogConsumer, config AggregatorConfig) LogAggregator { la := &NATSLogAggregator{ - hlc: hlc, - config: config, + hlc: hlc, + cfg: config, } // Start background cleanup goroutine @@ -118,8 +121,8 @@ func (la *NATSLogAggregator) AddLog(log buildlogs.Log) { newLogs = append(newLogs, log) // Trim if needed - if len(newLogs) > la.config.MaxJobLogs { - trimStart := len(newLogs) - la.config.MaxJobLogs + if len(newLogs) > la.cfg.MaxJobLogs { + trimStart := len(newLogs) - la.cfg.MaxJobLogs newLogs = newLogs[trimStart:] slog.Debug("Trimmed old logs", "job_id", log.JobID, "trimmed_count", trimStart) } @@ -166,7 +169,7 @@ func (la *NATSLogAggregator) FlushJob(jobID string) error { // MarkJobCompleted marks a job as completed and schedules it for cleanup after retention period. func (la *NATSLogAggregator) MarkJobCompleted(jobID string) { la.completed.Store(jobID, time.Now()) - slog.Info("Marked job as completed", "job_id", jobID, "retention", la.config.Retention) + slog.Info("Marked job as completed", "job_id", jobID, "retention", la.cfg.Retention) } // cleanupCompletedJobs removes logs for jobs that have exceeded the retention period. @@ -178,7 +181,7 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { jobID := key.(string) completedAt := value.(time.Time) - if now.Sub(completedAt) >= la.config.Retention { + if now.Sub(completedAt) >= la.cfg.Retention { slog.Debug("Retention expired, flushing job", "job_id", jobID) if err := la.FlushJob(jobID); err != nil { @@ -199,36 +202,27 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { func (la *NATSLogAggregator) SendJobLogs(jobID string) error { logs := la.GetJobLogs(jobID) - // Convert the DTO to JSON jsonData, err := json.Marshal(logs) if err != nil { - slog.Error("Error parsing logs to JSON", "error", err) + return fmt.Errorf("marshaling logs to JSON: %w", err) } slog.Debug("Parsed logs to JSON", "json", string(jsonData)) - slog.Info("Sending job logs to the API", "job_id", jobID, "batch_count", len(logs)) - req, err := http.NewRequest("POST", APIendpoint, bytes.NewBuffer(jsonData)) + req, err := http.NewRequest("POST", la.cfg.APIendpoint, bytes.NewBuffer(jsonData)) if err != nil { - slog.Error("Error creating the request", "error", err) + return fmt.Errorf("creating HTTP request: %w", err) } - // Set headers req.Header.Set("Content-Type", "application/json") - - // Create a Client - client := &http.Client{} - - // Send the request via a client + client := &http.Client{Timeout: httpClientTimeout} resp, err := client.Do(req) + if err != nil { - slog.Error("Error sending the request", "error", err) + return fmt.Errorf("sending HTTP request: %w", err) } - slog.Info("Request sent", "status", resp.Status) - - // Close response body defer resp.Body.Close() - + slog.Info("Sent job logs to adapter", "job_id", jobID, "status", resp.Status) return nil } From 57b87ebdb0443fd36685c0128e384fea7f98c7fa Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 20:06:30 +0100 Subject: [PATCH 10/36] rabiit comments --- .env.example | 4 ++-- HadesArtemisAdapter/artemis_adapter.go | 6 +++++- HadesArtemisAdapter/go.mod | 8 ++++---- HadesArtemisAdapter/main.go | 2 +- HadesLogManager/processor.go | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index b5424666..22ee9dcc 100644 --- a/.env.example +++ b/.env.example @@ -25,13 +25,13 @@ DELETE_ON_COMPLETE="true" MAX_PARALLELISM="100" ### HadesArtemisAdapter Options ### -ARTEMIS_ADAPTER_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) +API_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) ARTEMIS_BASE_URL=http://localhost:8080 ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result ARTEMIS_AUTH_TOKEN=superduperlongtestingsecrettoken ### HadesLogManager Options ### -API_ENDPOINT="http://localhost:8082/adapter/logs" +API_ENDPOINT=http://localhost:8082/adapter/logs ### DockerScheduler Options ### DOCKER_HOST="unix:///var/run/docker.sock" # Docker host to connect to (default: unix:///var/run/docker.sock) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index eb1e9915..cbfac5de 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -79,7 +79,7 @@ func NewAdapter(ctx context.Context, cfg AdapterConfig) *ArtemisAdapter { func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { executionLogs := []buildlogs.LogEntry{} if len(logs) < 2 { - slog.Error("Execution logs missing", "jobID", jobID) + slog.Warn("Execution logs missing", "jobID", jobID) executionLogs = []buildlogs.LogEntry{} } else { executionLogs = logs[1].Logs @@ -158,6 +158,10 @@ func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { } defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("adapter returned HTTP %d for job %s", resp.StatusCode, dto.UUID) + } + slog.Info("Request sent", "status", resp.Status) return nil diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod index 633c9daf..d338a782 100644 --- a/HadesArtemisAdapter/go.mod +++ b/HadesArtemisAdapter/go.mod @@ -26,14 +26,14 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/nats-io/nats.go v1.48.0 // indirect github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.57.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect go.uber.org/mock v0.5.0 // indirect diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index bcfd582f..a43dd8f5 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -131,7 +131,7 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { jobs.POST("/logs", func(c *gin.Context) { var newLogs []buildlogs.Log if err := c.BindJSON(&newLogs); err != nil { - slog.Error("Failed to bind test results JSON", "error", err) + slog.Error("Failed to bind logs JSON", "error", err) return } diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index ff93415d..c2fdc33e 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -206,7 +206,7 @@ func (la *NATSLogAggregator) SendJobLogs(jobID string) error { if err != nil { return fmt.Errorf("marshaling logs to JSON: %w", err) } - slog.Debug("Parsed logs to JSON", "json", string(jsonData)) + slog.Debug("Marshaled logs to JSON", "job_id", jobID) req, err := http.NewRequest("POST", la.cfg.APIendpoint, bytes.NewBuffer(jsonData)) if err != nil { From ed8e2f17c95832ce5954825f391fd811425c1ab0 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 20:20:33 +0100 Subject: [PATCH 11/36] go mod tidy, add docstrings --- HadesArtemisAdapter/artemis_adapter.go | 52 ++++++++++++++++++++------ HadesArtemisAdapter/go.mod | 4 -- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index cbfac5de..54ed1e91 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -13,6 +13,9 @@ import ( "github.com/ls1intum/hades/shared/buildlogs" ) +// ArtemisAdapter handles the collection and synchronization of build logs and test results +// before sending them to Artemis. It uses sync.Map for thread-safe storage and per-job +// mutexes to ensure exactly-once delivery of combined results. type ArtemisAdapter struct { logs sync.Map // jobID (string) -> []buildlogs.LogEntry (only Step 2:execution logs) results sync.Map // jobID (string) -> results @@ -21,7 +24,16 @@ type ArtemisAdapter struct { cfg AdapterConfig } -// Result DTOs used by Artemis +// ResultDTO is the complete data transfer object sent to Artemis. +// This struct is adapted from Artemis. +// It combines metadata, test results, and build logs into a single payload. +type ResultDTO struct { + ResultMetadata + Results []TestSuiteDTO `json:"results"` + BuildLogs []buildlogs.LogEntry `json:"logs"` +} + +// ResultMetadata contains the metadata about a build job's execution and outcome. type ResultMetadata struct { JobName string `json:"jobName"` UUID string `json:"uuid"` @@ -33,6 +45,8 @@ type ResultMetadata struct { Passed int `json:"passed"` } +// TestSuiteDTO represents a collection of test cases that were executed together. +// It aggregates statistics about the test run including total tests, failures, and execution time. type TestSuiteDTO struct { Name string `json:"name"` Time float64 `json:"time"` @@ -43,6 +57,8 @@ type TestSuiteDTO struct { TestCases []TestCaseDTO `json:"testCases"` } +// TestCaseDTO represents a single test case execution result. +// It contains either failure/error details for failed tests or success information for passed tests. type TestCaseDTO struct { Name string `json:"name"` Classname string `json:"classname"` @@ -52,18 +68,14 @@ type TestCaseDTO struct { Successes []TestCaseDetailMessageDTO `json:"successInfos"` // empty for failing tests } +// TestCaseDetailMessageDTO provides detailed information about a test case result, +// including error messages, stack traces, and result types. type TestCaseDetailMessageDTO struct { Message string `json:"message"` Type string `json:"type"` MessageWithStackTrace string `json:"messageWithStackTrace"` } -type ResultDTO struct { - ResultMetadata - Results []TestSuiteDTO `json:"results"` - BuildLogs []buildlogs.LogEntry `json:"logs"` -} - const requestTimeout = 10 * time.Second func NewAdapter(ctx context.Context, cfg AdapterConfig) *ArtemisAdapter { @@ -75,7 +87,11 @@ func NewAdapter(ctx context.Context, cfg AdapterConfig) *ArtemisAdapter { return &aa } -// StoreLogs only stores Step 2 (execution) logs for a job ID and checks if results are ready +// StoreLogs stores execution logs (Step 2) for the given job ID and triggers a check +// to see if results are also ready. If both logs and results are available, they are +// combined and sent to Artemis. +// +// If logs array has fewer than 2 elements, an empty log array is stored and a warning is logged. func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { executionLogs := []buildlogs.LogEntry{} if len(logs) < 2 { @@ -89,15 +105,22 @@ func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { return aa.checkAndSendIfReady(jobID) } -// StoreResults stores results for a job ID and checks if logs are ready +// StoreResults stores test results for the given job ID and triggers a check +// to see if logs are also ready. If both logs and results are available, they are +// combined and sent to Artemis. func (aa *ArtemisAdapter) StoreResults(jobID string, results ResultDTO) error { aa.results.Store(jobID, results) slog.Debug("Stored results", "jobID", jobID, "results", results.Results) return aa.checkAndSendIfReady(jobID) } -// checkAndSendIfReady checks if both logs and results exist for a jobID -// If both are present, combines them and sends to Artemis +// checkAndSendIfReady checks if both logs and results exist for the given jobID. +// If both are present, it combines them into a single ResultDTO and sends it to Artemis. +// After a successful send, all data for the job (logs, results, and mutex) is cleaned up. +// +// This method uses a per-job mutex to ensure that even if StoreLogs and StoreResults +// are called concurrently for the same job, the check-combine-send-delete sequence +// is atomic, preventing duplicate sends. func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { // Get or create mutex for this specific job lockVal, _ := aa.jobLocks.LoadOrStore(jobID, &sync.Mutex{}) @@ -135,7 +158,12 @@ func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { return nil } -// sendToArtemis sends the combined result DTO to the Artemis endpoint +// sendToArtemis sends the combined ResultDTO to the configured Artemis endpoint. +// It constructs the full endpoint URL using the job name, marshals the DTO to JSON, +// and sends an authenticated POST request. +// +// Returns an error if marshaling fails, request creation fails, the HTTP request fails, +// or if Artemis responds with a status code >= 400. func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { endpoint := fmt.Sprintf("%s/%s/%s", aa.cfg.ArtemisBaseURL, aa.cfg.NewResultEndpoint, dto.JobName) diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod index d338a782..eacd888b 100644 --- a/HadesArtemisAdapter/go.mod +++ b/HadesArtemisAdapter/go.mod @@ -36,14 +36,10 @@ require ( github.com/quic-go/quic-go v0.57.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect - go.uber.org/mock v0.5.0 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect - golang.org/x/tools v0.38.0 // indirect google.golang.org/protobuf v1.36.9 // indirect ) From e999aeac84062207c37e1e1d780d167e194c5197 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 20:21:12 +0100 Subject: [PATCH 12/36] add comments --- HadesArtemisAdapter/artemis_adapter.go | 4 ++-- HadesArtemisAdapter/go.sum | 26 ++++++++++++-------------- HadesArtemisAdapter/main.go | 1 - 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 54ed1e91..218d7f72 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -35,8 +35,8 @@ type ResultDTO struct { // ResultMetadata contains the metadata about a build job's execution and outcome. type ResultMetadata struct { - JobName string `json:"jobName"` - UUID string `json:"uuid"` + JobName string `json:"jobName"` // contains participationId + UUID string `json:"uuid"` // JobId AssignmentRepoBranchName string `json:"assignmentRepoBranchName"` IsBuildSuccessful bool `json:"isBuildSuccessful"` AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash"` diff --git a/HadesArtemisAdapter/go.sum b/HadesArtemisAdapter/go.sum index 6436c0b7..8d357048 100644 --- a/HadesArtemisAdapter/go.sum +++ b/HadesArtemisAdapter/go.sum @@ -47,10 +47,12 @@ github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b h1:qr4I+vH9q github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b/go.mod h1:dtZoDaLxyhyNlMNGZ/zHJ2pEwvVcv62uPZD4f4y1jKY= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= @@ -62,10 +64,10 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= -github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= -github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE= +github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -79,25 +81,21 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index a43dd8f5..d483db20 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -80,7 +80,6 @@ func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg } }() - // Wait for shutdown signal or error return waitForShutdown(ctx, cancel, server, &wg, errChan) } From 3c3323c5258470389ce66c116c58c7fc10c006c4 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 20:47:51 +0100 Subject: [PATCH 13/36] rabbit --- .env.example | 5 +++-- HadesAPI/main.go | 2 +- HadesArtemisAdapter/main.go | 2 +- HadesLogManager/main.go | 2 +- Readme.md | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 22ee9dcc..4a8ee1e3 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,7 @@ NATS_PASSWORD= # NATS password (optional) NATS_TLS_ENABLED=false # Enable TLS for NATS connection (default: false) ### HadesAPI Options ### -API_PORT=8080 # Port for the API to listen on (default: 8080) +HADESAPI_API_PORT=8080 # Port for the API to listen on (default: 8080) AUTH_KEY="" # Key used for Basic Auth with the API (default: "" for no auth) PROMETHEUS_ADDRESS="" # Address for Prometheus metrics to be exposed (default: "" for no metrics) RETENTION_IN_MIN=30 # Time in minutes to keep completed jobs for monitoring (default: 30) @@ -25,12 +25,13 @@ DELETE_ON_COMPLETE="true" MAX_PARALLELISM="100" ### HadesArtemisAdapter Options ### -API_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) +HADESADAPTER_API_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) ARTEMIS_BASE_URL=http://localhost:8080 ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result ARTEMIS_AUTH_TOKEN=superduperlongtestingsecrettoken ### HadesLogManager Options ### +HADESLOGMANAGER_API_PORT=8081 # Port for the Log Manager to listen on (default: 8081) API_ENDPOINT=http://localhost:8082/adapter/logs ### DockerScheduler Options ### diff --git a/HadesAPI/main.go b/HadesAPI/main.go index d507858c..bab5abca 100644 --- a/HadesAPI/main.go +++ b/HadesAPI/main.go @@ -10,7 +10,7 @@ import ( ) type HadesAPIConfig struct { - APIPort uint `env:"API_PORT,notEmpty" envDefault:"8080"` + APIPort uint `env:"HADESAPI_API_PORT,notEmpty" envDefault:"8080"` NatsConfig utils.NatsConfig AuthKey string `env:"AUTH_KEY"` PrometheusAddress string `env:"PROMETHEUS_ADDRESS" envDefault:""` diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index d483db20..ea494771 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -21,7 +21,7 @@ const ( ) type AdapterConfig struct { - APIPort string `env:"API_PORT" envDefault:"8082"` + APIPort string `env:"HADESADAPTER_API_PORT" envDefault:"8082"` ArtemisBaseURL string `env:"ARTEMIS_BASE_URL"` NewResultEndpoint string `env:"ARTEMIS_NEW_RESULT_ENDPOINT"` ArtemisAuthToken string `env:"ARTEMIS_AUTH_TOKEN"` diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 3d3c4516..b7fa6912 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -25,7 +25,7 @@ const ( // HadesLogManagerConfig holds the configuration for the log manager type HadesLogManagerConfig struct { NatsConfig utils.NatsConfig - APIPort string `env:"API_PORT" envDefault:"8081"` + APIPort string `env:"HADESLOGMANAGER_API_PORT" envDefault:"8081"` } func main() { diff --git a/Readme.md b/Readme.md index eeb4ffe5..31ed216d 100644 --- a/Readme.md +++ b/Readme.md @@ -185,7 +185,7 @@ Hades can be configured through environment variables or a `.env` file: |----------|-------------|---------| | `HADES_EXECUTOR` | Execution platform: `docker` or `kubernetes` | `docker` | | `CONCURRENCY` | Number of jobs to process concurrently | `1` | -| `API_PORT` | Port for the Hades API | `8080` | +| `HADESAPI_API_PORT` | Port for the Hades API | `8080` | ## Deployment From 614a46c55b63e14a94af97f6b4aa9963e09355c5 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 16 Feb 2026 20:48:07 +0100 Subject: [PATCH 14/36] rabbit --- HadesLogManager/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index c2fdc33e..86ff390b 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -79,7 +79,7 @@ func NewLogAggregator(ctx context.Context, hlc *buildlogs.HadesLogConsumer, conf // cleanupLoop runs periodic cleanup of completed jobs func (la *NATSLogAggregator) cleanupLoop(ctx context.Context) { - ticker := time.NewTicker(time.Minute) + ticker := time.NewTicker(time.Hour) defer ticker.Stop() for { From 105983a600979f1f8dad2b3d772b4818ccd595ce Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Tue, 17 Feb 2026 00:04:07 +0100 Subject: [PATCH 15/36] add Dockerfiles, update docker compose files, update build.yml --- .env.example | 2 +- .github/workflows/build.yml | 4 ++++ HadesArtemisAdapter/Dockerfile | 30 ++++++++++++++++++++++++++++++ HadesLogManager/Dockerfile | 30 ++++++++++++++++++++++++++++++ Readme.md | 2 +- compose.yml | 34 ++++++++++++++++++++++++++++++++++ docker-compose.deploy.yml | 26 ++++++++++++++++++++++++++ docker-compose.dev.yml | 18 +++++++++++++++++- docker-compose.k8s.yml | 1 - 9 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 HadesArtemisAdapter/Dockerfile create mode 100644 HadesLogManager/Dockerfile diff --git a/.env.example b/.env.example index 4a8ee1e3..5956d891 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ MAX_PARALLELISM="100" ### HadesArtemisAdapter Options ### HADESADAPTER_API_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) ARTEMIS_BASE_URL=http://localhost:8080 -ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result +ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result # this is the base. it is used with /{participationId} ARTEMIS_AUTH_TOKEN=superduperlongtestingsecrettoken ### HadesLogManager Options ### diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3fa1dbd6..aaaee5d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,10 @@ jobs: - dockerfile: ./HadesScheduler/HadesOperator/Dockerfile image: ls1intum/hades/hades-operator context: ./HadesScheduler/HadesOperator + - dockerfile: ./HadesScheduler/HadesLogManager/Dockerfile + image: ls1intum/hades/hades-log-manager + - dockerfile: ./HadesArtemisAdapter/Dockerfile + image: ls1intum/hades/hades-artemis-adapter uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.1.1 with: image-name: ${{ matrix.image }} diff --git a/HadesArtemisAdapter/Dockerfile b/HadesArtemisAdapter/Dockerfile new file mode 100644 index 00000000..f16b2ace --- /dev/null +++ b/HadesArtemisAdapter/Dockerfile @@ -0,0 +1,30 @@ +# Use an official Go runtime as a parent image +FROM golang:1.25-alpine AS builder + +# Set the working directory inside the container +WORKDIR /app + +# Copy go.mod and go.sum files +COPY ./HadesArtemisAdapter/go.mod ./HadesArtemisAdapter/go.sum ./HadesArtemisAdapter/ +COPY ./shared/go.mod ./shared/go.sum ./shared/ +RUN cd HadesArtemisAdapter && go mod download + +# Copy the Go application source code into the container +COPY ./HadesArtemisAdapter ./HadesArtemisAdapter +COPY ./shared ./shared + +# Build the Go application +WORKDIR /app/HadesArtemisAdapter +RUN CGO_ENABLED=0 go build -o hades-artemis-adapter . + +# Start a new stage for the minimal runtime container +FROM gcr.io/distroless/static-debian12 + +# Set the working directory inside the minimal runtime container +WORKDIR /app + +# Copy the built binary from the builder container into the minimal runtime container +COPY --from=builder /app/HadesArtemisAdapter/hades-artemis-adapter . + +# Run your Go application +CMD ["/app/hades-artemis-adapter"] \ No newline at end of file diff --git a/HadesLogManager/Dockerfile b/HadesLogManager/Dockerfile new file mode 100644 index 00000000..4fb3f6d2 --- /dev/null +++ b/HadesLogManager/Dockerfile @@ -0,0 +1,30 @@ +# Use an official Go runtime as a parent image +FROM golang:1.25-alpine AS builder + +# Set the working directory inside the container +WORKDIR /app + +# Copy go.mod and go.sum files +COPY ./HadesLogManager/go.mod ./HadesLogManager/go.sum ./HadesLogManager/ +COPY ./shared/go.mod ./shared/go.sum ./shared/ +RUN cd HadesLogManager && go mod download + +# Copy the Go application source code into the container +COPY ./HadesLogManager ./HadesLogManager +COPY ./shared ./shared + +# Build the Go application +WORKDIR /app/HadesLogManager +RUN CGO_ENABLED=0 go build -o hades-log-manager . + +# Start a new stage for the minimal runtime container +FROM gcr.io/distroless/static-debian12 + +# Set the working directory inside the minimal runtime container +WORKDIR /app + +# Copy the built binary from the builder container into the minimal runtime container +COPY --from=builder /app/HadesLogManager/hades-log-manager . + +# Run your Go application +CMD ["/app/hades-log-manager"] \ No newline at end of file diff --git a/Readme.md b/Readme.md index 31ed216d..c64ec37a 100644 --- a/Readme.md +++ b/Readme.md @@ -179,7 +179,7 @@ For more complex workflows, you can define multi-step jobs where each step runs ## Configuration Options -Hades can be configured through environment variables or a `.env` file: +Hades can be configured through environment variables or through a `.env` file: | Variable | Description | Default | |----------|-------------|---------| diff --git a/compose.yml b/compose.yml index 711acc7b..5250d0ea 100644 --- a/compose.yml +++ b/compose.yml @@ -37,6 +37,40 @@ services: environment: - NATS_URL=nats://nats:4222 - HADES_EXECUTOR=docker + + hadesLogManager: + image: ghcr.io/ls1intum/hades/hades-log-manager + build: + context: . + dockerfile: ./HadesLogManager/Dockerfile + ports: + - "8083:8081" # Map to 8083 to avoid conflicts + networks: + - hades + depends_on: + hadesAPI: + condition: service_started + environment: + - HADESLOGMANAGER_API_PORT=8081 + - API_ENDPOINT=${API_ENDPOINT} + + hadesArtemisAdapter: + image: ghcr.io/ls1intum/hades/hades-artemis-adapter + build: + context: . + dockerfile: ./HadesArtemisAdapter/Dockerfile + ports: + - "8082:8082" + networks: + - hades + depends_on: + hadesAPI: + condition: service_started + environment: + - HADESADAPTER_API_PORT=8082 + - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} + - ARTEMIS_NEW_RESULT_ENDPOINT=${ARTEMIS_NEW_RESULT_ENDPOINT} + - ARTEMIS_AUTH_TOKEN=${ARTEMIS_AUTH_TOKEN} nats: image: nats:2.11.4 diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index 3e6c807f..78d52292 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -35,6 +35,32 @@ services: networks: - hades + hadesLogManager: + ports: [] + expose: + - "8081" + networks: + - hades + labels: + - "traefik.enable=true" + - "traefik.http.routers.hadeslogmanager.rule=Host(`${HADES_LOGMANAGER_HOST:-logs.${HADES_API_HOST}}`)" + - "traefik.http.routers.hadeslogmanager.entrypoints=web,websecure" + - "traefik.http.routers.hadeslogmanager.tls.certresolver=letsencrypt" + - "traefik.http.services.hadeslogmanager.loadbalancer.server.port=8081" + + hadesArtemisAdapter: + ports: [] + expose: + - "8082" + networks: + - hades + labels: + - "traefik.enable=true" + - "traefik.http.routers.hadesadapter.rule=Host(`${HADES_ADAPTER_HOST:-adapter.${HADES_API_HOST}}`)" + - "traefik.http.routers.hadesadapter.entrypoints=web,websecure" + - "traefik.http.routers.hadesadapter.tls.certresolver=letsencrypt" + - "traefik.http.services.hadesadapter.loadbalancer.server.port=8082" + nats: ports: [] networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 69830f17..43a0e6f6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -15,4 +15,20 @@ services: - DOCKER_CONTAINER_AUTOREMOVE=false - DEBUG=true volumes: - - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file + - /var/run/docker.sock:/var/run/docker.sock + hadesLogManager: + ports: + - "8083:8081" # Expose for local access + environment: + - DEBUG=true + - HADESLOGMANAGER_API_PORT=8081 + - API_ENDPOINT=http://hadesArtemisAdapter:8082/adapter/logs + + hadesArtemisAdapter: + ports: + - "8082:8082" # Expose for local access + environment: + - DEBUG=true + - HADESADAPTER_API_PORT=8082 + # Override these in your .env file for local dev + - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} \ No newline at end of file diff --git a/docker-compose.k8s.yml b/docker-compose.k8s.yml index 43f00bef..d2378783 100644 --- a/docker-compose.k8s.yml +++ b/docker-compose.k8s.yml @@ -1,7 +1,6 @@ version: '3.7' services: - hadesScheduler: environment: - NATS_URL=nats://nats:4222 From d166bf8457be4e1d21a4502848987231b9fc8f1e Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Tue, 17 Feb 2026 00:07:08 +0100 Subject: [PATCH 16/36] fix build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aaaee5d8..9c34e26d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: - dockerfile: ./HadesScheduler/HadesOperator/Dockerfile image: ls1intum/hades/hades-operator context: ./HadesScheduler/HadesOperator - - dockerfile: ./HadesScheduler/HadesLogManager/Dockerfile + - dockerfile: ./HadesLogManager/Dockerfile image: ls1intum/hades/hades-log-manager - dockerfile: ./HadesArtemisAdapter/Dockerfile image: ls1intum/hades/hades-artemis-adapter From 655d3e8c3b82c12979db0f7c96385b0239ac46b6 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Tue, 17 Feb 2026 00:12:55 +0100 Subject: [PATCH 17/36] fix compose files --- compose.yml | 7 +++---- docker-compose.dev.yml | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compose.yml b/compose.yml index 5250d0ea..d0455bd6 100644 --- a/compose.yml +++ b/compose.yml @@ -38,7 +38,7 @@ services: - NATS_URL=nats://nats:4222 - HADES_EXECUTOR=docker - hadesLogManager: + hadesLogManager: image: ghcr.io/ls1intum/hades/hades-log-manager build: context: . @@ -48,6 +48,8 @@ services: networks: - hades depends_on: + nats: + condition: service_healthy hadesAPI: condition: service_started environment: @@ -63,9 +65,6 @@ services: - "8082:8082" networks: - hades - depends_on: - hadesAPI: - condition: service_started environment: - HADESADAPTER_API_PORT=8082 - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 43a0e6f6..8c3a62b5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -31,4 +31,6 @@ services: - DEBUG=true - HADESADAPTER_API_PORT=8082 # Override these in your .env file for local dev - - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} \ No newline at end of file + - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} + - ARTEMIS_NEW_RESULT_ENDPOINT=${ARTEMIS_NEW_RESULT_ENDPOINT} + - ARTEMIS_AUTH_TOKEN=${ARTEMIS_AUTH_TOKEN} From b198e380c92c9910258f843b84125d5d44f16e73 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Tue, 17 Feb 2026 00:19:48 +0100 Subject: [PATCH 18/36] go mod tidy --- HadesLogManager/go.mod | 2 +- HadesLogManager/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HadesLogManager/go.mod b/HadesLogManager/go.mod index 6cfee337..dc67f254 100644 --- a/HadesLogManager/go.mod +++ b/HadesLogManager/go.mod @@ -40,7 +40,7 @@ require ( require ( github.com/klauspost/compress v1.18.0 // indirect - github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e + github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398 github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.45.0 // indirect diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index af77b292..fc5a8500 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -43,8 +43,8 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e h1:7wC0iDQ4m/gcLn2gYdxmKAMxIXxasKY5PJ0DJVAnLDM= -github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e/go.mod h1:1uOGnAwXXd3UTaO+jOEcbTv21KXFMMwE70gD1057miM= +github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398 h1:hAVvbiD1m/NMZywecQdDR2pJ6sx2f9Nza5NgKgVxz7A= +github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398/go.mod h1:dtZoDaLxyhyNlMNGZ/zHJ2pEwvVcv62uPZD4f4y1jKY= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From da9d373c5eb0b15288b04726142579aa9d8337ea Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 23 Feb 2026 17:10:36 +0100 Subject: [PATCH 19/36] aggregate and send logs when build fails too --- HadesArtemisAdapter/artemis_adapter.go | 2 +- HadesLogManager/log_subscriber.go | 16 +++- HadesScheduler/docker/job.go | 12 ++- HadesScheduler/docker/step.go | 32 +++---- ...Job (Test Fail) with Result Parse Step.bru | 83 +++++++++++++++++++ ... (Test Succeed) with Result Parse Step.bru | 2 +- 6 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 docs/api/Create Build Job (Test Fail) with Result Parse Step.bru diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go index 218d7f72..e966cd12 100644 --- a/HadesArtemisAdapter/artemis_adapter.go +++ b/HadesArtemisAdapter/artemis_adapter.go @@ -187,7 +187,7 @@ func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { defer resp.Body.Close() if resp.StatusCode >= 400 { - return fmt.Errorf("adapter returned HTTP %d for job %s", resp.StatusCode, dto.UUID) + return fmt.Errorf("Artemis returned HTTP %d for job %s", resp.StatusCode, dto.UUID) } slog.Info("Request sent", "status", resp.Status) diff --git a/HadesLogManager/log_subscriber.go b/HadesLogManager/log_subscriber.go index 36f0752c..17426c57 100644 --- a/HadesLogManager/log_subscriber.go +++ b/HadesLogManager/log_subscriber.go @@ -37,6 +37,7 @@ type DynamicLogManager struct { type watcherState struct { ctx context.Context cancel context.CancelFunc + wg *sync.WaitGroup } // NewDynamicLogManager creates a new DynamicLogManager instance with the provided dependencies. @@ -169,6 +170,8 @@ func (dlm *DynamicLogManager) cleanupSubscriptions(subs []*nats.Subscription) { func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID string) { // Create new context for this job outside the lock jobCtx, cancel := context.WithCancel(ctx) + wg := &sync.WaitGroup{} + wg.Add(1) // Minimize critical section - only lock for map operations dlm.mu.Lock() @@ -176,16 +179,19 @@ func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID st dlm.watchers[jobID] = watcherState{ ctx: jobCtx, cancel: cancel, + wg: wg, } dlm.mu.Unlock() // Cancel old watcher outside the lock to avoid potential deadlock if exists { oldWatcher.cancel() + oldWatcher.wg.Wait() } // Start watching logs for this job go func() { + defer wg.Done() defer func() { // Use a more efficient cleanup check dlm.mu.Lock() @@ -223,12 +229,16 @@ func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID st // - jobID: Unique identifier of the job to stop watching logs for func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) { dlm.mu.Lock() - defer dlm.mu.Unlock() - - if watcher, exists := dlm.watchers[jobID]; exists { + watcher, exists := dlm.watchers[jobID] + if exists { delete(dlm.watchers, jobID) + } + dlm.mu.Unlock() + + if exists { slog.Info("Stopping log watch", "job_id", jobID) watcher.cancel() + watcher.wg.Wait() // Wait outside the lock dlm.logAggregator.MarkJobCompleted(jobID) dlm.logAggregator.SendJobLogs(jobID) diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 45cb9631..84d66476 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -2,6 +2,7 @@ package docker import ( "context" + "fmt" "log/slog" "maps" @@ -10,6 +11,8 @@ import ( "github.com/ls1intum/hades/shared/payload" ) +const ContinueOnError = "CONTINUE_ON_ERROR" + type Job struct { cli *client.Client logger *slog.Logger @@ -21,6 +24,8 @@ type Job struct { type jobIDContextKey string func (d Job) execute(ctx context.Context) error { + stepErr := error(nil) + for _, step := range d.Steps { d.logger.Info("Executing step", slog.Any("step", step)) @@ -43,8 +48,13 @@ func (d Job) execute(ctx context.Context) error { err := dockerStep.execute(stepCtx) if err != nil { d.logger.Error("Failed to execute step", slog.Any("error", err)) + if step.Metadata[ContinueOnError] == "true" { + d.logger.Info("Next step should be executed despite error due to ContinueOnError setting", slog.Any("step", step)) + stepErr = fmt.Errorf("Step %v failed with error but ContinueOnError is set: %w", step.ID, err) + continue + } return err } } - return nil + return stepErr } diff --git a/HadesScheduler/docker/step.go b/HadesScheduler/docker/step.go index 9c530990..af5c8aa1 100644 --- a/HadesScheduler/docker/step.go +++ b/HadesScheduler/docker/step.go @@ -87,6 +87,12 @@ func (s Step) execute(ctx context.Context) error { return err } + defer func() { + if err := removeContainer(ctx, s.cli, resp.ID); err != nil { + s.logger.Error("Failed to cleanup container", slog.Any("error", err), slog.Any("container_id", resp.ID)) + } + }() + // Start the container err = s.cli.ContainerStart(ctx, resp.ID, container.StartOptions{}) if err != nil { @@ -95,7 +101,7 @@ func (s Step) execute(ctx context.Context) error { } // Wait for the container to finish - statusCh, errCh := s.cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + statusCh, errCh := s.cli.ContainerWait(ctx, resp.ID, container.WaitCondition(container.WaitConditionNotRunning)) select { case err := <-errCh: if err != nil { @@ -103,6 +109,15 @@ func (s Step) execute(ctx context.Context) error { return err } case status := <-statusCh: + // Write the container logs to NATS before checking status + // Logs should be written even if the container fails to start or crashes during execution + if err := processContainerLogs(ctx, s.cli, s.publisher, resp.ID, jobId); err != nil { + s.logger.Error("Failed to write container logs to NATS", slog.Any("error", err), slog.Any("container_id", resp.ID)) + return err + } else { + s.logger.Debug("Container logs written to NATS", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) + } + if status.StatusCode != 0 { s.logger.Error("Container exited with status", slog.Any("status", status.StatusCode), slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) return fmt.Errorf("container exited with status %d", status.StatusCode) @@ -110,20 +125,5 @@ func (s Step) execute(ctx context.Context) error { } s.logger.Debug("Container completed", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) - - // Write the container logs to NATS - err = processContainerLogs(ctx, s.cli, s.publisher, resp.ID, jobId) - - if err != nil { - s.logger.Error("Failed to write container logs to NATS", slog.Any("error", err), slog.Any("container_id", resp.ID)) - return err - } else { - s.logger.Debug("Container logs written to NATS", slog.Any("container_id", resp.ID), slog.Any("image", s.Image)) - } - - if err := removeContainer(ctx, s.cli, resp.ID); err != nil { - s.logger.Error("Failed to cleanup container", slog.Any("error", err), slog.Any("container_id", resp.ID)) - } - return nil } diff --git a/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru new file mode 100644 index 00000000..8f29436a --- /dev/null +++ b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru @@ -0,0 +1,83 @@ +meta { + name: Create Build Job (Test Fail) with Result Parse Step + type: http + seq: 6 +} + +post { + url: http://{{hostname}}/build + body: json + auth: none +} + +body:json { + { + "name": "Example Job", + "volumes": [ + { + "name": "shared", + "emptyDir": {} + } + ], + "metadata": { + "GLOBAL": "test" + }, + "timestamp": "2021-01-01T00:00:00.000Z", + "priority": 3, // optional, default 3 + "steps": [ + { + "id": 1, // mandatory to declare the order of execution + "name": "Clone", + "image": "ghcr.io/ls1intum/hades/hades-clone-container:latest", // mandatory + "volumeMounts": [ + { "name": "shared", "mountPath": "/shared" } + ], + "workingDir": "/shared", + "metadata": { + "REPOSITORY_DIR": "/shared", + "HADES_TEST_USERNAME": "{{user}}", + "HADES_TEST_PASSWORD": "{{password}}", + "HADES_TEST_URL": "{{test_repo}}", + "HADES_TEST_PATH": "./example", + "HADES_TEST_ORDER": "1", + "HADES_ASSIGNMENT_USERNAME": "{{user}}", + "HADES_ASSIGNMENT_PASSWORD": "{{password}}", + "HADES_ASSIGNMENT_URL": "{{assignment_repo}}", + "HADES_ASSIGNMENT_PATH": "./example/assignment", + "HADES_ASSIGNMENT_ORDER": "2" + } + }, + { + "id": 2, // mandatory to declare the order of execution + "name": "Execute", + "image": "ls1tum/artemis-maven-template:java17-18", // mandatory + "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test", + "metadata": { + "CONTINUE_ON_ERROR": "true" + } + }, + { + "id": 3, + "name": "Parse Results", + "image": "ghcr.io/ls1intum/hades/junit-result-parser:latest", + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], + "metadata": { + "API_ENDPOINT": "http://host.docker.internal:8083/adapter/test-results", + "INGEST_DIR": "/shared/example/build/test-results/test" + } + } + ] + } +} + +vars:pre-request { + user: + password: + test_repo: https://github.com/Mtze/Artemis-Java-Test.git + assignment_repo: https://github.com/Mtze/Artemis-Java-Template.git +} diff --git a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru index 292958b0..5250ff68 100644 --- a/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru +++ b/docs/api/Create Build Job (Test Succeed) with Result Parse Step.bru @@ -73,7 +73,7 @@ body:json { } ], "metadata": { - "API_ENDPOINT": "http://host.docker.internal:8082/adapter/test-results", + "API_ENDPOINT": "http://host.docker.internal:8083/adapter/test-results", "INGEST_DIR": "/shared/example/build/test-results/test" } } From 13ca7ecccd6e4024df5352f6d866d36a0df7e196 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 23 Feb 2026 17:31:28 +0100 Subject: [PATCH 20/36] update docstrings --- HadesArtemisAdapter/main.go | 23 ++++++++++++++++++----- HadesLogManager/main.go | 7 +++++++ HadesLogManager/processor.go | 7 +++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go index ea494771..bed180c6 100644 --- a/HadesArtemisAdapter/main.go +++ b/HadesArtemisAdapter/main.go @@ -20,6 +20,7 @@ const ( shutdownTimeout = 30 * time.Second ) +// AdapterConfig holds the configuration for the Artemis adapter service. type AdapterConfig struct { APIPort string `env:"HADESADAPTER_API_PORT" envDefault:"8082"` ArtemisBaseURL string `env:"ARTEMIS_BASE_URL"` @@ -27,6 +28,8 @@ type AdapterConfig struct { ArtemisAuthToken string `env:"ARTEMIS_AUTH_TOKEN"` } +// main is the application entry point. It initialises logging, loads configuration from environment variables, +// and delegates to run for the core application logic. func main() { // Setup logging utils.SetupLogging() @@ -41,7 +44,8 @@ func main() { } } -// run contains the main application logic with proper error handling +// run initialises the application context, creates the ArtemisAdapter, and starts the server with graceful shutdown support. +// It returns any fatal error encountered during the lifetime of the application. func run(cfg AdapterConfig) error { // Create context for application lifecycle @@ -54,7 +58,8 @@ func run(cfg AdapterConfig) error { return runWithGracefulShutdown(ctx, cancel, cfg, aa) } -// runWithGracefulShutdown starts services and handles graceful shutdown +// runWithGracefulShutdown configures and starts the HTTP API server in a background goroutine, +// then delegates to waitForShutdown to block until either an OS signal or a fatal server error triggers a graceful teardown. func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg AdapterConfig, aa *ArtemisAdapter) error { var wg sync.WaitGroup errChan := make(chan error, 2) @@ -83,7 +88,9 @@ func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg return waitForShutdown(ctx, cancel, server, &wg, errChan) } -// waitForShutdown waits for OS signal or error and performs graceful shutdown +// waitForShutdown blocks until it receives either an OS signal (SIGINT/SIGTERM) or an error from errChan. +// Once triggered, it cancels the application context, attempts a graceful HTTP server shutdown within shutdownTimeout, +// and waits for all goroutines tracked by wg to complete before returning. func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error) error { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) @@ -122,11 +129,17 @@ func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *htt return shutdownErr } +// setupAPIRoute creates and configures the Gin router with all adapter endpoints. +// It registers the following routes: +// +// - POST /adapter/logs — accepts a JSON array of buildlogs.Log, stores it, key: jobID of the first log entry. +// - POST /adapter/test-results — accepts a JSON ResultDTO, stores it, and forwards it to Artemis using the configured result endpoint. +// - GET /health — returns a simple liveness check response. func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { r := gin.Default() jobs := r.Group("/adapter") { - // post logs for specific job + // Post logs for specific job jobs.POST("/logs", func(c *gin.Context) { var newLogs []buildlogs.Log if err := c.BindJSON(&newLogs); err != nil { @@ -148,7 +161,7 @@ func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { c.JSON(http.StatusCreated, gin.H{"status": "stored", "job_id": jobID}) }) - // post test results for specific job + // Post test results for specific job jobs.POST("/test-results", func(c *gin.Context) { var newResults ResultDTO if err := c.BindJSON(&newResults); err != nil { diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 12422360..c3b6d3ad 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -169,6 +169,13 @@ func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *htt return shutdownErr } +// setupAPIRoute creates and configures the Gin router with all log manager endpoints. +// It registers the following routes: +// +// - GET /jobs/:jobId/logs — returns all aggregated log entries for the given job ID. (Used for testing purposes) +// - GET /jobs/:jobId/status — returns the current build status for the given job ID, or 404 if the job is not found. +// - GET /jobs — returns a list of all known job IDs (active and completed). +// - GET /health — liveness probe returning a static OK response. func setupAPIRoute(aggregator LogAggregator) *gin.Engine { r := gin.Default() jobs := r.Group("/jobs") diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index 86ff390b..73b716c0 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -199,6 +199,9 @@ func (la *NATSLogAggregator) cleanupCompletedJobs() { } } +// SendJobLogs retrieves all stored logs for jobID, marshals them to JSON, and +// sends them via an HTTP POST to the configured APIendpoint. It returns an error +// if marshalling, request creation, or the HTTP call itself fails. func (la *NATSLogAggregator) SendJobLogs(jobID string) error { logs := la.GetJobLogs(jobID) @@ -266,10 +269,14 @@ func (la *NATSLogAggregator) GetAllJobs() []string { return jobs } +// UpdateJobStatus stores or overwrites the build status for jobID. +// This method is thread-safe via sync.Map.Store. func (la *NATSLogAggregator) UpdateJobStatus(jobID string, status buildstatus.JobStatus) { la.status.Store(jobID, status) } +// GetJobStatus returns the string representation of the current build status for jobID. +// It returns an error if no status has been recorded for the given job. func (la *NATSLogAggregator) GetJobStatus(jobID string) (string, error) { value, exists := la.status.Load(jobID) if !exists { From 1724e509c85720408fb4dc45bf297008237d61e2 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 23 Feb 2026 18:06:03 +0100 Subject: [PATCH 21/36] rabbit --- HadesLogManager/log_subscriber.go | 4 +++- HadesScheduler/docker/job.go | 3 ++- HadesScheduler/docker/step.go | 3 +++ compose.yml | 1 + .../Create Build Job (Test Fail) with Result Parse Step.bru | 6 ++++++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/HadesLogManager/log_subscriber.go b/HadesLogManager/log_subscriber.go index 17426c57..c2ae86fd 100644 --- a/HadesLogManager/log_subscriber.go +++ b/HadesLogManager/log_subscriber.go @@ -241,6 +241,8 @@ func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) { watcher.wg.Wait() // Wait outside the lock dlm.logAggregator.MarkJobCompleted(jobID) - dlm.logAggregator.SendJobLogs(jobID) + if err := dlm.logAggregator.SendJobLogs(jobID); err != nil { + slog.Error("Failed to send job logs", "job_id", jobID, "error", err) + } } } diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 84d66476..734a5bf6 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -2,6 +2,7 @@ package docker import ( "context" + "errors" "fmt" "log/slog" "maps" @@ -50,7 +51,7 @@ func (d Job) execute(ctx context.Context) error { d.logger.Error("Failed to execute step", slog.Any("error", err)) if step.Metadata[ContinueOnError] == "true" { d.logger.Info("Next step should be executed despite error due to ContinueOnError setting", slog.Any("step", step)) - stepErr = fmt.Errorf("Step %v failed with error but ContinueOnError is set: %w", step.ID, err) + stepErr = errors.Join(stepErr, fmt.Errorf("step %v failed with ContinueOnError set: %w", step.ID, err)) continue } return err diff --git a/HadesScheduler/docker/step.go b/HadesScheduler/docker/step.go index af5c8aa1..f7292274 100644 --- a/HadesScheduler/docker/step.go +++ b/HadesScheduler/docker/step.go @@ -88,6 +88,9 @@ func (s Step) execute(ctx context.Context) error { } defer func() { + if s.Options.containerAutoremove { + return + } if err := removeContainer(ctx, s.cli, resp.ID); err != nil { s.logger.Error("Failed to cleanup container", slog.Any("error", err), slog.Any("container_id", resp.ID)) } diff --git a/compose.yml b/compose.yml index d0455bd6..d0b216bf 100644 --- a/compose.yml +++ b/compose.yml @@ -55,6 +55,7 @@ services: environment: - HADESLOGMANAGER_API_PORT=8081 - API_ENDPOINT=${API_ENDPOINT} + - NATS_URL=nats://nats:4222 hadesArtemisAdapter: image: ghcr.io/ls1intum/hades/hades-artemis-adapter diff --git a/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru index 8f29436a..174c5682 100644 --- a/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru +++ b/docs/api/Create Build Job (Test Fail) with Result Parse Step.bru @@ -51,6 +51,12 @@ body:json { "id": 2, // mandatory to declare the order of execution "name": "Execute", "image": "ls1tum/artemis-maven-template:java17-18", // mandatory + "volumeMounts": [ + { + "name": "shared", + "mountPath": "/shared" + } + ], "script": "set -e && cd /shared/example && ./gradlew --status && ./gradlew clean test", "metadata": { "CONTINUE_ON_ERROR": "true" From bdda573f7831f45bec083cad480d80f802c0ec4d Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 12:43:21 +0100 Subject: [PATCH 22/36] rename env var, move ContinueOnError to Step struct --- HadesAPI/main.go | 2 +- HadesScheduler/docker/job.go | 4 +--- shared/payload/payload.go | 15 ++++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/HadesAPI/main.go b/HadesAPI/main.go index e32c5cc6..c6e67b8e 100644 --- a/HadesAPI/main.go +++ b/HadesAPI/main.go @@ -12,7 +12,7 @@ import ( ) type HadesAPIConfig struct { - APIPort uint `env:"HADESAPI_API_PORT,notEmpty" envDefault:"8080"` + APIPort uint `env:"API_PORT,notEmpty" envDefault:"8080"` NatsConfig hadesnats.ConnectionConfig AuthKey string `env:"AUTH_KEY"` PrometheusAddress string `env:"PROMETHEUS_ADDRESS" envDefault:""` diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 734a5bf6..2d815762 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -12,8 +12,6 @@ import ( "github.com/ls1intum/hades/shared/payload" ) -const ContinueOnError = "CONTINUE_ON_ERROR" - type Job struct { cli *client.Client logger *slog.Logger @@ -49,7 +47,7 @@ func (d Job) execute(ctx context.Context) error { err := dockerStep.execute(stepCtx) if err != nil { d.logger.Error("Failed to execute step", slog.Any("error", err)) - if step.Metadata[ContinueOnError] == "true" { + if step.ContinueOnError == true { d.logger.Info("Next step should be executed despite error due to ContinueOnError setting", slog.Any("step", step)) stepErr = errors.Join(stepErr, fmt.Errorf("step %v failed with ContinueOnError set: %w", step.ID, err)) continue diff --git a/shared/payload/payload.go b/shared/payload/payload.go index 4c24296c..4d8d00fd 100644 --- a/shared/payload/payload.go +++ b/shared/payload/payload.go @@ -32,13 +32,14 @@ type QueuePayload struct { // Step represents a single execution step in a job. // Each step runs in its own container with the specified image and resources. type Step struct { - ID int `json:"id"` // Step execution order (starts at 1) - Name string `json:"name"` // Human-readable step name - Image string `json:"image"` // Container image to use (e.g., "alpine:latest") - Script string `json:"script"` // Shell script to execute in the container - Metadata map[string]string `json:"metadata"` // Step-specific environment variables and metadata - CPULimit uint `json:"cpu_limit"` // CPU limit in millicores (e.g., 1000 = 1 CPU core) - MemoryLimit string `json:"memory_limit"` // Memory limit (e.g., "512M", "2G") + ID int `json:"id"` // Step execution order (starts at 1) + Name string `json:"name"` // Human-readable step name + Image string `json:"image"` // Container image to use (e.g., "alpine:latest") + Script string `json:"script"` // Shell script to execute in the container + ContinueOnError bool `json:"continue_on_error"` // Whether to continue with the next step if this step fails + Metadata map[string]string `json:"metadata"` // Step-specific environment variables and metadata + CPULimit uint `json:"cpu_limit"` // CPU limit in millicores (e.g., 1000 = 1 CPU core) + MemoryLimit string `json:"memory_limit"` // Memory limit (e.g., "512M", "2G") } // IDString returns the step ID as a string. From 045d70ba2715d895e1d8c77c86cacb06bb15fa4b Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 12:52:08 +0100 Subject: [PATCH 23/36] update ci.yml --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5bb40d..71c8748e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,17 +89,17 @@ jobs: if: github.actor != 'dependabot[bot]' uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 with: - image-name: ls1intum/hades/hades-operator - docker-file: ./HadesScheduler/HadesOperator/Dockerfile + image-name: ls1intum/hades/hades-log-manager + docker-file: ./HadesLogManager/Dockerfile secrets: inherit build-artemis-adapter: - needs: test + needs: test if: github.actor != 'dependabot[bot]' uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 with: - image-name: ls1intum/hades/hades-operator - docker-file: ./HadesScheduler/HadesOperator/Dockerfile + image-name: ls1intum/hades/hades-artemis-adapter + docker-file: ./HadesArtemisAdapter/Dockerfile secrets: inherit deploy: From d9029e1fe44d44f42159c356a72eddbca362d0d0 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 14:05:30 +0100 Subject: [PATCH 24/36] move adapter to seperate repo --- .env.example | 6 - .github/workflows/ci.yml | 18 --- HadesArtemisAdapter/Dockerfile | 30 ---- HadesArtemisAdapter/artemis_adapter.go | 196 ------------------------- HadesArtemisAdapter/go.mod | 45 ------ HadesArtemisAdapter/go.sum | 104 ------------- HadesArtemisAdapter/main.go | 189 ------------------------ HadesLogManager/go.mod | 16 +- HadesLogManager/go.sum | 21 +-- HadesScheduler/HadesOperator/go.mod | 18 +-- HadesScheduler/HadesOperator/go.sum | 33 ++--- compose.yml | 15 -- docker-compose.deploy.yml | 13 -- docker-compose.dev.yml | 12 +- go.work | 1 - 15 files changed, 36 insertions(+), 681 deletions(-) delete mode 100644 HadesArtemisAdapter/Dockerfile delete mode 100644 HadesArtemisAdapter/artemis_adapter.go delete mode 100644 HadesArtemisAdapter/go.mod delete mode 100644 HadesArtemisAdapter/go.sum delete mode 100644 HadesArtemisAdapter/main.go diff --git a/.env.example b/.env.example index 5956d891..2c9d1970 100644 --- a/.env.example +++ b/.env.example @@ -24,12 +24,6 @@ WATCH_NAMESPACE="hades" DELETE_ON_COMPLETE="true" MAX_PARALLELISM="100" -### HadesArtemisAdapter Options ### -HADESADAPTER_API_PORT=8082 # Port for the Artemis Adapter to listen on (default: 8082) -ARTEMIS_BASE_URL=http://localhost:8080 -ARTEMIS_NEW_RESULT_ENDPOINT=api/programming/public/programming-exercises/new-result # this is the base. it is used with /{participationId} -ARTEMIS_AUTH_TOKEN=superduperlongtestingsecrettoken - ### HadesLogManager Options ### HADESLOGMANAGER_API_PORT=8081 # Port for the Log Manager to listen on (default: 8081) API_ENDPOINT=http://localhost:8082/adapter/logs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71c8748e..f10120f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,15 +75,6 @@ jobs: docker-file: ./HadesScheduler/HadesOperator/Dockerfile secrets: inherit - build-operator: - needs: test - if: github.actor != 'dependabot[bot]' - uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 - with: - image-name: ls1intum/hades/hades-operator - docker-file: ./HadesScheduler/HadesOperator/Dockerfile - secrets: inherit - build-log-manager: needs: test if: github.actor != 'dependabot[bot]' @@ -93,15 +84,6 @@ jobs: docker-file: ./HadesLogManager/Dockerfile secrets: inherit - build-artemis-adapter: - needs: test - if: github.actor != 'dependabot[bot]' - uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 - with: - image-name: ls1intum/hades/hades-artemis-adapter - docker-file: ./HadesArtemisAdapter/Dockerfile - secrets: inherit - deploy: needs: [build-api, build-scheduler] runs-on: ubuntu-latest diff --git a/HadesArtemisAdapter/Dockerfile b/HadesArtemisAdapter/Dockerfile deleted file mode 100644 index f16b2ace..00000000 --- a/HadesArtemisAdapter/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# Use an official Go runtime as a parent image -FROM golang:1.25-alpine AS builder - -# Set the working directory inside the container -WORKDIR /app - -# Copy go.mod and go.sum files -COPY ./HadesArtemisAdapter/go.mod ./HadesArtemisAdapter/go.sum ./HadesArtemisAdapter/ -COPY ./shared/go.mod ./shared/go.sum ./shared/ -RUN cd HadesArtemisAdapter && go mod download - -# Copy the Go application source code into the container -COPY ./HadesArtemisAdapter ./HadesArtemisAdapter -COPY ./shared ./shared - -# Build the Go application -WORKDIR /app/HadesArtemisAdapter -RUN CGO_ENABLED=0 go build -o hades-artemis-adapter . - -# Start a new stage for the minimal runtime container -FROM gcr.io/distroless/static-debian12 - -# Set the working directory inside the minimal runtime container -WORKDIR /app - -# Copy the built binary from the builder container into the minimal runtime container -COPY --from=builder /app/HadesArtemisAdapter/hades-artemis-adapter . - -# Run your Go application -CMD ["/app/hades-artemis-adapter"] \ No newline at end of file diff --git a/HadesArtemisAdapter/artemis_adapter.go b/HadesArtemisAdapter/artemis_adapter.go deleted file mode 100644 index e966cd12..00000000 --- a/HadesArtemisAdapter/artemis_adapter.go +++ /dev/null @@ -1,196 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "log/slog" - "net/http" - "sync" - "time" - - "github.com/ls1intum/hades/shared/buildlogs" -) - -// ArtemisAdapter handles the collection and synchronization of build logs and test results -// before sending them to Artemis. It uses sync.Map for thread-safe storage and per-job -// mutexes to ensure exactly-once delivery of combined results. -type ArtemisAdapter struct { - logs sync.Map // jobID (string) -> []buildlogs.LogEntry (only Step 2:execution logs) - results sync.Map // jobID (string) -> results - jobLocks sync.Map // jobID (string) -> *sync.Mutex - httpClient *http.Client - cfg AdapterConfig -} - -// ResultDTO is the complete data transfer object sent to Artemis. -// This struct is adapted from Artemis. -// It combines metadata, test results, and build logs into a single payload. -type ResultDTO struct { - ResultMetadata - Results []TestSuiteDTO `json:"results"` - BuildLogs []buildlogs.LogEntry `json:"logs"` -} - -// ResultMetadata contains the metadata about a build job's execution and outcome. -type ResultMetadata struct { - JobName string `json:"jobName"` // contains participationId - UUID string `json:"uuid"` // JobId - AssignmentRepoBranchName string `json:"assignmentRepoBranchName"` - IsBuildSuccessful bool `json:"isBuildSuccessful"` - AssignmentRepoCommitHash string `json:"assignmentRepoCommitHash"` - TestsRepoCommitHash string `json:"testsRepoCommitHash"` - BuildCompletionTime string `json:"buildCompletionTime"` - Passed int `json:"passed"` -} - -// TestSuiteDTO represents a collection of test cases that were executed together. -// It aggregates statistics about the test run including total tests, failures, and execution time. -type TestSuiteDTO struct { - Name string `json:"name"` - Time float64 `json:"time"` - Errors int `json:"errors"` - Skipped int `json:"skipped"` - Failures int `json:"failures"` - Tests int `json:"tests"` - TestCases []TestCaseDTO `json:"testCases"` -} - -// TestCaseDTO represents a single test case execution result. -// It contains either failure/error details for failed tests or success information for passed tests. -type TestCaseDTO struct { - Name string `json:"name"` - Classname string `json:"classname"` - Time float64 `json:"time"` - Failures []TestCaseDetailMessageDTO `json:"failures"` // empty for passing tests - Errors []TestCaseDetailMessageDTO `json:"errors"` // empty for passing tests - Successes []TestCaseDetailMessageDTO `json:"successInfos"` // empty for failing tests -} - -// TestCaseDetailMessageDTO provides detailed information about a test case result, -// including error messages, stack traces, and result types. -type TestCaseDetailMessageDTO struct { - Message string `json:"message"` - Type string `json:"type"` - MessageWithStackTrace string `json:"messageWithStackTrace"` -} - -const requestTimeout = 10 * time.Second - -func NewAdapter(ctx context.Context, cfg AdapterConfig) *ArtemisAdapter { - aa := ArtemisAdapter{ - httpClient: &http.Client{Timeout: requestTimeout}, - cfg: cfg, - } - - return &aa -} - -// StoreLogs stores execution logs (Step 2) for the given job ID and triggers a check -// to see if results are also ready. If both logs and results are available, they are -// combined and sent to Artemis. -// -// If logs array has fewer than 2 elements, an empty log array is stored and a warning is logged. -func (aa *ArtemisAdapter) StoreLogs(jobID string, logs []buildlogs.Log) error { - executionLogs := []buildlogs.LogEntry{} - if len(logs) < 2 { - slog.Warn("Execution logs missing", "jobID", jobID) - executionLogs = []buildlogs.LogEntry{} - } else { - executionLogs = logs[1].Logs - } - - aa.logs.Store(jobID, executionLogs) - return aa.checkAndSendIfReady(jobID) -} - -// StoreResults stores test results for the given job ID and triggers a check -// to see if logs are also ready. If both logs and results are available, they are -// combined and sent to Artemis. -func (aa *ArtemisAdapter) StoreResults(jobID string, results ResultDTO) error { - aa.results.Store(jobID, results) - slog.Debug("Stored results", "jobID", jobID, "results", results.Results) - return aa.checkAndSendIfReady(jobID) -} - -// checkAndSendIfReady checks if both logs and results exist for the given jobID. -// If both are present, it combines them into a single ResultDTO and sends it to Artemis. -// After a successful send, all data for the job (logs, results, and mutex) is cleaned up. -// -// This method uses a per-job mutex to ensure that even if StoreLogs and StoreResults -// are called concurrently for the same job, the check-combine-send-delete sequence -// is atomic, preventing duplicate sends. -func (aa *ArtemisAdapter) checkAndSendIfReady(jobID string) error { - // Get or create mutex for this specific job - lockVal, _ := aa.jobLocks.LoadOrStore(jobID, &sync.Mutex{}) - lock := lockVal.(*sync.Mutex) - - lock.Lock() - defer lock.Unlock() - - logs, logsExist := aa.logs.Load(jobID) - results, resultsExist := aa.results.Load(jobID) - - // If both logs and results exist, combine and send - if logsExist && resultsExist { - logs := logs.([]buildlogs.LogEntry) - results := results.(ResultDTO) - - results.BuildLogs = logs - slog.Debug("Combined logs and results", "jobID", jobID) - - // Send to Artemis - if err := aa.sendToArtemis(results); err != nil { - slog.Error("Failed to send results with logs to Artemis", "jobID", jobID, "error", err) - return err - } - slog.Info("Successfully sent results with logs to Artemis", "jobID", jobID) - - // Clean up after successful send - aa.logs.Delete(jobID) - aa.results.Delete(jobID) - aa.jobLocks.Delete(jobID) - } else { - slog.Debug("Waiting for complete data", "jobID", jobID, "hasLogs", logsExist, "hasResults", resultsExist) - } - - return nil -} - -// sendToArtemis sends the combined ResultDTO to the configured Artemis endpoint. -// It constructs the full endpoint URL using the job name, marshals the DTO to JSON, -// and sends an authenticated POST request. -// -// Returns an error if marshaling fails, request creation fails, the HTTP request fails, -// or if Artemis responds with a status code >= 400. -func (aa *ArtemisAdapter) sendToArtemis(dto ResultDTO) error { - endpoint := fmt.Sprintf("%s/%s/%s", aa.cfg.ArtemisBaseURL, aa.cfg.NewResultEndpoint, dto.JobName) - - jsonData, err := json.Marshal(dto) - if err != nil { - return fmt.Errorf("marshaling DTO to JSON: %w", err) - } - - req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData)) - if err != nil { - return fmt.Errorf("creating request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", aa.cfg.ArtemisAuthToken) - - resp, err := aa.httpClient.Do(req) - if err != nil { - return fmt.Errorf("sending request to Artemis: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode >= 400 { - return fmt.Errorf("Artemis returned HTTP %d for job %s", resp.StatusCode, dto.UUID) - } - - slog.Info("Request sent", "status", resp.Status) - - return nil -} diff --git a/HadesArtemisAdapter/go.mod b/HadesArtemisAdapter/go.mod deleted file mode 100644 index eacd888b..00000000 --- a/HadesArtemisAdapter/go.mod +++ /dev/null @@ -1,45 +0,0 @@ -module HadesArtemisAdapter - -go 1.25.3 - -require ( - github.com/gin-gonic/gin v1.11.0 - github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b -) - -require ( - github.com/bytedance/sonic v1.14.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect - github.com/caarlos0/env/v11 v11.3.1 // indirect - github.com/cloudwego/base64x v0.1.6 // indirect - github.com/gabriel-vasile/mimetype v1.4.8 // indirect - github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/goccy/go-yaml v1.18.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/joho/godotenv v1.5.1 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/leodido/go-urn v1.4.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/nats-io/nats.go v1.48.0 // indirect - github.com/nats-io/nkeys v0.4.11 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.0 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect - golang.org/x/arch v0.20.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - google.golang.org/protobuf v1.36.9 // indirect -) diff --git a/HadesArtemisAdapter/go.sum b/HadesArtemisAdapter/go.sum deleted file mode 100644 index 8d357048..00000000 --- a/HadesArtemisAdapter/go.sum +++ /dev/null @@ -1,104 +0,0 @@ -github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= -github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= -github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= -github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= -github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= -github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= -github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b h1:qr4I+vH9qvuUIDMNyhcovwSapdd1AAbVjr2D8M1rlbw= -github.com/ls1intum/hades/shared v0.0.0-20260116114842-b44546727e4b/go.mod h1:dtZoDaLxyhyNlMNGZ/zHJ2pEwvVcv62uPZD4f4y1jKY= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= -github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= -github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= -github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE= -github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= -golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= -golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/HadesArtemisAdapter/main.go b/HadesArtemisAdapter/main.go deleted file mode 100644 index bed180c6..00000000 --- a/HadesArtemisAdapter/main.go +++ /dev/null @@ -1,189 +0,0 @@ -package main - -import ( - "context" - "errors" - "log/slog" - "net/http" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/gin-gonic/gin" - "github.com/ls1intum/hades/shared/buildlogs" - "github.com/ls1intum/hades/shared/utils" -) - -const ( - shutdownTimeout = 30 * time.Second -) - -// AdapterConfig holds the configuration for the Artemis adapter service. -type AdapterConfig struct { - APIPort string `env:"HADESADAPTER_API_PORT" envDefault:"8082"` - ArtemisBaseURL string `env:"ARTEMIS_BASE_URL"` - NewResultEndpoint string `env:"ARTEMIS_NEW_RESULT_ENDPOINT"` - ArtemisAuthToken string `env:"ARTEMIS_AUTH_TOKEN"` -} - -// main is the application entry point. It initialises logging, loads configuration from environment variables, -// and delegates to run for the core application logic. -func main() { - // Setup logging - utils.SetupLogging() - - var cfg AdapterConfig - utils.LoadConfig(&cfg) - - // Run main application - if err := run(cfg); err != nil { - slog.Error("Application error", "error", err) - os.Exit(1) - } -} - -// run initialises the application context, creates the ArtemisAdapter, and starts the server with graceful shutdown support. -// It returns any fatal error encountered during the lifetime of the application. -func run(cfg AdapterConfig) error { - - // Create context for application lifecycle - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - aa := NewAdapter(ctx, cfg) - - // Set up graceful shutdown - return runWithGracefulShutdown(ctx, cancel, cfg, aa) -} - -// runWithGracefulShutdown configures and starts the HTTP API server in a background goroutine, -// then delegates to waitForShutdown to block until either an OS signal or a fatal server error triggers a graceful teardown. -func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg AdapterConfig, aa *ArtemisAdapter) error { - var wg sync.WaitGroup - errChan := make(chan error, 2) - - // Start API server - router := setupAPIRoute(aa) - server := &http.Server{ - Addr: ":" + cfg.APIPort, - Handler: router, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, - ReadHeaderTimeout: 5 * time.Second, - } - - wg.Add(1) - go func() { - defer wg.Done() - slog.Info("Starting API server", "port", cfg.APIPort) - - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - slog.Error("API server failed", "error", err) - errChan <- err - } - }() - - return waitForShutdown(ctx, cancel, server, &wg, errChan) -} - -// waitForShutdown blocks until it receives either an OS signal (SIGINT/SIGTERM) or an error from errChan. -// Once triggered, it cancels the application context, attempts a graceful HTTP server shutdown within shutdownTimeout, -// and waits for all goroutines tracked by wg to complete before returning. -func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error) error { - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - var shutdownErr error - - select { - case sig := <-sigChan: - slog.Info("Received shutdown signal", "signal", sig.String()) - case err := <-errChan: - slog.Error("Error during operation", "error", err) - shutdownErr = err - } - - // Cancel context to stop background goroutines - slog.Info("Starting graceful shutdown...") - cancel() - - // Shutdown API server with timeout - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout) - defer shutdownCancel() - - if err := server.Shutdown(shutdownCtx); err != nil { - slog.Error("API server shutdown error", "error", err) - if shutdownErr == nil { - shutdownErr = err - } - } else { - slog.Info("API server shutdown complete") - } - - // Wait for all goroutines to finish - wg.Wait() - slog.Info("Graceful shutdown complete") - - return shutdownErr -} - -// setupAPIRoute creates and configures the Gin router with all adapter endpoints. -// It registers the following routes: -// -// - POST /adapter/logs — accepts a JSON array of buildlogs.Log, stores it, key: jobID of the first log entry. -// - POST /adapter/test-results — accepts a JSON ResultDTO, stores it, and forwards it to Artemis using the configured result endpoint. -// - GET /health — returns a simple liveness check response. -func setupAPIRoute(aa *ArtemisAdapter) *gin.Engine { - r := gin.Default() - jobs := r.Group("/adapter") - { - // Post logs for specific job - jobs.POST("/logs", func(c *gin.Context) { - var newLogs []buildlogs.Log - if err := c.BindJSON(&newLogs); err != nil { - slog.Error("Failed to bind logs JSON", "error", err) - return - } - - if len(newLogs) == 0 { - c.JSON(http.StatusBadRequest, gin.H{"error": "empty logs array"}) - return - } - - jobID := newLogs[0].JobID - if err := aa.StoreLogs(jobID, newLogs); err != nil { - slog.Error("Failed to store logs", "job_id", jobID, "error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to store logs"}) - return - } - c.JSON(http.StatusCreated, gin.H{"status": "stored", "job_id": jobID}) - }) - - // Post test results for specific job - jobs.POST("/test-results", func(c *gin.Context) { - var newResults ResultDTO - if err := c.BindJSON(&newResults); err != nil { - slog.Error("Failed to bind test results JSON", "error", err) - return - } - - if err := aa.StoreResults(newResults.UUID, newResults); err != nil { - slog.Error("Failed to store/send results", "uuid", newResults.UUID, "error", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to process results"}) - return - } - - slog.Debug("Stored new test results", "uuid", newResults.UUID) - c.IndentedJSON(http.StatusCreated, newResults) - }) - } - - // Health check endpoint - r.GET("/health", func(c *gin.Context) { - c.JSON(200, gin.H{"status": "ok"}) - }) - - return r -} diff --git a/HadesLogManager/go.mod b/HadesLogManager/go.mod index dc67f254..926b6488 100644 --- a/HadesLogManager/go.mod +++ b/HadesLogManager/go.mod @@ -1,16 +1,16 @@ -module HadesAdapter +module github.com/ls1intum/hades/hadesLogManager go 1.24.4 require ( github.com/gin-gonic/gin v1.11.0 - github.com/nats-io/nats.go v1.48.0 + github.com/nats-io/nats.go v1.49.0 ) require ( github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect - github.com/caarlos0/env/v11 v11.3.1 // indirect + github.com/caarlos0/env/v11 v11.4.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect @@ -34,15 +34,15 @@ require ( github.com/ugorji/go/codec v1.3.0 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/text v0.32.0 // indirect google.golang.org/protobuf v1.36.9 // indirect ) require ( - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398 - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/sys v0.39.0 // indirect ) diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index fc5a8500..b7f2c9c4 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -2,8 +2,7 @@ github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQ github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= -github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoGAc= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -37,8 +36,7 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -53,10 +51,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= -github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -85,15 +81,12 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= diff --git a/HadesScheduler/HadesOperator/go.mod b/HadesScheduler/HadesOperator/go.mod index b5a239ec..2a00087f 100644 --- a/HadesScheduler/HadesOperator/go.mod +++ b/HadesScheduler/HadesOperator/go.mod @@ -11,7 +11,7 @@ replace github.com/ls1intum/hades/hadesScheduler => ../../HadesScheduler require ( github.com/ls1intum/hades/hadesScheduler v0.0.0-00010101000000-000000000000 github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e - github.com/nats-io/nats.go v1.48.0 + github.com/nats-io/nats.go v1.49.0 k8s.io/api v0.35.1 k8s.io/apimachinery v0.35.1 k8s.io/client-go v0.35.1 @@ -20,7 +20,7 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/caarlos0/env/v11 v11.3.1 // indirect + github.com/caarlos0/env/v11 v11.4.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect @@ -40,12 +40,12 @@ require ( github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.22.0 // indirect @@ -58,13 +58,13 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.45.0 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.9 // indirect diff --git a/HadesScheduler/HadesOperator/go.sum b/HadesScheduler/HadesOperator/go.sum index ec5f5a46..346dcaa0 100644 --- a/HadesScheduler/HadesOperator/go.sum +++ b/HadesScheduler/HadesOperator/go.sum @@ -8,8 +8,7 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= -github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoGAc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -89,8 +88,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -127,10 +125,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= -github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= @@ -207,12 +203,10 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -224,27 +218,22 @@ golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKl golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/compose.yml b/compose.yml index d0b216bf..7983d00c 100644 --- a/compose.yml +++ b/compose.yml @@ -57,21 +57,6 @@ services: - API_ENDPOINT=${API_ENDPOINT} - NATS_URL=nats://nats:4222 - hadesArtemisAdapter: - image: ghcr.io/ls1intum/hades/hades-artemis-adapter - build: - context: . - dockerfile: ./HadesArtemisAdapter/Dockerfile - ports: - - "8082:8082" - networks: - - hades - environment: - - HADESADAPTER_API_PORT=8082 - - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} - - ARTEMIS_NEW_RESULT_ENDPOINT=${ARTEMIS_NEW_RESULT_ENDPOINT} - - ARTEMIS_AUTH_TOKEN=${ARTEMIS_AUTH_TOKEN} - nats: image: nats:2.11.4 command: ["-js", "-m", "8222"] diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index 78d52292..38a469e5 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -48,19 +48,6 @@ services: - "traefik.http.routers.hadeslogmanager.tls.certresolver=letsencrypt" - "traefik.http.services.hadeslogmanager.loadbalancer.server.port=8081" - hadesArtemisAdapter: - ports: [] - expose: - - "8082" - networks: - - hades - labels: - - "traefik.enable=true" - - "traefik.http.routers.hadesadapter.rule=Host(`${HADES_ADAPTER_HOST:-adapter.${HADES_API_HOST}}`)" - - "traefik.http.routers.hadesadapter.entrypoints=web,websecure" - - "traefik.http.routers.hadesadapter.tls.certresolver=letsencrypt" - - "traefik.http.services.hadesadapter.loadbalancer.server.port=8082" - nats: ports: [] networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8c3a62b5..6cee18b9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -23,14 +23,4 @@ services: - DEBUG=true - HADESLOGMANAGER_API_PORT=8081 - API_ENDPOINT=http://hadesArtemisAdapter:8082/adapter/logs - - hadesArtemisAdapter: - ports: - - "8082:8082" # Expose for local access - environment: - - DEBUG=true - - HADESADAPTER_API_PORT=8082 - # Override these in your .env file for local dev - - ARTEMIS_BASE_URL=${ARTEMIS_BASE_URL} - - ARTEMIS_NEW_RESULT_ENDPOINT=${ARTEMIS_NEW_RESULT_ENDPOINT} - - ARTEMIS_AUTH_TOKEN=${ARTEMIS_AUTH_TOKEN} + \ No newline at end of file diff --git a/go.work b/go.work index daf80089..2d7b38f2 100644 --- a/go.work +++ b/go.work @@ -2,7 +2,6 @@ go 1.25.3 use ( ./HadesAPI - ./HadesArtemisAdapter ./HadesLogManager ./HadesScheduler ./HadesScheduler/HadesOperator From f0161c30d77fb81208664039f5c50f6dff659986 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 14:09:19 +0100 Subject: [PATCH 25/36] rollback readme --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 99141046..478f519e 100644 --- a/Readme.md +++ b/Readme.md @@ -157,13 +157,13 @@ For more complex workflows, you can define multi-step jobs where each step runs ## Configuration Options -Hades can be configured through environment variables or through a `.env` file: +Hades can be configured through environment variables or a `.env` file: | Variable | Description | Default | |----------|-------------|---------| | `HADES_EXECUTOR` | Execution platform: `docker` or `kubernetes` | `docker` | | `CONCURRENCY` | Number of jobs to process concurrently | `1` | -| `HADESAPI_API_PORT` | Port for the Hades API | `8080` | +| `API_PORT` | Port for the Hades API | `8080` | ## Deployment From b9e5c881a85fcaed80d418b255c15b4cd808f455 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 14:25:24 +0100 Subject: [PATCH 26/36] edit adapter endpoint config in docker compose --- .env.example | 4 ++-- docker-compose.dev.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 2c9d1970..068d3ef6 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,7 @@ NATS_PASSWORD= # NATS password (optional) NATS_TLS_ENABLED=false # Enable TLS for NATS connection (default: false) ### HadesAPI Options ### -HADESAPI_API_PORT=8080 # Port for the API to listen on (default: 8080) +API_PORT=8080 # Port for the API to listen on (default: 8080) AUTH_KEY="" # Key used for Basic Auth with the API (default: "" for no auth) PROMETHEUS_ADDRESS="" # Address for Prometheus metrics to be exposed (default: "" for no metrics) RETENTION_IN_MIN=30 # Time in minutes to keep completed jobs for monitoring (default: 30) @@ -26,7 +26,7 @@ MAX_PARALLELISM="100" ### HadesLogManager Options ### HADESLOGMANAGER_API_PORT=8081 # Port for the Log Manager to listen on (default: 8081) -API_ENDPOINT=http://localhost:8082/adapter/logs +ARTEMIS_ADAPTER_URL=http://localhost:8082/adapter/logs # URL for the Artemis Adapter to send logs to (default: http://localhost:8082/adapter/logs) ### DockerScheduler Options ### DOCKER_HOST="unix:///var/run/docker.sock" # Docker host to connect to (default: unix:///var/run/docker.sock) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 6cee18b9..adbb52b8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -22,5 +22,5 @@ services: environment: - DEBUG=true - HADESLOGMANAGER_API_PORT=8081 - - API_ENDPOINT=http://hadesArtemisAdapter:8082/adapter/logs + - API_ENDPOINT=${ARTEMIS_ADAPTER_URL} \ No newline at end of file From 3a70a3d3c21f1c1d0a425becd268cb10c15eda79 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Mon, 9 Mar 2026 15:03:39 +0100 Subject: [PATCH 27/36] go mod tidy --- HadesLogManager/go.sum | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index b7f2c9c4..f4866bf1 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -3,6 +3,7 @@ github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxl github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoGAc= +github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -37,6 +38,7 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -52,7 +54,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE= +github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw= github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= +github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -82,11 +86,14 @@ go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= From 1d6427d95384ffd89cfe061ad4390cefafa11266 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 13 Mar 2026 16:16:42 +0100 Subject: [PATCH 28/36] adapt traefik values to #388 --- .github/workflows/ci.yml | 13 ++++++++++--- docker-compose.test.yml | 22 ++++------------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f10120f2..db0c8daa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,11 +84,20 @@ jobs: docker-file: ./HadesLogManager/Dockerfile secrets: inherit + deploy-traefik: + needs: [build-api, build-scheduler] + uses: ls1intum/.github/.github/workflows/deploy-docker-traefik.yml@main + with: + environment: hades-test + image-tag: v3.6 + secrets: inherit + deploy: needs: [build-api, build-scheduler] runs-on: ubuntu-latest environment: hades-test steps: + - name: Checkout repository uses: actions/checkout@v6 @@ -114,7 +123,7 @@ jobs: proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }} proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} - source: "docker-compose.test.yml,traefik/traefik.yml" + source: "docker-compose.test.yml" target: /opt/hades strip_components: 0 @@ -129,8 +138,6 @@ jobs: proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }} proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }} script: | - touch /opt/hades/traefik/acme.json - chmod 600 /opt/hades/traefik/acme.json install -m 600 /dev/null /opt/hades/.env cat > /opt/hades/.env <<'EOF' IMAGE_TAG=${{ needs.build-api.outputs.image_tag }} diff --git a/docker-compose.test.yml b/docker-compose.test.yml index e260f619..0f5d0e5a 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,29 +1,13 @@ version: "3.8" services: - traefik: - image: traefik:v3.6 - container_name: traefik - command: - - --configFile=/etc/traefik/traefik.yml - - --certificatesresolvers.letsencrypt.acme.email=${LETSENCRYPT_EMAIL} - ports: - - "80:80" - - "443:443" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./traefik/traefik.yml:/etc/traefik/traefik.yml:ro - - ./traefik/acme.json:/etc/traefik/acme.json - restart: unless-stopped - networks: - - hades - hadesAPI: image: ghcr.io/ls1intum/hades/hades-api:${IMAGE_TAG:?IMAGE_TAG is required} expose: - "8080" networks: - hades + - traefik depends_on: nats: condition: service_healthy @@ -37,7 +21,7 @@ services: - "traefik.enable=true" - "traefik.http.routers.hadesapi.rule=Host(`${HADES_API_HOST}`)" - "traefik.http.routers.hadesapi.entrypoints=websecure" - - "traefik.http.routers.hadesapi.tls.certresolver=letsencrypt" + - "traefik.http.routers.hadesapi.tls.certresolver=harica" - "traefik.http.services.hadesapi.loadbalancer.server.port=8080" restart: unless-stopped @@ -77,3 +61,5 @@ services: networks: hades: external: false + traefik: + external: true From 01758aa9c62de5ee6855382fe769e0471436994f Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 5 Jun 2026 13:17:07 +0200 Subject: [PATCH 29/36] chore: deploy hadesLogManager in test environment --- .github/workflows/ci.yml | 2 +- docker-compose.test.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c904e8e1..a0196228 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,7 +145,7 @@ jobs: secrets: inherit deploy: - needs: [build-api, build-scheduler] + needs: [build-api, build-scheduler, build-log-manager] runs-on: ubuntu-latest environment: hades-test steps: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 0f5d0e5a..58e5f9b9 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -47,6 +47,20 @@ services: - DOCKER_MEMORY_LIMIT=${DOCKER_MEMORY_LIMIT:-2g} restart: unless-stopped + hadesLogManager: + image: ghcr.io/ls1intum/hades/hades-log-manager:${IMAGE_TAG:?IMAGE_TAG is required} + expose: + - "8081" + networks: + - hades + depends_on: + nats: + condition: service_healthy + environment: + - NATS_URL=nats://nats:4222 + - ARTEMIS_ADAPTER_URL=https://hades-artemis-connector.student.k8s.aet.cit.tum.de/adapter/logs + restart: unless-stopped + nats: image: nats:2.11.4 command: ["-js", "-m", "8222"] From 11d8b961da055b34433b2f05607b413ff404b336 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 5 Jun 2026 17:16:28 +0200 Subject: [PATCH 30/36] * fix duplicate traefik job * HadesLogManager to use refactored interface *go mod tidy --- .github/workflows/ci.yml | 10 +--------- HadesAPI/go.sum | 8 ++++++++ HadesLogManager/go.sum | 1 + HadesLogManager/main.go | 4 ++-- HadesLogManager/processor.go | 16 ++-------------- HadesScheduler/HadesOperator/go.sum | 25 +++++++++++++++++++++++++ HadesScheduler/go.sum | 3 +++ shared/buildlogs/interfaces.go | 1 + 8 files changed, 43 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0196228..434221bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,7 +128,7 @@ jobs: secrets: inherit build-log-manager: - needs: test + needs: [lint, build, test] if: github.actor != 'dependabot[bot]' uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0 with: @@ -136,14 +136,6 @@ jobs: docker-file: ./HadesLogManager/Dockerfile secrets: inherit - deploy-traefik: - needs: [build-api, build-scheduler] - uses: ls1intum/.github/.github/workflows/deploy-docker-traefik.yml@main - with: - environment: hades-test - image-tag: v3.6 - secrets: inherit - deploy: needs: [build-api, build-scheduler, build-log-manager] runs-on: ubuntu-latest diff --git a/HadesAPI/go.sum b/HadesAPI/go.sum index 5af2341f..719d7875 100644 --- a/HadesAPI/go.sum +++ b/HadesAPI/go.sum @@ -39,6 +39,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= @@ -177,11 +178,17 @@ go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzyb go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= @@ -196,6 +203,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index fae1cb5f..0282878e 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -88,6 +88,7 @@ github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8= go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= diff --git a/HadesLogManager/main.go b/HadesLogManager/main.go index 8ee6c0ae..b226a3ec 100644 --- a/HadesLogManager/main.go +++ b/HadesLogManager/main.go @@ -80,7 +80,7 @@ func connectNATS(config hadesnats.ConnectionConfig) (*nats.Conn, error) { } // runWithGracefulShutdown starts services and handles graceful shutdown -func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesLogManagerConfig, dynamicManager LogManager, logAggregator LogAggregator, +func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesLogManagerConfig, dynamicManager buildlogs.LogManager, logAggregator buildlogs.LogAggregator, ) error { var wg sync.WaitGroup errChan := make(chan error, 2) @@ -169,7 +169,7 @@ func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *htt // - GET /jobs/:jobId/status — returns the current build status for the given job ID, or 404 if the job is not found. // - GET /jobs — returns a list of all known job IDs (active and completed). // - GET /health — liveness probe returning a static OK response. -func setupAPIRoute(aggregator LogAggregator) *gin.Engine { +func setupAPIRoute(aggregator buildlogs.LogAggregator) *gin.Engine { r := gin.Default() jobs := r.Group("/jobs") { diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index 8dedbc31..70807eeb 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -18,18 +18,6 @@ const ( httpClientTimeout = 10 * time.Second ) -// LogAggregator defines the interface for aggregating and managing job logs -type LogAggregator interface { - AddLog(log buildlogs.Log) - FlushJob(jobID string) error - GetJobLogs(jobID string) []buildlogs.Log - GetJobStatus(jobID string) (string, error) - GetAllJobs() []string - SendJobLogs(jobID string) error - MarkJobCompleted(jobID string) - UpdateJobStatus(jobID string, status buildstatus.JobStatus) -} - // NATSLogAggregator implements LogAggregator using in-memory storage for fast log retrieval. // It provides thread-safe log aggregation with configurable batching, automatic log rotation, // and memory management. Thread-safety is provided by sync.Map for all operations. @@ -277,10 +265,10 @@ func (la *NATSLogAggregator) UpdateJobStatus(jobID string, status buildstatus.Jo // GetJobStatus returns the string representation of the current build status for jobID. // It returns an error if no status has been recorded for the given job. -func (la *NATSLogAggregator) GetJobStatus(jobID string) (string, error) { +func (la *NATSLogAggregator) GetJobStatus(jobID string) (buildstatus.JobStatus, error) { value, exists := la.status.Load(jobID) if !exists { - return "", fmt.Errorf("job not found: %s", jobID) + return buildstatus.JobStatus(""), fmt.Errorf("job not found: %s", jobID) } return value.(buildstatus.JobStatus), nil } diff --git a/HadesScheduler/HadesOperator/go.sum b/HadesScheduler/HadesOperator/go.sum index 732b9e9e..e63bff97 100644 --- a/HadesScheduler/HadesOperator/go.sum +++ b/HadesScheduler/HadesOperator/go.sum @@ -1,6 +1,7 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -30,9 +31,11 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= @@ -70,6 +73,7 @@ github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTe github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA= github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y= github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= @@ -83,7 +87,9 @@ github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4Y github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE= github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4= +github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= @@ -116,9 +122,13 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -126,6 +136,7 @@ github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85 github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -154,6 +165,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -165,7 +177,9 @@ github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4Ul github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -175,18 +189,27 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= github.com/testcontainers/testcontainers-go/modules/k3s v0.42.0 h1:bTVmcnYaSHesN6HXXxV/k0+BMkyfo3VBy4w4yRqOIgE= +github.com/testcontainers/testcontainers-go/modules/k3s v0.42.0/go.mod h1:2O8+V4WzMb/bjg/Sez+aYci9LpGUbT5cSz7ildfTIb8= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -200,6 +223,7 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -215,6 +239,7 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/HadesScheduler/go.sum b/HadesScheduler/go.sum index 744a3524..7224a7af 100644 --- a/HadesScheduler/go.sum +++ b/HadesScheduler/go.sum @@ -70,6 +70,7 @@ github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTe github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA= github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y= github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= @@ -83,7 +84,9 @@ github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4Y github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE= github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4= +github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/shared/buildlogs/interfaces.go b/shared/buildlogs/interfaces.go index 2e7095e4..e2b630d3 100644 --- a/shared/buildlogs/interfaces.go +++ b/shared/buildlogs/interfaces.go @@ -12,6 +12,7 @@ type LogAggregator interface { FlushJob(jobID string) error GetJobLogs(jobID string) []Log GetAllJobs() []string + SendJobLogs(jobID string) error MarkJobCompleted(jobID string) UpdateJobStatus(jobID string, status buildstatus.JobStatus) GetJobStatus(jobID string) (buildstatus.JobStatus, error) From 43f940be893563ce0dd2cdaba432f5c5433e0083 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 5 Jun 2026 17:40:32 +0200 Subject: [PATCH 31/36] update shared module reference to include SendJobLogs --- HadesLogManager/go.mod | 2 +- HadesLogManager/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HadesLogManager/go.mod b/HadesLogManager/go.mod index a8c0a66a..41c06554 100644 --- a/HadesLogManager/go.mod +++ b/HadesLogManager/go.mod @@ -42,7 +42,7 @@ require ( require ( github.com/klauspost/compress v1.18.6 // indirect - github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2 + github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556 github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.52.0 // indirect diff --git a/HadesLogManager/go.sum b/HadesLogManager/go.sum index 0282878e..be6830f8 100644 --- a/HadesLogManager/go.sum +++ b/HadesLogManager/go.sum @@ -45,8 +45,8 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2 h1:vrqgmPmtTBRBBZEAEZMKqcxQK5OFmM4IE7+nDUOKd2w= -github.com/ls1intum/hades/shared v0.0.0-20260505155304-a178a43cb5c2/go.mod h1:qB8+4591QBf959Hd6DGmE0ATXsF0SsR1UDTGLKCbC1M= +github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556 h1:kfdGL7z/A4o7rr3Vi8fUkqAC+i6am/IUMkBjNnPX5Jg= +github.com/ls1intum/hades/shared v0.0.0-20260528212350-4905a32aa556/go.mod h1:rB2Xc7vj0gHoGp8Jq91fE9rCakAfBFqNqlJfWwcQ72A= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From a0a53d2beed1a7c3f27e86780601f1bca0d112f4 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 5 Jun 2026 18:04:09 +0200 Subject: [PATCH 32/36] esolve shared module via replace directive for Docker build --- HadesLogManager/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/HadesLogManager/go.mod b/HadesLogManager/go.mod index 41c06554..56f4d805 100644 --- a/HadesLogManager/go.mod +++ b/HadesLogManager/go.mod @@ -1,4 +1,5 @@ module github.com/ls1intum/hades/hadesLogManager +replace github.com/ls1intum/hades/shared => ../shared go 1.25.0 From dc571139b13f7d7239f5716a3da1a3313541f885 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Fri, 5 Jun 2026 20:02:02 +0200 Subject: [PATCH 33/36] fix: use correct env var name for adapter URL in AggregatorConfig --- HadesLogManager/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HadesLogManager/processor.go b/HadesLogManager/processor.go index 70807eeb..411acb2f 100644 --- a/HadesLogManager/processor.go +++ b/HadesLogManager/processor.go @@ -40,7 +40,7 @@ type AggregatorConfig struct { BatchSize int `env:"LOG_BATCH_SIZE" envDefault:"100"` Retention time.Duration `env:"LOG_RETENTION" envDefault:"1h"` MaxJobLogs int `env:"MAX_JOB_LOGS" envDefault:"1000"` - APIendpoint string `env:"API_ENDPOINT"` + APIendpoint string `env:"ARTEMIS_ADAPTER_URL"` } // NewLogAggregator creates a new NATS-based LogAggregator instance with the specified configuration. From 85ee41d74ff5753053e8aa026c002130e58ec1f3 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Sat, 6 Jun 2026 00:04:19 +0200 Subject: [PATCH 34/36] inject JOB_NAME (participationId) into step container environment --- HadesScheduler/docker/job.go | 1 + 1 file changed, 1 insertion(+) diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index 2d815762..be9205fe 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -33,6 +33,7 @@ func (d Job) execute(ctx context.Context) error { maps.Copy(envs, d.Metadata) maps.Copy(envs, step.Metadata) envs["UUID"] = d.ID.String() + envs["JOB_NAME"] = d.Name step.Metadata = envs dockerStep := Step{ From dcfc29d166203d9ada2e96470e64740d08ca0bdb Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Sat, 4 Jul 2026 21:33:24 +0200 Subject: [PATCH 35/36] Support continueOnError for Kubernetes BuildJob steps Mirrors the Docker executor's continue-past-failure behavior: the init container script now appends "; exit 0" when the step is marked continueOnError, and the CRD schema is regenerated via make manifests so the field isn't pruned by the API server. --- HadesScheduler/HadesOperator/api/v1/buildjob_types.go | 5 +++++ .../internal/controller/buildjob_controller.go | 7 ++++++- HadesScheduler/k8s/k8s.go | 7 ++++--- helm/hades/crds/build.hades.tum.de_buildjobs.yaml | 6 ++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/HadesScheduler/HadesOperator/api/v1/buildjob_types.go b/HadesScheduler/HadesOperator/api/v1/buildjob_types.go index 658216b2..9d3964f8 100644 --- a/HadesScheduler/HadesOperator/api/v1/buildjob_types.go +++ b/HadesScheduler/HadesOperator/api/v1/buildjob_types.go @@ -89,6 +89,11 @@ type BuildStep struct { // Example: "set -e && cd /shared/example && ./gradlew clean test" Script string `json:"script,omitempty"` + // If true, the BuildJob pipeline continues to the next step even if this step's container + // exits with a non-zero status. Useful for steps that must always run regardless of prior + // failures, such as result parsing or cleanup. + ContinueOnError bool `json:"continueOnError,omitempty"` + // Step-specific key-value pairs injected as environment variables into this step's container. // Use for step-specific configuration like repository URLs, file paths, or credentials. // Variables can reference placeholders (e.g., "{{user}}") that get substituted at runtime. diff --git a/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go b/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go index cd11cc5b..bc63885a 100644 --- a/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go +++ b/HadesScheduler/HadesOperator/internal/controller/buildjob_controller.go @@ -343,13 +343,18 @@ func buildK8sJob(bj *buildv1.BuildJob, jobName string, deleteOnComplete bool, su Env: append( envFromMeta(s.Metadata), corev1.EnvVar{Name: "UUID", Value: bj.Name}, + corev1.EnvVar{Name: "JOB_NAME", Value: bj.Spec.Name}, ), // Convert metadata to environment variables VolumeMounts: []corev1.VolumeMount{sharedMount}, } if strings.TrimSpace(s.Script) != "" { c.Command = []string{"/bin/sh", "-c"} - c.Args = []string{s.Script} + if s.ContinueOnError { + c.Args = []string{s.Script + "; exit 0"} + } else { + c.Args = []string{s.Script} + } } // Set resource limits if specified diff --git a/HadesScheduler/k8s/k8s.go b/HadesScheduler/k8s/k8s.go index a04d3fef..33fd5bc9 100644 --- a/HadesScheduler/k8s/k8s.go +++ b/HadesScheduler/k8s/k8s.go @@ -212,9 +212,10 @@ func (k Scheduler) createBuildJobCR(ctx context.Context, job payload.QueuePayloa steps := make([]map[string]interface{}, 0, len(job.Steps)) for _, s := range job.Steps { sm := map[string]interface{}{ - "id": s.ID, - "name": s.Name, - "image": s.Image, + "id": s.ID, + "name": s.Name, + "image": s.Image, + "continueOnError": s.ContinueOnError, } if s.Script != "" { sm["script"] = s.Script diff --git a/helm/hades/crds/build.hades.tum.de_buildjobs.yaml b/helm/hades/crds/build.hades.tum.de_buildjobs.yaml index 7f8e006d..97df19f2 100644 --- a/helm/hades/crds/build.hades.tum.de_buildjobs.yaml +++ b/helm/hades/crds/build.hades.tum.de_buildjobs.yaml @@ -70,6 +70,12 @@ spec: Typical pipeline: Step 1 clones code, Step 2 builds it, Step 3 runs tests. items: properties: + continueOnError: + description: |- + If true, the BuildJob pipeline continues to the next step even if this step's container + exits with a non-zero status. Useful for steps that must always run regardless of prior + failures, such as result parsing or cleanup. + type: boolean cpuLimit: anyOf: - type: integer From 2ed83a3eba3812ef24e240eab7f48d73bc37ec48 Mon Sep 17 00:00:00 2001 From: Pao Xin Tan Date: Sun, 5 Jul 2026 19:43:19 +0200 Subject: [PATCH 36/36] async log sends, docker continueOnError, adapter env var --- HadesLogManager/log_subscriber.go | 13 ++++++++++--- HadesScheduler/docker/job.go | 13 +++++++++++-- compose.yml | 2 +- docker-compose.dev.yml | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/HadesLogManager/log_subscriber.go b/HadesLogManager/log_subscriber.go index 5d0bbbc0..4172c25d 100644 --- a/HadesLogManager/log_subscriber.go +++ b/HadesLogManager/log_subscriber.go @@ -236,8 +236,15 @@ func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) { watcher.wg.Wait() // Wait outside the lock dlm.logAggregator.MarkJobCompleted(jobID) - if err := dlm.logAggregator.SendJobLogs(jobID); err != nil { - slog.Error("Failed to send job logs", "job_id", jobID, "error", err) - } + go func() { + defer func() { + if r := recover(); r != nil { + slog.Error("Panic while sending job logs", "job_id", jobID, "panic", r) + } + }() + if err := dlm.logAggregator.SendJobLogs(jobID); err != nil { + slog.Error("Failed to send job logs", "job_id", jobID, "error", err) + } + }() } } diff --git a/HadesScheduler/docker/job.go b/HadesScheduler/docker/job.go index be9205fe..9a038c00 100644 --- a/HadesScheduler/docker/job.go +++ b/HadesScheduler/docker/job.go @@ -23,6 +23,11 @@ type Job struct { type jobIDContextKey string func (d Job) execute(ctx context.Context) error { + // Failures from steps marked ContinueOnError are tolerated by design (e.g. a test step whose + // failure is a normal, expected outcome that a later step, such as result parsing, still needs + // to report). They're joined here purely for logging and must not fail the overall job - that + // would defeat the point of ContinueOnError and would be inconsistent with the Kubernetes + // executor, where such a step never fails the job at all. stepErr := error(nil) for _, step := range d.Steps { @@ -48,7 +53,7 @@ func (d Job) execute(ctx context.Context) error { err := dockerStep.execute(stepCtx) if err != nil { d.logger.Error("Failed to execute step", slog.Any("error", err)) - if step.ContinueOnError == true { + if step.ContinueOnError { d.logger.Info("Next step should be executed despite error due to ContinueOnError setting", slog.Any("step", step)) stepErr = errors.Join(stepErr, fmt.Errorf("step %v failed with ContinueOnError set: %w", step.ID, err)) continue @@ -56,5 +61,9 @@ func (d Job) execute(ctx context.Context) error { return err } } - return stepErr + + if stepErr != nil { + d.logger.Warn("Job completed with tolerated step failures", slog.Any("error", stepErr)) + } + return nil } diff --git a/compose.yml b/compose.yml index 7983d00c..76f7844c 100644 --- a/compose.yml +++ b/compose.yml @@ -54,7 +54,7 @@ services: condition: service_started environment: - HADESLOGMANAGER_API_PORT=8081 - - API_ENDPOINT=${API_ENDPOINT} + - ARTEMIS_ADAPTER_URL=${ARTEMIS_ADAPTER_URL} - NATS_URL=nats://nats:4222 nats: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index adbb52b8..57f2fe99 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -22,5 +22,5 @@ services: environment: - DEBUG=true - HADESLOGMANAGER_API_PORT=8081 - - API_ENDPOINT=${ARTEMIS_ADAPTER_URL} + - ARTEMIS_ADAPTER_URL=${ARTEMIS_ADAPTER_URL} \ No newline at end of file