From e9985d64918b31d39d026fb85667379ac79cc78c Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 23 Aug 2024 14:13:20 +0200 Subject: [PATCH 1/9] Health endpoint --- src/cmd/delegation_backend/main_bpu.go | 19 +++++++++++++++++++ src/delegation_backend/submit.go | 1 + 2 files changed, 20 insertions(+) diff --git a/src/cmd/delegation_backend/main_bpu.go b/src/cmd/delegation_backend/main_bpu.go index a4ab983..8c56995 100644 --- a/src/cmd/delegation_backend/main_bpu.go +++ b/src/cmd/delegation_backend/main_bpu.go @@ -3,6 +3,7 @@ package main import ( . "block_producers_uptime/delegation_backend" "context" + "encoding/json" "net/http" "time" @@ -14,6 +15,11 @@ import ( sheets "google.golang.org/api/sheets/v4" ) +// HealthStatus represents the JSON response structure for the /health endpoint +type HealthStatus struct { + Status string `json:"status"` +} + func main() { // Setup logging logging.SetupLogging(logging.Config{ @@ -30,6 +36,7 @@ func main() { ctx := context.Background() appCfg := LoadEnv(log) app := new(App) + app.IsReady = false app.Log = log awsctx := AwsContext{} kc := KeyspaceContext{} @@ -118,6 +125,17 @@ func main() { }) http.Handle("/v1/submit", app.NewSubmitH()) + // Health check endpoint + http.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) { + if app.IsReady { + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(HealthStatus{Status: "ok"}) + } else { + rw.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(rw).Encode(HealthStatus{Status: "unavailable"}) + } + }) + // Sheets service and whitelist loop app.WhitelistDisabled = appCfg.DelegationWhitelistDisabled if app.WhitelistDisabled { @@ -150,5 +168,6 @@ func main() { } // Start server + app.IsReady = true log.Fatal(http.ListenAndServe(DELEGATION_BACKEND_LISTEN_TO, nil)) } diff --git a/src/delegation_backend/submit.go b/src/delegation_backend/submit.go index b4324a7..9138277 100644 --- a/src/delegation_backend/submit.go +++ b/src/delegation_backend/submit.go @@ -107,6 +107,7 @@ type App struct { NetworkId uint8 Save func(ObjectsToSave) Now nowFunc + IsReady bool } type SubmitH struct { From 55f0d7c9dbc364bdec2d640f4ed7a233ac6f5816 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 23 Aug 2024 14:25:36 +0200 Subject: [PATCH 2/9] extract health handler into function --- src/cmd/delegation_backend/main_bpu.go | 18 +++--------------- src/delegation_backend/health.go | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 src/delegation_backend/health.go diff --git a/src/cmd/delegation_backend/main_bpu.go b/src/cmd/delegation_backend/main_bpu.go index 8c56995..7e83d65 100644 --- a/src/cmd/delegation_backend/main_bpu.go +++ b/src/cmd/delegation_backend/main_bpu.go @@ -3,7 +3,6 @@ package main import ( . "block_producers_uptime/delegation_backend" "context" - "encoding/json" "net/http" "time" @@ -15,11 +14,6 @@ import ( sheets "google.golang.org/api/sheets/v4" ) -// HealthStatus represents the JSON response structure for the /health endpoint -type HealthStatus struct { - Status string `json:"status"` -} - func main() { // Setup logging logging.SetupLogging(logging.Config{ @@ -126,15 +120,9 @@ func main() { http.Handle("/v1/submit", app.NewSubmitH()) // Health check endpoint - http.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) { - if app.IsReady { - rw.WriteHeader(http.StatusOK) - json.NewEncoder(rw).Encode(HealthStatus{Status: "ok"}) - } else { - rw.WriteHeader(http.StatusServiceUnavailable) - json.NewEncoder(rw).Encode(HealthStatus{Status: "unavailable"}) - } - }) + http.HandleFunc("/health", HealthHandler(func() bool { + return app.IsReady + })) // Sheets service and whitelist loop app.WhitelistDisabled = appCfg.DelegationWhitelistDisabled diff --git a/src/delegation_backend/health.go b/src/delegation_backend/health.go new file mode 100644 index 0000000..e694dc5 --- /dev/null +++ b/src/delegation_backend/health.go @@ -0,0 +1,24 @@ +package delegation_backend + +import ( + "encoding/json" + "net/http" +) + +// HealthStatus represents the JSON response structure for the /health endpoint +type HealthStatus struct { + Status string `json:"status"` +} + +// HealthHandler handles the /health endpoint, checking if the application is ready. +func HealthHandler(isReady func() bool) http.HandlerFunc { + return func(rw http.ResponseWriter, r *http.Request) { + if isReady() { + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(HealthStatus{Status: "ok"}) + } else { + rw.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(rw).Encode(HealthStatus{Status: "unavailable"}) + } + } +} From 235383f6747599b7264887d0f858bf0e5a780ba4 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Fri, 23 Aug 2024 14:25:49 +0200 Subject: [PATCH 3/9] add tests --- src/delegation_backend/health_test.go | 81 +++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/delegation_backend/health_test.go diff --git a/src/delegation_backend/health_test.go b/src/delegation_backend/health_test.go new file mode 100644 index 0000000..4b5bfca --- /dev/null +++ b/src/delegation_backend/health_test.go @@ -0,0 +1,81 @@ +package delegation_backend + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// Mock App structure with IsReady flag +type MockApp struct { + IsReady bool +} + +// TestHealthEndpointBeforeReady tests the /health endpoint before the application is ready. +func TestHealthEndpointBeforeReady(t *testing.T) { + app := &MockApp{IsReady: false} + + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := HealthHandler(func() bool { return app.IsReady }) + + handler.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusServiceUnavailable { + t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusServiceUnavailable) + } + +} + +// TestHealthEndpointAfterReady tests the /health endpoint after the application is ready. +func TestHealthEndpointAfterReady(t *testing.T) { + app := &MockApp{IsReady: true} + + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := HealthHandler(func() bool { return app.IsReady }) + + handler.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK) + } + +} + +// TestHealthEndpointTransition tests the /health endpoint during the transition from not ready to ready. +func TestHealthEndpointTransition(t *testing.T) { + app := &MockApp{IsReady: false} + + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + handler := HealthHandler(func() bool { return app.IsReady }) + + // Initially not ready + handler.ServeHTTP(rr, req) + if status := rr.Code; status != http.StatusServiceUnavailable { + t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusServiceUnavailable) + } + + // Simulate the application becoming ready + app.IsReady = true + rr = httptest.NewRecorder() // Reset the recorder + handler.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK) + } + +} From a332ccd347dec1ec17f059dfc60ac7c1d2067894 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Mon, 26 Aug 2024 11:20:09 +0200 Subject: [PATCH 4/9] don't run tests on tagging --- .github/workflows/build.yml | 4 ++-- .github/workflows/integration.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6800db7..82dfa05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,8 +4,8 @@ on: push: branches: - main - tags: - - '*' + # tags: + # - '*' pull_request: branches: - main diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8521348..edd04b8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,8 +4,8 @@ on: push: branches: - main - tags: - - '*' + # tags: + # - '*' pull_request: branches: - main From 65b1e9d69498512bde88000bc02a9b579d595b6a Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Mon, 26 Aug 2024 11:34:42 +0200 Subject: [PATCH 5/9] check local first --- src/integration_tests/integration_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/integration_tests/integration_test.go b/src/integration_tests/integration_test.go index 4cd24a2..b579f7f 100644 --- a/src/integration_tests/integration_test.go +++ b/src/integration_tests/integration_test.go @@ -71,11 +71,6 @@ func TestIntegration_BP_Uptime_Storage(t *testing.T) { miniminaNetworkStart(networkName) defer miniminaNetworkStop(networkName) - err = WaitUntilPostgresHasSubmissions(postgresConn) - if err != nil { - t.Fatalf("Failed to wait until Postgres has submissions: %v", err) - } - err = waitUntilLocalStorageHasBlocksAndSubmissions(uptimeStorageDir) defer emptyLocalFilesystemStorage(uptimeStorageDir) if err != nil { @@ -88,6 +83,11 @@ func TestIntegration_BP_Uptime_Storage(t *testing.T) { t.Fatalf("Failed to wait until S3 bucket is not empty: %v", err) } + err = WaitUntilPostgresHasSubmissions(postgresConn) + if err != nil { + t.Fatalf("Failed to wait until Postgres has submissions: %v", err) + } + // err = waitUntilKeyspacesHasBlocksAndSubmissions(config) // if err != nil { // t.Fatalf("Failed to wait until Keyspaces has blocks and submissions: %v", err) From 2a822de967bf4c9c4a5a999af92e5b7102dc917f Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Sep 2024 16:55:44 +0200 Subject: [PATCH 6/9] integration test: wait for the PostgreSQL to be ready --- src/integration_tests/postgres_helper.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/integration_tests/postgres_helper.go b/src/integration_tests/postgres_helper.go index 7bb2d29..0614bd0 100644 --- a/src/integration_tests/postgres_helper.go +++ b/src/integration_tests/postgres_helper.go @@ -53,6 +53,22 @@ func StartPostgresContainerAndSetupSchema(config delegation_backend.PostgreSQLCo return nil, fmt.Errorf("failed to connect to the database: %v", err) } + // wait for the PostgreSQL to be ready + timeout := time.After(TIMEOUT_IN_S * time.Second) + tick := time.Tick(5 * time.Second) + ready := false + for !ready { + select { + case <-timeout: + return nil, fmt.Errorf("timeout reached while waiting for PostgreSQL to be ready") + case <-tick: + if err = db.Ping(); err == nil { + log.Println("PostgreSQL is ready") + ready = true + } + } + } + submissions_schema := `CREATE TABLE IF NOT EXISTS submissions ( id SERIAL PRIMARY KEY, submitted_at_date DATE NOT NULL, From 8b37121b4e711492e44fd00f0c952323822fff8b Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Sep 2024 08:12:57 +0200 Subject: [PATCH 7/9] timeout-minutes: 30 on integration test step workflow --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index edd04b8..2ebca0d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -45,6 +45,7 @@ jobs: # UPTIME_SERVICE_SECRET is a passphrase needed to decrypt uptime service config files UPTIME_SERVICE_SECRET: ${{ secrets.UPTIME_SERVICE_SECRET }} run: nix-shell --run "make integration-test" + timeout-minutes: 30 - name: 📖 Get logs if: always() From be3b4bb9fc52a25c26a52398d2e6602bcce0eeaf Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Sep 2024 13:28:32 +0200 Subject: [PATCH 8/9] remove 1 node from topology --- .../topology/genesis_keys/node-c-key.json | 1 - test/integration/topology/genesis_ledger.json | 5 ----- .../topology/libp2p_keys/node-c.json | 1 - .../topology/libp2p_keys/node-c.json.peerid | 1 - test/integration/topology/topology.json | 22 ------------------- 5 files changed, 30 deletions(-) delete mode 100644 test/integration/topology/genesis_keys/node-c-key.json delete mode 100644 test/integration/topology/libp2p_keys/node-c.json delete mode 100644 test/integration/topology/libp2p_keys/node-c.json.peerid diff --git a/test/integration/topology/genesis_keys/node-c-key.json b/test/integration/topology/genesis_keys/node-c-key.json deleted file mode 100644 index 86a09b9..0000000 --- a/test/integration/topology/genesis_keys/node-c-key.json +++ /dev/null @@ -1 +0,0 @@ -{"keypair":"B62qoztWpP8cJbknVWCcW2u56Y44wLNqpsompb5dDGoG1o6nfeatTKX","keypair_name":"node-c-key","privkey_password":"naughty blue worm","public_key":"B62qoztWpP8cJbknVWCcW2u56Y44wLNqpsompb5dDGoG1o6nfeatTKX","private_key":"{\"box_primitive\":\"xsalsa20poly1305\",\"pw_primitive\":\"argon2i\",\"nonce\":\"8gdecixWQ1MkXZ6g2vmScBcdmcL5dwYh3ZYpomN\",\"pwsalt\":\"9RPqeGfYJrXUnNEqTVwgy6P6aig9\",\"pwdiff\":[134217728,6],\"ciphertext\":\"CBEicGNcMMMmmPuezZ6zutQvDfYXixisDgjgBhMMXUJCt3wPEHYQHrmrLB5trzbcQT683ZpTc\"}"} \ No newline at end of file diff --git a/test/integration/topology/genesis_ledger.json b/test/integration/topology/genesis_ledger.json index 65b60f2..22c47a8 100644 --- a/test/integration/topology/genesis_ledger.json +++ b/test/integration/topology/genesis_ledger.json @@ -40,11 +40,6 @@ "sk": "EKF5AkJQX1B6CgmmaXgsyw4tZFp1M27TRcJrzw67Mtnp7o45rEvv", "balance": "300000" }, - { - "pk": "B62qoztWpP8cJbknVWCcW2u56Y44wLNqpsompb5dDGoG1o6nfeatTKX", - "sk": "EKF44LaJVQKTT1iuSJ3XifSFFtBndMb741kHPzVcYAtYEyJbZ11s", - "balance": "300000" - }, { "pk": "B62qjwzawZqMmm27zwDg5xF8XtrH1TcQedsS3EVStct8wr1FcpRZFbm", "sk": "EKFBPN3My4Y5GqaYj9pMYvtfzvrzuSQyvTEgxTZmCx4QZg7vJePB", diff --git a/test/integration/topology/libp2p_keys/node-c.json b/test/integration/topology/libp2p_keys/node-c.json deleted file mode 100644 index 3307083..0000000 --- a/test/integration/topology/libp2p_keys/node-c.json +++ /dev/null @@ -1 +0,0 @@ -{"box_primitive":"xsalsa20poly1305","pw_primitive":"argon2i","nonce":"7VnXm6xTdL6vtBBBopMf7wpmQFNnGrDnYkhMCun","pwsalt":"9Jisfm3uHzfX7fbnPhVBd3vCr98y","pwdiff":[134217728,6],"ciphertext":"8zWqpCUDvRoBhPnXdp7QRmwVPUivDJQh2rQ5NUkXMMHrNyhPQfUHtAjtNeCcwni9kBjP7ZGtdDckiPjYSL4YKApLmu7NtqD8RbnW9GDUQfrtmG28hMfqhJLcTTR3bX3YUZahppiDHoFF86x2vYHjXU9gzwspseJZcqS7sLH9bwKkoEoFj1LrC8SXoNhUseR3sJSJzmcfs3Hbcc6fjG3TsvcdPF7vY793p96L8FTJXePmmGjYAGnNhkQuFjKx9apA1nbuHcWqC3VowzyTd1UWU7N6cSutqTupswgyf"} \ No newline at end of file diff --git a/test/integration/topology/libp2p_keys/node-c.json.peerid b/test/integration/topology/libp2p_keys/node-c.json.peerid deleted file mode 100644 index cc986cc..0000000 --- a/test/integration/topology/libp2p_keys/node-c.json.peerid +++ /dev/null @@ -1 +0,0 @@ -12D3KooWC3KLTGZygn5MYukqR1yVtSNR9mzM6BaaGKjoJTaaP4oi \ No newline at end of file diff --git a/test/integration/topology/topology.json b/test/integration/topology/topology.json index 3677aec..2a71cf3 100644 --- a/test/integration/topology/topology.json +++ b/test/integration/topology/topology.json @@ -66,28 +66,6 @@ }, "libp2p_peerid": "12D3KooWGFjBhvkwj6jGTfhQxWMq4rTvw22vgbgnc2KPF4xhNhAv" }, - "node-c": { - "pk": "B62qoztWpP8cJbknVWCcW2u56Y44wLNqpsompb5dDGoG1o6nfeatTKX", - "sk": "EKF44LaJVQKTT1iuSJ3XifSFFtBndMb741kHPzVcYAtYEyJbZ11s", - "privkey_path": "../../test/integration/topology/block_producer_keys/node-c.json", - "role": "Block_producer", - "docker_image": "gcr.io/o1labs-192920/mina-daemon:2.0.0berkeley-rc1-1551e2f-bullseye-berkeley", - "git_build": null, - "libp2p_pass": "naughty blue worm", - "libp2p_keyfile": "../../test/integration/topology/libp2p_keys/node-c.json", - "libp2p_keypair": { - "box_primitive": "xsalsa20poly1305", - "pw_primitive": "argon2i", - "nonce": "7VnXm6xTdL6vtBBBopMf7wpmQFNnGrDnYkhMCun", - "pwsalt": "9Jisfm3uHzfX7fbnPhVBd3vCr98y", - "pwdiff": [ - 134217728, - 6 - ], - "ciphertext": "8zWqpCUDvRoBhPnXdp7QRmwVPUivDJQh2rQ5NUkXMMHrNyhPQfUHtAjtNeCcwni9kBjP7ZGtdDckiPjYSL4YKApLmu7NtqD8RbnW9GDUQfrtmG28hMfqhJLcTTR3bX3YUZahppiDHoFF86x2vYHjXU9gzwspseJZcqS7sLH9bwKkoEoFj1LrC8SXoNhUseR3sJSJzmcfs3Hbcc6fjG3TsvcdPF7vY793p96L8FTJXePmmGjYAGnNhkQuFjKx9apA1nbuHcWqC3VowzyTd1UWU7N6cSutqTupswgyf" - }, - "libp2p_peerid": "12D3KooWC3KLTGZygn5MYukqR1yVtSNR9mzM6BaaGKjoJTaaP4oi" - }, "uptime-service-backend": { "role": "Uptime_service_backend", "app_config_path": "../../test/integration/topology/uptime_service_config/app_config.json", From 5ded609199c3e32cc3243e79a3dd694b824c0a2b Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Sep 2024 13:46:56 +0200 Subject: [PATCH 9/9] update integration workflow - remove node-c --- .github/workflows/integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2ebca0d..cdc7957 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,7 +56,6 @@ jobs: minimina node logs -n integration-test -i uptime-service-backend -r > integration-test/logs/uptime-service-backend.log minimina node logs -n integration-test -i node-a -r > integration-test/logs/node-a.log minimina node logs -n integration-test -i node-b -r > integration-test/logs/node-b.log - minimina node logs -n integration-test -i node-c -r > integration-test/logs/node-c.log - name: 📎 Upload logs uses: actions/upload-artifact@v3