Repository navigation
.github/workflows/ixuca-smoke-watchdog.yml #49
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: IXUCA-Smoke-Watchdog | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "*/15 * * * *" | |
| permissions: | |
| actions: write | |
| contents: read | |
| concurrency: | |
| group: ixuca-smoke-watchdog | |
| cancel-in-progress: true | |
| jobs: | |
| check-and-alert: | |
| name: Check IXUCA smoke status and alert | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch previous watchdog state artifact | |
| id: fetch_state | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const artifactsResp = await github.rest.actions.listArtifactsForRepo({ | |
| owner, | |
| repo, | |
| per_page: 100 | |
| }); | |
| const artifacts = artifactsResp.data.artifacts || []; | |
| const candidates = artifacts | |
| .filter(a => a.name === "ixuca-smoke-watchdog-state" && !a.expired) | |
| .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); | |
| if (candidates.length > 0) { | |
| core.setOutput("artifact_id", String(candidates[0].id)); | |
| core.info(`Found previous state artifact: ${candidates[0].id}`); | |
| } else { | |
| core.info("No previous state artifact found."); | |
| } | |
| - name: Download previous watchdog state zip | |
| if: steps.fetch_state.outputs.artifact_id != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const artifactId = Number("${{ steps.fetch_state.outputs.artifact_id }}"); | |
| const fs = require("fs"); | |
| const response = await github.rest.actions.downloadArtifact({ | |
| owner, | |
| repo, | |
| artifact_id: artifactId, | |
| archive_format: "zip" | |
| }); | |
| fs.writeFileSync("state.zip", Buffer.from(response.data)); | |
| core.info("Downloaded previous state artifact to state.zip"); | |
| - name: Extract previous watchdog state | |
| if: steps.fetch_state.outputs.artifact_id != '' | |
| run: | | |
| python3 - <<'PY' | |
| import zipfile | |
| with zipfile.ZipFile("state.zip") as zf: | |
| if "state.json" not in zf.namelist(): | |
| raise SystemExit("state.json not found in artifact") | |
| with zf.open("state.json") as src, open("prev_state.json", "wb") as dst: | |
| dst.write(src.read()) | |
| print("Extracted previous state to prev_state.json") | |
| PY | |
| - name: Evaluate recent smoke runs | |
| id: evaluate | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const workflow_id = "ixuca-smoke-schedule.yml"; | |
| const fs = require("fs"); | |
| const now = new Date(); | |
| const oneDayMs = 24 * 60 * 60 * 1000; | |
| let prevState = { | |
| status: "UNKNOWN", | |
| last_alerted_at: null, | |
| last_alert_type: "" | |
| }; | |
| if (fs.existsSync("prev_state.json")) { | |
| try { | |
| prevState = JSON.parse(fs.readFileSync("prev_state.json", "utf8")); | |
| } catch (err) { | |
| core.warning(`Failed to parse prev_state.json: ${err.message}`); | |
| } | |
| } | |
| const runsResp = await github.rest.actions.listWorkflowRuns({ | |
| owner, | |
| repo, | |
| workflow_id, | |
| status: "completed", | |
| per_page: 10 | |
| }); | |
| const runs = runsResp.data.workflow_runs || []; | |
| const recent = runs.slice(0, 2); | |
| if (recent.length < 2) { | |
| core.info("Not enough completed runs to evaluate. Need at least 2."); | |
| core.setOutput("should_alert", "false"); | |
| return; | |
| } | |
| const latest = recent[0]; | |
| const previous = recent[1]; | |
| const schedulingIssue = | |
| latest.conclusion === "cancelled" && | |
| previous.conclusion === "cancelled"; | |
| const smokeFailed = latest.conclusion === "failure"; | |
| let shouldAlert = false; | |
| let alertType = ""; | |
| let alertText = ""; | |
| let status = "OK"; | |
| if (schedulingIssue) { | |
| status = "SCHEDULING_ISSUE"; | |
| } else if (smokeFailed) { | |
| status = "DOCKER_CHECK_FAILED"; | |
| } | |
| const prevAlertAt = prevState.last_alerted_at | |
| ? new Date(prevState.last_alerted_at) | |
| : null; | |
| const sameIssue = prevState.status === status && status !== "OK"; | |
| const canRepeat = prevAlertAt && now - prevAlertAt >= oneDayMs; | |
| if (status !== "OK") { | |
| if (sameIssue && prevAlertAt && !canRepeat) { | |
| shouldAlert = false; | |
| alertText = | |
| `已告警且状态未变化，暂不重复告警：status=${status}。\n` + | |
| `latest: ${latest.html_url}\n` + | |
| `previous: ${previous.html_url}`; | |
| } else { | |
| shouldAlert = true; | |
| alertType = status; | |
| if (status === "SCHEDULING_ISSUE") { | |
| alertText = | |
| `调度异常：最近两次已完成运行均为 cancelled。\n` + | |
| `latest: ${latest.html_url}\n` + | |
| `previous: ${previous.html_url}`; | |
| } else { | |
| alertText = | |
| `容器检查失败：最近一次运行为 failure。\n` + | |
| `run_url: ${latest.html_url}`; | |
| } | |
| if (sameIssue && canRepeat) { | |
| alertText = | |
| `持续未修复超过1天，重复告警。\n` + | |
| alertText; | |
| } | |
| } | |
| } else if (prevState.status && prevState.status !== "OK") { | |
| if (latest.conclusion === "success") { | |
| shouldAlert = true; | |
| alertType = "RECOVERED"; | |
| alertText = | |
| `问题已修复：previous=${prevState.status}，latest=${latest.conclusion}。\n` + | |
| `run_url: ${latest.html_url}`; | |
| } else { | |
| alertText = | |
| `问题尚未修复：previous=${prevState.status}，latest=${latest.conclusion}。\n` + | |
| `run_url: ${latest.html_url}`; | |
| } | |
| } else { | |
| alertText = | |
| `IXUCA-Smoke-Schedule 正常：latest=${latest.conclusion}, previous=${previous.conclusion}`; | |
| } | |
| core.info(alertText); | |
| core.setOutput("should_alert", shouldAlert ? "true" : "false"); | |
| core.setOutput("alert_type", alertType); | |
| core.setOutput("alert_text", alertText); | |
| const nextState = { | |
| status, | |
| updated_at: now.toISOString(), | |
| last_alerted_at: prevState.last_alerted_at || null, | |
| last_alert_type: prevState.last_alert_type || "" | |
| }; | |
| if (status === "OK") { | |
| nextState.last_alerted_at = null; | |
| nextState.last_alert_type = ""; | |
| } else if (shouldAlert && alertType !== "RECOVERED") { | |
| nextState.last_alerted_at = now.toISOString(); | |
| nextState.last_alert_type = alertType; | |
| } | |
| fs.writeFileSync("state.json", JSON.stringify(nextState, null, 2)); | |
| - name: Send alert to Baidu IM robot | |
| if: steps.evaluate.outputs.should_alert == 'true' | |
| env: | |
| WEBHOOK_URL: ${{ secrets.BAIDU_IM_WEBHOOK_URL }} | |
| ALERT_TEXT: ${{ steps.evaluate.outputs.alert_text }} | |
| ALERT_TYPE: ${{ steps.evaluate.outputs.alert_type }} | |
| TO_ID: "11992367" | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| import time | |
| import urllib.request | |
| webhook_url = os.environ.get("WEBHOOK_URL", "").strip() | |
| if not webhook_url: | |
| raise SystemExit("Need secret BAIDU_IM_WEBHOOK_URL for alerting.") | |
| alert_text = os.environ["ALERT_TEXT"] | |
| alert_type = os.environ["ALERT_TYPE"] | |
| to_id = int(os.environ.get("TO_ID", "11992367")) | |
| payload = { | |
| "message": { | |
| "header": { | |
| "toid": to_id, | |
| "totype": "GROUP", | |
| "msgtype": "TEXT", | |
| "clientmsgid": int(time.time() * 1000), | |
| "role": "robot" | |
| }, | |
| "body": [ | |
| { | |
| "content": f"[{alert_type}] {alert_text}", | |
| "type": "TEXT" | |
| } | |
| ] | |
| } | |
| } | |
| req = urllib.request.Request( | |
| webhook_url, | |
| data=json.dumps(payload).encode("utf-8"), | |
| headers={"Content-Type": "application/json"}, | |
| method="POST", | |
| ) | |
| with urllib.request.urlopen(req, timeout=20) as resp: | |
| print(f"alert sent, status={resp.status}") | |
| PY | |
| - name: Upload watchdog state | |
| if: steps.evaluate.conclusion == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ixuca-smoke-watchdog-state | |
| path: state.json | |
| retention-days: 7 |