feat(triggers): DLQ classification + UI Replay + instrumentation (PR-3)

.claude/skills/int-evolution-go/scripts/evolution_go_client.py

@@ -6,7 +6,11 @@
 import argparse
 import json
 import os
+import random
+import socket
 import sys
+import time
+import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
@@ -43,38 +47,104 @@ def get_config():
     return url.rstrip("/"), key
 
 
+def _retry_http_call_client(do_call, max_attempts=3, base_delay=2.0, max_delay=8.0):
+    """Exponential backoff + jitter for Evolution Go API calls.
+
+    Retries on HTTP 5xx, urllib.error.URLError, and socket.timeout (transient).
+    NEVER retries on HTTP 4xx (deterministic client errors).
+
+    Returns the result of do_call() on success.
+    Raises the last exception after max_attempts are exhausted.
+    Raises immediately on HTTP 4xx (no retry).
+    """
+    last_exc = None
+    for attempt in range(max_attempts):
+        try:
+            return do_call()
+        except urllib.error.HTTPError as e:
+            if e.code < 500:
+                # 4xx — deterministic, raise immediately (caller decides sys.exit vs raise)
+                raise
+            last_exc = e
+            if attempt < max_attempts - 1:
+                delay = min(base_delay ** attempt + random.uniform(0, 0.5), max_delay)
+                print(
+                    json.dumps({
+                        "evt": "api_request_retry",
+                        "attempt": attempt + 1,
+                        "max_attempts": max_attempts,
+                        "http_status": e.code,
+                        "delay_s": round(delay, 2),
+                    })
+                )
+                time.sleep(delay)
+            else:
+                print(
+                    json.dumps({
+                        "evt": "api_request_failed",
+                        "attempt": attempt + 1,
+                        "max_attempts": max_attempts,
+                        "http_status": e.code,
+                        "category": "transient",
+                    })
+                )
+        except (urllib.error.URLError, socket.timeout) as e:
+            last_exc = e
+            if attempt < max_attempts - 1:
+                delay = min(base_delay ** attempt + random.uniform(0, 0.5), max_delay)
+                print(
+                    json.dumps({
+                        "evt": "api_request_retry",
+                        "attempt": attempt + 1,
+                        "max_attempts": max_attempts,
+                        "error": str(e),
+                        "delay_s": round(delay, 2),
+                    })
+                )
+                time.sleep(delay)
+            else:
+                print(
+                    json.dumps({
+                        "evt": "api_request_failed",
+                        "attempt": attempt + 1,
+                        "max_attempts": max_attempts,
+                        "error": str(e),
+                        "category": "transient",
+                    })
+                )
+    raise last_exc
+
+
 def api_request(method, path, data=None):
-    """Make an HTTP request to the Evolution Go API."""
+    """Make an HTTP request to the Evolution Go API.
+
+    Applies exponential backoff + jitter on HTTP 5xx / network errors (up to 3 attempts).
+    On HTTP 4xx: raises urllib.error.HTTPError immediately (no retry, deterministic error).
+    On persistent failure after retries: raises the last exception instead of sys.exit(1),
+    allowing library callers to handle it; CLI __main__ catches and sys.exit(1) as before.
+    """
     base_url, api_key = get_config()
     url = f"{base_url}{path}"
 
     body = json.dumps(data).encode("utf-8") if data else None
-    req = urllib.request.Request(
-        url,
-        data=body,
-        method=method,
-        headers={
-            "apikey": api_key,
-            "Content-Type": "application/json",
-        },
-    )
 
-    try:
+    def _do_call():
+        req = urllib.request.Request(
+            url,
+            data=body,
+            method=method,
+            headers={
+                "apikey": api_key,
+                "Content-Type": "application/json",
+            },
+        )
         with urllib.request.urlopen(req) as resp:
             raw = resp.read()
             if raw:
                 return json.loads(raw)
             return {"message": "success"}
-    except urllib.error.HTTPError as e:
-        try:
-            error_body = json.loads(e.read())
-        except Exception:
-            error_body = {"error": str(e)}
-        print(json.dumps({"error": f"HTTP {e.code}", "details": error_body}, indent=2))
-        sys.exit(1)
-    except urllib.error.URLError as e:
-        print(json.dumps({"error": f"Connection failed: {e.reason}"}))
-        sys.exit(1)
+
+    return _retry_http_call_client(_do_call)
 
 
 def to_jid(number):
@@ -523,12 +593,23 @@ def main():
     }
 
     handler = commands.get(args.command)
-    if handler:
-        handler(args)
-    else:
+    if not handler:
         print(json.dumps({"error": f"Unknown command: {args.command}"}))
         sys.exit(1)
 
+    try:
+        handler(args)
+    except urllib.error.HTTPError as e:
+        try:
+            error_body = json.loads(e.read())
+        except Exception:
+            error_body = {"error": str(e)}
+        print(json.dumps({"error": f"HTTP {e.code}", "details": error_body}, indent=2))
+        sys.exit(1)
+    except (urllib.error.URLError, socket.timeout) as e:
+        print(json.dumps({"error": f"Connection failed: {e}"}))
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     main()

dashboard/backend/app.py

@@ -613,6 +613,43 @@ def _cors_allowed_origins():
         except Exception:
             pass
     _conn.commit()
+
+    # --- WhatsApp retry pattern: idempotency_key + error_category + last_replay_at (PR-1 2026-05-11) ---
+    # Rollback: DROP INDEX uq_trigger_idem; DROP INDEX ix_trigger_executions_idem_key
+    # Columns are nullable — old code ignores them without breaking.
+    _te_cols = {row[1] for row in _cur.execute("PRAGMA table_info(trigger_executions)").fetchall()}
+    if "idempotency_key" not in _te_cols:
+        _cur.execute("ALTER TABLE trigger_executions ADD COLUMN idempotency_key TEXT")
+        _conn.commit()
+    if "error_category" not in _te_cols:
+        _cur.execute("ALTER TABLE trigger_executions ADD COLUMN error_category TEXT")
+        _conn.commit()
+    if "last_replay_at" not in _te_cols:
+        _cur.execute("ALTER TABLE trigger_executions ADD COLUMN last_replay_at TIMESTAMP")
+        _conn.commit()
+    # Basic index for idempotency lookups by key alone
+    try:
+        _cur.execute(
+            "CREATE INDEX IF NOT EXISTS ix_trigger_executions_idem_key "
+            "ON trigger_executions (idempotency_key)"
+        )
+        _conn.commit()
+    except Exception:
+        pass
+    # Partial unique index: enforces (trigger_id, idempotency_key) uniqueness only when key IS NOT NULL.
+    # SQLite >= 3.8 supports partial indices natively; our runtime is 3.51 (confirmed).
+    # This is the DB-level guard against race-condition duplicates (Step 2 handles app-level dedup).
+    try:
+        _cur.execute(
+            "CREATE UNIQUE INDEX IF NOT EXISTS uq_trigger_idem "
+            "ON trigger_executions (trigger_id, idempotency_key) "
+            "WHERE idempotency_key IS NOT NULL"
+        )
+        _conn.commit()
+    except Exception:
+        pass
+    # --- End WhatsApp retry pattern migration ---
+
     _conn.close()
     # --- End auto-migrate ---
 

dashboard/backend/models.py

@@ -333,12 +333,17 @@ class TriggerExecution(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     trigger_id = db.Column(db.Integer, db.ForeignKey("triggers.id", ondelete="CASCADE"), nullable=False)
     event_data = db.Column(db.Text, nullable=True, default="{}")  # JSON payload received
-    status = db.Column(db.String(20), nullable=False, default="pending")  # pending, running, completed, failed
+    status = db.Column(db.String(20), nullable=False, default="pending")  # pending, running, completed, failed, failed_retryable
     result_summary = db.Column(db.Text, nullable=True)
     error = db.Column(db.Text, nullable=True)
     duration_seconds = db.Column(db.Float, nullable=True)
     started_at = db.Column(db.DateTime, default=lambda: datetime.now(timezone.utc))
     completed_at = db.Column(db.DateTime, nullable=True)
+    # WhatsApp retry pattern (PR-1: migration 2026-05-11)
+    # rollback: DROP indices uq_trigger_idem + ix_trigger_executions_idem_key; columns are nullable, ignored by old code
+    idempotency_key = db.Column(db.String(255), nullable=True, index=True)  # messageId WPP or other source dedup key
+    error_category = db.Column(db.String(20), nullable=True)  # transient | permanent | validation | unknown
+    last_replay_at = db.Column(db.DateTime, nullable=True)  # rate-limit: 60s between replays of the same execution
 
     @property
     def event_data_dict(self) -> dict:
@@ -358,6 +363,9 @@ def to_dict(self):
             "duration_seconds": self.duration_seconds,
             "started_at": self.started_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ") if self.started_at else None,
             "completed_at": self.completed_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ") if self.completed_at else None,
+            "idempotency_key": self.idempotency_key,
+            "error_category": self.error_category,
+            "last_replay_at": self.last_replay_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ") if self.last_replay_at else None,
         }