-
Notifications
You must be signed in to change notification settings - Fork 56
feat: PostgreSQL Full Migration #315
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| [alembic] | ||
| script_location = migrations/alembic | ||
| sqlalchemy.url = postgresql+asyncpg://postgres:postgres@localhost/solfoundry | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| """SQLAlchemy ORM tables for payouts, buybacks, reputation (Issue #162). | ||
|
|
||
| Registered with the shared ``Base`` from ``app.database`` so | ||
| ``Base.metadata.create_all`` creates them during ``init_db()``. | ||
| """ | ||
|
|
||
| import uuid | ||
| from datetime import datetime, timezone | ||
|
|
||
| from sqlalchemy import Boolean, Column, DateTime, Float, Index, Integer, String | ||
| from sqlalchemy.dialects.postgresql import UUID | ||
|
|
||
| from app.database import Base | ||
|
|
||
|
|
||
| class PayoutTable(Base): | ||
| """On-chain payout records with tx_hash uniqueness constraint.""" | ||
|
|
||
| __tablename__ = "payouts" | ||
| id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) | ||
| recipient = Column(String(100), nullable=False, index=True) | ||
| recipient_wallet = Column(String(64)) | ||
| amount = Column(Float, nullable=False) | ||
| token = Column(String(20), nullable=False, server_default="FNDRY") | ||
| bounty_id = Column(String(64), index=True) | ||
| bounty_title = Column(String(200)) | ||
| tx_hash = Column(String(128), unique=True, index=True) | ||
| status = Column(String(20), nullable=False, server_default="pending") | ||
| solscan_url = Column(String(256)) | ||
| created_at = Column(DateTime(timezone=True), nullable=False, | ||
| default=lambda: datetime.now(timezone.utc), index=True) | ||
|
|
||
|
|
||
| class BuybackTable(Base): | ||
| """Treasury buyback records (SOL exchanged for $FNDRY).""" | ||
|
|
||
| __tablename__ = "buybacks" | ||
| id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) | ||
| amount_sol = Column(Float, nullable=False) | ||
| amount_fndry = Column(Float, nullable=False) | ||
| price_per_fndry = Column(Float, nullable=False) | ||
| tx_hash = Column(String(128), unique=True, index=True) | ||
| solscan_url = Column(String(256)) | ||
| created_at = Column(DateTime(timezone=True), nullable=False, | ||
| default=lambda: datetime.now(timezone.utc), index=True) | ||
|
|
||
|
|
||
| class ReputationHistoryTable(Base): | ||
| """Per-bounty reputation entries with contributor+bounty uniqueness.""" | ||
|
|
||
| __tablename__ = "reputation_history" | ||
| id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) | ||
| contributor_id = Column(String(64), nullable=False, index=True) | ||
| bounty_id = Column(String(64), nullable=False, index=True) | ||
| bounty_title = Column(String(200), nullable=False) | ||
| bounty_tier = Column(Integer, nullable=False) | ||
| review_score = Column(Float, nullable=False) | ||
| earned_reputation = Column(Float, nullable=False, server_default="0") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🌐 Web query:
💡 Result: In SQLAlchemy, from sqlalchemy import Column, Float, text
value = Column(Float, nullable=False, server_default=text("0.0"))If you want to be explicit for PostgreSQL, you can cast:
SQLAlchemy documents Citations:
Use Line 58 uses earned_reputation = Column(Float, nullable=False, server_default=text("0.0"))This is consistent with SQLAlchemy best practices and ensures proper SQL type coercion for the Float column. 🤖 Prompt for AI Agents |
||
| anti_farming_applied = Column(Boolean, nullable=False, server_default="false") | ||
| created_at = Column(DateTime(timezone=True), nullable=False, | ||
| default=lambda: datetime.now(timezone.utc), index=True) | ||
| __table_args__ = ( | ||
| Index("ix_rep_cid_bid", "contributor_id", "bounty_id", unique=True),) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,17 @@ | ||
| """In-memory bounty service for MVP (Issue #3). | ||
| """Bounty service with PostgreSQL write-through persistence (Issue #162). | ||
|
|
||
| Provides CRUD operations and solution submission. | ||
| Claim lifecycle is out of scope (see Issue #16). | ||
| In-memory cache is the hot read path; every mutation is written through | ||
| to PostgreSQL via ``pg_store`` so the database is the durable source of | ||
| truth. Write errors are logged but do not block the API response | ||
| (fire-and-forget via ``asyncio.create_task``). | ||
| """ | ||
|
|
||
| import asyncio | ||
| import logging | ||
| from datetime import datetime, timezone | ||
| from typing import Optional | ||
| from app.core.audit import audit_event | ||
|
|
||
| from app.core.audit import audit_event | ||
| from app.models.bounty import ( | ||
| BountyCreate, | ||
| BountyDB, | ||
|
|
@@ -24,19 +28,37 @@ | |
| VALID_STATUS_TRANSITIONS, | ||
| ) | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # In-memory store (replaced by a database in production) | ||
| # --------------------------------------------------------------------------- | ||
| logger = logging.getLogger(__name__) | ||
|
|
||
| # In-memory cache (write-through to PostgreSQL) | ||
| _bounty_store: dict[str, BountyDB] = {} | ||
|
|
||
|
|
||
| def _fire_and_forget(coro) -> None: | ||
| """Schedule an async coroutine as a background task. | ||
|
|
||
| Attaches a done-callback that logs exceptions so write failures | ||
| are never silently swallowed. No-ops when called outside an | ||
| async context (e.g. synchronous tests). | ||
| """ | ||
| try: | ||
| loop = asyncio.get_running_loop() | ||
| task = loop.create_task(coro) | ||
| task.add_done_callback( | ||
| lambda t: logger.error("pg_store background write failed", exc_info=t.exception()) | ||
| if t.exception() else None | ||
| ) | ||
| except RuntimeError: | ||
| pass # No event loop (sync tests) | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Internal helpers | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| def _to_submission_response(s: SubmissionRecord) -> SubmissionResponse: | ||
| """Convert a SubmissionRecord to the public API response schema.""" | ||
| return SubmissionResponse( | ||
| id=s.id, | ||
| bounty_id=s.bounty_id, | ||
|
|
@@ -50,6 +72,7 @@ def _to_submission_response(s: SubmissionRecord) -> SubmissionResponse: | |
|
|
||
|
|
||
| def _to_bounty_response(b: BountyDB) -> BountyResponse: | ||
| """Convert a BountyDB record to the full API response schema.""" | ||
| subs = [_to_submission_response(s) for s in b.submissions] | ||
| return BountyResponse( | ||
| id=b.id, | ||
|
|
@@ -70,6 +93,7 @@ def _to_bounty_response(b: BountyDB) -> BountyResponse: | |
|
|
||
|
|
||
| def _to_list_item(b: BountyDB) -> BountyListItem: | ||
| """Convert a BountyDB record to a lightweight list item.""" | ||
| subs = [_to_submission_response(s) for s in b.submissions] | ||
| return BountyListItem( | ||
| id=b.id, | ||
|
|
@@ -105,6 +129,8 @@ def create_bounty(data: BountyCreate) -> BountyResponse: | |
| created_by=data.created_by, | ||
| ) | ||
| _bounty_store[bounty.id] = bounty | ||
| from app.services.pg_store import persist_bounty | ||
| _fire_and_forget(persist_bounty(bounty)) | ||
| return _to_bounty_response(bounty) | ||
|
Comment on lines
+132
to
134
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick | 🔵 Trivial Inline imports for pg_store are acceptable but create repetition. The imports inside each function ( ♻️ Alternative: lazy import at module level# At module level
_pg_store = None
def _get_pg_store():
global _pg_store
if _pg_store is None:
from app.services import pg_store as _pg_store
return _pg_store
# Then in functions:
_fire_and_forget(_get_pg_store().persist_bounty(bounty))🤖 Prompt for AI Agents |
||
|
|
||
|
|
||
|
|
@@ -186,14 +212,18 @@ def update_bounty( | |
| updated_by=bounty.created_by # In a real app, this would be the current user | ||
| ) | ||
|
|
||
| from app.services.pg_store import persist_bounty | ||
| _fire_and_forget(persist_bounty(bounty)) | ||
| return _to_bounty_response(bounty), None | ||
|
|
||
|
|
||
| def delete_bounty(bounty_id: str) -> bool: | ||
| """Delete a bounty by ID. Returns True if deleted, False if not found.""" | ||
| """Delete a bounty by ID. Removes from cache and PostgreSQL.""" | ||
| deleted = _bounty_store.pop(bounty_id, None) is not None | ||
| if deleted: | ||
| audit_event("bounty_deleted", bounty_id=bounty_id) | ||
| from app.services.pg_store import delete_bounty as pg_delete | ||
| _fire_and_forget(pg_delete(bounty_id)) | ||
| return deleted | ||
|
|
||
|
|
||
|
|
@@ -230,6 +260,8 @@ def submit_solution( | |
| ) | ||
| bounty.submissions.append(submission) | ||
| bounty.updated_at = datetime.now(timezone.utc) | ||
| from app.services.pg_store import persist_bounty | ||
| _fire_and_forget(persist_bounty(bounty)) | ||
| return _to_submission_response(submission), None | ||
|
|
||
|
|
||
|
|
@@ -271,7 +303,9 @@ def update_submission( | |
| submission_id=submission_id, | ||
| new_status=status | ||
| ) | ||
|
|
||
|
|
||
| from app.services.pg_store import persist_bounty | ||
| _fire_and_forget(persist_bounty(bounty)) | ||
| return _to_submission_response(sub), None | ||
|
|
||
| return None, "Submission not found" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| """In-memory contributor service for MVP.""" | ||
| """Contributor service with in-memory store (Issue #162: shared Base).""" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Contributors not persisted to PostgreSQL despite PR objectives. The docstring references Issue Contributors created/updated via this service will be lost on restart, while contributor data from GitHub sync (in 🤖 Prompt for AI Agents |
||
|
|
||
| import uuid | ||
| from datetime import datetime, timezone | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -327,6 +327,15 @@ async def sync_bounties() -> int: | |
| _bounty_store.clear() | ||
| _bounty_store.update(new_store) | ||
|
|
||
| # Persist synced bounties to PostgreSQL (write-through) | ||
| try: | ||
| from app.services.pg_store import persist_bounty | ||
|
|
||
| for bounty in new_store.values(): | ||
| await persist_bounty(bounty) | ||
| except Exception as exc: | ||
| logger.warning("DB persistence during sync failed: %s", exc) | ||
|
Comment on lines
+330
to
+337
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick | 🔵 Trivial Sequential persistence is inefficient; consider parallel execution. Persisting bounties sequentially with ♻️ Suggested improvement try:
from app.services.pg_store import persist_bounty
- for bounty in new_store.values():
- await persist_bounty(bounty)
+ await asyncio.gather(
+ *(persist_bounty(bounty) for bounty in new_store.values()),
+ return_exceptions=True
+ )
except Exception as exc:
logger.warning("DB persistence during sync failed: %s", exc)Additionally, 🤖 Prompt for AI Agents |
||
|
|
||
| _last_sync = datetime.now(timezone.utc) | ||
| logger.info("Synced %d bounties from GitHub Issues", len(new_store)) | ||
| return len(new_store) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,18 @@ | ||
| """In-memory payout service (MVP -- data lost on restart, DB coming later).""" | ||
| """Payout service with PostgreSQL write-through persistence (Issue #162). | ||
|
|
||
| In-memory cache is the hot read path. On startup ``hydrate_from_database`` | ||
| loads all rows from PostgreSQL so the database is the durable source of | ||
| truth. Every create operation fires a background write to PostgreSQL. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import asyncio | ||
| import logging | ||
| import threading | ||
| from typing import Optional | ||
| from app.core.audit import audit_event | ||
|
|
||
| from app.core.audit import audit_event | ||
| from app.models.payout import ( | ||
| BuybackCreate, | ||
| BuybackRecord, | ||
|
|
@@ -18,13 +25,52 @@ | |
| PayoutStatus, | ||
| ) | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| _lock = threading.Lock() | ||
| _payout_store: dict[str, PayoutRecord] = {} | ||
| _buyback_store: dict[str, BuybackRecord] = {} | ||
|
|
||
| SOLSCAN_TX_BASE = "https://solscan.io/tx" | ||
|
|
||
|
|
||
| async def hydrate_from_database() -> None: | ||
| """Load payouts and buybacks from PostgreSQL into in-memory cache. | ||
|
|
||
| Called once during application startup. Errors propagate so the | ||
| lifespan handler can log them and decide on fallback behaviour. | ||
| """ | ||
| from app.services.pg_store import load_payouts, load_buybacks | ||
|
|
||
| payouts = await load_payouts() | ||
| buybacks = await load_buybacks() | ||
| with _lock: | ||
| _payout_store.update(payouts) | ||
| _buyback_store.update(buybacks) | ||
|
|
||
|
|
||
| def _fire_db(record, kind: str) -> None: | ||
| """Schedule an async DB write as a background task. | ||
|
|
||
| Logs errors via a done-callback so failures are never silent. | ||
| """ | ||
| try: | ||
| loop = asyncio.get_running_loop() | ||
| if kind == "payout": | ||
| from app.services.pg_store import insert_payout | ||
| coro = insert_payout(record) | ||
| else: | ||
| from app.services.pg_store import insert_buyback | ||
| coro = insert_buyback(record) | ||
| task = loop.create_task(coro) | ||
| task.add_done_callback( | ||
| lambda t: logger.error("pg_store %s write failed", kind, exc_info=t.exception()) | ||
| if t.exception() else None | ||
| ) | ||
| except RuntimeError: | ||
| pass # No event loop (sync tests) | ||
|
Comment on lines
+52
to
+71
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick | 🔵 Trivial Missing type annotation for The 💡 Suggested improvement-def _fire_db(record, kind: str) -> None:
+def _fire_db(record: Any, kind: str) -> None:Add 🤖 Prompt for AI Agents |
||
|
|
||
|
|
||
| def _solscan_url(tx_hash: Optional[str]) -> Optional[str]: | ||
| """Return a Solscan explorer link for *tx_hash*, or ``None``.""" | ||
| if not tx_hash: | ||
|
|
@@ -92,6 +138,7 @@ def create_payout(data: PayoutCreate) -> PayoutResponse: | |
| token=record.token, | ||
| tx_hash=record.tx_hash | ||
| ) | ||
| _fire_db(record, "payout") | ||
| return _payout_to_response(record) | ||
|
|
||
|
|
||
|
|
@@ -174,6 +221,7 @@ def create_buyback(data: BuybackCreate) -> BuybackResponse: | |
| amount_fndry=record.amount_fndry, | ||
| tx_hash=record.tx_hash | ||
| ) | ||
| _fire_db(record, "buyback") | ||
| return _buyback_to_response(record) | ||
|
|
||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hardcoded database credentials in configuration file.
The
sqlalchemy.urlcontains hardcoded credentials (postgres:postgres). This poses security risks if the file is committed to version control, and creates inconsistency withbackend/app/database.pywhich usesos.getenv("DATABASE_URL", ...).Alembic supports environment variable interpolation via
env.py. The standard approach is to leavesqlalchemy.urlempty or use a placeholder, then override it inenv.pyusing the sameDATABASE_URLenvironment variable.🤖 Prompt for AI Agents