Skip to content

Commit

Permalink
Add scheduler to be able to schedule worker jobs to certain times
Browse files Browse the repository at this point in the history
  • Loading branch information
razzeee committed Jul 16, 2024
1 parent dfe6707 commit d97364f
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 24 deletions.
2 changes: 2 additions & 0 deletions backend/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,5 +80,7 @@ class Settings(BaseSettings):

backend_node_url: str | None = None

is_worker: bool = False


settings = Settings()
16 changes: 16 additions & 0 deletions backend/app/cron.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from apscheduler.triggers.cron import CronTrigger

JOBS = []


def cron(crontab):
"""Wrap a Dramatiq actor in a cron schedule."""
trigger = CronTrigger.from_crontab(crontab)

def decorator(actor):
module_path = actor.fn.__module__
func_name = actor.fn.__name__
JOBS.append((trigger, module_path, func_name))
return actor

return decorator
1 change: 0 additions & 1 deletion backend/app/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ def register_to_app(app: FastAPI):
async def update():
worker.update.send()
worker.update_quality_moderation.send()
worker.update_app_picks.send()


@router.post("/stats", tags=["update"])
Expand Down
15 changes: 15 additions & 0 deletions backend/app/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@
import requests
import sentry_dramatiq
import sentry_sdk
from apscheduler.schedulers.blocking import BlockingScheduler
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker

from . import (
apps,
config,
cron,
db,
exceptions,
logins,
Expand Down Expand Up @@ -325,6 +327,7 @@ def refresh_github_repo_list(gh_access_token: str, accountId: int):
sqldb.session.commit()


@cron.cron("0 3 * * *") # every day at 3am
@dramatiq.actor
def update_app_picks():
with WorkerDB() as sqldb:
Expand Down Expand Up @@ -387,3 +390,15 @@ def pick_app_of_the_day_automatically(sqldb, day):

if len(oldest_apps) > 0:
models.AppOfTheDay.set_app_of_the_day(sqldb, oldest_apps[0], day)


if settings.is_worker:
scheduler = BlockingScheduler()
for trigger, module_path, func_name in cron.JOBS:
job_path = f"{module_path}:{func_name}.send"
job_name = f"{module_path}.{func_name}"
scheduler.add_job(job_path, trigger=trigger, name=job_name)
try:
scheduler.start()
except KeyboardInterrupt:
scheduler.shutdown()
1 change: 1 addition & 0 deletions backend/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ services:
- SMTP_PASSWORD=$SMTP_PASSWORD
- EMAIL_FROM=$EMAIL_FROM
- BACKEND_NODE_URL=${BACKEND_NODE_URL-http://backend-node:8001}
- IS_WORKER=true
depends_on:
- redis
- db
Expand Down
92 changes: 69 additions & 23 deletions backend/poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions backend/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ sqlalchemy = "2.0.31"
sentry-dramatiq = "^0.3.3"
pydantic-settings = "^2.3.4"
psycopg2-binary = "^2.9.9"
apscheduler = "^3.10.4"

[tool.poetry.group.dev.dependencies]
httpx = "^0.27.0"
Expand Down

0 comments on commit d97364f

Please sign in to comment.