diff --git a/.github/workflows/defog_daily_update.yml b/.github/workflows/defog_daily_update.yml new file mode 100644 index 000000000..3dcd182cd --- /dev/null +++ b/.github/workflows/defog_daily_update.yml @@ -0,0 +1,72 @@ +name: Run DEFOG Daily Update + +on: + workflow_call: + secrets: + SF_USERNAME: + required: true + SF_PASSWORD: + required: true + SF_ACCOUNT: + required: true + +jobs: + defog-daily-update: + name: DEFOG Daily Update + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install Snowflake Connector + run: pip install snowflake-connector-python + + - name: Run DEFOG_DAILY_UPDATE if table is older than 1 day + env: + SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} + SF_USERNAME: ${{ secrets.SF_USERNAME }} + SF_PASSWORD: ${{ secrets.SF_PASSWORD }} + run: | + python << 'EOF' + import snowflake.connector + import os + + conn = snowflake.connector.connect( + user=os.environ['SF_USERNAME'], + password=os.environ['SF_PASSWORD'], + account=os.environ['SF_ACCOUNT'], + warehouse='DEMO_WH', + database='DEFOG', + schema='BROKER', + ) + + cursor = conn.cursor() + + # Check if table data is a day or more old. + # If so, run the stored procedure to update it. + cursor.execute(""" + DECLARE last_mod DATE; + + BEGIN + -- Get table last modified date + SELECT DATE(LAST_ALTERED) INTO last_mod + FROM INFORMATION_SCHEMA.TABLES + WHERE table_catalog='DEFOG' + AND table_schema = 'BROKER' + AND table_name = 'SBDAILYPRICE'; + + -- If last modified is before today, call the procedure + IF (last_mod < CURRENT_DATE()) THEN + CALL DEFOG.BROKER.DEFOG_DAILY_UPDATE(); + END IF; + END; + """) + + cursor.close() + conn.close() + EOF diff --git a/.github/workflows/pr_testing.yml b/.github/workflows/pr_testing.yml index fbd884ee5..d718f6bfb 100644 --- a/.github/workflows/pr_testing.yml +++ b/.github/workflows/pr_testing.yml @@ -150,9 +150,22 @@ jobs: - name: Run Tests run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked)" -rs + run-defog-daily-update: + name: Run DEFOG Daily Update + needs: [get-msg] + if: | + (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || + (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run sf]')) || + (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-sf)) + uses: ./.github/workflows/defog_daily_update.yml + secrets: + SF_USERNAME: ${{ secrets.SF_USERNAME }} + SF_PASSWORD: ${{ secrets.SF_PASSWORD }} + SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} + run-sf-tests: name: Snowflake Tests - needs: [get-msg, get-py-ver-matrix] + needs: [run-defog-daily-update, get-msg, get-py-ver-matrix] if: | (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run sf]')) || diff --git a/tests/conftest.py b/tests/conftest.py index 92add5437..2dcc763c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -728,25 +728,27 @@ def _impl(database_name: str, schema_name: str) -> DatabaseContext: database=database_name, schema=schema_name, ) - # Run DEFOG_DAILY_UPDATE() only if data is older than 1 day - with connection.cursor() as cur: - cur.execute(""" - DECLARE last_mod DATE; - - BEGIN - -- Get table last modified date - SELECT DATE(LAST_ALTERED) INTO last_mod - FROM INFORMATION_SCHEMA.TABLES - WHERE table_catalog='DEFOG' - AND table_schema = 'BROKER' - AND table_name = 'SBDAILYPRICE'; - - -- If last modified is before today, call the procedure - IF (last_mod < CURRENT_DATE()) THEN - CALL DEFOG.BROKER.DEFOG_DAILY_UPDATE(); - END IF; - END; - """) + + if not is_ci(): + # Run DEFOG_DAILY_UPDATE() only if data is older than 1 day + with connection.cursor() as cur: + cur.execute(""" + DECLARE last_mod DATE; + + BEGIN + -- Get table last modified date + SELECT DATE(LAST_ALTERED) INTO last_mod + FROM INFORMATION_SCHEMA.TABLES + WHERE table_catalog='DEFOG' + AND table_schema = 'BROKER' + AND table_name = 'SBDAILYPRICE'; + + -- If last modified is before today, call the procedure + IF (last_mod < CURRENT_DATE()) THEN + CALL DEFOG.BROKER.DEFOG_DAILY_UPDATE(); + END IF; + END; + """) return load_database_context("snowflake", connection=connection) diff --git a/tests/gen_data/sf_task.sql b/tests/gen_data/sf_task.sql index 7a62b0ee0..d9b8e53bb 100644 --- a/tests/gen_data/sf_task.sql +++ b/tests/gen_data/sf_task.sql @@ -527,6 +527,75 @@ VALUES ('TX054', 'C018', 'T004', DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '2 months' + INTERVAL '4 days', 'sell', 30, 180.00, 5400.00, 'USD', 27.00, 5.00, 'KP054', TO_CHAR(DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '2 months' + INTERVAL '4 days', '%Y%m%d %H:%i:%s'), 'success'), ('TX055', 'C019', 'T005', DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 month' + INTERVAL '5 days', 'buy', 10, 2500.00, 25000.00, 'USD', 125.00, 15.00, 'KP055', TO_CHAR(DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 month' + INTERVAL '5 days', '%Y%m%d %H:%i:%s'), 'success'), ('TX056', 'C002', 'T006', DATE_TRUNC('month', CURRENT_DATE) + INTERVAL '1 day', 'sell', 20, 200.00, 4000.00, 'USD', 20.00, 10.00, 'KP056', TO_CHAR(DATE_TRUNC('month', CURRENT_DATE) + INTERVAL '1 day', '%Y%m%d %H:%i:%s'), 'success'); +------------------------------------------------------------------------------- + DELETE FROM DEFOG.DERMTREATMENT.DOCTORS; + INSERT INTO DEFOG.DERMTREATMENT.DOCTORS (doc_id, first_name, last_name, specialty, year_reg, med_school_name, loc_city, loc_state, loc_zip, bd_cert_num) +VALUES +(1, 'John', 'Doe', 'dermatology', EXTRACT(YEAR FROM CURRENT_DATE) - 2, 'Johns Hopkins University', 'Baltimore', 'MD', '21201', 'ABC123'), +(2,'Jane', 'Smith', 'immunology', EXTRACT(YEAR FROM CURRENT_DATE) - 2, 'Harvard Medical School', 'Boston', 'MA', '02115', 'XYZ789'), +(3, 'David', 'Johnson', 'general', 1998, 'University of Pennsylvania', 'Philadelphia', 'PA', '19104', 'DEF456'), +(4, 'Emily', 'Brown', 'dermatology', 2015, 'Stanford University', 'Palo Alto', 'CA', '94304', 'GHI012'), +(5, 'Michael', 'Davis', 'immunology', 2008, 'Duke University', 'Durham', 'NC', '27708', 'JKL345'), +(6, 'Sarah', 'Wilson', 'oncology', EXTRACT(YEAR FROM CURRENT_DATE) - 1, 'University of California, San Francisco', 'San Francisco', 'CA', '94143', 'MNO678'), +(7, 'Robert', 'Taylor', 'dermatology', 2012, 'Yale University', 'New Haven', 'CT', '06510', 'PQR901'), +(8, 'Laura', 'Martinez', 'immunology', 2006, 'University of Michigan', 'Ann Arbor', 'MI', '48109', 'STU234'), +(9, 'Daniel', 'Garcia', 'general', EXTRACT(YEAR FROM CURRENT_DATE) - 3, 'University of Chicago', 'Chicago', 'IL', '60637', 'VWX567'), +(10, 'Olivia', 'Anderson', 'dermatology', 2018, 'Columbia University', 'New York', 'NY', '10027', 'YZA890'); + + DELETE FROM DEFOG.DERMTREATMENT.TREATMENTS; + INSERT INTO DEFOG.DERMTREATMENT.TREATMENTS (treatment_id, patient_id, doc_id, drug_id, diag_id, start_dt, end_dt, is_placebo, tot_drug_amt, drug_unit) +VALUES +(1, 1, 1, 1, 1, '2022-01-01', '2022-06-30', false, 240, 'mg'), +(2, 2, 2, 2, 2, '2022-02-15', '2022-08-14', true, 180, 'mg'), +(3, 3, 3, 3, 3, '2022-03-10', '2022-09-09', false, 360, 'g'), +(4, 4, 4, 4, 4, '2022-04-01', NULL, false, 200, 'mg'), +(5, 5, 5, 5, 5, '2022-05-01', '2022-10-31', false, 180, 'mg'), +(6, 6, 6, 6, 6, '2022-06-15', '2022-12-14', false, 720, 'g'), +(7, 1, 7, 1, 7, '2022-07-01', '2022-12-31', true, 240, 'mg'), +(8, 2, 1, 2, 8, '2022-08-01', '2023-01-31', false, 180, 'mg'), +(9, 3, 2, 3, 9, '2022-09-01', '2023-02-28', false, 360, 'g'), +(10, 4, 3, 4, 10, '2022-10-01', NULL, true, 0, NULL), +(11, 5, 4, 5, 1, '2022-11-01', '2023-04-30', true, 180, 'mg'), +(12, 6, 5, 6, 2, '2022-12-01', '2023-05-31', false, 720, 'g'), +(13, 7, 6, 1, 3, '2023-01-01', '2023-06-30', false, 240, 'mg'), +(14, 1, 7, 2, 4, '2023-02-01', '2023-07-31', false, 180, 'mg'), +(15, 2, 1, 3, 5, '2023-03-01', '2023-08-31', false, 360, 'g'), +(16, 1, 2, 4, 6, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '2 year', DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '2 months', false, 300, 'mg'), +(17, 2, 5, 1, 8, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 year', DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '4 months', false, 80, 'mg'), +(18, 3, 6, 2, 9, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '5 months', NULL, true, 200, 'mg'), +(19, 1, 7, 3, 10, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '4 months', NULL, false, 150, 'g'), +(20, 2, 1, 4, 1, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '3 months', NULL, false, 100, 'mg'), +(21, 3, 2, 5, 2, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '2 months', NULL, false, 250, 'mg'), +(22, 1, 3, 6, 3, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 month', NULL, false, 300, 'g'), +(23, 2, 4, 1, 4, CURRENT_DATE, NULL, true, 200, 'mg'), +(24, 3, 5, 2, 5, CURRENT_DATE, NULL, false, 150, 'mg'), +(25, 9, 1, 1, 1, CURRENT_DATE - INTERVAL '6 months', CURRENT_DATE - INTERVAL '3 months', false, 240, 'mg'), +(26, 10, 2, 2, 2, CURRENT_DATE - INTERVAL '5 months', CURRENT_DATE - INTERVAL '2 months', false, 180, 'mg'); + + DELETE FROM DEFOG.DERMTREATMENT.OUTCOMES; + INSERT INTO DEFOG.DERMTREATMENT.OUTCOMES (outcome_id, treatment_id, assess_dt, day7_lesion_cnt, day30_lesion_cnt, day100_lesion_cnt, day7_pasi_score, day30_pasi_score, day100_pasi_score, day7_tewl, day30_tewl, day100_tewl, day7_itch_vas, day30_itch_vas, day100_itch_vas, day7_hfg, day30_hfg, day100_hfg) +VALUES +(1, 1, '2022-01-08', 20, 15, 5, 12.5, 8.2, 2.1, 18.2, 15.6, 12.1, 60, 40, 20, 1.5, 2.5, 4.0), +(2, 2, '2022-02-22', 25, 18, 8, 15.0, 10.1, 3.5, 20.1, 17.2, 13.5, 70, 50, 30, 1.0, 2.0, 3.5), +(3, 3, '2022-03-17', 18, 12, 3, 10.8, 6.4, 1.2, 16.5, 14.0, 10.8, 55, 35, 15, 2.0, 3.0, 4.5), +(4, 4, '2022-04-08', 30, 25, 12, 18.2, 13.9, 5.8, 22.4, 19.1, 15.2, 80, 60, 40, 0.5, 1.5, 3.0), +(5, 5, '2022-05-08', 22, 16, 6, 13.1, 8.7, 2.6, 19.0, 16.3, 12.7, 65, 45, 25, 1.2, 2.2, 3.8), +(6, 6, '2022-06-22', 28, 21, 10, 16.7, 11.5, 4.3, 21.3, 18.1, 14.3, 75, 55, 35, 0.8, 1.8, 3.3), +(7, 7, '2022-07-08', 19, 13, 4, 11.2, 6.9, 1.5, 17.1, 14.5, 11.2, 58, 38, 18, 1.8, 2.8, 4.3), +(8, 8, '2022-08-08', 26, 19, 9, 15.6, 10.6, 3.8, 20.7, 17.6, 13.9, 72, 52, 32, 0.7, 1.7, 3.2), +(9, 9, '2022-09-08', 21, 15, 5, 12.3, 8.0, 2.0, 18.6, 15.9, 12.4, 62, 42, 22, 1.4, 2.4, 3.9), +(10, 10, '2022-10-08', 32, 30, 25, 19.5, 17.8, 14.1, 23.2, 21.4, 18.7, 85, 80, 70, 0.2, 0.4, 0.8), +(11, 11, '2022-11-08', 23, 17, 7, 13.7, 9.2, 2.9, 19.5, 16.8, 13.1, 68, 48, 28, 1.1, 2.1, 3.6), +(12, 12, '2022-12-08', 29, 23, 11, 17.4, 12.3, 4.9, 21.8, 18.7, 14.8, 78, 58, 38, 0.6, 1.6, 3.1), +(13, 13, '2023-01-08', 18, 12, 3, 10.5, 6.1, 1.0, 16.9, 14.3, 11.0, 56, 36, 16, 1.9, 2.9, 4.4), +(14, 14, '2023-02-08', 27, 20, 10, 16.2, 11.1, 4.1, 21.0, 17.9, 14.1, 74, 54, 34, 0.5, 1.5, 3.0), +(15, 15, '2023-03-08', 20, 14, 4, 11.8, 7.3, 1.7, 17.8, 15.2, 11.8, 60, 40, 20, 1.6, 2.6, 4.1), +(16, 16, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '5 months' + INTERVAL '7 days', 24, 18, 8, 14.4, 9.6, 3.2, 20.4, 17.4, 13.7, 70, 50, 30, 0.9, 1.9, 3.4), +(17, 17, DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 month' + INTERVAL '7 days', 22, 16, NULL, 13.2, 8.8, NULL, 19.1, 16.3, NULL, 65, 45, NULL, 1.3, 2.3, NULL), +(18, 25, CURRENT_DATE - INTERVAL '6 months' + INTERVAL '7 days', 30, NULL, NULL, 18.0, NULL, NULL, 22.0, NULL, NULL, 80, NULL, NULL, 1.0, NULL, NULL), +(19, 25, CURRENT_DATE - INTERVAL '2 months', 30, 18, 10, 18.0, 12.0, 4.0, 22.0, 19.0, 15.0, 80, 60, 40, 1.0, 2.0, 3.0), +(20, 26, CURRENT_DATE - INTERVAL '5 months' + INTERVAL '7 days', 25, NULL, NULL, 15.0, NULL, NULL, 20.0, NULL, NULL, 75, NULL, NULL, 0.5, NULL, NULL), +(21, 26, CURRENT_DATE - INTERVAL '1 month', 25, 18, 10, 15.0, 10.0, 5.0, 20.0, 17.0, 13.0, 75, 55, 35, 0.5, 1.5, 3.0); END $$;