Skip to content

Commit

Permalink
More fixes to e2e tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmccully
cmccully committed Aug 16, 2024
1 parent 7ae8bc2 commit f2ef342
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 59 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,19 @@ jobs:
- name: Test Super Bias Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
- name: Test Super Dark Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark
- name: Test Super Flat Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat
kubectl exec banzai-e2e-test -c banzai-listener -- pytest-s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat
- name: Test Science Frame Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files
- name: Cleanup
run: |
Expand Down
2 changes: 2 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,5 @@ COPY --chown=10087:10000 . /lco/banzai
ENV PATH /home/archive/envs/banzai/bin:$PATH

RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/

RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini
2 changes: 1 addition & 1 deletion banzai/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
'elp': {'minute': 0, 'hour': 23},
'ogg': {'minute': 0, 'hour': 3}}

ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', ' ')
ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/')

CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename',
'banzai.utils.file_utils.ccdsum_to_filename'),
Expand Down
4 changes: 2 additions & 2 deletions banzai/tests/data/test_data.dat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,13 +304,13 @@ frameid filename filter site instrument
33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04
33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04
33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04
73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34
73890113 lsc0m476-sq34-20240809-0144-e00.fits.fz B lsc sq34
73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34
73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34
73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34
73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34
73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34
73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34
73409786 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34
73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34
73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34
73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34
Expand Down
84 changes: 42 additions & 42 deletions banzai/tests/data/test_obs_portal_response_lsc_20240723.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@
"extra_params": {},
"guide_camera_name": "",
"guiding_config": {
"exposure_time": None,
"exposure_time": null,
"extra_params": {},
"mode": "OFF",
"optical_elements": {},
"optional": True
"optional": true
},
"id": 11542373,
"instrument_configs": [
Expand All @@ -59,7 +59,7 @@
"instrument_name": "sq34",
"instrument_type": "0M4-SCICAM-QHY600",
"priority": 3,
"repeat_duration": None,
"repeat_duration": null,
"state": "COMPLETED",
"summary": {
"end": "2024-07-23T22:22:19Z",
Expand Down Expand Up @@ -343,14 +343,14 @@
},
"target": {
"dec": 0.0,
"epoch": None,
"epoch": null,
"extra_params": {},
"hour_angle": 1.0,
"name": "No target",
"parallax": None,
"proper_motion_dec": None,
"proper_motion_ra": None,
"ra": None,
"parallax": null,
"proper_motion_dec": null,
"proper_motion_ra": null,
"ra": null,
"type": "HOUR_ANGLE"
},
"type": "SKY_FLAT"
Expand All @@ -372,20 +372,20 @@
"telescope": "0m4a"
},
{
"id": 631577495,
"id": 631287808,
"request": {
"id": 3597581,
"id": 3596764,
"observation_note": "",
"optimization_type": "TIME",
"state": "COMPLETED",
"acceptability_threshold": 90.0,
"configuration_repeats": 1,
"extra_params": {},
"modified": "2024-07-24T21:21:09.473360Z",
"modified": "2024-07-23T21:20:17.110554Z",
"duration": 3427,
"configurations": [
{
"id": 11549305,
"id": 11543917,
"instrument_type": "0M4-SCICAM-QHY600",
"type": "BIAS",
"repeat_duration": null,
Expand Down Expand Up @@ -436,63 +436,63 @@
"hour_angle": 1.0,
"extra_params": {}
},
"configuration_status": 813028208,
"configuration_status": 812665084,
"state": "COMPLETED",
"instrument_name": "sq34",
"guide_camera_name": "",
"summary": {
"id": 4818287,
"start": "2024-07-24T20:52:40Z",
"end": "2024-07-24T20:55:16Z",
"id": 4815555,
"start": "2024-07-23T20:52:00Z",
"end": "2024-07-23T20:54:31Z",
"state": "COMPLETED",
"reason": "",
"time_completed": 0.0,
"events": [
{
"time": "2024-07-24T20:52:40",
"time": "2024-07-23T20:52:00",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand"
},
{
"time": "2024-07-24T20:52:41",
"time": "2024-07-23T20:52:01",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand"
},
{
"time": "2024-07-24T20:52:41",
"time": "2024-07-23T20:52:01",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand"
},
{
"time": "2024-07-24T20:52:42",
"time": "2024-07-23T20:52:03",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand"
},
{
"time": "2024-07-24T20:52:43",
"time": "2024-07-23T20:52:03",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.StartExposureCommand"
},
{
"time": "2024-07-24T20:55:14",
"time": "2024-07-23T20:54:30",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.StartExposureCommand"
},
{
"time": "2024-07-24T20:55:15",
"time": "2024-07-23T20:54:30",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.AbortCommand"
},
{
"time": "2024-07-24T20:55:16",
"time": "2024-07-23T20:54:31",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.AbortCommand"
}
]
}
},
{
"id": 11549306,
"id": 11543918,
"instrument_type": "0M4-SCICAM-QHY600",
"type": "DARK",
"repeat_duration": null,
Expand Down Expand Up @@ -543,55 +543,55 @@
"hour_angle": 1.0,
"extra_params": {}
},
"configuration_status": 813028209,
"configuration_status": 812665085,
"state": "COMPLETED",
"instrument_name": "sq34",
"guide_camera_name": "",
"summary": {
"id": 4818307,
"start": "2024-07-24T20:55:19Z",
"end": "2024-07-24T21:21:09Z",
"id": 4815574,
"start": "2024-07-23T20:54:34Z",
"end": "2024-07-23T21:20:16Z",
"state": "COMPLETED",
"reason": "",
"time_completed": 1500.0,
"events": [
{
"time": "2024-07-24T20:55:19",
"time": "2024-07-23T20:54:34",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand"
},
{
"time": "2024-07-24T20:55:20",
"time": "2024-07-23T20:54:34",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.ConfigureCCDCommand"
},
{
"time": "2024-07-24T20:55:20",
"time": "2024-07-23T20:54:35",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand"
},
{
"time": "2024-07-24T20:55:22",
"time": "2024-07-23T20:54:36",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.ConfigureFiltersCommand"
},
{
"time": "2024-07-24T20:55:23",
"time": "2024-07-23T20:54:37",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.StartExposureCommand"
},
{
"time": "2024-07-24T21:21:07",
"time": "2024-07-23T21:20:14",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.StartExposureCommand"
},
{
"time": "2024-07-24T21:21:07",
"time": "2024-07-23T21:20:15",
"state": "PROCESSING",
"description": "org.lcogt.sequencer.command.instrument.AbortCommand"
},
{
"time": "2024-07-24T21:21:08",
"time": "2024-07-23T21:20:16",
"state": "DONE",
"description": "org.lcogt.sequencer.command.instrument.AbortCommand"
}
Expand All @@ -603,18 +603,18 @@
"site": "lsc",
"enclosure": "aqwa",
"telescope": "0m4a",
"start": "2024-07-24T20:26:04.169574Z",
"end": "2024-07-24T21:32:04.169574Z",
"start": "2024-07-23T20:25:25.334568Z",
"end": "2024-07-23T21:31:25.334568Z",
"priority": 10,
"state": "COMPLETED",
"proposal": "calibrate",
"submitter": "eng",
"name": "LCOGT",
"ipp_value": 1.0,
"observation_type": "DIRECT",
"request_group_id": 2012784,
"created": "2024-07-24T15:05:49.265713Z",
"modified": "2024-07-24T21:21:09.532093Z"
"request_group_id": 2012021,
"created": "2024-07-23T15:05:56.175376Z",
"modified": "2024-07-23T21:20:17.151237Z"
}
]
}
18 changes: 9 additions & 9 deletions banzai/tests/e2e-k8s.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ spec:
readOnly: false
env:
- name: DB_ADDRESS
value: "sqlite:////archive/engineering/test.db?timeout=10"
value: "sqlite:////archive/engineering/test.db?timeout=15"
- name: RETRY_DELAY
value: "0"
- name: TASK_HOST
Expand All @@ -75,7 +75,7 @@ spec:
- name: API_ROOT
value: "https://archive-api.lco.global/"
- name: OMP_NUM_THREADS
value: "2"
value: "1"
- name: FITS_EXCHANGE
value: "fits_files"
- name: OPENTSDB_PYTHON_METRICS_TEST_MODE
Expand Down Expand Up @@ -125,7 +125,7 @@ spec:
readOnly: false
env:
- name: DB_ADDRESS
value: "sqlite:////archive/engineering/test.db?timeout=10"
value: "sqlite:////archive/engineering/test.db?timeout=15"
- name: RETRY_DELAY
value: "0"
- name: TASK_HOST
Expand Down Expand Up @@ -175,10 +175,10 @@ spec:
resources:
requests:
cpu: 2
memory: 5Gi
memory: 7Gi
limits:
cpu: 3
memory: 5Gi
memory: 7Gi
- name: banzai-listener
image: banzai:test-latest
imagePullPolicy: IfNotPresent
Expand All @@ -189,7 +189,7 @@ spec:
readOnly: false
env:
- name: DB_ADDRESS
value: "sqlite:////archive/engineering/test.db?timeout=10"
value: "sqlite:////archive/engineering/test.db?timeout=15"
- name: FITS_BROKER
value: "localhost"
- name: TASK_HOST
Expand All @@ -212,9 +212,9 @@ spec:
value: "e2e_large_task_queue"
command:
- banzai_run_realtime_pipeline
- "--db-address='sqlite:////archive/engineering/test.db?timeout=10'"
- "--fpack"
- "--broker-url=localhost"
- --db-address=$(DB_ADDRESS)
- --fpack
- --broker-url=localhost
resources:
requests:
cpu: 0.1
Expand Down
3 changes: 2 additions & 1 deletion environment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ dependencies:
- pygments=2.16.1
- pyparsing=3.1.1
- pysocks=1.7.1
- pytest=8.3.2
- pytest=8.0.0
- pytest-arraydiff=0.5.0
- pytest-astropy=0.10.0
- pytest-astropy-header=0.2.2
Expand Down Expand Up @@ -288,6 +288,7 @@ dependencies:
- cloudpickle==2.2.1
- cosmic-conn==0.4.1
- dask==2023.9.3
- extenion_helpers==1.1.1
- fasteners==0.19
- fsspec==2023.9.2
- kombu==4.6.11
Expand Down
2 changes: 2 additions & 0 deletions pytest.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ markers =
dark_comparer
dark_maker
dark_normalizer
dark_subtractor
date_utils
dbs
fits_utils
Expand All @@ -48,6 +49,7 @@ markers =
pattern_noise_qc
pointing
quick_select
read_noise
runtime_context
saturation_qc
saving_qc
Expand Down

0 comments on commit f2ef342

Please sign in to comment.