From d290173fcc6ab2b7c825a65e5532b913659d71d7 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Thu, 8 Jun 2023 14:55:17 -0700 Subject: [PATCH 1/6] Add dark scaling, add tests --- banzai/dark.py | 7 +++ banzai/lco.py | 6 ++- banzai/tests/test_dark_subtractor.py | 72 ++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 banzai/tests/test_dark_subtractor.py diff --git a/banzai/dark.py b/banzai/dark.py index 3bb5f44b..637fc15f 100755 --- a/banzai/dark.py +++ b/banzai/dark.py @@ -1,5 +1,7 @@ import logging +import numpy as np + from banzai.stages import Stage from banzai.calibrations import CalibrationStacker, CalibrationUser, CalibrationComparer from banzai.utils import qc @@ -41,9 +43,14 @@ def calibration_type(self): def apply_master_calibration(self, image, master_calibration_image): master_calibration_image *= image.exptime + temp_scaling_factor = np.exp(master_calibration_image.dark_temperature_coefficient * \ + (image.measured_ccd_temperature - master_calibration_image.measured_ccd_temperature)) + master_calibration_image *= temp_scaling_factor + image -= master_calibration_image image.meta['L1IDDARK'] = master_calibration_image.filename, 'ID of dark frame' image.meta['L1STATDA'] = 1, 'Status flag for dark frame correction' + image.meta['DARKTSCL'] = temp_scaling_factor, 'Temperature scaling factor applied to dark image' return image diff --git a/banzai/lco.py b/banzai/lco.py index 3dba0328..ee80ad35 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -176,7 +176,7 @@ def to_db_record(self, output_product): for attribute in self.grouping_criteria: record_attributes['attributes'][attribute] = str(getattr(self, attribute)) return dbs.CalibrationImage(**record_attributes) - + @property def is_master(self): return self.meta.get('ISMASTER', False) @@ -185,6 +185,10 @@ def is_master(self): def is_master(self, value): self.meta['ISMASTER'] = value + @property + def dark_temperature_coefficient(self): + return self.meta.get('DARKCOEF', 0.0) + def write(self, runtime_context): output_products = LCOObservationFrame.write(self, runtime_context) CalibrationFrame.write(self, output_products, runtime_context) diff --git a/banzai/tests/test_dark_subtractor.py b/banzai/tests/test_dark_subtractor.py new file mode 100644 index 00000000..2410f0e9 --- /dev/null +++ b/banzai/tests/test_dark_subtractor.py @@ -0,0 +1,72 @@ +import pytest +import numpy as np +import mock + +from banzai.dark import DarkSubtractor +from banzai.tests.utils import FakeCCDData, FakeLCOObservationFrame, FakeContext +from banzai.lco import LCOCalibrationFrame +from banzai.data import CCDData + + +pytestmark = pytest.mark.dark_subtractor + + +@pytest.fixture(scope='module') +def set_random_seed(): + np.random.seed(7298374) + + +def test_null_input_images(): + normalizer = DarkSubtractor(None) + image = normalizer.run(None) + assert image is None + + +@mock.patch('banzai.lco.LCOFrameFactory.open') +@mock.patch('banzai.calibrations.CalibrationUser.get_calibration_file_info', return_value='test.fits') +def test_reasonable_dark_subtraction(mock_super_cal_name, mock_super_frame): + + mock_super_cal_name.return_value = {'filename': 'test.fits'} + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'ISMASTER': True, + 'CCDATEMP': -100})], file_path='/tmp') + image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'CCDATEMP': -100})]) + subtractor = DarkSubtractor(FakeContext()) + image = subtractor.do_stage(image) + np.testing.assert_allclose(image.data, 3*np.ones((100,100))) + + +@mock.patch('banzai.lco.LCOFrameFactory.open') +@mock.patch('banzai.calibrations.CalibrationUser.get_calibration_file_info', return_value='test.fits') +def test_reasonable_dark_subtraction_with_scaling(mock_super_cal_name, mock_super_frame): + dark_temperature_coefficient = 0.5 + image_measured_temp = 0 + super_measured_temp = -5 + + + mock_super_cal_name.return_value = {'filename': 'test.fits'} + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'ISMASTER': True, + 'DARKCOEF': dark_temperature_coefficient, + 'CCDATEMP': -5})], file_path='/tmp') + image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'CCDATEMP': 0})]) + + dark_scaling_factor = np.exp(dark_temperature_coefficient * (image_measured_temp - super_measured_temp)) + subtracted_data = image.data - np.ones((100,100)) * dark_scaling_factor + subtractor = DarkSubtractor(FakeContext()) + image = subtractor.do_stage(image) + np.testing.assert_allclose(image.data, subtracted_data) From 0297f1dc1e301c7f6679094b825201ebfa5f1a1c Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Thu, 8 Jun 2023 15:09:13 -0700 Subject: [PATCH 2/6] Fix code style errors --- banzai/lco.py | 2 +- banzai/tests/test_dark_subtractor.py | 39 +++++++++++++++------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/banzai/lco.py b/banzai/lco.py index ee80ad35..587d083c 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -176,7 +176,7 @@ def to_db_record(self, output_product): for attribute in self.grouping_criteria: record_attributes['attributes'][attribute] = str(getattr(self, attribute)) return dbs.CalibrationImage(**record_attributes) - + @property def is_master(self): return self.meta.get('ISMASTER', False) diff --git a/banzai/tests/test_dark_subtractor.py b/banzai/tests/test_dark_subtractor.py index 2410f0e9..0aa0d89a 100644 --- a/banzai/tests/test_dark_subtractor.py +++ b/banzai/tests/test_dark_subtractor.py @@ -27,12 +27,13 @@ def test_null_input_images(): def test_reasonable_dark_subtraction(mock_super_cal_name, mock_super_frame): mock_super_cal_name.return_value = {'filename': 'test.fits'} - mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, - 'SATURATE': 35000, - 'GAIN': 1.0, - 'MAXLIN': 35000, - 'ISMASTER': True, - 'CCDATEMP': -100})], file_path='/tmp') + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), + meta={'EXPTIME': 1.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'ISMASTER': True, + 'CCDATEMP': -100})], file_path='/tmp') image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, 'SATURATE': 35000, 'GAIN': 1.0, @@ -52,18 +53,20 @@ def test_reasonable_dark_subtraction_with_scaling(mock_super_cal_name, mock_supe mock_super_cal_name.return_value = {'filename': 'test.fits'} - mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, - 'SATURATE': 35000, - 'GAIN': 1.0, - 'MAXLIN': 35000, - 'ISMASTER': True, - 'DARKCOEF': dark_temperature_coefficient, - 'CCDATEMP': -5})], file_path='/tmp') - image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, - 'SATURATE': 35000, - 'GAIN': 1.0, - 'MAXLIN': 35000, - 'CCDATEMP': 0})]) + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), + meta={'EXPTIME': 1.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'ISMASTER': True, + 'DARKCOEF': dark_temperature_coefficient, + 'CCDATEMP': -5})], file_path='/tmp') + image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), + meta={'EXPTIME': 2.0, + 'SATURATE': 35000, + 'GAIN': 1.0, + 'MAXLIN': 35000, + 'CCDATEMP': 0})]) dark_scaling_factor = np.exp(dark_temperature_coefficient * (image_measured_temp - super_measured_temp)) subtracted_data = image.data - np.ones((100,100)) * dark_scaling_factor From a952eb6a1e4071036f049dbbe4ca6f3e92bf1536 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Thu, 8 Jun 2023 15:14:14 -0700 Subject: [PATCH 3/6] More whitespace fixes --- banzai/tests/test_dark_subtractor.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/banzai/tests/test_dark_subtractor.py b/banzai/tests/test_dark_subtractor.py index 0aa0d89a..58803b56 100644 --- a/banzai/tests/test_dark_subtractor.py +++ b/banzai/tests/test_dark_subtractor.py @@ -27,7 +27,7 @@ def test_null_input_images(): def test_reasonable_dark_subtraction(mock_super_cal_name, mock_super_frame): mock_super_cal_name.return_value = {'filename': 'test.fits'} - mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, 'SATURATE': 35000, 'GAIN': 1.0, @@ -51,9 +51,8 @@ def test_reasonable_dark_subtraction_with_scaling(mock_super_cal_name, mock_supe image_measured_temp = 0 super_measured_temp = -5 - mock_super_cal_name.return_value = {'filename': 'test.fits'} - mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), + mock_super_frame.return_value = LCOCalibrationFrame(hdu_list=[CCDData(data=0.5*np.ones((100,100)), meta={'EXPTIME': 1.0, 'SATURATE': 35000, 'GAIN': 1.0, @@ -61,13 +60,13 @@ def test_reasonable_dark_subtraction_with_scaling(mock_super_cal_name, mock_supe 'ISMASTER': True, 'DARKCOEF': dark_temperature_coefficient, 'CCDATEMP': -5})], file_path='/tmp') - image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), + image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, 'SATURATE': 35000, 'GAIN': 1.0, 'MAXLIN': 35000, 'CCDATEMP': 0})]) - + dark_scaling_factor = np.exp(dark_temperature_coefficient * (image_measured_temp - super_measured_temp)) subtracted_data = image.data - np.ones((100,100)) * dark_scaling_factor subtractor = DarkSubtractor(FakeContext()) From 7111f5deb16ad30184c56ede2e0393f83e32f894 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 14 Jun 2023 13:36:43 -0700 Subject: [PATCH 4/6] Adjust header keyword used for temp scaling Naming things is always the hardest --- banzai/dark.py | 2 +- banzai/lco.py | 2 +- banzai/tests/test_dark_subtractor.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/banzai/dark.py b/banzai/dark.py index 637fc15f..a5db9704 100755 --- a/banzai/dark.py +++ b/banzai/dark.py @@ -50,7 +50,7 @@ def apply_master_calibration(self, image, master_calibration_image): image -= master_calibration_image image.meta['L1IDDARK'] = master_calibration_image.filename, 'ID of dark frame' image.meta['L1STATDA'] = 1, 'Status flag for dark frame correction' - image.meta['DARKTSCL'] = temp_scaling_factor, 'Temperature scaling factor applied to dark image' + image.meta['DRKTSCAL'] = temp_scaling_factor, 'Temperature scaling factor applied to dark image' return image diff --git a/banzai/lco.py b/banzai/lco.py index 587d083c..a769000c 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -187,7 +187,7 @@ def is_master(self, value): @property def dark_temperature_coefficient(self): - return self.meta.get('DARKCOEF', 0.0) + return self.meta.get('DRKTCOEF', 0.0) def write(self, runtime_context): output_products = LCOObservationFrame.write(self, runtime_context) diff --git a/banzai/tests/test_dark_subtractor.py b/banzai/tests/test_dark_subtractor.py index 58803b56..8286a579 100644 --- a/banzai/tests/test_dark_subtractor.py +++ b/banzai/tests/test_dark_subtractor.py @@ -58,7 +58,7 @@ def test_reasonable_dark_subtraction_with_scaling(mock_super_cal_name, mock_supe 'GAIN': 1.0, 'MAXLIN': 35000, 'ISMASTER': True, - 'DARKCOEF': dark_temperature_coefficient, + 'DRKTCOEF': dark_temperature_coefficient, 'CCDATEMP': -5})], file_path='/tmp') image = FakeLCOObservationFrame(hdu_list=[CCDData(data=4*np.ones((100,100)), meta={'EXPTIME': 2.0, From 05b8bc3a7f9aef39defddb53ea3fc99b3dcfb885 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Tue, 27 Jun 2023 16:17:13 +0000 Subject: [PATCH 5/6] Rename temperature variable for clarity temp is a very overloaded name in CS --- banzai/dark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/banzai/dark.py b/banzai/dark.py index a5db9704..dbd18469 100755 --- a/banzai/dark.py +++ b/banzai/dark.py @@ -43,14 +43,14 @@ def calibration_type(self): def apply_master_calibration(self, image, master_calibration_image): master_calibration_image *= image.exptime - temp_scaling_factor = np.exp(master_calibration_image.dark_temperature_coefficient * \ - (image.measured_ccd_temperature - master_calibration_image.measured_ccd_temperature)) - master_calibration_image *= temp_scaling_factor + temperature_scaling_factor = np.exp(master_calibration_image.dark_temperature_coefficient * \ + (image.measured_ccd_temperature - master_calibration_image.measured_ccd_temperature)) + master_calibration_image *= temperature_scaling_factor image -= master_calibration_image image.meta['L1IDDARK'] = master_calibration_image.filename, 'ID of dark frame' image.meta['L1STATDA'] = 1, 'Status flag for dark frame correction' - image.meta['DRKTSCAL'] = temp_scaling_factor, 'Temperature scaling factor applied to dark image' + image.meta['DRKTSCAL'] = temperature_scaling_factor, 'Temperature scaling factor applied to dark image' return image From 7f63b942765e200b86350a99423a96a7998562a4 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Tue, 27 Jun 2023 16:52:56 -0700 Subject: [PATCH 6/6] Add prod deploy to jenkinsfile on tag --- Jenkinsfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 6229cc90..a35ee305 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,6 +42,25 @@ pipeline { } } } + stage('DeployProdStack') { + agent { + label 'helm' + } + when { + buildingTag(); + } + steps { + script { + withKubeConfig([credentialsId: "prod-kube-config"]) { + sh('helm repo update && helm dependency update helm-chart/banzai/ '+ + '&& helm package helm-chart/banzai --app-version="${GIT_DESCRIPTION}" --version="${GIT_DESCRIPTION}" ' + + '&& helm upgrade --install banzai banzai-"${GIT_DESCRIPTION}".tgz --namespace=prod ' + + '--set image.tag="${GIT_DESCRIPTION}" --values=helm-chart/banzai/values-prod.yaml ' + + '--force --atomic --timeout=3600s') + } + } + } + } stage('DeployTestStack') { when { anyOf {