From a2b1e554c0e5811b252bdd66ce8c262f2c3ce83c Mon Sep 17 00:00:00 2001 From: Carolina Capetillo Date: Wed, 11 Sep 2024 11:37:54 -0700 Subject: [PATCH 1/2] adds normalization operation and corrects typo --- .../data_operations/data_operation.py | 2 +- .../data_operations/normalization.py | 65 +++++++++++++++++++ datalab/datalab_session/s3_utils.py | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 datalab/datalab_session/data_operations/normalization.py diff --git a/datalab/datalab_session/data_operations/data_operation.py b/datalab/datalab_session/data_operations/data_operation.py index 5f9fc87..106bf04 100644 --- a/datalab/datalab_session/data_operations/data_operation.py +++ b/datalab/datalab_session/data_operations/data_operation.py @@ -53,7 +53,7 @@ def operate(self): """ def perform_operation(self): - """ The generic method to perform perform the operation if its not in progress """ + """ The generic method to perform the operation if its not in progress """ status = self.get_status() if status == 'PENDING' or status == 'FAILED': self.set_status('IN_PROGRESS') diff --git a/datalab/datalab_session/data_operations/normalization.py b/datalab/datalab_session/data_operations/normalization.py new file mode 100644 index 0000000..70df52b --- /dev/null +++ b/datalab/datalab_session/data_operations/normalization.py @@ -0,0 +1,65 @@ +import logging + +import numpy as np + +from datalab.datalab_session.data_operations.data_operation import BaseDataOperation +from datalab.datalab_session.file_utils import create_fits, create_jpgs +from datalab.datalab_session.s3_utils import save_fits_and_thumbnails + +log = logging.getLogger() +log.setLevel(logging.INFO) + + +class Normalization(BaseDataOperation): + + @staticmethod + def name(): + return 'Normalization' + + @staticmethod + def description(): + return """The normalize operation takes in 1..n input images and calculates each image's median value and divides every pixel by that value. + +The output is a normalized image for the n input images. This operation is commonly used as a precursor step for flat removal.""" + + @staticmethod + def wizard_description(): + return { + 'name': Normalization.name(), + 'description': Normalization.description(), + 'category': 'image', + 'inputs': { + 'input_files': { + 'name': 'Input Files', + 'description': 'The input files to operate on', + 'type': 'file', + 'minimum': 1, + 'maximum': 999 + } + } + } + + def operate(self): + + input = self.input_data.get('input_files', []) + + log.info(f'Executing normalization operation on {len(input)} file(s) {input}') + + image_data_list = self.get_fits_npdata(input, percent=0.4, cur_percent=0.0) + log.info(f'image data list: {image_data_list}') + output_files = [] + for i, image in enumerate(image_data_list): + median = np.median(image) + normalized_image = image / median + fits_file = create_fits(self.cache_key, normalized_image) + log.info(f'fits_file: {fits_file}') + large_jpg_path, small_jpg_path = create_jpgs(self.cache_key, fits_file) + output_file = save_fits_and_thumbnails(self.cache_key, fits_file, large_jpg_path, small_jpg_path, index=i) + output_files.append(output_file) + log.info(f'x: {output_files}') + + output = {'output_files': output_files} + + self.set_percent_completion(1.0) + self.set_output(output) + log.info(f'Normalization output: {self.get_output()}') diff --git a/datalab/datalab_session/s3_utils.py b/datalab/datalab_session/s3_utils.py index c461b97..3c3734c 100644 --- a/datalab/datalab_session/s3_utils.py +++ b/datalab/datalab_session/s3_utils.py @@ -152,7 +152,7 @@ def save_fits_and_thumbnails(cache_key, fits_path, large_jpg_path, thumbnail_jpg 'fits_url': fits_url, 'large_url': large_jpg_url, 'thumbnail_url': thumbnail_jpg_url, - 'basename': f'{cache_key}', + 'basename': f'{bucket_key}', 'source': 'datalab'} ) From 9a5d3fcaf1cc5ba2909a1d1c0023c2741d9e016f Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 16 Sep 2024 12:17:27 -0700 Subject: [PATCH 2/2] adjust percentages, remove logging messages --- .../data_operations/normalization.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/datalab/datalab_session/data_operations/normalization.py b/datalab/datalab_session/data_operations/normalization.py index 70df52b..aff5752 100644 --- a/datalab/datalab_session/data_operations/normalization.py +++ b/datalab/datalab_session/data_operations/normalization.py @@ -20,7 +20,7 @@ def name(): def description(): return """The normalize operation takes in 1..n input images and calculates each image's median value and divides every pixel by that value. -The output is a normalized image for the n input images. This operation is commonly used as a precursor step for flat removal.""" +The output is a normalized image. This operation is commonly used as a precursor step for flat removal.""" @staticmethod def wizard_description(): @@ -43,23 +43,24 @@ def operate(self): input = self.input_data.get('input_files', []) - log.info(f'Executing normalization operation on {len(input)} file(s) {input}') + log.info(f'Executing normalization operation on {len(input)} file(s)') + + image_data_list = self.get_fits_npdata(input) + self.set_percent_completion(0.40) - image_data_list = self.get_fits_npdata(input, percent=0.4, cur_percent=0.0) - log.info(f'image data list: {image_data_list}') output_files = [] - for i, image in enumerate(image_data_list): + for index, image in enumerate(image_data_list): median = np.median(image) normalized_image = image / median + fits_file = create_fits(self.cache_key, normalized_image) - log.info(f'fits_file: {fits_file}') large_jpg_path, small_jpg_path = create_jpgs(self.cache_key, fits_file) - output_file = save_fits_and_thumbnails(self.cache_key, fits_file, large_jpg_path, small_jpg_path, index=i) + output_file = save_fits_and_thumbnails(self.cache_key, fits_file, large_jpg_path, small_jpg_path, index=index) output_files.append(output_file) - log.info(f'x: {output_files}') + + self.set_percent_completion(self.get_percent_completion() + .40 * (index + 1) / len(input)) output = {'output_files': output_files} - self.set_percent_completion(1.0) self.set_output(output) log.info(f'Normalization output: {self.get_output()}')