From b7ae06285f990520e59bfd686a875f0f0176671d Mon Sep 17 00:00:00 2001 From: stephen-riggs Date: Thu, 23 May 2024 16:57:46 +0100 Subject: [PATCH 1/2] Remove processing files which aren't required for further processing --- src/cryoemservices/services/cryolo.py | 6 ++++++ src/cryoemservices/services/ctffind.py | 14 ++++++++++++++ src/cryoemservices/services/extract.py | 6 +++++- src/cryoemservices/services/icebreaker.py | 1 + src/cryoemservices/services/motioncorr.py | 5 +++++ src/cryoemservices/services/select_particles.py | 3 +++ src/cryoemservices/util/spa_output_files.py | 17 ++++++++--------- src/cryoemservices/wrappers/class2d_wrapper.py | 8 ++++++++ 8 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/cryoemservices/services/cryolo.py b/src/cryoemservices/services/cryolo.py index dffab49..b1a668b 100644 --- a/src/cryoemservices/services/cryolo.py +++ b/src/cryoemservices/services/cryolo.py @@ -410,5 +410,11 @@ def dummy(self, *args, **kwargs): }, ) + # Remove unnecessary files + eman_file = ( + job_dir / f"EMAN/{Path(cryolo_params.output_path).with_suffix('.box').name}" + ) + eman_file.unlink(missing_ok=True) + self.log.info(f"Done {self.job_type} for {cryolo_params.input_path}.") rw.transport.ack(header) diff --git a/src/cryoemservices/services/ctffind.py b/src/cryoemservices/services/ctffind.py index 2ba6b01..f4019ed 100644 --- a/src/cryoemservices/services/ctffind.py +++ b/src/cryoemservices/services/ctffind.py @@ -204,6 +204,13 @@ def dummy(self, *args, **kwargs): ), "stdout": result.stdout.decode("utf8", "replace"), "stderr": result.stderr.decode("utf8", "replace"), + "results": { + "defocus1": str(self.defocus1), + "defocus2": str(self.defocus2), + "astigmatism_angle": str(self.astigmatism_angle), + "cc_value": str(self.cc_value), + "estimated_resolution": str(self.estimated_resolution), + }, } if result.returncode: node_creator_parameters["success"] = False @@ -333,5 +340,12 @@ def dummy(self, *args, **kwargs): else: rw.send_to("cryolo", ctf_params.autopick) + # Remove unnecessary files + log_file = Path(ctf_params.output_image).parent / ( + Path(ctf_params.output_image).stem + "_ctffind4.log" + ) + log_file.unlink(missing_ok=True) + Path(ctf_params.output_image).with_suffix(".txt").unlink(missing_ok=True) + self.log.info(f"Done {self.job_type} for {ctf_params.input_image}.") rw.transport.ack(header) diff --git a/src/cryoemservices/services/extract.py b/src/cryoemservices/services/extract.py index 3ccfca1..b5b8124 100644 --- a/src/cryoemservices/services/extract.py +++ b/src/cryoemservices/services/extract.py @@ -224,6 +224,10 @@ def dummy(self, *args, **kwargs): extracted_parts_doc.write_file( extract_params.output_file, style=cif.Style.Simple ) + file_for_selection = Path(extract_params.output_file).parent / ( + Path(extract_params.output_file).stem + "_to_select.star" + ) + extracted_parts_doc.write_file(str(file_for_selection), style=cif.Style.Simple) # Extraction with mrcfile.open(extract_params.micrographs_file) as input_micrograph: @@ -422,7 +426,7 @@ def dummy(self, *args, **kwargs): # Register the files needed for selection and batching self.log.info("Sending to particle selection") select_params = { - "input_file": extract_params.output_file, + "input_file": str(file_for_selection), "batch_size": extract_params.batch_size, "image_size": box_len, "relion_options": dict(extract_params.relion_options), diff --git a/src/cryoemservices/services/icebreaker.py b/src/cryoemservices/services/icebreaker.py index 97103f9..a0d63c8 100644 --- a/src/cryoemservices/services/icebreaker.py +++ b/src/cryoemservices/services/icebreaker.py @@ -193,6 +193,7 @@ def dummy(self, *args, **kwargs): summary_results = five_fig_csv.split(",") if len(summary_results) != 6: summary_results = ["0", "0", "0", "0", "0", "0"] + (Path("IB_input") / mic_from_project.name).unlink(missing_ok=True) # Create the command this replicates command = [ diff --git a/src/cryoemservices/services/motioncorr.py b/src/cryoemservices/services/motioncorr.py index 11a3dfa..16728c0 100644 --- a/src/cryoemservices/services/motioncorr.py +++ b/src/cryoemservices/services/motioncorr.py @@ -327,6 +327,11 @@ def dummy(self, *args, **kwargs): Path(mc_params.mrc_out).stem + "_DW.mrc" ) if dose_weighted.is_file(): + # Remove DWS file and set DW as the output + dws = Path(mc_params.mrc_out).parent / ( + Path(mc_params.mrc_out).stem + "_DWS.mrc" + ) + dws.unlink(missing_ok=True) mc_params.mrc_out = str(dose_weighted) else: # Construct the command for Relion motion correction diff --git a/src/cryoemservices/services/select_particles.py b/src/cryoemservices/services/select_particles.py index a4df277..3a71eaa 100644 --- a/src/cryoemservices/services/select_particles.py +++ b/src/cryoemservices/services/select_particles.py @@ -291,5 +291,8 @@ def dummy(self, *args, **kwargs): else: rw.send_to("murfey_feedback", murfey_confirmation) + # Remove unnecessary files + Path(select_params.input_file).unlink(missing_ok=True) + self.log.info(f"Done {self.job_type} for {select_params.input_file}.") rw.transport.ack(header) diff --git a/src/cryoemservices/util/spa_output_files.py b/src/cryoemservices/util/spa_output_files.py index 93cf490..da8df42 100644 --- a/src/cryoemservices/util/spa_output_files.py +++ b/src/cryoemservices/util/spa_output_files.py @@ -145,19 +145,17 @@ def _ctffind_output_files( """Ctf estimation saves a list of micrographs and their ctf parameters""" star_file = job_dir / "micrographs_ctf.star" - # Results needed in the star file are stored in a txt file with the output - with open(output_file.with_suffix(".txt"), "r") as f: - ctf_results = f.readlines()[-1].split() + # Results sent across need to be written to the star file added_line = [ str(input_file), "1", str(output_file.with_suffix(".ctf")) + ":mrc", - ctf_results[1], - ctf_results[2], - str(abs(float(ctf_results[1]) - float(ctf_results[2]))), - ctf_results[3], - ctf_results[5], - ctf_results[6], + results["defocus1"], + results["defocus2"], + str(abs(float(results["defocus1"]) - float(results["defocus2"]))), + results["astimatism_angle"], + results["cc_value"], + results["estimated_resolution"], ] # Read and append to the existing output file, or otherwise create one @@ -328,6 +326,7 @@ def _extract_output_files( for new_row in added_lines: if new_row[:1].isdigit(): output_cif.write(new_row) + output_file.unlink() def _select_output_files( diff --git a/src/cryoemservices/wrappers/class2d_wrapper.py b/src/cryoemservices/wrappers/class2d_wrapper.py index d2d72b2..e236cbe 100644 --- a/src/cryoemservices/wrappers/class2d_wrapper.py +++ b/src/cryoemservices/wrappers/class2d_wrapper.py @@ -373,6 +373,14 @@ def run(self): } self.recwrap.send_to("murfey_feedback", murfey_params) + # Remove unnecessary files + for iteration in range(class2d_params.class2d_nr_iter): + (job_dir / f"run_it{iteration:03}_classes.mrcs").unlink(missing_ok=True) + (job_dir / f"run_it{iteration:03}_data.star").unlink(missing_ok=True) + (job_dir / f"run_it{iteration:03}_model.star").unlink(missing_ok=True) + (job_dir / f"run_it{iteration:03}_optimiser.star").unlink(missing_ok=True) + (job_dir / f"run_it{iteration:03}_sampling.star").unlink(missing_ok=True) + (job_dir / "RELION_JOB_EXIT_SUCCESS").touch(exist_ok=True) self.log.info(f"Done {job_type} for {class2d_params.particles_file}.") return True From 8785755245776c86799c295c4db98d3d5a43b973 Mon Sep 17 00:00:00 2001 From: stephen-riggs Date: Fri, 24 May 2024 09:52:15 +0100 Subject: [PATCH 2/2] Update tests to account for removed files --- src/cryoemservices/services/node_creator.py | 1 + src/cryoemservices/util/spa_output_files.py | 2 +- tests/services/test_ctffind_service.py | 7 +++++++ tests/services/test_extract_service.py | 4 +++- tests/services/test_node_creator.py | 11 ++++++++--- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/cryoemservices/services/node_creator.py b/src/cryoemservices/services/node_creator.py index 0871368..5ab5225 100644 --- a/src/cryoemservices/services/node_creator.py +++ b/src/cryoemservices/services/node_creator.py @@ -331,6 +331,7 @@ def dummy(self, *args, **kwargs): pipeliner_job.create_output_nodes() pipeliner_job.write_runjob(pipeliner_job.output_dir) pipeliner_job.write_jobstar(pipeliner_job.output_dir) + pipeliner_job.write_jobstar(".gui_" + job_info.job_type.replace(".", "_")) pipeliner_job.write_jobstar( f"{pipeliner_job.output_dir}/continue_", is_continue=True ) diff --git a/src/cryoemservices/util/spa_output_files.py b/src/cryoemservices/util/spa_output_files.py index da8df42..4b27549 100644 --- a/src/cryoemservices/util/spa_output_files.py +++ b/src/cryoemservices/util/spa_output_files.py @@ -153,7 +153,7 @@ def _ctffind_output_files( results["defocus1"], results["defocus2"], str(abs(float(results["defocus1"]) - float(results["defocus2"]))), - results["astimatism_angle"], + results["astigmatism_angle"], results["cc_value"], results["estimated_resolution"], ] diff --git a/tests/services/test_ctffind_service.py b/tests/services/test_ctffind_service.py index 0cad04a..894da2a 100644 --- a/tests/services/test_ctffind_service.py +++ b/tests/services/test_ctffind_service.py @@ -191,6 +191,13 @@ def test_ctffind_service( "stdout": "stdout", "stderr": "stderr", "success": True, + "results": { + "defocus1": str(service.defocus1), + "defocus2": str(service.defocus2), + "astigmatism_angle": str(service.astigmatism_angle), + "cc_value": str(service.cc_value), + "estimated_resolution": str(service.estimated_resolution), + }, }, "content": "dummy", }, diff --git a/tests/services/test_extract_service.py b/tests/services/test_extract_service.py index 4eb5815..14498a6 100644 --- a/tests/services/test_extract_service.py +++ b/tests/services/test_extract_service.py @@ -109,7 +109,9 @@ def test_extract_service(mock_mrcfile, mock_environment, offline_transport, tmp_ destination="select_particles", message={ "parameters": { - "input_file": extract_test_message["parameters"]["output_file"], + "input_file": str( + tmp_path / "Extract/job008/Movies/sample_to_select.star" + ), "relion_options": output_relion_options, "batch_size": output_relion_options["batch_size"], "image_size": 64, diff --git a/tests/services/test_node_creator.py b/tests/services/test_node_creator.py index a247e87..79ff02a 100644 --- a/tests/services/test_node_creator.py +++ b/tests/services/test_node_creator.py @@ -346,9 +346,13 @@ def test_node_creator_ctffind(mock_environment, offline_transport, tmp_path): input_file = tmp_path / "MotionCorr/job002/Movies/sample.mrc" output_file = tmp_path / job_dir / "Movies/sample.ctf" - output_file.parent.mkdir(parents=True) - with open(output_file.with_suffix(".txt"), "w") as f: - f.write("0.0 1.0 2.0 3.0 4.0 5.0 6.0") + ctf_results = { + "defocus1": "1.0", + "defocus2": "2.0", + "astigmatism_angle": "3.0", + "cc_value": "5.0", + "estimated_resolution": "6.0", + } setup_and_run_node_creation( mock_environment, @@ -358,6 +362,7 @@ def test_node_creator_ctffind(mock_environment, offline_transport, tmp_path): "relion.ctffind.ctffind4", input_file, output_file, + ctf_results, ) # Check the output file structure