Remove some files to reduce the number in project

src/cryoemservices/services/cryolo.py

@@ -410,5 +410,11 @@ def dummy(self, *args, **kwargs):
                 },
             )
 
+        # Remove unnecessary files
+        eman_file = (
+            job_dir / f"EMAN/{Path(cryolo_params.output_path).with_suffix('.box').name}"
+        )
+        eman_file.unlink(missing_ok=True)
+
         self.log.info(f"Done {self.job_type} for {cryolo_params.input_path}.")
         rw.transport.ack(header)

src/cryoemservices/services/ctffind.py

@@ -204,6 +204,13 @@ def dummy(self, *args, **kwargs):
                 ),
                 "stdout": result.stdout.decode("utf8", "replace"),
                 "stderr": result.stderr.decode("utf8", "replace"),
+                "results": {
+                    "defocus1": str(self.defocus1),
+                    "defocus2": str(self.defocus2),
+                    "astigmatism_angle": str(self.astigmatism_angle),
+                    "cc_value": str(self.cc_value),
+                    "estimated_resolution": str(self.estimated_resolution),
+                },
             }
             if result.returncode:
                 node_creator_parameters["success"] = False
@@ -333,5 +340,12 @@ def dummy(self, *args, **kwargs):
             else:
                 rw.send_to("cryolo", ctf_params.autopick)
 
+        # Remove unnecessary files
+        log_file = Path(ctf_params.output_image).parent / (
+            Path(ctf_params.output_image).stem + "_ctffind4.log"
+        )
+        log_file.unlink(missing_ok=True)
+        Path(ctf_params.output_image).with_suffix(".txt").unlink(missing_ok=True)
+
         self.log.info(f"Done {self.job_type} for {ctf_params.input_image}.")
         rw.transport.ack(header)

src/cryoemservices/services/extract.py

@@ -224,6 +224,10 @@ def dummy(self, *args, **kwargs):
         extracted_parts_doc.write_file(
             extract_params.output_file, style=cif.Style.Simple
         )
+        file_for_selection = Path(extract_params.output_file).parent / (
+            Path(extract_params.output_file).stem + "_to_select.star"
+        )
+        extracted_parts_doc.write_file(str(file_for_selection), style=cif.Style.Simple)
 
         # Extraction
         with mrcfile.open(extract_params.micrographs_file) as input_micrograph:
@@ -422,7 +426,7 @@ def dummy(self, *args, **kwargs):
         # Register the files needed for selection and batching
         self.log.info("Sending to particle selection")
         select_params = {
-            "input_file": extract_params.output_file,
+            "input_file": str(file_for_selection),
             "batch_size": extract_params.batch_size,
             "image_size": box_len,
             "relion_options": dict(extract_params.relion_options),

src/cryoemservices/services/icebreaker.py

@@ -193,6 +193,7 @@ def dummy(self, *args, **kwargs):
             summary_results = five_fig_csv.split(",")
             if len(summary_results) != 6:
                 summary_results = ["0", "0", "0", "0", "0", "0"]
+            (Path("IB_input") / mic_from_project.name).unlink(missing_ok=True)
 
             # Create the command this replicates
             command = [

src/cryoemservices/services/motioncorr.py

@@ -327,6 +327,11 @@ def dummy(self, *args, **kwargs):
                 Path(mc_params.mrc_out).stem + "_DW.mrc"
             )
             if dose_weighted.is_file():
+                # Remove DWS file and set DW as the output
+                dws = Path(mc_params.mrc_out).parent / (
+                    Path(mc_params.mrc_out).stem + "_DWS.mrc"
+                )
+                dws.unlink(missing_ok=True)
                 mc_params.mrc_out = str(dose_weighted)
         else:
             # Construct the command for Relion motion correction

src/cryoemservices/services/node_creator.py

@@ -331,6 +331,7 @@ def dummy(self, *args, **kwargs):
         pipeliner_job.create_output_nodes()
         pipeliner_job.write_runjob(pipeliner_job.output_dir)
         pipeliner_job.write_jobstar(pipeliner_job.output_dir)
+        pipeliner_job.write_jobstar(".gui_" + job_info.job_type.replace(".", "_"))
         pipeliner_job.write_jobstar(
             f"{pipeliner_job.output_dir}/continue_", is_continue=True
         )

src/cryoemservices/services/select_particles.py

@@ -291,5 +291,8 @@ def dummy(self, *args, **kwargs):
         else:
             rw.send_to("murfey_feedback", murfey_confirmation)
 
+        # Remove unnecessary files
+        Path(select_params.input_file).unlink(missing_ok=True)
+
         self.log.info(f"Done {self.job_type} for {select_params.input_file}.")
         rw.transport.ack(header)

src/cryoemservices/util/spa_output_files.py

@@ -145,19 +145,17 @@ def _ctffind_output_files(
     """Ctf estimation saves a list of micrographs and their ctf parameters"""
     star_file = job_dir / "micrographs_ctf.star"
 
-    # Results needed in the star file are stored in a txt file with the output
-    with open(output_file.with_suffix(".txt"), "r") as f:
-        ctf_results = f.readlines()[-1].split()
+    # Results sent across need to be written to the star file
     added_line = [
         str(input_file),
         "1",
         str(output_file.with_suffix(".ctf")) + ":mrc",
-        ctf_results[1],
-        ctf_results[2],
-        str(abs(float(ctf_results[1]) - float(ctf_results[2]))),
-        ctf_results[3],
-        ctf_results[5],
-        ctf_results[6],
+        results["defocus1"],
+        results["defocus2"],
+        str(abs(float(results["defocus1"]) - float(results["defocus2"]))),
+        results["astigmatism_angle"],
+        results["cc_value"],
+        results["estimated_resolution"],
     ]
 
     # Read and append to the existing output file, or otherwise create one
@@ -328,6 +326,7 @@ def _extract_output_files(
             for new_row in added_lines:
                 if new_row[:1].isdigit():
                     output_cif.write(new_row)
+    output_file.unlink()
 
 
 def _select_output_files(

src/cryoemservices/wrappers/class2d_wrapper.py

@@ -373,6 +373,14 @@ def run(self):
             }
             self.recwrap.send_to("murfey_feedback", murfey_params)
 
+        # Remove unnecessary files
+        for iteration in range(class2d_params.class2d_nr_iter):
+            (job_dir / f"run_it{iteration:03}_classes.mrcs").unlink(missing_ok=True)
+            (job_dir / f"run_it{iteration:03}_data.star").unlink(missing_ok=True)
+            (job_dir / f"run_it{iteration:03}_model.star").unlink(missing_ok=True)
+            (job_dir / f"run_it{iteration:03}_optimiser.star").unlink(missing_ok=True)
+            (job_dir / f"run_it{iteration:03}_sampling.star").unlink(missing_ok=True)
+
         (job_dir / "RELION_JOB_EXIT_SUCCESS").touch(exist_ok=True)
         self.log.info(f"Done {job_type} for {class2d_params.particles_file}.")
         return True

tests/services/test_ctffind_service.py

@@ -191,6 +191,13 @@ def test_ctffind_service(
                 "stdout": "stdout",
                 "stderr": "stderr",
                 "success": True,
+                "results": {
+                    "defocus1": str(service.defocus1),
+                    "defocus2": str(service.defocus2),
+                    "astigmatism_angle": str(service.astigmatism_angle),
+                    "cc_value": str(service.cc_value),
+                    "estimated_resolution": str(service.estimated_resolution),
+                },
             },
             "content": "dummy",
         },

tests/services/test_extract_service.py

@@ -109,7 +109,9 @@ def test_extract_service(mock_mrcfile, mock_environment, offline_transport, tmp_
         destination="select_particles",
         message={
             "parameters": {
-                "input_file": extract_test_message["parameters"]["output_file"],
+                "input_file": str(
+                    tmp_path / "Extract/job008/Movies/sample_to_select.star"
+                ),
                 "relion_options": output_relion_options,
                 "batch_size": output_relion_options["batch_size"],
                 "image_size": 64,

tests/services/test_node_creator.py

@@ -346,9 +346,13 @@ def test_node_creator_ctffind(mock_environment, offline_transport, tmp_path):
     input_file = tmp_path / "MotionCorr/job002/Movies/sample.mrc"
     output_file = tmp_path / job_dir / "Movies/sample.ctf"
 
-    output_file.parent.mkdir(parents=True)
-    with open(output_file.with_suffix(".txt"), "w") as f:
-        f.write("0.0 1.0 2.0 3.0 4.0 5.0 6.0")
+    ctf_results = {
+        "defocus1": "1.0",
+        "defocus2": "2.0",
+        "astigmatism_angle": "3.0",
+        "cc_value": "5.0",
+        "estimated_resolution": "6.0",
+    }
 
     setup_and_run_node_creation(
         mock_environment,
@@ -358,6 +362,7 @@ def test_node_creator_ctffind(mock_environment, offline_transport, tmp_path):
         "relion.ctffind.ctffind4",
         input_file,
         output_file,
+        ctf_results,
     )
 
     # Check the output file structure