From bba35af5d04e8b81120ebb255f9cf33b475832f0 Mon Sep 17 00:00:00 2001
From: Yuanpeng Zhang <zyroc1990@gmail.com>
Date: Thu, 25 Mar 2021 16:52:38 -0400
Subject: [PATCH 1/5] add redpar preparation for Joerg new processing routine

---
 addie/processing/idl/run_sum_scans.py         | 82 +++++++++++++++++--
 addie/processing/idl/step2_gui_handler.py     |  5 ++
 .../periodic_table/material_handler.py        | 10 ++-
 3 files changed, 89 insertions(+), 8 deletions(-)

diff --git a/addie/processing/idl/run_sum_scans.py b/addie/processing/idl/run_sum_scans.py
index f5cd77e2..f0c5bc59 100644
--- a/addie/processing/idl/run_sum_scans.py
+++ b/addie/processing/idl/run_sum_scans.py
@@ -2,11 +2,19 @@
 import os
 from qtpy.QtCore import Qt
 from addie.processing.idl.step2_gui_handler import Step2GuiHandler
+from addie.processing.mantid.master_table.periodic_table.material_handler import get_periodictable_formatted_element_and_number_of_atoms
+import numpy as np
+import re
 
 
 class RunSumScans(object):
 
     output_file = ''
+    # Variable for Jeorg's new routine, controlling maximum r one could get to.
+    # For almost all situations, we don't need to change it, but it is good to
+    # put the variable explicitly just in case we may need to change it for
+    # whatever reason in the future.
+    q_interval_min = 0.02
 
     def __init__(self, parent=None):
         self.parent = parent.ui.postprocessing_ui
@@ -22,7 +30,14 @@ def set_sum_scans_script(self):
 
     def run(self):
         self._background = self.collect_background_runs()
-        self._runs = self.collect_runs_checked()
+        collect_return = self.collect_runs_checked()
+        self._runs = collect_return[0]
+        self._runs_name = collect_return[1]
+        self._sam_formula = collect_return[2]
+        self._mass_den = collect_return[3]
+        self._radius = collect_return[4]
+        self._pack_frac = collect_return[5]
+        self._geom = collect_return[6]
         self.create_output_file()
         self.run_script()
 
@@ -94,17 +109,61 @@ def create_output_file(self):
         f.close()
         print("[LOG] created file %s" % _full_output_file_name)
 
+        # redpar file
+        for _label in self._runs:
+            f = open(self._runs_name[_label] + ".redpar", "w")
+            chem_form_temp = self._sam_formula[_label]
+            list_element = chem_form_temp.split(" ")
+            formated_ele_list = []
+            for _element in list_element:
+                [formated_element, number_of_atoms, case] = get_periodictable_formatted_element_and_number_of_atoms(_element)
+                formated_ele_list.append(formated_element)
+            sample_form_str = ""
+            for item in formated_ele_list:
+                if case == 1:
+                    to_append = item + "_"
+                elif case == 2:
+                    to_append = re.findall('[A-Z][a-z]*', item)[-1]
+                    to_append += ("_" + re.findall('[0-9]+', item)[-1] + "_")
+                else:
+                    to_append = item.split("[")[1].split("]")[0] + item.split("[")[0] + "_" + item.split("]")[1] + "_"
+                sample_form_str += to_append
+            sample_form_str = sample_form_str[:-1]
+            f.write("{0:13s}{1:<s}\n".format("formula", sample_form_str))
+            f.write("{0:13s}{1:<s}\n".format("massdensity", self._mass_den[_label]))
+            f.write("{0:13s}{1:<s}\n".format("radius", self._radius[_label]))
+            f.write("{0:13s}{1:<s}\n".format("packfrac", self._pack_frac[_label]))
+            f.write("{0:13s}{1:<s}\n".format("geometry", self._geom[_label]))
+            [r_range_min, r_range_max] = o_gui_handler.get_r_range()
+            r_max_possible = "{0:6.2F}".format(np.pi / self.q_interval_min)
+            if r_range_min and r_range_max:
+                f.write("{0:13s}{1:<s}\n".format("rfilter", r_range_min + "," + r_range_max + "," + r_max_possible))
+            if q_range_min and q_range_max:
+                f.write("{0:13s}{1:<s}\n".format("qfilter", q_range_min + "," + q_range_max))
+            f.close()
+
     def collect_runs_checked(self):
         table = self.parent.table
+        _runs_name = {}
         _runs = {}
+        _sam_formula = {}
+        _mass_den = {}
+        _radius = {}
+        _pack_frac = {}
+        _geom = {}
         for _row_index in range(table.rowCount()):
             _selected_widget = table.cellWidget(_row_index, 0).children()[1]
             if (_selected_widget.checkState() == Qt.Checked):
-                _label = str(table.item(_row_index, 1).text())
-                _value = str(table.item(_row_index, 2).text())
-                _runs[_label] = _value
+                _runs = self.load_table(_runs, table, _row_index, 2)
+                # for Joerg's new script.
+                _runs_name = self.load_table(_runs_name, table, _row_index, 1)
+                _sam_formula = self.load_table(_sam_formula, table, _row_index, 3)
+                _mass_den = self.load_table(_mass_den, table, _row_index, 4)
+                _radius = self.load_table(_radius, table, _row_index, 5)
+                _pack_frac= self.load_table(_pack_frac, table, _row_index, 6)
+                _geom = self.load_table(_geom, table, _row_index, 7)
 
-        return _runs
+        return [_runs, _runs_name, _sam_formula, _mass_den, _radius, _pack_frac, _geom]
 
     def collect_background_runs(self):
         if self.parent.background_no.isChecked():
@@ -112,3 +171,16 @@ def collect_background_runs(self):
         else:
             _background = str(self.parent.background_line_edit.text())
         return _background
+
+    def load_table(self, dict_in, table_in, row, col):
+        _label = str(table_in.item(row, 1).text())
+        if col != 7:
+            _value = str(table_in.item(row, col).text())
+        else:
+            _widget = table_in.cellWidget(row, col)
+            _selected_index = _widget.currentIndex()
+            _value = _widget.itemText(_selected_index)
+        dict_in[_label] = _value
+
+        return dict_in
+
diff --git a/addie/processing/idl/step2_gui_handler.py b/addie/processing/idl/step2_gui_handler.py
index ddb405a8..c964f2da 100644
--- a/addie/processing/idl/step2_gui_handler.py
+++ b/addie/processing/idl/step2_gui_handler.py
@@ -60,6 +60,11 @@ def get_q_range(self):
         q_range_max = self.main_window.postprocessing_ui.q_range_max.text().strip()
         return [q_range_min, q_range_max]
 
+    def get_r_range(self):
+        r_range_min = self.main_window.postprocessing_ui.fourier_filter_from.text().strip()
+        r_range_max = self.main_window.postprocessing_ui.fourier_filter_to.text().strip()
+        return [r_range_min, r_range_max]
+
     def step2_background_flag(self):
         if self.main_window.postprocessing_ui.background_no.isChecked():
             self.no_background_clicked()
diff --git a/addie/processing/mantid/master_table/periodic_table/material_handler.py b/addie/processing/mantid/master_table/periodic_table/material_handler.py
index 8100fbda..4d48b531 100644
--- a/addie/processing/mantid/master_table/periodic_table/material_handler.py
+++ b/addie/processing/mantid/master_table/periodic_table/material_handler.py
@@ -21,27 +21,31 @@ def get_periodictable_formatted_element_and_number_of_atoms(element):
     regular_expression_1 = r'^(?P<stable_element>[A-Z]{1}[a-z]{0,1}$)'
     m1 = re.search(regular_expression_1, element)
     if m1 is not None:
-        return [m1.group('stable_element'), 1.]
+        case = 1
+        return [m1.group('stable_element'), 1., case]
 
     # stable with stochiometric coefficient
     regular_expression_2 = r'^(?P<stable_element>[A-Z]{1}[a-z]{0,1})(?P<stochiometric_coefficient>\d*\.{0,1}\d*)$'
     m2 = re.search(regular_expression_2, element)
     if m2 is not None:
+        case = 2
         return ["{}{}".format(m2.group('stable_element'), m2.group('stochiometric_coefficient')),
-                np.float(m2.group('stochiometric_coefficient'))]
+                np.float(m2.group('stochiometric_coefficient')), case]
 
     # isotope with or without stochiometric coefficient
     regular_expression_3 = r'\((?P<isotope_element>[A-Z]{1}[a-z]{0,1})(?P<isotope_number>\d+)\)' \
                            r'(?P<stochiometric_coefficient>\d*\.{0,1}\d*)'
     m3 = re.search(regular_expression_3, element)
     if m3 is not None:
+        case = 3
         if m3.group('stochiometric_coefficient') == "":
             number_of_atoms = 1.
         else:
             number_of_atoms = np.float(
                 m3.group('stochiometric_coefficient'))
         return ["{}[{}]{}".format(m3.group('isotope_element'), m3.group('isotope_number'),
-                                  m3.group('stochiometric_coefficient')), number_of_atoms]
+                                  m3.group('stochiometric_coefficient')),
+                                  number_of_atoms, case]
 
     raise ValueError
 

From 007125920082350986f383b7ffa20720b8de35da Mon Sep 17 00:00:00 2001
From: Yuanpeng Zhang <zyroc1990@gmail.com>
Date: Thu, 25 Mar 2021 17:06:59 -0400
Subject: [PATCH 2/5] fix the flake8 error

---
 .../mantid/master_table/periodic_table/material_handler.py     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/addie/processing/mantid/master_table/periodic_table/material_handler.py b/addie/processing/mantid/master_table/periodic_table/material_handler.py
index 4d48b531..8e677259 100644
--- a/addie/processing/mantid/master_table/periodic_table/material_handler.py
+++ b/addie/processing/mantid/master_table/periodic_table/material_handler.py
@@ -44,8 +44,7 @@ def get_periodictable_formatted_element_and_number_of_atoms(element):
             number_of_atoms = np.float(
                 m3.group('stochiometric_coefficient'))
         return ["{}[{}]{}".format(m3.group('isotope_element'), m3.group('isotope_number'),
-                                  m3.group('stochiometric_coefficient')),
-                                  number_of_atoms, case]
+                                  m3.group('stochiometric_coefficient')), number_of_atoms, case]
 
     raise ValueError
 

From 79ecb841e68d4a8abe95fa14e30b8b5c5a77ffdd Mon Sep 17 00:00:00 2001
From: Yuanpeng Zhang <zyroc1990@gmail.com>
Date: Thu, 25 Mar 2021 21:45:56 -0400
Subject: [PATCH 3/5] cleaning up codes for redpar preparation

---
 addie/processing/idl/run_sum_scans.py | 59 ++++++++++++---------------
 1 file changed, 27 insertions(+), 32 deletions(-)

diff --git a/addie/processing/idl/run_sum_scans.py b/addie/processing/idl/run_sum_scans.py
index f0c5bc59..3b72e251 100644
--- a/addie/processing/idl/run_sum_scans.py
+++ b/addie/processing/idl/run_sum_scans.py
@@ -30,14 +30,7 @@ def set_sum_scans_script(self):
 
     def run(self):
         self._background = self.collect_background_runs()
-        collect_return = self.collect_runs_checked()
-        self._runs = collect_return[0]
-        self._runs_name = collect_return[1]
-        self._sam_formula = collect_return[2]
-        self._mass_den = collect_return[3]
-        self._radius = collect_return[4]
-        self._pack_frac = collect_return[5]
-        self._geom = collect_return[6]
+        self._runs = self.collect_runs_checked()
         self.create_output_file()
         self.run_script()
 
@@ -75,7 +68,7 @@ def create_output_file(self):
         f = open(_full_output_file_name, 'w')
 
         for _label in self._runs:
-            f.write("%s %s\n" % (_label, self._runs[_label]))
+            f.write("%s %s\n" % (_label, self._runs[_label]["runs"]))
         f.write("endsamples\n")
         f.write("Background %s\n" % self._background)
 
@@ -111,8 +104,8 @@ def create_output_file(self):
 
         # redpar file
         for _label in self._runs:
-            f = open(self._runs_name[_label] + ".redpar", "w")
-            chem_form_temp = self._sam_formula[_label]
+            f = open(self._runs[_label]["sam_name"] + ".redpar", "w")
+            chem_form_temp = self._runs[_label]["sam_formula"]
             list_element = chem_form_temp.split(" ")
             formated_ele_list = []
             for _element in list_element:
@@ -130,10 +123,10 @@ def create_output_file(self):
                 sample_form_str += to_append
             sample_form_str = sample_form_str[:-1]
             f.write("{0:13s}{1:<s}\n".format("formula", sample_form_str))
-            f.write("{0:13s}{1:<s}\n".format("massdensity", self._mass_den[_label]))
-            f.write("{0:13s}{1:<s}\n".format("radius", self._radius[_label]))
-            f.write("{0:13s}{1:<s}\n".format("packfrac", self._pack_frac[_label]))
-            f.write("{0:13s}{1:<s}\n".format("geometry", self._geom[_label]))
+            f.write("{0:13s}{1:<s}\n".format("massdensity", self._runs[_label]["mass_density"]))
+            f.write("{0:13s}{1:<s}\n".format("radius", self._runs[_label]["radius"]))
+            f.write("{0:13s}{1:<s}\n".format("packfrac", self._runs[_label]["packing_fraction"]))
+            f.write("{0:13s}{1:<s}\n".format("geometry", self._runs[_label]["geometry"]))
             [r_range_min, r_range_max] = o_gui_handler.get_r_range()
             r_max_possible = "{0:6.2F}".format(np.pi / self.q_interval_min)
             if r_range_min and r_range_max:
@@ -144,26 +137,17 @@ def create_output_file(self):
 
     def collect_runs_checked(self):
         table = self.parent.table
-        _runs_name = {}
         _runs = {}
-        _sam_formula = {}
-        _mass_den = {}
-        _radius = {}
-        _pack_frac = {}
-        _geom = {}
         for _row_index in range(table.rowCount()):
             _selected_widget = table.cellWidget(_row_index, 0).children()[1]
+            _label = str(table.item(_row_index, 1).text())
+            _runs[_label] = {}
             if (_selected_widget.checkState() == Qt.Checked):
-                _runs = self.load_table(_runs, table, _row_index, 2)
                 # for Joerg's new script.
-                _runs_name = self.load_table(_runs_name, table, _row_index, 1)
-                _sam_formula = self.load_table(_sam_formula, table, _row_index, 3)
-                _mass_den = self.load_table(_mass_den, table, _row_index, 4)
-                _radius = self.load_table(_radius, table, _row_index, 5)
-                _pack_frac= self.load_table(_pack_frac, table, _row_index, 6)
-                _geom = self.load_table(_geom, table, _row_index, 7)
+                for i in range(7):
+                    _runs = self.load_table(_runs, table, _label, _row_index, i + 1)
 
-        return [_runs, _runs_name, _sam_formula, _mass_den, _radius, _pack_frac, _geom]
+        return _runs
 
     def collect_background_runs(self):
         if self.parent.background_no.isChecked():
@@ -172,15 +156,26 @@ def collect_background_runs(self):
             _background = str(self.parent.background_line_edit.text())
         return _background
 
-    def load_table(self, dict_in, table_in, row, col):
-        _label = str(table_in.item(row, 1).text())
+    def load_table(self, dict_in, table_in, _label, row, col):
         if col != 7:
             _value = str(table_in.item(row, col).text())
+            if col == 1:
+                dict_in[_label]["sam_name"] = _value
+            elif col == 2:
+                dict_in[_label]["runs"] = _value
+            elif col == 3:
+                dict_in[_label]["sam_formula"] = _value
+            elif col == 4:
+                dict_in[_label]["mass_density"] = _value
+            elif col == 5:
+                dict_in[_label]["radius"] = _value
+            elif col == 6:
+                dict_in[_label]["packing_fraction"] = _value
         else:
             _widget = table_in.cellWidget(row, col)
             _selected_index = _widget.currentIndex()
             _value = _widget.itemText(_selected_index)
-        dict_in[_label] = _value
+            dict_in[_label]["geometry"] = _value
 
         return dict_in
 

From 7cb2b1b45949c7b116a52623908dad8b6070e149 Mon Sep 17 00:00:00 2001
From: Yuanpeng Zhang <zyroc1990@gmail.com>
Date: Thu, 25 Mar 2021 21:58:34 -0400
Subject: [PATCH 4/5] fix flake8 issue

---
 addie/processing/idl/run_sum_scans.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/addie/processing/idl/run_sum_scans.py b/addie/processing/idl/run_sum_scans.py
index 3b72e251..27f543d2 100644
--- a/addie/processing/idl/run_sum_scans.py
+++ b/addie/processing/idl/run_sum_scans.py
@@ -2,7 +2,8 @@
 import os
 from qtpy.QtCore import Qt
 from addie.processing.idl.step2_gui_handler import Step2GuiHandler
-from addie.processing.mantid.master_table.periodic_table.material_handler import get_periodictable_formatted_element_and_number_of_atoms
+from addie.processing.mantid.master_table.periodic_table.material_handler \
+    import get_periodictable_formatted_element_and_number_of_atoms
 import numpy as np
 import re
 
@@ -178,4 +179,3 @@ def load_table(self, dict_in, table_in, _label, row, col):
             dict_in[_label]["geometry"] = _value
 
         return dict_in
-

From 7ea26bcf16844bd24d774078a7e5bb29ea1384f9 Mon Sep 17 00:00:00 2001
From: Yuanpeng Zhang <zyroc1990@gmail.com>
Date: Fri, 26 Mar 2021 11:17:42 -0400
Subject: [PATCH 5/5] clean up codes and add in test file

---
 addie/processing/idl/run_sum_scans.py | 73 +++++++++++++++------------
 tests/test_files/SiO2.redpar          |  7 +++
 2 files changed, 49 insertions(+), 31 deletions(-)
 create mode 100644 tests/test_files/SiO2.redpar

diff --git a/addie/processing/idl/run_sum_scans.py b/addie/processing/idl/run_sum_scans.py
index 27f543d2..ed945e79 100644
--- a/addie/processing/idl/run_sum_scans.py
+++ b/addie/processing/idl/run_sum_scans.py
@@ -3,7 +3,7 @@
 from qtpy.QtCore import Qt
 from addie.processing.idl.step2_gui_handler import Step2GuiHandler
 from addie.processing.mantid.master_table.periodic_table.material_handler \
-    import get_periodictable_formatted_element_and_number_of_atoms
+    import get_periodictable_formatted_element_and_number_of_atoms as format_ele
 import numpy as np
 import re
 
@@ -104,37 +104,48 @@ def create_output_file(self):
         print("[LOG] created file %s" % _full_output_file_name)
 
         # redpar file
+        r_max_possible = "{0:6.2F}".format(np.pi / self.q_interval_min)
+        r_range = o_gui_handler.get_r_range(r_max_possible)
+        self._runs["r_range"] = r_range
+        self._runs["q_range"] = [q_range_min, q_range_max]
         for _label in self._runs:
-            f = open(self._runs[_label]["sam_name"] + ".redpar", "w")
-            chem_form_temp = self._runs[_label]["sam_formula"]
-            list_element = chem_form_temp.split(" ")
-            formated_ele_list = []
-            for _element in list_element:
-                [formated_element, number_of_atoms, case] = get_periodictable_formatted_element_and_number_of_atoms(_element)
-                formated_ele_list.append(formated_element)
-            sample_form_str = ""
-            for item in formated_ele_list:
-                if case == 1:
-                    to_append = item + "_"
-                elif case == 2:
-                    to_append = re.findall('[A-Z][a-z]*', item)[-1]
-                    to_append += ("_" + re.findall('[0-9]+', item)[-1] + "_")
-                else:
-                    to_append = item.split("[")[1].split("]")[0] + item.split("[")[0] + "_" + item.split("]")[1] + "_"
-                sample_form_str += to_append
-            sample_form_str = sample_form_str[:-1]
-            f.write("{0:13s}{1:<s}\n".format("formula", sample_form_str))
-            f.write("{0:13s}{1:<s}\n".format("massdensity", self._runs[_label]["mass_density"]))
-            f.write("{0:13s}{1:<s}\n".format("radius", self._runs[_label]["radius"]))
-            f.write("{0:13s}{1:<s}\n".format("packfrac", self._runs[_label]["packing_fraction"]))
-            f.write("{0:13s}{1:<s}\n".format("geometry", self._runs[_label]["geometry"]))
-            [r_range_min, r_range_max] = o_gui_handler.get_r_range()
-            r_max_possible = "{0:6.2F}".format(np.pi / self.q_interval_min)
-            if r_range_min and r_range_max:
-                f.write("{0:13s}{1:<s}\n".format("rfilter", r_range_min + "," + r_range_max + "," + r_max_possible))
-            if q_range_min and q_range_max:
-                f.write("{0:13s}{1:<s}\n".format("qfilter", q_range_min + "," + q_range_max))
-            f.close()
+            if _label != "r_range" and _label != "q_range":
+                self._runs = self.collect_info_for_redpar(self._runs, _label)
+                self.write_redpar(self._runs, _label)
+
+    def collect_info_for_redpar(self, runs_info, _label):
+        chem_form_temp = runs_info[_label]["sam_formula"]
+        list_element = chem_form_temp.split(" ")
+        formated_ele_list = []
+        for _element in list_element:
+            [formated_element, number_of_atoms, case] = format_ele(_element)
+            formated_ele_list.append(formated_element)
+        sample_form_str = ""
+        for item in formated_ele_list:
+            if case == 1:
+                to_append = item + "_"
+            elif case == 2:
+                to_append = re.findall('[A-Z][a-z]*', item)[-1]
+                to_append += ("_" + re.findall('[0-9]+', item)[-1] + "_")
+            else:
+                to_append = item.split("[")[1].split("]")[0] + item.split("[")[0] + "_" + item.split("]")[1] + "_"
+            sample_form_str += to_append
+        runs_info[_label]["sam_form_used"] = sample_form_str[:-1]
+
+        return runs_info
+
+    def write_redpar(self, runs_info, _label):
+        file_name = runs_info[_label]["sam_name"] + ".redpar"
+        with open(file_name, 'w') as f:
+            f.write("{0:13s}{1:<s}\n".format("formula", runs_info[_label]["sam_form_used"]))
+            f.write("{0:13s}{1:<s}\n".format("massdensity", runs_info[_label]["mass_density"]))
+            f.write("{0:13s}{1:<s}\n".format("radius", runs_info[_label]["radius"]))
+            f.write("{0:13s}{1:<s}\n".format("packfrac", runs_info[_label]["packing_fraction"]))
+            f.write("{0:13s}{1:<s}\n".format("geometry", runs_info[_label]["geometry"]))
+            if None not in runs_info["r_range"]:
+                f.write("{0:13s}{1:<s}\n".format("rfilter", ",".join(runs_info["r_range"])))
+            if None not in runs_info["q_range"]:
+                f.write("{0:13s}{1:<s}\n".format("qfilter", ",".join(runs_info["q_range"])))
 
     def collect_runs_checked(self):
         table = self.parent.table
diff --git a/tests/test_files/SiO2.redpar b/tests/test_files/SiO2.redpar
new file mode 100644
index 00000000..dc293a88
--- /dev/null
+++ b/tests/test_files/SiO2.redpar
@@ -0,0 +1,7 @@
+formula      Si_1_O_2
+massdensity  2.5
+radius       0.15
+packfrac     0.7
+geometry     Cylinder
+rfilter      1.5,50,157.08
+qfilter      0.2,31.4