Merge pull request #4 from alexdrydew/master

Linters support

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+max-line-length = 120
+extend-ignore = E203
+exclude =
+    model_export/tf2xla_pb2.py
+per-file-ignores =
+    legacy_code/test_script_data_v0.py:E402
+    legacy_code/test_script_data_v1_normed.py:E402

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto
+
+*.sh text eol=crlf

.github/workflows/push.yml

@@ -0,0 +1,22 @@
+name: linter
+on: [push]
+jobs:
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Python 3
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+      - name: Install flake8
+        run: python -m pip install flake8
+      - name: Run flake8
+        run: flake8
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: psf/black@stable
+        with:
+          version: '19.10b0'

combine_images.py

@@ -5,45 +5,46 @@
 
 
 def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('path_to_images', type=str)
-  parser.add_argument('--output_name', type=str, default='plots.png')
-
-  args = parser.parse_args()
-
-  variables = [
-      'crossing_angle',
-      'dip_angle',
-      'drift_length',
-      'pad_coord_fraction',
-      'time_bin_fraction',
-  ]
-
-  stats = [
-      'Mean0',
-      'Mean1',
-      'Sigma0^2',
-      'Sigma1^2',
-      'Cov01',
-      'Sum',
-  ]
-
-  img_path = Path(args.path_to_images)
-  images = [[Image.open(img_path / f'{s} vs {v}_amp_gt_1.png') for v in variables] for s in stats]
-
-  width, height = images[0][0].size
-
-  new_image = Image.new('RGB', (width * len(stats), height * len(variables)))
-
-  x_offset = 0
-  for img_line in images:
-      y_offset = 0
-      for img in img_line:
-          new_image.paste(img, (x_offset, y_offset))
-          y_offset += img.size[1]
-      x_offset += img.size[0]
-
-  new_image.save(img_path / args.output_name)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('path_to_images', type=str)
+    parser.add_argument('--output_name', type=str, default='plots.png')
+
+    args = parser.parse_args()
+
+    variables = [
+        'crossing_angle',
+        'dip_angle',
+        'drift_length',
+        'pad_coord_fraction',
+        'time_bin_fraction',
+    ]
+
+    stats = [
+        'Mean0',
+        'Mean1',
+        'Sigma0^2',
+        'Sigma1^2',
+        'Cov01',
+        'Sum',
+    ]
+
+    img_path = Path(args.path_to_images)
+    images = [[Image.open(img_path / f'{s} vs {v}_amp_gt_1.png') for v in variables] for s in stats]
+
+    width, height = images[0][0].size
+
+    new_image = Image.new('RGB', (width * len(stats), height * len(variables)))
+
+    x_offset = 0
+    for img_line in images:
+        y_offset = 0
+        for img in img_line:
+            new_image.paste(img, (x_offset, y_offset))
+            y_offset += img.size[1]
+        x_offset += img.size[0]
+
+    new_image.save(img_path / args.output_name)
+
 
 if __name__ == '__main__':
-  main()
+    main()

cuda_gpu_config.py

@@ -2,6 +2,7 @@
 
 import tensorflow as tf
 
+
 def setup_gpu(gpu_num=None):
     os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
     if gpu_num is not None:

data/preprocessing.py

@@ -7,6 +7,7 @@
 _THIS_PATH = Path(os.path.realpath(__file__)).parent
 _VERSION = 'data_v1'
 
+
 class Reader:
     def __init__(self, variables, types):
         assert len(variables) == len(types), 'Reader.__init__: variables and types have different length'
@@ -25,7 +26,6 @@ def build(self):
         return pd.DataFrame(self.data, columns=['evtId'] + self.vars).set_index('evtId')
 
 
-
 def raw_to_csv(fname_in=None, fname_out=None):
     if fname_in is None:
         fname_in = str(_THIS_PATH.joinpath(_VERSION, 'raw', 'digits.dat'))
@@ -38,10 +38,7 @@ def raw_to_csv(fname_in=None, fname_out=None):
     with open(fname_in, 'r') as f:
         lines = f.readlines()
 
-    reader_main = Reader(
-        variables = ['ipad', 'itime', 'amp'],
-        types     = [int   , int    , float]
-    )
+    reader_main = Reader(variables=['ipad', 'itime', 'amp'], types=[int, int, float])
 
     data_sources = [lines]
     readers = [reader_main]
@@ -50,54 +47,49 @@ def raw_to_csv(fname_in=None, fname_out=None):
         assert len(lines) % 2 == 0, 'raw_to_csv: Odd number of lines when expected even'
 
         if _VERSION == 'data_v2':
-            reader_features = Reader(
-                variables = ["crossing_angle", "dip_angle"],
-                types     = [float           , float      ]
-            )
+            reader_features = Reader(variables=["crossing_angle", "dip_angle"], types=[float, float])
         elif _VERSION == 'data_v3':
             reader_features = Reader(
-                variables = ["crossing_angle", "dip_angle", "drift_length"],
-                types     = [float           , float      , float         ]
+                variables=["crossing_angle", "dip_angle", "drift_length"], types=[float, float, float]
             )
         elif _VERSION == 'data_v4':
             reader_features = Reader(
-                variables = ["crossing_angle", "dip_angle", "drift_length", "pad_coordinate"],
-                types     = [float           , float      , float         , float           ]
+                variables=["crossing_angle", "dip_angle", "drift_length", "pad_coordinate"],
+                types=[float, float, float, float],
             )
         else:
             raise NotImplementedError
 
         lines, lines_angles = lines[1::2], lines[::2]
-        lines_angles = [' '.join(l.split()[1:]) for l in lines_angles]
+        lines_angles = [' '.join(line.split()[1:]) for line in lines_angles]
 
         data_sources = [lines, lines_angles]
         readers = [reader_main, reader_features]
 
     for evt_id, lines_tuple in enumerate(zip(*data_sources)):
         for r, l in zip(readers, lines_tuple):
             r.read_line(l, evt_id)
-            
+
     result = pd.concat([r.build() for r in readers], axis=1).reset_index()
     result.to_csv(fname_out, index=False)
 
+
 def read_csv_2d(filename=None, pad_range=(40, 50), time_range=(265, 280), strict=True):
     if filename is None:
         filename = str(_THIS_PATH.joinpath(_VERSION, 'csv', 'digits.csv'))
 
     df = pd.read_csv(filename)
 
-    sel = lambda df, col, limits: (df[col] >= limits[0]) & (df[col] < limits[1])
+    def sel(df, col, limits):
+        return (df[col] >= limits[0]) & (df[col] < limits[1])
 
     if 'drift_length' in df.columns:
         df['itime'] -= df['drift_length'].astype(int)
 
     if 'pad_coordinate' in df.columns:
         df['ipad'] -= df['pad_coordinate'].astype(int)
 
-    selection = (
-        sel(df, 'itime', time_range) &
-        sel(df, 'ipad' , pad_range )
-    )
+    selection = sel(df, 'itime', time_range) & sel(df, 'ipad', pad_range)
 
     if not selection.all():
         msg = f"WARNING: current selection ignores {(~selection).sum() / len(selection) * 100}% of the data!"
@@ -107,9 +99,8 @@ def read_csv_2d(filename=None, pad_range=(40, 50), time_range=(265, 280), strict
     g = df[selection].groupby('evtId')
 
     def convert_event(event):
-        result = np.zeros(dtype=float, shape=(pad_range [1] - pad_range [0],
-                                              time_range[1] - time_range[0]))
-
+        result = np.zeros(dtype=float, shape=(pad_range[1] - pad_range[0], time_range[1] - time_range[0]))
+
         indices = tuple(event[['ipad', 'itime']].values.T - np.array([[pad_range[0]], [time_range[0]]]))
         result[indices] = event.amp.values
 

dump_graph_model_v4.py

@@ -1,4 +1,4 @@
-import argparse, re
+import argparse
 from pathlib import Path
 
 import tensorflow as tf
@@ -9,6 +9,7 @@
 from models.utils import load_weights
 from run_model_v4 import load_config
 
+
 def main():
     parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
     parser.add_argument('--checkpoint_name', type=str, required=True)
@@ -31,36 +32,38 @@ def main():
     print("----" * 10)
     print("")
 
-    def epoch_from_name(name):
-        epoch, = re.findall('\d+', name)
-        return int(epoch)
-
     model_path = Path('saved_models') / args.checkpoint_name
 
     full_model = Model_v4(load_config(model_path / 'config.yaml'))
     load_weights(full_model, model_path)
     model = full_model.generator
 
     if args.constant_seed is None:
+
         def preprocess(x):
             size = tf.shape(x)[0]
             latent_input = tf.random.normal(shape=(size, args.latent_dim), dtype='float32')
             return tf.concat([preprocess_features(x), latent_input], axis=-1)
+
     else:
+
         def preprocess(x):
             size = tf.shape(x)[0]
             latent_input = tf.ones(shape=(size, args.latent_dim), dtype='float32') * args.constant_seed
             return tf.concat([preprocess_features(x), latent_input], axis=-1)
 
     def postprocess(x):
         x = 10 ** x - 1
-        return tf.where(x < 1., 0., x)
+        return tf.where(x < 1.0, 0.0, x)
 
     dump_graph.model_to_graph(
-        model, preprocess, postprocess,
+        model,
+        preprocess,
+        postprocess,
         input_signature=[tf.TensorSpec(shape=[None, 4], dtype=tf.float32)],
-        output_file=args.output_path, test_input=args.test_input,
-        hack_upsampling=not args.dont_hack_upsampling_op
+        output_file=args.output_path,
+        test_input=args.test_input,
+        hack_upsampling=not args.dont_hack_upsampling_op,
     )