compatibilty for tensorflow-v2

dataset_tool.py

@@ -16,7 +16,8 @@
 import six.moves.queue as Queue # pylint: disable=import-error
 import traceback
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 import PIL.Image
 import dnnlib.tflib as tflib
 
@@ -500,13 +501,42 @@ def create_celeba(tfrecord_dir, celeba_dir, cx=89, cy=121):
 
 #----------------------------------------------------------------------------
 
+import pandas as pd
+from sklearn import preprocessing
+from ipaddress import ip_address
+
+def myfunc(img):
+    return np.reshape(img, (-1, 16))
+
 def create_from_images(tfrecord_dir, image_dir, shuffle):
-    print('Loading images from "%s"' % image_dir)
+    print('Loading csv from "%s"' % image_dir)
     image_filenames = sorted(glob.glob(os.path.join(image_dir, '*')))
     if len(image_filenames) == 0:
-        error('No input images found')
-
-    img = np.asarray(PIL.Image.open(image_filenames[0]))
+        error('No input csv found')
+    csvs = []
+    for f in image_filenames:
+        df = pd.read_csv(f)
+        df = df.drop(df.columns[0], axis=1)
+        try:
+            df['src'] = df['src'].apply(lambda x: int(ip_address(x)))
+            df['dst'] = df['dst'].apply(lambda x: int(ip_address(x)))
+        except: pass
+        df = df.fillna(0)
+        scaler = preprocessing.MinMaxScaler((0, 255))
+        scaler.fit(df)
+        df = scaler.transform(df)
+        csvs.append(df)
+
+    img = np.concatenate((csvs[0], csvs[1], csvs[2]), axis=1)
+    # print(img.shape)
+
+    # img = np.reshape(img, (-1, 2))
+    data = np.apply_along_axis(myfunc, axis=1, arr=img)
+    # print(data.shape)
+
+    img = data[0]
+    print(img.shape)
+
     resolution = img.shape[0]
     channels = img.shape[2] if img.ndim == 3 else 1
     if img.shape[1] != resolution:
@@ -517,9 +547,9 @@ def create_from_images(tfrecord_dir, image_dir, shuffle):
         error('Input images must be stored as RGB or grayscale')
 
     with TFRecordExporter(tfrecord_dir, len(image_filenames)) as tfr:
-        order = tfr.choose_shuffled_order() if shuffle else np.arange(len(image_filenames))
-        for idx in range(order.size):
-            img = np.asarray(PIL.Image.open(image_filenames[order[idx]]))
+        # order = tfr.choose_shuffled_order() if shuffle else np.arange(len(image_filenames))
+        for img in data:
+            # img = np.asarray(PIL.Image.open(image_filenames[order[idx]]))
             if channels == 1:
                 img = img[np.newaxis, :, :] # HW => CHW
             else:

dnnlib/tflib/autosummary.py

@@ -24,9 +24,11 @@
 
 from collections import OrderedDict
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 from tensorboard import summary as summary_lib
 from tensorboard.plugins.custom_scalar import layout_pb2
+from tensorboard.plugins.custom_scalar.summary import pb as custom_scalar_pb
 
 from . import tfutil
 from .tfutil import TfExpression
@@ -164,7 +166,7 @@ def finalize_autosummaries() -> None:
             margin = layout_pb2.MarginChartContent(series=series)
             charts.append(layout_pb2.Chart(title=chart_name, margin=margin))
         categories.append(layout_pb2.Category(title=cat_name, chart=charts))
-    layout = summary_lib.custom_scalar_pb(layout_pb2.Layout(category=categories))
+    layout = custom_scalar_pb(layout_pb2.Layout(category=categories))
     return layout
 
 def save_summaries(file_writer, global_step=None):

dnnlib/tflib/network.py

@@ -13,7 +13,8 @@
 import uuid
 import sys
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 
 from collections import OrderedDict
 from typing import Any, List, Tuple, Union

dnnlib/tflib/optimizer.py

@@ -8,7 +8,8 @@
 """Helper wrapper for a Tensorflow optimizer."""
 
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 
 from collections import OrderedDict
 from typing import List, Union
@@ -39,7 +40,7 @@ class Optimizer:
 
     def __init__(self,
                  name: str = "Train",
-                 tf_optimizer: str = "tf.train.AdamOptimizer",
+                 tf_optimizer: str = "tf.compat.v1.train.AdamOptimizer",
                  learning_rate: TfExpressionEx = 0.001,
                  use_loss_scaling: bool = False,
                  loss_scaling_init: float = 64.0,

dnnlib/tflib/tfutil.py

@@ -9,7 +9,8 @@
 
 import os
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 
 from typing import Any, Iterable, List, Union
 

ertza.md

@@ -0,0 +1,23 @@
+<img src="purdue-cs-logo.jpg" alt="drawing" width="450"/>
+
+# Title: Network Dataset Generation Using GANs
+
+**Problem Statement**
+Network datasets form the basis of many research problems falling under the computer networks domain. Despite the need, network datasets are rare due to privacy concerns and sensitive information leakage. We present this work as a way to overcome this problem and propose a Generative Adversarial Network approach to create new computer network datatsets.
+
+**Design and Implementation**
+We implement an LSTM-based GAN that learns on distinct 11 flow-based features extracted from network traffic. The GAN is then able to output features that our custom framework uses to communicate and capture real network traffic. This collection of traffic combined creates new datasets that can then be used in any application. In the second task of the project we implement a custom StyleGAN that converts network capture traffic into 16x16 grayscale images and then learns their most prominent features. We do this to identify the "face" of the network dataset as identified by StyleGAN (which is primarily used to generate images of human faces), and then generate new dataset features as images to visualize its face.
+
+
+**Results**
+We evaluated the traffic generated by our GAN to see its effectiveness in being able to represent the specific class of traffic it represents without actually being a part of that class. We were successful in showing that and GAN generated traffic was classified as the normal class by 6 highly effective and accurate classifiers. We also evaluated the features that were identified by StyleGAN as the face of the network data. 
+The most important features identified were:
+90th & 80th Percentile of PacketTimes, 90th & 80th Percentile of PacketTimesIn, skewPacketTimesIn, variancePacketTimesIn, skewPacketTimesOut, variancePacketTimesOut.
+Similarly, the least important features identified were:
+InBurst Kurtosis, InBurst Skewness, InBurst Stdev, InBurst Variance, InBurst total.
+
+Extrapolating the image generated by StyleGAN helps us visualize the face of the network data as shown below. 
+<img src="face.png" alt="drawing" width="200"/>
+
+> Link to GitHub repo: https://github.com/ertza/stylegan
+>

metrics/frechet_inception_distance.py

@@ -9,8 +9,10 @@
 
 import os
 import numpy as np
+import pickle
 import scipy
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 import dnnlib.tflib as tflib
 
 from metrics import metric_base
@@ -26,7 +28,9 @@ def __init__(self, num_images, minibatch_per_gpu, **kwargs):
 
     def _evaluate(self, Gs, num_gpus):
         minibatch_size = num_gpus * self.minibatch_per_gpu
-        inception = misc.load_pkl('https://drive.google.com/uc?id=1MzTY44rLToO5APn8TZmfR7_ENSe5aZUn') # inception_v3_features.pkl
+        # inception = misc.load_pkl('https://drive.google.com/file/d/1UFvPYshotkUnqcVgy50DZn5IzJ95ubkx/') # inception_v3_features.pkl
+        with open("inception_v3_features.pkl", 'rb') as file:
+            inception = pickle.load(file, encoding='latin1')
         activations = np.empty([self.num_images, inception.output_shape[1]], dtype=np.float32)
 
         # Calculate statistics for reals.

metrics/linear_separability.py

@@ -10,7 +10,8 @@
 from collections import defaultdict
 import numpy as np
 import sklearn.svm
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 import dnnlib.tflib as tflib
 
 from metrics import metric_base

metrics/metric_base.py

@@ -11,7 +11,8 @@
 import time
 import hashlib
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 import dnnlib
 import dnnlib.tflib as tflib
 

metrics/perceptual_path_length.py

@@ -8,7 +8,8 @@
 """Perceptual Path Length (PPL)."""
 
 import numpy as np
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 import dnnlib.tflib as tflib
 
 from metrics import metric_base