Merge remote-tracking branch 'origin/master'

.gitignore

@@ -113,7 +113,7 @@ venv.bak/
 .mypy_cache/
 
 
-datasets/dataset_images/**
+datasets/dataset_images/*/
 datasets/dataset_images/dataset_archives/*
 
 models/cnn_1/saved_models

datasets/dataset_images/download_links.txt

@@ -0,0 +1,8 @@
+http://services.futurelabsolutions.com:6112/dataset_download?dataset_folder_name=wielder_non-wielder
+
+http://services.futurelabsolutions.com:6112/dataset_download?dataset_folder_name=cifar10
+
+http://services.futurelabsolutions.com:6112/dataset_download?dataset_folder_name=congested_non-congested_resized
+
+http://services.futurelabsolutions.com:6112/dataset_download?dataset_folder_name=congested_non-congested
+

datasets/datasets.json

@@ -56,9 +56,9 @@
 	"license":"apache 2"
 	},
 	{
-	"dataset_name":"Cifar 10",
+	"dataset_name":"CIFAR-10",
 	"download_link":"",
-	"folder":"cifar10",
+	"folder":"cifar-10",
 	"labels":[
 				{"label":"airplane","images":4100},
 				{"label":"automobile","images":4107},
@@ -74,11 +74,11 @@
 	"image_x":32,
 	"image_y":32,
 	"image_channels":3,
-	"ground_truth_csv_path":"congested_non-congested_resized.csv",
-	"default_training_allocation_path":"congested_non-congested_resized_train_split.csv",
-	"interesting_images":["00140_congested.jpg","00182_congested.jpg","00605_congested.jpg","01989_non-congested.jpg","01708_non-congested.jpg"],
-	"references":["https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk", "https://osf.io/wm3t9/"],
-	"description":"Resized version of the first traffic congestion image classification dataset.Image classification of traffic camera imagery collected from Transport for London's 'Jamcams' api. The specific set of images collected and labeled is hosted on OSF. The images have been cropped to remove logo and overlaid text.",
+	"ground_truth_csv_path":"cifar-10.csv",
+	"default_training_allocation_path":"cifar-10_train_split.csv",
+	"interesting_images":["39485_truck.jpg","35796_ship.jpg","01228_airplane.jpg","16669_deer.jpg","24730_frog.jpg"],
+	"references":[""],
+	"description":"Dataset commonly used for benchmarking Machine Learning techniques.",
 	"license":"apache 2"
 	}
 

explanations/explanations.json

@@ -5,16 +5,15 @@
 		"class_name":"LimeExplainer",
 		"compatible_models":[
 			{"model_name":"conv_svm"},
-			{"model_name":"cnn_1"},
-			{"model_name":"keras_cnn"},
-			{"model_name":"keras_vgg"},
-			{"model_name":"keras_api_vgg"},
-			{"model_name":"keras_vgg"}
+			{"model_name":"keras_api_vgg"}
 				],
 		"compatible_datasets":[
 			{"dataset_name":"Gun Wielding Image Classification"},
 			{"dataset_name":"Traffic Congestion Image Classification"},
-			{"dataset_name":"Traffic Congestion Image Classification (Resized)"}
+			{"dataset_name":"Traffic Congestion Image Classification (Resized)"},
+			{"dataset_name":"CIFAR-10"}
+
+
 				],
 		"modality":"imagery",
 		"description":"A local (example specific) decision-boundary explanation of evidence towards classes",
@@ -50,14 +49,13 @@
 		"class_name":"ShapExplainer",
 		"compatible_models":[
 			{"model_name":"keras_cnn"},
-			{"model_name":"keras_vgg"},
-			{"model_name":"keras_api_vgg"},
-			{"model_name":"keras_vgg"}
-				],
+			{"model_name":"keras_api_vgg"}
+			],
 		"compatible_datasets":[
 			{"dataset_name":"Gun Wielding Image Classification"},
 			{"dataset_name":"Traffic Congestion Image Classification"},
-			{"dataset_name":"Traffic Congestion Image Classification (Resized)"}
+			{"dataset_name":"Traffic Congestion Image Classification (Resized)"},
+			{"dataset_name":"CIFAR-10"}
 				],
 		"modality":"imagery",
 		"description":"",
@@ -84,13 +82,12 @@
 		"class_name":"InfluenceExplainer",
 		"compatible_models":[
 			{"model_name":"conv_svm"},
-			{"model_name":"keras_vgg"},
-			{"model_name":"keras_api_vgg"},
-			{"model_name":"keras_vgg"}				],
+			{"model_name":"keras_api_vgg"}				],
 		"compatible_datasets":[
 			{"dataset_name":"Gun Wielding Image Classification"},
 			{"dataset_name":"Traffic Congestion Image Classification"},
-			{"dataset_name":"Traffic Congestion Image Classification (Resized)"}
+			{"dataset_name":"Traffic Congestion Image Classification (Resized)"},
+			{"dataset_name":"CIFAR-10"}
 				],
 		"modality":"imagery",
 		"description":"An explanation by example method that finds accurate approximations of the difference in loss at a test image due caused by retraining the model with the exclusion of a train image",
@@ -122,7 +119,8 @@
 		"compatible_datasets":[
 			{"dataset_name":"Gun Wielding Image Classification"},
 			{"dataset_name":"Traffic Congestion Image Classification"},
-			{"dataset_name":"Traffic Congestion Image Classification (Resized)"}
+			{"dataset_name":"Traffic Congestion Image Classification (Resized)"},
+			{"dataset_name":"CIFAR-10"}
 				],
 		"modality":"imagery",
 		"description":"",

explanations/influence_functions/influence_explanations.py

@@ -11,7 +11,7 @@
 import PIL.Image as Pimage
 
 class InfluenceExplainer(object):
-    def __init__(self,model, train_imgs, train_lbls, additional_args):
+    def __init__(self,model):
         super(InfluenceExplainer, self).__init__()
         self.model = model
         self.sess = model.sess
@@ -28,21 +28,11 @@ def __init__(self,model, train_imgs, train_lbls, additional_args):
         ]
         self.grad_loss = [tf.reshape(grad, [-1,]) for grad in self.model.GetGradLoss()]
         self.hvp = self._get_approx_hvp(self.grad_loss, self.params, self.v_placeholder)
-        self.train_imgs = train_imgs
-        self.train_lbls = train_lbls
         self.vec_to_list = self._get_vec_to_list()
         self.cached_influence = {}
-        if "damping" in additional_args:
-            self.damping = additional_args["damping"]
-        else:
-            self.damping = 0.0    
-        if "mini_batch" in additional_args:
-            self.mini_batch = additional_args["mini_batch"]
-        else:
-            self.mini_batch = True
 
 
-    def Explain(self, test_img, n_max, additional_args):
+    def Explain(self, test_img, additional_args):
         """
         Explain test image by showing the training sampels with most influence
         on the models classification of the image.
@@ -59,12 +49,48 @@ def Explain(self, test_img, n_max, additional_args):
             n_max : number of train images to select for explanation, i.e.
             first n_max images from train set in order of descending 
         """
+
+        if len(test_img.shape) < 4:
+            test_img = np.expand_dims(test_img, axis = 0) 
+
+        ### no default additional arguments (REQUIRED)
+        print("influence function explain")
+
+        if "train_x" in additional_args:
+            self.train_imgs = additional_args["train_x"]
+        else:
+            raise ValueError('no training images passed in additional arguments')
+
+        if "train_y" in additional_args:
+            self.train_lbls = additional_args["train_y"]
+        else:
+            raise ValueError('no training labels passed in additional arguments')
+
+
         compute = True
+        if "max_n_influence_images" in additional_args:
+            self.n_max = additional_args["max_n_influence_images"]
+        else:
+            self.n_max = 9
+
         if "label" in additional_args:
             label = additional_args["label"]
         else:
             label = self.model.Predict(test_img)
+
+        if(isinstance(label,list)):
+            label = np.array(label)
         label = label.reshape(-1, 1)
+
+        if "damping" in additional_args:
+            self.damping = additional_args["damping"]
+        else:
+            self.damping = 0.0
+        if "mini_batch" in additional_args:
+            self.mini_batch = additional_args["mini_batch"]
+        else:
+            self.mini_batch = True
+
         """
         If cache is true, and the image has not been cached before, the resulting max_imgs list will be cached under the test_img
         If cache is true, and the image has been cached previously, the previously cached results are returned
@@ -81,8 +107,8 @@ def Explain(self, test_img, n_max, additional_args):
         else:
             cache = False
 
-        if len(test_img.shape) < 4:
-            test_img = np.expand_dims(test_img, axis = 0) 
+
+        print("test_img.shape",test_img.shape)
         feed_dict = {
             self.input_: test_img,
             self.labels_: label
@@ -102,7 +128,7 @@ def Explain(self, test_img, n_max, additional_args):
 
         max_imgs = self.FormCollage(max_imgs)
 
-        return max_imgs
+        return max_imgs, "", np.argmax(label), {}
 
 
 

explanations/lrp/lrp_explanation.py

@@ -57,7 +57,7 @@ def GenerateLRPExplanationImage(self,input_image,LRP_values):
         abs_vals = np.stack([np.abs(LRP_values[i]) for i in range(len(LRP_values))], 0).flatten()
     else:
         abs_vals = np.stack([np.abs(LRP_values[i].sum(-1)) for i in range(len(LRP_values))], 0).flatten()
-    max_val = np.nanpercentile(abs_vals, 95)
+    max_val = np.nanpercentile(abs_vals, 70)
 
     # i = 0
 

explanations/shap/shap_explanation.py

@@ -15,6 +15,9 @@
 import sys
 import json
 
+import tensorflow as tf
+from keras.backend.tensorflow_backend import set_session
+
 class ShapExplainer(object):
   """docstring for LimeExplainer"""
   def __init__(self, model):
@@ -163,7 +166,7 @@ def Explain(self,input_image, additional_args = {}):
     if("num_background_samples" in additional_args):
       num_background_samples=additional_args["num_background_samples"]
     else:
-      num_background_samples=100
+      num_background_samples=10
 
     if("background_image_pool" in additional_args):
       background_image_pool=additional_args["background_image_pool"]
@@ -188,7 +191,11 @@ def Explain(self,input_image, additional_args = {}):
     else:
       print("generating new shap explainer")
       background = background_image_pool[np.random.choice(background_image_pool.shape[0], num_background_samples, replace=False)]
-
+      print("backgroun images selected, creating explainer")
+      # config = tf.ConfigProto()
+      # config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU
+      # sess = tf.Session(config=config)
+      # set_session(sess)  # set this TensorFlow session as the default session for Keras
       e = shap.DeepExplainer(self.model.model, background)
       self.shap_explainers_dict[additional_args["dataset_name"]][self.model.__class__.__name__] = e
 
@@ -209,12 +216,14 @@ def Explain(self,input_image, additional_args = {}):
 
     ## for testing:
     # shap.image_plot(shap_values, np.multiply(input_image,255.0))
+    show_explanation_image = False
 
-    cv2_image = cv2.cvtColor(explanation_image, cv2.COLOR_RGB2BGR)
-    cv2.imshow("explanation image",cv2_image)
-    cv2.waitKey(0)
-    cv2.destroyAllWindows()
-
+    if(show_explanation_image):
+      cv2_image = cv2.cvtColor(explanation_image, cv2.COLOR_RGB2BGR)
+      cv2.imshow("explanation image",cv2_image)
+      cv2.waitKey(0)
+      cv2.destroyAllWindows()
+
     additional_outputs = {"shap_values":[shap_value.tolist() for shap_value in shap_values]}
 
     explanation_text = "Evidence towards predicted class shown in red, evidence against shown in blue."

flask_services/DatasetsAAS.py

@@ -132,10 +132,11 @@ def EncodeTestImages(dataset_name,num_images=1, model_name = ""):
 		dataset_tools[dataset_name] = {"dataset_tool":dataset_tool,"label_names":label_names}
 
 	label_names = dataset_tools[dataset_name]["label_names"]
+	n_classes = len(label_names)
 	source = "test"
-
-	x, y, batch = dataset_tools[dataset_name]["dataset_tool"].GetBatch(batch_size = math.ceil(num_images / 2.) * 2,even_examples=True, y_labels_to_use=label_names, split_batch = True,split_one_hot = False, batch_source = source, return_batch_data=True)
-
+	print("label_names",label_names)
+	x, y, batch = dataset_tools[dataset_name]["dataset_tool"].GetBatch(batch_size = math.ceil(num_images / float(n_classes)) * n_classes,even_examples=True, y_labels_to_use=label_names, split_batch = True,split_one_hot = False, batch_source = source, return_batch_data=True)
+	print("len(x)",len(x))
 	random_i_list = random.sample(list(range(0,len(x))),num_images)
 
 	enc_image_list = []