AISOLA23 code, medminst3d verification

code/nnv/engine/nn/layers/AveragePooling3DLayer.m

@@ -200,7 +200,7 @@ function set_padding(obj, padding)
             %
             % @y: high-dimensional array (output volume), depth of output = number of filters
 
-            x = dlarray(input, "SSSC");
+            x = dlarray(input, "SSSCB");
             y = avgpool(x, obj.PoolSize, 'Stride', obj.Stride, 'Padding', obj.PaddingSize);
             y = extractdata(y);
 

code/nnv/engine/nn/layers/BatchNormalizationLayer.m

@@ -295,7 +295,9 @@
             % date: 1/7/2020
 
             n = length(in_images);
-            if isa(in_images(n), 'ImageStar')
+            if isa(in_images(n), 'VolumeStar')
+                images(n) = VolumeStar;
+            elseif isa(in_images(n), 'ImageStar')
                 images(n) = ImageStar; 
             else
                 images(n) = Star; 

code/nnv/engine/nn/layers/Conv3DLayer.m

@@ -407,7 +407,7 @@ function set_name(obj, name)
             % compute output sets
             c = dlconv(dlarray(input.V(:,:,:,:,1), "SSSC"), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Padding', obj.PaddingSize, 'DilationFactor', obj.DilationFactor);
             V = dlconv(dlarray(input.V(:,:,:,:,2:input.numPred + 1), "SSSCB"), obj.Weights, 0, 'Stride', obj.Stride, 'Padding', obj.PaddingSize, 'DilationFactor', obj.DilationFactor);        
-            Y = cat(4, extractdata(c), extractdata(V));
+            Y = cat(5, extractdata(c), extractdata(V));
             S = VolumeStar(Y, input.C, input.d, input.pred_lb, input.pred_ub);
 
         end

code/nnv/engine/nn/layers/FlattenLayer.m

@@ -158,7 +158,7 @@
             % author: Dung Tran
             % date: 6/9/2020
 
-            if isa(in_set, 'ImageStar') || ~isa(in_set, 'ImageZono')
+            if isa(in_set, 'ImageStar') || isa(in_set, 'ImageZono')
                 N = in_set.height*in_set.width*in_set.numChannel;
                 n = in_set.numPred;
                 V(1, 1, :, in_set.numPred + 1) = zeros(N, 1);        
@@ -196,7 +196,7 @@
             elseif isa(inputs(1), 'ImageZono')
                 S(n) = ImageZono;
             elseif isa(inputs(1), 'VolumeStar')
-                S(n) = VolumeStar;
+                S(n) = Star; % convert to Star as we move to a 2D set
             else
                 error('Unknown input data set');
             end

code/nnv/engine/nn/layers/FullyConnectedLayer.m

@@ -96,7 +96,16 @@
             % update: add support for sequence evaluation (neuralode, RNN)
             %   -date: 03/17/2023 (Diego Manzanas)
 
-            x = squeeze(x); % remove higher dimensions of 1
+            n = size(x);
+            if length(n) == 2
+                x = reshape(x, [n(1)*n(2) 1]);
+            elseif length(n) == 3
+                x = reshape(x, [n(1)*n(2)*n(3) 1]);
+            elseif length(n) == 4
+                x = reshape(x, [n(1)*n(2)*n(3)*n(4) 1]);
+            else
+                error('Invalid input');
+            end
 
             if size(x, 1) ~= size(obj.Weights, 2)
                 error('Inconsistent input vector')

code/nnv/examples/NN/medmnist/verify_3D_multipleAttacks.m

@@ -0,0 +1,76 @@
+%% Verify all possible 3D classification models for medmnist data
+
+
+medmnist_path = "data/mat_files/"; % path to data
+
+datasets = dir(medmnist_path+"*.mat");
+
+for i=1:length(datasets)
+
+    if endsWith(datasets(i).name, "3d.mat")
+
+        % get current dataset to verify
+        dataset = medmnist_path + datasets(i).name;
+
+        try % verification
+
+            disp("Begin verification of " + datasets(i).name);
+
+            % Load data
+            load(dataset);
+
+            % data to verify (test set)
+            test_images = permute(test_images, [2 3 4 5 1]);
+            test_labels = test_labels + 1;
+
+            % load network
+            load("models/model_"+string(datasets(i).name));
+            matlabNet = net;
+            net = matlab2nnv(net);
+
+            % adversarial attacks
+            names = ["dark";"bright"];
+            max_pixels = [50;100;200];
+            noise_vals = [1;2;3];
+
+            % select volumes to verify
+            N = 200;
+            inputs = test_images(:,:,:,:,1:N);
+            targets = test_labels(1:N);
+
+            % Initialize results 
+            results = zeros(2, N, length(names), length(max_pixels), length(noise_vals));
+
+            % verify volumes with all attack combos
+            for a=1:length(names)
+                for b=1:length(max_pixels)
+                    for c=1:length(noise_vals)
+                        % create attack from variables
+                        adv_attack = struct;
+                        adv_attack.Name = names(a);
+                        adv_attack.max_pixels = max_pixels(b);
+                        adv_attack.noise_de = noise_vals(c);
+                        if strcmp(names(a),'dark')
+                            adv_attack.threshold = 150;
+                        elseif strcmp(names(a), 'bright')
+                            adv_attack.threshold = 100;
+                        end
+                        % Compute verification
+                        results(:,:,a,b,c) = verify_medmnist3d(net, inputs, targets, adv_attack);
+                    end
+                end
+            end
+
+            % save results
+            save("results/verification_multipleAttacks_"+datasets(i).name, "results");
+
+        catch ME
+
+            warning("Failed!!")
+            warning(ME.message);
+            disp(medmnist_path+datasets(i).name)
+        end
+
+    end
+
+end

code/nnv/examples/NN/medmnist/verify_all_3D.m

@@ -29,22 +29,40 @@
             net = matlab2nnv(net);
 
             % adversarial attack
-            adv_attack = struct;
-            adv_attack.Name = "linf";
-            adv_attack.epsilon = 1; % {epsilon} color values
-            % adv_attack.max_pixels = 784; % Max number of pixels to modify from input image
-            adv_attack.max_pixels = 1; % Max number of pixels to modify from input image
+            attack_option = 'bright'; % default (darkening attack)
+            switch attack_option
+                case 'linf'
+                    adv_attack = struct;
+                    adv_attack.Name = "linf";
+                    adv_attack.epsilon = 1; % {epsilon} color values
+                    % adv_attack.max_pixels = 784; % Max number of pixels to modify from input image
+                    adv_attack.max_pixels = 1; % Max number of pixels to modify from input image
+                case 'dark'
+                    adv_attack = struct;
+                    adv_attack.Name = "dark";
+                    adv_attack.threshold = 150; % perturb pixels over this value
+                    adv_attack.max_pixels = 50; % Max number of pixels to modify from input image
+                    adv_attack.noise_de = 1; % disturbance (noise) on pixels
+                case 'bright'
+                    adv_attack = struct;
+                    adv_attack.Name = "bright";
+                    adv_attack.threshold = 100; % perturb pixels below this value
+                    adv_attack.max_pixels = 50; % Max number of pixels to modify from input image
+                    adv_attack.noise_de = 1; % disturbance (noise) on pixels
+                otherwise
+                    error("Wrong attack");
+            end
 
             % select volumes to verify
-            N = 5;
+            N = 200;
             inputs = test_images(:,:,:,:,1:N);
             targets = test_labels(1:N);
 
             % verify volumes
-            results = verify_medmnist3d(net, matlabNet, inputs, targets, adv_attack);
+            results = verify_medmnist3d(net, inputs, targets, adv_attack);
 
             % save results
-            save("results/verification_"+datasets(i).name, "results", "adv_attack");
+            save("results/verification_"+adv_attack.Name+"_"+datasets(i).name, "results", "adv_attack");
 
             % print results to screen
             disp("======= ROBUSTNESS RESULTS ==========")

code/nnv/examples/NN/medmnist/verify_medmnist3d.m

@@ -1,4 +1,4 @@
-function results = verify_medmnist3d(net, matlabNet, inputs, targets, attack, max_value, min_value)
+function results = verify_medmnist3d(net, inputs, targets, attack, max_value, min_value)
     % verify medmnist with inputs (input images), targets (labels) and attack
     % (struct with adversarial attack info)
     % results =  verify_medmnist(net, matlabNet, inputs, targets, attack, max_value*, min_value*)
@@ -7,6 +7,10 @@
     if strcmp(attack.Name, 'linf')
         epsilon = attack.epsilon;
         max_pixels = attack.max_pixels;
+    elseif strcmp(attack.Name, 'dark') || strcmp(attack.Name, 'bright')
+        max_pixels = attack.max_pixels;
+        threshold = attack.threshold;
+        noise_disturbance = attack.noise_de;
     else
         error("Adversarial attack not supported.");
     end
@@ -31,7 +35,7 @@
     % Define reachability parameters
     reachOptions = struct;
     reachOptions.reachMethod = 'approx-star';
-    reachOptions.dis_opt = 'display';
+    % reachOptions.dis_opt = 'display';
 
     % Evaluate all images
     for i = 1:N
@@ -43,7 +47,14 @@
 
         % Create set of volume images
         vol = inputs(:,:,:,:,i);
-        I = L_inf_attack(vol, epsilon, max_pixels, max_value, min_value);
+        % Choose attack
+        if strcmp(attack.Name,'linf')
+            I = L_inf_attack(vol, epsilon, max_pixels, max_value, min_value);
+        elseif strcmp(attack.Name, 'dark')
+            I = dark_attack(vol, max_pixels, threshold, noise_disturbance);
+        elseif strcmp(attack.Name, 'bright')
+            I = bright_attack(vol, max_pixels, threshold, noise_disturbance);
+        end
 
         t = tic; % start timer
 
@@ -58,24 +69,31 @@
         end
 
         % Check for falsification with upper and lower bounds
-        yUpper = net.evaluate(I.vol_ub);
-        [~, yUpper] = max(yUpper);
-        yLower = net.evaluate(I.vol_lb);
-        [~, yLower] = max(yLower);
-        if yUpper ~= targets(i) || yLower ~= targets(i)
-            results(1,i) = 0; % not robust
-            results(2,i) = toc(t);
-            continue;
+        if strcmp(attack.Name,'linf')
+            yUpper = net.evaluate(I.vol_ub);
+            [~, yUpper] = max(yUpper);
+            yLower = net.evaluate(I.vol_lb);
+            [~, yLower] = max(yLower);
+            if yUpper ~= targets(i) || yLower ~= targets(i)
+                results(1,i) = 0; % not robust
+                results(2,i) = toc(t);
+                continue;
+            end
+        else
+            % disp("TODO: Add some sampling from the initial set to check for falsification.")
         end
 
         % Compute reachability for verification
         results(1,i) = net.verify_robustness(I, reachOptions, targets(i));
         results(2,i) = toc(t);
+        % disp(" ");
 
     end
 
 end
 
+
+%% Helper Functions
 % Return a VolumeStar of an Linf attack
 function I = L_inf_attack(vol, epsilon, max_pixels, max_value, min_value)
     volSize = size(vol);
@@ -88,3 +106,85 @@
     I = VolumeStar(single(lb), single(ub)); % default: single (assume onnx input models)
 end
 
+% Return a VolumeStar of a drkening attack on a few pixels
+function I = dark_attack(vol, max_pixels, threshold, noise_disturbance)
+
+    % Initialize vars
+    ct = 0; % keep track of pixels modified
+    flag = 0; % determine when to stop modifying pixels
+    vol = single(vol);
+    at_vol = vol;
+
+    % Create darkening attack
+    for i=1:size(vol,1)
+        for j=1:size(vol,2)
+            for k=1:size(vol,3)
+                if vol(i,j,k) > threshold
+                    at_vol(i,j,k) = 0;
+                    ct = ct + 1;
+                    if ct >= max_pixels
+                        flag = 1;
+                        break;
+                    end
+                end
+            end
+            if flag == 1
+                break
+            end
+        end
+        if flag == 1
+            break;
+        end
+    end
+
+    % Define input set as VolumeStar
+    dif_vol = vol - at_vol;
+    noise = -dif_vol;
+    V(:,:,:,:,1) = vol; % center of set
+    V(:,:,:,:,2) = noise; % basis vectors
+    C = [1; -1]; % constraints
+    d = [255; noise_disturbance-255];
+    I = VolumeStar(V, C, d, 255-noise_disturbance, 255); % input set
+
+end
+
+% Return a VolumeStar of a brightening attack on a few pixels
+function I = bright_attack(vol, max_pixels, threshold, noise_disturbance)
+
+    % Initialize vars
+    ct = 0; % keep track of pixels modified
+    flag = 0; % determine when to stop modifying pixels
+    vol = single(vol);
+    at_vol = vol;
+
+    % Create brightening attack
+    for i=1:size(vol,1)
+        for j=1:size(vol,2)
+            for k=1:size(vol,3)
+                if vol(i,j,k) < threshold
+                    at_vol(i,j,k) = 1;
+                    ct = ct + 1;
+                    if ct >= max_pixels
+                        flag = 1;
+                        break;
+                    end
+                end
+            end
+            if flag == 1
+                break
+            end
+        end
+        if flag == 1
+            break;
+        end
+    end
+
+    % Define input set as VolumeStar
+    dif_vol = vol - at_vol;
+    noise = -dif_vol;
+    V(:,:,:,:,1) = vol;   % center of set
+    V(:,:,:,:,2) = noise; % basis vectors
+    C = [1; -1];          % constraints
+    d = [255; noise_disturbance-255]; % constraints
+    I = VolumeStar(V, C, d, 255-noise_disturbance, 255); % input set
+end