Merge pull request #15 from ispras/attack-defense_pipeline

Attack defense pipeline

demo/back_explainers_demo.py

@@ -16,15 +16,13 @@ def explainers_test():
     dataset_mg_small, _, results_dataset_path_mg_small = DatasetManager.get_by_full_name(
         full_name=("multiple-graphs", "custom", "small",),
         features={'attr': {'a': 'as_is'}},
-        dataset_attack_type='original',
         labeling='binary',
         dataset_ver_ind=0
     )
 
     dataset_sg_example, _, results_dataset_path_sg_example = DatasetManager.get_by_full_name(
         full_name=("single-graph", "custom", "example",),
         features={'attr': {'a': 'as_is'}},
-        dataset_attack_type='original',
         labeling='binary',
         dataset_ver_ind=0
     )
@@ -36,7 +34,6 @@ def explainers_test():
             graph="small"),
         DatasetVarConfig(features={'attr': {'a': 'as_is'}},
                          labeling='binary',
-                         dataset_attack_type='original',
                          dataset_ver_ind=0)
     )
     gen_dataset_sg_example = DatasetManager.get_by_config(
@@ -46,7 +43,6 @@ def explainers_test():
             graph="example"),
         DatasetVarConfig(features={'attr': {'a': 'as_is'}},
                          labeling='binary',
-                         dataset_attack_type='original',
                          dataset_ver_ind=0)
     )
     gen_dataset_mg_small.train_test_split(percent_train_class=0.6, percent_test_class=0.4)

demo/backend_demo.py

@@ -14,17 +14,14 @@ def backend_demo():
     # Init datasets VK and Cora
     # dataset_cora, _, results_dataset_path_cora = DatasetManager.get_by_full_name(
     #     full_name=("single-graph", "Planetoid", 'Cora'),
-    #     dataset_attack_type='original',
     #     dataset_ver_ind=0)
     # dataset_comp, _, results_dataset_path_comp = DatasetManager.get_by_full_name(
     #     full_name=("single-graph", "Amazon", "Computers",),
-    #     dataset_attack_type='original',
     #     dataset_ver_ind=0
     # )
     # dataset_mg_example, _, results_dataset_path_mg_example = DatasetManager.get_by_full_name(
     #     full_name=("multiple-graphs", "custom", "example",),
     #     features={'attr': {'type': 'as_is'}},
-    #     dataset_attack_type='original',
     #     labeling='binary',
     #     dataset_ver_ind=0
     # )
@@ -53,7 +50,6 @@ def backend_demo():
             graph="example"),
         DatasetVarConfig(features={'attr': {'type': 'as_is'}},
                          labeling='binary',
-                         dataset_attack_type='original',
                          dataset_ver_ind=0)
     )
     gen_dataset_mg_example.train_test_split(percent_train_class=0.6, percent_test_class=0.4)

demo/user_exp_example.py

@@ -26,7 +26,6 @@ def test_Konst_model():
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=full_name,
         features={'attr': {'a': 'as_is', 'b': 'as_is'}},
-        dataset_attack_type='original',
         labeling='threeClasses',
         dataset_ver_ind=0
     )

experiments/GNNExplainerDIG_exp_example.py

@@ -17,40 +17,12 @@ def dig_GNNExplainer_test():
     my_device = device('cuda' if is_available() else 'cpu')
     my_device = device('cpu')
 
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "Planetoid", 'Cora'),
-    #     dataset_attack_type='original')
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "pytorch-geometric-other", 'KarateClub'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0)
-
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=("single-graph", "Planetoid", 'Cora'),
-        dataset_attack_type='original',
         dataset_ver_ind=0)
 
     # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
     #     full_name=("multiple-graphs", "TUDataset", 'PROTEINS'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0
-    # )
-
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
-    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
-    #     # features=('sex',),
-    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
-    #         # "('personal', 'political')": 'one_hot',
-    #         # "('occupation', 'type')": 'one_hot', # Don't work now
-    #         # "('relation',)": 'one_hot',
-    #         # "('age',)": 'one_hot',
-    #         "('sex',)": 'one_hot',
-    #     }},
-    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
-    #     labeling='sex1',
-    #     dataset_attack_type='original',
     #     dataset_ver_ind=0
     # )
 

experiments/GNNExplainerGEOM_exp_example.py

@@ -17,41 +17,9 @@ def geom_GNNExplainer_test():
     my_device = device('cuda' if is_available() else 'cpu')
     my_device = device('cpu')
 
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "Planetoid", 'Cora'),
-    #     dataset_attack_type='original')
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "pytorch-geometric-other", 'KarateClub'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0)
-
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=("single-graph", "Planetoid", 'Cora'),
-        dataset_attack_type='original',
         dataset_ver_ind=0)
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("multiple-graphs", "TUDataset", 'PROTEINS'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0
-    # )
-
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
-    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
-    #     # features=('sex',),
-    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
-    #         # "('personal', 'political')": 'one_hot',
-    #         # "('occupation', 'type')": 'one_hot', # Don't work now
-    #         # "('relation',)": 'one_hot',
-    #         # "('age',)": 'one_hot',
-    #         "('sex',)": 'one_hot',
-    #     }},
-    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
-    #     labeling='sex1',
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0
-    # )
 
     gcn2 = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
 

experiments/GradCAM_exp_example.py

@@ -17,7 +17,6 @@ def GradCAM_test():
 
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=("single-graph", "Planetoid", 'Cora'),
-        dataset_attack_type='original',
         dataset_ver_ind=0)
 
     # print(dataset)

experiments/GraphMask_exp_example.py

@@ -32,41 +32,9 @@ def test_graph_mask(configuration):
     my_device = device('cuda' if is_available() else 'cpu')
     my_device = device('cpu')
 
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "Planetoid", 'Cora'),
-    #     dataset_attack_type='original')
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("single-graph", "pytorch-geometric-other", 'KarateClub'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0)
-
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=("single-graph", "Planetoid", 'Cora'),
-        dataset_attack_type='original',
         dataset_ver_ind=0)
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     full_name=("multiple-graphs", "TUDataset", 'PROTEINS'),
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0
-    # )
-
-    # dataset, data, results_dataset_path = DatasetManager.get_pytorch_geometric(
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
-    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
-    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
-    #     # features=('sex',),
-    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
-    #         # "('personal', 'political')": 'one_hot',
-    #         # "('occupation', 'type')": 'one_hot', # Don't work now
-    #         # "('relation',)": 'one_hot',
-    #         # "('age',)": 'one_hot',
-    #         "('sex',)": 'one_hot',
-    #     }},
-    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
-    #     labeling='sex1',
-    #     dataset_attack_type='original',
-    #     dataset_ver_ind=0
-    # )
 
     gcn2 = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
 

experiments/NeuralAnalysis_our_exp_example.py

@@ -15,7 +15,6 @@ def test_neural_analysis(percent_train_class: float = 0.8, percent_test_class: f
     full_name = ("multiple-graphs", "TUDataset", 'MUTAG')
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=full_name,
-        dataset_attack_type='original',
         dataset_ver_ind=0
     )
 

experiments/ProtGNN_exp.py

@@ -34,7 +34,6 @@ def test_prot(i=None, conv=None, batch_size=24, seed=5,
     """
     dataset, data, results_dataset_path = Datasets.get_pytorch_geometric(
         full_name=("single-graph", "TUDataset", 'MUTAG'),
-        dataset_attack_type='original',
         dataset_ver_ind=0)
     """
 
@@ -50,7 +49,6 @@ def test_prot(i=None, conv=None, batch_size=24, seed=5,
     full_name = ("multiple-graphs", "TUDataset", 'MUTAG')
     dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
         full_name=full_name,
-        dataset_attack_type='original',
         dataset_ver_ind=0
     )
 

experiments/attack_defense_test.py

@@ -0,0 +1,161 @@
+import torch
+
+import warnings
+
+from torch import device
+
+from src.aux.utils import POISON_ATTACK_PARAMETERS_PATH, POISON_DEFENSE_PARAMETERS_PATH
+from src.models_builder.gnn_models import FrameworkGNNModelManager, Metric
+from src.aux.configs import ModelModificationConfig, ConfigPattern
+from src.base.datasets_processing import DatasetManager
+from src.models_builder.models_zoo import model_configs_zoo
+
+
+def test_attack_defense():
+    # my_device = device('cuda' if is_available() else 'cpu')
+    my_device = device('cpu')
+
+    full_name = None
+
+    # full_name = ("multiple-graphs", "TUDataset", 'MUTAG')
+    # full_name = ("single-graph", "custom", 'karate')
+    full_name = ("single-graph", "Planetoid", 'Cora')
+    # full_name = ("multiple-graphs", "TUDataset", 'PROTEINS')
+
+    dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+        full_name=full_name,
+        dataset_ver_ind=0
+    )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     full_name=("single-graph", "custom", "example",),
+    #     features={'attr': {'a': 'as_is', 'b': 'as_is'}},
+    #     labeling='threeClasses',
+    #     dataset_ver_ind=0
+    # )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
+    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
+    #     # features=('sex',),
+    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
+    #         # "('personal', 'political')": 'one_hot',
+    #         # "('occupation', 'type')": 'one_hot', # Don't work now
+    #         # "('relation',)": 'one_hot',
+    #         # "('age',)": 'one_hot',
+    #         "('sex',)": 'one_hot',
+    #     }},
+    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
+    #     labeling='sex1',
+    #     dataset_ver_ind=0
+    # )
+
+    # print(data.train_mask)
+
+    gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='test_gnn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin_prot')
+
+    manager_config = ConfigPattern(
+        _config_class="ModelManagerConfig",
+        _config_kwargs={
+            "mask_features": [],
+            "optimizer": {
+                # "_config_class": "Config",
+                "_class_name": "Adam",
+                # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+                # "_class_import_info": ["torch.optim"],
+                "_config_kwargs": {},
+            }
+        }
+    )
+    # manager_config = ModelManagerConfig(**{
+    #         "mask_features": [],
+    #         "optimizer": {
+    #             # "_config_class": "Config",
+    #             "_class_name": "Adam",
+    #             # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+    #             # "_class_import_info": ["torch.optim"],
+    #             "_config_kwargs": {},
+    #         }
+    #     }
+    # )
+
+    # train_test_split = [0.8, 0.2]
+    # train_test_split = [0.6, 0.4]
+    steps_epochs = 200
+    gnn_model_manager = FrameworkGNNModelManager(
+        gnn=gnn,
+        dataset_path=results_dataset_path,
+        manager_config=manager_config,
+        modification=ModelModificationConfig(model_ver_ind=0, epochs=steps_epochs)
+    )
+
+    # save_model_flag = False
+    save_model_flag = True
+
+    # data.x = data.x.float()
+    gnn_model_manager.gnn.to(my_device)
+    data = data.to(my_device)
+
+    poison_attack_config = ConfigPattern(
+        _class_name="RandomPoisonAttack",
+        _import_path=POISON_ATTACK_PARAMETERS_PATH,
+        _config_class="PoisonAttackConfig",
+        _config_kwargs={
+            "n_edges_percent": 0.1,
+        }
+    )
+
+    # poison_defense_config = ConfigPattern(
+    #     _class_name="BadRandomPoisonDefender",
+    #     _import_path=POISON_DEFENSE_PARAMETERS_PATH,
+    #     _config_class="PoisonDefenseConfig",
+    #     _config_kwargs={
+    #         "n_edges_percent": 0.1,
+    #     }
+    # )
+    poison_defense_config = ConfigPattern(
+        _class_name="EmptyPoisonDefender",
+        _import_path=POISON_DEFENSE_PARAMETERS_PATH,
+        _config_class="PoisonDefenseConfig",
+        _config_kwargs={
+        }
+    )
+
+    gnn_model_manager.set_poison_attacker(poison_attack_config=poison_attack_config)
+    gnn_model_manager.set_poison_defender(poison_defense_config=poison_defense_config)
+
+    warnings.warn("Start training")
+    dataset.train_test_split()
+
+    try:
+        raise FileNotFoundError()
+        # gnn_model_manager.load_model_executor()
+    except FileNotFoundError:
+        gnn_model_manager.epochs = gnn_model_manager.modification.epochs = 0
+        train_test_split_path = gnn_model_manager.train_model(gen_dataset=dataset, steps=steps_epochs,
+                                                              save_model_flag=save_model_flag,
+                                                              metrics=[Metric("F1", mask='train', average=None)])
+
+        if train_test_split_path is not None:
+            dataset.save_train_test_mask(train_test_split_path)
+            train_mask, val_mask, test_mask, train_test_sizes = torch.load(train_test_split_path / 'train_test_split')[
+                                                                :]
+            dataset.train_mask, dataset.val_mask, dataset.test_mask = train_mask, val_mask, test_mask
+            data.percent_train_class, data.percent_test_class = train_test_sizes
+
+    warnings.warn("Training was successful")
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='test', average='macro')])
+    print(metric_loc)
+
+
+if __name__ == '__main__':
+    test_attack_defense()
+
+