diff --git a/model3_test_control.tsv b/model3_test_control.tsv new file mode 100644 index 0000000..fff2b22 --- /dev/null +++ b/model3_test_control.tsv @@ -0,0 +1,10 @@ +1 +0 +1 +1 +1 +1 +0 +1 +1 +1 diff --git a/probe_ably/core/flows/__init__.py b/probe_ably/core/flows/__init__.py index e69de29..6e05287 100644 --- a/probe_ably/core/flows/__init__.py +++ b/probe_ably/core/flows/__init__.py @@ -0,0 +1 @@ +from .probe_from_dataloaders import probe_from_dataloaders \ No newline at end of file diff --git a/probe_ably/core/flows/probe_from_dataloaders.py b/probe_ably/core/flows/probe_from_dataloaders.py new file mode 100644 index 0000000..93b4f51 --- /dev/null +++ b/probe_ably/core/flows/probe_from_dataloaders.py @@ -0,0 +1,20 @@ +import click +import prefect +from dynaconf import settings +from loguru import logger +from prefect import Flow +from prefect.engine.flow_runner import FlowRunner +from probe_ably.core.tasks.probing import TrainProbingTask +from probe_ably.core.tasks.metric_task import ProcessMetricTask + +INPUT_FILE = "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" +train_probing_task = TrainProbingTask() +process_metric_task = ProcessMetricTask() + +def probe_from_dataloaders(config_dict, prepared_data): + with Flow("Running Probe") as flow1: + train_results = train_probing_task(prepared_data, config_dict["probing_setup"]) + processed_results = process_metric_task( + train_results, config_dict["probing_setup"] + ) + FlowRunner(flow=flow1).run \ No newline at end of file diff --git a/probe_ably/core/models/__init__.py b/probe_ably/core/models/__init__.py index 4f0e7a3..468b0ae 100644 --- a/probe_ably/core/models/__init__.py +++ b/probe_ably/core/models/__init__.py @@ -1,3 +1,4 @@ from .abstract_model import AbstractModel from .linear import LinearModel -from .mlp import MLPModel \ No newline at end of file +from .mlp import MLPModel +from .model_params import ModelParams \ No newline at end of file diff --git a/probe_ably/core/models/linear.py b/probe_ably/core/models/linear.py index bef473b..b300a2f 100644 --- a/probe_ably/core/models/linear.py +++ b/probe_ably/core/models/linear.py @@ -73,8 +73,8 @@ def get_norm(self) -> Tensor: return penalty -# def get_rank(self): -# ext_matrix = torch.cat([self.linear.weight, self.linear.bias.unsqueeze(-1)], dim=1) -# _, svd_matrix, _ = np.linalg.svd(ext_matrix.cpu().numpy()) -# rank = np.sum(svd_matrix > 1e-3) -# return rank + def get_rank(self): + ext_matrix = torch.cat([self.linear.weight, self.linear.bias.unsqueeze(-1)], dim=1) + _, svd_matrix, _ = np.linalg.svd(ext_matrix.cpu().numpy()) + rank = np.sum(svd_matrix > 1e-3) + return rank diff --git a/probe_ably/core/models/model_params.py b/probe_ably/core/models/model_params.py new file mode 100644 index 0000000..b579553 --- /dev/null +++ b/probe_ably/core/models/model_params.py @@ -0,0 +1,50 @@ +from typing import Dict + +class ModelParams(): + def __init__(self)->Dict: + self.default_params = { + "probe_ably.core.models.linear.LinearModel": { + "params": [ + { + "name": "dropout", + "type": "float_range", + "options": [0.0, 0.51] + }, + { + "name": "alpha", + "type": "function", + "function_location": "probe_ably.core.utils.param_functions.nuclear_norm_alpha_generation", + "options": [-10.0, 3] + }] + }, + "probe_ably.core.models.mlp.MLPModel": { + "params": [ + { + "name": "hidden_size", + "type": "function", + "step": 0.01, + "function_location": "probe_ably.core.utils.param_functions.hidden_size_generation", + "options": [ + 2, + 5 + ] + }, + { + "name": "n_layers", + "type": "int_range", + "options": [ + 1, + 2 + ] + }, + { + "name": "dropout", + "type": "float_range", + "options": [ + 0.0, + 0.5 + ] + } + ] + } + } \ No newline at end of file diff --git a/probe_ably/core/tasks/control_task/generate_control_task.py b/probe_ably/core/tasks/control_task/generate_control_task.py index 0069767..7618ba3 100644 --- a/probe_ably/core/tasks/control_task/generate_control_task.py +++ b/probe_ably/core/tasks/control_task/generate_control_task.py @@ -16,7 +16,6 @@ def generate_random_labels(unique_labels, labels_size): return random_labels - @overrides def run(self, input_data, input_labels): unique_labels = self.get_unique_labels(input_labels) diff --git a/probe_ably/core/tasks/metric_task/process_metric_task.py b/probe_ably/core/tasks/metric_task/process_metric_task.py index 1caa099..cd04bbc 100644 --- a/probe_ably/core/tasks/metric_task/process_metric_task.py +++ b/probe_ably/core/tasks/metric_task/process_metric_task.py @@ -12,7 +12,6 @@ class ProcessMetricTask(Task): - @overrides def run( self, train_results: Dict[str, Dict], probing_configuration: Dict[str, Dict] ): diff --git a/probe_ably/core/tasks/probing/deprobe_task.py b/probe_ably/core/tasks/probing/deprobe_task.py new file mode 100644 index 0000000..af46853 --- /dev/null +++ b/probe_ably/core/tasks/probing/deprobe_task.py @@ -0,0 +1,353 @@ +import random +from typing import Dict, List +from copy import copy, deepcopy +import numpy as np +import torch +from loguru import logger +from overrides import overrides +from prefect import Task +from probe_ably.core.metrics import AbstractIntraModelMetric +from probe_ably.core.models import LinearModel +from probe_ably.core.utils import GridModelFactory +from sklearn.metrics import accuracy_score +from torch.utils.data import DataLoader +from tqdm import tqdm, trange +from colorama import Fore +import threading + + +class TrainProbingTask(Task): + def __init__(self, **kwargs): + self.cuda = kwargs.pop("cuda", True) + self.logging_steps = kwargs.get("logging_steps", 5) + super(TrainProbingTask, self).__init__(**kwargs) + + def set_seed(self, n_gpu, seed=42): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if n_gpu > 0: + torch.cuda.manual_seed_all(seed) + + def start_training_process( + self, + train, + test, + dev, + train_batch_size, + model, + device, + n_gpu, + num_epochs, + eval_fn, + ): + outputs = {} + # logger.info("Running train mode") + train_dataloader = DataLoader( + train, + batch_size=train_batch_size, + shuffle=True, + ) + dev_dataloader = DataLoader( + dev, + batch_size=train_batch_size, + shuffle=False, + ) + test_dataloader = DataLoader( + test, + batch_size=train_batch_size, + shuffle=False, + ) + + model = model.to(device) + if n_gpu > 1: + model = torch.nn.DataParallel(model) + best_model = self.train( + model, + train_dataloader, + dev_dataloader, + device, + n_gpu, + num_epochs, + eval_fn, + ) + + score_test, preds_test = self.eval( + best_model, + test_dataloader, + device, + n_gpu, + eval_fn, + ) + + return preds_test + + def run(self, tasks: Dict, probing_setup: Dict, thread: threading.Thread) -> Dict: + """Runs the Probing models + + :param tasks: Data content of the models for probing. + :type tasks: Dict + :param probing_setup: Experiment setup for probing. + :type probing_setup: Dict + :return: Dictionary containing the following values: + {int(task id) : + "models": { + int(model id) : { + str (name of probing model) : { + int (run number) : { + "complexity": { + str (name of metric): float (value for the complexity) + } + "model" : { + "labels": array (original labels for the auxiliary task) + "preds": array (predicted labels for the auxiliary task) + } + "control": { + "labels": array (original labels for the control task) + "preds": array (predicted labels for the control task) + + } + } + } + } + } + + :rtype: Dict + """ + + logger.debug("Starting the Probing Training Task") + torch.cuda.empty_cache() + device = torch.device( + "cuda" if torch.cuda.is_available() and self.cuda else "cpu" + ) + + n_gpu = torch.cuda.device_count() + self.set_seed(n_gpu) + self.logger.info(f"GPUs used {n_gpu}") + + output_results = dict() + intra_metric_class = AbstractIntraModelMetric.subclasses[ + probing_setup["intra_metric"] + ] + intra_metric_object = intra_metric_class() + + thread.task_loop_bar = tqdm( + tasks.items(), + desc=f"Task progress", + bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.GREEN, Fore.RESET), + ) + + for id_task, content_tasks in thread.task_loop_bar: + + thread.task_loop_bar.set_description( + f"Task: {content_tasks['task_name']} progress" + ) + output_results[id_task] = dict() + output_results[id_task]["models"] = dict() + output_results[id_task]["task_name"] = content_tasks["task_name"] + thread.model_loop_bar = tqdm( + content_tasks["models"].items(), + desc=f"Model progress", + bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.BLUE, Fore.RESET), + leave=False, + ) + for id_model, model_content in thread.model_loop_bar: + thread.model_loop_bar.set_description( + f"Model: {model_content['model_name']} progress" + ) + output_results[id_task]["models"][id_model] = dict() + output_results[id_task]["models"][id_model][ + "model_name" + ] = model_content["model_name"] + + model_params = { + "representation_size": model_content["representation_size"], + "n_classes": model_content["number_of_classes"], + } + + for id_prob_model, probe_content in probing_setup[ + "probing_models" + ].items(): + probe_model_name = probe_content["probing_model_name"] + + probing_models = GridModelFactory.create_models( + probe_model_name, + probe_content["number_of_models"], + model_params, + ) + + output_results[id_task]["models"][id_model][ + probe_model_name + ] = dict() + run_number = 0 + train_batch_size = probe_content["batch_size"] * max(1, n_gpu) + + thread.probes_loop_bar = tqdm( + probing_models, + desc="Probe Progress", + leave=False, + bar_format="{l_bar}%s{bar}%s{r_bar}" + % (Fore.YELLOW, Fore.RESET), + ) + for probe in thread.probes_loop_bar: + thread.probes_loop_bar.set_description( + f"Probe: {probe_model_name} progress" + ) + probe_for_model = deepcopy(probe) + probe_for_control = deepcopy(probe) + preds_model = self.start_training_process( + train=model_content["model"]["train"], + test=model_content["model"]["test"], + dev=model_content["model"]["dev"], + train_batch_size=train_batch_size, + model=probe_for_model, + device=device, + num_epochs=probe_content["epochs"], + n_gpu=n_gpu, + eval_fn=intra_metric_object.calculate_metrics, + ) + + if model_content["default_control"]: + test_control_set = model_content["control"]["train"] + else: + test_control_set = model_content["control"]["test"] + preds_control = self.start_training_process( + train=model_content["control"]["train"], + test=test_control_set, + dev=model_content["control"]["dev"], + train_batch_size=train_batch_size, + model=probe_for_control, + device=device, + num_epochs=probe_content["epochs"], + n_gpu=n_gpu, + eval_fn=intra_metric_object.calculate_metrics, + ) + output_results[id_task]["models"][id_model][probe_model_name][ + run_number + ] = { + "complexity": probe_for_model.get_complexity(), + "model": { + "labels": model_content["model"]["test"].labels, + "preds": preds_model, + }, + "control": { + "labels": test_control_set.labels, + "preds": preds_control, + }, + } + run_number += 1 + return output_results + + def train( + self, + model, + train_dataloader, + dev_dataloader, + device, + n_gpu, + num_epochs, + eval_fn, + ): + best_score = -1.0 + + optimizer = torch.optim.Adam(model.parameters()) + + global_step = 0 + epochs_trained = 0 + + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = range( + epochs_trained, + int(num_epochs), + ) + + for epoch in train_iterator: + # epoch_iterator = tqdm(train_dataloader, desc="Iteration") + for step, batch in enumerate(train_dataloader): + model.train() + batch = tuple(t.to(device) for t in batch) + + input_model = { + "representation": batch[0], + "labels": batch[1], + } + + output = model(**input_model) + + loss = output["loss"] + + if n_gpu > 1: + loss = ( + loss.mean() + ) # mean() to average on multi-gpu parallel training + + loss.backward() + + tr_loss += loss.item() + + optimizer.step() + model.zero_grad() + global_step += 1 + + if self.logging_steps > 0 and global_step % self.logging_steps == 0: + loss_scalar = (tr_loss - logging_loss) / self.logging_steps + + # epoch_iterator.set_description(f"Loss :{loss_scalar}") + + logging_loss = tr_loss + + score, _ = self.eval( + model, + dev_dataloader, + device, + n_gpu, + eval_fn, + ) + + with torch.no_grad(): + if score > best_score: + # logger.success(f"Saving new model with best score: {score}") + best_model = model + best_score = score + + return best_model + + def eval(self, model, dataloader, device, n_gpu, eval_fn): + if n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): + model = torch.nn.DataParallel(model) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + + for batch in dataloader: + model.eval() + batch = tuple(t.to(device) for t in batch) + + with torch.no_grad(): + input_model = { + "representation": batch[0], + "labels": batch[1], + } + + output = model(**input_model) + nb_eval_steps += 1 + if preds is None: + preds = output["preds"].detach().cpu().numpy() + out_label_ids = input_model["labels"].detach().cpu().numpy() + + else: + + preds = np.append(preds, output["preds"].detach().cpu().numpy(), axis=0) + + out_label_ids = np.append( + out_label_ids, input_model["labels"].detach().cpu().numpy(), axis=0 + ) + + eval_loss = eval_loss / nb_eval_steps + + score = eval_fn(preds, out_label_ids) + # logger.info(f"Score:{score}") + + return score, preds + diff --git a/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py b/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py index 890e9d4..e43109f 100644 --- a/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py +++ b/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py @@ -8,7 +8,6 @@ from torch.utils.data import Dataset import numpy as np - class PrepareDataForProbingTask(Task): @staticmethod def prepare_entries(vectors, labels): @@ -34,7 +33,6 @@ def train_val_test_split(X, y, train_size, val_size, test_size, seed=42): return X_train, X_val, X_test, y_train, y_val, y_test - @overrides def run(self, tasks_data: Dict, experiment_setup: Dict) -> Dict: """Reads the task_data and experiment_setup, splits into train/dev/test and creates a TorchDataset for each. @@ -94,9 +92,7 @@ def run(self, tasks_data: Dict, experiment_setup: Dict) -> Dict: model_labels_train, model_labels_val, model_labels_test, - ) = self.train_val_test_split( - X=model_content["model_vectors"], - y=model_content["model_labels"], + ) = self.train_val_test_split( X=model_content["model_vectors"], y=model_content["model_labels"], train_size=experiment_setup["train_size"], val_size=experiment_setup["dev_size"], test_size=experiment_setup["test_size"], @@ -159,4 +155,4 @@ def get_id(self, index): return self.keys[index] def __len__(self): - return len(self.dataset) + return len(self.dataset) \ No newline at end of file diff --git a/probe_ably/core/tasks/probing/train_probing_task.py b/probe_ably/core/tasks/probing/train_probing_task.py index fb0fe18..d9e6a54 100644 --- a/probe_ably/core/tasks/probing/train_probing_task.py +++ b/probe_ably/core/tasks/probing/train_probing_task.py @@ -81,7 +81,6 @@ def start_training_process( return preds_test - @overrides def run(self, tasks: Dict, probing_setup: Dict) -> Dict: """Runs the Probing models @@ -163,6 +162,7 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: "representation_size": model_content["representation_size"], "n_classes": model_content["number_of_classes"], } + print(model_params) for id_prob_model, probe_content in probing_setup[ "probing_models" @@ -352,4 +352,4 @@ def eval(self, model, dataloader, device, n_gpu, eval_fn): score = eval_fn(preds, out_label_ids) # logger.info(f"Score:{score}") - return score, preds + return score, preds \ No newline at end of file diff --git a/probe_ably/core/tasks/utils/read_input_task.py b/probe_ably/core/tasks/utils/read_input_task.py index 4f7c676..c8f73f3 100644 --- a/probe_ably/core/tasks/utils/read_input_task.py +++ b/probe_ably/core/tasks/utils/read_input_task.py @@ -16,7 +16,6 @@ SCHEMA_TEMPLATE_FILE = settings["input_json_schema"] - class ModelRepresentationFileNotFound(Exception): def __init__(self, model_location): self.model_location = model_location @@ -35,7 +34,6 @@ def __init__(self, type_of_class, class_name): class ReadInputTask(Task): - @overrides def run(self, input_file_location: str) -> Dict: """Function that parses the input configuration file provided by the user. diff --git a/probe_ably/core/utils/grid_model_factory.py b/probe_ably/core/utils/grid_model_factory.py index a8d6550..9514f9a 100644 --- a/probe_ably/core/utils/grid_model_factory.py +++ b/probe_ably/core/utils/grid_model_factory.py @@ -6,7 +6,7 @@ from typing import Any, Dict, List import sys import numpy as np -from probe_ably.core.models import AbstractModel +from probe_ably.core.models import AbstractModel, ModelParams class GridModelFactory: @@ -24,19 +24,9 @@ def create_models( Returns: List[AbstractModel]: Creates a Grid with the prescribed parameter ranges and returns list of models with size num_models by random initialization """ - paths = glob.glob("./config/params/*.json") - - params = None - for path in paths: - with open(path, "r") as f: - maybe_params = json.load(f) - if model_class in maybe_params: - params = maybe_params[model_class]["params"] - break - if not params: - raise FileNotFoundError("No parameters specified.") ModelClass = AbstractModel.subclasses[model_class] + params = ModelParams().default_params[model_class]["params"] generated_params = dict() for param in params: diff --git a/probe_ably/service/package.json b/probe_ably/service/package.json index ba40a9d..70ab509 100644 --- a/probe_ably/service/package.json +++ b/probe_ably/service/package.json @@ -3,28 +3,28 @@ "version": "0.1.0", "private": true, "dependencies": { - "@fortawesome/fontawesome-free": "^5.15.1", - "@fortawesome/fontawesome-svg-core": "^1.2.30", - "@fortawesome/free-brands-svg-icons": "^5.14.0", - "@fortawesome/free-regular-svg-icons": "^5.14.0", - "@fortawesome/free-solid-svg-icons": "^5.14.0", - "@fortawesome/react-fontawesome": "^0.1.11", - "@nivo/bar": "^0.67.0", - "@nivo/core": "^0.67.0", - "@nivo/line": "^0.67.0", - "@nivo/scatterplot": "^0.67.0", - "@testing-library/jest-dom": "^4.2.4", - "@testing-library/react": "^9.3.2", - "@testing-library/user-event": "^7.1.2", + "@fortawesome/fontawesome-free": "^5.15.3", + "@fortawesome/fontawesome-svg-core": "^1.2.35", + "@fortawesome/free-brands-svg-icons": "^5.15.3", + "@fortawesome/free-regular-svg-icons": "^5.15.3", + "@fortawesome/free-solid-svg-icons": "^5.15.3", + "@fortawesome/react-fontawesome": "^0.1.14", + "@nivo/bar": "^0.73.1", + "@nivo/core": "^0.73.0", + "@nivo/line": "^0.73.0", + "@nivo/scatterplot": "^0.73.0", + "@testing-library/jest-dom": "^5.14.1", + "@testing-library/react": "^12.0.0", + "@testing-library/user-event": "^13.2.1", "@themesberg/react-bootstrap": "^1.4.1", - "bootstrap": "5.0.0-beta1", + "bootstrap": "5.0.2", "chartist": "^0.11.4", "chartist-plugin-tooltips-updated": "^0.1.4", "jspdf": "^2.3.1", - "moment-timezone": "^0.5.31", - "node-sass": "^4.14.1", - "react": "^16.13.1", - "react-chartist": "^0.14.3", + "moment-timezone": "^0.5.33", + "node-sass": "^6.0.1", + "react": "^17.0.2", + "react-chartist": "^0.14.4", "react-copy-to-clipboard": "^5.0.3", "react-datetime": "^3.0.4", "react-dom": "^17.0.2", @@ -32,20 +32,18 @@ "react-github-btn": "^1.2.0", "react-live": "^2.2.3", "react-router-dom": "^5.2.0", - "react-router-hash-link": "^2.3.1", - "react-scripts": "3.4.3", - "react-transition-group": "^4.4.1", - "simplebar-react": "^2.3.0", + "react-router-hash-link": "^2.4.3", + "react-scripts": "4.0.3", + "react-transition-group": "^4.4.2", + "simplebar-react": "^2.3.5", "svg2pdf.js": "^2.1.0", - "typescript": "^4.2.3" + "typescript": "^4.3.5" }, "scripts": { "start": "react-scripts start", "build": "react-scripts build", "test": "react-scripts test", - "eject": "react-scripts eject", - "storybook": "start-storybook -p 6006 -s public", - "build-storybook": "build-storybook -s public" + "eject": "react-scripts eject" }, "eslintConfig": { "extends": [ @@ -66,11 +64,5 @@ ] }, "devDependencies": { - "@storybook/addon-actions": "6.1.14", - "@storybook/addon-essentials": "6.1.14", - "@storybook/addon-links": "6.1.14", - "@storybook/node-logger": "6.1.14", - "@storybook/preset-create-react-app": "^3.1.7", - "@storybook/react": "6.1.14" } } diff --git a/probe_ably/service/tsconfig.json b/probe_ably/service/tsconfig.json index f2850b7..e18c413 100644 --- a/probe_ably/service/tsconfig.json +++ b/probe_ably/service/tsconfig.json @@ -17,7 +17,8 @@ "resolveJsonModule": true, "isolatedModules": true, "noEmit": true, - "jsx": "react" + "jsx": "react-jsx", + "noFallthroughCasesInSwitch": true }, "include": [ "src" diff --git a/requirements.txt b/requirements.txt index 5c765bb..d4dbcbc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ dynaconf[all] prefect loguru ujson -overrides sklearn pandas numpy