diff --git a/config/params/linear.json b/config/params/linear.json deleted file mode 100644 index 7713709..0000000 --- a/config/params/linear.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "probe_ably.core.models.linear.LinearModel": { - "params": [ - { - "name": "dropout", - "type": "float_range", - "options": [0.0, 0.51] - }, - { - "name": "alpha", - "type": "function", - "function_location": "probe_ably.core.utils.param_functions.nuclear_norm_alpha_generation", - "options": [-10.0, 3] - }] - } -} diff --git a/config/params/mlp.json b/config/params/mlp.json deleted file mode 100644 index c3ba2b1..0000000 --- a/config/params/mlp.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "probe_ably.core.models.mlp.MLPModel": { - "params": [ - { - "name": "hidden_size", - "type": "function", - "step": 0.01, - "function_location": "probe_ably.core.utils.param_functions.hidden_size_generation", - "options": [ - 2, - 5 - ] - }, - { - "name": "n_layers", - "type": "int_range", - "options": [ - 2, - 3 - ] - }, - { - "name": "dropout", - "type": "float_range", - "options": [ - 0.0, - 0.5 - ] - } - ] - } -} \ No newline at end of file diff --git a/config/probing_setup/default_probing_setup.json b/config/probing_setup/default_probing_setup.json deleted file mode 100644 index a60f636..0000000 --- a/config/probing_setup/default_probing_setup.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "train_size":0.6, - "dev_size":0.2, - "test_size":0.2, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", - "probing_models":{ - "0":{ - "probing_model_name":"probe_ably.core.models.linear.LinearModel", - "batch_size":32, - "epochs":25, - "number_of_models":50 - }, - "1":{ - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", - "batch_size":32, - "epochs":25, - "number_of_models":50 - } - } -} \ No newline at end of file diff --git a/config/settings.toml b/config/settings.toml deleted file mode 100644 index 4a86fea..0000000 --- a/config/settings.toml +++ /dev/null @@ -1,3 +0,0 @@ -[default] -input_json_schema = "./config/json_schema/input_file.json" -default_probing_setup = "./config/probing_setup/default_probing_setup.json" diff --git a/docs/.buildinfo b/docs/.buildinfo deleted file mode 100644 index 67fb0e0..0000000 --- a/docs/.buildinfo +++ /dev/null @@ -1,4 +0,0 @@ -# Sphinx build info version 1 -# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 8787bd51e235bc680470d8971692b5e9 -tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docsrc/quicktour.md b/docsrc/quicktour.md index 097c024..19f2b4e 100644 --- a/docsrc/quicktour.md +++ b/docsrc/quicktour.md @@ -1,5 +1,31 @@ # Quick Start +## Application +We recommend using the browser-based application interface if you are new to probing. + +Run the app using: +``` +python -m probe_ably.app +``` + + +## As a Python Library +Follow the installation guidelines. + +### Default Probing Configuration +''' +from probe_ably import ProbingExperiment +experiment = ProbingExperiment.from_config("default") +experiment.representations = data # dataframe, torch tensor or np.array of vector representations +experiment.labels = labels # array-like +experiment.run() +''' + +### Custom Probing Configuration + +### Custom Probing Configuration with Data Paths + +## CLI ## Representation File The representation file that you are going to test in expected to be a tab seperated `tsv` file with one column for each dimension, separated by \t, last column is assumed to be the label. @@ -67,8 +93,3 @@ In this configuration file. At the end of the probe you will be directed or prompted to visit the following address: `http://127.0.0.1:8031/` where you can see the visualzation output. By default probing will run the `Linear Model` and `MLP Model` with Accuracy and Selectivity as the default metrics. If you want to change the run configurations of these models see [Probing Configurations](/advanced/configurations.md). - - - - - diff --git a/model3_test_control.tsv b/model3_test_control.tsv deleted file mode 100644 index fff2b22..0000000 --- a/model3_test_control.tsv +++ /dev/null @@ -1,10 +0,0 @@ -1 -0 -1 -1 -1 -1 -0 -1 -1 -1 diff --git a/probe_ably/__init__.py b/probe_ably/__init__.py index e69de29..92f0327 100644 --- a/probe_ably/__init__.py +++ b/probe_ably/__init__.py @@ -0,0 +1 @@ +from .experiment_setup import ProbingExperiment \ No newline at end of file diff --git a/probe_ably/__main__.py b/probe_ably/__main__.py new file mode 100644 index 0000000..d3b88b6 --- /dev/null +++ b/probe_ably/__main__.py @@ -0,0 +1,13 @@ +import click +from probe_ably import ProbingExperiment + +@click.command() +@click.option("--config_file", + help="Probing Configuration File", + default="./tests/sample_files/test_input/multi_task_multi_model_with_control.json") +def main(config_file): + experiment = ProbingExperiment.from_json(config_file) + experiment.run() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/probe_ably/service/.gitignore b/probe_ably/app/.gitignore similarity index 100% rename from probe_ably/service/.gitignore rename to probe_ably/app/.gitignore diff --git a/probe_ably/service/.prettierrc.js b/probe_ably/app/.prettierrc.js similarity index 100% rename from probe_ably/service/.prettierrc.js rename to probe_ably/app/.prettierrc.js diff --git a/probe_ably/core/__init__.py b/probe_ably/app/__init__.py similarity index 100% rename from probe_ably/core/__init__.py rename to probe_ably/app/__init__.py diff --git a/probe_ably/app/__main__.py b/probe_ably/app/__main__.py new file mode 100644 index 0000000..5081e2a --- /dev/null +++ b/probe_ably/app/__main__.py @@ -0,0 +1,85 @@ +from probe_ably.metrics import ProcessMetricTask +from probe_ably.utils import ReadInputTask +from probe_ably.probing import TrainProbingTask +from probe_ably import ProbingExperiment +from pathlib import Path +import threading +import os + +from fastapi import FastAPI, UploadFile, File, BackgroundTasks +from fastapi.staticfiles import StaticFiles +from fastapi.concurrency import run_in_threadpool +import uvicorn + +app_dir = Path(os.path.abspath(__file__)).parent +build_dir = app_dir.joinpath('build') + +# INPUT_FILE = "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" +probing_task = TrainProbingTask() +read_input_task = ReadInputTask() +process_metric_task = ProcessMetricTask() + + +class ProbingThread(threading.Thread): + def __init__(self): + super().__init__() + self.results = None + self.task_loop_bar = None + self.reps_loop_bar = None + self.probes_loop_bar = None + + + async def set_config(self, config_file): + self.parsed_input = await read_input_task.run(config_file) + + + def run(self): + experiment = ProbingExperiment.from_parsed_input(self.parsed_input, thread=self) + results = experiment.run() + return results + +app = FastAPI() +app.probing_thread = ProbingThread() + + +@app.post("/start_probing") +async def start_probing(background_tasks: BackgroundTasks, config_file: UploadFile = File(...), ): + await app.probing_thread.set_config(config_file) + results = await run_in_threadpool(app.probing_thread.run) + return results + + +@app.get('/model_progress') +def model_progress(): + bar = app.probing_thread.reps_loop_bar + if bar: + prog = bar.format_dict + return prog + else: + return {'n': 0, 'total':0} + + +@app.get('/task_progress') +async def task_progress(): + bar = app.probing_thread.task_loop_bar + if bar: + prog = bar.format_dict + return prog + else: + return {'n': 0, 'total':0} + + +@app.get('/probes_progress') +def probes_progress(): + bar = app.probing_thread.probes_loop_bar + if bar: + prog = bar.format_dict + return prog + else: + return {'n': 0, 'total':0} + + +app.mount("/", StaticFiles(directory=build_dir, html = True), name="static") + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/probe_ably/service/package.json b/probe_ably/app/package.json similarity index 76% rename from probe_ably/service/package.json rename to probe_ably/app/package.json index 70ab509..0de20e4 100644 --- a/probe_ably/service/package.json +++ b/probe_ably/app/package.json @@ -4,11 +4,11 @@ "private": true, "dependencies": { "@fortawesome/fontawesome-free": "^5.15.3", - "@fortawesome/fontawesome-svg-core": "^1.2.35", - "@fortawesome/free-brands-svg-icons": "^5.15.3", - "@fortawesome/free-regular-svg-icons": "^5.15.3", - "@fortawesome/free-solid-svg-icons": "^5.15.3", - "@fortawesome/react-fontawesome": "^0.1.14", + "@fortawesome/fontawesome-svg-core": "^1.2.36", + "@fortawesome/free-brands-svg-icons": "^5.15.4", + "@fortawesome/free-regular-svg-icons": "^5.15.4", + "@fortawesome/free-solid-svg-icons": "^5.15.4", + "@fortawesome/react-fontawesome": "^0.1.15", "@nivo/bar": "^0.73.1", "@nivo/core": "^0.73.0", "@nivo/line": "^0.73.0", @@ -17,20 +17,21 @@ "@testing-library/react": "^12.0.0", "@testing-library/user-event": "^13.2.1", "@themesberg/react-bootstrap": "^1.4.1", - "bootstrap": "5.0.2", + "bootstrap": "5.1.0", "chartist": "^0.11.4", "chartist-plugin-tooltips-updated": "^0.1.4", + "core-js": "^3.19.1", "jspdf": "^2.3.1", "moment-timezone": "^0.5.33", "node-sass": "^6.0.1", "react": "^17.0.2", "react-chartist": "^0.14.4", - "react-copy-to-clipboard": "^5.0.3", + "react-copy-to-clipboard": "^5.0.4", "react-datetime": "^3.0.4", "react-dom": "^17.0.2", "react-download-svg": "^0.0.4", - "react-github-btn": "^1.2.0", - "react-live": "^2.2.3", + "react-github-btn": "^1.2.1", + "react-live": "^2.3.0", "react-router-dom": "^5.2.0", "react-router-hash-link": "^2.4.3", "react-scripts": "4.0.3", @@ -63,6 +64,5 @@ "last 1 safari version" ] }, - "devDependencies": { - } + "devDependencies": {} } diff --git a/probe_ably/service/public/android-chrome-192x192.png b/probe_ably/app/public/android-chrome-192x192.png similarity index 100% rename from probe_ably/service/public/android-chrome-192x192.png rename to probe_ably/app/public/android-chrome-192x192.png diff --git a/probe_ably/service/public/android-chrome-512x512.png b/probe_ably/app/public/android-chrome-512x512.png similarity index 100% rename from probe_ably/service/public/android-chrome-512x512.png rename to probe_ably/app/public/android-chrome-512x512.png diff --git a/probe_ably/service/public/apple-touch-icon.png b/probe_ably/app/public/apple-touch-icon.png similarity index 100% rename from probe_ably/service/public/apple-touch-icon.png rename to probe_ably/app/public/apple-touch-icon.png diff --git a/probe_ably/service/public/browserconfig.xml b/probe_ably/app/public/browserconfig.xml similarity index 100% rename from probe_ably/service/public/browserconfig.xml rename to probe_ably/app/public/browserconfig.xml diff --git a/probe_ably/service/public/favicon-16x16.png b/probe_ably/app/public/favicon-16x16.png similarity index 100% rename from probe_ably/service/public/favicon-16x16.png rename to probe_ably/app/public/favicon-16x16.png diff --git a/probe_ably/service/public/favicon-32x32.png b/probe_ably/app/public/favicon-32x32.png similarity index 100% rename from probe_ably/service/public/favicon-32x32.png rename to probe_ably/app/public/favicon-32x32.png diff --git a/probe_ably/service/public/favicon.ico b/probe_ably/app/public/favicon.ico similarity index 100% rename from probe_ably/service/public/favicon.ico rename to probe_ably/app/public/favicon.ico diff --git a/probe_ably/service/public/index.html b/probe_ably/app/public/index.html similarity index 100% rename from probe_ably/service/public/index.html rename to probe_ably/app/public/index.html diff --git a/probe_ably/service/public/logo192.png b/probe_ably/app/public/logo192.png similarity index 100% rename from probe_ably/service/public/logo192.png rename to probe_ably/app/public/logo192.png diff --git a/probe_ably/service/public/logo512.png b/probe_ably/app/public/logo512.png similarity index 100% rename from probe_ably/service/public/logo512.png rename to probe_ably/app/public/logo512.png diff --git a/probe_ably/service/public/manifest.json b/probe_ably/app/public/manifest.json similarity index 100% rename from probe_ably/service/public/manifest.json rename to probe_ably/app/public/manifest.json diff --git a/probe_ably/service/public/mstile-150x150.png b/probe_ably/app/public/mstile-150x150.png similarity index 100% rename from probe_ably/service/public/mstile-150x150.png rename to probe_ably/app/public/mstile-150x150.png diff --git a/probe_ably/service/public/robots.txt b/probe_ably/app/public/robots.txt similarity index 100% rename from probe_ably/service/public/robots.txt rename to probe_ably/app/public/robots.txt diff --git a/probe_ably/service/public/safari-pinned-tab.svg b/probe_ably/app/public/safari-pinned-tab.svg similarity index 100% rename from probe_ably/service/public/safari-pinned-tab.svg rename to probe_ably/app/public/safari-pinned-tab.svg diff --git a/probe_ably/service/public/site.webmanifest b/probe_ably/app/public/site.webmanifest similarity index 100% rename from probe_ably/service/public/site.webmanifest rename to probe_ably/app/public/site.webmanifest diff --git a/probe_ably/app/src/assets/fireparrot.gif b/probe_ably/app/src/assets/fireparrot.gif new file mode 100644 index 0000000..e25bd71 Binary files /dev/null and b/probe_ably/app/src/assets/fireparrot.gif differ diff --git a/probe_ably/service/src/charts/LineChartWidget.js b/probe_ably/app/src/charts/LineChartWidget.js similarity index 96% rename from probe_ably/service/src/charts/LineChartWidget.js rename to probe_ably/app/src/charts/LineChartWidget.js index a83ae24..3d5d47d 100644 --- a/probe_ably/service/src/charts/LineChartWidget.js +++ b/probe_ably/app/src/charts/LineChartWidget.js @@ -1,16 +1,17 @@ -import { ResponsiveScatterPlot } from "@nivo/scatterplot"; +import { ResponsiveLine } from '@nivo/line'; import { Button, ButtonGroup, Card, Col, - Row, + Row } from "@themesberg/react-bootstrap"; import { jsPDF } from "jspdf"; import React, { useEffect, useRef } from "react"; import ReactDOM from "react-dom"; import "svg2pdf.js"; + export default (props) => { const { title, probing_data, task_name } = props; const linechartRefs = useRef([]); @@ -61,8 +62,8 @@ export default (props) => { style={{ height: 400 }} className="ct-series-g ct-major-tent" > - { tickRotation: 90, legend: p_data.x_axis, legendPosition: "middle", - format: (value) => value.toExponential(2), + // format: (value) => value, + format: (value) => value < 0.001 || value > 10000 ? value.toExponential(2) : value, legendOffset: 60, }} axisLeft={{ diff --git a/probe_ably/service/src/index.css b/probe_ably/app/src/index.css similarity index 100% rename from probe_ably/service/src/index.css rename to probe_ably/app/src/index.css diff --git a/probe_ably/service/src/index.js b/probe_ably/app/src/index.js similarity index 99% rename from probe_ably/service/src/index.js rename to probe_ably/app/src/index.js index 2b8c27f..ae9f086 100644 --- a/probe_ably/service/src/index.js +++ b/probe_ably/app/src/index.js @@ -22,6 +22,7 @@ import HomePage from "./pages/HomePage"; // core styles import "./scss/volt.scss"; + ReactDOM.render(
{/* */} diff --git a/probe_ably/service/src/logo.svg b/probe_ably/app/src/logo.svg similarity index 100% rename from probe_ably/service/src/logo.svg rename to probe_ably/app/src/logo.svg diff --git a/probe_ably/app/src/pages/HomePage.js b/probe_ably/app/src/pages/HomePage.js new file mode 100644 index 0000000..f1b7249 --- /dev/null +++ b/probe_ably/app/src/pages/HomePage.js @@ -0,0 +1,57 @@ +import { + Col, Container, + Navbar, + Row, + Spinner, + Card, +} from "@themesberg/react-bootstrap"; + +import React, { useEffect, useState } from "react"; +import Report from "./dashboard/Report.js" +import ConfigDashboard from "./forms/ConfigDashboard"; + +export default () => { + const [isProbing, setIsProbing] = useState(false); + const [results, setResults] = useState(null); + const [taskProgress, setTaskProgress] = useState(0); + const [modelProgress, setModelProgress] = useState(0); + const [probesProgress, setProbesProgress] = useState(0); + + + const startProbing = async (formData) => { + setIsProbing(true); + await fetch("/start_probing", {method: "POST", body: formData}) + .then(response => response.json()) + .then(data => setResults(data)); + setIsProbing(false); + } + + return ( +
+ + + + Probe_Ably + + + +
+ + + +
+
+ ) + }; \ No newline at end of file diff --git a/probe_ably/service/src/pages/dashboard/DashboardOverview.js b/probe_ably/app/src/pages/dashboard/DashboardOverview.js similarity index 78% rename from probe_ably/service/src/pages/dashboard/DashboardOverview.js rename to probe_ably/app/src/pages/dashboard/DashboardOverview.js index 7be5ef0..b4de95e 100644 --- a/probe_ably/service/src/pages/dashboard/DashboardOverview.js +++ b/probe_ably/app/src/pages/dashboard/DashboardOverview.js @@ -1,21 +1,21 @@ -import { Col, Nav, Row } from "@themesberg/react-bootstrap"; +import { Col, Nav, Row, Container } from "@themesberg/react-bootstrap"; import React, { useState } from "react"; import LineChartWidget from "../../charts/LineChartWidget"; -export default ({ aux_tasks }) => { +export default ({ tasks }) => { const [selectedTask, setSelectedTask] = useState(0); return ( -
+
- - {aux_tasks[selectedTask].probings.map((probe, i) => ( + {tasks[selectedTask].probings.map((probe, i) => ( { style={{ padding: "10px" }} > { ))} -
- ); -}; + +); +}; \ No newline at end of file diff --git a/probe_ably/app/src/pages/dashboard/ProgressBar.js b/probe_ably/app/src/pages/dashboard/ProgressBar.js new file mode 100644 index 0000000..47f8f90 --- /dev/null +++ b/probe_ably/app/src/pages/dashboard/ProgressBar.js @@ -0,0 +1,70 @@ +import React from "react"; + +const ProgressBar = ({ taskProgress, modelProgress, probesProgress }) => { + + const containerStyles = { + height: 20, + width: '400px', + backgroundColor: "#e0e0de", + borderRadius: 50, + margin: 50 + } + + const taskFillerStyles = { + height: '100%', + width: `${(taskProgress.n * 100) / taskProgress.total}%`, + backgroundColor: 'green', + borderRadius: 'inherit', + textAlign: 'right', + transition: 'width 1s ease-in-out', + } + const modelFillerStyles = { + height: '100%', + width: `${(modelProgress.n * 100) / modelProgress.total}%`, + backgroundColor: 'red', + borderRadius: 'inherit', + textAlign: 'right', + } + const probesFillerStyles = { + height: '100%', + width: `${(probesProgress.n * 100) / probesProgress.total}%`, + backgroundColor: 'red', + borderRadius: 'inherit', + textAlign: 'right', + } + + const labelStyles = { + color: 'white', + fontWeight: 'bold', + height: '90%', + paddingRight: '3px', + } + const imgStyle = { + paddingRight: '3px', + height: '90%', + } + + return ( + <> +
"Tasks:"
+
+
+ {`${taskProgress.n}/${taskProgress.total}`} +
+
+
"Representations:"
+
+
+ {`${modelProgress.n}/${modelProgress.total}`} +
+
+
"Training Probes:"
+
+
+ {`${probesProgress.n}/${probesProgress.total}`} +
+
+ + )}; + +export default ProgressBar; diff --git a/probe_ably/app/src/pages/dashboard/Report.js b/probe_ably/app/src/pages/dashboard/Report.js new file mode 100644 index 0000000..3a59b2d --- /dev/null +++ b/probe_ably/app/src/pages/dashboard/Report.js @@ -0,0 +1,94 @@ +import React, { useEffect } from "react"; +import ProgressBar from "./ProgressBar" +import Dashboard from "./DashboardOverview"; +import fireparrot from "../../assets/fireparrot.gif" +import { + Col, Container, + Navbar, + Row, + Spinner, + Card, + Nav, +} from "@themesberg/react-bootstrap"; + + +const Report = ({ taskProgress, modelProgress, probesProgress, setTaskProgress, setModelProgress, setProbesProgress, results, isProbing }) => { + +const imgStyle = { + paddingRight: '3px', + height: '90%', +} + useEffect(() => { + const interval = setInterval(() => { + if (isProbing) { + fetch("/task_progress") + .then((res) => res.json()) + .then(progress => {setTaskProgress(progress)}) + } + }, 500); + + return () => clearInterval(interval); + }, [taskProgress, setTaskProgress, isProbing]); + + useEffect(() => { + const interval = setInterval(() => { + if (isProbing) { + fetch("/model_progress") + .then((res) => res.json()) + .then((prog)=>{setModelProgress(prog)}); + } + }, 500); + + return () => clearInterval(interval); + }, [modelProgress, setModelProgress, isProbing]); + + useEffect(() => { + const interval = setInterval(() => { + if (isProbing) { + fetch("/probes_progress") + .then((res) => res.json()) + .then((prog)=>{setProbesProgress(prog)}); + } + }, 500); + + return () => clearInterval(interval); + }, [probesProgress, setProbesProgress, isProbing]); + + return ( + + + {results == null ? ( + isProbing == false ? (<>) : ( + + + + + {/* */} +
+ + +
+ + + + )) : ( + + +
+
Probing Results
+
+ + + + )} + + + ) + }; + + +export default Report diff --git a/probe_ably/app/src/pages/forms/Button.js b/probe_ably/app/src/pages/forms/Button.js new file mode 100644 index 0000000..930cf05 --- /dev/null +++ b/probe_ably/app/src/pages/forms/Button.js @@ -0,0 +1,11 @@ +import React from 'react'; + +const Button = ({ text, onClick }) => { + + +return ( + +) +} + +export default Button diff --git a/probe_ably/app/src/pages/forms/ConfigDashboard.js b/probe_ably/app/src/pages/forms/ConfigDashboard.js new file mode 100644 index 0000000..1c89e80 --- /dev/null +++ b/probe_ably/app/src/pages/forms/ConfigDashboard.js @@ -0,0 +1,53 @@ +import React, { useState } from "react"; +import { + Card, + Col, + Row, + Tab, + Nav, + Form, + Container +} from "@themesberg/react-bootstrap"; +import ConfigFileLoader from "./ConfigFileLoader"; +import ConfigForm from "./ConfigForm" + +const ConfigDashboard = ({ startProbing }) => { + return( +
+ + + + + + + + + + + + + + + + + + + + + +
+ ) + } + + export default ConfigDashboard \ No newline at end of file diff --git a/probe_ably/app/src/pages/forms/ConfigFileLoader.js b/probe_ably/app/src/pages/forms/ConfigFileLoader.js new file mode 100644 index 0000000..170e925 --- /dev/null +++ b/probe_ably/app/src/pages/forms/ConfigFileLoader.js @@ -0,0 +1,37 @@ +import React from "react"; +import { + Button, + Col, + Row, + Form, + Container +} from "@themesberg/react-bootstrap"; + +const ConfigFileLoader = ({ startProbing }) => { + + const submit = async () => { + var files = document.getElementById("config_file").files; + var formData = new FormData(); + formData.append('config_file', files[0]); + startProbing(formData); + } + + return( + + Choose Config File (JSON): + + + + + + + + +

+ (File Format Guide) +

+ +)} +export default ConfigFileLoader diff --git a/probe_ably/app/src/pages/forms/ConfigForm.js b/probe_ably/app/src/pages/forms/ConfigForm.js new file mode 100644 index 0000000..ad980d2 --- /dev/null +++ b/probe_ably/app/src/pages/forms/ConfigForm.js @@ -0,0 +1,189 @@ +import { Button, Card, Col, Form, Row } from "@themesberg/react-bootstrap"; +import React, { useState } from "react"; + +const ConfigForm = ({ startProbing }) => { + const submit = async () => { + var files = document.getElementById("config_file").files; + var formData = new FormData(); + formData.append('config_file', files[0]); + startProbing(formData); + } + + return( +
+ + +
+ ) +}; + +// return ( +//
+// +// +//
+// +// Task Name +// +// +// +//
+// +// + +// {tasks.map((task_data, i) => ( +// +// +// +//
+//
{task_data.task_name}
+//
+// +// +// +// +// +//
+// +// Model Name +// +// + +// +// Representations +// +// + +// +// Representations Labels +// +// +// +// Control Task Class +// +// +// +//
+// +// +// +// ))} + +// +// +// +// +// +// +//
+// ); +// }; + +// return ( +//
+// +// +//
+// +// Task Name +// +// +// +//
+// +// + +// {tasks.map((task_data, i) => ( +// +// +// +//
+//
{task_data.task_name}
+//
+// +// +// +// +// +//
+// +// Model Name +// +// + +// +// Representations +// +// + +// +// Representations Labels +// +// +// +// Control Task Class +// +// +// +//
+// +// +// +// ))} + +// +// +// +// +// +// +//
+// ); +// }; +export default ConfigForm \ No newline at end of file diff --git a/probe_ably/service/src/react-app-env.d.ts b/probe_ably/app/src/react-app-env.d.ts similarity index 100% rename from probe_ably/service/src/react-app-env.d.ts rename to probe_ably/app/src/react-app-env.d.ts diff --git a/probe_ably/service/src/reportWebVitals.js b/probe_ably/app/src/reportWebVitals.js similarity index 100% rename from probe_ably/service/src/reportWebVitals.js rename to probe_ably/app/src/reportWebVitals.js diff --git a/probe_ably/service/src/scss/volt.scss b/probe_ably/app/src/scss/volt.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt.scss rename to probe_ably/app/src/scss/volt.scss diff --git a/probe_ably/service/src/scss/volt/_components.scss b/probe_ably/app/src/scss/volt/_components.scss similarity index 100% rename from probe_ably/service/src/scss/volt/_components.scss rename to probe_ably/app/src/scss/volt/_components.scss diff --git a/probe_ably/service/src/scss/volt/_functions.scss b/probe_ably/app/src/scss/volt/_functions.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/_functions.scss rename to probe_ably/app/src/scss/volt/_functions.scss diff --git a/probe_ably/service/src/scss/volt/_layout.scss b/probe_ably/app/src/scss/volt/_layout.scss similarity index 100% rename from probe_ably/service/src/scss/volt/_layout.scss rename to probe_ably/app/src/scss/volt/_layout.scss diff --git a/probe_ably/service/src/scss/volt/_mixins.scss b/probe_ably/app/src/scss/volt/_mixins.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/_mixins.scss rename to probe_ably/app/src/scss/volt/_mixins.scss diff --git a/probe_ably/service/src/scss/volt/_reboot.scss b/probe_ably/app/src/scss/volt/_reboot.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/_reboot.scss rename to probe_ably/app/src/scss/volt/_reboot.scss diff --git a/probe_ably/service/src/scss/volt/_utilities.scss b/probe_ably/app/src/scss/volt/_utilities.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/_utilities.scss rename to probe_ably/app/src/scss/volt/_utilities.scss diff --git a/probe_ably/service/src/scss/volt/_variables.scss b/probe_ably/app/src/scss/volt/_variables.scss similarity index 100% rename from probe_ably/service/src/scss/volt/_variables.scss rename to probe_ably/app/src/scss/volt/_variables.scss diff --git a/probe_ably/service/src/scss/volt/_vendor.scss b/probe_ably/app/src/scss/volt/_vendor.scss similarity index 100% rename from probe_ably/service/src/scss/volt/_vendor.scss rename to probe_ably/app/src/scss/volt/_vendor.scss diff --git a/probe_ably/service/src/scss/volt/components/_accordions.scss b/probe_ably/app/src/scss/volt/components/_accordions.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_accordions.scss rename to probe_ably/app/src/scss/volt/components/_accordions.scss diff --git a/probe_ably/service/src/scss/volt/components/_alerts.scss b/probe_ably/app/src/scss/volt/components/_alerts.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_alerts.scss rename to probe_ably/app/src/scss/volt/components/_alerts.scss diff --git a/probe_ably/service/src/scss/volt/components/_avatars.scss b/probe_ably/app/src/scss/volt/components/_avatars.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_avatars.scss rename to probe_ably/app/src/scss/volt/components/_avatars.scss diff --git a/probe_ably/service/src/scss/volt/components/_badge.scss b/probe_ably/app/src/scss/volt/components/_badge.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_badge.scss rename to probe_ably/app/src/scss/volt/components/_badge.scss diff --git a/probe_ably/service/src/scss/volt/components/_blog-cards.scss b/probe_ably/app/src/scss/volt/components/_blog-cards.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_blog-cards.scss rename to probe_ably/app/src/scss/volt/components/_blog-cards.scss diff --git a/probe_ably/service/src/scss/volt/components/_breadcrumb.scss b/probe_ably/app/src/scss/volt/components/_breadcrumb.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_breadcrumb.scss rename to probe_ably/app/src/scss/volt/components/_breadcrumb.scss diff --git a/probe_ably/service/src/scss/volt/components/_buttons.scss b/probe_ably/app/src/scss/volt/components/_buttons.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_buttons.scss rename to probe_ably/app/src/scss/volt/components/_buttons.scss diff --git a/probe_ably/service/src/scss/volt/components/_card.scss b/probe_ably/app/src/scss/volt/components/_card.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_card.scss rename to probe_ably/app/src/scss/volt/components/_card.scss diff --git a/probe_ably/service/src/scss/volt/components/_carousel.scss b/probe_ably/app/src/scss/volt/components/_carousel.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_carousel.scss rename to probe_ably/app/src/scss/volt/components/_carousel.scss diff --git a/probe_ably/service/src/scss/volt/components/_charts.scss b/probe_ably/app/src/scss/volt/components/_charts.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_charts.scss rename to probe_ably/app/src/scss/volt/components/_charts.scss diff --git a/probe_ably/service/src/scss/volt/components/_close.scss b/probe_ably/app/src/scss/volt/components/_close.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_close.scss rename to probe_ably/app/src/scss/volt/components/_close.scss diff --git a/probe_ably/service/src/scss/volt/components/_counters.scss b/probe_ably/app/src/scss/volt/components/_counters.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_counters.scss rename to probe_ably/app/src/scss/volt/components/_counters.scss diff --git a/probe_ably/service/src/scss/volt/components/_custom-forms.scss b/probe_ably/app/src/scss/volt/components/_custom-forms.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_custom-forms.scss rename to probe_ably/app/src/scss/volt/components/_custom-forms.scss diff --git a/probe_ably/service/src/scss/volt/components/_datepicker.scss b/probe_ably/app/src/scss/volt/components/_datepicker.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_datepicker.scss rename to probe_ably/app/src/scss/volt/components/_datepicker.scss diff --git a/probe_ably/service/src/scss/volt/components/_dropdown.scss b/probe_ably/app/src/scss/volt/components/_dropdown.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_dropdown.scss rename to probe_ably/app/src/scss/volt/components/_dropdown.scss diff --git a/probe_ably/service/src/scss/volt/components/_dropzone.scss b/probe_ably/app/src/scss/volt/components/_dropzone.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_dropzone.scss rename to probe_ably/app/src/scss/volt/components/_dropzone.scss diff --git a/probe_ably/service/src/scss/volt/components/_forms.scss b/probe_ably/app/src/scss/volt/components/_forms.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_forms.scss rename to probe_ably/app/src/scss/volt/components/_forms.scss diff --git a/probe_ably/service/src/scss/volt/components/_icon-box.scss b/probe_ably/app/src/scss/volt/components/_icon-box.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_icon-box.scss rename to probe_ably/app/src/scss/volt/components/_icon-box.scss diff --git a/probe_ably/service/src/scss/volt/components/_images.scss b/probe_ably/app/src/scss/volt/components/_images.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_images.scss rename to probe_ably/app/src/scss/volt/components/_images.scss diff --git a/probe_ably/service/src/scss/volt/components/_input-group.scss b/probe_ably/app/src/scss/volt/components/_input-group.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_input-group.scss rename to probe_ably/app/src/scss/volt/components/_input-group.scss diff --git a/probe_ably/service/src/scss/volt/components/_list-group.scss b/probe_ably/app/src/scss/volt/components/_list-group.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_list-group.scss rename to probe_ably/app/src/scss/volt/components/_list-group.scss diff --git a/probe_ably/service/src/scss/volt/components/_modal.scss b/probe_ably/app/src/scss/volt/components/_modal.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_modal.scss rename to probe_ably/app/src/scss/volt/components/_modal.scss diff --git a/probe_ably/service/src/scss/volt/components/_nav.scss b/probe_ably/app/src/scss/volt/components/_nav.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_nav.scss rename to probe_ably/app/src/scss/volt/components/_nav.scss diff --git a/probe_ably/service/src/scss/volt/components/_pagination.scss b/probe_ably/app/src/scss/volt/components/_pagination.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_pagination.scss rename to probe_ably/app/src/scss/volt/components/_pagination.scss diff --git a/probe_ably/service/src/scss/volt/components/_popover.scss b/probe_ably/app/src/scss/volt/components/_popover.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_popover.scss rename to probe_ably/app/src/scss/volt/components/_popover.scss diff --git a/probe_ably/service/src/scss/volt/components/_preloader.scss b/probe_ably/app/src/scss/volt/components/_preloader.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_preloader.scss rename to probe_ably/app/src/scss/volt/components/_preloader.scss diff --git a/probe_ably/service/src/scss/volt/components/_pricing-cards.scss b/probe_ably/app/src/scss/volt/components/_pricing-cards.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_pricing-cards.scss rename to probe_ably/app/src/scss/volt/components/_pricing-cards.scss diff --git a/probe_ably/service/src/scss/volt/components/_progress.scss b/probe_ably/app/src/scss/volt/components/_progress.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_progress.scss rename to probe_ably/app/src/scss/volt/components/_progress.scss diff --git a/probe_ably/service/src/scss/volt/components/_scrollbar.scss b/probe_ably/app/src/scss/volt/components/_scrollbar.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_scrollbar.scss rename to probe_ably/app/src/scss/volt/components/_scrollbar.scss diff --git a/probe_ably/service/src/scss/volt/components/_shapes.scss b/probe_ably/app/src/scss/volt/components/_shapes.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_shapes.scss rename to probe_ably/app/src/scss/volt/components/_shapes.scss diff --git a/probe_ably/service/src/scss/volt/components/_tables.scss b/probe_ably/app/src/scss/volt/components/_tables.scss similarity index 100% rename from probe_ably/service/src/scss/volt/components/_tables.scss rename to probe_ably/app/src/scss/volt/components/_tables.scss diff --git a/probe_ably/service/src/scss/volt/components/_timelines.scss b/probe_ably/app/src/scss/volt/components/_timelines.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_timelines.scss rename to probe_ably/app/src/scss/volt/components/_timelines.scss diff --git a/probe_ably/service/src/scss/volt/components/_tooltip.scss b/probe_ably/app/src/scss/volt/components/_tooltip.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_tooltip.scss rename to probe_ably/app/src/scss/volt/components/_tooltip.scss diff --git a/probe_ably/service/src/scss/volt/components/_type.scss b/probe_ably/app/src/scss/volt/components/_type.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/components/_type.scss rename to probe_ably/app/src/scss/volt/components/_type.scss diff --git a/probe_ably/service/src/scss/volt/layout/_footer.scss b/probe_ably/app/src/scss/volt/layout/_footer.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/layout/_footer.scss rename to probe_ably/app/src/scss/volt/layout/_footer.scss diff --git a/probe_ably/service/src/scss/volt/layout/_navbar.scss b/probe_ably/app/src/scss/volt/layout/_navbar.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/layout/_navbar.scss rename to probe_ably/app/src/scss/volt/layout/_navbar.scss diff --git a/probe_ably/service/src/scss/volt/layout/_section.scss b/probe_ably/app/src/scss/volt/layout/_section.scss similarity index 100% rename from probe_ably/service/src/scss/volt/layout/_section.scss rename to probe_ably/app/src/scss/volt/layout/_section.scss diff --git a/probe_ably/service/src/scss/volt/layout/_sidebar.scss b/probe_ably/app/src/scss/volt/layout/_sidebar.scss similarity index 100% rename from probe_ably/service/src/scss/volt/layout/_sidebar.scss rename to probe_ably/app/src/scss/volt/layout/_sidebar.scss diff --git a/probe_ably/service/src/scss/volt/layout/_sidenav.scss b/probe_ably/app/src/scss/volt/layout/_sidenav.scss similarity index 100% rename from probe_ably/service/src/scss/volt/layout/_sidenav.scss rename to probe_ably/app/src/scss/volt/layout/_sidenav.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_animations.scss b/probe_ably/app/src/scss/volt/mixins/_animations.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_animations.scss rename to probe_ably/app/src/scss/volt/mixins/_animations.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_background-variant.scss b/probe_ably/app/src/scss/volt/mixins/_background-variant.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_background-variant.scss rename to probe_ably/app/src/scss/volt/mixins/_background-variant.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_icon.scss b/probe_ably/app/src/scss/volt/mixins/_icon.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_icon.scss rename to probe_ably/app/src/scss/volt/mixins/_icon.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_modals.scss b/probe_ably/app/src/scss/volt/mixins/_modals.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_modals.scss rename to probe_ably/app/src/scss/volt/mixins/_modals.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_popover.scss b/probe_ably/app/src/scss/volt/mixins/_popover.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_popover.scss rename to probe_ably/app/src/scss/volt/mixins/_popover.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_transform.scss b/probe_ably/app/src/scss/volt/mixins/_transform.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_transform.scss rename to probe_ably/app/src/scss/volt/mixins/_transform.scss diff --git a/probe_ably/service/src/scss/volt/mixins/_utilities.scss b/probe_ably/app/src/scss/volt/mixins/_utilities.scss similarity index 100% rename from probe_ably/service/src/scss/volt/mixins/_utilities.scss rename to probe_ably/app/src/scss/volt/mixins/_utilities.scss diff --git a/probe_ably/service/src/scss/volt/themes/_variables-dark.scss b/probe_ably/app/src/scss/volt/themes/_variables-dark.scss similarity index 100% rename from probe_ably/service/src/scss/volt/themes/_variables-dark.scss rename to probe_ably/app/src/scss/volt/themes/_variables-dark.scss diff --git a/probe_ably/service/src/scss/volt/themes/_variables-light.scss b/probe_ably/app/src/scss/volt/themes/_variables-light.scss similarity index 100% rename from probe_ably/service/src/scss/volt/themes/_variables-light.scss rename to probe_ably/app/src/scss/volt/themes/_variables-light.scss diff --git a/probe_ably/service/src/scss/volt/themes/_variables-sunset.scss b/probe_ably/app/src/scss/volt/themes/_variables-sunset.scss similarity index 100% rename from probe_ably/service/src/scss/volt/themes/_variables-sunset.scss rename to probe_ably/app/src/scss/volt/themes/_variables-sunset.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_animations.scss b/probe_ably/app/src/scss/volt/utilities/_animations.scss similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_animations.scss rename to probe_ably/app/src/scss/volt/utilities/_animations.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_backgrounds.scss b/probe_ably/app/src/scss/volt/utilities/_backgrounds.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_backgrounds.scss rename to probe_ably/app/src/scss/volt/utilities/_backgrounds.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_helper.scss b/probe_ably/app/src/scss/volt/utilities/_helper.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_helper.scss rename to probe_ably/app/src/scss/volt/utilities/_helper.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_position.scss b/probe_ably/app/src/scss/volt/utilities/_position.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_position.scss rename to probe_ably/app/src/scss/volt/utilities/_position.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_shadows.scss b/probe_ably/app/src/scss/volt/utilities/_shadows.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_shadows.scss rename to probe_ably/app/src/scss/volt/utilities/_shadows.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_sizing.scss b/probe_ably/app/src/scss/volt/utilities/_sizing.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_sizing.scss rename to probe_ably/app/src/scss/volt/utilities/_sizing.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_text.scss b/probe_ably/app/src/scss/volt/utilities/_text.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_text.scss rename to probe_ably/app/src/scss/volt/utilities/_text.scss diff --git a/probe_ably/service/src/scss/volt/utilities/_transform.scss b/probe_ably/app/src/scss/volt/utilities/_transform.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/utilities/_transform.scss rename to probe_ably/app/src/scss/volt/utilities/_transform.scss diff --git a/probe_ably/service/src/scss/volt/vendor/_datepicker.scss b/probe_ably/app/src/scss/volt/vendor/_datepicker.scss similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/_datepicker.scss rename to probe_ably/app/src/scss/volt/vendor/_datepicker.scss diff --git a/probe_ably/service/src/scss/volt/vendor/_headroom.scss b/probe_ably/app/src/scss/volt/vendor/_headroom.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/_headroom.scss rename to probe_ably/app/src/scss/volt/vendor/_headroom.scss diff --git a/probe_ably/service/src/scss/volt/vendor/_prism.scss b/probe_ably/app/src/scss/volt/vendor/_prism.scss similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/_prism.scss rename to probe_ably/app/src/scss/volt/vendor/_prism.scss diff --git a/probe_ably/service/src/scss/volt/vendor/chartist/_chartist.scss b/probe_ably/app/src/scss/volt/vendor/chartist/_chartist.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/chartist/_chartist.scss rename to probe_ably/app/src/scss/volt/vendor/chartist/_chartist.scss diff --git a/probe_ably/service/src/scss/volt/vendor/chartist/settings/_chartist-settings.scss b/probe_ably/app/src/scss/volt/vendor/chartist/settings/_chartist-settings.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/chartist/settings/_chartist-settings.scss rename to probe_ably/app/src/scss/volt/vendor/chartist/settings/_chartist-settings.scss diff --git a/probe_ably/service/src/scss/volt/vendor/wizard/_form.scss b/probe_ably/app/src/scss/volt/vendor/wizard/_form.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/wizard/_form.scss rename to probe_ably/app/src/scss/volt/vendor/wizard/_form.scss diff --git a/probe_ably/service/src/scss/volt/vendor/wizard/_mixins.scss b/probe_ably/app/src/scss/volt/vendor/wizard/_mixins.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/wizard/_mixins.scss rename to probe_ably/app/src/scss/volt/vendor/wizard/_mixins.scss diff --git a/probe_ably/service/src/scss/volt/vendor/wizard/_variables.scss b/probe_ably/app/src/scss/volt/vendor/wizard/_variables.scss old mode 100755 new mode 100644 similarity index 100% rename from probe_ably/service/src/scss/volt/vendor/wizard/_variables.scss rename to probe_ably/app/src/scss/volt/vendor/wizard/_variables.scss diff --git a/probe_ably/service/tsconfig.json b/probe_ably/app/tsconfig.json similarity index 100% rename from probe_ably/service/tsconfig.json rename to probe_ably/app/tsconfig.json diff --git a/probe_ably/constants/__init__.py b/probe_ably/constants/__init__.py new file mode 100644 index 0000000..01c9e24 --- /dev/null +++ b/probe_ably/constants/__init__.py @@ -0,0 +1,5 @@ +from pathlib import Path + +CONFIG_FOLDER = Path(__file__).parent +SCHEMA_TEMPLATE_FILE = CONFIG_FOLDER.joinpath('input_json_schema.json') +DEFAULT_PROBING_SETUP = CONFIG_FOLDER.joinpath('default_probing_setup.json') \ No newline at end of file diff --git a/probe_ably/constants/default_probing_setup.json b/probe_ably/constants/default_probing_setup.json new file mode 100644 index 0000000..97f782e --- /dev/null +++ b/probe_ably/constants/default_probing_setup.json @@ -0,0 +1,21 @@ +{ + "train_size":0.6, + "dev_size":0.2, + "test_size":0.2, + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", + "probing_models":[ + { + "probing_model_name":"probe_ably.models.linear.LinearModel", + "batch_size":16, + "epochs":10, + "number_of_models":20 + }, + { + "probing_model_name":"probe_ably.models.mlp.MLPModel", + "batch_size":16, + "epochs":10, + "number_of_models":20 + } + ] +} \ No newline at end of file diff --git a/config/json_schema/input_file.json b/probe_ably/constants/input_json_schema.json similarity index 89% rename from config/json_schema/input_file.json rename to probe_ably/constants/input_json_schema.json index 999f4e5..711e0e7 100644 --- a/config/json_schema/input_file.json +++ b/probe_ably/constants/input_json_schema.json @@ -9,12 +9,12 @@ "task_name":{ "type":"string" }, - "models":{ + "representations":{ "type":"array", "items":{ "type":"object", "properties":{ - "model_name":{ + "representation_name":{ "type":"string" }, "file_location":{ @@ -25,7 +25,7 @@ } }, "required":[ - "model_name", + "representation_name", "file_location" ] } @@ -33,11 +33,11 @@ }, "required":[ "task_name", - "models" + "representations" ] } }, - "probing_setup":{ + "probing_config":{ "type":"object", "properties":{ "train_size":{ @@ -89,16 +89,9 @@ } }, "required":[ - "train_size", - "dev_size", - "test_size", "intra_metric", - "inter_metric", "probing_models" ] } - }, - "required":[ - "tasks" - ] + } } \ No newline at end of file diff --git a/probe_ably/core/flows/__init__.py b/probe_ably/core/flows/__init__.py deleted file mode 100644 index 6e05287..0000000 --- a/probe_ably/core/flows/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .probe_from_dataloaders import probe_from_dataloaders \ No newline at end of file diff --git a/probe_ably/core/flows/probe_from_dataloaders.py b/probe_ably/core/flows/probe_from_dataloaders.py deleted file mode 100644 index 93b4f51..0000000 --- a/probe_ably/core/flows/probe_from_dataloaders.py +++ /dev/null @@ -1,20 +0,0 @@ -import click -import prefect -from dynaconf import settings -from loguru import logger -from prefect import Flow -from prefect.engine.flow_runner import FlowRunner -from probe_ably.core.tasks.probing import TrainProbingTask -from probe_ably.core.tasks.metric_task import ProcessMetricTask - -INPUT_FILE = "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" -train_probing_task = TrainProbingTask() -process_metric_task = ProcessMetricTask() - -def probe_from_dataloaders(config_dict, prepared_data): - with Flow("Running Probe") as flow1: - train_results = train_probing_task(prepared_data, config_dict["probing_setup"]) - processed_results = process_metric_task( - train_results, config_dict["probing_setup"] - ) - FlowRunner(flow=flow1).run \ No newline at end of file diff --git a/probe_ably/core/flows/run_probing.py b/probe_ably/core/flows/run_probing.py deleted file mode 100644 index d84fcf7..0000000 --- a/probe_ably/core/flows/run_probing.py +++ /dev/null @@ -1,36 +0,0 @@ -import click -import prefect -from dynaconf import settings -from loguru import logger -from prefect import Flow, task -from prefect.engine.flow_runner import FlowRunner -from probe_ably.core.tasks.metric_task import ProcessMetricTask -from probe_ably.core.tasks.probing import PrepareDataForProbingTask, TrainProbingTask -from probe_ably.core.tasks.utils import ReadInputTask, VisualiaztionTask - -INPUT_FILE = "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" -read_input_task = ReadInputTask() -prepare_data_probing = PrepareDataForProbingTask() -train_probing_task = TrainProbingTask() -process_metric_task = ProcessMetricTask() -visualization_task = VisualiaztionTask() - - -@click.command() -@click.option("--config_file", help="Probing Configuration File") -def run_probing(config_file): - with Flow("Running Probe") as flow1: - parsed_input = read_input_task(config_file) - prepared_data = prepare_data_probing( - parsed_input["tasks"], parsed_input["probing_setup"] - ) - train_results = train_probing_task(prepared_data, parsed_input["probing_setup"]) - processed_results = process_metric_task( - train_results, parsed_input["probing_setup"] - ) - visualization_task(processed_results) - FlowRunner(flow=flow1).run() - - -if __name__ == "__main__": - run_probing() diff --git a/probe_ably/core/tasks/control_task/__init__.py b/probe_ably/core/tasks/control_task/__init__.py deleted file mode 100644 index 4d90a48..0000000 --- a/probe_ably/core/tasks/control_task/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .generate_control_task import GenerateControlTask \ No newline at end of file diff --git a/probe_ably/core/tasks/metric_task/__init__.py b/probe_ably/core/tasks/metric_task/__init__.py deleted file mode 100644 index e7876ef..0000000 --- a/probe_ably/core/tasks/metric_task/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .process_metric_task import ProcessMetricTask \ No newline at end of file diff --git a/probe_ably/core/tasks/probing/__init__.py b/probe_ably/core/tasks/probing/__init__.py deleted file mode 100644 index b26d98b..0000000 --- a/probe_ably/core/tasks/probing/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .prepare_data_for_probing_task import PrepareDataForProbingTask -from .train_probing_task import TrainProbingTask \ No newline at end of file diff --git a/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py b/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py deleted file mode 100644 index e43109f..0000000 --- a/probe_ably/core/tasks/probing/prepare_data_for_probing_task.py +++ /dev/null @@ -1,158 +0,0 @@ -from overrides import overrides -from prefect import Task -from loguru import logger -import sklearn -from sklearn.model_selection import train_test_split -from typing import Dict -import torch -from torch.utils.data import Dataset -import numpy as np - -class PrepareDataForProbingTask(Task): - @staticmethod - def prepare_entries(vectors, labels): - dataset = dict() - for i in range(0, len(vectors)): - dataset[i] = {"representation": vectors[i], "label": labels[i]} - - return TorchDataset(dataset) - - @staticmethod - def train_val_test_split(X, y, train_size, val_size, test_size, seed=42): - X_train_val, X_test, y_train_val, y_test = train_test_split( - X, y, test_size=test_size, random_state=seed - ) - relative_train_size = train_size / (val_size + train_size) - - X_train, X_val, y_train, y_val = train_test_split( - X_train_val, - y_train_val, - test_size=1 - relative_train_size, - random_state=seed, - ) - - return X_train, X_val, X_test, y_train, y_val, y_test - - def run(self, tasks_data: Dict, experiment_setup: Dict) -> Dict: - """Reads the task_data and experiment_setup, splits into train/dev/test and - creates a TorchDataset for each. - - :param tasks_data: Tasks info obtained from user input - :type tasks_data: Dict - :param experiment_setup: Experiment setup obtained from default file or user input - :type experiment_setup: Dict - :return: Dictonary of processed data in the format: - { task_id: - {'task_name': str, - 'models': - {model_id: - {"model_name": str, - "model": {"train": numpy.ndarray, "dev": numpy.ndarray, "test": numpy.ndarray}, - "control": {"train": numpy.ndarray, "dev": numpy.ndarray, "test": numpy.ndarray}, - "representation_size": int, - "number_of_classes": int, - "default_control": boolean (False if user inputs control task) - } - } - } - } - :rtype: Dict - """ - - logger.debug("Prepare the data for probing.") - output_data = dict() - - for id_task, task_content in tasks_data.items(): - output_data[id_task] = dict() - output_data[id_task]["task_name"] = task_content["task_name"] - output_data[id_task]["models"] = dict() - for model_id, model_content in task_content["models"].items(): - output_data[id_task]["models"][model_id] = dict() - - output_data[id_task]["models"][model_id][ - "representation_size" - ] = model_content["representation_size"] - - output_data[id_task]["models"][model_id][ - "number_of_classes" - ] = model_content["number_of_classes"] - - output_data[id_task]["models"][model_id][ - "default_control" - ] = model_content["default_control"] - - output_data[id_task]["models"][model_id]["model_name"] = model_content[ - "model_name" - ] - - ( - model_vectors_train, - model_vectors_val, - model_vectors_test, - model_labels_train, - model_labels_val, - model_labels_test, - ) = self.train_val_test_split( X=model_content["model_vectors"], y=model_content["model_labels"], - train_size=experiment_setup["train_size"], - val_size=experiment_setup["dev_size"], - test_size=experiment_setup["test_size"], - ) - - output_data[id_task]["models"][model_id]["model"] = { - "train": self.prepare_entries( - model_vectors_train, model_labels_train - ), - "dev": self.prepare_entries(model_vectors_val, model_labels_val), - "test": self.prepare_entries(model_vectors_test, model_labels_test), - } - - ( - control_vectors_train, - control_vectors_val, - control_vectors_test, - control_labels_train, - control_labels_val, - control_labels_test, - ) = self.train_val_test_split( - X=model_content["model_vectors"], - y=model_content["control_labels"], - train_size=experiment_setup["train_size"], - val_size=experiment_setup["dev_size"], - test_size=experiment_setup["test_size"], - ) - - output_data[id_task]["models"][model_id]["control"] = { - "train": self.prepare_entries( - control_vectors_train, control_labels_train - ), - "dev": self.prepare_entries( - control_vectors_val, control_labels_val - ), - "test": self.prepare_entries( - control_vectors_test, control_labels_test - ), - } - - return output_data - - -class TorchDataset(Dataset): - def __init__(self, dataset): - - self.dataset = list(dataset.values()) - self.labels = np.array([data["label"] for data in self.dataset]) - self.keys = list(dataset.keys()) - - def __getitem__(self, index): - instance = self.dataset[index] - return ( - torch.FloatTensor(instance["representation"]), - instance["label"], - index, - ) - - def get_id(self, index): - return self.keys[index] - - def __len__(self): - return len(self.dataset) \ No newline at end of file diff --git a/probe_ably/core/tasks/utils/__init__.py b/probe_ably/core/tasks/utils/__init__.py deleted file mode 100644 index 45bc50b..0000000 --- a/probe_ably/core/tasks/utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .read_input_task import ReadInputTask -from .visualization_task import VisualiaztionTask diff --git a/probe_ably/core/tasks/utils/read_input_task.py b/probe_ably/core/tasks/utils/read_input_task.py deleted file mode 100644 index c8f73f3..0000000 --- a/probe_ably/core/tasks/utils/read_input_task.py +++ /dev/null @@ -1,263 +0,0 @@ -from overrides import overrides -from prefect import Task -from typing import Dict -from loguru import logger -import os.path -import json -import sys -import jsonschema -import os.path -import pandas as pd -import numpy as np -from dynaconf import settings -from probe_ably.core.tasks.control_task import GenerateControlTask -from probe_ably.core.models import AbstractModel -from probe_ably.core.metrics import AbstractInterModelMetric, AbstractIntraModelMetric - -SCHEMA_TEMPLATE_FILE = settings["input_json_schema"] - -class ModelRepresentationFileNotFound(Exception): - def __init__(self, model_location): - self.model_location = model_location - - -class ControlSizeMissmatch(Exception): - def __init__(self, task_name, model_name): - self.task_name = task_name - self.model_name = model_name - - -class InputClassNotFound(Exception): - def __init__(self, type_of_class, class_name): - self.type_of_class = type_of_class - self.class_name = class_name - - -class ReadInputTask(Task): - def run(self, input_file_location: str) -> Dict: - """Function that parses the input configuration file provided by the user. - - :param input_file_location: Input json file containing the representations for probing and probing setup. - The file should follow the template in settings["input_json_schema"] - :type input_file_location: str - :return: Dictionary of the parsed input, in the following format: - { - "tasks": { - int (task_id): - { - "task_name": str, - "models": { - int (model_id): { - "model_name": str, - "model_vectors": array (representations being probed), - "model_labels": array (labels for the auxiliary task), - "control_labels": array (labels for the control task), - "representation_size": int (size of each representation), - "number_of_classes": int (number of unique labels), - "default_control": boolean (False if user inputs control task) - } - } - } - } - - "probing_setup": { - "inter_metric": string (class for metric to compare model and control task), - "intra_metric": string (class for metric that will be used for measuring - the best model), - "dev_size": int, - "train_size": int, - "test_size": int, - "probing_models": { - int (probing_models_id): { - "probing_model_name": string (class for the probing model), - "batch_size": int, - "epochs": int, - "number_of_models": int (number of probe models for generation) - } - } - } - } - :rtype: Dict - """ - - generate_control_task = GenerateControlTask() - logger.debug("Reading input file.") - try: - with open(input_file_location, "r") as f: - input_data = json.load(f) - logger.debug(f"Opening file located in {input_file_location}") - - with open(SCHEMA_TEMPLATE_FILE, "r") as f: - input_template = json.load(f) - jsonschema.validate(instance=input_data, schema=input_template) - - output_dict = dict() - current_task_id = 0 - task_list = input_data["tasks"] - ## Getting input info - for task_content in task_list: - output_dict[current_task_id] = dict() - output_dict[current_task_id]["task_name"] = task_content["task_name"] - output_dict[current_task_id]["models"] = dict() - models_list = task_content["models"] - - current_model_id = 0 - for model_content in models_list: - if not os.path.isfile(model_content["file_location"]): - raise ModelRepresentationFileNotFound( - model_content["file_location"] - ) - - output_dict[current_task_id]["models"][ - current_model_id - ] = self.parse_model_info( - model_content, task_content["task_name"], generate_control_task - ) - - current_model_id += 1 - current_task_id += 1 - - ## Getting probe info - probing_setup = self.parse_probing_setup(input_data) - - except FileNotFoundError: - sys.exit(f"Input file not found: {input_file_location}") - except json.JSONDecodeError as e: - sys.exit( - f"Input file is not a properly foramtted json file: {input_file_location}" - ) - except jsonschema.ValidationError as e: - logger.error(e) - sys.exit( - f"Input file ({input_file_location}) does not follow correct template. Please refer to README file." - ) - except ModelRepresentationFileNotFound as e: - sys.exit(f"Representation file ({e.model_location}) not found.") - - except ControlSizeMissmatch as e: - sys.exit( - f"Control task for task {e.task_name} and model {e.model_name} does not match the number of labels of the aux task." - ) - except InputClassNotFound as e: - sys.exit( - f"Error in probing setup: Element {e.type_of_class} with content {e.class_name} not found." - ) - except ValueError as e: - sys.exit(e) - return {"tasks": output_dict, "probing_setup": probing_setup} - - @staticmethod - def parse_model_info(model_content, task_name, generate_control_task): - model_representation = pd.read_csv( - model_content["file_location"], sep="\t", header=None - ) - model_labels = model_representation.iloc[:, -1].to_numpy() - - model_representation.drop( - model_representation.columns[-1], axis=1, inplace=True - ) - model_representation = model_representation.to_numpy() - - if "control_location" in model_content: - default_control = False - if not os.path.isfile(model_content["control_location"]): - raise ModelRepresentationFileNotFound(model_content["control_location"]) - - control_labels = np.loadtxt( - fname=model_content["control_location"], - delimiter="\t", - dtype=int, - ) - - if len(control_labels) != len(model_labels): - raise ControlSizeMissmatch( - task_name, - model_content["model_name"], - ) - else: - default_control = True - logger.info( - f"No control labels provided for task {task_name} and model {model_content['model_name']}, generating random control task." - ) - control_labels = generate_control_task.run( - model_representation, model_labels - ) - total_number_of_classes = ( - np.amax(np.concatenate((model_labels, control_labels))) + 1 - ) - return { - "model_name": model_content["model_name"], - "model_vectors": model_representation, - "model_labels": model_labels, - "control_labels": control_labels, - "representation_size": model_representation.shape[1], - "number_of_classes": total_number_of_classes, - "default_control": default_control, - } - - @staticmethod - def parse_probing_setup(input_data): - if "probing_setup" not in input_data: - with open(settings["default_probing_setup"], "r") as f: - probing_setup = json.load(f) - - logger.info( - "No experiment setup provided, using the following default values:" - ) - logger.info(probing_setup) - else: - available_inter_metrics = AbstractInterModelMetric.subclasses - available_intra_metrics = AbstractIntraModelMetric.subclasses - available_probing_models = AbstractModel.subclasses - - if ( - input_data["probing_setup"]["inter_metric"] - not in available_inter_metrics - ): - raise InputClassNotFound( - "inter_metric", input_data["probing_setup"]["inter_metric"] - ) - - if ( - input_data["probing_setup"]["intra_metric"] - not in available_intra_metrics - ): - raise InputClassNotFound( - "intra_metric", input_data["probing_setup"]["intra_metric"] - ) - - split_distribution = ( - input_data["probing_setup"]["train_size"] - + input_data["probing_setup"]["test_size"] - + input_data["probing_setup"]["dev_size"] - ) - - if split_distribution != 1: - raise ValueError( - f"Train + Test + Dev size should be equals to 1, got {split_distribution}" - ) - probing_setup = { - "inter_metric": input_data["probing_setup"]["inter_metric"], - "intra_metric": input_data["probing_setup"]["intra_metric"], - "train_size": input_data["probing_setup"]["train_size"], - "dev_size": input_data["probing_setup"]["dev_size"], - "test_size": input_data["probing_setup"]["test_size"], - "probing_models": dict(), - } - num_probing_models = 0 - - for probe_model in input_data["probing_setup"]["probing_models"]: - if probe_model["probing_model_name"] not in available_probing_models: - raise InputClassNotFound( - "probing_model_name", probe_model["probing_model_name"] - ) - - probing_setup["probing_models"][num_probing_models] = probe_model - num_probing_models += 1 - - logger.info( - "Using the experiment setup provided in the input file with values:" - ) - logger.info(probing_setup) - - return probing_setup \ No newline at end of file diff --git a/probe_ably/core/tasks/utils/visualization_task.py b/probe_ably/core/tasks/utils/visualization_task.py deleted file mode 100644 index 55f69d4..0000000 --- a/probe_ably/core/tasks/utils/visualization_task.py +++ /dev/null @@ -1,26 +0,0 @@ -import json -import webbrowser -from typing import Dict - -from loguru import logger -from overrides import overrides -from prefect import Task -from probe_ably.service.server.web_server import WebServer - - -class VisualiaztionTask(Task): - def run(self, processed_data: Dict): - logger.info("Launching server for visualization") - web_server = WebServer(processed_data) - # with open(f"{web_server.get_path()}/data.json", "w") as f: - # json.dump(processed_data, f) - web_server.start() - - ip_address = "http://127.0.0.1:8031/" - - try: - webbrowser.get("google-chrome").open(ip_address) - except: - logger.info( - f"Tried to launch Google Chrom and Failed. Visit: {ip_address} to view the visualization" - ) diff --git a/probe_ably/core/utils/__init__.py b/probe_ably/core/utils/__init__.py deleted file mode 100644 index 9214628..0000000 --- a/probe_ably/core/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .grid_model_factory import GridModelFactory \ No newline at end of file diff --git a/probe_ably/core/utils/util_functions.py b/probe_ably/core/utils/util_functions.py deleted file mode 100644 index e69de29..0000000 diff --git a/probe_ably/experiment_setup.py b/probe_ably/experiment_setup.py new file mode 100644 index 0000000..303d46e --- /dev/null +++ b/probe_ably/experiment_setup.py @@ -0,0 +1,64 @@ +import asyncio +import pathlib +from typing import List, Union +from probe_ably.probing import TrainProbingTask +from probe_ably.metrics import ProcessMetricTask +from probe_ably.probing.prepare_data import prep_data_from_parsed_json +from probe_ably.utils import ReadInputTask, ProbingConfig, ProbingTask, ProbingInput +from probe_ably.constants import DEFAULT_PROBING_SETUP +from loguru import logger +from dataclasses import dataclass +from torch.utils.data import Dataset +import threading + +read_input_task = ReadInputTask() +train_probing_task = TrainProbingTask() +process_metric_task = ProcessMetricTask() + +class ProbingExperiment: + def __init__(self, + probing_config: ProbingConfig, + tasks: Union[ProbingTask, List[ProbingTask]] = [], + thread: threading.Thread = None): + self.probing_config = probing_config + self.tasks = tasks + self.thread = thread + + def load_tasks(self, tasks: Union[ProbingTask, List[ProbingTask]]): + self.tasks = tasks + + @classmethod + def from_parsed_input(cls, parsed_input: ProbingInput, thread=None): + probing_config = parsed_input["probing_config"] + + if parsed_input["tasks"]: + logger.info("Loading data from paths in config") + + tasks = prep_data_from_parsed_json( + parsed_input["tasks"], + parsed_input["probing_config"] + ) + else: + tasks = [] + + return cls(probing_config, tasks, thread) + + @classmethod + def from_json(cls, config_path: Union[str, pathlib.Path], thread=None): + if config_path == "default": + config_path = DEFAULT_PROBING_SETUP + + with open(config_path) as config_file: + parsed_input = asyncio.run(read_input_task.run(config_file)) + + return cls.from_parsed_input(parsed_input, thread=thread) + + def run(self): + train_results = train_probing_task.run(self.tasks, self.probing_config, self.thread) + processed_results = process_metric_task.run( + train_results, self.probing_config + ) + + # visualization_task.run(processed_results) + return processed_results + # visualization_task.run(processed_results) diff --git a/probe_ably/core/metrics/__init__.py b/probe_ably/metrics/__init__.py similarity index 63% rename from probe_ably/core/metrics/__init__.py rename to probe_ably/metrics/__init__.py index d6b3150..2ee4605 100644 --- a/probe_ably/core/metrics/__init__.py +++ b/probe_ably/metrics/__init__.py @@ -1,4 +1,5 @@ from .abstract_inter_model_metric import AbstractInterModelMetric from .abstract_intra_model_metric import AbstractIntraModelMetric from .accuracy import AccuracyMetric -from .selectivity import SelectivityMetric \ No newline at end of file +from .selectivity import SelectivityMetric +from .process_metric_task import ProcessMetricTask \ No newline at end of file diff --git a/probe_ably/core/metrics/abstract_inter_model_metric.py b/probe_ably/metrics/abstract_inter_model_metric.py similarity index 100% rename from probe_ably/core/metrics/abstract_inter_model_metric.py rename to probe_ably/metrics/abstract_inter_model_metric.py diff --git a/probe_ably/core/metrics/abstract_intra_model_metric.py b/probe_ably/metrics/abstract_intra_model_metric.py similarity index 100% rename from probe_ably/core/metrics/abstract_intra_model_metric.py rename to probe_ably/metrics/abstract_intra_model_metric.py diff --git a/probe_ably/core/metrics/accuracy.py b/probe_ably/metrics/accuracy.py similarity index 91% rename from probe_ably/core/metrics/accuracy.py rename to probe_ably/metrics/accuracy.py index 2bdcdde..7840534 100644 --- a/probe_ably/core/metrics/accuracy.py +++ b/probe_ably/metrics/accuracy.py @@ -1,5 +1,5 @@ import numpy as np -from probe_ably.core.metrics import AbstractIntraModelMetric +from probe_ably.metrics import AbstractIntraModelMetric from sklearn.metrics import accuracy_score diff --git a/probe_ably/core/tasks/metric_task/process_metric_task.py b/probe_ably/metrics/process_metric_task.py similarity index 78% rename from probe_ably/core/tasks/metric_task/process_metric_task.py rename to probe_ably/metrics/process_metric_task.py index cd04bbc..5110e0b 100644 --- a/probe_ably/core/tasks/metric_task/process_metric_task.py +++ b/probe_ably/metrics/process_metric_task.py @@ -4,14 +4,13 @@ import numpy as np from loguru import logger -from overrides import overrides from prefect import Task -from probe_ably.core.metrics.abstract_inter_model_metric import AbstractInterModelMetric -from probe_ably.core.metrics.abstract_intra_model_metric import AbstractIntraModelMetric +from probe_ably.metrics.abstract_inter_model_metric import AbstractInterModelMetric +from probe_ably.metrics.abstract_intra_model_metric import AbstractIntraModelMetric from tqdm import tqdm -class ProcessMetricTask(Task): +class ProcessMetricTask(): def run( self, train_results: Dict[str, Dict], probing_configuration: Dict[str, Dict] ): @@ -29,16 +28,16 @@ def run( intra_eval_metric, intra_eval_name = intra_metric, intra_metric.metric_name() processed_aux_tasks = [] - probing_models, models, x_axis_data, y_axis_data = set(), set(), set(), set() + probing_models, reps, x_axis_data, y_axis_data = set(), set(), set(), set() for task_id, task_data in tqdm(train_results.items(), "Processing task data"): processed_task = {} processed_task["id"] = task_id processed_task["name"] = task_data["task_name"] processed_task["probings"] = defaultdict(lambda: []) - for _, model_data in task_data["models"].items(): - model_name = model_data["model_name"] - for probing_model, probing_data in model_data.items(): - if probing_model != "model_name": + for _, rep_data in task_data["representations"].items(): + rep_name = rep_data["representation_name"] + for probing_model, probing_data in rep_data.items(): + if probing_model != "representation_name": for p_data in probing_data.values(): for cplx_key, cplx_data in p_data["complexity"].items(): model_gold, model_preds = ( @@ -62,7 +61,7 @@ def run( processed_task["probings"][ ( probing_model, - model_name, + rep_name, cplx_key, inter_eval_name, ) @@ -74,7 +73,7 @@ def run( ) probing_models.add(probing_model) - models.add(model_name) + reps.add(rep_name) x_axis_data.add(cplx_key) y_axis_data.add(inter_eval_name) y_axis_data.add(intra_eval_name) @@ -82,7 +81,7 @@ def run( processed_task["probings"][ ( probing_model, - model_name, + rep_name, cplx_key, intra_eval_name, ) @@ -102,28 +101,28 @@ def run( visual_data_task["probings"] = [] for probing_model in probing_models: probing_data = {} - probing_data["model_name"] = probing_model + probing_data["representation_name"] = probing_model probing_data["probing_results"] = [] for x_axis in x_axis_data: for y_axis in y_axis_data: - model_data = {} - for model_name in models: + rep_data = {} + for rep_name in reps: if ( len( processed_task["probings"][ - (probing_model, model_name, x_axis, y_axis) + (probing_model, rep_name, x_axis, y_axis) ] ) > 0 ): m_data = { - "id": model_name, + "id": rep_name, "color": f"hsl({random.randint(1,360)}, {int(random.randint(1,9))*10}%, {int(random.randint(1,9))*10}%)", } m_data["data"] = [] x, y = [], [] for point in processed_task["probings"][ - (probing_model, model_name, x_axis, y_axis) + (probing_model, rep_name, x_axis, y_axis) ]: x.append(point["x"]) y.append(point["y"]) @@ -133,13 +132,13 @@ def run( {"x": x[index], "y": y[index]} ) - if "chart_data" not in model_data: - model_data["x_axis"] = x_axis - model_data["y_axis"] = y_axis - model_data["chart_data"] = [] - model_data["chart_data"].append(m_data) - if len(model_data) > 0: - probing_data["probing_results"].append(model_data) + if "chart_data" not in rep_data: + rep_data["x_axis"] = x_axis + rep_data["y_axis"] = y_axis + rep_data["chart_data"] = [] + rep_data["chart_data"].append(m_data) + if len(rep_data) > 0: + probing_data["probing_results"].append(rep_data) visual_data_task["probings"].append(probing_data) visual_data_tasks.append(visual_data_task) - return visual_data_tasks + return visual_data_tasks \ No newline at end of file diff --git a/probe_ably/core/metrics/selectivity.py b/probe_ably/metrics/selectivity.py similarity index 94% rename from probe_ably/core/metrics/selectivity.py rename to probe_ably/metrics/selectivity.py index be373c6..0fe5074 100644 --- a/probe_ably/core/metrics/selectivity.py +++ b/probe_ably/metrics/selectivity.py @@ -1,5 +1,5 @@ import numpy as np -from probe_ably.core.metrics import AbstractInterModelMetric +from probe_ably.metrics import AbstractInterModelMetric from sklearn.metrics import accuracy_score diff --git a/probe_ably/core/models/__init__.py b/probe_ably/models/__init__.py similarity index 100% rename from probe_ably/core/models/__init__.py rename to probe_ably/models/__init__.py diff --git a/probe_ably/core/models/abstract_model.py b/probe_ably/models/abstract_model.py similarity index 100% rename from probe_ably/core/models/abstract_model.py rename to probe_ably/models/abstract_model.py diff --git a/probe_ably/core/models/linear.py b/probe_ably/models/linear.py similarity index 98% rename from probe_ably/core/models/linear.py rename to probe_ably/models/linear.py index b300a2f..12d165d 100644 --- a/probe_ably/core/models/linear.py +++ b/probe_ably/models/linear.py @@ -3,7 +3,7 @@ import math import numpy as np import torch -from probe_ably.core.models import AbstractModel +from probe_ably.models import AbstractModel from torch import Tensor, nn diff --git a/probe_ably/core/models/mlp.py b/probe_ably/models/mlp.py similarity index 98% rename from probe_ably/core/models/mlp.py rename to probe_ably/models/mlp.py index 184d04a..18b9e1a 100644 --- a/probe_ably/core/models/mlp.py +++ b/probe_ably/models/mlp.py @@ -1,10 +1,9 @@ ## ADAPTED FROM https://github.com/rycolab/pareto-probing/blob/master/src/h02_learn/model/mlp.py - from typing import Dict import numpy as np import torch -from probe_ably.core.models import AbstractModel +from probe_ably.models import AbstractModel from torch import Tensor, nn diff --git a/probe_ably/core/models/model_params.py b/probe_ably/models/model_params.py similarity index 80% rename from probe_ably/core/models/model_params.py rename to probe_ably/models/model_params.py index b579553..53416a9 100644 --- a/probe_ably/core/models/model_params.py +++ b/probe_ably/models/model_params.py @@ -3,7 +3,7 @@ class ModelParams(): def __init__(self)->Dict: self.default_params = { - "probe_ably.core.models.linear.LinearModel": { + "probe_ably.models.linear.LinearModel": { "params": [ { "name": "dropout", @@ -13,17 +13,17 @@ def __init__(self)->Dict: { "name": "alpha", "type": "function", - "function_location": "probe_ably.core.utils.param_functions.nuclear_norm_alpha_generation", + "function_location": "probe_ably.utils.param_functions.nuclear_norm_alpha_generation", "options": [-10.0, 3] }] }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { "params": [ { "name": "hidden_size", "type": "function", "step": 0.01, - "function_location": "probe_ably.core.utils.param_functions.hidden_size_generation", + "function_location": "probe_ably.utils.param_functions.hidden_size_generation", "options": [ 2, 5 diff --git a/probe_ably/probing/__init__.py b/probe_ably/probing/__init__.py new file mode 100644 index 0000000..f8f983c --- /dev/null +++ b/probe_ably/probing/__init__.py @@ -0,0 +1,3 @@ +from .prepare_data import prep_data_from_parsed_json +from .train_probing_task import TrainProbingTask +# from .experiment_setup import ProbingExperiment \ No newline at end of file diff --git a/probe_ably/core/tasks/probing/calculate_metric_task.py b/probe_ably/probing/calculate_metric_task.py similarity index 100% rename from probe_ably/core/tasks/probing/calculate_metric_task.py rename to probe_ably/probing/calculate_metric_task.py diff --git a/probe_ably/core/tasks/probing/deprobe_task.py b/probe_ably/probing/deprobe_task.py similarity index 100% rename from probe_ably/core/tasks/probing/deprobe_task.py rename to probe_ably/probing/deprobe_task.py diff --git a/probe_ably/core/tasks/control_task/generate_control_task.py b/probe_ably/probing/generate_control_task.py similarity index 90% rename from probe_ably/core/tasks/control_task/generate_control_task.py rename to probe_ably/probing/generate_control_task.py index 7618ba3..a285d8a 100644 --- a/probe_ably/core/tasks/control_task/generate_control_task.py +++ b/probe_ably/probing/generate_control_task.py @@ -1,11 +1,10 @@ -from overrides import overrides from prefect import Task import numpy as np import pandas as pd # TODO properly comment this -class GenerateControlTask(Task): +class GenerateControlTask(): @staticmethod def get_unique_labels(labels): return np.unique(labels) diff --git a/probe_ably/probing/prepare_data.py b/probe_ably/probing/prepare_data.py new file mode 100644 index 0000000..8065aab --- /dev/null +++ b/probe_ably/probing/prepare_data.py @@ -0,0 +1,145 @@ +from prefect import Task +from loguru import logger +import sklearn +from sklearn.model_selection import train_test_split +from typing import Dict +import torch +from torch.utils.data import Dataset +import numpy as np +from probe_ably.utils.input_types import SplitProbingDataset, ProbingRepresentation, ProbingInput, ProbingConfig, ProbingTask + +def prepare_probing_dataset(vectors, labels) -> Dataset: + dataset = dict() + for i in range(0, len(vectors)): + dataset[i] = {"representation": vectors[i], "label": labels[i]} + return TorchDataset(dataset) + +def train_val_test_split(X, y, train_size, val_size, test_size, seed=42): + X_train_val, X_test, y_train_val, y_test = train_test_split( + X, y, test_size=test_size, random_state=seed + ) + relative_train_size = train_size / (val_size + train_size) + + X_train, X_val, y_train, y_val = train_test_split( + X_train_val, + y_train_val, + test_size=1 - relative_train_size, + random_state=seed, + ) + + return X_train, X_val, X_test, y_train, y_val, y_test + +def unpack_rep_data(rep_content: dict, probing_config: ProbingConfig) -> ProbingRepresentation: + ( + model_vectors_train, + model_vectors_val, + model_vectors_test, + model_labels_train, + model_labels_val, + model_labels_test, + ) = train_val_test_split( X=rep_content["representation_vectors"], y=rep_content["representation_labels"], + train_size=probing_config["train_size"], + val_size=probing_config["dev_size"], + test_size=probing_config["test_size"], + ) + ( + control_vectors_train, + control_vectors_val, + control_vectors_test, + control_labels_train, + control_labels_val, + control_labels_test, + ) = train_val_test_split( + X=rep_content["representation_vectors"], + y=rep_content["control_labels"], + train_size=probing_config["train_size"], + val_size=probing_config["dev_size"], + test_size=probing_config["test_size"], + ) + + representation_dataset: SplitProbingDataset = { + "train": prepare_probing_dataset( + model_vectors_train, model_labels_train + ), + "dev": prepare_probing_dataset(model_vectors_val, model_labels_val), + "test": prepare_probing_dataset(model_vectors_test, model_labels_test), + } + control_dataset: SplitProbingDataset = { + "train": prepare_probing_dataset( + control_vectors_train, control_labels_train + ), + "dev": prepare_probing_dataset( + control_vectors_val, control_labels_val + ), + "test": prepare_probing_dataset( + control_vectors_test, control_labels_test + ), + } + + representation: ProbingRepresentation = { + "representation_name": rep_content["representation_name"], + "representation_size": rep_content["representation_size"], + "number_of_classes": rep_content["number_of_classes"], + "default_control": rep_content["default_control"], + "representation": representation_dataset, + "control":control_dataset, + } + return representation + +def prep_data_from_parsed_json(parsed_tasks_data: Dict, probing_config: ProbingConfig, return_trained_model: bool = False) -> Dict: + """Reads the task_data and experiment_setup, splits into train/dev/test and + creates a TorchDataset for each. + + # :param tasks_data: Tasks info obtained from user input + # :type tasks_data: Dict + # :param experiment_setup: Experiment setup obtained from default file or user input + # :type experiment_setup: Dict + # :return: Dictonary of processed data in the format: + # { task_id: + # {'task_name': str, + # 'models': + # {model_id: + # {"model_name": str, + # "model": {"train": numpy.ndarray, "dev": numpy.ndarray, "test": numpy.ndarray}, + # "control": {"train": numpy.ndarray, "dev": numpy.ndarray, "test": numpy.ndarray}, + # "representation_size": int, + # "number_of_classes": int, + # "default_control": boolean (False if user inputs control task) + # } + # } + # } + # } + # :rtype: Dict + """ + + logger.debug("Prepare the data for probing.") + + tasks = list() + for id_task, task_content in parsed_tasks_data.items(): + probing_task : ProbingTask = { + "task_name": task_content["task_name"], + "representations": [unpack_rep_data(rep_content, probing_config) for rep_id, rep_content in task_content["representations"].items()] + } + tasks.append(probing_task) + + return tasks + +class TorchDataset(Dataset): + def __init__(self, dataset): + self.dataset = list(dataset.values()) + self.labels = np.array([data["label"] for data in self.dataset]) + self.keys = list(dataset.keys()) + + def __getitem__(self, index): + instance = self.dataset[index] + return ( + torch.FloatTensor(instance["representation"]), + instance["label"], + index, + ) + + def get_id(self, index): + return self.keys[index] + + def __len__(self): + return len(self.dataset) \ No newline at end of file diff --git a/probe_ably/core/tasks/probing/train_probing_task.py b/probe_ably/probing/train_probing_task.py similarity index 74% rename from probe_ably/core/tasks/probing/train_probing_task.py rename to probe_ably/probing/train_probing_task.py index d9e6a54..e60cac1 100644 --- a/probe_ably/core/tasks/probing/train_probing_task.py +++ b/probe_ably/probing/train_probing_task.py @@ -1,21 +1,17 @@ import random -from typing import Dict +from typing import Dict, List from copy import copy, deepcopy import numpy as np import torch from loguru import logger -from overrides import overrides from prefect import Task -from probe_ably.core.metrics import AbstractIntraModelMetric -from probe_ably.core.models import LinearModel -from probe_ably.core.utils import GridModelFactory -from sklearn.metrics import accuracy_score +from probe_ably.metrics import AbstractIntraModelMetric +from probe_ably.utils import GridModelFactory, ProbingTask, ProbingConfig, ProbingInput from torch.utils.data import DataLoader -from tqdm import tqdm, trange +from tqdm import tqdm from colorama import Fore - -class TrainProbingTask(Task): +class TrainProbingTask(): def __init__(self, **kwargs): self.cuda = kwargs.pop("cuda", True) self.logging_steps = kwargs.get("logging_steps", 5) @@ -39,6 +35,7 @@ def start_training_process( n_gpu, num_epochs, eval_fn, + return_trained_model=False ): outputs = {} # logger.info("Running train mode") @@ -79,9 +76,18 @@ def start_training_process( eval_fn, ) - return preds_test + if return_trained_model: + return { + "preds_test": preds_test, + "trained_model": best_model + } - def run(self, tasks: Dict, probing_setup: Dict) -> Dict: + else: + return { + "preds_test": preds_test + } + + def run(self, tasks: List[ProbingTask], probing_setup: ProbingConfig, thread=None, return_trained_model=False) -> Dict: """Runs the Probing models :param tasks: Data content of the models for probing. @@ -91,7 +97,7 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: :return: Dictionary containing the following values: {int(task id) : "models": { - int(model id) : { + int (model id) : { str (name of probing model) : { int (run number) : { "complexity": { @@ -122,7 +128,7 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: n_gpu = torch.cuda.device_count() self.set_seed(n_gpu) - self.logger.info(f"GPUs used {n_gpu}") + logger.info(f"GPUs used {n_gpu}") output_results = dict() intra_metric_class = AbstractIntraModelMetric.subclasses[ @@ -131,42 +137,45 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: intra_metric_object = intra_metric_class() task_loop_bar = tqdm( - tasks.items(), + tasks, desc=f"Task progress", bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.GREEN, Fore.RESET), ) - for id_task, content_tasks in task_loop_bar: - + if thread: + thread.task_loop_bar = task_loop_bar + + for id_task, content_tasks in enumerate(task_loop_bar): task_loop_bar.set_description( f"Task: {content_tasks['task_name']} progress" ) output_results[id_task] = dict() - output_results[id_task]["models"] = dict() + output_results[id_task]["representations"] = dict() output_results[id_task]["task_name"] = content_tasks["task_name"] - model_loop_bar = tqdm( - content_tasks["models"].items(), + + reps_loop_bar = tqdm( + content_tasks["representations"], desc=f"Model progress", bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.BLUE, Fore.RESET), leave=False, ) - for id_model, model_content in model_loop_bar: - model_loop_bar.set_description( - f"Model: {model_content['model_name']} progress" + if thread: + thread.reps_loop_bar = reps_loop_bar + for id_model, rep_content in enumerate(reps_loop_bar): + reps_loop_bar.set_description( + f"Model: {rep_content['representation_name']} progress" ) - output_results[id_task]["models"][id_model] = dict() - output_results[id_task]["models"][id_model][ - "model_name" - ] = model_content["model_name"] + + output_results[id_task]["representations"][id_model] = dict() + output_results[id_task]["representations"][id_model][ + "representation_name" + ] = rep_content["representation_name"] model_params = { - "representation_size": model_content["representation_size"], - "n_classes": model_content["number_of_classes"], + "representation_size": rep_content["representation_size"], + "n_classes": rep_content["number_of_classes"], } - print(model_params) - for id_prob_model, probe_content in probing_setup[ - "probing_models" - ].items(): + for id_prob_model, probe_content in enumerate(probing_setup["probing_models"]): probe_model_name = probe_content["probing_model_name"] probing_models = GridModelFactory.create_models( @@ -175,7 +184,7 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: model_params, ) - output_results[id_task]["models"][id_model][ + output_results[id_task]["representations"][id_model][ probe_model_name ] = dict() run_number = 0 @@ -188,45 +197,54 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.YELLOW, Fore.RESET), ) + if thread: + thread.probes_loop_bar = probes_loop_bar for probe in probes_loop_bar: probes_loop_bar.set_description( f"Probe: {probe_model_name} progress" ) probe_for_model = deepcopy(probe) probe_for_control = deepcopy(probe) - preds_model = self.start_training_process( - train=model_content["model"]["train"], - test=model_content["model"]["test"], - dev=model_content["model"]["dev"], + train_output = self.start_training_process( + train=rep_content["representation"]["train"], + test=rep_content["representation"]["test"], + dev=rep_content["representation"]["dev"], train_batch_size=train_batch_size, model=probe_for_model, device=device, num_epochs=probe_content["epochs"], n_gpu=n_gpu, eval_fn=intra_metric_object.calculate_metrics, + return_trained_model=return_trained_model ) - if model_content["default_control"]: - test_control_set = model_content["control"]["train"] + preds_model = train_output["preds_test"] + if return_trained_model: + trained_probe_model = train_output["trained_model"] + + if rep_content["default_control"]: + test_control_set = rep_content["control"]["train"] else: - test_control_set = model_content["control"]["test"] + test_control_set = rep_content["control"]["test"] preds_control = self.start_training_process( - train=model_content["control"]["train"], + train=rep_content["control"]["train"], test=test_control_set, - dev=model_content["control"]["dev"], + dev=rep_content["control"]["dev"], train_batch_size=train_batch_size, model=probe_for_control, device=device, num_epochs=probe_content["epochs"], n_gpu=n_gpu, eval_fn=intra_metric_object.calculate_metrics, - ) - output_results[id_task]["models"][id_model][probe_model_name][ + return_trained_model=False + )["preds_test"] + + output_results[id_task]["representations"][id_model][probe_model_name][ run_number ] = { "complexity": probe_for_model.get_complexity(), "model": { - "labels": model_content["model"]["test"].labels, + "labels": rep_content["representation"]["test"].labels, "preds": preds_model, }, "control": { @@ -234,6 +252,11 @@ def run(self, tasks: Dict, probing_setup: Dict) -> Dict: "preds": preds_control, }, } + #TODO: adjust this for returned model + # if return_trained_model: + # output_results[id_task]["models"][id_model][probe_model_name][ + # run_number + # ][model]["trained_model"] = trained_probe_model run_number += 1 return output_results diff --git a/probe_ably/service/.storybook/main.js b/probe_ably/service/.storybook/main.js deleted file mode 100644 index cd74759..0000000 --- a/probe_ably/service/.storybook/main.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = { - "stories": [ - "../src/**/*.stories.mdx", - "../src/**/*.stories.@(js|jsx|ts|tsx)" - ], - "addons": [ - "@storybook/addon-links", - "@storybook/addon-essentials", - "@storybook/preset-create-react-app" - ] -} \ No newline at end of file diff --git a/probe_ably/service/.storybook/preview.js b/probe_ably/service/.storybook/preview.js deleted file mode 100644 index 5d00c02..0000000 --- a/probe_ably/service/.storybook/preview.js +++ /dev/null @@ -1,4 +0,0 @@ - -export const parameters = { - actions: { argTypesRegex: "^on[A-Z].*" }, -} \ No newline at end of file diff --git a/probe_ably/service/server/web_server.py b/probe_ably/service/server/web_server.py deleted file mode 100644 index 7dbbddb..0000000 --- a/probe_ably/service/server/web_server.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from pathlib import Path - -from flask import Flask, jsonify, render_template, send_from_directory -from flask_cors import CORS, cross_origin - -this_filepath = Path(os.path.abspath(__file__)) -this_dirpath = this_filepath.parent.parent -import json - - -class WebServer: - def __init__(self, processed_data): - self.app = Flask( - __name__, - static_folder=str(this_dirpath.joinpath("build")), - static_url_path="", - ) - self.cors = CORS(self.app) - self.app.config["CORS_HEADERS"] = "Content-Type" - self.processed_data = processed_data - - self.app.add_url_rule("/", view_func=self.serve) - self.app.add_url_rule("/sample", view_func=self.serve_data) - - def get_path(self): - return this_dirpath - - def serve(self): - print(this_dirpath) - return send_from_directory(self.app.static_folder, "index.html") - - def serve_data(self): - # with open(f"{self.get_path()}/data.json", "r") as f: - # data = json.load(f) - return jsonify({"aux_tasks": self.processed_data}) - - def start(self, port: int = 8031): - self.app.run(port=port, host="0.0.0.0") diff --git a/probe_ably/service/src/pages/HomePage.js b/probe_ably/service/src/pages/HomePage.js deleted file mode 100644 index 6be7a93..0000000 --- a/probe_ably/service/src/pages/HomePage.js +++ /dev/null @@ -1,55 +0,0 @@ -import { - Col, - Container, - Navbar, - Row, - Spinner, -} from "@themesberg/react-bootstrap"; -import React, { useEffect, useState } from "react"; -import Dashboard from "./dashboard/DashboardOverview"; - -export default () => { - const [auxTasks, setAuxTasks] = useState(null); - - useEffect(() => { - fetch("/sample") - .then((res) => res.json()) - .then((data) => { - setAuxTasks(data.aux_tasks); - }); - }, []); - - return ( -
- - - ProBe-Ably - - -
- {auxTasks == null ? ( - - - - - Loading... - - - - - ) : ( - - )} -
-
- ); -}; diff --git a/probe_ably/service/src/pages/forms/ConfigurationForm.js b/probe_ably/service/src/pages/forms/ConfigurationForm.js deleted file mode 100644 index ead0f18..0000000 --- a/probe_ably/service/src/pages/forms/ConfigurationForm.js +++ /dev/null @@ -1,90 +0,0 @@ -import { Button, Card, Col, Form, Row } from "@themesberg/react-bootstrap"; -import React, { useState } from "react"; - -export default () => { - const [tasks, setTasks] = useState([{ task_name: "task_name" }]); - - return ( -
- - -
- - Task Name - - - -
- - - - {tasks.map((task_data, i) => ( - - - -
-
{task_data.task_name}
-
- - - - - -
- - Model Name - - - - - Representations - - - - - Representations Labels - - - - Control Task Class - - - -
- - - - ))} - - - - - - - -
- ); -}; diff --git a/probe_ably/service/src/stories/ConfigurationForm.stories.js b/probe_ably/service/src/stories/ConfigurationForm.stories.js deleted file mode 100644 index 7dad7b8..0000000 --- a/probe_ably/service/src/stories/ConfigurationForm.stories.js +++ /dev/null @@ -1,17 +0,0 @@ -// vendor styles -import "@fortawesome/fontawesome-free/css/all.css"; -import React from "react"; -import "react-datetime/css/react-datetime.css"; -import ConfigurationForm from "../pages/forms/ConfigurationForm"; -import "../scss/volt.scss"; - -export default { - title: "ConfigurationForm", - component: ConfigurationForm, -}; - -const Template = (args) => ; - -export const SimpleForm = Template.bind({}); - -SimpleForm.args = {}; diff --git a/probe_ably/service/src/stories/Dahsboard.stories.js b/probe_ably/service/src/stories/Dahsboard.stories.js deleted file mode 100644 index 8c0b109..0000000 --- a/probe_ably/service/src/stories/Dahsboard.stories.js +++ /dev/null @@ -1,308 +0,0 @@ -// vendor styles -import "@fortawesome/fontawesome-free/css/all.css"; -import React from "react"; -import "react-datetime/css/react-datetime.css"; -import Dashboard from "../pages/dashboard/DashboardOverview"; -import "../scss/volt.scss"; - -export default { - title: "Dashboard", - component: Dashboard, -}; - -const Template = (args) => ; - -export const SingleTab = Template.bind({}); - -SingleTab.args = { - aux_tasks: [ - { - id: "1", - name: "Task name 1", - probings: [ - { - model_name: "MLP", - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - { - x: "boat", - y: 54, - }, - ], - }, - ], - }, - { - x_axis: "Number of Parameter", - y_axis: "Selectivity", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 150, - }, - { - x: "helicopter", - y: 200, - }, - { - x: "boat", - y: 54, - }, - { - x: "train", - y: 126, - }, - ], - }, - { - id: "china", - color: "hsl(12, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - { - x: "boat", - y: 54, - }, - { - x: "train", - y: 126, - }, - ], - }, - ], - }, - ], - }, - { - model_name: "Linear", - probing_types: [{ index: 0, name: "Accuracy" }], - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - ], - }, - ], - }, - ], - }, - ], - }, - ], -}; - -export const SingleTabValues = Template.bind({}); - -SingleTabValues.args = { - aux_tasks: [ - { - id: "1", - name: "Task name 1", - probings: [ - { - model_name: "MLP", - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: 100000000, - y: 175, - }, - { - x: 110000000, - y: 295, - }, - { - x: 120000000, - y: 54, - }, - ], - }, - ], - }, - ], - }, - ], - }, - ], -}; - -export const MultipleTask = Template.bind({}); - -MultipleTask.args = { - aux_tasks: [ - { - id: "1", - name: "Task name 1", - probings: [ - { - model_name: "MLP", - probing_types: [ - { index: 0, name: "Accuracy" }, - { index: 1, name: "Selectivity" }, - ], - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - { - x: "boat", - y: 54, - }, - ], - }, - ], - }, - { - x_axis: "Number of Parameter", - y_axis: "Selectivity", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - { - x: "boat", - y: 54, - }, - { - x: "train", - y: 126, - }, - ], - }, - ], - }, - ], - }, - { - model_name: "Linear", - probing_types: [{ index: 0, name: "Accuracy" }], - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - ], - }, - ], - }, - ], - }, - ], - }, - { - id: "2", - name: "Task name 2", - probings: [ - { - model_name: "MLP", - probing_types: [ - { index: 0, name: "Accuracy" }, - { index: 1, name: "Selectivity" }, - ], - probing_results: [ - { - x_axis: "Number of Parameter", - y_axis: "Accuracy", - chart_data: [ - { - id: "japan", - color: "hsl(190, 70%, 50%)", - data: [ - { - x: "plane", - y: 175, - }, - { - x: "helicopter", - y: 295, - }, - ], - }, - ], - }, - ], - }, - ], - }, - ], -}; diff --git a/probe_ably/utils/__init__.py b/probe_ably/utils/__init__.py new file mode 100644 index 0000000..758f0c6 --- /dev/null +++ b/probe_ably/utils/__init__.py @@ -0,0 +1,4 @@ +from .grid_model_factory import GridModelFactory +from .read_input_task import ReadInputTask +from .input_types import ProbingConfig, ProbingTask, ProbingInput, SplitProbingDataset, ProbingRepresentation +# from .visualization_task import VisualiaztionTask \ No newline at end of file diff --git a/probe_ably/core/utils/grid_model_factory.py b/probe_ably/utils/grid_model_factory.py similarity index 95% rename from probe_ably/core/utils/grid_model_factory.py rename to probe_ably/utils/grid_model_factory.py index 9514f9a..e3231be 100644 --- a/probe_ably/core/utils/grid_model_factory.py +++ b/probe_ably/utils/grid_model_factory.py @@ -1,12 +1,8 @@ -import glob import importlib -import json -import os import random from typing import Any, Dict, List -import sys import numpy as np -from probe_ably.core.models import AbstractModel, ModelParams +from probe_ably.models import AbstractModel, ModelParams class GridModelFactory: @@ -17,7 +13,7 @@ def create_models( """Creates a list of models provided from static param ranges Args: - model_class (str): Probing model class. For example: :code:`probe_ably.core.models.linear` + model_class (str): Probing model class. For example: :code:`probe_ably.models.linear` num_models (int, optional): Number of Models to create. Defaults to 50. param_args (Dict, optional): Paramter ranges to choose from the model to create. Defaults to {}. diff --git a/probe_ably/utils/input_types.py b/probe_ably/utils/input_types.py new file mode 100644 index 0000000..aac3490 --- /dev/null +++ b/probe_ably/utils/input_types.py @@ -0,0 +1,33 @@ +from typing import Type, TypedDict, List, Union, Dict +from torch.utils.data import Dataset + +class ProbingConfig(TypedDict): + probing_models: List[Dict] + inter_metric: Union[str, List[str]] + intra_metric: Union[str, List[str]] + own_splits: bool + train_size: float + dev_size: float + test_size: float + +class SplitProbingDataset(TypedDict): + train: Dataset + dev: Dataset + test: Dataset + +class ProbingRepresentation(TypedDict): + representation_name: str + representation: SplitProbingDataset + control: SplitProbingDataset + representation_size: int + number_of_classes: int + default_control: bool + + +class ProbingTask(TypedDict): + task_name: str + representations: List[ProbingRepresentation] + +class ProbingInput(TypedDict): + tasks: List[ProbingTask] + probing_config: ProbingConfig \ No newline at end of file diff --git a/probe_ably/core/utils/param_functions.py b/probe_ably/utils/param_functions.py similarity index 100% rename from probe_ably/core/utils/param_functions.py rename to probe_ably/utils/param_functions.py diff --git a/probe_ably/utils/read_input_task.py b/probe_ably/utils/read_input_task.py new file mode 100644 index 0000000..9fe3cf6 --- /dev/null +++ b/probe_ably/utils/read_input_task.py @@ -0,0 +1,298 @@ +from posixpath import split +from prefect import Task +from typing import Dict +from loguru import logger +import os.path +import json +import sys +import jsonschema +import os.path +import pandas as pd +import numpy as np +from probe_ably.probing.generate_control_task import GenerateControlTask +from probe_ably.models import AbstractModel +from probe_ably.metrics import AbstractInterModelMetric, AbstractIntraModelMetric +from probe_ably.constants import DEFAULT_PROBING_SETUP, SCHEMA_TEMPLATE_FILE +from .input_types import ProbingConfig, ProbingInput + +class ModelRepresentationFileNotFound(Exception): + def __init__(self, model_location): + self.model_location = model_location + +class ControlSizeMissmatch(Exception): + def __init__(self, task_name, model_name): + self.task_name = task_name + self.model_name = model_name + + +class InputClassNotFound(Exception): + def __init__(self, type_of_class, class_name): + self.type_of_class = type_of_class + self.class_name = class_name + + +def load_input(input_file_location): + """Function that parses the input configuration file provided by the user. + + :param input_file_location: Input json file containing the representations for probing and probing setup. + The file should follow the template in probe_ably/config/json_schema/input_file.json + :type input_file_location: str + :return: Dictionary of the parsed input, in the following format: + { + "tasks": { + int (task_id): + { + "task_name": str, + "models": { + int (model_id): { + "model_name": str, + "model_vectors": array (representations being probed), + "model_labels": array (labels for the auxiliary task), + "control_labels": array (labels for the control task), + "representation_size": int (size of each representation), + "number_of_classes": int (number of unique labels), + "default_control": boolean (False if user inputs control task) + } + } + } + } + + "probing_config": { + "inter_metric": string (class for metric to compare model and control task), + "intra_metric": string (class for metric that will be used for measuring + the best model), + "dev_size": int, + "train_size": int, + "test_size": int, + "probing_models": { + int (probing_models_id): { + "probing_model_name": string (class for the probing model), + "batch_size": int, + "epochs": int, + "number_of_models": int (number of probe models for generation) + } + } + } + } + :rtype: Dict + """ + + with open(input_file_location, "r") as f: + input_data = json.load(f) + logger.debug(f"Opening file located in {input_file_location}") + + return input_data + +class ReadInputTask(): + async def run(self, input_file) -> Dict: + + generate_control_task = GenerateControlTask() + logger.debug("Reading input file.") + + try: + input_data = load_input(input_file) + except TypeError: + # UploadFile handling + input_data = input_file.read() #await removed, maybe needed for app? + try: + input_data = json.loads(input_data) + except TypeError: + input_data = json.loads(await input_data) + + except FileNotFoundError: + sys.exit(f"Input file not found: {input_file}") + + except json.JSONDecodeError as e: + sys.exit( + f"Input file is not a properly foramtted json file: {input_file}" + ) + + print('Here it is Julia!', input_data) + try: + with open(SCHEMA_TEMPLATE_FILE, "r") as f: + input_template = json.load(f) + jsonschema.validate(instance=input_data, schema=input_template) + + output_dict = dict() + current_task_id = 0 + try: + task_list = input_data["tasks"] + + ## Getting input info + for task_content in task_list: + output_dict[current_task_id] = dict() + output_dict[current_task_id]["task_name"] = task_content["task_name"] + output_dict[current_task_id]["representations"] = dict() + models_list = task_content["representations"] + + current_model_id = 0 + for model_content in models_list: + if not os.path.isfile(model_content["file_location"]): + raise ModelRepresentationFileNotFound( + model_content["file_location"] + ) + + output_dict[current_task_id]["representations"][ + current_model_id + ] = self.parse_model_info( + model_content, task_content["task_name"], generate_control_task + ) + + current_model_id += 1 + current_task_id += 1 + + except KeyError: + logger.info("No task files referenced in json config, awaiting configuration of probing data.") + pass + + ## Getting probe info + probing_config = self.parse_probing_config(input_data) + + + except jsonschema.ValidationError as e: + logger.error(e) + sys.exit( + f"Input file ({input_file}) does not follow correct template. Please refer to README file." + ) + except ModelRepresentationFileNotFound as e: + sys.exit(f"Representation file ({e.model_location}) not found.") + + except ControlSizeMissmatch as e: + sys.exit( + f"Control task for task {e.task_name} and model {e.model_name} does not match the number of labels of the aux task." + ) + except InputClassNotFound as e: + sys.exit( + f"Error in probing setup: Element {e.type_of_class} with content {e.class_name} not found." + ) + except ValueError as e: + sys.exit(e) + + parsed_input = {"tasks": output_dict, "probing_config": probing_config} + + return parsed_input + + @staticmethod + def parse_model_info(model_content, task_name, generate_control_task): + model_representation = pd.read_csv( + model_content["file_location"], sep="\t", header=None + ) + model_labels = model_representation.iloc[:, -1].to_numpy() + + model_representation.drop( + model_representation.columns[-1], axis=1, inplace=True + ) + model_representation = model_representation.to_numpy() + + if "control_location" in model_content: + default_control = False + if not os.path.isfile(model_content["control_location"]): + raise ModelRepresentationFileNotFound(model_content["control_location"]) + + control_labels = np.loadtxt( + fname=model_content["control_location"], + delimiter="\t", + dtype=int, + ) + + if len(control_labels) != len(model_labels): + raise ControlSizeMissmatch( + task_name, + model_content["model_name"], + ) + else: + default_control = True + logger.info( + f"No control labels provided for task {task_name} and model {model_content['representation_name']}, generating random control task." + ) + control_labels = generate_control_task.run( + model_representation, model_labels + ) + total_number_of_classes = ( + np.amax(np.concatenate((model_labels, control_labels))) + 1 + ) + + #TODO: match variable name model_labels to representation_labels, to match the key here + return { + "representation_name": model_content["representation_name"], + "representation_vectors": model_representation, + "representation_labels": model_labels, + "control_labels": control_labels, + "representation_size": model_representation.shape[1], + "number_of_classes": total_number_of_classes, + "default_control": default_control, + } + + @staticmethod + def parse_probing_config(input_data) -> ProbingConfig: + if "probing_config" not in input_data: + with open(DEFAULT_PROBING_SETUP, "r") as f: + probing_config = json.load(f) + + logger.info( + "No experiment setup provided, using the default values. To see the default values, run 'experiment.probing_config'." + ) + else: + available_inter_metrics = AbstractInterModelMetric.subclasses + available_intra_metrics = AbstractIntraModelMetric.subclasses + available_probing_models = AbstractModel.subclasses + + if ( + input_data["probing_config"]["inter_metric"] + not in available_inter_metrics + ): + raise InputClassNotFound( + "inter_metric", input_data["probing_config"]["inter_metric"] + ) + + if ( + input_data["probing_config"]["intra_metric"] + not in available_intra_metrics + ): + raise InputClassNotFound( + "intra_metric", input_data["probing_config"]["intra_metric"] + ) + + try: + train_size = input_data["probing_config"]["train_size"] + dev_size = input_data["probing_config"]["dev_size"] + test_size = input_data["probing_config"]["test_size"] + split_distribution = ( + train_size + + test_size + + dev_size + ) + + if split_distribution != 1: + raise ValueError( + f"Train + Test + Dev size should be equals to 1, got {split_distribution}" + ) + probing_config : ProbingConfig = { + "inter_metric": input_data["probing_config"]["inter_metric"], + "intra_metric": input_data["probing_config"]["intra_metric"], + "train_size": input_data["probing_config"]["train_size"], + "dev_size": input_data["probing_config"]["dev_size"], + "test_size": input_data["probing_config"]["test_size"], + "probing_models": list(), + } + + except KeyError: + probing_config : ProbingConfig = { + "inter_metric": input_data["probing_config"]["inter_metric"], + "intra_metric": input_data["probing_config"]["intra_metric"], + "probing_models": list(), + "own_splits": True + } + + + for probe_model in input_data["probing_config"]["probing_models"]: + if probe_model["probing_model_name"] not in available_probing_models: + raise InputClassNotFound( + "probing_model_name", probe_model["probing_model_name"] + ) + + probing_config["probing_models"].append(probe_model) + + logger.info(f"Using the experiment setup provided in the input file with values: \n {probing_config}") + + return probing_config \ No newline at end of file diff --git a/probe_ably/utils/visualization_task.py b/probe_ably/utils/visualization_task.py new file mode 100644 index 0000000..ce8a272 --- /dev/null +++ b/probe_ably/utils/visualization_task.py @@ -0,0 +1,26 @@ +# import json +# import webbrowser +from typing import Dict + +from loguru import logger +from prefect import Task +from probe_ably.app.__main__ import app + + +class VisualiaztionTask(Task): + def run(self, processed_data: Dict): + pass + # logger.info("Launching server for visualization") + # web_server = WebServer(processed_data) + # # with open(f"{web_server.get_path()}/data.json", "w") as f: + # # json.dump(processed_data, f) + # web_server.start() + + # ip_address = "http://127.0.0.1:8031/" + + # try: + # webbrowser.get("google-chrome").open(ip_address) + # except: + # logger.info( + # f"Tried to launch Google Chrom and Failed. Visit: {ip_address} to view the visualization" + # ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d35727..1db1ee0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ sphinx sphinx-autodoc-typehints sphinx_rtd_theme recommonmark +sphincemoji diff --git a/requirements.txt b/requirements.txt index d4dbcbc..d06531d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,15 @@ tqdm -dynaconf[all] prefect loguru ujson -sklearn +scikit-learn pandas numpy click torch colorama jsonschema -flask -flask-cors +aiofiles +fastapi +python-multipart +asyncio \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index b6e4002..44e1cce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,6 @@ cover-package = aigraphsuite [tool:pytest] addopts = --verbose -log_print = 1 log_cli = 1 [build_sphinx] diff --git a/tests/core/__init__.py b/tests/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/tasks/__init__.py b/tests/core/tasks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/tasks/metric/__init__.py b/tests/core/tasks/metric/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/tasks/probing/__init__.py b/tests/core/tasks/probing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/tasks/probing/test_prepare_data_for_probing_task.py b/tests/core/tasks/probing/test_prepare_data_for_probing_task.py deleted file mode 100644 index 5bba430..0000000 --- a/tests/core/tasks/probing/test_prepare_data_for_probing_task.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest -from loguru import logger -from probe_ably.core.tasks.utils import ReadInputTask -from probe_ably.core.tasks.probing import PrepareDataForProbingTask - - -class PrepareProbingDataTest(unittest.TestCase): - def test_prepare_data_probing(self): - TEST_INPUT = ( - "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" - ) - read_input_task = ReadInputTask() - - output = read_input_task.run(TEST_INPUT) - - prepare_data_probing_task = PrepareDataForProbingTask() - - dataset = prepare_data_probing_task.run( - output["tasks"], output["probing_setup"] - ) - - total_size = ( - dataset[0]["models"][0]["model"]["train"].__len__() - + dataset[0]["models"][0]["model"]["dev"].__len__() - + dataset[0]["models"][0]["model"]["test"].__len__() - ) - - original_size = len(output["tasks"][0]["models"][0]["model_vectors"]) - - model_element = dataset[0]["models"][0]["model"]["train"].__getitem__(0)[0][0] - control_element = dataset[0]["models"][0]["control"]["train"].__getitem__(0)[0][ - 0 - ] - - self.assertEquals(model_element, control_element) - self.assertEquals(total_size, original_size) diff --git a/tests/core/tasks/utils/__init__.py b/tests/core/tasks/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/tasks/utils/test_read_input_task.py b/tests/core/tasks/utils/test_read_input_task.py deleted file mode 100644 index 100124b..0000000 --- a/tests/core/tasks/utils/test_read_input_task.py +++ /dev/null @@ -1,87 +0,0 @@ -import unittest -from loguru import logger -from probe_ably.core.tasks.utils import ReadInputTask - - -class PrepareRicoScaTest(unittest.TestCase): - def test_wrong_split_size(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_split_size.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_multi_task_multi_model_with_control_with_setup(self): - TEST_INPUT = ( - "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" - ) - read_input_task = ReadInputTask() - - output = read_input_task.run(TEST_INPUT) - - self.assertEquals(len(output["tasks"][0]["models"][0]["model_labels"]), 10) - - def test_multi_task_multi_model_with_control_with_no_setup(self): - TEST_INPUT = "./tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json" - read_input_task = ReadInputTask() - - output = read_input_task.run(TEST_INPUT) - - self.assertEquals(output["probing_setup"]["train_size"], 0.60) - - def test_wrong_setup(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_setup.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_multi_task_wrong_probing_model(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_probing_model.json" - read_input_task = ReadInputTask() - - # output = read_input_task.run(TEST_INPUT) - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_multi_task_wrong_inter_metric(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_inter.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_multi_task_wrong_intra_metric(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_intra.json" - read_input_task = ReadInputTask() - - # output = read_input_task.run(TEST_INPUT) - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_wrong_control_size(self): - TEST_INPUT = "./tests/sample_files/test_input/wrong_control_size.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_missing_key(self): - - TEST_INPUT_1 = "./tests/sample_files/test_input/missing_tasks_key.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT_1) - - def test_wrong_key_name(self): - - TEST_INPUT_2 = "./tests/sample_files/test_input/wrong_template_format.json" - read_input_task = ReadInputTask() - - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT_2) - - def test_file_does_not_exist(self): - TEST_INPUT = "./this/does/not/exist" - read_input_task = ReadInputTask() - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) - - def test_wrong_json_format(self): - TEST_INPUT = "./tests/sample_files/test_input/problematic_json_file.json" - read_input_task = ReadInputTask() - self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) diff --git a/tests/core/utils/__init__.py b/tests/core/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/core/utils/test_factory.py b/tests/core/utils/test_factory.py deleted file mode 100644 index 64ce3c4..0000000 --- a/tests/core/utils/test_factory.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest -from probe_ably.core.utils import GridModelFactory - - -class FactoryTest(unittest.TestCase): - def test_create_models(self): - thing = GridModelFactory.create_models('probe_ably.core.models.linear.LinearModel', num_models=2, param_args={'representation_size':768, 'n_classes':2}) - - print(thing) - diff --git a/probe_ably/core/tasks/__init__.py b/tests/metrics/__init__.py similarity index 100% rename from probe_ably/core/tasks/__init__.py rename to tests/metrics/__init__.py diff --git a/tests/core/tasks/metric/process_metric_task_test.py b/tests/metrics/test_process_metric_task.py similarity index 91% rename from tests/core/tasks/metric/process_metric_task_test.py rename to tests/metrics/test_process_metric_task.py index 0b3f6ce..fa60f86 100644 --- a/tests/core/tasks/metric/process_metric_task_test.py +++ b/tests/metrics/test_process_metric_task.py @@ -1,17 +1,17 @@ import unittest from numpy import array -from probe_ably.core.tasks.metric_task import ProcessMetricTask +from probe_ably.metrics import ProcessMetricTask class ProcessMetricTaskTest(unittest.TestCase): def test_process_metric_task(self): metric_results = { 0: { - "models": { + "representations": { 0: { - "model_name": "AUX TASK 1 - MODEL 1", - "probe_ably.core.models.linear.LinearModel": { + "representation_name": "AUX TASK 1 - MODEL 1", + "probe_ably.models.linear.LinearModel": { 0: { "complexity": {"norm": 1.1320844888687134}, "model": { @@ -35,7 +35,7 @@ def test_process_metric_task(self): }, }, }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { 0: { "complexity": {"nparams": 18080}, "model": { @@ -61,8 +61,8 @@ def test_process_metric_task(self): }, }, 1: { - "model_name": "AUX TASK 1 - MODEL 2", - "probe_ably.core.models.linear.LinearModel": { + "representation_name": "AUX TASK 1 - MODEL 2", + "probe_ably.models.linear.LinearModel": { 0: { "complexity": {"norm": 1.2004743814468384}, "model": { @@ -86,7 +86,7 @@ def test_process_metric_task(self): }, }, }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { 0: { "complexity": {"nparams": 3536}, "model": { @@ -115,10 +115,10 @@ def test_process_metric_task(self): "task_name": "AUX TASK 1", }, 1: { - "models": { + "representations": { 0: { - "model_name": "AUX TASK 2 - MODEL 1", - "probe_ably.core.models.linear.LinearModel": { + "representation_name": "AUX TASK 2 - MODEL 1", + "probe_ably.models.linear.LinearModel": { 0: { "complexity": {"norm": 1.0756611824035645}, "model": { @@ -142,7 +142,7 @@ def test_process_metric_task(self): }, }, }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { 0: { "complexity": {"nparams": 32937}, "model": { @@ -168,8 +168,8 @@ def test_process_metric_task(self): }, }, 1: { - "model_name": "AUX TASK 2 - MODEL 2", - "probe_ably.core.models.linear.LinearModel": { + "representation_name": "AUX TASK 2 - MODEL 2", + "probe_ably.models.linear.LinearModel": { 0: { "complexity": {"norm": 1.1543824672698975}, "model": { @@ -193,7 +193,7 @@ def test_process_metric_task(self): }, }, }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { 0: { "complexity": {"nparams": 291363}, "model": { @@ -219,8 +219,8 @@ def test_process_metric_task(self): }, }, 2: { - "model_name": "AUX TASK 2 - MODEL 3", - "probe_ably.core.models.linear.LinearModel": { + "representation_name": "AUX TASK 2 - MODEL 3", + "probe_ably.models.linear.LinearModel": { 0: { "complexity": {"norm": 1.1083829402923584}, "model": { @@ -244,7 +244,7 @@ def test_process_metric_task(self): }, }, }, - "probe_ably.core.models.mlp.MLPModel": { + "probe_ably.models.mlp.MLPModel": { 0: { "complexity": {"nparams": 303989}, "model": { @@ -274,13 +274,14 @@ def test_process_metric_task(self): }, } + print(metric_results) process_metric_task = ProcessMetricTask() processed_data = process_metric_task.run( metric_results, { - "intra_metric": "probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric": "probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric": "probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric": "probe_ably.metrics.selectivity.SelectivityMetric", }, ) self.assertEqual(len(processed_data), 2) diff --git a/probe_ably/core/tasks/probing/metrics/__init__.py b/tests/probing/__init__.py similarity index 100% rename from probe_ably/core/tasks/probing/metrics/__init__.py rename to tests/probing/__init__.py diff --git a/tests/probing/test_prepare_data_for_probing_task.py b/tests/probing/test_prepare_data_for_probing_task.py new file mode 100644 index 0000000..8f29d2c --- /dev/null +++ b/tests/probing/test_prepare_data_for_probing_task.py @@ -0,0 +1,36 @@ +from unittest import IsolatedAsyncioTestCase +from loguru import logger +from probe_ably.utils import ReadInputTask + +#TODO convert to test for from_parsed_json + +# class PrepareProbingDataTest(IsolatedAsyncioTestCase): +# async def test_prepare_data_probing(self): +# TEST_INPUT = ( +# "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" +# ) +# read_input_task = ReadInputTask() + +# output = await read_input_task.run(TEST_INPUT) + +# prepare_data_probing_task = PrepareDataForProbingTask() + +# dataset = prepare_data_probing_task.run( +# output["tasks"], output["probing_setup"] +# ) + +# total_size = ( +# dataset[0]["models"][0]["model"]["train"].__len__() +# + dataset[0]["models"][0]["model"]["dev"].__len__() +# + dataset[0]["models"][0]["model"]["test"].__len__() +# ) + +# original_size = len(output["tasks"][0]["models"][0]["model_vectors"]) + +# model_element = dataset[0]["models"][0]["model"]["train"].__getitem__(0)[0][0] +# control_element = dataset[0]["models"][0]["control"]["train"].__getitem__(0)[0][ +# 0 +# ] + +# self.assertEqual(model_element, control_element) +# self.assertEqual(total_size, original_size) diff --git a/tests/core/tasks/probing/test_train_probing.py b/tests/probing/test_train_probing_task.py similarity index 62% rename from tests/core/tasks/probing/test_train_probing.py rename to tests/probing/test_train_probing_task.py index baeaa5f..0c151b4 100644 --- a/tests/core/tasks/probing/test_train_probing.py +++ b/tests/probing/test_train_probing_task.py @@ -1,17 +1,17 @@ -import unittest +from unittest import IsolatedAsyncioTestCase from loguru import logger -from probe_ably.core.tasks.utils import ReadInputTask -from probe_ably.core.tasks.probing import PrepareDataForProbingTask, TrainProbingTask +from probe_ably.utils import ReadInputTask +from probe_ably.probing import PrepareDataForProbingTask, TrainProbingTask -class TrainProbingTest(unittest.TestCase): - def test_train_probing(self): +class TrainProbingTest(IsolatedAsyncioTestCase): + async def test_train_probing(self): TEST_INPUT = ( "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" ) read_input_task = ReadInputTask() - output = read_input_task.run(TEST_INPUT) + output = await read_input_task.run(TEST_INPUT) prepare_data_probing_task = PrepareDataForProbingTask() diff --git a/tests/sample_files/test_input/bp_test_input.json b/tests/sample_files/test_input/bp_test_input.json index d070be9..28d44e1 100644 --- a/tests/sample_files/test_input/bp_test_input.json +++ b/tests/sample_files/test_input/bp_test_input.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,17 +17,17 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] @@ -35,17 +35,17 @@ ], "probing_setup":{ - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", "probing_models":[ { - "probing_model_name":"probe_ably.core.models.linear.LinearModel", + "probing_model_name":"probe_ably.models.linear.LinearModel", "batch_size":5, "epochs":10, "number_of_models":5 }, { - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", + "probing_model_name":"probe_ably.models.mlp.MLPModel", "batch_size":5, "epochs":20, "number_of_models":10 diff --git a/tests/sample_files/test_input/missing_tasks_key.json b/tests/sample_files/test_input/missing_tasks_key.json index 48456ef..773a286 100644 --- a/tests/sample_files/test_input/missing_tasks_key.json +++ b/tests/sample_files/test_input/missing_tasks_key.json @@ -1,16 +1,16 @@ -{"0":{ +[{ "task_name": "AUX TASK 1", - "models": { - "0": { - "model_name": "AUX TASK 1 - MODEL 1", + "representations":[ + { + "representation_name": "AUX TASK 1 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model1_test.tsv", "control_location": "./tests/sample_files/test_representation/model1_test_control.tsv" }, - "1": { - "model_name": "AUX TASK 1 - MODEL 2", + { + "representation_name": "AUX TASK 1 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model2_test.tsv", "control_location": "./tests/sample_files/test_representation/model2_test_control.tsv" } - } + ] } -} \ No newline at end of file +] \ No newline at end of file diff --git a/tests/sample_files/test_input/multi_task_multi_model_with_control.json b/tests/sample_files/test_input/multi_task_multi_model_with_control.json index 884b53e..dbd24b0 100644 --- a/tests/sample_files/test_input/multi_task_multi_model_with_control.json +++ b/tests/sample_files/test_input/multi_task_multi_model_with_control.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,40 +17,40 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.50, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", "probing_models":[ { - "probing_model_name":"probe_ably.core.models.linear.LinearModel", + "probing_model_name":"probe_ably.models.linear.LinearModel", "batch_size":5, "epochs":10, "number_of_models":5 }, { - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", + "probing_model_name":"probe_ably.models.mlp.MLPModel", "batch_size":5, - "epochs":20, - "number_of_models":10 + "epochs":10, + "number_of_models":5 } ] } diff --git a/tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json b/tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json index 928d22c..223ffc3 100644 --- a/tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json +++ b/tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,17 +17,17 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] diff --git a/tests/sample_files/test_input/problematic_json_file.json b/tests/sample_files/test_input/problematic_json_file.json index cead652..e07e521 100644 --- a/tests/sample_files/test_input/problematic_json_file.json +++ b/tests/sample_files/test_input/problematic_json_file.json @@ -1,16 +1,16 @@ { - "tasks": {"0":{ + "tasks": [ "task_name": "AUX TASK 1", - "models": { - "0": { + "representations": [ + { "model_name": "AUX TASK 1 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model1_test.tsv", "control_location": "./tests/sample_files/test_representation/model1_test_control.tsv" }, - "1": { + { "model_name": "AUX TASK 1 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model2_test.tsv", "control_location": "./tests/sample_files/test_representation/model2_test_control.tsv" - } - } - }, \ No newline at end of file + } + ], +} \ No newline at end of file diff --git a/tests/sample_files/test_input/sample_config3.json b/tests/sample_files/test_input/sample_config3.json deleted file mode 100644 index e69de29..0000000 diff --git a/tests/sample_files/test_input/sample_config4.json b/tests/sample_files/test_input/sample_config4.json deleted file mode 100644 index e69de29..0000000 diff --git a/tests/sample_files/test_input/wrong_control_size.json b/tests/sample_files/test_input/wrong_control_size.json index a7fd18d..d9e412a 100644 --- a/tests/sample_files/test_input/wrong_control_size.json +++ b/tests/sample_files/test_input/wrong_control_size.json @@ -1,39 +1,35 @@ { "tasks": [{ "task_name": "AUX TASK 1", - "models": + "representations": [ { - "model_name": "AUX TASK 1 - MODEL 1", + "representation_name": "AUX TASK 1 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model1_test.tsv", "control_location": "./tests/sample_files/test_representation/model3_test_control.tsv" }, { - "model_name": "AUX TASK 1 - MODEL 2", + "representation_name": "AUX TASK 1 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model2_test.tsv", "control_location": "./tests/sample_files/test_representation/model2_test_control.tsv" } ] }, - { "task_name": "AUX TASK 2", "models": [{ - - "model_name": "AUX TASK 2 - MODEL 1", + "representation_name": "AUX TASK 2 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name": "AUX TASK 2 - MODEL 2", + "representation_name": "AUX TASK 2 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name": "AUX TASK 2 - MODEL 3", + "representation_name": "AUX TASK 2 - MODEL 3", "file_location": "./tests/sample_files/test_representation/model5_test.tsv" - } ] } - ] } \ No newline at end of file diff --git a/tests/sample_files/test_input/wrong_inter.json b/tests/sample_files/test_input/wrong_inter.json index a368f26..c204bc6 100644 --- a/tests/sample_files/test_input/wrong_inter.json +++ b/tests/sample_files/test_input/wrong_inter.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,28 +17,28 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.50, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivitMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivitMetric", "probing_models":[ { "probing_model_name":"probe_ably.core.models.linear.LinearModel", diff --git a/tests/sample_files/test_input/wrong_intra.json b/tests/sample_files/test_input/wrong_intra.json index 97dda75..110cb19 100644 --- a/tests/sample_files/test_input/wrong_intra.json +++ b/tests/sample_files/test_input/wrong_intra.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,28 +17,28 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.50, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", "probing_models":[ { "probing_model_name":"probe_ably.core.models.linear.LinearModel", diff --git a/tests/sample_files/test_input/wrong_probing_model.json b/tests/sample_files/test_input/wrong_probing_model.json index d95115c..8a156a3 100644 --- a/tests/sample_files/test_input/wrong_probing_model.json +++ b/tests/sample_files/test_input/wrong_probing_model.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,37 +17,37 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.50, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", "probing_models":[ { - "probing_model_name":"probe_ably.core.models.linear.Linearodel", + "probing_model_name":"probe_ably.models.linear.Linearodel", "batch_size":5, "epochs":10, "number_of_models":5 }, { - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", + "probing_model_name":"probe_ably.models.mlp.MLPModel", "batch_size":5, "epochs":20, "number_of_models":10 diff --git a/tests/sample_files/test_input/wrong_setup.json b/tests/sample_files/test_input/wrong_setup.json index 08975bf..1875796 100644 --- a/tests/sample_files/test_input/wrong_setup.json +++ b/tests/sample_files/test_input/wrong_setup.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,36 +17,36 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.50, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", "probing_models":[ { - "probing_model_name":"probe_ably.core.models.linear.LinearModel", + "probing_model_name":"probe_ably.models.linear.LinearModel", "epochs":10, "number_of_models":5 }, { - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", + "probing_model_name":"probe_ably.models.mlp.MLPModel", "batch_size":5, "epochs":20, "number_of_models":10 diff --git a/tests/sample_files/test_input/wrong_split_size.json b/tests/sample_files/test_input/wrong_split_size.json index e60cb70..aa95ca1 100644 --- a/tests/sample_files/test_input/wrong_split_size.json +++ b/tests/sample_files/test_input/wrong_split_size.json @@ -2,14 +2,14 @@ "tasks":[ { "task_name":"AUX TASK 1", - "models":[ + "representations":[ { - "model_name":"AUX TASK 1 - MODEL 1", + "representation_name":"AUX TASK 1 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model1_test.tsv", "control_location":"./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name":"AUX TASK 1 - MODEL 2", + "representation_name":"AUX TASK 1 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model2_test.tsv", "control_location":"./tests/sample_files/test_representation/model2_test_control.tsv" } @@ -17,37 +17,37 @@ }, { "task_name":"AUX TASK 2", - "models":[ + "representations":[ { - "model_name":"AUX TASK 2 - MODEL 1", + "representation_name":"AUX TASK 2 - MODEL 1", "file_location":"./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 2", + "representation_name":"AUX TASK 2 - MODEL 2", "file_location":"./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name":"AUX TASK 2 - MODEL 3", + "representation_name":"AUX TASK 2 - MODEL 3", "file_location":"./tests/sample_files/test_representation/model5_test.tsv" } ] } ], - "probing_setup":{ + "probing_config":{ "train_size":0.60, "dev_size":0.25, "test_size":0.25, - "intra_metric":"probe_ably.core.metrics.accuracy.AccuracyMetric", - "inter_metric":"probe_ably.core.metrics.selectivity.SelectivityMetric", + "intra_metric":"probe_ably.metrics.accuracy.AccuracyMetric", + "inter_metric":"probe_ably.metrics.selectivity.SelectivityMetric", "probing_models":[ { - "probing_model_name":"probe_ably.core.models.linear.LinearModel", + "probing_model_name":"probe_ably.models.linear.LinearModel", "batch_size":5, "epochs":10, "number_of_models":5 }, { - "probing_model_name":"probe_ably.core.models.mlp.MLPModel", + "probing_model_name":"probe_ably.models.mlp.MLPModel", "batch_size":5, "epochs":20, "number_of_models":10 diff --git a/tests/sample_files/test_input/wrong_template_format.json b/tests/sample_files/test_input/wrong_template_format.json index b814d5e..a71470d 100644 --- a/tests/sample_files/test_input/wrong_template_format.json +++ b/tests/sample_files/test_input/wrong_template_format.json @@ -1,39 +1,35 @@ { "tasks": [{ "name": "AUX TASK 1", - "models": + "representations": [ { - "model_name": "AUX TASK 1 - MODEL 1", + "representation_name": "AUX TASK 1 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model1_test.tsv", "control_location": "./tests/sample_files/test_representation/model1_test_control.tsv" }, { - "model_name": "AUX TASK 1 - MODEL 2", + "representation_name": "AUX TASK 1 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model2_test.tsv", "control_location": "./tests/sample_files/test_representation/model2_test_control.tsv" } ] - }, - { "task_name": "AUX TASK 2", - "models": [{ + "representations": [{ - "model_name": "AUX TASK 2 - MODEL 1", + "representation_name": "AUX TASK 2 - MODEL 1", "file_location": "./tests/sample_files/test_representation/model3_test.tsv" }, { - "model_name": "AUX TASK 2 - MODEL 2", + "representation_name": "AUX TASK 2 - MODEL 2", "file_location": "./tests/sample_files/test_representation/model4_test.tsv" }, { - "model_name": "AUX TASK 2 - MODEL 3", + "representation_name": "AUX TASK 2 - MODEL 3", "file_location": "./tests/sample_files/test_representation/model5_test.tsv" - } ] } - ] } \ No newline at end of file diff --git a/probe_ably/service/server/__init__.py b/tests/utils/__init__.py similarity index 100% rename from probe_ably/service/server/__init__.py rename to tests/utils/__init__.py diff --git a/tests/utils/test_read_input_task.py b/tests/utils/test_read_input_task.py new file mode 100644 index 0000000..dc4f006 --- /dev/null +++ b/tests/utils/test_read_input_task.py @@ -0,0 +1,101 @@ +from unittest import IsolatedAsyncioTestCase +from loguru import logger +from probe_ably.utils import ReadInputTask +from probe_ably.utils.read_input_task import InputClassNotFound + + +class PrepareRicoScaTest(IsolatedAsyncioTestCase): + async def test_wrong_split_size(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_split_size.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + # self.assertRaises(SystemExit, read_input_task.run, TEST_INPUT) + + async def test_multi_task_multi_model_with_control_with_setup(self): + TEST_INPUT = ( + "./tests/sample_files/test_input/multi_task_multi_model_with_control.json" + ) + read_input_task = ReadInputTask() + + output = await read_input_task.run(TEST_INPUT) + + self.assertEqual(len(output["tasks"][0]["representations"][0]["representation_labels"]), 10) + + async def test_multi_task_multi_model_with_control_with_no_setup(self): + TEST_INPUT = "./tests/sample_files/test_input/multi_task_multi_model_with_control_no_setup.json" + read_input_task = ReadInputTask() + + output = await read_input_task.run(TEST_INPUT) + + self.assertEqual(output["probing_config"]["train_size"], 0.60) + + async def test_wrong_setup(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_setup.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_multi_task_wrong_probing_model(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_probing_model.json" + read_input_task = ReadInputTask() + + # output = read_input_task.run(TEST_INPUT) + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_multi_task_wrong_inter_metric(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_inter.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_multi_task_wrong_intra_metric(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_intra.json" + read_input_task = ReadInputTask() + + # output = read_input_task.run(TEST_INPUT) + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_wrong_control_size(self): + TEST_INPUT = "./tests/sample_files/test_input/wrong_control_size.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_missing_key(self): + + TEST_INPUT_1 = "./tests/sample_files/test_input/missing_tasks_key.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT_1) + + async def test_wrong_key_name(self): + + TEST_INPUT_2 = "./tests/sample_files/test_input/wrong_template_format.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT_2) + + async def test_file_does_not_exist(self): + TEST_INPUT = "./this/does/not/exist" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) + + async def test_wrong_json_format(self): + TEST_INPUT = "./tests/sample_files/test_input/problematic_json_file.json" + read_input_task = ReadInputTask() + + with self.assertRaises(SystemExit): + await read_input_task.run(TEST_INPUT) \ No newline at end of file