From e673fde2bf8331b2b5540b324ea14b71c0f015e7 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Mon, 1 Apr 2019 15:02:58 -0400 Subject: [PATCH 01/40] direct copy from pyodide-0.10 build this adds the original pyodide.js with no mods. doesn't work out of the box. gitignore src files for building. the good place if we need to remove from git history if better solution come for importing --- .gitignore | 2 + app/utils/pyodide/pyodide.js | 404 +++++++++++++++++++++++++++++++++++ 2 files changed, 406 insertions(+) create mode 100644 app/utils/pyodide/pyodide.js diff --git a/.gitignore b/.gitignore index 86b8b644..1dddca1d 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ npm-debug.log.* *.sass.d.ts *.scss.d.ts keys.js + +app/utils/pyodide/src diff --git a/app/utils/pyodide/pyodide.js b/app/utils/pyodide/pyodide.js new file mode 100644 index 00000000..5360dda4 --- /dev/null +++ b/app/utils/pyodide/pyodide.js @@ -0,0 +1,404 @@ +/** + * The main bootstrap script for loading pyodide. + */ + +var languagePluginLoader = new Promise((resolve, reject) => { + // This is filled in by the Makefile to be either a local file or the + // deployed location. TODO: This should be done in a less hacky + // way. + var baseURL = self.languagePluginUrl || 'https://iodide.io/pyodide-demo/'; + baseURL = baseURL.substr(0, baseURL.lastIndexOf('/')) + '/'; + + //////////////////////////////////////////////////////////// + // Package loading + let loadedPackages = new Array(); + var loadPackagePromise = new Promise((resolve) => resolve()); + // Regexp for validating package name and URI + var package_name_regexp = '[a-z0-9_][a-z0-9_\-]*' + var package_uri_regexp = + new RegExp('^https?://.*?(' + package_name_regexp + ').js$', 'i'); + var package_name_regexp = new RegExp('^' + package_name_regexp + '$', 'i'); + + let _uri_to_package_name = (package_uri) => { + // Generate a unique package name from URI + + if (package_name_regexp.test(package_uri)) { + return package_uri; + } else if (package_uri_regexp.test(package_uri)) { + let match = package_uri_regexp.exec(package_uri); + // Get the regexp group corresponding to the package name + return match[1]; + } else { + return null; + } + }; + + // clang-format off + let preloadWasm = () => { + // On Chrome, we have to instantiate wasm asynchronously. Since that + // can't be done synchronously within the call to dlopen, we instantiate + // every .so that comes our way up front, caching it in the + // `preloadedWasm` dictionary. + + let promise = new Promise((resolve) => resolve()); + let FS = pyodide._module.FS; + + function recurseDir(rootpath) { + let dirs; + try { + dirs = FS.readdir(rootpath); + } catch { + return; + } + for (let entry of dirs) { + if (entry.startsWith('.')) { + continue; + } + const path = rootpath + entry; + if (entry.endsWith('.so')) { + if (Module['preloadedWasm'][path] === undefined) { + promise = promise + .then(() => Module['loadWebAssemblyModule']( + FS.readFile(path), {loadAsync: true})) + .then((module) => { + Module['preloadedWasm'][path] = module; + }); + } + } else if (FS.isDir(FS.lookupPath(path).node.mode)) { + recurseDir(path + '/'); + } + } + } + + recurseDir('/'); + + return promise; + } + // clang-format on + + function loadScript(url, onload, onerror) { + if (self.document) { // browser + const script = self.document.createElement('script'); + script.src = url; + script.onload = (e) => { onload(); }; + script.onerror = (e) => { onerror(); }; + self.document.head.appendChild(script); + } else if (self.importScripts) { // webworker + try { + self.importScripts(url); + onload(); + } catch { + onerror(); + } + } + } + + let _loadPackage = (names, messageCallback) => { + // DFS to find all dependencies of the requested packages + let packages = self.pyodide._module.packages.dependencies; + let loadedPackages = self.pyodide.loadedPackages; + let queue = [].concat(names || []); + let toLoad = new Array(); + while (queue.length) { + let package_uri = queue.pop(); + + const package = _uri_to_package_name(package_uri); + + if (package == null) { + console.error(`Invalid package name or URI '${package_uri}'`); + return; + } else if (package == package_uri) { + package_uri = 'default channel'; + } + + if (package in loadedPackages) { + if (package_uri != loadedPackages[package]) { + console.error(`URI mismatch, attempting to load package ` + + `${package} from ${package_uri} while it is already ` + + `loaded from ${loadedPackages[package]}!`); + return; + } + } else if (package in toLoad) { + if (package_uri != toLoad[package]) { + console.error(`URI mismatch, attempting to load package ` + + `${package} from ${package_uri} while it is already ` + + `being loaded from ${toLoad[package]}!`); + return; + } + } else { + console.log(`Loading ${package} from ${package_uri}`); + + toLoad[package] = package_uri; + if (packages.hasOwnProperty(package)) { + packages[package].forEach((subpackage) => { + if (!(subpackage in loadedPackages) && !(subpackage in toLoad)) { + queue.push(subpackage); + } + }); + } else { + console.error(`Unknown package '${package}'`); + } + } + } + + self.pyodide._module.locateFile = (path) => { + // handle packages loaded from custom URLs + let package = path.replace(/\.data$/, ""); + if (package in toLoad) { + let package_uri = toLoad[package]; + if (package_uri != 'default channel') { + return package_uri.replace(/\.js$/, ".data"); + }; + }; + return baseURL + path; + }; + + let promise = new Promise((resolve, reject) => { + if (Object.keys(toLoad).length === 0) { + resolve('No new packages to load'); + return; + } + + const packageList = Array.from(Object.keys(toLoad)).join(', '); + if (messageCallback !== undefined) { + messageCallback(`Loading ${packageList}`); + } + + // monitorRunDependencies is called at the beginning and the end of each + // package being loaded. We know we are done when it has been called + // exactly "toLoad * 2" times. + var packageCounter = Object.keys(toLoad).length * 2; + + self.pyodide._module.monitorRunDependencies = () => { + packageCounter--; + if (packageCounter === 0) { + for (let package in toLoad) { + self.pyodide.loadedPackages[package] = toLoad[package]; + } + delete self.pyodide._module.monitorRunDependencies; + self.removeEventListener('error', windowErrorHandler); + if (!isFirefox) { + preloadWasm().then(() => {resolve(`Loaded ${packageList}`)}); + } else { + resolve(`Loaded ${packageList}`); + } + } + }; + + // Add a handler for any exceptions that are thrown in the process of + // loading a package + var windowErrorHandler = (err) => { + delete self.pyodide._module.monitorRunDependencies; + self.removeEventListener('error', windowErrorHandler); + // Set up a new Promise chain, since this one failed + loadPackagePromise = new Promise((resolve) => resolve()); + reject(err.message); + }; + self.addEventListener('error', windowErrorHandler); + + for (let package in toLoad) { + let scriptSrc; + let package_uri = toLoad[package]; + if (package_uri == 'default channel') { + scriptSrc = `${baseURL}${package}.js`; + } else { + scriptSrc = `${package_uri}`; + } + loadScript(scriptSrc, () => {}, () => { + // If the package_uri fails to load, call monitorRunDependencies twice + // (so packageCounter will still hit 0 and finish loading), and remove + // the package from toLoad so we don't mark it as loaded. + console.error(`Couldn't load package from URL ${scriptSrc}`) + let index = toLoad.indexOf(package); + if (index !== -1) { + toLoad.splice(index, 1); + } + for (let i = 0; i < 2; i++) { + self.pyodide._module.monitorRunDependencies(); + } + }); + } + + // We have to invalidate Python's import caches, or it won't + // see the new files. This is done here so it happens in parallel + // with the fetching over the network. + self.pyodide.runPython('import importlib as _importlib\n' + + '_importlib.invalidate_caches()\n'); + }); + + return promise; + }; + + let loadPackage = (names, messageCallback) => { + /* We want to make sure that only one loadPackage invocation runs at any + * given time, so this creates a "chain" of promises. */ + loadPackagePromise = + loadPackagePromise.then(() => _loadPackage(names, messageCallback)); + return loadPackagePromise; + }; + + //////////////////////////////////////////////////////////// + // Fix Python recursion limit + function fixRecursionLimit(pyodide) { + // The Javascript/Wasm call stack may be too small to handle the default + // Python call stack limit of 1000 frames. This is generally the case on + // Chrom(ium), but not on Firefox. Here, we determine the Javascript call + // stack depth available, and then divide by 50 (determined heuristically) + // to set the maximum Python call stack depth. + + let depth = 0; + function recurse() { + depth += 1; + recurse(); + } + try { + recurse(); + } catch (err) { + ; + } + + let recursionLimit = depth / 50; + if (recursionLimit > 1000) { + recursionLimit = 1000; + } + pyodide.runPython( + `import sys; sys.setrecursionlimit(int(${recursionLimit}))`); + }; + + //////////////////////////////////////////////////////////// + // Rearrange namespace for public API + let PUBLIC_API = [ + 'globals', + 'loadPackage', + 'loadedPackages', + 'pyimport', + 'repr', + 'runPython', + 'runPythonAsync', + 'checkABI', + 'version', + ]; + + function makePublicAPI(module, public_api) { + var namespace = {_module : module}; + for (let name of public_api) { + namespace[name] = module[name]; + } + return namespace; + } + + //////////////////////////////////////////////////////////// + // Loading Pyodide + let wasmURL = `${baseURL}pyodide.asm.wasm`; + let Module = {}; + self.Module = Module; + + Module.noImageDecoding = true; + Module.noAudioDecoding = true; + Module.noWasmDecoding = true; + Module.preloadedWasm = {}; + let isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; + + let wasm_promise = WebAssembly.compileStreaming(fetch(wasmURL)); + Module.instantiateWasm = (info, receiveInstance) => { + wasm_promise.then(module => WebAssembly.instantiate(module, info)) + .then(instance => receiveInstance(instance)); + return {}; + }; + + Module.checkABI = function(ABI_number) { + if (ABI_number !== parseInt('1')) { + var ABI_mismatch_exception = + `ABI numbers differ. Expected 1, got ${ABI_number}`; + console.error(ABI_mismatch_exception); + throw ABI_mismatch_exception; + } + return true; + }; + + Module.locateFile = (path) => baseURL + path; + var postRunPromise = new Promise((resolve, reject) => { + Module.postRun = () => { + delete self.Module; + fetch(`${baseURL}packages.json`) + .then((response) => response.json()) + .then((json) => { + fixRecursionLimit(self.pyodide); + self.pyodide.globals = + self.pyodide.runPython('import sys\nsys.modules["__main__"]'); + self.pyodide = makePublicAPI(self.pyodide, PUBLIC_API); + self.pyodide._module.packages = json; + resolve(); + }); + }; + }); + + var dataLoadPromise = new Promise((resolve, reject) => { + Module.monitorRunDependencies = + (n) => { + if (n === 0) { + delete Module.monitorRunDependencies; + resolve(); + } + } + }); + + Promise.all([ postRunPromise, dataLoadPromise ]).then(() => resolve()); + + const data_script_src = `${baseURL}pyodide.asm.data.js`; + loadScript(data_script_src, () => { + const scriptSrc = `${baseURL}pyodide.asm.js`; + loadScript(scriptSrc, () => { + // The emscripten module needs to be at this location for the core + // filesystem to install itself. Once that's complete, it will be replaced + // by the call to `makePublicAPI` with a more limited public API. + self.pyodide = pyodide(Module); + self.pyodide.loadedPackages = new Array(); + self.pyodide.loadPackage = loadPackage; + }, () => {}); + }, () => {}); + + //////////////////////////////////////////////////////////// + // Iodide-specific functionality, that doesn't make sense + // if not using with Iodide. + if (self.iodide !== undefined) { + // Load the custom CSS for Pyodide + let link = document.createElement('link'); + link.rel = 'stylesheet'; + link.type = 'text/css'; + link.href = `${baseURL}renderedhtml.css`; + document.getElementsByTagName('head')[0].appendChild(link); + + // Add a custom output handler for Python objects + self.iodide.addOutputRenderer({ + shouldRender : (val) => { + return (typeof val === 'function' && + pyodide._module.PyProxy.isPyProxy(val)); + }, + + render : (val) => { + let div = document.createElement('div'); + div.className = 'rendered_html'; + var element; + if (val._repr_html_ !== undefined) { + let result = val._repr_html_(); + if (typeof result === 'string') { + div.appendChild(new DOMParser() + .parseFromString(result, 'text/html') + .body.firstChild); + element = div; + } else { + element = result; + } + } else { + let pre = document.createElement('pre'); + pre.textContent = val.toString(); + div.appendChild(pre); + element = div; + } + return element.outerHTML; + } + }); + } +}); +languagePluginLoader From d4f788300d449e66ce90ee701fc7cf699cc72b00 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 25 Apr 2020 20:12:33 -0400 Subject: [PATCH 02/40] Rebased with makebrainwaves/master --- app/components/HomeComponent/index.tsx | 4 + app/utils/pyodide/pyodide.js | 102 ++++++------------- configs/webpack.config.renderer.dev.babel.js | 6 +- 3 files changed, 37 insertions(+), 75 deletions(-) diff --git a/app/components/HomeComponent/index.tsx b/app/components/HomeComponent/index.tsx index 618331fe..d1d18884 100644 --- a/app/components/HomeComponent/index.tsx +++ b/app/components/HomeComponent/index.tsx @@ -40,9 +40,13 @@ import OverviewComponent from './OverviewComponent'; import EEGExplorationComponent from '../EEGExplorationComponent'; import { SignalQualityData } from '../../constants/interfaces'; import { getExperimentFromType } from '../../utils/labjs/functions'; +import { languagePluginLoader } from '../../utils/pyodide/pyodide'; const { dialog } = remote; +// this initiates pyodide +languagePluginLoader; + const HOME_STEPS = { // TODO: maybe change the recent and new labels, but not necessary right now RECENT: 'MY EXPERIMENTS', diff --git a/app/utils/pyodide/pyodide.js b/app/utils/pyodide/pyodide.js index 5360dda4..2ada05ae 100644 --- a/app/utils/pyodide/pyodide.js +++ b/app/utils/pyodide/pyodide.js @@ -6,7 +6,8 @@ var languagePluginLoader = new Promise((resolve, reject) => { // This is filled in by the Makefile to be either a local file or the // deployed location. TODO: This should be done in a less hacky // way. - var baseURL = self.languagePluginUrl || 'https://iodide.io/pyodide-demo/'; + var baseURL = 'http://localhost:1212/src/'; + // var baseURL = self.languagePluginUrl || 'https://iodide.io/pyodide-demo/'; baseURL = baseURL.substr(0, baseURL.lastIndexOf('/')) + '/'; //////////////////////////////////////////////////////////// @@ -47,7 +48,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { let dirs; try { dirs = FS.readdir(rootpath); - } catch { + } catch (err) { return; } for (let entry of dirs) { @@ -87,7 +88,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { try { self.importScripts(url); onload(); - } catch { + } catch (err) { onerror(); } } @@ -102,50 +103,50 @@ var languagePluginLoader = new Promise((resolve, reject) => { while (queue.length) { let package_uri = queue.pop(); - const package = _uri_to_package_name(package_uri); + const packageName = _uri_to_package_name(package_uri); - if (package == null) { + if (packageName == null) { console.error(`Invalid package name or URI '${package_uri}'`); return; - } else if (package == package_uri) { + } else if (packageName == package_uri) { package_uri = 'default channel'; } - if (package in loadedPackages) { - if (package_uri != loadedPackages[package]) { + if (packageName in loadedPackages) { + if (package_uri != loadedPackages[packageName]) { console.error(`URI mismatch, attempting to load package ` + - `${package} from ${package_uri} while it is already ` + - `loaded from ${loadedPackages[package]}!`); + `${packageName} from ${package_uri} while it is already ` + + `loaded from ${loadedPackages[packageName]}!`); return; } - } else if (package in toLoad) { - if (package_uri != toLoad[package]) { + } else if (packageName in toLoad) { + if (package_uri != toLoad[packageName]) { console.error(`URI mismatch, attempting to load package ` + - `${package} from ${package_uri} while it is already ` + - `being loaded from ${toLoad[package]}!`); + `${packageName} from ${package_uri} while it is already ` + + `being loaded from ${toLoad[packageName]}!`); return; } } else { - console.log(`Loading ${package} from ${package_uri}`); + console.log(`Loading ${packageName} from ${package_uri}`); - toLoad[package] = package_uri; - if (packages.hasOwnProperty(package)) { - packages[package].forEach((subpackage) => { + toLoad[packageName] = package_uri; + if (packages.hasOwnProperty(packageName)) { + packages[packageName].forEach((subpackage) => { if (!(subpackage in loadedPackages) && !(subpackage in toLoad)) { queue.push(subpackage); } }); } else { - console.error(`Unknown package '${package}'`); + console.error(`Unknown package '${packageName}'`); } } } self.pyodide._module.locateFile = (path) => { // handle packages loaded from custom URLs - let package = path.replace(/\.data$/, ""); - if (package in toLoad) { - let package_uri = toLoad[package]; + let packageName = path.replace(/\.data$/, ""); + if (packageName in toLoad) { + let package_uri = toLoad[packageName]; if (package_uri != 'default channel') { return package_uri.replace(/\.js$/, ".data"); }; @@ -172,8 +173,8 @@ var languagePluginLoader = new Promise((resolve, reject) => { self.pyodide._module.monitorRunDependencies = () => { packageCounter--; if (packageCounter === 0) { - for (let package in toLoad) { - self.pyodide.loadedPackages[package] = toLoad[package]; + for (let packageName in toLoad) { + self.pyodide.loadedPackages[packageName] = toLoad[packageName]; } delete self.pyodide._module.monitorRunDependencies; self.removeEventListener('error', windowErrorHandler); @@ -196,11 +197,11 @@ var languagePluginLoader = new Promise((resolve, reject) => { }; self.addEventListener('error', windowErrorHandler); - for (let package in toLoad) { + for (let packageName in toLoad) { let scriptSrc; - let package_uri = toLoad[package]; + let package_uri = toLoad[packageName]; if (package_uri == 'default channel') { - scriptSrc = `${baseURL}${package}.js`; + scriptSrc = `${baseURL}${packageName}.js`; } else { scriptSrc = `${package_uri}`; } @@ -209,7 +210,7 @@ var languagePluginLoader = new Promise((resolve, reject) => { // (so packageCounter will still hit 0 and finish loading), and remove // the package from toLoad so we don't mark it as loaded. console.error(`Couldn't load package from URL ${scriptSrc}`) - let index = toLoad.indexOf(package); + let index = toLoad.indexOf(packageName); if (index !== -1) { toLoad.splice(index, 1); } @@ -357,48 +358,5 @@ var languagePluginLoader = new Promise((resolve, reject) => { self.pyodide.loadPackage = loadPackage; }, () => {}); }, () => {}); - - //////////////////////////////////////////////////////////// - // Iodide-specific functionality, that doesn't make sense - // if not using with Iodide. - if (self.iodide !== undefined) { - // Load the custom CSS for Pyodide - let link = document.createElement('link'); - link.rel = 'stylesheet'; - link.type = 'text/css'; - link.href = `${baseURL}renderedhtml.css`; - document.getElementsByTagName('head')[0].appendChild(link); - - // Add a custom output handler for Python objects - self.iodide.addOutputRenderer({ - shouldRender : (val) => { - return (typeof val === 'function' && - pyodide._module.PyProxy.isPyProxy(val)); - }, - - render : (val) => { - let div = document.createElement('div'); - div.className = 'rendered_html'; - var element; - if (val._repr_html_ !== undefined) { - let result = val._repr_html_(); - if (typeof result === 'string') { - div.appendChild(new DOMParser() - .parseFromString(result, 'text/html') - .body.firstChild); - element = div; - } else { - element = result; - } - } else { - let pre = document.createElement('pre'); - pre.textContent = val.toString(); - div.appendChild(pre); - element = div; - } - return element.outerHTML; - } - }); - } }); -languagePluginLoader +// languagePluginLoader diff --git a/configs/webpack.config.renderer.dev.babel.js b/configs/webpack.config.renderer.dev.babel.js index 093ba545..1112bdeb 100644 --- a/configs/webpack.config.renderer.dev.babel.js +++ b/configs/webpack.config.renderer.dev.babel.js @@ -246,8 +246,8 @@ export default merge(baseConfig, { inline: true, lazy: false, hot: true, - headers: { 'Access-Control-Allow-Origin': '*' }, - contentBase: path.join(__dirname, 'dist'), + headers: { "Access-Control-Allow-Origin": "*" }, + contentBase: [path.join(__dirname, "app", "dist"), path.join(__dirname, "app", "utils", "pyodide")], watchOptions: { aggregateTimeout: 300, ignored: /node_modules/, @@ -257,7 +257,7 @@ export default merge(baseConfig, { verbose: true, disableDotRule: false, }, - before() { + before(app) { if (process.env.START_HOT) { console.log('Starting Main Process...'); spawn('npm', ['run', 'start-main-dev'], { From 40b08b3d8a70d3cc3b9899b59ed5b6e9e1cac7b6 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Wed, 3 Apr 2019 11:16:46 -0400 Subject: [PATCH 03/40] Update pyodide.js --- app/utils/pyodide/pyodide.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/utils/pyodide/pyodide.js b/app/utils/pyodide/pyodide.js index 2ada05ae..d1601285 100644 --- a/app/utils/pyodide/pyodide.js +++ b/app/utils/pyodide/pyodide.js @@ -1,12 +1,11 @@ /** * The main bootstrap script for loading pyodide. */ +const port = process.env.PORT || 1212; -var languagePluginLoader = new Promise((resolve, reject) => { - // This is filled in by the Makefile to be either a local file or the - // deployed location. TODO: This should be done in a less hacky - // way. - var baseURL = 'http://localhost:1212/src/'; +export const languagePluginLoader = new Promise((resolve, reject) => { + + var baseURL = 'http://localhost:' + port + '/src/'; // var baseURL = self.languagePluginUrl || 'https://iodide.io/pyodide-demo/'; baseURL = baseURL.substr(0, baseURL.lastIndexOf('/')) + '/'; @@ -359,4 +358,3 @@ var languagePluginLoader = new Promise((resolve, reject) => { }, () => {}); }, () => {}); }); -// languagePluginLoader From 4a3abfe8f74de41b553487f597bdcc870b1536ba Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 7 Apr 2019 00:12:12 -0400 Subject: [PATCH 04/40] rewriting the python dependency within brainwaves the objective is to clean up and organize the python calls. - pyimport contains all the python modules to be used - utils is for all the function definitions - cells are independent commands to be made (maybe rename to commands) pipes.js and function.js were responsible for managing the state of the python kernel. they won't be needed in this new paradigm --- app/utils/pyodide/commands.py | 90 +++++++++++++++++ app/utils/pyodide/pyimport.py | 16 ++++ app/utils/pyodide/utils.py | 175 ++++++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100644 app/utils/pyodide/commands.py create mode 100644 app/utils/pyodide/pyimport.py create mode 100644 app/utils/pyodide/utils.py diff --git a/app/utils/pyodide/commands.py b/app/utils/pyodide/commands.py new file mode 100644 index 00000000..83f349c4 --- /dev/null +++ b/app/utils/pyodide/commands.py @@ -0,0 +1,90 @@ +import * as path from 'path'; +import { readFileSync } from 'fs'; + +// ----------------------------- +// Imports and Utility functions + +export const imports = () => + readFileSync(path.join(__dirname, '/utils/pyodide/pyimport.py'), 'utf8'); + +export const utils = () => + readFileSync(path.join(__dirname, '/utils/jupyter/utils.py'), 'utf8'); + +export const loadCSV = (filePathArray: Array) => + [ + `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, + `replace_ch_names = None`, + `raw = load_data(files, replace_ch_names)` + ].join('\n'); + +// --------------------------- +// MNE-Related Data Processing +export const loadCleanedEpochs = (filePathArray: Array) => + [ + `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, + `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, + `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` + ].join('\n'); + +// NOTE: this command includes a ';' to prevent returning data +export const filterIIR = (lowCutoff: number, highCutoff: number) => + `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`; + +export const epochEvents = ( + eventIDs: { [string]: number }, + tmin: number, + tmax: number, + reject?: Array | string = 'None' +) => { + const command = [ + `event_id = ${JSON.stringify(eventIDs)}`, + `tmin=${tmin}`, + `tmax=${tmax}`, + `baseline= (tmin, tmax)`, + `picks = None`, + `reject = ${reject}`, + 'events = find_events(raw)', + `raw_epochs = Epochs(raw, events=events, event_id=event_id, + tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, + verbose=False, picks=picks)`, + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` + ].join('\n'); + return command; +}; + +export const requestEpochsInfo = (variableName: string) => + `get_epochs_info(${variableName})`; + +export const requestChannelInfo = () => + `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; + +// ----------------------------- +// Plot functions + +export const cleanEpochsPlot = () => + `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)`; + +export const plotPSD = () => `raw.plot_psd(fmin=1, fmax=30)`; + +export const plotTopoMap = () => `plot_topo(clean_epochs, conditions)`; + +export const plotERP = (channelIndex: number) => + `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, + ci=97.5, n_boot=1000, title='', diff_waveform=None)`; + +export const saveEpochs = (workspaceDir: string, subject: string) => + `raw_epochs.save(${formatFilePath( + path.join( + workspaceDir, + 'Data', + subject, + 'EEG', + `${subject}-cleaned-epo.fif` + ) + )})`; + +// ------------------------------------------- +// Helper methods + +const formatFilePath = (filePath: string) => + `"${filePath.replace(/\\/g, '/')}"`; diff --git a/app/utils/pyodide/pyimport.py b/app/utils/pyodide/pyimport.py new file mode 100644 index 00000000..2149f2c7 --- /dev/null +++ b/app/utils/pyodide/pyimport.py @@ -0,0 +1,16 @@ +from time import time, strftime, gmtime +import os +from collections import OrderedDict +from glob import glob + +import numpy as np +import pandas as pd # maybe we can remove this dependency +import seaborn as sns +from matplotlib import pyplot as plt + +from mne import (Epochs, RawArray, concatenate_raws, concatenate_epochs, + create_info, find_events, read_epochs, set_eeg_reference) +from mne.channels import read_montage + + +plt.style.use(fivethirtyeight) diff --git a/app/utils/pyodide/utils.py b/app/utils/pyodide/utils.py new file mode 100644 index 00000000..c09aa583 --- /dev/null +++ b/app/utils/pyodide/utils.py @@ -0,0 +1,175 @@ +from glob import glob +import os +from collections import OrderedDict +from mne import create_info, concatenate_raws, viz +from mne.io import RawArray +from mne.channels import read_montage +import pandas as pd +import numpy as np +import seaborn as sns +from matplotlib import pyplot as plt + +sns.set_context('talk') +sns.set_style('white') + + +def load_data(fnames, sfreq=128., replace_ch_names=None): + """Load CSV files from the /data directory into a Raw object. + + Args: + fnames (array): CSV filepaths from which to load data + + Keyword Args: + sfreq (float): EEG sampling frequency + replace_ch_names (dict or None): dictionary containing a mapping to + rename channels. Useful when an external electrode was used. + + Returns: + (mne.io.array.array.RawArray): loaded EEG + """ + + raw = [] + print(fnames) + for fname in fnames: + # read the file + data = pd.read_csv(fname, index_col=0) + + data = data.dropna() + + # get estimation of sampling rate and use to determine sfreq + # yes, this could probably be improved + srate = 1000 / (data.index.values[1] - data.index.values[0]) + if srate >= 200: + sfreq = 256 + else: + sfreq = 128 + + # name of each channel + ch_names = list(data.columns) + + # indices of each channel + ch_ind = list(range(len(ch_names))) + + if replace_ch_names is not None: + ch_names = [c if c not in replace_ch_names.keys() + else replace_ch_names[c] for c in ch_names] + + # type of each channels + ch_types = ['eeg'] * (len(ch_ind) - 1) + ['stim'] + montage = read_montage('standard_1005') + + # get data and exclude Aux channel + data = data.values[:, ch_ind].T + + # create MNE object + info = create_info(ch_names=ch_names, ch_types=ch_types, + sfreq=sfreq, montage=montage) + raw.append(RawArray(data=data, info=info)) + + # concatenate all raw objects + raws = concatenate_raws(raw) + + return raws + + +def plot_topo(epochs, conditions=OrderedDict()): + palette = sns.color_palette("hls", len(conditions) + 1) + evokeds = [epochs[name].average() for name in (conditions)] + + evoked_topo = viz.plot_evoked_topo( + evokeds, vline=None, color=palette[0:len(conditions)], show=False) + evoked_topo.patch.set_alpha(0) + evoked_topo.set_size_inches(10, 8) + for axis in evoked_topo.axes: + for line in axis.lines: + line.set_linewidth(2) + + legend_loc = 0 + labels = [e.comment if e.comment else 'Unknown' for e in evokeds] + legend = plt.legend(labels, loc=legend_loc, prop={'size': 20}) + txts = legend.get_texts() + for txt, col in zip(txts, palette): + txt.set_color(col) + + return evoked_topo + + +def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, n_boot=1000, + title='', palette=None, + diff_waveform=(4, 3)): + """Plot Averaged Epochs with ERP conditions. + + Args: + epochs (mne.epochs): EEG epochs + + Keyword Args: + conditions (OrderedDict): dictionary that contains the names of the + conditions to plot as keys, and the list of corresponding marker + numbers as value. E.g., + + conditions = {'Non-target': [0, 1], + 'Target': [2, 3, 4]} + + ch_ind (int): index of channel to plot data from + ci (float): confidence interval in range [0, 100] + n_boot (int): number of bootstrap samples + title (str): title of the figure + palette (list): color palette to use for conditions + ylim (tuple): (ymin, ymax) + diff_waveform (tuple or None): tuple of ints indicating which + conditions to subtract for producing the difference waveform. + If None, do not plot a difference waveform + + Returns: + (matplotlib.figure.Figure): figure object + (list of matplotlib.axes._subplots.AxesSubplot): list of axes + """ + if isinstance(conditions, dict): + conditions = OrderedDict(conditions) + + if palette is None: + palette = sns.color_palette("hls", len(conditions) + 1) + + X = epochs.get_data() + times = epochs.times + y = pd.Series(epochs.events[:, -1]) + fig, ax = plt.subplots() + + for cond, color in zip(conditions.values(), palette): + sns.tsplot(X[y.isin(cond), ch_ind], time=times, color=color, + n_boot=n_boot, ci=ci) + + if diff_waveform: + diff = (np.nanmean(X[y == diff_waveform[1], ch_ind], axis=0) - + np.nanmean(X[y == diff_waveform[0], ch_ind], axis=0)) + ax.plot(times, diff, color='k', lw=1) + + ax.set_title(epochs.ch_names[ch_ind]) + ax.axvline(x=0, color='k', lw=1, label='_nolegend_') + + ax.set_xlabel('Time (s)') + ax.set_ylabel('Amplitude (uV)') + ax.set_xlabel('Time (s)') + ax.set_ylabel('Amplitude (uV)') + + # Round y axis tick labels to 2 decimal places + # ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f')) + + if diff_waveform: + legend = (['{} - {}'.format(diff_waveform[1], diff_waveform[0])] + + list(conditions.keys())) + else: + legend = conditions.keys() + ax.legend(legend) + sns.despine() + plt.tight_layout() + + if title: + fig.suptitle(title, fontsize=20) + + fig.set_size_inches(10, 8) + + return fig, ax + +def get_epochs_info(epochs): + return [*[{x: len(epochs[x])} for x in epochs.event_id], {"Drop Percentage": round((1 - len(epochs.events)/len(epochs.drop_log)) * 100, 2)}, {"Total Epochs": len(epochs.events)}] From ff7646586f4d5145da2856261ecc8093d150343f Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 25 Apr 2020 20:14:55 -0400 Subject: [PATCH 05/40] fixed package script conflict --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fbd07eb5..1db897eb 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "package-mac": "yarn build && electron-builder build --mac", "package-linux": "yarn build && electron-builder build --linux", "package-win": "yarn build && electron-builder build --win --x64", - "postinstall": "node -r @babel/register internals/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn build-dll && opencollective-postinstall", + "postinstall": "node -r @babel/register internals/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn build-dll && opencollective-postinstall && mkdir app/utils/pyodide/src && cd app/utils/pyodide/src && curl -LJO https://github.com/iodide-project/pyodide/releases/download/0.12.0/pyodide-build-0.12.0.tar.bz2 && tar xjf pyodide-build-0.12.0.tar.bz2 && rm pyodide-build-0.12.0.tar.bz2", "postlint-fix": "prettier --ignore-path .eslintignore --single-quote --write '**/*.{js,jsx,json,html,css,less,scss,yml}'", "postlint-styles-fix": "prettier --ignore-path .eslintignore --single-quote --write '**/*.{css,scss}'", "preinstall": "node ./internals/scripts/CheckYarn.js", From 147326d66d63adf48a7e2232612e7bf62f7e1ef8 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 25 Apr 2020 20:15:18 -0400 Subject: [PATCH 06/40] fixed package script conflict --- package.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 1db897eb..2ff2a541 100644 --- a/package.json +++ b/package.json @@ -37,9 +37,7 @@ "test-watch": "yarn test --watch" }, "lint-staged": { - "*.{js,jsx,ts,tsx}": [ - "cross-env NODE_ENV=development eslint --cache" - ], + "*.{js,jsx,ts,tsx}": ["cross-env NODE_ENV=development eslint --cache"], "{*.json,.{babelrc,eslintrc,prettierrc,stylelintrc}}": [ "prettier --ignore-path .eslintignore --parser json --write" ], @@ -47,9 +45,7 @@ "stylelint --ignore-path .eslintignore --syntax scss --fix", "prettier --ignore-path .eslintignore --single-quote --write" ], - "*.{html,md,yml}": [ - "prettier --ignore-path .eslintignore --single-quote --write" - ] + "*.{html,md,yml}": ["prettier --ignore-path .eslintignore --single-quote --write"] }, "build": { "productName": "BrainWaves", From 9a03113dc77a9de74e17d9a7ce8cbf799062881e Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 7 Apr 2019 00:36:19 -0400 Subject: [PATCH 07/40] rename files from jupyter to pyodide to make it clear that python is being served using WebAssembly and not the jupyter, changing all the references --- app/actions/pyodideActions.js | 52 +++++ app/epics/pyodideEpics.js | 412 +++++++++++++++++++++++++++++++++ app/reducers/pyodideReducer.js | 107 +++++++++ 3 files changed, 571 insertions(+) create mode 100644 app/actions/pyodideActions.js create mode 100644 app/epics/pyodideEpics.js create mode 100644 app/reducers/pyodideReducer.js diff --git a/app/actions/pyodideActions.js b/app/actions/pyodideActions.js new file mode 100644 index 00000000..f3234fd3 --- /dev/null +++ b/app/actions/pyodideActions.js @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------- +// Action Types + +export const LAUNCH_KERNEL = 'LAUNCH_KERNEL'; +export const REQUEST_KERNEL_INFO = 'REQUEST_KERNEL_INFO'; +export const SEND_EXECUTE_REQUEST = 'SEND_EXECUTE_REQUEST'; +export const LOAD_EPOCHS = 'LOAD_EPOCHS'; +export const LOAD_CLEANED_EPOCHS = 'LOAD_CLEANED_EPOCHS'; +export const LOAD_PSD = 'LOAD_PSD'; +export const LOAD_ERP = 'LOAD_ERP'; +export const LOAD_TOPO = 'LOAD_TOPO'; +export const CLEAN_EPOCHS = 'CLEAN_EPOCHS'; +export const CLOSE_KERNEL = 'CLOSE_KERNEL'; + +// ------------------------------------------------------------------------- +// Actions + +export const launchKernel = () => ({ type: LAUNCH_KERNEL }); + +export const requestKernelInfo = () => ({ type: REQUEST_KERNEL_INFO }); + +export const sendExecuteRequest = (payload: string) => ({ + payload, + type: SEND_EXECUTE_REQUEST, +}); + +export const loadEpochs = (payload: Array) => ({ + payload, + type: LOAD_EPOCHS, +}); + +export const loadCleanedEpochs = (payload: Array) => ({ + payload, + type: LOAD_CLEANED_EPOCHS, +}); + +export const loadPSD = () => ({ + type: LOAD_PSD, +}); + +export const loadERP = (payload: ?string) => ({ + payload, + type: LOAD_ERP, +}); + +export const loadTopo = () => ({ + type: LOAD_TOPO, +}); + +export const cleanEpochs = () => ({ type: CLEAN_EPOCHS }); + +export const closeKernel = () => ({ type: CLOSE_KERNEL }); diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js new file mode 100644 index 00000000..e000751a --- /dev/null +++ b/app/epics/pyodideEpics.js @@ -0,0 +1,412 @@ +import { combineEpics } from 'redux-observable'; +import { from, of } from 'rxjs'; +import { map, mergeMap, tap, pluck, ignoreElements, filter, take } from 'rxjs/operators'; +import { find } from 'kernelspecs'; +import { launchSpec } from 'spawnteract'; +import { createMainChannel } from 'enchannel-zmq-backend'; +import { isNil } from 'lodash'; +import { kernelInfoRequest, executeRequest } from '@nteract/messaging'; +import { toast } from 'react-toastify'; +import { getWorkspaceDir } from '../utils/filesystem/storage'; +import { + LAUNCH_KERNEL, + REQUEST_KERNEL_INFO, + LOAD_EPOCHS, + LOAD_CLEANED_EPOCHS, + LOAD_PSD, + LOAD_ERP, + LOAD_TOPO, + CLEAN_EPOCHS, + CLOSE_KERNEL, + loadTopo, + loadERP +} from '../actions/pyodideActions'; +import { + imports, + utils, + loadCSV, + loadCleanedEpochs, + filterIIR, + epochEvents, + requestEpochsInfo, + requestChannelInfo, + cleanEpochsPlot, + plotPSD, + plotERP, + plotTopoMap, + saveEpochs +} from '../utils/pyodide/commands'; +import { + EMOTIV_CHANNELS, + EVENTS, + DEVICES, + MUSE_CHANNELS, + JUPYTER_VARIABLE_NAMES, +} from '../constants/constants'; +import { + parseSingleQuoteJSON, + parseKernelStatus, + debugParseMessage +} from '../utils/pyodide/functions'; + +export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; +export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; +export const SET_KERNEL = 'SET_KERNEL'; +export const SET_KERNEL_STATUS = 'SET_KERNEL_STATUS'; +export const SET_KERNEL_INFO = 'SET_KERNEL_INFO'; +export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; +export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; +export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; +export const SET_PSD_PLOT = 'SET_PSD_PLOT'; +export const SET_ERP_PLOT = 'SET_ERP_PLOT'; +export const SET_TOPO_PLOT = 'SET_TOPO_PLOT'; +export const RECEIVE_EXECUTE_REPLY = 'RECEIVE_EXECUTE_REPLY'; +export const RECEIVE_EXECUTE_RESULT = 'RECEIVE_EXECUTE_RESULT'; +export const RECEIVE_STREAM = 'RECEIVE_STREAM'; +export const RECEIVE_DISPLAY_DATA = 'RECEIVE_DISPLAY_DATA'; + +// ------------------------------------------------------------------------- +// Action Creators + +const getEpochsInfo = (payload) => ({ payload, type: GET_EPOCHS_INFO }); + +const getChannelInfo = () => ({ type: GET_CHANNEL_INFO }); + +const setKernel = (payload) => ({ + payload, + type: SET_KERNEL, +}); + +const setKernelStatus = (payload) => ({ + payload, + type: SET_KERNEL_STATUS, +}); + +const setKernelInfo = (payload) => ({ + payload, + type: SET_KERNEL_INFO, +}); + +const setMainChannel = (payload) => ({ + payload, + type: SET_MAIN_CHANNEL, +}); + +const setEpochInfo = (payload) => ({ + payload, + type: SET_EPOCH_INFO, +}); + +const setChannelInfo = (payload) => ({ + payload, + type: SET_CHANNEL_INFO, +}); + +const setPSDPlot = (payload) => ({ + payload, + type: SET_PSD_PLOT, +}); + +const setTopoPlot = (payload) => ({ + payload, + type: SET_TOPO_PLOT, +}); + +const setERPPlot = (payload) => ({ + payload, + type: SET_ERP_PLOT, +}); + +const receiveExecuteReply = (payload) => ({ + payload, + type: RECEIVE_EXECUTE_REPLY, +}); + +const receiveExecuteResult = (payload) => ({ + payload, + type: RECEIVE_EXECUTE_RESULT, +}); + +const receiveDisplayData = (payload) => ({ + payload, + type: RECEIVE_DISPLAY_DATA, +}); + +const receiveStream = (payload) => ({ + payload, + type: RECEIVE_STREAM, +}); + +// ------------------------------------------------------------------------- +// Epics + +const launchEpic = (action$) => + action$.ofType(LAUNCH_KERNEL).pipe( + mergeMap(() => from(find('brainwaves'))), + tap((kernelInfo) => { + if (isNil(kernelInfo)) { + toast.error("Could not find 'brainwaves' jupyter kernel. Have you installed Python?"); + } + }), + filter((kernelInfo) => !isNil(kernelInfo)), + mergeMap((kernelInfo) => + from( + launchSpec(kernelInfo.spec, { + // No STDIN, opt in to STDOUT and STDERR as node streams + stdio: ['ignore', 'pipe', 'pipe'], + }) + ) + ), + tap((kernel) => { + // Route everything that we won't get in messages to our own stdout + kernel.spawn.stdout.on('data', (data) => { + const text = data.toString(); + console.log('KERNEL STDOUT: ', text); + }); + kernel.spawn.stderr.on('data', (data) => { + const text = data.toString(); + console.log('KERNEL STDERR: ', text); + toast.error('Jupyter: ', text); + }); + + kernel.spawn.on('close', () => { + console.log('Kernel closed'); + }); + }), + map(setKernel) + ); + +const setUpChannelEpic = (action$) => + action$.ofType(SET_KERNEL).pipe( + pluck('payload'), + mergeMap((kernel) => from(createMainChannel(kernel.config))), + tap((mainChannel) => mainChannel.next(executeRequest(imports()))), + tap((mainChannel) => mainChannel.next(executeRequest(utils()))), + map(setMainChannel) + ); + +const receiveChannelMessageEpic = (action$, state$) => + action$.ofType(SET_MAIN_CHANNEL).pipe( + mergeMap(() => + state$.value.jupyter.mainChannel.pipe( + map((msg) => { + console.log(debugParseMessage(msg)); + switch (msg['header']['msg_type']) { + case 'kernel_info_reply': + return setKernelInfo(msg); + case 'status': + return setKernelStatus(parseKernelStatus(msg)); + case 'stream': + return receiveStream(msg); + case 'execute_reply': + return receiveExecuteReply(msg); + case 'execute_result': + return receiveExecuteResult(msg); + case 'display_data': + return receiveDisplayData(msg); + default: + } + }), + filter((action) => !isNil(action)) + ) + ) + ); + +const requestKernelInfoEpic = (action$, state$) => + action$.ofType(REQUEST_KERNEL_INFO).pipe( + filter(() => state$.value.jupyter.mainChannel), + map(() => state$.value.jupyter.mainChannel.next(kernelInfoRequest())), + ignoreElements() + ); + +const loadEpochsEpic = (action$, state$) => + action$.ofType(LOAD_EPOCHS).pipe( + pluck('payload'), + filter((filePathsArray) => filePathsArray.length >= 1), + map((filePathsArray) => + state$.value.jupyter.mainChannel.next(executeRequest(loadCSV(filePathsArray))) + ), + awaitOkMessage(action$), + execute(filterIIR(1, 30), state$), + awaitOkMessage(action$), + map(() => + epochEvents( + { + [state$.value.experiment.params.stimulus1.title]: EVENTS.STIMULUS_1, + [state$.value.experiment.params.stimulus2.title]: EVENTS.STIMULUS_2, + [state$.value.experiment.params.stimulus3.title]: EVENTS.STIMULUS_3, + [state$.value.experiment.params.stimulus4.title]: EVENTS.STIMULUS_4, + }, + -0.1, + 0.8 + ) + ), + tap((e)=> {console.log('e', e)}), + map((epochEventsCommand) => + state$.value.jupyter.mainChannel.next(executeRequest(epochEventsCommand)) + ), + awaitOkMessage(action$), + map(() => getEpochsInfo(JUPYTER_VARIABLE_NAMES.RAW_EPOCHS)) + ); + +const loadCleanedEpochsEpic = (action$, state$) => + action$.ofType(LOAD_CLEANED_EPOCHS).pipe( + pluck('payload'), + filter((filePathsArray) => filePathsArray.length >= 1), + map((filePathsArray) => + state$.value.jupyter.mainChannel.next(executeRequest(loadCleanedEpochs(filePathsArray))) + ), + awaitOkMessage(action$), + mergeMap(() => + of(getEpochsInfo(JUPYTER_VARIABLE_NAMES.CLEAN_EPOCHS), getChannelInfo(), loadTopo()) + ) + ); + +const cleanEpochsEpic = (action$, state$) => + action$.ofType(CLEAN_EPOCHS).pipe( + execute(cleanEpochsPlot(), state$), + mergeMap(() => + action$.ofType(RECEIVE_STREAM).pipe( + pluck('payload'), + filter( + (msg) => msg.channel === 'iopub' && msg.content.text.includes('Channels marked as bad') + ), + take(1) + ) + ), + map(() => + state$.value.jupyter.mainChannel.next( + executeRequest( + saveEpochs( + getWorkspaceDir(state$.value.experiment.title), + state$.value.experiment.subject + ) + ) + ) + ), + awaitOkMessage(action$), + map(() => getEpochsInfo(JUPYTER_VARIABLE_NAMES.RAW_EPOCHS)) + ); + +const getEpochsInfoEpic = (action$, state$) => + action$.ofType(GET_EPOCHS_INFO).pipe( + pluck('payload'), + map((variableName) => + state$.value.jupyter.mainChannel.next(executeRequest(requestEpochsInfo(variableName))) + ), + mergeMap(() => + action$.ofType(RECEIVE_EXECUTE_RESULT).pipe( + pluck('payload'), + filter((msg) => msg.channel === 'iopub' && !isNil(msg.content.data)), + pluck('content', 'data', 'text/plain'), + filter((msg) => msg.includes('Drop Percentage')), + take(1) + ) + ), + map((epochInfoString) => + parseSingleQuoteJSON(epochInfoString).map((infoObj) => ({ + name: Object.keys(infoObj)[0], + value: infoObj[Object.keys(infoObj)[0]], + })) + ), + map(setEpochInfo) + ); + +const getChannelInfoEpic = (action$, state$) => + action$.ofType(GET_CHANNEL_INFO).pipe( + execute(requestChannelInfo(), state$), + mergeMap(() => + action$.ofType(RECEIVE_EXECUTE_RESULT).pipe( + pluck('payload'), + filter((msg) => msg.channel === 'iopub' && !isNil(msg.content.data)), + pluck('content', 'data', 'text/plain'), + // Filter to prevent this from reading requestEpochsInfo returns + filter((msg) => !msg.includes('Drop Percentage')), + take(1) + ) + ), + map((channelInfoString) => setChannelInfo(parseSingleQuoteJSON(channelInfoString))) + ); + +const loadPSDEpic = (action$, state$) => + action$.ofType(LOAD_PSD).pipe( + execute(plotPSD(), state$), + mergeMap(() => + action$.ofType(RECEIVE_DISPLAY_DATA).pipe( + pluck('payload'), + // PSD graphs should have two axes + filter((msg) => msg.content.data['text/plain'].includes('2 Axes')), + pluck('content', 'data'), + take(1) + ) + ), + map(setPSDPlot) + ); + +const loadTopoEpic = (action$, state$) => + action$.ofType(LOAD_TOPO).pipe( + execute(plotTopoMap(), state$), + mergeMap(() => + action$.ofType(RECEIVE_DISPLAY_DATA).pipe(pluck('payload'), pluck('content', 'data'), take(1)) + ), + mergeMap((topoPlot) => + of( + setTopoPlot(topoPlot), + loadERP( + state$.value.device.deviceType === DEVICES.EMOTIV ? EMOTIV_CHANNELS[0] : MUSE_CHANNELS[0] + ) + ) + ) + ); + +const loadERPEpic = (action$, state$) => + action$.ofType(LOAD_ERP).pipe( + pluck('payload'), + map((channelName) => { + if (MUSE_CHANNELS.includes(channelName)) { + return MUSE_CHANNELS.indexOf(channelName); + } else if (EMOTIV_CHANNELS.includes(channelName)) { + return EMOTIV_CHANNELS.indexOf(channelName); + } + console.warn('channel name supplied to loadERPEpic does not belong to either device'); + return EMOTIV_CHANNELS[0]; + }), + map((channelIndex) => + state$.value.jupyter.mainChannel.next(executeRequest(plotERP(channelIndex))) + ), + mergeMap(() => + action$.ofType(RECEIVE_DISPLAY_DATA).pipe( + pluck('payload'), + // ERP graphs should have 1 axis according to MNE + filter((msg) => msg.content.data['text/plain'].includes('1 Axes')), + pluck('content', 'data'), + take(1) + ) + ), + map(setERPPlot) + ); + +const closeKernelEpic = (action$, state$) => + action$.ofType(CLOSE_KERNEL).pipe( + map(() => { + state$.value.jupyter.kernel.spawn.kill(); + state$.value.jupyter.mainChannel.complete(); + }), + ignoreElements() + ); + +export default combineEpics( + launchEpic, + setUpChannelEpic, + requestKernelInfoEpic, + receiveChannelMessageEpic, + loadEpochsEpic, + loadCleanedEpochsEpic, + cleanEpochsEpic, + getEpochsInfoEpic, + getChannelInfoEpic, + loadPSDEpic, + loadTopoEpic, + loadERPEpic, + closeKernelEpic +); diff --git a/app/reducers/pyodideReducer.js b/app/reducers/pyodideReducer.js new file mode 100644 index 00000000..fa395798 --- /dev/null +++ b/app/reducers/pyodideReducer.js @@ -0,0 +1,107 @@ +// @flow +import { + SET_KERNEL, + SET_KERNEL_STATUS, + SET_MAIN_CHANNEL, + SET_KERNEL_INFO, + SET_EPOCH_INFO, + SET_CHANNEL_INFO, + SET_PSD_PLOT, + SET_TOPO_PLOT, + SET_ERP_PLOT, + RECEIVE_EXECUTE_RETURN +} from '../epics/pyodideEpics'; +import { ActionType, Kernel } from '../constants/interfaces'; +import { KERNEL_STATUS } from '../constants/constants'; +import { EXPERIMENT_CLEANUP } from '../epics/experimentEpics'; + +export interface JupyterStateType { + +kernel: ?Kernel; + +kernelStatus: KERNEL_STATUS; + +mainChannel: ?any; + +epochsInfo: ?Array<{ [string]: number | string }>; + +channelInfo: ?Array; + +psdPlot: ?{ [string]: string }; + +topoPlot: ?{ [string]: string }; + +erpPlot: ?{ [string]: string }; +} + +const initialState = { + kernel: null, + kernelStatus: KERNEL_STATUS.OFFLINE, + mainChannel: null, + epochsInfo: null, + channelInfo: [], + psdPlot: null, + topoPlot: null, + erpPlot: null, +}; + +export default function jupyter(state: JupyterStateType = initialState, action: ActionType) { + switch (action.type) { + case SET_KERNEL: + return { + ...state, + kernel: action.payload, + }; + + case SET_KERNEL_STATUS: + return { + ...state, + kernelStatus: action.payload, + }; + + case SET_MAIN_CHANNEL: + return { + ...state, + mainChannel: action.payload, + }; + + case SET_KERNEL_INFO: + return state; + + case SET_EPOCH_INFO: + return { + ...state, + epochsInfo: action.payload, + }; + + case SET_CHANNEL_INFO: + return { + ...state, + channelInfo: action.payload, + }; + + case SET_PSD_PLOT: + return { + ...state, + psdPlot: action.payload, + }; + + case SET_TOPO_PLOT: + return { + ...state, + topoPlot: action.payload, + }; + + case SET_ERP_PLOT: + return { + ...state, + erpPlot: action.payload, + }; + + case EXPERIMENT_CLEANUP: + return { + ...state, + epochsInfo: null, + psdPlot: null, + erpPlot: null, + }; + + case RECEIVE_EXECUTE_RETURN: + return state; + + default: + return state; + } +} From 6be006b0d03b86708b9084b0756fe0888093e83d Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Tue, 9 Apr 2019 16:04:08 -0400 Subject: [PATCH 08/40] WIP cleanup and overhaul --- app/actions/pyodideActions.js | 24 +++---- app/constants/constants.ts | 4 +- app/constants/interfaces.js | 81 ++++++++++++++++++++++++ app/epics/pyodideEpics.js | 114 ++++++---------------------------- app/utils/pyodide/commands.py | 2 +- app/utils/pyodide/utils.py | 96 ++++++++++++++++------------ 6 files changed, 168 insertions(+), 153 deletions(-) create mode 100644 app/constants/interfaces.js diff --git a/app/actions/pyodideActions.js b/app/actions/pyodideActions.js index f3234fd3..df2b87b3 100644 --- a/app/actions/pyodideActions.js +++ b/app/actions/pyodideActions.js @@ -1,24 +1,18 @@ // ------------------------------------------------------------------------- // Action Types -export const LAUNCH_KERNEL = 'LAUNCH_KERNEL'; -export const REQUEST_KERNEL_INFO = 'REQUEST_KERNEL_INFO'; -export const SEND_EXECUTE_REQUEST = 'SEND_EXECUTE_REQUEST'; -export const LOAD_EPOCHS = 'LOAD_EPOCHS'; -export const LOAD_CLEANED_EPOCHS = 'LOAD_CLEANED_EPOCHS'; -export const LOAD_PSD = 'LOAD_PSD'; -export const LOAD_ERP = 'LOAD_ERP'; -export const LOAD_TOPO = 'LOAD_TOPO'; -export const CLEAN_EPOCHS = 'CLEAN_EPOCHS'; -export const CLOSE_KERNEL = 'CLOSE_KERNEL'; +export const SEND_EXECUTE_REQUEST = "SEND_EXECUTE_REQUEST"; +export const LOAD_EPOCHS = "LOAD_EPOCHS"; +export const LOAD_CLEANED_EPOCHS = "LOAD_CLEANED_EPOCHS"; +export const LOAD_PSD = "LOAD_PSD"; +export const LOAD_ERP = "LOAD_ERP"; +export const LOAD_TOPO = "LOAD_TOPO"; +export const CLEAN_EPOCHS = "CLEAN_EPOCHS"; +export const CLOSE_KERNEL = "CLOSE_KERNEL"; // ------------------------------------------------------------------------- // Actions -export const launchKernel = () => ({ type: LAUNCH_KERNEL }); - -export const requestKernelInfo = () => ({ type: REQUEST_KERNEL_INFO }); - export const sendExecuteRequest = (payload: string) => ({ payload, type: SEND_EXECUTE_REQUEST, @@ -48,5 +42,3 @@ export const loadTopo = () => ({ }); export const cleanEpochs = () => ({ type: CLEAN_EPOCHS }); - -export const closeKernel = () => ({ type: CLOSE_KERNEL }); diff --git a/app/constants/constants.ts b/app/constants/constants.ts index c7444663..4b573d4d 100644 --- a/app/constants/constants.ts +++ b/app/constants/constants.ts @@ -50,8 +50,8 @@ export enum DEVICE_AVAILABILITY { AVAILABLE = 'AVAILABLE', } -// Names of variables in the jupyter kernel -export enum JUPYTER_VARIABLE_NAMES { +// Names of variables in the pyodide kernel +export enum PYODIDE_VARIABLE_NAMES { RAW_EPOCHS = 'raw_epochs', CLEAN_EPOCHS = 'clean_epochs', } diff --git a/app/constants/interfaces.js b/app/constants/interfaces.js new file mode 100644 index 00000000..455823a3 --- /dev/null +++ b/app/constants/interfaces.js @@ -0,0 +1,81 @@ +/* + * This file contains all the custom types that we use for Flow type checking + */ + +import { EVENTS } from './constants'; + +// TODO: Write interfaces for device objects (Observables, Classes, etc) + +// ------------------------------------------------------------------ +// lab.js Experiment + +export type ExperimentParameters = { + trialDuration: number, + nbTrials: number, + iti: number, + jitter: number, + sampleType: string, + intro: string, + // Setting this to any prevents ridiculous flow runtime errors + showProgessBar: any, + stimulus1: { dir: string, type: EVENTS, title: string, response: string }, + stimulus2: { dir: string, type: EVENTS, title: string, response: string }, +}; + +export type ExperimentDescription = { + question: string, + hypothesis: string, + methods: string, +}; + +// Array of timeline and trial ids that will be presented in experiment +export type MainTimeline = Array; + +// jsPsych trial presented as part of an experiment +export interface Trial { + id: string; + type: string; + stimulus?: string | StimulusVariable; + trial_duration?: number | (() => number); + post_trial_gap?: number; + on_load?: (string) => void | StimulusVariable; + choices?: Array; +} + +// Timeline of jsPsych trials +export type Timeline = { + id: string, + timeline: Array, + sample?: SampleParameter, + timeline_variables?: Array, +}; + +export interface SampleParameter { + type: string; + size?: number; + fn?: () => Array; +} + +export type StimulusVariable = () => any; + +// -------------------------------------------------------------------- +// Device + +export interface EEGData { + data: Array; + timestamp: number; + marker?: string | number; +} + +export interface DeviceInfo { + name: string; + samplingRate: number; +} + +// -------------------------------------------------------------------- +// General + +export interface ActionType { + +payload: any; + +type: string; +} diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index e000751a..781966f9 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -1,23 +1,24 @@ import { combineEpics } from 'redux-observable'; import { from, of } from 'rxjs'; -import { map, mergeMap, tap, pluck, ignoreElements, filter, take } from 'rxjs/operators'; -import { find } from 'kernelspecs'; -import { launchSpec } from 'spawnteract'; -import { createMainChannel } from 'enchannel-zmq-backend'; +import { + map, + mergeMap, + tap, + pluck, + ignoreElements, + filter, + take +} from 'rxjs/operators'; import { isNil } from 'lodash'; -import { kernelInfoRequest, executeRequest } from '@nteract/messaging'; import { toast } from 'react-toastify'; import { getWorkspaceDir } from '../utils/filesystem/storage'; import { - LAUNCH_KERNEL, - REQUEST_KERNEL_INFO, LOAD_EPOCHS, LOAD_CLEANED_EPOCHS, LOAD_PSD, LOAD_ERP, LOAD_TOPO, CLEAN_EPOCHS, - CLOSE_KERNEL, loadTopo, loadERP } from '../actions/pyodideActions'; @@ -41,13 +42,9 @@ import { EVENTS, DEVICES, MUSE_CHANNELS, - JUPYTER_VARIABLE_NAMES, + PYODIDE_VARIABLE_NAMES } from '../constants/constants'; -import { - parseSingleQuoteJSON, - parseKernelStatus, - debugParseMessage -} from '../utils/pyodide/functions'; + export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; @@ -140,85 +137,6 @@ const receiveStream = (payload) => ({ // ------------------------------------------------------------------------- // Epics -const launchEpic = (action$) => - action$.ofType(LAUNCH_KERNEL).pipe( - mergeMap(() => from(find('brainwaves'))), - tap((kernelInfo) => { - if (isNil(kernelInfo)) { - toast.error("Could not find 'brainwaves' jupyter kernel. Have you installed Python?"); - } - }), - filter((kernelInfo) => !isNil(kernelInfo)), - mergeMap((kernelInfo) => - from( - launchSpec(kernelInfo.spec, { - // No STDIN, opt in to STDOUT and STDERR as node streams - stdio: ['ignore', 'pipe', 'pipe'], - }) - ) - ), - tap((kernel) => { - // Route everything that we won't get in messages to our own stdout - kernel.spawn.stdout.on('data', (data) => { - const text = data.toString(); - console.log('KERNEL STDOUT: ', text); - }); - kernel.spawn.stderr.on('data', (data) => { - const text = data.toString(); - console.log('KERNEL STDERR: ', text); - toast.error('Jupyter: ', text); - }); - - kernel.spawn.on('close', () => { - console.log('Kernel closed'); - }); - }), - map(setKernel) - ); - -const setUpChannelEpic = (action$) => - action$.ofType(SET_KERNEL).pipe( - pluck('payload'), - mergeMap((kernel) => from(createMainChannel(kernel.config))), - tap((mainChannel) => mainChannel.next(executeRequest(imports()))), - tap((mainChannel) => mainChannel.next(executeRequest(utils()))), - map(setMainChannel) - ); - -const receiveChannelMessageEpic = (action$, state$) => - action$.ofType(SET_MAIN_CHANNEL).pipe( - mergeMap(() => - state$.value.jupyter.mainChannel.pipe( - map((msg) => { - console.log(debugParseMessage(msg)); - switch (msg['header']['msg_type']) { - case 'kernel_info_reply': - return setKernelInfo(msg); - case 'status': - return setKernelStatus(parseKernelStatus(msg)); - case 'stream': - return receiveStream(msg); - case 'execute_reply': - return receiveExecuteReply(msg); - case 'execute_result': - return receiveExecuteResult(msg); - case 'display_data': - return receiveDisplayData(msg); - default: - } - }), - filter((action) => !isNil(action)) - ) - ) - ); - -const requestKernelInfoEpic = (action$, state$) => - action$.ofType(REQUEST_KERNEL_INFO).pipe( - filter(() => state$.value.jupyter.mainChannel), - map(() => state$.value.jupyter.mainChannel.next(kernelInfoRequest())), - ignoreElements() - ); - const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck('payload'), @@ -246,7 +164,7 @@ const loadEpochsEpic = (action$, state$) => state$.value.jupyter.mainChannel.next(executeRequest(epochEventsCommand)) ), awaitOkMessage(action$), - map(() => getEpochsInfo(JUPYTER_VARIABLE_NAMES.RAW_EPOCHS)) + map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); const loadCleanedEpochsEpic = (action$, state$) => @@ -258,7 +176,11 @@ const loadCleanedEpochsEpic = (action$, state$) => ), awaitOkMessage(action$), mergeMap(() => - of(getEpochsInfo(JUPYTER_VARIABLE_NAMES.CLEAN_EPOCHS), getChannelInfo(), loadTopo()) + of( + getEpochsInfo(PYODIDE_VARIABLE_NAMES.CLEAN_EPOCHS), + getChannelInfo(), + loadTopo() + ) ) ); @@ -285,7 +207,7 @@ const cleanEpochsEpic = (action$, state$) => ) ), awaitOkMessage(action$), - map(() => getEpochsInfo(JUPYTER_VARIABLE_NAMES.RAW_EPOCHS)) + map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); const getEpochsInfoEpic = (action$, state$) => diff --git a/app/utils/pyodide/commands.py b/app/utils/pyodide/commands.py index 83f349c4..7c048286 100644 --- a/app/utils/pyodide/commands.py +++ b/app/utils/pyodide/commands.py @@ -8,7 +8,7 @@ readFileSync(path.join(__dirname, '/utils/pyodide/pyimport.py'), 'utf8'); export const utils = () => - readFileSync(path.join(__dirname, '/utils/jupyter/utils.py'), 'utf8'); + readFileSync(path.join(__dirname, '/utils/pyodide/utils.py'), 'utf8'); export const loadCSV = (filePathArray: Array) => [ diff --git a/app/utils/pyodide/utils.py b/app/utils/pyodide/utils.py index c09aa583..52a5eae0 100644 --- a/app/utils/pyodide/utils.py +++ b/app/utils/pyodide/utils.py @@ -14,18 +14,22 @@ def load_data(fnames, sfreq=128., replace_ch_names=None): - """Load CSV files from the /data directory into a Raw object. - - Args: - fnames (array): CSV filepaths from which to load data - - Keyword Args: - sfreq (float): EEG sampling frequency - replace_ch_names (dict or None): dictionary containing a mapping to - rename channels. Useful when an external electrode was used. - - Returns: - (mne.io.array.array.RawArray): loaded EEG + """Load CSV files from the /data directory into a RawArray object. + + Parameters + ---------- + fnames : list + CSV filepaths from which to load data + sfreq : float + EEG sampling frequency + replace_ch_names : dict | None + A dict containing a mapping to rename channels. + Useful when an external electrode was used during recording. + + Returns + ------- + raw : an instance of mne.io.RawArray + The loaded data. """ raw = [] @@ -94,35 +98,48 @@ def plot_topo(epochs, conditions=OrderedDict()): return evoked_topo -def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, n_boot=1000, - title='', palette=None, - diff_waveform=(4, 3)): +def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, + n_boot=1000, title='', palette=None, diff_waveform=(4, 3)): """Plot Averaged Epochs with ERP conditions. - Args: - epochs (mne.epochs): EEG epochs - - Keyword Args: - conditions (OrderedDict): dictionary that contains the names of the - conditions to plot as keys, and the list of corresponding marker - numbers as value. E.g., - - conditions = {'Non-target': [0, 1], - 'Target': [2, 3, 4]} - - ch_ind (int): index of channel to plot data from - ci (float): confidence interval in range [0, 100] - n_boot (int): number of bootstrap samples - title (str): title of the figure - palette (list): color palette to use for conditions - ylim (tuple): (ymin, ymax) - diff_waveform (tuple or None): tuple of ints indicating which - conditions to subtract for producing the difference waveform. + Parameters + ---------- + epochs : an instance of mne.epochs + EEG epochs + conditions : an instance of OrderedDict + An ordered dictionary that contains the names of the + conditions to plot as keys, and the list of corresponding marker + numbers as value. + + E.g., + + conditions = {'Non-target': [0, 1], + 'Target': [2, 3, 4]} + + ch_ind : int + An index of channel to plot data from. + ci : float + The confidence interval of the measurement within + the range [0, 100]. + n_boot : int + Number of bootstrap samples. + title : str + Title of the figure. + palette : list + Color palette to use for conditions. + ylim : tuple + (ymin, ymax) + diff_waveform : tuple | None + tuple of ints indicating which conditions to subtract for + producing the difference waveform. If None, do not plot a difference waveform - Returns: - (matplotlib.figure.Figure): figure object - (list of matplotlib.axes._subplots.AxesSubplot): list of axes + Returns + ------- + fig : an instance of matplotlib.figure.Figure + A figure object. + ax : list of matplotlib.axes._subplots.AxesSubplot + A list of axes """ if isinstance(conditions, dict): conditions = OrderedDict(conditions) @@ -172,4 +189,7 @@ def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, n_boot= return fig, ax def get_epochs_info(epochs): - return [*[{x: len(epochs[x])} for x in epochs.event_id], {"Drop Percentage": round((1 - len(epochs.events)/len(epochs.drop_log)) * 100, 2)}, {"Total Epochs": len(epochs.events)}] + return [*[{x: len(epochs[x])} for x in epochs.event_id], + {"Drop Percentage": round((1 - len(epochs.events) / + len(epochs.drop_log)) * 100, 2)}, + {"Total Epochs": len(epochs.events)}] From 3a02731ab9eb64346210b85dac4444d5c115e4cb Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Thu, 11 Apr 2019 23:20:06 -0400 Subject: [PATCH 09/40] overhaul continues --- app/actions/pyodideActions.js | 1 - app/components/CleanComponent/index.tsx | 21 ++----- app/components/JupyterPlotWidget.tsx | 8 +-- app/containers/AnalyzeContainer.ts | 6 +- app/containers/CleanContainer.ts | 6 +- app/containers/HomeContainer.ts | 15 ++--- app/epics/index.ts | 4 +- app/epics/pyodideEpics.js | 66 +++++++-------------- app/reducers/index.js | 15 +++++ app/reducers/pyodideReducer.js | 32 ++-------- app/store/configureStore.dev.js | 77 +++++++++++++++++++++++++ app/utils/filesystem/storage.ts | 2 +- app/utils/jupyter/cells.ts | 1 + app/utils/jupyter/functions.ts | 14 ----- app/utils/jupyter/pipes.ts | 7 ++- 15 files changed, 147 insertions(+), 128 deletions(-) create mode 100644 app/reducers/index.js create mode 100644 app/store/configureStore.dev.js diff --git a/app/actions/pyodideActions.js b/app/actions/pyodideActions.js index df2b87b3..4fe1b3ba 100644 --- a/app/actions/pyodideActions.js +++ b/app/actions/pyodideActions.js @@ -8,7 +8,6 @@ export const LOAD_PSD = "LOAD_PSD"; export const LOAD_ERP = "LOAD_ERP"; export const LOAD_TOPO = "LOAD_TOPO"; export const CLEAN_EPOCHS = "CLEAN_EPOCHS"; -export const CLOSE_KERNEL = "CLOSE_KERNEL"; // ------------------------------------------------------------------------- // Actions diff --git a/app/components/CleanComponent/index.tsx b/app/components/CleanComponent/index.tsx index 0053191f..d6b40727 100644 --- a/app/components/CleanComponent/index.tsx +++ b/app/components/CleanComponent/index.tsx @@ -18,8 +18,7 @@ import { Link } from 'react-router-dom'; import { isNil, isArray, isString } from 'lodash'; import styles from '../styles/collect.css'; import commonStyles from '../styles/common.css'; -import { EXPERIMENTS, DEVICES, KERNEL_STATUS } from '../../constants/constants'; -import { Kernel } from '../../constants/interfaces'; +import { EXPERIMENTS, DEVICES } from '../../constants/constants'; import { readWorkspaceRawEEGData } from '../../utils/filesystem/storage'; import CleanSidebar from './CleanSidebar'; import { JupyterActions, ExperimentActions } from '../../actions'; @@ -29,12 +28,10 @@ export interface Props { title: string; deviceType: DEVICES; mainChannel?: any; - kernel?: Kernel; - kernelStatus: KERNEL_STATUS; epochsInfo: Array<{ [key: string]: number | string; }>; - JupyterActions: typeof JupyterActions; + PyodideActions: typeof PyodideActions; ExperimentActions: typeof ExperimentActions; subject: string; session: number; @@ -72,9 +69,6 @@ export default class Clean extends Component { async componentDidMount() { const workspaceRawData = await readWorkspaceRawEEGData(this.props.title); - if (this.props.kernelStatus === KERNEL_STATUS.OFFLINE) { - this.props.JupyterActions.LaunchKernel(); - } this.setState({ subjects: workspaceRawData .map( @@ -116,7 +110,7 @@ export default class Clean extends Component { handleLoadData() { this.props.ExperimentActions.SetSubject(this.state.selectedSubject); - this.props.JupyterActions.LoadEpochs(this.state.selectedFilePaths); + this.props.PyodideActions.LoadEpochs(this.state.selectedFilePaths); } handleSidebarToggle() { @@ -232,13 +226,6 @@ export default class Clean extends Component { diff --git a/app/components/JupyterPlotWidget.tsx b/app/components/JupyterPlotWidget.tsx index 3c196792..ef490988 100644 --- a/app/components/JupyterPlotWidget.tsx +++ b/app/components/JupyterPlotWidget.tsx @@ -6,7 +6,7 @@ import { standardTransforms, } from '@nteract/transforms'; import { isNil } from 'lodash'; -import { storeJupyterImage } from '../utils/filesystem/storage'; +import { storePyodideImage } from '../utils/filesystem/storage'; interface Props { title: string; @@ -24,7 +24,7 @@ interface State { mimeType: string; } -export default class JupyterPlotWidget extends Component { +export default class PyodidePlotWidget extends Component { // state: State; constructor(props: Props) { super(props); @@ -55,8 +55,8 @@ export default class JupyterPlotWidget extends Component { } handleSave() { - const buf = Buffer.from(this.state.rawData, 'base64'); - storeJupyterImage(this.props.title, this.props.imageTitle, buf); + const buf = Buffer.from(this.state.rawData, "base64"); + storePyodideImage(this.props.title, this.props.imageTitle, buf); } renderResults() { diff --git a/app/containers/AnalyzeContainer.ts b/app/containers/AnalyzeContainer.ts index 9d8ad30a..38d3f1e5 100644 --- a/app/containers/AnalyzeContainer.ts +++ b/app/containers/AnalyzeContainer.ts @@ -1,7 +1,7 @@ import { bindActionCreators } from 'redux'; import { connect } from 'react-redux'; import Analyze from '../components/AnalyzeComponent'; -import { JupyterActions, ExperimentActions } from '../actions'; +import { PyodideActions, ExperimentActions } from '../actions'; function mapStateToProps(state) { return { @@ -9,14 +9,14 @@ function mapStateToProps(state) { type: state.experiment.type, deviceType: state.device.deviceType, isEEGEnabled: state.experiment.isEEGEnabled, - ...state.jupyter, + ...state.pyodide }; } function mapDispatchToProps(dispatch) { return { ExperimentActions: bindActionCreators(ExperimentActions, dispatch), - JupyterActions: bindActionCreators(JupyterActions, dispatch), + PyodideActions: bindActionCreators(PyodideActions, dispatch), }; } diff --git a/app/containers/CleanContainer.ts b/app/containers/CleanContainer.ts index 0c96106d..f9d83ed1 100644 --- a/app/containers/CleanContainer.ts +++ b/app/containers/CleanContainer.ts @@ -1,7 +1,7 @@ import { bindActionCreators } from 'redux'; import { connect } from 'react-redux'; import CleanComponent from '../components/CleanComponent'; -import { JupyterActions, ExperimentActions } from '../actions'; +import { PyodideActions, ExperimentActions } from '../actions'; function mapStateToProps(state) { return { @@ -11,14 +11,14 @@ function mapStateToProps(state) { group: state.experiment.group, session: state.experiment.session, deviceType: state.device.deviceType, - ...state.jupyter, + ...state.pyodide }; } function mapDispatchToProps(dispatch) { return { ExperimentActions: bindActionCreators(ExperimentActions, dispatch), - JupyterActions: bindActionCreators(JupyterActions, dispatch), + PyodideActions: bindActionCreators(PyodideActions, dispatch), }; } diff --git a/app/containers/HomeContainer.ts b/app/containers/HomeContainer.ts index ee15803e..c57ba8f6 100644 --- a/app/containers/HomeContainer.ts +++ b/app/containers/HomeContainer.ts @@ -1,21 +1,16 @@ import { connect } from 'react-redux'; import { bindActionCreators } from 'redux'; import Home from '../components/HomeComponent'; -import { DeviceActions, ExperimentActions, JupyterActions } from '../actions'; - -function mapStateToProps(state) { - return { - ...state.device, - kernelStatus: state.jupyter.kernelStatus, - }; -} +import { DeviceActions, ExperimentActions, PyodideActions } from '../actions'; function mapDispatchToProps(dispatch) { return { DeviceActions: bindActionCreators(DeviceActions, dispatch), - JupyterActions: bindActionCreators(JupyterActions, dispatch), + PyodideActions: bindActionCreators(PyodideActions, dispatch), ExperimentActions: bindActionCreators(ExperimentActions, dispatch), }; } -export default connect(mapStateToProps, mapDispatchToProps)(Home); +export default connect( + mapDispatchToProps +)(Home); diff --git a/app/epics/index.ts b/app/epics/index.ts index bd28ed8a..c49f309b 100644 --- a/app/epics/index.ts +++ b/app/epics/index.ts @@ -1,6 +1,6 @@ import { combineEpics } from 'redux-observable'; -import jupyter from './jupyterEpics'; +import pyodide from './pyodideEpics'; import device from './deviceEpics'; import experiment from './experimentEpics'; -export default combineEpics(device, experiment, jupyter); +export default combineEpics(device, experiment, pyodide); diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index 781966f9..64f23111 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -48,9 +48,6 @@ import { export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; -export const SET_KERNEL = 'SET_KERNEL'; -export const SET_KERNEL_STATUS = 'SET_KERNEL_STATUS'; -export const SET_KERNEL_INFO = 'SET_KERNEL_INFO'; export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; @@ -69,22 +66,7 @@ const getEpochsInfo = (payload) => ({ payload, type: GET_EPOCHS_INFO }); const getChannelInfo = () => ({ type: GET_CHANNEL_INFO }); -const setKernel = (payload) => ({ - payload, - type: SET_KERNEL, -}); - -const setKernelStatus = (payload) => ({ - payload, - type: SET_KERNEL_STATUS, -}); - -const setKernelInfo = (payload) => ({ - payload, - type: SET_KERNEL_INFO, -}); - -const setMainChannel = (payload) => ({ +const setMainChannel = payload => ({ payload, type: SET_MAIN_CHANNEL, }); @@ -140,9 +122,11 @@ const receiveStream = (payload) => ({ const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck('payload'), - filter((filePathsArray) => filePathsArray.length >= 1), - map((filePathsArray) => - state$.value.jupyter.mainChannel.next(executeRequest(loadCSV(filePathsArray))) + filter(filePathsArray => filePathsArray.length >= 1), + map(filePathsArray => + state$.value.pyodide.mainChannel.next( + executeRequest(loadCSV(filePathsArray)) + ) ), awaitOkMessage(action$), execute(filterIIR(1, 30), state$), @@ -159,9 +143,8 @@ const loadEpochsEpic = (action$, state$) => 0.8 ) ), - tap((e)=> {console.log('e', e)}), - map((epochEventsCommand) => - state$.value.jupyter.mainChannel.next(executeRequest(epochEventsCommand)) + map(epochEventsCommand => + state$.value.pyodide.mainChannel.next(executeRequest(epochEventsCommand)) ), awaitOkMessage(action$), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) @@ -170,9 +153,11 @@ const loadEpochsEpic = (action$, state$) => const loadCleanedEpochsEpic = (action$, state$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( pluck('payload'), - filter((filePathsArray) => filePathsArray.length >= 1), - map((filePathsArray) => - state$.value.jupyter.mainChannel.next(executeRequest(loadCleanedEpochs(filePathsArray))) + filter(filePathsArray => filePathsArray.length >= 1), + map(filePathsArray => + state$.value.pyodide.mainChannel.next( + executeRequest(loadCleanedEpochs(filePathsArray)) + ) ), awaitOkMessage(action$), mergeMap(() => @@ -197,7 +182,7 @@ const cleanEpochsEpic = (action$, state$) => ) ), map(() => - state$.value.jupyter.mainChannel.next( + state$.value.pyodide.mainChannel.next( executeRequest( saveEpochs( getWorkspaceDir(state$.value.experiment.title), @@ -213,8 +198,10 @@ const cleanEpochsEpic = (action$, state$) => const getEpochsInfoEpic = (action$, state$) => action$.ofType(GET_EPOCHS_INFO).pipe( pluck('payload'), - map((variableName) => - state$.value.jupyter.mainChannel.next(executeRequest(requestEpochsInfo(variableName))) + map(variableName => + state$.value.pyodide.mainChannel.next( + executeRequest(requestEpochsInfo(variableName)) + ) ), mergeMap(() => action$.ofType(RECEIVE_EXECUTE_RESULT).pipe( @@ -293,8 +280,10 @@ const loadERPEpic = (action$, state$) => console.warn('channel name supplied to loadERPEpic does not belong to either device'); return EMOTIV_CHANNELS[0]; }), - map((channelIndex) => - state$.value.jupyter.mainChannel.next(executeRequest(plotERP(channelIndex))) + map(channelIndex => + state$.value.pyodide.mainChannel.next( + executeRequest(plotERP(channelIndex)) + ) ), mergeMap(() => action$.ofType(RECEIVE_DISPLAY_DATA).pipe( @@ -308,19 +297,9 @@ const loadERPEpic = (action$, state$) => map(setERPPlot) ); -const closeKernelEpic = (action$, state$) => - action$.ofType(CLOSE_KERNEL).pipe( - map(() => { - state$.value.jupyter.kernel.spawn.kill(); - state$.value.jupyter.mainChannel.complete(); - }), - ignoreElements() - ); - export default combineEpics( launchEpic, setUpChannelEpic, - requestKernelInfoEpic, receiveChannelMessageEpic, loadEpochsEpic, loadCleanedEpochsEpic, @@ -330,5 +309,4 @@ export default combineEpics( loadPSDEpic, loadTopoEpic, loadERPEpic, - closeKernelEpic ); diff --git a/app/reducers/index.js b/app/reducers/index.js new file mode 100644 index 00000000..6c7920aa --- /dev/null +++ b/app/reducers/index.js @@ -0,0 +1,15 @@ +// @flow +import { combineReducers } from 'redux'; +import { routerReducer as router } from 'react-router-redux'; +import pyodide from './pyodideReducer'; +import device from './deviceReducer'; +import experiment from './experimentReducer'; + +const rootReducer = combineReducers({ + pyodide, + device, + experiment, + router, +}); + +export default rootReducer; diff --git a/app/reducers/pyodideReducer.js b/app/reducers/pyodideReducer.js index fa395798..bb70eba5 100644 --- a/app/reducers/pyodideReducer.js +++ b/app/reducers/pyodideReducer.js @@ -1,9 +1,6 @@ // @flow import { - SET_KERNEL, - SET_KERNEL_STATUS, SET_MAIN_CHANNEL, - SET_KERNEL_INFO, SET_EPOCH_INFO, SET_CHANNEL_INFO, SET_PSD_PLOT, @@ -11,13 +8,10 @@ import { SET_ERP_PLOT, RECEIVE_EXECUTE_RETURN } from '../epics/pyodideEpics'; -import { ActionType, Kernel } from '../constants/interfaces'; -import { KERNEL_STATUS } from '../constants/constants'; +import { ActionType } from '../constants/interfaces'; import { EXPERIMENT_CLEANUP } from '../epics/experimentEpics'; -export interface JupyterStateType { - +kernel: ?Kernel; - +kernelStatus: KERNEL_STATUS; +export interface PyodideStateType { +mainChannel: ?any; +epochsInfo: ?Array<{ [string]: number | string }>; +channelInfo: ?Array; @@ -27,8 +21,6 @@ export interface JupyterStateType { } const initialState = { - kernel: null, - kernelStatus: KERNEL_STATUS.OFFLINE, mainChannel: null, epochsInfo: null, channelInfo: [], @@ -37,29 +29,17 @@ const initialState = { erpPlot: null, }; -export default function jupyter(state: JupyterStateType = initialState, action: ActionType) { +export default function pyodide( + state: PyodideStateType = initialState, + action: ActionType +) { switch (action.type) { - case SET_KERNEL: - return { - ...state, - kernel: action.payload, - }; - - case SET_KERNEL_STATUS: - return { - ...state, - kernelStatus: action.payload, - }; - case SET_MAIN_CHANNEL: return { ...state, mainChannel: action.payload, }; - case SET_KERNEL_INFO: - return state; - case SET_EPOCH_INFO: return { ...state, diff --git a/app/store/configureStore.dev.js b/app/store/configureStore.dev.js new file mode 100644 index 00000000..7b3cd13a --- /dev/null +++ b/app/store/configureStore.dev.js @@ -0,0 +1,77 @@ +import { createStore, applyMiddleware, compose } from 'redux'; +import thunk from 'redux-thunk'; +import { createEpicMiddleware } from 'redux-observable'; +import { createHashHistory } from 'history'; +import { routerMiddleware, routerActions } from 'react-router-redux'; +import { createLogger } from 'redux-logger'; +import rootReducer from '../reducers'; +import rootEpic from '../epics'; +import * as pyodideActions from '../actions/pyodideActions'; +import * as deviceActions from '../actions/deviceActions'; + +const history = createHashHistory(); + +const configureStore = (initialState?: AppState) => { + // Redux Configuration + const middleware = []; + const enhancers = []; + + // Thunk Middleware + middleware.push(thunk); + + // Redux Observable (Epic) Middleware + const epicMiddleware = createEpicMiddleware(); + middleware.push(epicMiddleware); + + // Logging Middleware + const logger = createLogger({ + level: 'info', + collapsed: true, + }); + + // Skip redux logs in console during the tests + if (process.env.NODE_ENV !== 'test') { + middleware.push(logger); + } + + // Router Middleware + const router = routerMiddleware(history); + middleware.push(router); + + // Redux DevTools Configuration + const actionCreators = { + ...deviceActions, + ...pyodideActions, + ...routerActions + }; + // If Redux DevTools Extension is installed use it, otherwise use Redux compose + /* eslint-disable no-underscore-dangle */ + const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ + ? window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__({ + // Options: http://zalmoxisus.github.io/redux-devtools-extension/API/Arguments.html + actionCreators, + }) + : compose; + /* eslint-enable no-underscore-dangle */ + + // Apply Middleware & Compose Enhancers + enhancers.push(applyMiddleware(...middleware)); + const enhancer = composeEnhancers(...enhancers); + + // Create Store + const store = createStore(rootReducer, initialState, enhancer); + + if (module.hot) { + module.hot.accept( + '../reducers', + () => store.replaceReducer(require('../reducers')) // eslint-disable-line global-require + ); + } + + // Redux Observable + epicMiddleware.run(rootEpic); + + return store; +}; + +export default { configureStore, history }; diff --git a/app/utils/filesystem/storage.ts b/app/utils/filesystem/storage.ts index 7ad4407d..fbf791cb 100644 --- a/app/utils/filesystem/storage.ts +++ b/app/utils/filesystem/storage.ts @@ -75,7 +75,7 @@ export const storeBehavioralData = ( }; // Stores an image to workspace dir -export const storeJupyterImage = ( +export const storePyodideImage = ( title: string, imageTitle: string, rawData: Buffer diff --git a/app/utils/jupyter/cells.ts b/app/utils/jupyter/cells.ts index f4424ac6..d81a17fe 100644 --- a/app/utils/jupyter/cells.ts +++ b/app/utils/jupyter/cells.ts @@ -91,6 +91,7 @@ export const plotTopoMap = () => export const plotERP = (channelIndex: number | string) => [ `%matplotlib inline`, + `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions`, `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, ci=97.5, n_boot=1000, title='', diff_waveform=None)`, ].join('\n'); diff --git a/app/utils/jupyter/functions.ts b/app/utils/jupyter/functions.ts index 95f73baa..58f7ab87 100644 --- a/app/utils/jupyter/functions.ts +++ b/app/utils/jupyter/functions.ts @@ -1,20 +1,6 @@ -import { KERNEL_STATUS } from '../../constants/constants'; - export const parseSingleQuoteJSON = (string: string) => JSON.parse(string.replace(/'/g, '"')); -export const parseKernelStatus = (msg: Record) => { - switch (msg.content.execution_state) { - case 'busy': - return KERNEL_STATUS.BUSY; - case 'idle': - return KERNEL_STATUS.IDLE; - case 'starting': - default: - return KERNEL_STATUS.STARTING; - } -}; - export const debugParseMessage = (msg: Record) => { let content = ''; switch (msg.channel) { diff --git a/app/utils/jupyter/pipes.ts b/app/utils/jupyter/pipes.ts index e74cf285..efcf94eb 100644 --- a/app/utils/jupyter/pipes.ts +++ b/app/utils/jupyter/pipes.ts @@ -1,18 +1,19 @@ import { pipe } from 'rxjs'; import { map, pluck, filter, take, mergeMap } from 'rxjs/operators'; import { executeRequest } from '@nteract/messaging'; -import { JupyterActions } from '../../actions'; +import { PyodideActions } from '../../actions'; +import { RECEIVE_EXECUTE_REPLY } from '../../epics/pyodideEpics'; // Refactor this so command can be calculated either up stream or inside pipe export const execute = (command, state$) => pipe( - map(() => state$.value.jupyter.mainChannel.next(executeRequest(command))) + map(() => state$.value.pyodide.mainChannel.next(executeRequest(command))) ); export const awaitOkMessage = (action$) => pipe( mergeMap(() => - action$.ofType(JupyterActions.ReceiveExecuteReply.type).pipe( + action$.ofType(PyodideActions.ReceiveExecuteReply.type).pipe( pluck('payload'), filter( (msg) => msg.channel === 'shell' && msg.content.status === 'ok' From 7c2c30b616ed366745a9471981729d3f567f9c35 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sun, 26 May 2019 11:54:07 -0400 Subject: [PATCH 10/40] Switched Cortex wrapper to debug verbosity --- app/utils/eeg/emotiv.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/utils/eeg/emotiv.ts b/app/utils/eeg/emotiv.ts index 9bfbd46a..06de6160 100644 --- a/app/utils/eeg/emotiv.ts +++ b/app/utils/eeg/emotiv.ts @@ -26,7 +26,8 @@ interface EmotivHeadset { } // Creates the Cortex object from SDK -const verbose = process.env.LOG_LEVEL || 1; +const verbose = 3; +// const verbose = process.env.LOG_LEVEL || 1; const options = { verbose }; // This global client is used in every Cortex API call From 73459a06eb73cdfdf9bd7ce62ee989683ce48cf3 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sun, 26 May 2019 12:24:27 -0400 Subject: [PATCH 11/40] Speculatively updated epics for pyodide --- app/actions/pyodideActions.js | 7 ++ app/components/HomeComponent/index.tsx | 5 +- app/epics/pyodideEpics.js | 123 ++++++------------------- 3 files changed, 38 insertions(+), 97 deletions(-) diff --git a/app/actions/pyodideActions.js b/app/actions/pyodideActions.js index 4fe1b3ba..f4c2efdb 100644 --- a/app/actions/pyodideActions.js +++ b/app/actions/pyodideActions.js @@ -1,6 +1,7 @@ // ------------------------------------------------------------------------- // Action Types +export const LAUNCH = "LAUNCH"; export const SEND_EXECUTE_REQUEST = "SEND_EXECUTE_REQUEST"; export const LOAD_EPOCHS = "LOAD_EPOCHS"; export const LOAD_CLEANED_EPOCHS = "LOAD_CLEANED_EPOCHS"; @@ -12,6 +13,12 @@ export const CLEAN_EPOCHS = "CLEAN_EPOCHS"; // ------------------------------------------------------------------------- // Actions + +export const launch = () => ({ + type: LAUNCH +}); + + export const sendExecuteRequest = (payload: string) => ({ payload, type: SEND_EXECUTE_REQUEST, diff --git a/app/components/HomeComponent/index.tsx b/app/components/HomeComponent/index.tsx index d1d18884..5ab3e0b9 100644 --- a/app/components/HomeComponent/index.tsx +++ b/app/components/HomeComponent/index.tsx @@ -29,7 +29,7 @@ import { deleteWorkspaceDir, } from '../../utils/filesystem/storage'; import { - JupyterActions, + PyodideActions, DeviceActions, ExperimentActions, } from '../../actions'; @@ -64,7 +64,7 @@ export interface Props { deviceType: DEVICES; ExperimentActions: typeof ExperimentActions; history: History; - JupyterActions: typeof JupyterActions; + PyodideActions: typeof PyodideActions; kernelStatus: KERNEL_STATUS; signalQualityObservable?: Observable; } @@ -98,6 +98,7 @@ export default class Home extends Component { } componentDidMount() { + this.props.pyodideActions.launch(); this.setState({ recentWorkspaces: readWorkspaces() }); } diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index 64f23111..f902b70b 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -12,7 +12,9 @@ import { import { isNil } from 'lodash'; import { toast } from 'react-toastify'; import { getWorkspaceDir } from '../utils/filesystem/storage'; +import { languagePluginLoader } from '../../utils/pyodide/pyodide'; import { + LAUNCH, LOAD_EPOCHS, LOAD_CLEANED_EPOCHS, LOAD_PSD, @@ -45,7 +47,6 @@ import { PYODIDE_VARIABLE_NAMES } from '../constants/constants'; - export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; @@ -119,18 +120,18 @@ const receiveStream = (payload) => ({ // ------------------------------------------------------------------------- // Epics +const launchEpic = (action$, state$) => + action$.ofType(LAUNCH).pipe( + mergeMap(languagePluginLoader), + tap(() => console.log('launched pyodide')) + ); + const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck('payload'), filter(filePathsArray => filePathsArray.length >= 1), - map(filePathsArray => - state$.value.pyodide.mainChannel.next( - executeRequest(loadCSV(filePathsArray)) - ) - ), - awaitOkMessage(action$), - execute(filterIIR(1, 30), state$), - awaitOkMessage(action$), + map(filePathsArray => loadCSV(filePathsArray)), + map(() => filterIIR(1, 30)), map(() => epochEvents( { @@ -143,10 +144,7 @@ const loadEpochsEpic = (action$, state$) => 0.8 ) ), - map(epochEventsCommand => - state$.value.pyodide.mainChannel.next(executeRequest(epochEventsCommand)) - ), - awaitOkMessage(action$), + map(epochEventsCommand => epochEventsCommand), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); @@ -154,12 +152,7 @@ const loadCleanedEpochsEpic = (action$, state$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( pluck('payload'), filter(filePathsArray => filePathsArray.length >= 1), - map(filePathsArray => - state$.value.pyodide.mainChannel.next( - executeRequest(loadCleanedEpochs(filePathsArray)) - ) - ), - awaitOkMessage(action$), + map(filePathsArray => loadCleanedEpochs(filePathsArray)), mergeMap(() => of( getEpochsInfo(PYODIDE_VARIABLE_NAMES.CLEAN_EPOCHS), @@ -171,49 +164,22 @@ const loadCleanedEpochsEpic = (action$, state$) => const cleanEpochsEpic = (action$, state$) => action$.ofType(CLEAN_EPOCHS).pipe( - execute(cleanEpochsPlot(), state$), - mergeMap(() => - action$.ofType(RECEIVE_STREAM).pipe( - pluck('payload'), - filter( - (msg) => msg.channel === 'iopub' && msg.content.text.includes('Channels marked as bad') - ), - take(1) - ) - ), + map(cleanEpochsPlot), map(() => - state$.value.pyodide.mainChannel.next( - executeRequest( - saveEpochs( - getWorkspaceDir(state$.value.experiment.title), - state$.value.experiment.subject - ) - ) + saveEpochs( + getWorkspaceDir(state$.value.experiment.title), + state$.value.experiment.subject ) ), - awaitOkMessage(action$), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); const getEpochsInfoEpic = (action$, state$) => action$.ofType(GET_EPOCHS_INFO).pipe( pluck('payload'), - map(variableName => - state$.value.pyodide.mainChannel.next( - executeRequest(requestEpochsInfo(variableName)) - ) - ), - mergeMap(() => - action$.ofType(RECEIVE_EXECUTE_RESULT).pipe( - pluck('payload'), - filter((msg) => msg.channel === 'iopub' && !isNil(msg.content.data)), - pluck('content', 'data', 'text/plain'), - filter((msg) => msg.includes('Drop Percentage')), - take(1) - ) - ), - map((epochInfoString) => - parseSingleQuoteJSON(epochInfoString).map((infoObj) => ({ + map(variableName => requestEpochsInfo(variableName)), + map(epochInfoString => + parseSingleQuoteJSON(epochInfoString).map(infoObj => ({ name: Object.keys(infoObj)[0], value: infoObj[Object.keys(infoObj)[0]], })) @@ -223,42 +189,22 @@ const getEpochsInfoEpic = (action$, state$) => const getChannelInfoEpic = (action$, state$) => action$.ofType(GET_CHANNEL_INFO).pipe( - execute(requestChannelInfo(), state$), - mergeMap(() => - action$.ofType(RECEIVE_EXECUTE_RESULT).pipe( - pluck('payload'), - filter((msg) => msg.channel === 'iopub' && !isNil(msg.content.data)), - pluck('content', 'data', 'text/plain'), - // Filter to prevent this from reading requestEpochsInfo returns - filter((msg) => !msg.includes('Drop Percentage')), - take(1) - ) - ), - map((channelInfoString) => setChannelInfo(parseSingleQuoteJSON(channelInfoString))) + map(requestChannelInfo), + map(channelInfoString => + setChannelInfo(parseSingleQuoteJSON(channelInfoString)) + ) ); const loadPSDEpic = (action$, state$) => action$.ofType(LOAD_PSD).pipe( - execute(plotPSD(), state$), - mergeMap(() => - action$.ofType(RECEIVE_DISPLAY_DATA).pipe( - pluck('payload'), - // PSD graphs should have two axes - filter((msg) => msg.content.data['text/plain'].includes('2 Axes')), - pluck('content', 'data'), - take(1) - ) - ), + map(plotPSD), map(setPSDPlot) ); const loadTopoEpic = (action$, state$) => action$.ofType(LOAD_TOPO).pipe( - execute(plotTopoMap(), state$), - mergeMap(() => - action$.ofType(RECEIVE_DISPLAY_DATA).pipe(pluck('payload'), pluck('content', 'data'), take(1)) - ), - mergeMap((topoPlot) => + map(plotTopoMap), + mergeMap(topoPlot => of( setTopoPlot(topoPlot), loadERP( @@ -280,20 +226,7 @@ const loadERPEpic = (action$, state$) => console.warn('channel name supplied to loadERPEpic does not belong to either device'); return EMOTIV_CHANNELS[0]; }), - map(channelIndex => - state$.value.pyodide.mainChannel.next( - executeRequest(plotERP(channelIndex)) - ) - ), - mergeMap(() => - action$.ofType(RECEIVE_DISPLAY_DATA).pipe( - pluck('payload'), - // ERP graphs should have 1 axis according to MNE - filter((msg) => msg.content.data['text/plain'].includes('1 Axes')), - pluck('content', 'data'), - take(1) - ) - ), + map(channelIndex => plotERP(channelIndex)), map(setERPPlot) ); @@ -308,5 +241,5 @@ export default combineEpics( getChannelInfoEpic, loadPSDEpic, loadTopoEpic, - loadERPEpic, + loadERPEpic ); From 83418cf6ab4c99f8499ad6ed1a4efb630d89aa11 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sun, 16 Jun 2019 20:17:11 -0400 Subject: [PATCH 12/40] Added node pyodide install script --- internals/scripts/InstallPyodide.js | 65 +++++++++++++++++++++++++++++ package.json | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 internals/scripts/InstallPyodide.js diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js new file mode 100644 index 00000000..6f1bd011 --- /dev/null +++ b/internals/scripts/InstallPyodide.js @@ -0,0 +1,65 @@ +//mkdir app/utils/pyodide/src +// && cd app/utils/pyodide/src +// curl -LJO https://github.com/iodide-project/pyodide/releases/download/0.12.0/pyodide-build-0.12.0.tar.bz2 +// tar xjf pyodide-build-0.12.0.tar.bz2 +// rm pyodide-build-0.12.0.tar.bz2", + +import chalk from "chalk"; +import os from "os"; +import fs from "fs"; +import https from "https"; +import mkdirp from "mkdirp"; +import tar from "tar-fs"; +import url from "url"; +import gunzip from "gunzip-maybe"; + +const PYODIDE_VERSION = "0.12.0"; +const TAR_NAME = `pyodide-build-${PYODIDE_VERSION}.tar.bz2`; +const PYODIDE_DIR = "app/utils/pyodide/src/"; + +const writeAndUnzipFile = response => { + const filePath = `${PYODIDE_DIR}${TAR_NAME}`; + const writeStream = fs.createWriteStream(filePath); + response.pipe(writeStream); + + writeStream.on("finish", () => { + console.log(`${chalk.green.bold(`Unzipping pyodide`)}`); + + const readStream = fs.createReadStream(filePath); + readStream.pipe(gunzip()).pipe(tar.extract(PYODIDE_DIR)); + + readStream.on("end", () => { + console.log(`${chalk.green.bold(`Unzip successful`)}`); + }); + }); +}; + +(() => { + console.log( + `${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}` + ); + + mkdirp.sync(`app/utils/pyodide/src`); + + https.get( + `https://github.com/iodide-project/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-build-${PYODIDE_VERSION}.tar.bz2`, + response => { + if ( + response.statusCode > 300 && + response.statusCode < 400 && + response.headers.location + ) { + if (url.parse(response.headers.location).hostname) { + https.get(response.headers.location, writeAndUnzipFile); + } else { + https.get( + url.resolve(url.parse(TAR_URL).hostname, response.headers.location), + writeToFile + ); + } + } else { + writeAndUnzipFile(response); + } + } + ); +})(); diff --git a/package.json b/package.json index 2ff2a541..43a5aed8 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "package-mac": "yarn build && electron-builder build --mac", "package-linux": "yarn build && electron-builder build --linux", "package-win": "yarn build && electron-builder build --win --x64", - "postinstall": "node -r @babel/register internals/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn build-dll && opencollective-postinstall && mkdir app/utils/pyodide/src && cd app/utils/pyodide/src && curl -LJO https://github.com/iodide-project/pyodide/releases/download/0.12.0/pyodide-build-0.12.0.tar.bz2 && tar xjf pyodide-build-0.12.0.tar.bz2 && rm pyodide-build-0.12.0.tar.bz2", + "postinstall": "node -r @babel/register internals/scripts/CheckNativeDep.js && electron-builder install-app-deps && yarn build-dll && opencollective-postinstall && node -r babel-register internals/scripts/InstallPyodide.js", "postlint-fix": "prettier --ignore-path .eslintignore --single-quote --write '**/*.{js,jsx,json,html,css,less,scss,yml}'", "postlint-styles-fix": "prettier --ignore-path .eslintignore --single-quote --write '**/*.{css,scss}'", "preinstall": "node ./internals/scripts/CheckYarn.js", From ffd5fab38f0ba5267cbcd2db466aa7a508642c9a Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Mon, 17 Jun 2019 23:10:38 -0400 Subject: [PATCH 13/40] Added loading of pyodide within app epics --- app/constants/constants.ts | 9 +- app/containers/HomeContainer.js | 20 ++++ app/epics/pyodideEpics.js | 99 +++++++++++-------- app/reducers/pyodideReducer.js | 17 +++- .../pyodide/{commands.py => commands.js} | 45 ++++++--- 5 files changed, 124 insertions(+), 66 deletions(-) create mode 100644 app/containers/HomeContainer.js rename app/utils/pyodide/{commands.py => commands.js} (72%) diff --git a/app/constants/constants.ts b/app/constants/constants.ts index 4b573d4d..64dac915 100644 --- a/app/constants/constants.ts +++ b/app/constants/constants.ts @@ -37,20 +37,13 @@ export enum CONNECTION_STATUS { BLUETOOTH_DISABLED = 'BLUETOOTH_DISABLED', } -export enum KERNEL_STATUS { - OFFLINE = 'Offline', - BUSY = 'Busy', - IDLE = 'Idle', - STARTING = 'Starting', -} - export enum DEVICE_AVAILABILITY { NONE = 'NONE', SEARCHING = 'SEARCHING', AVAILABLE = 'AVAILABLE', } -// Names of variables in the pyodide kernel +// Names of variables in pyodide export enum PYODIDE_VARIABLE_NAMES { RAW_EPOCHS = 'raw_epochs', CLEAN_EPOCHS = 'clean_epochs', diff --git a/app/containers/HomeContainer.js b/app/containers/HomeContainer.js new file mode 100644 index 00000000..78791b99 --- /dev/null +++ b/app/containers/HomeContainer.js @@ -0,0 +1,20 @@ +// @flow +import { connect } from "react-redux"; +import { bindActionCreators } from "redux"; +import Home from "../components/HomeComponent"; +import * as deviceActions from "../actions/deviceActions"; +import * as pyodideActions from "../actions/pyodideActions"; +import * as experimentActions from "../actions/experimentActions"; + +function mapDispatchToProps(dispatch) { + return { + deviceActions: bindActionCreators(deviceActions, dispatch), + pyodideActions: bindActionCreators(pyodideActions, dispatch), + experimentActions: bindActionCreators(experimentActions, dispatch) + }; +} + +export default connect( + null, + mapDispatchToProps +)(Home); diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index f902b70b..b5b438a3 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -1,18 +1,15 @@ -import { combineEpics } from 'redux-observable'; -import { from, of } from 'rxjs'; +import { combineEpics } from "redux-observable"; +import { of } from "rxjs"; import { map, mergeMap, tap, pluck, ignoreElements, - filter, - take -} from 'rxjs/operators'; -import { isNil } from 'lodash'; -import { toast } from 'react-toastify'; -import { getWorkspaceDir } from '../utils/filesystem/storage'; -import { languagePluginLoader } from '../../utils/pyodide/pyodide'; + filter +} from "rxjs/operators"; +import { getWorkspaceDir } from "../utils/filesystem/storage"; +import { languagePluginLoader } from "../utils/pyodide/pyodide"; import { LAUNCH, LOAD_EPOCHS, @@ -23,10 +20,10 @@ import { CLEAN_EPOCHS, loadTopo, loadERP -} from '../actions/pyodideActions'; +} from "../actions/pyodideActions"; import { + test, imports, - utils, loadCSV, loadCleanedEpochs, filterIIR, @@ -38,27 +35,29 @@ import { plotERP, plotTopoMap, saveEpochs -} from '../utils/pyodide/commands'; +} from "../utils/pyodide/commands"; import { EMOTIV_CHANNELS, EVENTS, DEVICES, MUSE_CHANNELS, - PYODIDE_VARIABLE_NAMES -} from '../constants/constants'; - -export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; -export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; -export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; -export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; -export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; -export const SET_PSD_PLOT = 'SET_PSD_PLOT'; -export const SET_ERP_PLOT = 'SET_ERP_PLOT'; -export const SET_TOPO_PLOT = 'SET_TOPO_PLOT'; -export const RECEIVE_EXECUTE_REPLY = 'RECEIVE_EXECUTE_REPLY'; -export const RECEIVE_EXECUTE_RESULT = 'RECEIVE_EXECUTE_RESULT'; -export const RECEIVE_STREAM = 'RECEIVE_STREAM'; -export const RECEIVE_DISPLAY_DATA = 'RECEIVE_DISPLAY_DATA'; + PYODIDE_VARIABLE_NAMES, + PYODIDE_STATUS +} from "../constants/constants"; + +export const GET_EPOCHS_INFO = "GET_EPOCHS_INFO"; +export const GET_CHANNEL_INFO = "GET_CHANNEL_INFO"; +export const SET_MAIN_CHANNEL = "SET_MAIN_CHANNEL"; +export const SET_EPOCH_INFO = "SET_EPOCH_INFO"; +export const SET_CHANNEL_INFO = "SET_CHANNEL_INFO"; +export const SET_PSD_PLOT = "SET_PSD_PLOT"; +export const SET_ERP_PLOT = "SET_ERP_PLOT"; +export const SET_TOPO_PLOT = "SET_TOPO_PLOT"; +export const SET_PYODIDE_STATUS = "SET_PYODIDE_STATUS"; +export const RECEIVE_EXECUTE_REPLY = "RECEIVE_EXECUTE_REPLY"; +export const RECEIVE_EXECUTE_RESULT = "RECEIVE_EXECUTE_RESULT"; +export const RECEIVE_STREAM = "RECEIVE_STREAM"; +export const RECEIVE_DISPLAY_DATA = "RECEIVE_DISPLAY_DATA"; // ------------------------------------------------------------------------- // Action Creators @@ -67,12 +66,7 @@ const getEpochsInfo = (payload) => ({ payload, type: GET_EPOCHS_INFO }); const getChannelInfo = () => ({ type: GET_CHANNEL_INFO }); -const setMainChannel = payload => ({ - payload, - type: SET_MAIN_CHANNEL, -}); - -const setEpochInfo = (payload) => ({ +const setEpochInfo = payload => ({ payload, type: SET_EPOCH_INFO, }); @@ -97,7 +91,12 @@ const setERPPlot = (payload) => ({ type: SET_ERP_PLOT, }); -const receiveExecuteReply = (payload) => ({ +const setPyodideStatus = payload => ({ + payload, + type: SET_PYODIDE_STATUS +}); + +const receiveExecuteReply = payload => ({ payload, type: RECEIVE_EXECUTE_REPLY, }); @@ -120,15 +119,22 @@ const receiveStream = (payload) => ({ // ------------------------------------------------------------------------- // Epics -const launchEpic = (action$, state$) => +const launchEpic = action$ => action$.ofType(LAUNCH).pipe( - mergeMap(languagePluginLoader), - tap(() => console.log('launched pyodide')) + tap(() => console.log("launching")), + mergeMap(async () => { + await languagePluginLoader; + console.log("loaded language plugin"); + // using window.pyodide instead of pyodide to get linter to stop yelling ;) + await window.pyodide.loadPackage(["mne"]); + console.log("loaded mne package"); + }), + map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) ); const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( - pluck('payload'), + pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), map(filePathsArray => loadCSV(filePathsArray)), map(() => filterIIR(1, 30)), @@ -150,7 +156,7 @@ const loadEpochsEpic = (action$, state$) => const loadCleanedEpochsEpic = (action$, state$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( - pluck('payload'), + pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), map(filePathsArray => loadCleanedEpochs(filePathsArray)), mergeMap(() => @@ -176,7 +182,7 @@ const cleanEpochsEpic = (action$, state$) => const getEpochsInfoEpic = (action$, state$) => action$.ofType(GET_EPOCHS_INFO).pipe( - pluck('payload'), + pluck("payload"), map(variableName => requestEpochsInfo(variableName)), map(epochInfoString => parseSingleQuoteJSON(epochInfoString).map(infoObj => ({ @@ -216,14 +222,25 @@ const loadTopoEpic = (action$, state$) => const loadERPEpic = (action$, state$) => action$.ofType(LOAD_ERP).pipe( +<<<<<<< HEAD pluck('payload'), map((channelName) => { +======= + pluck("payload"), + map(channelName => { +>>>>>>> Added loading of pyodide within app epics if (MUSE_CHANNELS.includes(channelName)) { return MUSE_CHANNELS.indexOf(channelName); } else if (EMOTIV_CHANNELS.includes(channelName)) { return EMOTIV_CHANNELS.indexOf(channelName); } +<<<<<<< HEAD console.warn('channel name supplied to loadERPEpic does not belong to either device'); +======= + console.warn( + "channel name supplied to loadERPEpic does not belong to either device" + ); +>>>>>>> Added loading of pyodide within app epics return EMOTIV_CHANNELS[0]; }), map(channelIndex => plotERP(channelIndex)), @@ -232,8 +249,6 @@ const loadERPEpic = (action$, state$) => export default combineEpics( launchEpic, - setUpChannelEpic, - receiveChannelMessageEpic, loadEpochsEpic, loadCleanedEpochsEpic, cleanEpochsEpic, diff --git a/app/reducers/pyodideReducer.js b/app/reducers/pyodideReducer.js index bb70eba5..7c1e993c 100644 --- a/app/reducers/pyodideReducer.js +++ b/app/reducers/pyodideReducer.js @@ -6,10 +6,12 @@ import { SET_PSD_PLOT, SET_TOPO_PLOT, SET_ERP_PLOT, - RECEIVE_EXECUTE_RETURN -} from '../epics/pyodideEpics'; -import { ActionType } from '../constants/interfaces'; -import { EXPERIMENT_CLEANUP } from '../epics/experimentEpics'; + RECEIVE_EXECUTE_RETURN, + SET_PYODIDE_STATUS +} from "../epics/pyodideEpics"; +import { ActionType } from "../constants/interfaces"; +import { PYODIDE_STATUS } from "../constants/constants"; +import { EXPERIMENT_CLEANUP } from "../epics/experimentEpics"; export interface PyodideStateType { +mainChannel: ?any; @@ -27,6 +29,7 @@ const initialState = { psdPlot: null, topoPlot: null, erpPlot: null, + status: PYODIDE_STATUS.NOT_LOADED }; export default function pyodide( @@ -81,6 +84,12 @@ export default function pyodide( case RECEIVE_EXECUTE_RETURN: return state; + case SET_PYODIDE_STATUS: + return { + ...state, + status: action.payload + }; + default: return state; } diff --git a/app/utils/pyodide/commands.py b/app/utils/pyodide/commands.js similarity index 72% rename from app/utils/pyodide/commands.py rename to app/utils/pyodide/commands.js index 7c048286..b4c71a65 100644 --- a/app/utils/pyodide/commands.py +++ b/app/utils/pyodide/commands.js @@ -1,21 +1,42 @@ -import * as path from 'path'; -import { readFileSync } from 'fs'; +import * as path from "path"; +import { readFileSync } from "fs"; +let pyodide; // ----------------------------- // Imports and Utility functions +export const test = async () => { + await window.pyodide.loadPackage(["mne"]); + + const mneCommands = [ + `import numpy as np`, + `import mne`, + `data = np.repeat(np.atleast_2d(np.arange(1000)), 8, axis=0)`, + `info = mne.create_info(8, 250)`, + `raw = mne.io.RawArray(data=data, info=info)`, + `raw.save("test_brainwaves.fif")` + ]; + await window.pyodide.runPython(mneCommands.join("; ")); +}; + +export const loadPackages = async () => window.pyodide.loadPackage(["mne"]); + export const imports = () => - readFileSync(path.join(__dirname, '/utils/pyodide/pyimport.py'), 'utf8'); + pyodide.runPython( + readFileSync(path.join(__dirname, "/utils/pyodide/pyimport.py"), "utf8") + ); export const utils = () => - readFileSync(path.join(__dirname, '/utils/pyodide/utils.py'), 'utf8'); + pyodide.runPython( + readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8") + ); export const loadCSV = (filePathArray: Array) => [ `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, `replace_ch_names = None`, `raw = load_data(files, replace_ch_names)` - ].join('\n'); + ].join("\n"); // --------------------------- // MNE-Related Data Processing @@ -24,7 +45,7 @@ `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` - ].join('\n'); + ].join("\n"); // NOTE: this command includes a ';' to prevent returning data export const filterIIR = (lowCutoff: number, highCutoff: number) => @@ -34,7 +55,7 @@ eventIDs: { [string]: number }, tmin: number, tmax: number, - reject?: Array | string = 'None' + reject?: Array | string = "None" ) => { const command = [ `event_id = ${JSON.stringify(eventIDs)}`, @@ -43,12 +64,12 @@ `baseline= (tmin, tmax)`, `picks = None`, `reject = ${reject}`, - 'events = find_events(raw)', + "events = find_events(raw)", `raw_epochs = Epochs(raw, events=events, event_id=event_id, tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, verbose=False, picks=picks)`, `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` - ].join('\n'); + ].join("\n"); return command; }; @@ -76,9 +97,9 @@ `raw_epochs.save(${formatFilePath( path.join( workspaceDir, - 'Data', + "Data", subject, - 'EEG', + "EEG", `${subject}-cleaned-epo.fif` ) )})`; @@ -87,4 +108,4 @@ // Helper methods const formatFilePath = (filePath: string) => - `"${filePath.replace(/\\/g, '/')}"`; + `"${filePath.replace(/\\/g, "/")}"`; From d5145e13a29a1bb909e1822560920b48e7d9f324 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Tue, 18 Jun 2019 21:54:31 -0700 Subject: [PATCH 14/40] moving files --- app/utils/jupyter/utils.py | 175 ------------------ .../{jupyter/cells.ts => pyodide/cell.js} | 0 .../functions.ts => pyodide/functions.js} | 0 .../{jupyter/pipes.ts => pyodide/pipes.js} | 0 4 files changed, 175 deletions(-) delete mode 100644 app/utils/jupyter/utils.py rename app/utils/{jupyter/cells.ts => pyodide/cell.js} (100%) rename app/utils/{jupyter/functions.ts => pyodide/functions.js} (100%) rename app/utils/{jupyter/pipes.ts => pyodide/pipes.js} (100%) diff --git a/app/utils/jupyter/utils.py b/app/utils/jupyter/utils.py deleted file mode 100644 index c09aa583..00000000 --- a/app/utils/jupyter/utils.py +++ /dev/null @@ -1,175 +0,0 @@ -from glob import glob -import os -from collections import OrderedDict -from mne import create_info, concatenate_raws, viz -from mne.io import RawArray -from mne.channels import read_montage -import pandas as pd -import numpy as np -import seaborn as sns -from matplotlib import pyplot as plt - -sns.set_context('talk') -sns.set_style('white') - - -def load_data(fnames, sfreq=128., replace_ch_names=None): - """Load CSV files from the /data directory into a Raw object. - - Args: - fnames (array): CSV filepaths from which to load data - - Keyword Args: - sfreq (float): EEG sampling frequency - replace_ch_names (dict or None): dictionary containing a mapping to - rename channels. Useful when an external electrode was used. - - Returns: - (mne.io.array.array.RawArray): loaded EEG - """ - - raw = [] - print(fnames) - for fname in fnames: - # read the file - data = pd.read_csv(fname, index_col=0) - - data = data.dropna() - - # get estimation of sampling rate and use to determine sfreq - # yes, this could probably be improved - srate = 1000 / (data.index.values[1] - data.index.values[0]) - if srate >= 200: - sfreq = 256 - else: - sfreq = 128 - - # name of each channel - ch_names = list(data.columns) - - # indices of each channel - ch_ind = list(range(len(ch_names))) - - if replace_ch_names is not None: - ch_names = [c if c not in replace_ch_names.keys() - else replace_ch_names[c] for c in ch_names] - - # type of each channels - ch_types = ['eeg'] * (len(ch_ind) - 1) + ['stim'] - montage = read_montage('standard_1005') - - # get data and exclude Aux channel - data = data.values[:, ch_ind].T - - # create MNE object - info = create_info(ch_names=ch_names, ch_types=ch_types, - sfreq=sfreq, montage=montage) - raw.append(RawArray(data=data, info=info)) - - # concatenate all raw objects - raws = concatenate_raws(raw) - - return raws - - -def plot_topo(epochs, conditions=OrderedDict()): - palette = sns.color_palette("hls", len(conditions) + 1) - evokeds = [epochs[name].average() for name in (conditions)] - - evoked_topo = viz.plot_evoked_topo( - evokeds, vline=None, color=palette[0:len(conditions)], show=False) - evoked_topo.patch.set_alpha(0) - evoked_topo.set_size_inches(10, 8) - for axis in evoked_topo.axes: - for line in axis.lines: - line.set_linewidth(2) - - legend_loc = 0 - labels = [e.comment if e.comment else 'Unknown' for e in evokeds] - legend = plt.legend(labels, loc=legend_loc, prop={'size': 20}) - txts = legend.get_texts() - for txt, col in zip(txts, palette): - txt.set_color(col) - - return evoked_topo - - -def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, n_boot=1000, - title='', palette=None, - diff_waveform=(4, 3)): - """Plot Averaged Epochs with ERP conditions. - - Args: - epochs (mne.epochs): EEG epochs - - Keyword Args: - conditions (OrderedDict): dictionary that contains the names of the - conditions to plot as keys, and the list of corresponding marker - numbers as value. E.g., - - conditions = {'Non-target': [0, 1], - 'Target': [2, 3, 4]} - - ch_ind (int): index of channel to plot data from - ci (float): confidence interval in range [0, 100] - n_boot (int): number of bootstrap samples - title (str): title of the figure - palette (list): color palette to use for conditions - ylim (tuple): (ymin, ymax) - diff_waveform (tuple or None): tuple of ints indicating which - conditions to subtract for producing the difference waveform. - If None, do not plot a difference waveform - - Returns: - (matplotlib.figure.Figure): figure object - (list of matplotlib.axes._subplots.AxesSubplot): list of axes - """ - if isinstance(conditions, dict): - conditions = OrderedDict(conditions) - - if palette is None: - palette = sns.color_palette("hls", len(conditions) + 1) - - X = epochs.get_data() - times = epochs.times - y = pd.Series(epochs.events[:, -1]) - fig, ax = plt.subplots() - - for cond, color in zip(conditions.values(), palette): - sns.tsplot(X[y.isin(cond), ch_ind], time=times, color=color, - n_boot=n_boot, ci=ci) - - if diff_waveform: - diff = (np.nanmean(X[y == diff_waveform[1], ch_ind], axis=0) - - np.nanmean(X[y == diff_waveform[0], ch_ind], axis=0)) - ax.plot(times, diff, color='k', lw=1) - - ax.set_title(epochs.ch_names[ch_ind]) - ax.axvline(x=0, color='k', lw=1, label='_nolegend_') - - ax.set_xlabel('Time (s)') - ax.set_ylabel('Amplitude (uV)') - ax.set_xlabel('Time (s)') - ax.set_ylabel('Amplitude (uV)') - - # Round y axis tick labels to 2 decimal places - # ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f')) - - if diff_waveform: - legend = (['{} - {}'.format(diff_waveform[1], diff_waveform[0])] + - list(conditions.keys())) - else: - legend = conditions.keys() - ax.legend(legend) - sns.despine() - plt.tight_layout() - - if title: - fig.suptitle(title, fontsize=20) - - fig.set_size_inches(10, 8) - - return fig, ax - -def get_epochs_info(epochs): - return [*[{x: len(epochs[x])} for x in epochs.event_id], {"Drop Percentage": round((1 - len(epochs.events)/len(epochs.drop_log)) * 100, 2)}, {"Total Epochs": len(epochs.events)}] diff --git a/app/utils/jupyter/cells.ts b/app/utils/pyodide/cell.js similarity index 100% rename from app/utils/jupyter/cells.ts rename to app/utils/pyodide/cell.js diff --git a/app/utils/jupyter/functions.ts b/app/utils/pyodide/functions.js similarity index 100% rename from app/utils/jupyter/functions.ts rename to app/utils/pyodide/functions.js diff --git a/app/utils/jupyter/pipes.ts b/app/utils/pyodide/pipes.js similarity index 100% rename from app/utils/jupyter/pipes.ts rename to app/utils/pyodide/pipes.js From 9783e7a04cbd06e926e90af512008a10c43a25f7 Mon Sep 17 00:00:00 2001 From: Dano Morrison Date: Mon, 8 Jul 2019 10:11:46 -0400 Subject: [PATCH 15/40] Reverted inadvertent logging change --- app/utils/eeg/emotiv.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/utils/eeg/emotiv.ts b/app/utils/eeg/emotiv.ts index 06de6160..9bfbd46a 100644 --- a/app/utils/eeg/emotiv.ts +++ b/app/utils/eeg/emotiv.ts @@ -26,8 +26,7 @@ interface EmotivHeadset { } // Creates the Cortex object from SDK -const verbose = 3; -// const verbose = process.env.LOG_LEVEL || 1; +const verbose = process.env.LOG_LEVEL || 1; const options = { verbose }; // This global client is used in every Cortex API call From 67734406939a967f0ffe2f990440ddb8ef1bdd1f Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sat, 3 Aug 2019 13:37:37 -0400 Subject: [PATCH 16/40] updates and rebase --- internals/scripts/InstallPyodide.js | 81 ++++++++++++++++------------- package.json | 11 +++- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js index 6f1bd011..68a03b87 100644 --- a/internals/scripts/InstallPyodide.js +++ b/internals/scripts/InstallPyodide.js @@ -1,65 +1,72 @@ -//mkdir app/utils/pyodide/src +// mkdir app/utils/pyodide/src // && cd app/utils/pyodide/src // curl -LJO https://github.com/iodide-project/pyodide/releases/download/0.12.0/pyodide-build-0.12.0.tar.bz2 // tar xjf pyodide-build-0.12.0.tar.bz2 // rm pyodide-build-0.12.0.tar.bz2", -import chalk from "chalk"; -import os from "os"; -import fs from "fs"; -import https from "https"; -import mkdirp from "mkdirp"; -import tar from "tar-fs"; -import url from "url"; -import gunzip from "gunzip-maybe"; +import chalk from 'chalk'; +import fs from 'fs'; +import https from 'https'; +import mkdirp from 'mkdirp'; +import tar from 'tar-fs'; +import url from 'url'; +import bz2 from 'unbzip2-stream'; -const PYODIDE_VERSION = "0.12.0"; +const PYODIDE_VERSION = '0.12.0'; const TAR_NAME = `pyodide-build-${PYODIDE_VERSION}.tar.bz2`; -const PYODIDE_DIR = "app/utils/pyodide/src/"; +const TAR_URL = `https://github.com/iodide-project/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-build-${PYODIDE_VERSION}.tar.bz2`; +const PYODIDE_DIR = 'app/utils/pyodide/src/'; const writeAndUnzipFile = response => { const filePath = `${PYODIDE_DIR}${TAR_NAME}`; const writeStream = fs.createWriteStream(filePath); response.pipe(writeStream); - writeStream.on("finish", () => { + writeStream.on('finish', () => { console.log(`${chalk.green.bold(`Unzipping pyodide`)}`); const readStream = fs.createReadStream(filePath); - readStream.pipe(gunzip()).pipe(tar.extract(PYODIDE_DIR)); + try { + readStream.pipe(bz2()).pipe(tar.extract(PYODIDE_DIR)); + } catch (e) { + throw new Error('Error in unzip:', e); + } - readStream.on("end", () => { + readStream.on('end', () => { console.log(`${chalk.green.bold(`Unzip successful`)}`); }); }); }; +const downloadFile = response => { + if ( + response.statusCode > 300 && + response.statusCode < 400 && + response.headers.location + ) { + if (url.parse(response.headers.location).hostname) { + https.get(response.headers.location, writeAndUnzipFile); + } else { + https.get( + url.resolve(url.parse(TAR_URL).hostname, response.headers.location), + writeAndUnzipFile + ); + } + } else { + writeAndUnzipFile(response); + } +}; + (() => { + if (fs.existsSync(`${PYODIDE_DIR}${TAR_NAME}`)) { + console.log( + `${chalk.green.bold(`Pyodide is already present: ${PYODIDE_VERSION}...`)}` + ); + return; + } console.log( `${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}` ); - mkdirp.sync(`app/utils/pyodide/src`); - - https.get( - `https://github.com/iodide-project/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-build-${PYODIDE_VERSION}.tar.bz2`, - response => { - if ( - response.statusCode > 300 && - response.statusCode < 400 && - response.headers.location - ) { - if (url.parse(response.headers.location).hostname) { - https.get(response.headers.location, writeAndUnzipFile); - } else { - https.get( - url.resolve(url.parse(TAR_URL).hostname, response.headers.location), - writeToFile - ); - } - } else { - writeAndUnzipFile(response); - } - } - ); + https.get(TAR_URL, downloadFile); })(); diff --git a/package.json b/package.json index 43a5aed8..e658d770 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,9 @@ "test-watch": "yarn test --watch" }, "lint-staged": { - "*.{js,jsx,ts,tsx}": ["cross-env NODE_ENV=development eslint --cache"], + "*.{js,jsx,ts,tsx}": [ + "cross-env NODE_ENV=development eslint --cache" + ], "{*.json,.{babelrc,eslintrc,prettierrc,stylelintrc}}": [ "prettier --ignore-path .eslintignore --parser json --write" ], @@ -45,7 +47,9 @@ "stylelint --ignore-path .eslintignore --syntax scss --fix", "prettier --ignore-path .eslintignore --single-quote --write" ], - "*.{html,md,yml}": ["prettier --ignore-path .eslintignore --single-quote --write"] + "*.{html,md,yml}": [ + "prettier --ignore-path .eslintignore --single-quote --write" + ] }, "build": { "productName": "BrainWaves", @@ -53,6 +57,9 @@ "files": [ "dist/", "node_modules/", + "assets/", + "utils/pyodide/utils.py", + "utils/pyodide/pyimport.py", "app.html", "main.prod.js", "main.prod.js.map", From 5f416ea7b9baa53cfab3e949a54424f862960faf Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sat, 3 Aug 2019 18:17:41 -0400 Subject: [PATCH 17/40] massive overhaul of pyodide in the app Co-authored-by: Dano Morrison --- app/epics/pyodideEpics.js | 60 ++++++++--------------- app/utils/filesystem/read.js | 18 +++++++ app/utils/pyodide/commands.js | 92 +++++++++++++++-------------------- app/utils/pyodide/pyimport.py | 15 ------ app/utils/pyodide/utils.py | 43 ++++++++++------ 5 files changed, 103 insertions(+), 125 deletions(-) create mode 100644 app/utils/filesystem/read.js diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index b5b438a3..166d4228 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -5,11 +5,11 @@ import { mergeMap, tap, pluck, - ignoreElements, filter } from "rxjs/operators"; import { getWorkspaceDir } from "../utils/filesystem/storage"; -import { languagePluginLoader } from "../utils/pyodide/pyodide"; +import { parseSingleQuoteJSON } from "../utils/pyodide/functions" +import { readFiles } from "../utils/filesystem/read"; import { LAUNCH, LOAD_EPOCHS, @@ -22,8 +22,8 @@ import { loadERP } from "../actions/pyodideActions"; import { - test, - imports, + loadPackages, + utils, loadCSV, loadCleanedEpochs, filterIIR, @@ -34,7 +34,7 @@ import { plotPSD, plotERP, plotTopoMap, - saveEpochs + saveEpochs, } from "../utils/pyodide/commands"; import { EMOTIV_CHANNELS, @@ -96,39 +96,14 @@ const setPyodideStatus = payload => ({ type: SET_PYODIDE_STATUS }); -const receiveExecuteReply = payload => ({ - payload, - type: RECEIVE_EXECUTE_REPLY, -}); - -const receiveExecuteResult = (payload) => ({ - payload, - type: RECEIVE_EXECUTE_RESULT, -}); - -const receiveDisplayData = (payload) => ({ - payload, - type: RECEIVE_DISPLAY_DATA, -}); - -const receiveStream = (payload) => ({ - payload, - type: RECEIVE_STREAM, -}); - // ------------------------------------------------------------------------- // Epics const launchEpic = action$ => action$.ofType(LAUNCH).pipe( tap(() => console.log("launching")), - mergeMap(async () => { - await languagePluginLoader; - console.log("loaded language plugin"); - // using window.pyodide instead of pyodide to get linter to stop yelling ;) - await window.pyodide.loadPackage(["mne"]); - console.log("loaded mne package"); - }), + mergeMap(loadPackages), + mergeMap(utils), map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) ); @@ -136,9 +111,12 @@ const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), - map(filePathsArray => loadCSV(filePathsArray)), - map(() => filterIIR(1, 30)), - map(() => + tap(files => console.log('files:', files)), + map((filePathsArray => readFiles(filePathsArray))), + tap(csvArray => console.log('csvs:', csvArray)), + mergeMap(csvArray => loadCSV(csvArray)), + mergeMap(() => filterIIR(1, 30)), + mergeMap(() => epochEvents( { [state$.value.experiment.params.stimulus1.title]: EVENTS.STIMULUS_1, @@ -150,11 +128,10 @@ const loadEpochsEpic = (action$, state$) => 0.8 ) ), - map(epochEventsCommand => epochEventsCommand), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const loadCleanedEpochsEpic = (action$, state$) => +const loadCleanedEpochsEpic = (action$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), @@ -180,12 +157,13 @@ const cleanEpochsEpic = (action$, state$) => map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const getEpochsInfoEpic = (action$, state$) => +const getEpochsInfoEpic = (action$) => action$.ofType(GET_EPOCHS_INFO).pipe( pluck("payload"), - map(variableName => requestEpochsInfo(variableName)), - map(epochInfoString => - parseSingleQuoteJSON(epochInfoString).map(infoObj => ({ + tap(payload => console.log('payload: ', payload)), + mergeMap(requestEpochsInfo), + map(epochInfoArray => + epochInfoArray.map(infoObj => ({ name: Object.keys(infoObj)[0], value: infoObj[Object.keys(infoObj)[0]], })) diff --git a/app/utils/filesystem/read.js b/app/utils/filesystem/read.js new file mode 100644 index 00000000..23bc63c4 --- /dev/null +++ b/app/utils/filesystem/read.js @@ -0,0 +1,18 @@ +const fs = require("fs"); + +export const readFiles = (filePathsArray) => { + return filePathsArray.map(path => { + console.log('about to read file') + const file = fs.readFileSync(path, 'utf8') + console.log('read file') + return file + }) +} + + + +// ------------------------------------------- +// Helper methods + +const formatFilePath = (filePath: string) => + `"${filePath.replace(/\\/g, "/")}"`; diff --git a/app/utils/pyodide/commands.js b/app/utils/pyodide/commands.js index b4c71a65..4b718834 100644 --- a/app/utils/pyodide/commands.js +++ b/app/utils/pyodide/commands.js @@ -1,80 +1,69 @@ import * as path from "path"; import { readFileSync } from "fs"; +import { languagePluginLoader } from "./pyodide"; + let pyodide; // ----------------------------- // Imports and Utility functions -export const test = async () => { - await window.pyodide.loadPackage(["mne"]); - - const mneCommands = [ - `import numpy as np`, - `import mne`, - `data = np.repeat(np.atleast_2d(np.arange(1000)), 8, axis=0)`, - `info = mne.create_info(8, 250)`, - `raw = mne.io.RawArray(data=data, info=info)`, - `raw.save("test_brainwaves.fif")` - ]; - await window.pyodide.runPython(mneCommands.join("; ")); -}; - -export const loadPackages = async () => window.pyodide.loadPackage(["mne"]); - -export const imports = () => - pyodide.runPython( - readFileSync(path.join(__dirname, "/utils/pyodide/pyimport.py"), "utf8") - ); +export const loadPackages = async () => { + await languagePluginLoader; + console.log("loaded language plugin"); + // using window.pyodide instead of pyodide to get linter to stop yelling ;) + await window.pyodide.loadPackage(["matplotlib", "mne", "pandas"]); + await window.pyodide.runPython("import js"); + console.log("loaded mne package"); -export const utils = () => - pyodide.runPython( +} + +export const utils = async () => + window.pyodide.runPython( readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8") ); -export const loadCSV = (filePathArray: Array) => - [ - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, - `replace_ch_names = None`, - `raw = load_data(files, replace_ch_names)` - ].join("\n"); +export const loadCSV = async (csvArray: Array) => { + window.csvArray = csvArray; + // TODO: Pass attached variable name as parameter to load_data + await window.pyodide.runPython( + `raw = load_data()`) +} // --------------------------- // MNE-Related Data Processing -export const loadCleanedEpochs = (filePathArray: Array) => +export const loadCleanedEpochs = (epocsArray: Array) => [ - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` ].join("\n"); // NOTE: this command includes a ';' to prevent returning data -export const filterIIR = (lowCutoff: number, highCutoff: number) => - `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`; +export const filterIIR = async (lowCutoff: number, highCutoff: number) => + window.pyodide.runPython(`raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`) -export const epochEvents = ( +export const epochEvents = async ( eventIDs: { [string]: number }, tmin: number, tmax: number, reject?: Array | string = "None" -) => { - const command = [ - `event_id = ${JSON.stringify(eventIDs)}`, - `tmin=${tmin}`, - `tmax=${tmax}`, - `baseline= (tmin, tmax)`, - `picks = None`, - `reject = ${reject}`, - "events = find_events(raw)", - `raw_epochs = Epochs(raw, events=events, event_id=event_id, +) => window.pyodide.runPython([ + `event_id = ${JSON.stringify(eventIDs)}`, + `tmin=${tmin}`, + `tmax=${tmax}`, + `baseline= (tmin, tmax)`, + `picks = None`, + `reject = ${reject}`, + "events = find_events(raw)", + `raw_epochs = Epochs(raw, events=events, event_id=event_id, tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, verbose=False, picks=picks)`, - `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` - ].join("\n"); - return command; -}; + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` +].join("\n")) -export const requestEpochsInfo = (variableName: string) => - `get_epochs_info(${variableName})`; +export const requestEpochsInfo = async (variableName: string) => { + const pyodideReturn = await window.pyodide.runPython(`get_epochs_info(${variableName})`); + return pyodideReturn +} export const requestChannelInfo = () => `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; @@ -104,8 +93,3 @@ export const saveEpochs = (workspaceDir: string, subject: string) => ) )})`; -// ------------------------------------------- -// Helper methods - -const formatFilePath = (filePath: string) => - `"${filePath.replace(/\\/g, "/")}"`; diff --git a/app/utils/pyodide/pyimport.py b/app/utils/pyodide/pyimport.py index 2149f2c7..8b137891 100644 --- a/app/utils/pyodide/pyimport.py +++ b/app/utils/pyodide/pyimport.py @@ -1,16 +1 @@ -from time import time, strftime, gmtime -import os -from collections import OrderedDict -from glob import glob -import numpy as np -import pandas as pd # maybe we can remove this dependency -import seaborn as sns -from matplotlib import pyplot as plt - -from mne import (Epochs, RawArray, concatenate_raws, concatenate_epochs, - create_info, find_events, read_epochs, set_eeg_reference) -from mne.channels import read_montage - - -plt.style.use(fivethirtyeight) diff --git a/app/utils/pyodide/utils.py b/app/utils/pyodide/utils.py index 52a5eae0..ce736458 100644 --- a/app/utils/pyodide/utils.py +++ b/app/utils/pyodide/utils.py @@ -1,25 +1,32 @@ from glob import glob import os +from time import time, strftime, gmtime from collections import OrderedDict -from mne import create_info, concatenate_raws, viz -from mne.io import RawArray -from mne.channels import read_montage -import pandas as pd + import numpy as np -import seaborn as sns from matplotlib import pyplot as plt +import pandas as pd # maybe we can remove this dependency +# import seaborn as sns + +from mne import (Epochs, concatenate_raws, concatenate_epochs, create_info, + find_events, read_epochs, set_eeg_reference, viz) +from mne.io import RawArray +from io import StringIO +from mne.channels import read_montage + + +# plt.style.use(fivethirtyeight) -sns.set_context('talk') -sns.set_style('white') +# sns.set_context('talk') +# sns.set_style('white') -def load_data(fnames, sfreq=128., replace_ch_names=None): +def load_data(sfreq=128., replace_ch_names=None): """Load CSV files from the /data directory into a RawArray object. Parameters ---------- - fnames : list - CSV filepaths from which to load data + sfreq : float EEG sampling frequency replace_ch_names : dict | None @@ -31,12 +38,13 @@ def load_data(fnames, sfreq=128., replace_ch_names=None): raw : an instance of mne.io.RawArray The loaded data. """ - + ## js is loaded in loadPackages + ## TODO: Received attached variable name raw = [] - print(fnames) - for fname in fnames: + for csv in js.csvArray: + string_io = StringIO(csv) # read the file - data = pd.read_csv(fname, index_col=0) + data = pd.read_csv(string_io, index_col=0) data = data.dropna() @@ -77,7 +85,11 @@ def load_data(fnames, sfreq=128., replace_ch_names=None): def plot_topo(epochs, conditions=OrderedDict()): - palette = sns.color_palette("hls", len(conditions) + 1) + # palette = sns.color_palette("hls", len(conditions) + 1) + # temp hack, just pull in the color palette from seaborn + palette = [(0.85999999999999999, 0.37119999999999997, 0.33999999999999997), + (0.33999999999999997, 0.85999999999999999, 0.37119999999999997), + (0.37119999999999997, 0.33999999999999997, 0.85999999999999999)] evokeds = [epochs[name].average() for name in (conditions)] evoked_topo = viz.plot_evoked_topo( @@ -189,6 +201,7 @@ def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, return fig, ax def get_epochs_info(epochs): + print('Get Epochs Info:') return [*[{x: len(epochs[x])} for x in epochs.event_id], {"Drop Percentage": round((1 - len(epochs.events) / len(epochs.drop_log)) * 100, 2)}, From 473b711496835a4adaff290c66912e87089b37ef Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 7 Mar 2020 13:28:17 -0500 Subject: [PATCH 18/40] Tidyed up pyodide files and removed some jupyter dependencies --- app/epics/pyodideEpics.js | 80 ++++++++++--------- app/package.json | 5 +- app/utils/pyodide/commands.js | 95 ----------------------- app/utils/pyodide/functions.js | 6 ++ app/utils/pyodide/index.js | 118 +++++++++++++++++++++++++++++ app/utils/pyodide/pythonStrings.js | 84 ++++++++++++++++++++ 6 files changed, 248 insertions(+), 140 deletions(-) delete mode 100644 app/utils/pyodide/commands.js create mode 100644 app/utils/pyodide/index.js create mode 100644 app/utils/pyodide/pythonStrings.js diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index 166d4228..f72b6eb6 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -1,15 +1,9 @@ -import { combineEpics } from "redux-observable"; -import { of } from "rxjs"; -import { - map, - mergeMap, - tap, - pluck, - filter -} from "rxjs/operators"; -import { getWorkspaceDir } from "../utils/filesystem/storage"; -import { parseSingleQuoteJSON } from "../utils/pyodide/functions" -import { readFiles } from "../utils/filesystem/read"; +import { combineEpics } from 'redux-observable'; +import { of } from 'rxjs'; +import { map, mergeMap, tap, pluck, filter } from 'rxjs/operators'; +import { getWorkspaceDir } from '../utils/filesystem/storage'; +import { parseSingleQuoteJSON } from '../utils/pyodide/functions'; +import { readFiles } from '../utils/filesystem/read'; import { LAUNCH, LOAD_EPOCHS, @@ -20,7 +14,7 @@ import { CLEAN_EPOCHS, loadTopo, loadERP -} from "../actions/pyodideActions"; +} from '../actions/pyodideActions'; import { loadPackages, utils, @@ -34,8 +28,8 @@ import { plotPSD, plotERP, plotTopoMap, - saveEpochs, -} from "../utils/pyodide/commands"; + saveEpochs +} from '../utils/pyodide'; import { EMOTIV_CHANNELS, EVENTS, @@ -43,21 +37,21 @@ import { MUSE_CHANNELS, PYODIDE_VARIABLE_NAMES, PYODIDE_STATUS -} from "../constants/constants"; - -export const GET_EPOCHS_INFO = "GET_EPOCHS_INFO"; -export const GET_CHANNEL_INFO = "GET_CHANNEL_INFO"; -export const SET_MAIN_CHANNEL = "SET_MAIN_CHANNEL"; -export const SET_EPOCH_INFO = "SET_EPOCH_INFO"; -export const SET_CHANNEL_INFO = "SET_CHANNEL_INFO"; -export const SET_PSD_PLOT = "SET_PSD_PLOT"; -export const SET_ERP_PLOT = "SET_ERP_PLOT"; -export const SET_TOPO_PLOT = "SET_TOPO_PLOT"; -export const SET_PYODIDE_STATUS = "SET_PYODIDE_STATUS"; -export const RECEIVE_EXECUTE_REPLY = "RECEIVE_EXECUTE_REPLY"; -export const RECEIVE_EXECUTE_RESULT = "RECEIVE_EXECUTE_RESULT"; -export const RECEIVE_STREAM = "RECEIVE_STREAM"; -export const RECEIVE_DISPLAY_DATA = "RECEIVE_DISPLAY_DATA"; +} from '../constants/constants'; + +export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; +export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; +export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; +export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; +export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; +export const SET_PSD_PLOT = 'SET_PSD_PLOT'; +export const SET_ERP_PLOT = 'SET_ERP_PLOT'; +export const SET_TOPO_PLOT = 'SET_TOPO_PLOT'; +export const SET_PYODIDE_STATUS = 'SET_PYODIDE_STATUS'; +export const RECEIVE_EXECUTE_REPLY = 'RECEIVE_EXECUTE_REPLY'; +export const RECEIVE_EXECUTE_RESULT = 'RECEIVE_EXECUTE_RESULT'; +export const RECEIVE_STREAM = 'RECEIVE_STREAM'; +export const RECEIVE_DISPLAY_DATA = 'RECEIVE_DISPLAY_DATA'; // ------------------------------------------------------------------------- // Action Creators @@ -101,7 +95,7 @@ const setPyodideStatus = payload => ({ const launchEpic = action$ => action$.ofType(LAUNCH).pipe( - tap(() => console.log("launching")), + tap(() => console.log('launching')), mergeMap(loadPackages), mergeMap(utils), map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) @@ -109,10 +103,10 @@ const launchEpic = action$ => const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( - pluck("payload"), + pluck('payload'), filter(filePathsArray => filePathsArray.length >= 1), tap(files => console.log('files:', files)), - map((filePathsArray => readFiles(filePathsArray))), + map(filePathsArray => readFiles(filePathsArray)), tap(csvArray => console.log('csvs:', csvArray)), mergeMap(csvArray => loadCSV(csvArray)), mergeMap(() => filterIIR(1, 30)), @@ -131,9 +125,9 @@ const loadEpochsEpic = (action$, state$) => map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const loadCleanedEpochsEpic = (action$) => +const loadCleanedEpochsEpic = action$ => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( - pluck("payload"), + pluck('payload'), filter(filePathsArray => filePathsArray.length >= 1), map(filePathsArray => loadCleanedEpochs(filePathsArray)), mergeMap(() => @@ -157,9 +151,9 @@ const cleanEpochsEpic = (action$, state$) => map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const getEpochsInfoEpic = (action$) => +const getEpochsInfoEpic = action$ => action$.ofType(GET_EPOCHS_INFO).pipe( - pluck("payload"), + pluck('payload'), tap(payload => console.log('payload: ', payload)), mergeMap(requestEpochsInfo), map(epochInfoArray => @@ -171,7 +165,7 @@ const getEpochsInfoEpic = (action$) => map(setEpochInfo) ); -const getChannelInfoEpic = (action$, state$) => +const getChannelInfoEpic = action$ => action$.ofType(GET_CHANNEL_INFO).pipe( map(requestChannelInfo), map(channelInfoString => @@ -179,7 +173,7 @@ const getChannelInfoEpic = (action$, state$) => ) ); -const loadPSDEpic = (action$, state$) => +const loadPSDEpic = action$ => action$.ofType(LOAD_PSD).pipe( map(plotPSD), map(setPSDPlot) @@ -198,13 +192,17 @@ const loadTopoEpic = (action$, state$) => ) ); -const loadERPEpic = (action$, state$) => +const loadERPEpic = action$ => action$.ofType(LOAD_ERP).pipe( +<<<<<<< HEAD <<<<<<< HEAD pluck('payload'), map((channelName) => { ======= pluck("payload"), +======= + pluck('payload'), +>>>>>>> Tidyed up pyodide files and removed some jupyter dependencies map(channelName => { >>>>>>> Added loading of pyodide within app epics if (MUSE_CHANNELS.includes(channelName)) { @@ -216,7 +214,7 @@ const loadERPEpic = (action$, state$) => console.warn('channel name supplied to loadERPEpic does not belong to either device'); ======= console.warn( - "channel name supplied to loadERPEpic does not belong to either device" + 'channel name supplied to loadERPEpic does not belong to either device' ); >>>>>>> Added loading of pyodide within app epics return EMOTIV_CHANNELS[0]; diff --git a/app/package.json b/app/package.json index 2d08fd75..afd3c108 100644 --- a/app/package.json +++ b/app/package.json @@ -19,10 +19,7 @@ "@neurosity/pipes": "^3.2.3", "@babel/runtime": "7.10.2", "@babel/runtime-corejs2": "^7.10.2", - "enchannel-zmq-backend": "^9.1.22", - "kernelspecs": "^2.0.0", - "node-pre-gyp": "^0.15.0", - "spawnteract": "^5.0.1" + "node-pre-gyp": "^0.15.0" }, "devDependencies": { "@babel/register": "^7.10.1" diff --git a/app/utils/pyodide/commands.js b/app/utils/pyodide/commands.js deleted file mode 100644 index 4b718834..00000000 --- a/app/utils/pyodide/commands.js +++ /dev/null @@ -1,95 +0,0 @@ -import * as path from "path"; -import { readFileSync } from "fs"; -import { languagePluginLoader } from "./pyodide"; - - -let pyodide; -// ----------------------------- -// Imports and Utility functions - -export const loadPackages = async () => { - await languagePluginLoader; - console.log("loaded language plugin"); - // using window.pyodide instead of pyodide to get linter to stop yelling ;) - await window.pyodide.loadPackage(["matplotlib", "mne", "pandas"]); - await window.pyodide.runPython("import js"); - console.log("loaded mne package"); - -} - -export const utils = async () => - window.pyodide.runPython( - readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8") - ); - -export const loadCSV = async (csvArray: Array) => { - window.csvArray = csvArray; - // TODO: Pass attached variable name as parameter to load_data - await window.pyodide.runPython( - `raw = load_data()`) -} - -// --------------------------- -// MNE-Related Data Processing -export const loadCleanedEpochs = (epocsArray: Array) => - [ - `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, - `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` - ].join("\n"); - -// NOTE: this command includes a ';' to prevent returning data -export const filterIIR = async (lowCutoff: number, highCutoff: number) => - window.pyodide.runPython(`raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`) - -export const epochEvents = async ( - eventIDs: { [string]: number }, - tmin: number, - tmax: number, - reject?: Array | string = "None" -) => window.pyodide.runPython([ - `event_id = ${JSON.stringify(eventIDs)}`, - `tmin=${tmin}`, - `tmax=${tmax}`, - `baseline= (tmin, tmax)`, - `picks = None`, - `reject = ${reject}`, - "events = find_events(raw)", - `raw_epochs = Epochs(raw, events=events, event_id=event_id, - tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, - verbose=False, picks=picks)`, - `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` -].join("\n")) - -export const requestEpochsInfo = async (variableName: string) => { - const pyodideReturn = await window.pyodide.runPython(`get_epochs_info(${variableName})`); - return pyodideReturn -} - -export const requestChannelInfo = () => - `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; - -// ----------------------------- -// Plot functions - -export const cleanEpochsPlot = () => - `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)`; - -export const plotPSD = () => `raw.plot_psd(fmin=1, fmax=30)`; - -export const plotTopoMap = () => `plot_topo(clean_epochs, conditions)`; - -export const plotERP = (channelIndex: number) => - `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, - ci=97.5, n_boot=1000, title='', diff_waveform=None)`; - -export const saveEpochs = (workspaceDir: string, subject: string) => - `raw_epochs.save(${formatFilePath( - path.join( - workspaceDir, - "Data", - subject, - "EEG", - `${subject}-cleaned-epo.fif` - ) - )})`; - diff --git a/app/utils/pyodide/functions.js b/app/utils/pyodide/functions.js index 58f7ab87..bf674756 100644 --- a/app/utils/pyodide/functions.js +++ b/app/utils/pyodide/functions.js @@ -1,3 +1,6 @@ +// ------------------------------------------- +// Helper & utility functions + export const parseSingleQuoteJSON = (string: string) => JSON.parse(string.replace(/'/g, '"')); @@ -31,3 +34,6 @@ export const debugParseMessage = (msg: Record) => { } return `${msg.channel} ${content}`; }; + +export const formatFilePath = (filePath: string) => + `"${filePath.replace(/\\/g, '/')}"`; diff --git a/app/utils/pyodide/index.js b/app/utils/pyodide/index.js new file mode 100644 index 00000000..424fec39 --- /dev/null +++ b/app/utils/pyodide/index.js @@ -0,0 +1,118 @@ +import * as path from 'path'; +import { readFileSync } from 'fs'; +import { languagePluginLoader } from './pyodide'; +import { formatFilePath } from './functions'; + +// --------------------------------- +// This file contains the JS functions that allow the app to access python-wasm through pyodide +// These functions wrap the python strings defined in the + +// ----------------------------- +// Imports and Utility functions + +// Note: this takes an incredibly long time +export const loadPackages = async () => { + await languagePluginLoader; + console.log('loaded language plugin'); + // using window.pyodide instead of pyodide to get linter to stop yelling ;) + await window.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']); + await window.pyodide.runPython('import js'); + console.log('loaded mne package'); +}; + +export const loadUtils = async () => + window.pyodide.runPython( + readFileSync(path.join(__dirname, '/utils/pyodide/utils.py'), 'utf8') + ); + +export const loadCSV = async (csvArray: Array) => { + window.csvArray = csvArray; + // TODO: Pass attached variable name as parameter to load_data + await window.pyodide.runPython(`raw = load_data()`); +}; + +// --------------------------- +// MNE-Related Data Processing + +// export const loadCleanedEpochs = (epocsArray: Array) => +// [ +// `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, +// `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` +// ].join("\n"); + +// NOTE: this command includes a ';' to prevent returning data +export const filterIIR = async (lowCutoff: number, highCutoff: number) => + window.pyodide.runPython( + `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');` + ); + +export const epochEvents = async ( + eventIDs: { [string]: number }, + tmin: number, + tmax: number, + reject?: Array | string = 'None' +) => + window.pyodide.runPython( + [ + `event_id = ${JSON.stringify(eventIDs)}`, + `tmin=${tmin}`, + `tmax=${tmax}`, + `baseline= (tmin, tmax)`, + `picks = None`, + `reject = ${reject}`, + 'events = find_events(raw)', + `raw_epochs = Epochs(raw, events=events, event_id=event_id, + tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, + verbose=False, picks=picks)`, + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` + ].join('\n') + ); + +export const requestEpochsInfo = async (variableName: string) => { + const pyodideReturn = await window.pyodide.runPython( + `get_epochs_info(${variableName})` + ); + return pyodideReturn; +}; + +export const requestChannelInfo = async () => + window.pyodide.runPython( + `[ch for ch in clean_epochs.ch_names if ch != 'Marker']` + ); + +// ----------------------------- +// Plot functions + +export const cleanEpochsPlot = async () => { + // TODO: Figure out how to get image results from pyodide + window.pyodide.runPython( + `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)` + ); +}; + +export const plotPSD = async () => { + // TODO: Figure out how to get image results from pyodide + window.pyodide.runPython(`raw.plot_psd(fmin=1, fmax=30)`); +}; + +export const plotTopoMap = async () => { + // TODO: Figure out how to get image results from pyodide + window.pyodide.runPython(`plot_topo(clean_epochs, conditions)`); +}; + +export const plotERP = (channelIndex: number) => + `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, + ci=97.5, n_boot=1000, title='', diff_waveform=None)`; + +export const saveEpochs = (workspaceDir: string, subject: string) => + window.pyodide.runPython( + `raw_epochs.save(${formatFilePath( + path.join( + workspaceDir, + 'Data', + subject, + 'EEG', + `${subject}-cleaned-epo.fif` + ) + )}` + ); diff --git a/app/utils/pyodide/pythonStrings.js b/app/utils/pyodide/pythonStrings.js new file mode 100644 index 00000000..03cc3694 --- /dev/null +++ b/app/utils/pyodide/pythonStrings.js @@ -0,0 +1,84 @@ +// DEPRECATED + +// import * as path from "path"; +// import { readFileSync } from "fs"; + +// // The output of the functions contained in this file are python commands encoded as strings +// // that would be run in a notebook environment in order to perform the experimental analyses underlying BrainWaves + +// export const utils = () => +// readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8"); + +// // export const loadCSV = (filePathArray: Array) => +// // [ +// // `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, +// // `replace_ch_names = None`, +// // `raw = load_data(files, replace_ch_names)` +// // ].join("\n"); + +// // export const loadCleanedEpochs = (filePathArray: Array) => +// // [ +// // `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, +// // `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, +// // `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` +// // ].join("\n"); + +// // NOTE: this command includes a ';' to prevent returning data +// export const filterIIR = (lowCutoff: number, highCutoff: number) => +// `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`; + +// export const plotPSD = () => +// [`%matplotlib inline`, `raw.plot_psd(fmin=1, fmax=30)`].join("\n"); + +// export const epochEvents = ( +// eventIDs: { [string]: number }, +// tmin: number, +// tmax: number, +// reject?: Array | string = "None" +// ) => +// [ +// `event_id = ${JSON.stringify(eventIDs)}`, +// `tmin=${tmin}`, +// `tmax=${tmax}`, +// `baseline= (tmin, tmax)`, +// `picks = None`, +// `reject = ${reject}`, +// "events = find_events(raw)", +// `raw_epochs = Epochs(raw, events=events, event_id=event_id, +// tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, +// verbose=False, picks=picks)`, +// `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` +// ].join("\n"); + +// export const requestEpochsInfo = (variableName: string) => +// `get_epochs_info(${variableName})`; + +// export const requestChannelInfo = () => +// `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; + +// export const cleanEpochsPlot = () => +// [ +// `%matplotlib`, +// `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)` +// ].join("\n"); + +// export const plotTopoMap = () => +// [`%matplotlib inline`, `plot_topo(clean_epochs, conditions)`].join("\n"); + +// export const plotERP = (channelIndex: number) => +// [ +// `%matplotlib inline`, +// `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, +// ci=97.5, n_boot=1000, title='', diff_waveform=None)` +// ].join("\n"); + +// export const saveEpochs = (workspaceDir: string, subject: string) => +// `raw_epochs.save(${formatFilePath( +// path.join( +// workspaceDir, +// "Data", +// subject, +// "EEG", +// `${subject}-cleaned-epo.fif` +// ) +// )})`; From 4092c5848d3530f7f75ccce219f4356403e451fb Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 7 Mar 2020 13:36:28 -0500 Subject: [PATCH 19/40] Updated pyodide version --- internals/scripts/InstallPyodide.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js index 68a03b87..90461ec8 100644 --- a/internals/scripts/InstallPyodide.js +++ b/internals/scripts/InstallPyodide.js @@ -12,7 +12,7 @@ import tar from 'tar-fs'; import url from 'url'; import bz2 from 'unbzip2-stream'; -const PYODIDE_VERSION = '0.12.0'; +const PYODIDE_VERSION = '0.14.3'; const TAR_NAME = `pyodide-build-${PYODIDE_VERSION}.tar.bz2`; const TAR_URL = `https://github.com/iodide-project/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-build-${PYODIDE_VERSION}.tar.bz2`; const PYODIDE_DIR = 'app/utils/pyodide/src/'; From 465a79da8ad2ad912db251caf0b08923f71ed2b4 Mon Sep 17 00:00:00 2001 From: jdpigeon Date: Sat, 25 Apr 2020 21:04:51 -0400 Subject: [PATCH 20/40] fixed rebase bug --- app/containers/HomeContainer.js | 25 +++++----- app/epics/pyodideEpics.js | 82 +++++++++++---------------------- 2 files changed, 41 insertions(+), 66 deletions(-) diff --git a/app/containers/HomeContainer.js b/app/containers/HomeContainer.js index 78791b99..aa57af8f 100644 --- a/app/containers/HomeContainer.js +++ b/app/containers/HomeContainer.js @@ -1,20 +1,23 @@ // @flow -import { connect } from "react-redux"; -import { bindActionCreators } from "redux"; -import Home from "../components/HomeComponent"; -import * as deviceActions from "../actions/deviceActions"; -import * as pyodideActions from "../actions/pyodideActions"; -import * as experimentActions from "../actions/experimentActions"; +import { connect } from 'react-redux'; +import { bindActionCreators } from 'redux'; +import Home from '../components/HomeComponent'; +import * as deviceActions from '../actions/deviceActions'; +import * as pyodideActions from '../actions/pyodideActions'; +import * as experimentActions from '../actions/experimentActions'; + +function mapStateToProps(state) { + return { + availableDevices: state.device.availableDevices, + }; +} function mapDispatchToProps(dispatch) { return { deviceActions: bindActionCreators(deviceActions, dispatch), pyodideActions: bindActionCreators(pyodideActions, dispatch), - experimentActions: bindActionCreators(experimentActions, dispatch) + experimentActions: bindActionCreators(experimentActions, dispatch), }; } -export default connect( - null, - mapDispatchToProps -)(Home); +export default connect(mapStateToProps, mapDispatchToProps)(Home); diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index f72b6eb6..3a11f4af 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -13,7 +13,7 @@ import { LOAD_TOPO, CLEAN_EPOCHS, loadTopo, - loadERP + loadERP, } from '../actions/pyodideActions'; import { loadPackages, @@ -28,7 +28,7 @@ import { plotPSD, plotERP, plotTopoMap, - saveEpochs + saveEpochs, } from '../utils/pyodide'; import { EMOTIV_CHANNELS, @@ -36,7 +36,7 @@ import { DEVICES, MUSE_CHANNELS, PYODIDE_VARIABLE_NAMES, - PYODIDE_STATUS + PYODIDE_STATUS, } from '../constants/constants'; export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; @@ -60,7 +60,7 @@ const getEpochsInfo = (payload) => ({ payload, type: GET_EPOCHS_INFO }); const getChannelInfo = () => ({ type: GET_CHANNEL_INFO }); -const setEpochInfo = payload => ({ +const setEpochInfo = (payload) => ({ payload, type: SET_EPOCH_INFO, }); @@ -85,15 +85,15 @@ const setERPPlot = (payload) => ({ type: SET_ERP_PLOT, }); -const setPyodideStatus = payload => ({ +const setPyodideStatus = (payload) => ({ payload, - type: SET_PYODIDE_STATUS + type: SET_PYODIDE_STATUS, }); // ------------------------------------------------------------------------- // Epics -const launchEpic = action$ => +const launchEpic = (action$) => action$.ofType(LAUNCH).pipe( tap(() => console.log('launching')), mergeMap(loadPackages), @@ -104,11 +104,11 @@ const launchEpic = action$ => const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck('payload'), - filter(filePathsArray => filePathsArray.length >= 1), - tap(files => console.log('files:', files)), - map(filePathsArray => readFiles(filePathsArray)), - tap(csvArray => console.log('csvs:', csvArray)), - mergeMap(csvArray => loadCSV(csvArray)), + filter((filePathsArray) => filePathsArray.length >= 1), + tap((files) => console.log('files:', files)), + map((filePathsArray) => readFiles(filePathsArray)), + tap((csvArray) => console.log('csvs:', csvArray)), + mergeMap((csvArray) => loadCSV(csvArray)), mergeMap(() => filterIIR(1, 30)), mergeMap(() => epochEvents( @@ -125,17 +125,13 @@ const loadEpochsEpic = (action$, state$) => map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const loadCleanedEpochsEpic = action$ => +const loadCleanedEpochsEpic = (action$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( pluck('payload'), - filter(filePathsArray => filePathsArray.length >= 1), - map(filePathsArray => loadCleanedEpochs(filePathsArray)), + filter((filePathsArray) => filePathsArray.length >= 1), + map((filePathsArray) => loadCleanedEpochs(filePathsArray)), mergeMap(() => - of( - getEpochsInfo(PYODIDE_VARIABLE_NAMES.CLEAN_EPOCHS), - getChannelInfo(), - loadTopo() - ) + of(getEpochsInfo(PYODIDE_VARIABLE_NAMES.CLEAN_EPOCHS), getChannelInfo(), loadTopo()) ) ); @@ -143,21 +139,18 @@ const cleanEpochsEpic = (action$, state$) => action$.ofType(CLEAN_EPOCHS).pipe( map(cleanEpochsPlot), map(() => - saveEpochs( - getWorkspaceDir(state$.value.experiment.title), - state$.value.experiment.subject - ) + saveEpochs(getWorkspaceDir(state$.value.experiment.title), state$.value.experiment.subject) ), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const getEpochsInfoEpic = action$ => +const getEpochsInfoEpic = (action$) => action$.ofType(GET_EPOCHS_INFO).pipe( pluck('payload'), - tap(payload => console.log('payload: ', payload)), + tap((payload) => console.log('payload: ', payload)), mergeMap(requestEpochsInfo), - map(epochInfoArray => - epochInfoArray.map(infoObj => ({ + map((epochInfoArray) => + epochInfoArray.map((infoObj) => ({ name: Object.keys(infoObj)[0], value: infoObj[Object.keys(infoObj)[0]], })) @@ -165,24 +158,18 @@ const getEpochsInfoEpic = action$ => map(setEpochInfo) ); -const getChannelInfoEpic = action$ => +const getChannelInfoEpic = (action$) => action$.ofType(GET_CHANNEL_INFO).pipe( map(requestChannelInfo), - map(channelInfoString => - setChannelInfo(parseSingleQuoteJSON(channelInfoString)) - ) + map((channelInfoString) => setChannelInfo(parseSingleQuoteJSON(channelInfoString))) ); -const loadPSDEpic = action$ => - action$.ofType(LOAD_PSD).pipe( - map(plotPSD), - map(setPSDPlot) - ); +const loadPSDEpic = (action$) => action$.ofType(LOAD_PSD).pipe(map(plotPSD), map(setPSDPlot)); const loadTopoEpic = (action$, state$) => action$.ofType(LOAD_TOPO).pipe( map(plotTopoMap), - mergeMap(topoPlot => + mergeMap((topoPlot) => of( setTopoPlot(topoPlot), loadERP( @@ -192,34 +179,19 @@ const loadTopoEpic = (action$, state$) => ) ); -const loadERPEpic = action$ => +const loadERPEpic = (action$) => action$.ofType(LOAD_ERP).pipe( -<<<<<<< HEAD -<<<<<<< HEAD pluck('payload'), map((channelName) => { -======= - pluck("payload"), -======= - pluck('payload'), ->>>>>>> Tidyed up pyodide files and removed some jupyter dependencies - map(channelName => { ->>>>>>> Added loading of pyodide within app epics if (MUSE_CHANNELS.includes(channelName)) { return MUSE_CHANNELS.indexOf(channelName); } else if (EMOTIV_CHANNELS.includes(channelName)) { return EMOTIV_CHANNELS.indexOf(channelName); } -<<<<<<< HEAD console.warn('channel name supplied to loadERPEpic does not belong to either device'); -======= - console.warn( - 'channel name supplied to loadERPEpic does not belong to either device' - ); ->>>>>>> Added loading of pyodide within app epics return EMOTIV_CHANNELS[0]; }), - map(channelIndex => plotERP(channelIndex)), + map((channelIndex) => plotERP(channelIndex)), map(setERPPlot) ); From 4961fa871cb3e1f66b12fcb46ae8434f5f9dd279 Mon Sep 17 00:00:00 2001 From: Dano Morrison Date: Mon, 11 May 2020 02:18:00 -0400 Subject: [PATCH 21/40] Switched to web worker with different epic pattern --- app/epics/pyodideEpics.js | 72 +++++++++++++++---- app/utils/pyodide/index.js | 69 +++++++----------- app/utils/pyodide/pyimport.py | 1 - .../{pythonStrings.js => statements.json} | 0 app/utils/pyodide/webworker.js | 37 ++++++++++ 5 files changed, 121 insertions(+), 58 deletions(-) delete mode 100644 app/utils/pyodide/pyimport.py rename app/utils/pyodide/{pythonStrings.js => statements.json} (100%) create mode 100644 app/utils/pyodide/webworker.js diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index 3a11f4af..2ac3bd23 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -1,5 +1,6 @@ import { combineEpics } from 'redux-observable'; -import { of } from 'rxjs'; +import { of, fromEvent } from 'rxjs'; +import { toast } from 'react-toastify'; import { map, mergeMap, tap, pluck, filter } from 'rxjs/operators'; import { getWorkspaceDir } from '../utils/filesystem/storage'; import { parseSingleQuoteJSON } from '../utils/pyodide/functions'; @@ -16,8 +17,7 @@ import { loadERP, } from '../actions/pyodideActions'; import { - loadPackages, - utils, + loadPyodide, loadCSV, loadCleanedEpochs, filterIIR, @@ -39,19 +39,22 @@ import { PYODIDE_STATUS, } from '../constants/constants'; -export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; export const GET_CHANNEL_INFO = 'GET_CHANNEL_INFO'; -export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; -export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; -export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; -export const SET_PSD_PLOT = 'SET_PSD_PLOT'; -export const SET_ERP_PLOT = 'SET_ERP_PLOT'; -export const SET_TOPO_PLOT = 'SET_TOPO_PLOT'; -export const SET_PYODIDE_STATUS = 'SET_PYODIDE_STATUS'; +export const GET_EPOCHS_INFO = 'GET_EPOCHS_INFO'; +export const RECEIVE_DISPLAY_DATA = 'RECEIVE_DISPLAY_DATA'; +export const RECEIVE_ERROR = 'RECEIVE_ERROR'; export const RECEIVE_EXECUTE_REPLY = 'RECEIVE_EXECUTE_REPLY'; export const RECEIVE_EXECUTE_RESULT = 'RECEIVE_EXECUTE_RESULT'; +export const RECEIVE_MESSAGE = 'RECEIVE_MESSAGE'; export const RECEIVE_STREAM = 'RECEIVE_STREAM'; -export const RECEIVE_DISPLAY_DATA = 'RECEIVE_DISPLAY_DATA'; +export const SET_CHANNEL_INFO = 'SET_CHANNEL_INFO'; +export const SET_EPOCH_INFO = 'SET_EPOCH_INFO'; +export const SET_ERP_PLOT = 'SET_ERP_PLOT'; +export const SET_MAIN_CHANNEL = 'SET_MAIN_CHANNEL'; +export const SET_PSD_PLOT = 'SET_PSD_PLOT'; +export const SET_PYODIDE_STATUS = 'SET_PYODIDE_STATUS'; +export const SET_PYODIDE_WORKER = 'SET_PYODIDE_WORKER'; +export const SET_TOPO_PLOT = 'SET_TOPO_PLOT'; // ------------------------------------------------------------------------- // Action Creators @@ -90,15 +93,52 @@ const setPyodideStatus = (payload) => ({ type: SET_PYODIDE_STATUS, }); +const setPyodideWorker = (payload: Worker) => ({ + payload, + type: SET_PYODIDE_WORKER, +}); + +const receivePyodideError = (payload) => ({ + payload, + type: RECEIVE_ERROR, +}); + +const receivePyodideMessage = (payload) => ({ + payload, + type: RECEIVE_MESSAGE, +}); + // ------------------------------------------------------------------------- // Epics const launchEpic = (action$) => action$.ofType(LAUNCH).pipe( tap(() => console.log('launching')), - mergeMap(loadPackages), - mergeMap(utils), - map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) + map(loadPyodide), + map(setPyodideWorker) + ); + +const pyodideError = (action$) => + action$.ofType(SET_PYODIDE_WORKER).pipe( + pluck('payload'), + tap((e) => + toast.error(`Error in pyodideWorker at ${e.filename}, Line: ${e.lineno}, ${e.message}`) + ), + map(receivePyodideError) + ); + +const pyodideMessage = (action$) => + action$.ofType(SET_PYODIDE_WORKER).pipe( + pluck('payload'), + tap((e) => { + const { results, error } = e.data; + if (results && !error) { + toast(`Pyodide: `, results); + } else if (error) { + toast.error('Pyodide: ', error); + } + }), + map(receivePyodideMessage) ); const loadEpochsEpic = (action$, state$) => @@ -196,6 +236,8 @@ const loadERPEpic = (action$) => ); export default combineEpics( + pyodideError, + pyodideMessage, launchEpic, loadEpochsEpic, loadCleanedEpochsEpic, diff --git a/app/utils/pyodide/index.js b/app/utils/pyodide/index.js index 424fec39..07dd267e 100644 --- a/app/utils/pyodide/index.js +++ b/app/utils/pyodide/index.js @@ -7,28 +7,23 @@ import { formatFilePath } from './functions'; // This file contains the JS functions that allow the app to access python-wasm through pyodide // These functions wrap the python strings defined in the + // ----------------------------- // Imports and Utility functions -// Note: this takes an incredibly long time -export const loadPackages = async () => { - await languagePluginLoader; - console.log('loaded language plugin'); - // using window.pyodide instead of pyodide to get linter to stop yelling ;) - await window.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']); - await window.pyodide.runPython('import js'); - console.log('loaded mne package'); +export const loadPyodide = () => { + return new Worker('./utils/pyodide/webworker.js'); }; export const loadUtils = async () => - window.pyodide.runPython( - readFileSync(path.join(__dirname, '/utils/pyodide/utils.py'), 'utf8') - ); + pyodideWorker.postMessage({ + data: readFileSync(path.join(__dirname, '/utils/pyodide/utils.py'), 'utf8'), + }); export const loadCSV = async (csvArray: Array) => { window.csvArray = csvArray; // TODO: Pass attached variable name as parameter to load_data - await window.pyodide.runPython(`raw = load_data()`); + await pyodideWorker.postMessage({ data: `raw = load_data()` }); }; // --------------------------- @@ -42,9 +37,7 @@ export const loadCSV = async (csvArray: Array) => { // NOTE: this command includes a ';' to prevent returning data export const filterIIR = async (lowCutoff: number, highCutoff: number) => - window.pyodide.runPython( - `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');` - ); + pyodideWorker.postMessage({ data: `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');` }); export const epochEvents = async ( eventIDs: { [string]: number }, @@ -52,8 +45,8 @@ export const epochEvents = async ( tmax: number, reject?: Array | string = 'None' ) => - window.pyodide.runPython( - [ + pyodideWorker.postMessage({ + data: [ `event_id = ${JSON.stringify(eventIDs)}`, `tmin=${tmin}`, `tmax=${tmax}`, @@ -64,40 +57,38 @@ export const epochEvents = async ( `raw_epochs = Epochs(raw, events=events, event_id=event_id, tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, verbose=False, picks=picks)`, - `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` - ].join('\n') - ); + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})`, + ].join('\n'), + }); export const requestEpochsInfo = async (variableName: string) => { - const pyodideReturn = await window.pyodide.runPython( - `get_epochs_info(${variableName})` - ); + const pyodideReturn = await pyodideWorker.postMessage({ + data: `get_epochs_info(${variableName})`, + }); return pyodideReturn; }; export const requestChannelInfo = async () => - window.pyodide.runPython( - `[ch for ch in clean_epochs.ch_names if ch != 'Marker']` - ); + pyodideWorker.postMessage({ data: `[ch for ch in clean_epochs.ch_names if ch != 'Marker']` }); // ----------------------------- // Plot functions export const cleanEpochsPlot = async () => { // TODO: Figure out how to get image results from pyodide - window.pyodide.runPython( - `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)` - ); + pyodideWorker.postMessage({ + data: `raw_epochs.plot(scalings='auto', n_epochs=6, title="Clean Data", events=None)`, + }); }; export const plotPSD = async () => { // TODO: Figure out how to get image results from pyodide - window.pyodide.runPython(`raw.plot_psd(fmin=1, fmax=30)`); + pyodideWorker.postMessage({ data: `raw.plot_psd(fmin=1, fmax=30)` }); }; export const plotTopoMap = async () => { // TODO: Figure out how to get image results from pyodide - window.pyodide.runPython(`plot_topo(clean_epochs, conditions)`); + pyodideWorker.postMessage({ data: `plot_topo(clean_epochs, conditions)` }); }; export const plotERP = (channelIndex: number) => @@ -105,14 +96,8 @@ export const plotERP = (channelIndex: number) => ci=97.5, n_boot=1000, title='', diff_waveform=None)`; export const saveEpochs = (workspaceDir: string, subject: string) => - window.pyodide.runPython( - `raw_epochs.save(${formatFilePath( - path.join( - workspaceDir, - 'Data', - subject, - 'EEG', - `${subject}-cleaned-epo.fif` - ) - )}` - ); + pyodideWorker.postMessage({ + data: `raw_epochs.save(${formatFilePath( + path.join(workspaceDir, 'Data', subject, 'EEG', `${subject}-cleaned-epo.fif`) + )}`, + }); diff --git a/app/utils/pyodide/pyimport.py b/app/utils/pyodide/pyimport.py deleted file mode 100644 index 8b137891..00000000 --- a/app/utils/pyodide/pyimport.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/app/utils/pyodide/pythonStrings.js b/app/utils/pyodide/statements.json similarity index 100% rename from app/utils/pyodide/pythonStrings.js rename to app/utils/pyodide/statements.json diff --git a/app/utils/pyodide/webworker.js b/app/utils/pyodide/webworker.js new file mode 100644 index 00000000..a5eccba5 --- /dev/null +++ b/app/utils/pyodide/webworker.js @@ -0,0 +1,37 @@ +/** + * This file has been copied from pyodide source and modified to allow + * pyodide to be used in a web worker within this + */ + +self.languagePluginUrl = './src'; +importScripts('./pyodide.js'); + +const onmessage = function(e) { + // eslint-disable-line no-unused-vars + languagePluginLoader.then(() => { + // Preloaded packages + self.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']).then(() => { + const data = e.data; + const keys = Object.keys(data); + for (let key of keys) { + if (key !== 'python') { + // Keys other than python must be arguments for the python script. + // Set them on self, so that `from js import key` works. + self[key] = data[key]; + } + } + + self.pyodide + .runPythonAsync(data.python, () => {}) + .then((results) => { + self.postMessage({ results }); + }) + .catch((err) => { + // if you prefer messages with the error + self.postMessage({ error: err.message }); + // if you prefer onerror events + // setTimeout(() => { throw err; }); + }); + }); + }); +}; From 4523001e2d6f5737672e9ca7d541b3bfebedaf41 Mon Sep 17 00:00:00 2001 From: Dano Morrison Date: Mon, 11 May 2020 02:18:46 -0400 Subject: [PATCH 22/40] Tidied up dependency bugs related to pyodide rebase --- internals/scripts/InstallPyodide.js | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js index 90461ec8..2b6f34a1 100644 --- a/internals/scripts/InstallPyodide.js +++ b/internals/scripts/InstallPyodide.js @@ -1,9 +1,3 @@ -// mkdir app/utils/pyodide/src -// && cd app/utils/pyodide/src -// curl -LJO https://github.com/iodide-project/pyodide/releases/download/0.12.0/pyodide-build-0.12.0.tar.bz2 -// tar xjf pyodide-build-0.12.0.tar.bz2 -// rm pyodide-build-0.12.0.tar.bz2", - import chalk from 'chalk'; import fs from 'fs'; import https from 'https'; @@ -17,7 +11,7 @@ const TAR_NAME = `pyodide-build-${PYODIDE_VERSION}.tar.bz2`; const TAR_URL = `https://github.com/iodide-project/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-build-${PYODIDE_VERSION}.tar.bz2`; const PYODIDE_DIR = 'app/utils/pyodide/src/'; -const writeAndUnzipFile = response => { +const writeAndUnzipFile = (response) => { const filePath = `${PYODIDE_DIR}${TAR_NAME}`; const writeStream = fs.createWriteStream(filePath); response.pipe(writeStream); @@ -38,12 +32,8 @@ const writeAndUnzipFile = response => { }); }; -const downloadFile = response => { - if ( - response.statusCode > 300 && - response.statusCode < 400 && - response.headers.location - ) { +const downloadFile = (response) => { + if (response.statusCode > 300 && response.statusCode < 400 && response.headers.location) { if (url.parse(response.headers.location).hostname) { https.get(response.headers.location, writeAndUnzipFile); } else { @@ -59,14 +49,10 @@ const downloadFile = response => { (() => { if (fs.existsSync(`${PYODIDE_DIR}${TAR_NAME}`)) { - console.log( - `${chalk.green.bold(`Pyodide is already present: ${PYODIDE_VERSION}...`)}` - ); + console.log(`${chalk.green.bold(`Pyodide is already present: ${PYODIDE_VERSION}...`)}`); return; } - console.log( - `${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}` - ); + console.log(`${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}`); mkdirp.sync(`app/utils/pyodide/src`); https.get(TAR_URL, downloadFile); })(); From 446bfa46c9c19c0b90579dd0b871c1ccb087bcaf Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Wed, 14 Oct 2020 22:13:19 -0400 Subject: [PATCH 23/40] additional cleanup from rebase TODO: - jupyterEpics.ts --> pyodideEpics.ts, rm pyodideEpics.js - combine pyodideActions.js into pyodideAcions.ts - pass linting --- .../{jupyterActions.ts => pyodideActions.ts} | 16 +- app/components/AnalyzeComponent.tsx | 17 +- app/components/CleanComponent/index.tsx | 7 +- app/components/HomeComponent/index.tsx | 9 +- ...erPlotWidget.tsx => PyodidePlotWidget.tsx} | 2 +- app/constants/interfaces.ts | 10 - app/containers/AnalyzeContainer.ts | 2 +- app/containers/CleanContainer.ts | 2 +- app/containers/HomeContainer.ts | 4 +- app/epics/jupyterEpics.ts | 15 +- app/epics/pyodideEpics.js | 33 ++- app/reducers/index.ts | 6 +- app/reducers/pyodideReducer.js | 96 ------- .../{jupyterReducer.ts => pyodideReducer.ts} | 34 +-- app/store/configureStore.dev.js | 2 +- app/utils/filesystem/read.js | 20 +- app/utils/pyodide/index.js | 18 +- app/utils/pyodide/pipes.js | 16 +- app/utils/pyodide/pyodide.js | 264 ++++++++++-------- app/utils/pyodide/webworker.js | 6 +- configs/webpack.config.renderer.dev.babel.js | 7 +- environment.yml | 1 - internals/scripts/InstallPyodide.js | 14 +- package.json | 2 + 24 files changed, 263 insertions(+), 340 deletions(-) rename app/actions/{jupyterActions.ts => pyodideActions.ts} (71%) rename app/components/{JupyterPlotWidget.tsx => PyodidePlotWidget.tsx} (96%) delete mode 100644 app/reducers/pyodideReducer.js rename app/reducers/{jupyterReducer.ts => pyodideReducer.ts} (57%) diff --git a/app/actions/jupyterActions.ts b/app/actions/pyodideActions.ts similarity index 71% rename from app/actions/jupyterActions.ts rename to app/actions/pyodideActions.ts index d8b1867b..7f789df0 100644 --- a/app/actions/jupyterActions.ts +++ b/app/actions/pyodideActions.ts @@ -1,13 +1,11 @@ import { createAction } from '@reduxjs/toolkit'; import { ActionType } from 'typesafe-actions'; -import { JUPYTER_VARIABLE_NAMES } from '../constants/constants'; +import { PYODIDE_VARIABLE_NAMES } from '../constants/constants'; // ------------------------------------------------------------------------- // Actions -export const JupyterActions = { - LaunchKernel: createAction('LAUNCH_KERNEL'), - RequestKernelInfo: createAction('REQUEST_KERNEL_INFO'), +export const PyodideActions = { SendExecuteRequest: createAction( 'SEND_EXECUTE_REQUEST' ), @@ -19,14 +17,10 @@ export const JupyterActions = { LoadERP: createAction('LOAD_ERP'), LoadTopo: createAction('LOAD_TOPO'), CleanEpochs: createAction('CLEAN_EPOCHS'), - CloseKernel: createAction('CLOSE_KERNEL'), - SetKernel: createAction('SET_KERNEL'), - GetEpochsInfo: createAction( + GetEpochsInfo: createAction( 'GET_EPOCHS_INFO' ), GetChannelInfo: createAction('GET_CHANNEL_INFO'), - SetKernelStatus: createAction('SET_KERNEL_STATUS'), - SetKernelInfo: createAction('SET_KERNEL_INFO'), SetMainChannel: createAction('SET_MAIN_CHANNEL'), SetEpochInfo: createAction('SET_EPOCH_INFO'), SetChannelInfo: createAction('SET_CHANNEL_INFO'), @@ -45,6 +39,6 @@ export const JupyterActions = { ReceiveStream: createAction('RECEIVE_STREAM'), } as const; -export type JupyterActionType = ActionType< - typeof JupyterActions[keyof typeof JupyterActions] +export type PyodideActionType = ActionType< + typeof PyodideActions[keyof typeof PyodideActions] >; diff --git a/app/components/AnalyzeComponent.tsx b/app/components/AnalyzeComponent.tsx index 69501f70..7a1a2b7c 100644 --- a/app/components/AnalyzeComponent.tsx +++ b/app/components/AnalyzeComponent.tsx @@ -34,10 +34,10 @@ import { } from '../utils/behavior/compute'; import SecondaryNavComponent from './SecondaryNavComponent'; import ClickableHeadDiagramSVG from './svgs/ClickableHeadDiagramSVG'; -import JupyterPlotWidget from './JupyterPlotWidget'; +import PyodidePlotWidget from './PyodidePlotWidget'; import { HelpButton } from './CollectComponent/HelpSidebar'; import { Kernel } from '../constants/interfaces'; -import { JupyterActions } from '../actions/jupyterActions'; +import { PyodideActions } from '../actions/pyodideActions'; const ANALYZE_STEPS = { OVERVIEW: 'OVERVIEW', @@ -74,7 +74,7 @@ interface Props { [key: string]: string; }; - JupyterActions: typeof JupyterActions; + PyodideActions: typeof PyodideActions; } interface State { @@ -163,9 +163,6 @@ export default class Analyze extends Component { const workspaceCleanData = await readWorkspaceCleanedEEGData( this.props.title ); - if (this.props.kernelStatus === KERNEL_STATUS.OFFLINE) { - this.props.JupyterActions.LaunchKernel(); - } const behavioralData = await readWorkspaceBehaviorData(this.props.title); this.setState({ eegFilePaths: workspaceCleanData.map((filepath) => ({ @@ -200,7 +197,7 @@ export default class Analyze extends Component { selectedFilePaths: data.value, selectedSubjects: getSubjectNamesFromFiles(data.value), }); - this.props.JupyterActions.LoadCleanedEpochs(data.value); + this.props.PyodideActions.LoadCleanedEpochs(data.value); } } @@ -343,7 +340,7 @@ export default class Analyze extends Component { handleChannelSelect(channelName: string) { this.setState({ selectedChannel: channelName }); - this.props.JupyterActions.LoadERP(channelName); + this.props.PyodideActions.LoadERP(channelName); } handleStepClick(step: string) { @@ -480,7 +477,7 @@ export default class Analyze extends Component { - { - {