diff --git a/.eslintrc.yml b/.eslintrc.yml index c0442c4006..257496302f 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -3,7 +3,7 @@ env: amd: true extends: 'eslint:recommended' parserOptions: - ecmaVersion: 5 + ecmaVersion: 6 rules: strict: - error diff --git a/.travis.yml b/.travis.yml index faedee21d9..4de9861ba3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic sudo: required language: python python: @@ -45,10 +45,10 @@ install: - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - source activate test-environment -before_script: - - "export DISPLAY=:99.0" - - "sh -e /etc/init.d/xvfb start" - - sleep 3 # give xvfb some time to start +# before_script: +# - "export DISPLAY=:99.0" +# - "sh -e /etc/init.d/xvfb start" +# - sleep 3 # give xvfb some time to start script: - make build-travis-narrative diff --git a/Makefile b/Makefile index 56bd9bf4b5..624f399260 100755 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ REPO_NAME = narrative # Installer script INSTALLER = ./scripts/install_narrative.sh -INSTALL_VENV = narrative-venv BACKEND_TEST_SCRIPT = scripts/narrative_backend_tests.sh FRONTEND_TEST_DIR = test @@ -18,12 +17,14 @@ build-narrative-container: docker_image: build-narrative-container # Per PR #1328, adding an option to skip minification -dev_image: +dev-image: SKIP_MINIFY=1 DOCKER_TAG=dev sh $(DOCKER_INSTALLER) +run-dev-image: + ENV=$(ENV) bash scripts/local-dev-run.sh + install: - @echo "Installing local Narrative in the $(INSTALL_VENV) virtual environment" - bash $(INSTALLER) -v $(INSTALL_VENV) + bash $(INSTALLER) # runs the installer to locally build the Narrative in a # local venv. diff --git a/README.md b/README.md index 8889fa5987..d039600485 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,17 @@ conda activate my_narrative_environment ./scripts/install_narrative.sh kbase-narrative ``` +If the previous instructions do not work, try +``` +# source ~/anaconda3/bin/activate or wherever you have python installed +conda create -n my_narrative_environment +conda activate my_narrative_environment +sh scripts/install_narrative.sh +# scripts/install_narrative.sh +kbase-narrative +``` + + ### Or, without conda - this installs lots of requirements of specific versions and may clobber things on your PYTHONPATH. ``` @@ -81,6 +92,16 @@ To run authenticated tests, you'll need to get an auth token from KBase servers, Note: **DO NOT CHECK YOUR TOKEN FILE IN TO GITHUB**. You'll be shamed without mercy. +## Manual Testing + +* It can be useful to immediately see your changes in the narrative. For javascript changes, you will just have to reload the page. You can print messages to the console with `console.log` + +* For python changes, it will require shutting down the notebook, running `scripts/install_narrative.sh -u` and then starting the notebook server up again with kbase-narrative. You can print messages to the terminal using + +``` +log = logging.getLogger("tornado.application") +log.info("Your Logs Go Here") +``` ## Submitting code diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f0b0811128..59e06b93e9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -3,8 +3,17 @@ The Narrative Interface allows users to craft KBase Narratives using a combinati This is built on the Jupyter Notebook v6.0.2 (more notes will follow). +### Version 4.2.0 +- Updated the Narrative interface to connect to the remade Execution Engine. +- Updated the Narrative interface to streamline events and cookies connected to the Traefik update. +- Fixed an issue where job log browser state (running, stopped, scrolling) could cross browser sessions. +- Added a viewer for the SampleSet object. +- PTV-1446 - fix bug preventing KBaseFeatureValues viewer apps from working and displaying data + ### Version 4.1.2 -- Redirect to the interstitial loading page after shutting down a Narrative session, instead of letting the backend server do the redirect. +- Improve display of job logs. +- Prevent App cell elements from overflowing the page. +- Job status is now inaccessible before a job enters the queue. ### Version 4.1.1 - Fix sort order in Narratives panel - should be by most recently saved. diff --git a/bower.json b/bower.json index a8d7d13bc2..7e795e86d1 100644 --- a/bower.json +++ b/bower.json @@ -1,6 +1,6 @@ { "name": "kbase-narrative", - "version": "4.1.2", + "version": "4.2.0", "homepage": "https://kbase.us", "dependencies": { "bluebird": "3.4.7", diff --git a/docs/design/app-cells/about.md b/docs/design/app-cells/about.md index e69de29bb2..7ba2b4f2fd 100644 --- a/docs/design/app-cells/about.md +++ b/docs/design/app-cells/about.md @@ -0,0 +1,20 @@ +# Flow and Modules + +## 1. nbextensions +### Jupyter Notebook extensions +< blurb about Jupyter Notebook extensions here and what we're doing > + +### App Cell extension pattern +Detection +Initialization +Cell transformation + +## 2. App Cell Widget as controller +Main controller +Finite State Machine + +## 3. App Cell Tabs + +### A. Configure Tab + +### B. Job Status Tab diff --git a/docs/design/job_architecture.md b/docs/design/job_architecture.md index 026fbeb017..6c9430cad9 100644 --- a/docs/design/job_architecture.md +++ b/docs/design/job_architecture.md @@ -1,158 +1,506 @@ # Job Management Architecture +The Narrative job manager is based on a data flow that operates between the browser, the IPython Kernel behind the Narrative, and the KBase Execution Engine (EE2) behind that. In general, each bit of data flows between those three stops. + +In general, there's a single point of information flow on the front end and one on the backend. + +# Comm channels +Jupyter provides a "Comm" object that allows for custom messaging between the frontend and the kernel ([details and documentation here](https://jupyter-notebook.readthedocs.io/en/stable/comms.html)). This provides an interface for the frontend to directly request information from the kernel, and to listen to asynchronous responses. On the kernel-side, it allows one or more modules to register message handlers to process those requests out of the band of the usual kernel invocation. These are used to implement the Jupyter Notebook's ipywidgets, for example. + +The Narrative Interface uses one of these channels to manage job information. These are funneled through an interface on the frontend side and a matching one in the kernel. + +## Frontend Comm Channel +On the frontend, there's a `jobCommChannel.js` module that uses the MiniBus system to communicate. So, the following happens. Frontend modules use the bus system to send one of the following messages over the main channel, which then get interpreted and crafted into a message that gets passed through a kernel comm object. Most of these take one or more inputs. These are listed below the command, where applicable. + +This section is broken into two parts - kernel requests and kernel responses. Both of these are from the perspective of the frontend Javascript stack, using an AMD module and the Runtime object. The request parameters and examples are given first, then the responses below. + +## Bus requests +These messages are sent to the `JobCommChannel` on the front end, to get processed into messages sent to the kernel. +`ping-comm-channel` - sees that the comm channel is open through the websocket + +`request-job-status` - gets the status for a job, one time + * `jobId` - a string, the job id + * `parentJobId` - (optional) a string, the id of the requested job's "parent" job + +`request-job-update` - request the status for a job, but start an update cycle so that it's continually requested. + * `jobId` - a string, the job id + * `parentJobId` - (optional) a string, the id of the requested job's "parent" job + +`request-job-completion` - signal that the front end doesn't need any more updates for that job, so stop sending them for each loop cycle. Doesn't actually end the job, only requests for updates. + * `jobId` - a string, the job id + * `parentJobId` - (optional) a string, the id of the requested job's "parent" job + +`request-job-info` - request information about the job, specifically app id, spec, input parameters and (if finished) outputs + * `jobId` - a string, the job id + * `parentJobId` - (optional) a string, the id of the requested job's "parent" job + +`request-job-cancellation` - request that the server cancel the running job. + * `jobId` - a string, the job id + * `parentJobId` - (optional) a string, the id of the requested job's "parent" job + +`request-job-log` - request the job logs starting at some given line. + * `jobId` - a string, the job id + * `options` - an object, with attributes: + * `first_line` - the first line (0-indexed) to request + * `num_lines` - the number of lines to request (will get back up to that many if there aren't more) + +`request-latest-job-log` - request the latest several job log lines + * `jobId` - a string, the job id + * `options` - an object, with attributes: + * `num_lines` - the number of lines to request (will get back up to that many if there aren't more) + +### Usage Example +The comm channel is used through the main Bus object that's instantiated through the global `Runtime` object. That needs to be included in the `define` statement for all AMD modules. The bus is then used with its `emit` function (you have the bus *emit* a message to its listeners), and any inputs are passed along with it. + +Generally, this is used as follows (without much detail. For a readable real example, check out the `jobLogViewer.js` module): + +```Javascript +define( + ['common/runtime', ... other modules ...], + function(Runtime, ...others...) { + let runtime = Runtime.make(); + runtime.bus().emit('some-request', { + inputKey: 'value' + }); + } +); +``` + +Or, a more specific usage that requests the first 10 job log lines: +```Javascript +define( + ['common/runtime'], + function(Runtime) { + let runtime = Runtime.make(); + runtime.bus().emit('request-job-log', { + jobId: 'some_job_id', + options: { + first_line: 0, + num_lines: 10 + } + }); + } +); +``` + +## Bus responses +When the kernel sends a message to the front end, the only module set up to listen to them is the `JobCommChannel` as mentioned above. This takes the responses, unpacks them, and turns them into a response message that is passed back over the bus to any frontend Javascript module that listens to them. The message types are described below, along with the content that gets sent, followed by an example of how to make use of them. + +`job-status` - contains the current job state + * `jobId` - string, the job id + * `jobState` - object, describes the job state (see the **Data Structures** section below for the structure) + * `outputWidgetInfo` - object, contains the parameters to be sent to an output widget. This will be different for all widgets, depending on the App that invokes them. + +`job-deleted` - sent when a job has been deleted, but some information about it has been requested + * `jobId` - the id of the deleted job + * `via` - a string about why it's been deleted (generally "no_longer_exists") + +`job-info` - contains information about the current job + * `jobId` - string, the job id + * `jobInfo` - object, the job information object (see the **Data Structures** section below) + +`run-status` - updates the run status of the job - this is part of the initial flow of starting a job through the AppManager. + * TODO + +`job-canceled` - sent when a job has been canceled in the kernel, as a response to other messages + * `jobId` - string, the job id + * `via` - string, generally "job_canceled" + +`job-logs` - sent with information about some job logs. + * `jobId` - string, the job id + * `logs` - the raw message data from the kernel. (see the **Data Structures** section below) + * `latest` - if truthy, then these are the latest logs, if falsy, then they don't have to be the latest logs. + +`job-error` - sent in response to an error that happened on job information lookup, or another error that happened while processing some other message to the JobManager. + * `jobId` - string, the job id + * `message` - string, some message about the error + +`job-cancel-error` - a cancel request has thrown an error + * `jobId` - string, the job id + * `message` - string, a reason for the error + +`job-log-deleted` - a log request has thrown an error + * `jobId` - string, the job id + * `message` - string, a reason for the error + +`job-status-error` - a status request as thrown an error + * `jobId` - string, the job id + * `message` - string, a reason for the error + +`job-does-not-exist` - sent in response to a request for information about a job that doesn't exist. Jobs might not exist if (1) they have been previously canceled, or (2) a malformed request was sent. + * `jobId` - string, the job id + * `source` - string, the source of the message in the kernel (what service, or module, was invoked. Usually "JobManager" or "ExecutionEngine2") + +### Usage example +As in the Bus requests section above, the front end response handling is done through the Runtime bus. The bus provides both an `on` and a `listen` function, examples will show how to use both. Generally, the `listen` function is more specific and binds the listener to a specific bus channel. These channels can invoke the jobId, or the cellId, to make sure that only information about specific jobs is listened for. + +The `listen` function takes an object with three attributes as input - a channel (either the cellId or jobId), a key (with the type of message to listen for), and a handle, which is a function to process the message. This is probably the easiest way to handle messages. A usage would look like this: + +```Javascript +define(['common/runtime'], + function(Runtime) { + let runtime = Runtime.make(); + let listenerId = runtime.bus().listen({ + channel: { + jobId: 'some_job_id' + }, + key: { + type: 'job-status' + }, + handle: (message) => { + ...process the message... + } + }) + } +); +``` + +The `on` function requires a constructed channel bus, premade and reusable for a given channel. So you would make a channel bus that would always receive messages for that channel, and instruct it on what to do when a message of a given type arrives. That looks like this: +```Javascript +define(['common/runtime'], + function(Runtime) { + let runtime = Runtime.make(); + let cellBus = runtime.bus().makeChannelBus({ + name: { + cell: 'some_cell_id' + } + }); + let listenerId = cellBus.on('run-status', (message) => { + ...process the message... + }); + } +); +``` +Note that both of these create events that get bound to the DOM, and when the widget is removed, they should be cleaned up. This can be done by calling `bus.removeListener(id)` with the created `listenerId`. If you created a channel bus, then that bus should be used, otherwise the main runtime.bus() object should be used. + +## Kernel Comm Channel +On the kernel side, a complementary comm channel is used. This is set up in the `biokbase.narrative.jobs.jobcomm.JobComm` class. On Narrative load, page reload, or kernel restart, this is initialized to handle any messages sent to the kernel. The structure here is slightly different than the structure used on the front end. Likewise, all the message names are different. They all have a request string, most involve a job id, and that's it. The job logs request also have which line to start with and how many lines to get back. + +Note that these are autogenerated by the frontend `JobCommChannel` object, using the `Jupyter.kernel.comm` package. + +The actual message that the JobComm sees in the kernel has this format: +``` +{ + "msg_id": "some random string", + "content": { + "data": { + "request_type": "a string - see below", + "job_id": "not required, but present in most" + ... other keys, depending on message ... + } + } +} +``` +The point here is that all messages have a `request_type`, most are accompanied by a `job_id`, and a few have some extra info. But they're in a flat structure that's formatted by the Jupyter kernel. + +## Messages sent to the kernel +These are organized by the `request_type` field, followed by the expected response message. Additional parameters and their formats are given as a list below the request name. E.g. the `job_status` message will be sent as: +```json +{ + "msg_id": "some string", + "content": { + "data": { + "request_type": "job_status", + "job_id": "a_job_id", + "parent_job_id": "another_job_id" + } + } +} +``` + +`all_status` - request the status of all currently running jobs, responds with `job_status_all` + +`job_status` - request a single job status, responds with `job_status` +* `job_id` - string, +* `parent_job_id` - optional string + +`start_update_loop` - request starting the global job status update thread, no specific response, but generally with `job_status_all` + +`stop_update_loop` - request stopping the global job status update thread, no response + +`start_job_update` - request updating a single job during the update thread, no specific response, but generally with `job_status` +* `job_id` - string +* `parent_job_id` - optional string + +`stop_job_update` - request halting update for a single job during the update thread, no response +* `job_id` - string +* `parent_job_id` - optional string + +`job_info` - request general information about a job, responds with `job_info` +* `job_id` - string +* `parent_job_id` - optional string + +`job_logs` - request job log information, responds with `job_logs` +* `job_id` - string +* `parent_job_id` - optional string +* `first_line` - int >= 0, +* `num_lines` - int > 0 + +`job_logs_latest` - request the latest set of lines from job logs, responds with `job_logs` +* `job_id` - string +* `parent_job_id` - optional string +* `num_lines` - int > 0 + + +## Messages sent from the kernel to the browser +These are all caught by the `JobCommChannel` on the browser side, then parsed and sent as the bus messages described above. Like other kernel messages, they have a `msg_type` field, and a `content` field containing data meant for the frontend to use. They have a rough structure like this: + +```json +{ + "data": { + "msg_type": "some_message", + "content": { + "key1": "value1", + "key2": "value2" + } + } +} +``` + +a specific example: +```json +{ + "msg_id": "some_string", + "data": { + "msg_type": "job_status", + "content": { + "state": { + "status": "running", + ... other state keys ... + }, + "spec": {}, + "widget_info": {} + } + } +} +``` + +These are described below. The name (`msg_type`) is given, followed by the keys given in the `content` block. + +By design, these should only be seen by the `JobCommChannel` instance, then sent into bus messages that get sent on specific channels. That information is also given in each block. + +### `job_does_not_exist` +This is an error message triggered when trying to get info/state/logs on a job that either doesn't exist in EE2 or that the JobManager doesn't have associated with the running narrative. + +**content** + * `job_id` - a string, the job id + * `source` - string, the source of the error + +**bus** `job-does-not-exist` + +### `job_comm_error` +A general job comm error, capturing most errors that get thrown by the kernel + +**content** (this varies, but usually includes the below) + * `request_type` - the original request message that wound up in an error + * `job_id` - string, the job id (if present) + * `message` - string, an error message + +**bus** one of `job-cancel-error`, `job-log-deleted`, `job-status-error`, `job-error` + +### `job_status_all` +The set of all job states for all running jobs, or at least the set that should be updated (those that are complete and not requested by the front end are not included - if a job is sitting in an error or finished state, it doesn't need ot have its app cell updated) + +**content** - all of the below are included, but the top-level keys are all job id strings, e.g.: +```json +{ + "job_id_1": { ...contents... }, + "job_id_2": { ...contents... } +} +``` + * `state` - the job state (see the **Data Structures** section below for details + * `widget_info` - the parameters to send to output widgets, only available for a completed job + * `owner` - string, username of user who submitted the job + +**bus** - a series of `job-status` or `job-deleted` messages + +### `job_info` +Includes information about the running job + +**content** + * `app_id` - string, the app id (format = `module_name/app_name`) + * `app_name` - string, the human-readable app name + * `job_id` - string, the job id + * `job_params` - the unstructured set of parameters sent to the execution engine + +**bus** - `job-info` + +### `job_status` +The current job state. This one is probably most common. + +**content** + * `state` - see **Data Structures** below for details (it's big and shouldn't be repeated all over this document) + * `widget_info` - the parameters to send to output widgets, only available for a completed job + * `owner` - string, username of user who submitted the job + +**bus** - `job-status` + +### `job_logs` +Includes log statement information for a given job. + +**content** + * `job_id` - string, the job id + * `latest` - boolean, `true` if this is just the latest set of logs + * `first` - int, the index of first line included in the set + * `max_lines` - int, the total log lines available in the server + * `lines` - list of log line objects, each one has the following keys: + * `line` - string, the log line + * `is_error` - 0 or 1, if 1 then the line is an "error" as reported by the server + +**bus** `job-logs` + +### `new_job` +Sent when a new job is launched and serialized. This just triggers a save/checkpoint on the frontend - no other bus message is sent + +### `run_status` +Sent during the job startup process. There are a few of these containing various startup status, including errors (if they happen). + +**content** +All cases: + * `event` - string, what's the run status + * `event_at` - string, timestamp + * `cell_id` - the app cell id (used for routing) + * `run_id` - the run id of the app (autogenerated by the cell) + +(if error) + * `event` - string, "error", + * `event_at` - string, timestamp + * `error_message` - string, the error + * `error_type` - string, the type of Exception that was raised. + * `error_stacktrace` - string, a stacktrace + * `error_code` - int, an error code + * `error_source` - string, the "source" of the error (generally "appmanager") + +(if ok) + * `job_id` - if the job was launched successfully + +**bus** `run-status` + +### `result` +Sent at the end of a `AppManager.run_dynamic_service` call (of which there aren't many). + +**content** + * `cell_id` - the app cell id (used for routing) + * `run_id` - the run id of the app (autogenerated by the cell) + * `event_at` - string, timestamp + * `result` - the result of the dynamic service call (some unspecified object) + +**bus** `result` - sent to `cell_id` channel ## Job Management flow on backend (in IPython kernel, biokbase.narrative.jobs package) -This is mostly linear. +These steps define the process of creating a new app running job. 1. User clicks "Run" on App Cell in browser. * Cell provides app_id, cell_id, run_id, and parameters. * Invokes biokbase.narrative.appmanager.AppManager.run_app. -2. `AppManager.run_app` validation: +2. `AppManager.run_app` validates the following bits of information before passing them on to EE2: * App (based on id, version, and spec). - * Parameters. + * Parameters (based on the app spec). 3. `AppManager.run_app` preparation and start: - * Convert app params from user-readable to machine-understandable (via spec input_mapping). + * Convert app params from user-readable to machine-understandable (via the spec input_mapping). * Fetch cell id, run id, workspace id, user token. - * Create an agent token on behalf of the user. + * Create an agent token on behalf of the user. This effectively makes a new authentication token that has a two-week lifetime, separate from the current login token. For example, if the user's current login token has a remaining lifespan of 1 hour, then the new job will be able to continue long past that. * Submit all of the above to `NarrativeJobService.run_job`. 4. Get response from `NarrativeJobService.run_job` * Combine with info from step 3, create `biokbase.narrative.jobs.job.Job` object. * submit new `Job` to `biokbase.narrative.jobs.jobmanager.JobManager` singleton object. -5. `JobManager` fetches job status and pushes it to browser. +5. `AppManager` tells the `JobComm` channel to (1) fetch the new job status and push it to the browser, and (2) start the job lookup loop for the newly created job. (calls `AppManager.register_new_job`) -JobManager initialization and startup. +## JobManager initialization and startup. +These steps take place whenever the user loads a narrative, or when the kernel is restarted. This ensures that the JobManager in the kernel is kept up-to-date on Job status. 1. User starts kernel (opens a Narrative, or clicks Kernel -> Restart) -2. `kbaseNarrativeJobPanel` (invisible front end widget) executes kernel call to `JobManager().initialize_jobs()` +2. `jobCommsChannel` (front end widget) executes the following kernel call: `JobManager().initialize_jobs(); JobComm().start_update_loop` 3. `JobManager` does: * Get current user and workspace id. - * `UserAndJobState.list_jobs2` with the workspace id - gets the list of jobs in that workspace - * `NarrativeJobService.check_jobs` with the list of job ids, and request for job params, esp. metadata. - * Call `_lookup_all_job_status` to push status of all jobs forward to front end. - * (optional) Call `_lookup_job_status_loop` to start a lookup thread (uses a Python `Timer` to lookup job status) - -JobManager status lookup loop. -1. Calls either `_lookup_all_job_status` or `_lookup_job_status_loop` (which itself calls the former) -2. Build a list of job ids to lookup - those that are flagged for lookup. -3. Calls `_construct_job_status_set` - * Calls `_get_all_job_states` + * `ExecutionEngine2.check_workspace_jobs` with the workspace id - gets the set of jobs in that workspace, and builds them into `Job` objects. +4. `JobComm` does: + * Starts the lookup loop thread. + * On the first pass, this looks up status of all jobs and pushes them forward to the browser. + * If any jobs are in a terminal state, they'll stopped being looked up automatically. If all jobs are terminated, then the loop thread itself stops. + +## JobComm status lookup loop. +1. Calls `JobComm._lookup_job_status_loop`, which in turn calls `JobComm.lookup_all_job_states`. This gets forwarded to `JobManager.lookup_all_job_states`, and the results pushed to the browser as a comm channel message. +2. Internal to `JobManager.lookup_all_job_states`, the following steps happen: + * Build a list of job ids to lookup - those that are flagged for lookup. + * Call `_construct_job_status_set` + * Call `_get_all_job_states` * Gets list of all job ids the JM is tracking. * Retrieves some states from cache (cache is used for finalized jobs). * Calls `NarrativeJobService.check_jobs` on everything that's not finalized. * Injects `run_id` and `cell_id` into states * Returns dict of states. -4. Sends result to browser over comm channel - +3. Sends result to browser over comm channel +4. Since this runs in the background on the kernel-side, it removes any need for the browser to constantly poll. ## Data Structures ### Job state -In kernel, as retrieved from NJS.check_job +In kernel, as retrieved from EE2.check_job (described by example) ```json { - "status": [ - "2019-05-07T22:42:41+0000", - "started", - "queued", - null, - null, - 0, - 0 - ], - "job_id": "5cd209dcaa5a4d298c5dc1c2", - "job_state": "queued", - "creation_time": 1557268961909, - "ujs_url": "https://ci.kbase.us/services/userandjobstate/", - "finished": 0, - "sub_jobs": [], - "awe_job_state": "queued" + "user": "wjriehl", + "authstrat": "kbaseworkspace", + "wsid": 46214, + "status": "queued", + "updated": 1583863267977, + "queued": 1583863267977, + "scheduler_type": "condor", + "scheduler_id": "14221", + "job_input": { + "wsid": 46214, + "method": "simpleapp.simple_add", + "params": [ + { + "workspace_name": "wjriehl:narrative_1580237536246", + "base_number": 5 + } + ], + "service_ver": "f5a7586776c31b05ae3cc6923c2d46c25990d20a", + "app_id": "simpleapp/example_method", + "source_ws_objects": [], + "parent_job_id": "None", + "requirements": { + "clientgroup": "njs", + "cpu": 4, + "memory": 23000, + "disk": 100 + }, + "narrative_cell_info": { + "run_id": "d7558838-a712-42d3-9511-c4b95f3651fe", + "token_id": "e80f4f81-b7bb-4483-a92b-b1e0200f8a20", + "tag": "beta", + "cell_id": "c04c19bd-20ce-41be-b793-50f84de8f60b" + } + }, + "job_id": "5e67d5e395d1f00a7cf4ea21", + "created": 1583863267000 } ``` As sent to browser, includes cell info and run info ``` { - owner: string (username), + owner: string (username, who started the job), spec: app spec (optional) widget_info: (if not finished, None, else...) job.get_viewer_params result state: { - job_state: string, - error (if present): dict of error info, - cell_id: string/None, - run_id: string/None, - awe_job_id: string/None, - canceled: 0/1 - creation_time: epoch second - exec_start_time: epoch/none, - finish_time: epoch/none, - finished: 0/1, job_id: string, - status: (from UJS) [ - timestamp(last_update, string), - stage (string), - status (string), - progress (string/None), - est_complete (string/None), - complete (0/1), - error (0/1) - ], - ujs_url: string + status: string, + created: epoch ms, + updated: epoch ms, + queued: optional - epoch ms, + finished: optional - epoch ms, + terminated_code: optional - int, + tag: string (release, beta, dev), + parent_job_id: optional - string or null, + run_id: string, + cell_id: string, + errormsg: optional - string, + error (optional): { + code: int, + name: string, + message: string (should be for the user to read), + error: string, (likely a stacktrace) + }, + error_code: optional - int } } ``` - - -## Application, execution cycle notes: - -User clicks App Panel -> App Cell inserted. Done! - -User clicks "Run" on App Cell (or otherwise executes a function with a cell_id stuck to it) -> -App Cell executes code (should disable the Run button) -> -Kernel goes through AppManager.run_app steps: - sends over comm channel: - 1. run_status with serialized events: - a. validating_app - b. validated_app - c. launching_job - d. launched_job - 2. job_status - 3. new_job (empty, triggers a save) - 4. run_app_error (on NJSW.run_job failure) --> -Job Panel catches these and translates messages before sending out over Bus: - run_status -> run-status - job_status -> job-status - new_job -> null - run_app_error -> ... nothing? Should get a message. - -App Cell catching messages: -run-status -> updates FSM from launching..... launched Job (with an id in job-status), now listens to jobId channel -job-status -> updates copy of job state, elapsed time, display of state -job-status (with terminal status) -> updates job state, expectes no more changes, creates output cell and area - -JobManager kernel loop: - loops over all jobs, gets state, sends job_status for all - sends job_err for individual jobs if - 1. job is missing (e.g. JobManager maintains a handle, but job.state() fails because NJS can't find it - 2. Network error - JobPanel translates messages - -Job Canceling: -Permissions! - View permissions - no permission to touch jobs - Edit permissions - can cancel jobs, start jobs, delete own started job, not delete other's started jobs - Admin - can cancel, delete, view, start - -User cancels job - either in App Cell Cancel button or Job Panel Cancel button - sends cancel-job to JobPanel - sends cancel_job to kernel - JobManager tries to cancel the job - sends job_canceled if successful (or already canceled) - sends job_err if not - with sensible reason - -User deletes job - either in App Cell or Job Panel - sends delete-job to JobPanel - sends delete_job to kernel - JobManager tries to cancel the job (if not in a completed state): - sends job_canceled if successful - sends job_err if not - JobManager tries to delete the job: - sends job_deleted if successful - sends job_err if not diff --git a/kbase-extension/static/ext_packages/jquery-extensions/js/jquery.cookie.min.js b/kbase-extension/static/ext_packages/jquery-extensions/js/jquery.cookie.min.js deleted file mode 100644 index 7401208d4e..0000000000 --- a/kbase-extension/static/ext_packages/jquery-extensions/js/jquery.cookie.min.js +++ /dev/null @@ -1,8 +0,0 @@ -/*! - * jQuery Cookie Plugin v1.3.1 - * https://github.com/carhartl/jquery-cookie - * - * Copyright 2013 Klaus Hartl - * Released under the MIT license - */ -(function(a,b,c){function e(a){return a}function f(a){return g(decodeURIComponent(a.replace(d," ")))}function g(a){return 0===a.indexOf('"')&&(a=a.slice(1,-1).replace(/\\"/g,'"').replace(/\\\\/g,"\\")),a}function h(a){return i.json?JSON.parse(a):a}var d=/\+/g,i=a.cookie=function(d,g,j){if(g!==c){if(j=a.extend({},i.defaults,j),null===g&&(j.expires=-1),"number"==typeof j.expires){var k=j.expires,l=j.expires=new Date;l.setDate(l.getDate()+k)}return g=i.json?JSON.stringify(g):g+"",b.cookie=[encodeURIComponent(d),"=",i.raw?g:encodeURIComponent(g),j.expires?"; expires="+j.expires.toUTCString():"",j.path?"; path="+j.path:"",j.domain?"; domain="+j.domain:"",j.secure?"; secure":""].join("")}for(var m=i.raw?e:f,n=b.cookie.split("; "),o=d?null:{},p=0,q=n.length;q>p;p++){var r=n[p].split("="),s=m(r.shift()),t=m(r.join("="));if(d&&d===s){o=h(t);break}d||(o[s]=h(t))}return o};i.defaults={},a.removeCookie=function(b,c){return null!==a.cookie(b)?(a.cookie(b,null,c),!0):!1}})(jQuery,document); \ No newline at end of file diff --git a/kbase-extension/static/kbase/css/kbaseJobLog.css b/kbase-extension/static/kbase/css/kbaseJobLog.css index 3da3e65501..e217362f96 100644 --- a/kbase-extension/static/kbase/css/kbaseJobLog.css +++ b/kbase-extension/static/kbase/css/kbaseJobLog.css @@ -16,7 +16,6 @@ .kblog-header { display: flex; font-family: monospace; - font-size: 85%; } .kblog-line { @@ -50,6 +49,7 @@ .kblog-text { word-wrap: break-word; + overflow-wrap: break-word; flex: 1; margin-left: 6px; } diff --git a/kbase-extension/static/kbase/js/api/auth.js b/kbase-extension/static/kbase/js/api/auth.js index 264b9bfcaa..37f62d9f81 100644 --- a/kbase-extension/static/kbase/js/api/auth.js +++ b/kbase-extension/static/kbase/js/api/auth.js @@ -1,14 +1,128 @@ define([ 'bluebird', 'jquery', - 'util/string', - 'jqueryCookie' -], function(Promise, $, StringUtil) { + 'narrativeConfig' +], function ( + Promise, + $, + Config +) { 'use strict'; function factory(config) { - var url = config.url; - var cookieName = 'kbase_session'; + const url = config.url; + const secureCookies = typeof config.secureCookies === 'undefined' ? true : config.secureCookies; + + /* + Each cookie is defined + */ + const cookieConfig = { + auth: { + name: 'kbase_session' + }, + backup: { + name: 'kbase_session_backup', + domain: 'kbase.us', + enableIn: ['prod'] + }, + narrativeSession: { + name: 'narrative_session' + } + }; + + const TOKEN_AGE = 14; // days + + /** + * Meant for managing auth or session cookies (mainly auth cookies as set by + * a developer working locally - which is why this is very very simple). + * Get a cookie "object" (key-value pairs) as input. + * If it's missing name or value, does nothing. + * Default expiration time is 14 days. + * domain, expires, and max-age are optional + * expires is expected to be in days + * auto set fields are: + * - path = '/' + * - expires = TOKEN_AGE (default 14) days + * @param {object} cookie + * - has the cookie keys: name, value, path, expires, max-age, domain + * - adds secure=true, samesite=none for KBase use. + */ + function setCookie(cookie) { + if (!cookie.name) { + return; + } + const name = encodeURIComponent(cookie.name); + const value = encodeURIComponent(cookie.value || ''); + const props = { + expires: TOKEN_AGE, // gets translated to GMT string + path: '/', + samesite: 'none' + }; + if (Number.isInteger(cookie.expires)) { + props.expires = cookie.expires; + } + + // Default to secure cookies global setting if not specified. + if (typeof cookie.secure === 'undefined') { + cookie.secure = secureCookies; + } + + if (cookie.domain) { + props.domain = cookie.domain; + } + props['max-age'] = 86400 * props.expires; + if (props.expires === 0) { + props.expires = new Date(0).toUTCString(); + } else { + props.expires = new Date(new Date().getTime() + (86400000*props.expires)).toUTCString(); + } + + const fields = Object.keys(props).map((key) => { + return `${key}=${props[key]}`; + }); + + if (cookie.secure) { + fields.push('secure'); + } + + const propStr = fields.join(';'); + + const newCookie = `${name}=${value}; ${propStr}`; + document.cookie=newCookie; + } + + /** + * If present in the browser, returns the value for the cookie. If not present, returns undefined. + * @param {string} name + */ + function getCookie(name) { + const allCookies = {}; + document.cookie.split(';').forEach((cookie) => { + const parts = cookie.trim().split('='); + allCookies[parts[0]] = parts[1]; + }); + return allCookies[name] || null; + } + + /** + * Removes a cookie from the browser. Meant for removing auth token and narrative session + * cookies on logout. + * @param {string} name + * @param {string} path + * @param {string || undefined} domain + */ + function removeCookie(name, path, domain) { + const cookieToRemove = { + name, + value: '', + path, + expires: 0 + }; + if (domain) { + cookieToRemove.domain = domain; + } + setCookie(cookieToRemove); + } /** * Does a GET request to get the profile of the currently logged in user. @@ -46,30 +160,57 @@ define([ * Returns null if not logged in. */ function getAuthToken() { - if (!$.cookie(cookieName)) { - return null; - } - return $.cookie(cookieName); + return getCookie(cookieConfig.auth.name); } /* Sets the given auth token into the browser's cookie. * Does nothing if the token is null. */ function setAuthToken(token) { - if (token) { - $.cookie(cookieName, token, {path: '/', domain: 'kbase.us', expires: 60}); - $.cookie(cookieName, token, {path: '/', expires: 60}); + const deployEnv = Config.get('environment'); + + function setToken(config) { + // Honor cookie host whitelist if present. + if (config.enableIn) { + if (config.enableIn.indexOf(deployEnv) === -1) { + return; + } + } + const cookieField = { + name: config.name, + value: token + }; + if (config.domain) { + cookieField.domain = config.domain; + } + setCookie(cookieField); } + + setToken(cookieConfig.auth); + setToken(cookieConfig.backup); } - /* Deletes the auth token cookie */ + /* Deletes the auth cookies */ function clearAuthToken() { - $.removeCookie(cookieName, {path: '/'}); - $.removeCookie(cookieName, {path: '/', domain: 'kbase.us'}); + const deployEnv = Config.get('environment'); + + function removeToken(config) { + // Honor the cookie host whitelist if present. + if (config.enableIn) { + if (config.enableIn.indexOf(deployEnv) === -1) { + return; + } + } + removeCookie(config.name, '/', config.domain); + } + + Object.keys(cookieConfig).forEach((name) => { + removeToken(cookieConfig[name]); + }); } function revokeAuthToken(token, id) { - var operation = '/tokens/revoke/' + id; + const operation = '/tokens/revoke/' + id; return makeAuthCall(token, { operation: operation, method: 'DELETE' @@ -82,8 +223,8 @@ define([ if (!token) { token = getAuthToken(); } - let encodedUsers = users.map(u => encodeURIComponent(u)); - var operation = '/users/?list=' + encodedUsers.join(','); + const encodedUsers = users.map((u) => encodeURIComponent(u)); + const operation = '/users/?list=' + encodedUsers.join(','); return makeAuthCall(token, { operation: operation, method: 'GET' @@ -94,7 +235,7 @@ define([ if (!token) { token = getAuthToken(); } - var operation = '/users/search/' + query; + let operation = '/users/search/' + query; if (options) { operation += '/?fields=' + options.join(','); } @@ -129,18 +270,19 @@ define([ * parameters as expected. */ function makeAuthCall(token, callParams) { - var version = callParams.version || 'V2', - callString = [ - url, - '/api/', - version, - callParams.operation - ].join(''); + const version = callParams.version || 'V2'; + const callString = [ + url, + '/api/', + version, + callParams.operation + ].join(''); return Promise.resolve($.ajax({ url: callString, method: callParams.method, dataType: 'json', + crossDomain: true, headers: { 'Authorization': token, 'Content-Type': 'application/json' @@ -165,36 +307,37 @@ define([ retries = 3; } return getTokenInfo(token) - .then(function(info) { - if (info.expires && info.expires > new Date().getTime()) { - return true; - } - return false; - }) - .catch(function(error) { - if (error.status === 401 || error.status === 403 || retries < 0) { + .then(function (info) { + if (info.expires && info.expires > new Date().getTime()) { + return true; + } return false; - } - else { - throw error; - } - }); + }) + .catch(function (error) { + if (error.status === 401 || error.status === 403 || retries < 0) { + return false; + } + else { + throw error; + } + }); } return { - putCurrentProfile: putCurrentProfile, - getCurrentProfile: getCurrentProfile, + putCurrentProfile, + getCurrentProfile, getUserProfile: getCurrentProfile, - getAuthToken: getAuthToken, - setAuthToken: setAuthToken, - clearAuthToken: clearAuthToken, - revokeAuthToken: revokeAuthToken, - getTokenInfo: getTokenInfo, - getUserNames: getUserNames, - searchUserNames: searchUserNames, - validateToken: validateToken + getAuthToken, + setAuthToken, + clearAuthToken, + revokeAuthToken, + getTokenInfo, + getUserNames, + searchUserNames, + validateToken, + setCookie, + getCookie }; - } return { diff --git a/kbase-extension/static/kbase/js/common/jobs.js b/kbase-extension/static/kbase/js/common/jobs.js index a8db5ad5f8..319c8ccce2 100644 --- a/kbase-extension/static/kbase/js/common/jobs.js +++ b/kbase-extension/static/kbase/js/common/jobs.js @@ -28,7 +28,7 @@ define([ /* * Strip out console commands from text captured from console: * http://search.cpan.org/~jlmorel/Win32-Console-ANSI-1.10/lib/Win32/Console/ANSI.pm - * + * */ function consoleToText(consoleText) { return consoleText.replace(/\[([\s\S]*?)m/g, ''); @@ -95,7 +95,7 @@ define([ } /* - * For a given job, returns the log lines after "skip" lines, as an Promise + * For a given job, returns the log lines after "skip" lines, as an Promise * which will deliver an array of strings. */ function getLogData(jobId, skip) { @@ -112,9 +112,28 @@ define([ throw new Error("Method is not supported anymore"); } + /** + * A jobState is deemed valid if + * 1. It's an object (not an array or atomic type) + * 2. It has a created key + * 3. It has a job_id key + * 4. There's others that are necessary, but the top two are sufficient to judge if it's valid + * enough and up to date. This function should be updated as necessary. + * + * This is intended to be used to make sure that jobStates are of the latest version of the + * execution engine. + * @param {object} jobState + */ + function isValidJobState(jobState) { + if (typeof jobState === 'object' && jobState !== null) { + return jobState.hasOwnProperty('created') && jobState.hasOwnProperty('job_id'); + } + return false; + } return { getLogData: getLogData, - deleteJob: deleteJob + deleteJob: deleteJob, + isValidJobState: isValidJobState }; -}); \ No newline at end of file +}); diff --git a/kbase-extension/static/kbase/js/common/props.js b/kbase-extension/static/kbase/js/common/props.js index cabaa129f8..55aae8e636 100644 --- a/kbase-extension/static/kbase/js/common/props.js +++ b/kbase-extension/static/kbase/js/common/props.js @@ -63,7 +63,7 @@ define([], function() { temp[propKey] = []; } temp = temp[propKey]; - // Finally set the property. + // Finally set the property. if (typeof temp === 'object' && temp.push) { temp.push(value); @@ -130,7 +130,7 @@ define([], function() { timer, api; /* - * In enabled by setting an update handler via the onUpdate factory + * In enabled by setting an update handler via the onUpdate factory * configuration property, this function should be run whenever the * property is updated. It will then run the update handler callback. * This is a way to enable essentially synchronization of the props @@ -271,37 +271,6 @@ define([], function() { return temp[propKey]; } - // function pushItem(path, value) { - // if (typeof path === 'string') { - // path = path.split('.'); - // } - // if (path.length === 0) { - // return; - // } - // var propKey = path.pop(), - // key, temp = obj; - // while (path.length > 0) { - // key = path.shift(); - // if (temp[key] === undefined) { - // temp[key] = {}; - // } - // temp = temp[key]; - // } - // ensureHistory(); - // if (temp[propKey] === undefined) { - // temp[propKey] = [value]; - // } else { - // if (temp[propKey]) - // if (isArray(temp[propKey])) { - // temp[propKey].push(value); - // } else { - // throw new Error('Can only push onto an Array'); - // } - // } - // run(); - // return temp[propKey]; - // } - function deleteItem(path) { if (typeof path === 'string') { path = path.split('.'); @@ -354,4 +323,4 @@ define([], function() { pushDataItem: pushDataItem, popDataItem: popDataItem }; -}); \ No newline at end of file +}); diff --git a/kbase-extension/static/kbase/js/kbaseNarrative.js b/kbase-extension/static/kbase/js/kbaseNarrative.js index 0d2f5857a7..db7addf7c0 100644 --- a/kbase-extension/static/kbase/js/kbaseNarrative.js +++ b/kbase-extension/static/kbase/js/kbaseNarrative.js @@ -15,6 +15,7 @@ define([ 'bluebird', 'handlebars', 'narrativeConfig', + 'jobCommChannel', 'kbaseNarrativeSidePanel', 'kbaseNarrativeOutputCell', 'kbaseNarrativeWorkspace', @@ -49,6 +50,7 @@ define([ Promise, Handlebars, Config, + JobCommChannel, KBaseNarrativeSidePanel, KBaseNarrativeOutputCell, KBaseNarrativeWorkspace, @@ -782,24 +784,20 @@ define([ this.sidePanel.render(); }.bind(this)); - $([Jupyter.events]).trigger('loaded.Narrative'); $([Jupyter.events]).on('kernel_ready.Kernel', - function () { - console.log('Kernel Ready! Initializing Job Channel...'); + (e) => { this.loadingWidget.updateProgress('kernel', true); + this.jobCommChannel = new JobCommChannel(); // TODO: This should be an event "kernel-ready", perhaps broadcast // on the default bus channel. - this.sidePanel.$jobsWidget.initCommChannel() - .then(function () { - this.loadingWidget.updateProgress('jobs', true); - }.bind(this)) - .catch(function (err) { + this.jobCommChannel.initCommChannel() + .then(() => this.loadingWidget.updateProgress('jobs', true)) + .catch((err) => { // TODO: put the narrative into a terminal state console.error('ERROR initializing kbase comm channel', err); KBFatal('Narrative.ini', 'KBase communication channel could not be initiated with the back end. TODO'); - // this.loadingWidget.remove(); - }.bind(this)); - }.bind(this) + }); + } ); }.bind(this)); }; diff --git a/kbase-extension/static/kbase/js/util/bootstrapSearch.js b/kbase-extension/static/kbase/js/util/bootstrapSearch.js index 3ffc846ac9..0219305f82 100644 --- a/kbase-extension/static/kbase/js/util/bootstrapSearch.js +++ b/kbase-extension/static/kbase/js/util/bootstrapSearch.js @@ -14,6 +14,7 @@ * inputFunction - gets fired off when a user inputs something. * addonFunction - gets fired off when a user clicks the addon area. default = clear input. * escFunction - gets fired off if escape (key 27) is hit while the input is focused. + * delay - time in ms before firing the input function (default 300) */ define([ @@ -46,6 +47,9 @@ define([ if (!options.filledIcon.startsWith('fa-')) { options.filledIcon = 'fa-' + options.filledIcon; } + if (!options.delay) { + options.delay = 300; + } this.options = options; this.initialize($target); @@ -89,7 +93,7 @@ define([ if (Jupyter && Jupyter.narrative) { Jupyter.narrative.disableKeyboardManager(); } - }).on('input change', function (e) { + }).on('input', function (e, ignoreDelay) { if ($input.val()) { $addonIcon.removeClass(self.options.emptyIcon); $addonIcon.addClass(self.options.filledIcon); @@ -99,7 +103,16 @@ define([ $addonIcon.addClass(self.options.emptyIcon); } if (self.options.inputFunction) { - self.options.inputFunction(e); + if (self.delayTimeout) { + clearTimeout(self.delayTimeout); + } + if (ignoreDelay) { + return self.options.inputFunction(e); + } + self.delayTimeout = setTimeout( + () => self.options.inputFunction(e), + self.options.delay + ); } }).on('keyup', function (e) { if (e.keyCode === 27) { @@ -112,7 +125,7 @@ define([ $addonBtn.click(function(e) { if (!self.options.addonFunction) { $input.val(''); - $input.trigger('input'); + $input.trigger('input', [true]); } else { self.options.addonFunction(e); @@ -130,7 +143,7 @@ define([ } else { retVal = this.$input.val(val); - this.$input.trigger('input'); + this.$input.trigger('input', [true]); } return retVal; }; diff --git a/kbase-extension/static/kbase/js/util/jobLogViewer.js b/kbase-extension/static/kbase/js/util/jobLogViewer.js index 2aa8854ef1..3e259aea10 100644 --- a/kbase-extension/static/kbase/js/util/jobLogViewer.js +++ b/kbase-extension/static/kbase/js/util/jobLogViewer.js @@ -33,13 +33,11 @@ define([ button = t('button'), span = t('span'), p = t('p'), - fsm, - currentSection, smallPanelHeight = '300px', largePanelHeight = '600px', - numLines = 10, - panel, + numLines = 100, panelHeight = smallPanelHeight, + // all the states possible, to be fed into the FSM. appStates = [{ state: { mode: 'new' @@ -340,24 +338,24 @@ define([ /** * The entrypoint to this widget. This creates the job log viewer and initializes it. - * Starting it is left as a lifecycle method for the calling object. + * Starting it is left as a lifecycle method for the caller. * */ function factory() { let runtime = Runtime.make(), - bus = runtime.bus().makeChannelBus({ description: 'Log Viewer Bus' }), container, jobId, - panelId, model, ui, - startingLine = 0, - linesPerPage = null, + linesPerPage = numLines, + fsm, loopFrequency = 5000, looping = false, stopped = false, - listeningForJob = false, - requestLoop = null; + listeningForJob = false, // if true, this means we're listening for job updates + awaitingLog = false, // if true, there's a log request fired that we're awaiting + requestLoop = null, // the timeout object + scrollToEndOnNext = false; // VIEW ACTIONS @@ -365,38 +363,43 @@ define([ if (!looping) { return; } - requestLoop = window.setTimeout(function() { - if (!looping) { - return; - } + requestLoop = window.setTimeout(() => { requestLatestJobLog(); }, loopFrequency); } function stopAutoFetch() { looping = false; + if (ui) { + ui.hideElement('spinner'); + } } + /** + * Starts the autofetch loop. After the first request, this starts a timeout that calls it again. + */ function startAutoFetch() { - if (looping) { - return; - } - if (stopped) { + if (looping || stopped) { return; } var state = fsm.getCurrentState().state; if (state.mode === 'active' && state.auto) { looping = true; fsm.newState({ mode: 'active', auto: true }); - runtime.bus().emit('request-latest-job-log', { - jobId: jobId, - options: { - num_lines: linesPerPage - } - }); + // stop the current timer if we have one. + if (requestLoop) { + clearTimeout(requestLoop); + } + requestLatestJobLog(); + // runtime.bus().emit('request-latest-job-log', { + // jobId: jobId, + // }); } } + /** + * Start automatically fetching logs - triggered by hitting the play button. + */ function doPlayLogs() { fsm.updateState({ auto: true @@ -405,21 +408,32 @@ define([ startAutoFetch(); } - function doStopPlayLogs() { + /** + * Stop automatically fetching logs - triggered by hitting the stop button. + */ + function doStopLogs() { fsm.updateState({ auto: false }); stopped = true; stopAutoFetch(); + if (requestLoop) { + clearTimeout(requestLoop); + } } - function requestJobLog(firstLine, numLines, params) { + /** + * Requests numLines (set in the factory method) log lines starting from the given firstLine. + * @param {int} firstLine + */ + function requestJobLog(firstLine) { ui.showElement('spinner'); + awaitingLog = true; runtime.bus().emit('request-job-log', { jobId: jobId, options: { first_line: firstLine, - num_lines: linesPerPage + // num_lines: linesPerPage } }); } @@ -429,14 +443,13 @@ define([ // load numLines at a time // otherwise load entire log let autoState = fsm.getCurrentState().state.auto; - if(autoState){ - linesPerPage = numLines; // set it to 10 - } + scrollToEndOnNext = true; + awaitingLog = true; ui.showElement('spinner'); runtime.bus().emit('request-latest-job-log', { jobId: jobId, options: { - num_lines: linesPerPage + // num_lines: linesPerPage } }); } @@ -445,21 +458,22 @@ define([ * Scroll to the top of the job log */ function doFetchFirstLogChunk() { - doStopPlayLogs(); - panel.scrollTo(0, 0); + doStopLogs(); + requestJobLog(0); + getLogPanel().scrollTo(0, 0); } /** * scroll to the bottom of the job log */ function doFetchLastLogChunk() { - doStopPlayLogs(); - panel.scrollTo(0, panel.lastChild.offsetTop); + doStopLogs(); + requestLatestJobLog(); } function toggleViewerSize() { panelHeight = panelHeight === smallPanelHeight ? largePanelHeight : smallPanelHeight; - getPanelNode().style.height = panelHeight; + getLogPanel().style.height = panelHeight; } // VIEW @@ -503,7 +517,7 @@ define([ title: 'Stop fetching logs', id: events.addEvent({ type: 'click', - handler: doStopPlayLogs + handler: doStopLogs }) }, [ span({ class: 'fa fa-stop' }) @@ -541,11 +555,40 @@ define([ ]); } + /** + * This is a step toward having scrollahead/scrollbehind. It doesn't work right, and we + * have to move on, but I'm leaving this in here for now. + * There's something minor that I'm missing, I think, about how the scrolling gets + * managed. + * @param {ScrollEvent} e + */ + function handlePanelScrolling(e) { + const panel = getLogPanel(); + // if scroll is at the bottom, and there are more lines, + // get the next chunk. + if (panel.scrollTop === (panel.scrollHeight - panel.offsetHeight)) { + const curLast = model.getItem('lastLine'); + if (curLast < model.getItem('totalLines')) { + requestJobLog(curLast); + } + } + // if it's at the top, and we're not at line 0, get + // the previous chunk. + else if (panel.scrollTop === 0) { + const curFirst = model.getItem('firstLine'); + if (curFirst > 0) { + const reqLine = Math.max(0, curFirst - numLines); + if (reqLine < curFirst) { + requestJobLog(reqLine); + } + } + } + } + /** * builds contents of panel-body class - * @param {string} panelId */ - function renderLayout(panelId) { + function renderLayout() { const events = Events.make(), content = div({ dataElement: 'kb-log', style: { marginTop: '10px'}}, [ div({ class: 'kblog-header' }, [ @@ -556,7 +599,7 @@ define([ renderControls(events) // header ]) ]), - div({ dataElement: 'panel', id: panelId, + div({ dataElement: 'log-panel', style: { 'overflow-y': 'scroll', height: panelHeight, @@ -571,24 +614,6 @@ define([ }; } - function sanitize(text) { - var longWord = 80; - var encoded = ui.htmlEncode(text); - - // try to make sane word length not break things. - var words = encoded.split(/ /); - - var fixed = words.map(function(word) { - if (word.length < longWord) { - return word; - } - return word.replace(/\//, '/') - .replace(/\./, '.'); - }); - - return fixed.join(' '); - } - /** * build and return div that displays * individual job log line @@ -616,7 +641,7 @@ define([ // text const textDiv = document.createElement('div'); textDiv.setAttribute('class', 'kblog-text'); - const lineText = document.createTextNode(line.text) + const lineText = document.createTextNode(line.text); textDiv.appendChild(lineText); // append line number and text wrapperDiv.appendChild(numDiv); @@ -627,15 +652,8 @@ define([ return kblogLine; } - /** - * Append div that displays job log lines - * to the panel - * @param {array} lines - */ - function makeLogChunkDiv(lines) { - for (let i=0; i panel.appendChild(buildLine(line))); - makeLogChunkDiv(viewLines); - if (fsm.getCurrentState().state.auto) { + // if we're autoscrolling, scroll to the bottom + if (fsm.getCurrentState().state.auto || scrollToEndOnNext) { panel.scrollTo(0, panel.lastChild.offsetTop); + scrollToEndOnNext = false; } } else { - ui.setContent('panel', 'Sorry, no log yet...'); + ui.setContent('log-panel', 'No log entries to show.'); } } function handleJobStatusUpdate(message) { // if the job is finished, we don't want to reflect // this in the ui, and disable play/stop controls. - var jobStatus = message.jobState.job_state, + var jobStatus = message.jobState.status, mode = fsm.getCurrentState().state.mode, newState; switch (mode) { case 'new': switch (jobStatus) { + case 'created': + case 'estimating': case 'queued': startJobUpdates(); newState = { @@ -687,7 +700,7 @@ define([ auto: true }; break; - case 'in-progress': + case 'running': startJobUpdates(); startAutoFetch(); newState = { @@ -696,22 +709,21 @@ define([ }; break; case 'completed': - requestLatestJobLog(); + requestJobLog(0); stopJobUpdates(); newState = { mode: 'complete' }; break; case 'error': - case 'suspend': - requestLatestJobLog(); + requestJobLog(0); stopJobUpdates(); newState = { mode: 'error' }; break; - case 'canceled': - requestLatestJobLog(); + case 'terminated': + requestJobLog(0); stopJobUpdates(); newState = { mode: 'canceled' @@ -725,10 +737,12 @@ define([ break; case 'queued': switch (jobStatus) { + case 'created': + case 'estimating': case 'queued': // no change break; - case 'in-progress': + case 'running': newState = { mode: 'active', auto: true @@ -741,12 +755,11 @@ define([ }; break; case 'error': - case 'suspend': newState = { mode: 'error' }; break; - case 'canceled': + case 'terminated': newState = { mode: 'canceled' }; @@ -761,7 +774,7 @@ define([ case 'queued': // this should not occur! break; - case 'in-progress': + case 'running': startAutoFetch(); break; case 'completed': @@ -770,12 +783,11 @@ define([ }; break; case 'error': - case 'suspend': newState = { mode: 'error' }; break; - case 'canceled': + case 'terminated': newState = { mode: 'canceled' }; @@ -795,7 +807,7 @@ define([ } case 'canceled': switch (jobStatus) { - case 'canceled': + case 'terminated': return; default: console.error('Unexpected log status ' + jobStatus + ' for "canceled" state'); @@ -836,32 +848,43 @@ define([ type: 'job-logs' }, handle: function(message) { + if (!awaitingLog) { + return; + } ui.hideElement('spinner'); + /* message has structure: + * { + * jobId: string, + * latest: bool, + * logs: { + * first: int (first line of the log batch), 0-indexed + * job_id: string, + * latest: bool, + * max_lines: int (total logs available - if job is done, so is this), + * lines: [{ + * is_error: 0 or 1, + * line: string, + * linepos: int, position in log. helpful! + * ts: timestamp + * }] + * } + * } + */ + awaitingLog = false; - if (message.logs.lines.length === 0) { - // TODO: add an alert area and show a dismissable alert. - if (!looping) { - // alert('No log entries returned'); - console.warn('No log entries returned', message); - } - } else { - var lines = model.getItem('lines'); - model.setItem('lines', message.logs.lines); - model.setItem('currentLine', message.logs.first); - model.setItem('latest', true); - model.setItem('fetchedAt', new Date().toUTCString()); - // Detect end of log. - var lastLine = model.getItem('lastLine'), - batchLastLine = message.logs.first + message.logs.lines.length; - if (!lastLine) { - lastLine = batchLastLine; - } else { - if (batchLastLine > lastLine) { - lastLine = batchLastLine; - } - } - model.setItem('lastLine', lastLine); - + if (message.logs.lines.length !== 0) { + const viewLines = message.logs.lines.map(function(line, index) { + return { + text: line.line, + isError: (line.is_error === 1 ? true : false), + lineNumber: line.linepos + }; + }); + model.setItem('lines', viewLines); + model.setItem('firstLine', message.logs.first + 1); + model.setItem('lastLine', message.logs.first + viewLines.length); + model.setItem('totalLines', message.logs.max_lines); + render(); } if (looping) { scheduleNextRequest(); @@ -885,7 +908,7 @@ define([ }, handle: function() { stopAutoFetch(); - console.warn('No job log :( -- it has been deleted'); + render(); } }); externalEventListeners.push(ev); @@ -953,7 +976,7 @@ define([ text: 'Job is queued, logs will be available when the job is running.' } const line = buildLine(noLogYet); - ui.setContent('kb-log.panel', line); + getLogPanel().appendChild(line); } function doExitQueued(message) { @@ -1005,9 +1028,6 @@ define([ } function startJobUpdates() { - if (listeningForJob) { - return; - } runtime.bus().emit('request-job-update', { jobId: jobId }); @@ -1020,10 +1040,6 @@ define([ } } - function getPanelNode() { - return document.getElementById(panelId); - } - /** * The main lifecycle event, called when its container node exists, and we want to start * running this widget. @@ -1046,11 +1062,9 @@ define([ container = hostNode.appendChild(document.createElement('div')); ui = UI.make({ node: container }); - panelId = html.genId(); - var layout = renderLayout(panelId); + var layout = renderLayout(); container.innerHTML = layout.content; layout.events.attachEvents(container); - panel = getPanelNode(); initializeFSM(); renderFSM(); @@ -1069,9 +1083,6 @@ define([ if (requestLoop) { clearTimeout(requestLoop); } - if (bus) { - bus.stop(); - } if (fsm) { fsm.stop(); } @@ -1086,17 +1097,25 @@ define([ } // MAIN + /* The data model for this widget contains all lines currently being shown, along with the indices for + * the first and last lines known. + * It also tracks the total lines currently available for the app, if returned. + * Lines is a list of small objects. Each object + * lines - list, each is a small object with keys (these are all post-processed after fetching): + * line - string, the line text + * isError - boolean, true if that line denotes an error + * ts - int timestamp + * lineNumber - int, what line this is + * firstLine - int, the first line we're tracking (inclusive) + * lastLine - int, the last line we're tracking (inclusive) + * totalLines - int, the total number of lines available from the server as of the last message. + */ model = Props.make({ data: { - cache: [], lines: [], - currentLine: null, + firstLine: null, lastLine: null, - linesPerPage: linesPerPage, - fetchedAt: null - }, - onUpdate: function() { - render(); + totalLines: null } }); @@ -1105,7 +1124,6 @@ define([ return Object.freeze({ start: start, stop: stop, - bus: bus, detach: detach }); } diff --git a/kbase-extension/static/kbase/js/widgets/function_output/kbaseSampleSet.js b/kbase-extension/static/kbase/js/widgets/function_output/kbaseSampleSet.js new file mode 100644 index 0000000000..769f8c6595 --- /dev/null +++ b/kbase-extension/static/kbase/js/widgets/function_output/kbaseSampleSet.js @@ -0,0 +1,231 @@ +/* +Sebastian Le Bras - April 2020 +*/ +define ([ + 'kbwidget', + 'bootstrap', + 'jquery', + 'kbase-client-api', + 'widgets/dynamicTable', + 'kbaseAuthenticatedWidget', + 'kbaseTabs', + 'kbase-generic-client-api', + 'narrativeConfig', + 'bluebird' +], function( + KBWidget, + bootstrap, + $, + kbase_client_api, + DynamicTable, + kbaseAuthenticatedWidget, + kbaseTabs, + GenericClient, + Config, + Promise +) { + 'use strict'; + + return KBWidget({ + name: 'kbaseSampleSetView', + parent: kbaseAuthenticatedWidget, + version: '1.0.0', + options: { + pageLimit: 10, + default_blank_value: "" + }, + + init: function (options) { + this._super(options); + + // var self = this; + this.obj_ref = this.options.upas.id; + this.link_ref = this.obj_ref; + + if(options._obj_info) { + this.ss_info = options._obj_info; + this.obj_ref = this.ss_info['ws_id'] + '/' + this.ss_info['id'] + '/' + this.ss_info['version']; + this.link_ref = this.ss_info['ws_id'] + '/' + this.ss_info['name'] + '/' + this.ss_info['version']; + } + + this.client = new GenericClient(Config.url('service_wizard'), {token: this.authToken()}); + this.ws = new Workspace(Config.url('workspace'),{'token': this.authToken()}); + + this.$elem.append($('
').attr('align', 'center').append($('