Merge pull request #868 from StackStorm/dont_load_large_execution_result

Don't display and render large execution results, fix lint gulp task and lint violations, use latest up to date docker compose setup for e2e tests

Author: Kami

Diff for: .circleci/config.yml

@@ -12,9 +12,11 @@ jobs:
       DEPLOY_PACKAGES: 1
       DEB: xenial bionic
       RPM: el7 el8
+      ST2_VERSION: "3.5dev"
       ST2_HOST: localhost
-      ST2_USERNAME: admin
-      ST2_PASSWORD: 123
+      ST2_PROTOCOL: http
+      ST2_USERNAME: st2admin
+      ST2_PASSWORD: Ch@ngeMe
       ST2_TEST_ENVIRONMENT: https://github.com/StackStorm/st2-docker
     steps:
       - checkout
@@ -68,48 +70,42 @@ jobs:
           name: Update Docker Compose
           command: |
             set -x
-            sudo sh -c "curl -L https://github.com/docker/compose/releases/download/1.14.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose"
+            sudo sh -c "curl -L https://github.com/docker/compose/releases/download/1.28.6/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose"
             sudo chmod +x /usr/local/bin/docker-compose
       - run:
           name: Clone test containers
           command: |
-            # Use DEPRECATED/all-in-one for now, we'll have to circle back around
-            # and fix this to use the master branch
-            echo "Cloning ${ST2_DOCKER_BRANCH:-DEPRECATED/all-in-one} branch of st2-docker"
-            git clone --branch ${ST2_DOCKER_BRANCH:-DEPRECATED/all-in-one} --depth 1 ${ST2_TEST_ENVIRONMENT} ~/st2-docker
+            echo "Cloning ${ST2_DOCKER_BRANCH:-master} branch of st2-docker"
+            git clone --branch ${ST2_DOCKER_BRANCH:-master} --depth 1 ${ST2_TEST_ENVIRONMENT} ~/st2-docker
       - run:
-          name: Update env variables for test containers
+          name: Configufe docker compose config
           command: |
-            make -C ~/st2-docker env
-            echo -e "ST2_USER=${ST2_USERNAME}\nST2_PASSWORD=${ST2_PASSWORD}" > ~/st2-docker/conf/stackstorm.env
-            cat ~/st2-docker/conf/stackstorm.env
+            # Configure allow origin in the user config
+            echo "[api]" > ~/st2-docker/files/st2.user.conf
+            echo "allow_origin = *" >> ~/st2-docker/files/st2.user.conf
       - run:
           name: Start test containers
           command: |
             docker-compose -f ~/st2-docker/docker-compose.yml up -d
-            sleep 60
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm crudini --set /etc/st2/st2.conf api allow_origin "*"
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm st2ctl restart
+            sleep 100
       - run:
           name: Check test containers
           command: |
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm st2 run core.noop
+            docker-compose -f ~/st2-docker/docker-compose.yml exec st2client st2 run core.noop
       - run:
           name: Run functional tests
           command: npm run test-functional
       - run:
           name: Reset test containers
           command: |
-            docker-compose -f ~/st2-docker/docker-compose.yml down
+            docker-compose -f ~/st2-docker/docker-compose.yml down --rmi
             docker-compose -f ~/st2-docker/docker-compose.yml up -d
-            sleep 60
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm crudini --set /etc/st2/st2.conf api allow_origin "*"
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm st2ctl restart
+            sleep 100
       - run:
           name: Recheck test containers
           command: |
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm st2 run core.noop
-            docker-compose -f ~/st2-docker/docker-compose.yml exec stackstorm st2 execution list
+            docker-compose -f ~/st2-docker/docker-compose.yml exec st2client st2 run core.noop
+            docker-compose -f ~/st2-docker/docker-compose.yml exec st2client st2 execution list
       - run:
           name: Run tests on production version
           command: npm run test-production

Diff for: .eslintignore

@@ -3,3 +3,4 @@ modules/*/node_modules
 tasks/*/node_modules
 node_modules
 js
+package.meta.js

Diff for: README.md

@@ -123,9 +123,9 @@ We're using [zombie](https://github.com/assaf/zombie) as our headless browser fo
 
 First of all, you need to make sure you have a running copy of st2 to run tests against. We're using [official docker images](https://github.com/stackstorm/st2-docker) for our automated tests, but the [AIO](https://docs.stackstorm.com/install/index.html) deployment will work just as good (though will take more time to deploy).
 
-To let test runner know the details of your st2 installation, you need to set ST2_HOST, ST2_USERNAME and ST2_PASSWORD env variables, then call `gulp test`.
+To let test runner know the details of your st2 installation, you need to set ST2_PROTOCOL, ST2_HOST, ST2_USERNAME and ST2_PASSWORD env variables, then call `gulp test`.
 
-    $ ST2_HOST=localhost ST2_USERNAME=admin ST2_PASSWORD=123 gulp test
+    $ ST2_PROTOCOL=http ST2_HOST=localhost ST2_USERNAME=admin ST2_PASSWORD=123 gulp test
 
 Copyright, License, and Contributors Agreement
 ----------------------------------------------

Diff for: apps/st2-actions/actions-details.component.js

@@ -213,9 +213,11 @@ export default class ActionsDetails extends React.Component {
       },
     });
   }
-setWindowName(e){
-   window.name="parent"
-}
+
+  setWindowName(e) {
+    window.name = 'parent';
+  }
+
   handleRun(e, ...args) {
     e.preventDefault();
 
@@ -259,7 +261,7 @@ setWindowName(e){
                   target="_blank"
                   to={`/action/${action.ref}`}
                   className="st2-forms__button st2-details__toolbar-button"
-                  onClick ={e => this.setWindowName(e)}
+                  onClick={e => this.setWindowName(e)}
                 >
                   Edit
                 </Link>

Diff for: apps/st2-history/history-details.component.js

@@ -47,6 +47,7 @@ import HistoryPopup from './history-popup.component';
   const { execution } = state;
   return { execution };
 })
+
 export default class HistoryDetails extends React.Component {
   static propTypes = {
     handleNavigate: PropTypes.func.isRequired,
@@ -55,7 +56,7 @@ export default class HistoryDetails extends React.Component {
 
     id: PropTypes.string,
     section: PropTypes.string,
-    execution: PropTypes.object,
+    execution: PropTypes.object, // eslint-disable-line react/no-unused-prop-types
     displayUTC: PropTypes.bool.isRequired,
     handleToggleUTC: PropTypes.func,
   }
@@ -81,9 +82,15 @@ export default class HistoryDetails extends React.Component {
   }
 
   fetchExecution(id) {
+    // We utilize ?max_result_size query parameter filter so we don't retrieve
+    // large results which we don't render due to that being very slow and
+    // freezing the browser window
+    const maxResultSizeForRender = ActionReporter.utils.getMaxExecutionResultSizeForRender();
+    const path = `/executions/${id}?max_result_size=${maxResultSizeForRender}`;
+
     store.dispatch({
       type: 'FETCH_EXECUTION',
-      promise: api.request({ path: `/executions/${id}` }),
+      promise: api.request({ path: path }),
     })
       .catch((err) => {
         notification.error(`Unable to retrieve execution "${id}".`, { err });
@@ -183,7 +190,7 @@ export default class HistoryDetails extends React.Component {
                   </Link>
                 </DetailsPanelHeading>
                 <DetailsPanelBody data-test="action_output">
-                  <ActionReporter runner={execution.runner.name} execution={execution} />
+                  <ActionReporter runner={execution.runner.name} execution={execution} api={api} />
                 </DetailsPanelBody>
               </DetailsPanel>
               { execution.rule ? (

Diff for: apps/st2-history/history-panel.component.js

@@ -94,7 +94,7 @@ export default class HistoryPanel extends React.Component {
     }).isRequired,
 
     filter: PropTypes.string,
-    filters: PropTypes.object,
+    filters: PropTypes.array,
     childExecutions: PropTypes.object,
     groups: PropTypes.array,
     collapsed: PropTypes.bool,

Diff for: apps/st2-workflows/workflows.component.js

@@ -73,7 +73,7 @@ export default class Workflows extends Component {
    pack: PropTypes.string,
    meta: PropTypes.object,
    metaSource: PropTypes.string,
-   setMeta: PropTypes.func,
+   setMeta: PropTypes.func,  // eslint-disable-line react/no-unused-prop-types
    input: PropTypes.array,
    workflowSource: PropTypes.string,
    dirty: PropTypes.bool,
@@ -243,7 +243,7 @@ export default class Workflows extends Component {
  }
 
  save() {
-   const { pack, meta, actions, workflowSource, metaSource, setMeta } = this.props;
+   const { pack, meta, actions, workflowSource, metaSource } = this.props;
    const existingAction = actions.find(e => e.name === meta.name && e.pack === pack);
 
    if (!meta.name) {

Diff for: config.js

@@ -18,6 +18,15 @@
 angular.module('main')
   .constant('st2Config', {
 
+    // In case you want to override default value for the result sizes we still render in the
+    // history details widget. Keep in mind that anything above 200-500 KB will take a long time to
+    // render and likely freeze the browser window for deeply nested JSON object results.
+    // Value is in bytes.
+    // max_execution_result_size_for_render: 200 * 1024,
+    //
+    // Set to true to display StackStorm and st2web version in the header
+    //show_version_in_header: false;
+
     // hosts: [
     //   {
     //     name: 'Dev Env',

Diff for: index.html

@@ -19,12 +19,18 @@
             throw new Error('The st2web angular-config-polyfill only supports the "main" module.');
           }
 
-          if (constant !== 'st2Config') {
-            throw new Error('The st2web angular-config-polyfill only supports the "st2Config" constant.');
+          if (constant !== 'st2Config' && constant !== 'st2PackageMeta') {
+            throw new Error('The st2web angular-config-polyfill only supports the "st2Config" and "st2PackageMeta" constant.');
           }
 
           window.st2constants = window.st2constants || {};
-          window.st2constants.st2Config = value;
+
+          if (constant === 'st2Config') {
+            window.st2constants.st2Config = value;
+          }
+          else if (constant === 'st2PackageMeta') {
+            window.st2constants.st2PackageMeta = value;
+          }
         },
         run: (fn) => {
           if (module !== 'main') {
@@ -33,6 +39,7 @@
 
           window.st2constants = window.st2constants || {};
           window.st2constants.st2Config = window.st2constants.st2Config || {};
+          window.st2constants.st2PackageMeta = window.st2constants.st2PackageMeta || {};
 
           fn(window.st2constants.st2Config);
         },
@@ -44,6 +51,7 @@
 <body>
   <div class="wrapper" id="container"></div>
   <script src="config.js"></script>
+  <script src="package.meta.js"></script>
   <script src="js/main.js"></script>
 </body>
 

Diff for: modules/st2-action-reporter/action-reporter.component.js

@@ -20,21 +20,114 @@ import reporters from './reporters';
 
 import style from './style.css';
 
+// If action execution result is larger than this value (in bytes) we won't try to render it in
+// the code highlighter widget, but display a link to the raw result output instead.
+// This way we avoid large results freezing and blocking the browser window.
+// Keep in mind that rendering time also depends on the result type (aka deeply
+// nested JSON object vs a more flat one).
+// Based on testing, any larger and more nested JSON object over 100 KB will
+// take a while to render and consume a lot of memory (and in case of even
+// larger objects, freeze / block the whole browser window).
+// Technically we could still display and render results up to 300 KB, but the
+// whole code widget and browser window gets lagy and slow.
+// Testing was also performed on relatively high end PC so on older ones, even
+// lower limit may be more appropriate.
+// Can be overriden in the config, but values over 50-100 KB (depending on the client
+// resources and how nested the result objects are) are not recommended.
+const DEFAULT_MAX_RESULT_SIZE = 100 * 1024;  // 100 KB
+
+
+/**
+ * Return base URL to the API service based on the config value.
+ */
+function getBaseAPIUrl(api) {
+  if (!api.server) {
+    console.log('config.js is not correctly configured - it\'s missing API server URL entry');
+    return null;
+  }
+
+  if (!api.server.api) {
+    console.log('config.js is not correctly configured - it\'s missing API server URL entry');
+    return null;
+  }
+
+  const url = api.server.api;
+  let baseUrl;
+
+  if (!url.startsWith('http://') && !(url.startsWith('https://'))) {
+    baseUrl = `${window.location.protocol}${url}`;
+  }
+  else {
+    baseUrl = `${url}`;
+  }
+
+  return baseUrl;
+}
+
+/**
+ * Return value for the ?max_result_size query parameter aka the maximum number for the result size
+ * (in bytes) we will still try to render and display.
+ *
+ * We specify a default value which can be overriden inside the config.
+ */
+function getMaxExecutionResultSizeForRender() {
+  let maxResultSizeForRender;
+
+  try {
+    maxResultSizeForRender = window.st2constants.st2Config.max_execution_result_size_for_render || DEFAULT_MAX_RESULT_SIZE;
+  }
+  catch (e) {
+    maxResultSizeForRender = DEFAULT_MAX_RESULT_SIZE;
+  }
+
+  return maxResultSizeForRender;
+}
+
 export default class ActionReporter extends React.Component {
   static propTypes = {
     className: PropTypes.string,
     runner: PropTypes.string.isRequired,
     execution: PropTypes.object.isRequired,
+    api: PropTypes.object.isRequired,
+  }
+
+  static utils = {
+    getMaxExecutionResultSizeForRender: getMaxExecutionResultSizeForRender,
+    getBaseAPIUrl: getBaseAPIUrl,
   }
 
   render() {
-    const { className, runner, execution, ...props } = this.props;
+    const { className, runner, execution, api, ...props } = this.props;
     const reporter = reporters[runner] || reporters.debug;
 
     if (!execution) {
       return null;
     }
 
+    // For backward compatibility with older executions which may not have result_size attribute
+    // we fall back to execution.result (if available - would only be available when using newer
+    // st2web with older version of other StackStorm components).
+    const resultSize = execution.result_size || JSON.stringify(execution.result || {}).length;
+    const resultSizeMB = ((resultSize / 1024 / 1024)).toFixed(2);
+    const maxResultSizeForRender = getMaxExecutionResultSizeForRender();
+
+    if (resultSize && resultSize > maxResultSizeForRender) {
+      // TODO: Add methods to the client to retrieve full correct URL?
+      const baseApiUrl = getBaseAPIUrl(api);
+      const viewRawResultUrl = `${baseApiUrl}/v1/executions/${execution.id}/result?pretty_format=1`;
+      const downloadRawResultUrl = `${baseApiUrl}/v1/executions/${execution.id}/result?download=1&pretty_format=1`;
+      const downloadCompressedRawResultUrl = `${baseApiUrl}/v1/executions/${execution.id}/result?download=1&pretty_format=1&compress=1`;
+
+      return (
+        <div {...props} className={cx(style.component, className)}>
+          <div key="output" className={style.source}>Output</div>
+          <p>
+            Action output is too large to be displayed here ({`${resultSizeMB}`} MB).<br /><br />You can view raw execution output by clicking <a href={`${viewRawResultUrl}`} target="_blank" rel="noopener noreferrer">here</a> or you can download the output by clicking <a href={`${downloadRawResultUrl}`} target="_blank" rel="noopener noreferrer">here (uncompressed)</a> or <a href={`${downloadCompressedRawResultUrl}`} target="_blank" rel="noopener noreferrer">here (compressed)</a>.
+          </p>
+        </div>
+      );
+    }
+
     return (
       <div {...props} className={cx(style.component, className)}>
         { reporter(execution) }