Merge branch '2.x' into backport-1276-to-2.x

opensearch-project · Sep 3, 2024 · 3a83157 · 3a83157
2 parents 5beb70c + 3253225
commit 3a83157
Show file tree

Hide file tree

Showing 119 changed files with 43,773 additions and 1,712 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+* -text
diff --git a/.github/workflows/assistant-release-e2e-workflow.yml b/.github/workflows/assistant-release-e2e-workflow.yml
@@ -34,3 +34,13 @@ jobs:
       test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*'
       osd-serve-args: --assistant.chat.enabled=true
       security-enabled: false
+  tests-with-multiple-data-source-and-disabled-local-cluster:
+    needs: changes
+    if: ${{ needs.changes.outputs.tests == 'true' }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: dashboards assistant
+      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true
+      security-enabled: true
+
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml
@@ -0,0 +1,111 @@
+name: Cypress test for core Dashboards
+on:
+  pull_request:
+    branches: ['**']
+    paths:
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml'
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+      - 'package.json'
+  push:
+    branches: ['**']
+    paths:
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based-ci-groups.yml'
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+      - 'package.json'
+
+env:
+  CI_GROUPS: "1,2,3,4,5,6,7,8,9"
+
+jobs:
+  get_spec:
+    runs-on: 'ubuntu-latest'
+    outputs:
+      MATRIX_INCLUDES: ${{ steps.get_spec.outputs.MATRIX_INCLUDES }}
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+      - name: Checkout cypress-test
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.repository}}
+          path: spec-detect
+      - id: get_spec
+        name: Get specs array
+        run: |
+          cd spec-detect
+          source ./test_finder.sh
+          result="["
+          ## split CI_Groups into array
+          IFS="," read -a groups <<< "${{ env.CI_GROUPS }}"
+          for group in "${groups[@]}"; do
+            item="{\"ciGroup\": \"${group}\", \"specs\": "
+            specs_of_ci_group=`ci_grouped_specs_finder ${group}`
+            FORMATTED_SPEC="\"${specs_of_ci_group}\" }"
+            if [[ "${result}" != "[" ]];
+            then
+                result+=","
+            fi
+            item+="${FORMATTED_SPEC}"
+            result+="${item}"
+          done
+
+          result+=']';
+
+          echo ${result}
+          echo "MATRIX_INCLUDES=${result}" >> "$GITHUB_OUTPUT"
+
+  tests-with-security:
+    needs: ["get_spec"]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+    name: (security)osd:ciGroup${{ matrix.ciGroup }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec '${{ matrix.specs }}'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
+
+  tests-without-security:
+    needs: ["get_spec"]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+    name: (non-security)osd:ciGroup${{ matrix.ciGroup }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec '${{ matrix.specs }}'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
+      security-enabled: false
+
+  # Hold on windows test cases
+  # https://github.com/opensearch-project/opensearch-dashboards-functional-test/actions/runs/9377445544/job/25819022121?pr=1352
+  # tests-with-security-windows:
+  #   needs: ["get_spec"]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+  #   name: (security)osd:ciGroup${{ matrix.ciGroup }}
+  #   uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
+  #   with:
+  #     test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+  #     test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec '${{ matrix.specs }}'
+
+  # tests-without-security-windows:
+  #   needs: ["get_spec"]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include: ${{ fromJSON(needs.get_spec.outputs.MATRIX_INCLUDES) }}
+  #   name: (non-security)osd:ciGroup${{ matrix.ciGroup }}
+  #   uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
+  #   with:
+  #     test-name: "osd:ciGroup${{ matrix.ciGroup }}"
+  #     test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec '${{ matrix.specs }}'
+  #     security-enabled: false
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml
@@ -27,4 +27,4 @@ jobs:
       test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
-      security-enabled: false
+      security-enabled: false
diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml
@@ -1,28 +1,30 @@
 name: Bundle Snapshot based E2E Cypress tests workflow for core Dashboards
 on:
   pull_request:
-    branches: [ '**' ]
+    branches: ['**']
     paths:
       - 'cypress/**/core-opensearch-dashboards/**'
       - 'cypress/utils/dashboards/**'
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based.yml'
   push:
-    branches: [ '**' ]
+    branches: ['**']
     paths:
       - 'cypress/**/core-opensearch-dashboards/**'
       - 'cypress/utils/dashboards/**'
+      - '.github/workflows/cypress-workflow-bundle-snapshot-based.yml'
 
 jobs:
   tests-with-security:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
-      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true 
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true
 
   tests-without-security:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
-      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true
       security-enabled: false
diff --git a/.github/workflows/mds-release-e2e-workflow.yml b/.github/workflows/mds-release-e2e-workflow.yml
@@ -0,0 +1,19 @@
+name: E2E Cypress tests workflow for MDS
+on:
+  pull_request:
+    branches: ['**']
+    paths:
+      - 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+  push:
+    branches: ['**']
+    paths:
+      - 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+
+jobs:
+  tests-with-multiple-data-source-and-disabled-local-cluster:
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: MDS enabled and local cluster disabled
+      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/mds*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true
+      security-enabled: true
diff --git a/.github/workflows/ml-commons-release-e2e-workflow.yml b/.github/workflows/ml-commons-release-e2e-workflow.yml
@@ -24,11 +24,11 @@ jobs:
       test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/ml-commons-dashboards/*'
       osd-serve-args: --ml_commons_dashboards.enabled=true
 
-  tests-with-multi-data-source:
+  tests-with-multiple-data-source-and-disabled-local-cluster:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: ML Commons With Multi Data Source
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/ml-commons-dashboards/*'
-      osd-serve-args: --ml_commons_dashboards.enabled=true --data_source.enabled=true
+      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/ml-commons-dashboards/mds*.js'
+      osd-serve-args: --ml_commons_dashboards.enabled=true --data_source.enabled=true --data_source.hideLocalCluster=true
diff --git a/.github/workflows/workspace-release-e2e-workflow.yml b/.github/workflows/workspace-release-e2e-workflow.yml
@@ -15,14 +15,15 @@ jobs:
           filters: |
             tests:
               - 'cypress/**/workspace-plugin/**'
+              - '.github/workflows/workspace-release-e2e-workflow.yml'
   tests-with-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: dashboards workspace
       test-command: env CYPRESS_WORKSPACE_ENABLED=true CYPRESS_SAVED_OBJECTS_PERMISSION_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/workspace-plugin/*'
-      osd-serve-args: --workspace.enabled=true --savedObjects.permission.enabled=true --opensearch_security.multitenancy.enabled=false
+      osd-serve-args: --workspace.enabled=true --savedObjects.permission.enabled=true --opensearch_security.multitenancy.enabled=false --opensearchDashboards.dashboardAdmin.users='["admin"]'
   tests-without-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
@@ -32,3 +33,12 @@ jobs:
       test-command: env CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/workspace-plugin/*'
       osd-serve-args: --workspace.enabled=true --savedObjects.permission.enabled=false
       security-enabled: false
+  tests-with-multiple-data-source-and-disabled-local-cluster:
+    needs: changes
+    if: ${{ needs.changes.outputs.tests == 'true' }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: dashboards workspace
+      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true CYPRESS_SAVED_OBJECTS_PERMISSION_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/workspace-plugin/mds*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --workspace.enabled=true --savedObjects.permission.enabled=true --opensearch_security.multitenancy.enabled=false --opensearchDashboards.dashboardAdmin.users='["admin"]'
+      security-enabled: true
diff --git a/SECURITY.md b/SECURITY.md
@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do not create a public GitHub issue.
diff --git a/cypress.json b/cypress.json
@@ -16,6 +16,7 @@
     "remoteDataSourceBasicAuthUsername": "admin",
     "remoteDataSourceBasicAuthPassword": "admin",
     "SECURITY_ENABLED": false,
+    "MULTITENANCY_ENABLED": true,
     "AGGREGATION_VIEW": false,
     "MULTITENANCY_ENABLED": true,
     "username": "admin",
@@ -29,8 +30,10 @@
     "DASHBOARDS_ASSISTANT_ENABLED": false,
     "WORKSPACE_ENABLED": false,
     "SAVED_OBJECTS_PERMISSION_ENABLED": false,
+    "DISABLE_LOCAL_CLUSTER": false,
     "browserPermissions": {
       "clipboard": "allow"
-    }
+    },
+    "UIMETRIC_ENABLED": false
   }
 }
diff --git a/cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json b/cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json
@@ -0,0 +1 @@
+{"report":{"reportVersion":1,"uiStatsMetrics":{"console-count-GET_cat.indices":{"key":"console-count-GET_cat.indices","appName":"console","eventName":"GET_cat.indices","type":"count","stats":{"min":0,"max":1,"avg":0.5,"sum":21}}}}}