Merge remote-tracking branch 'origin/main' into stack-cloud-provider

.buildkite/pipeline.trigger.integration.tests.sh

@@ -25,7 +25,7 @@ for test in ${STACK_COMMAND_TESTS[@]}; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log"
+    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
     echo "          - build/elastic-stack-status/*/*"
 done
 
@@ -43,7 +43,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
     echo "          - build/elastic-stack-status/*/*"
     if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
@@ -63,7 +63,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log"
+    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
 done
 
 popd > /dev/null

.buildkite/pipeline.yml

@@ -1,6 +1,6 @@
 env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
-  DOCKER_COMPOSE_VERSION: "1.25.5" # "v2.15.1"
+  DOCKER_COMPOSE_VERSION: "v2.17.2"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'

.buildkite/scripts/integration_tests.sh

@@ -111,19 +111,36 @@ fi
 echo "--- Run integration test ${TARGET}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
     make install
+
+    # allow to fail this command, to be able to upload safe logs
+    set +e
     make PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET}
+    testReturnCode=$?
+    set -e
 
     if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
-            "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*" \
-            "insecure-logs/${PACKAGE}/"
+            "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \
+            "insecure-logs/${PACKAGE}/elastic-agent-logs/"
+
+        # required for <8.6.0
+        upload_safe_logs \
+            "${JOB_GCS_BUCKET_INTERNAL}" \
+            "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \
+            "insecure-logs/${PACKAGE}/elastic-agent-logs/default/"
 
         upload_safe_logs \
             "${JOB_GCS_BUCKET_INTERNAL}" \
             "build/container-logs/*.log" \
             "insecure-logs/${PACKAGE}/container-logs/"
     fi
+
+    if [ $testReturnCode != 0 ]; then
+        echo "make PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
+        exit ${testReturnCode}
+    fi
+
     make check-git-clean
     exit 0
 fi

catalog-info.yaml

@@ -1,3 +1,22 @@
+# Declare a Backstage Component that represents your application.
+---
+# yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: elastic-package
+  description: elastic-package - Command line tool for developing Elastic Integrations
+  links:
+    - title: Developer Documentation
+      icon: file-doc
+      url: https://www.elastic.co/guide/en/integrations-developer/current/elastic-package.html
+
+spec:
+  type: tool
+  owner: group:ingest-fp
+  system: platform-ingest
+  lifecycle: production
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -24,7 +43,11 @@ spec:
       pipeline_file: ".buildkite/pipeline.yml"
       provider_settings:
         build_pull_request_forks: false
+        build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: true
+        filter_enabled: true
+        filter_condition: |
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-package
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -33,9 +56,10 @@ spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
       teams:
-        ecosystem: {}
+        ecosystem:
+          access_level: MANAGE_BUILD_AND_READ
         ingest-fp:
-          access_level: BUILD_AND_READ
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
 
@@ -58,22 +82,27 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: elastic-package-package-stroage-publish
+      name: elastic-package-package-storage-publish
       description: 'Minimal Jenkins pipeline to exercise publishing a package to Package Storage (for testing only)'
     spec:
       branch_configuration: main
       pipeline_file: ".buildkite/pipeline.package-storage-publish.yml"
       provider_settings:
         build_pull_request_forks: false
+        build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: true
+        filter_enabled: true
+        filter_condition: |
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-package
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: '!main'
       teams:
-        ecosystem: {}
+        ecosystem:
+          access_level: MANAGE_BUILD_AND_READ
         ingest-fp:
-          access_level: BUILD_AND_READ
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY

cmd/benchmark.go

@@ -269,7 +269,8 @@ func generateDataStreamCorpusCommandAction(cmd *cobra.Command, _ []string) error
 		return cobraext.FlagParsingError(err, cobraext.GenerateCorpusRallyTrackOutputDirFlagName)
 	}
 
-	generator, err := corpusgenerator.NewGenerator(packageName, dataSetName, commit, totSizeInBytes)
+	genLibClient := corpusgenerator.NewClient(commit)
+	generator, err := corpusgenerator.NewGenerator(genLibClient, packageName, dataSetName, totSizeInBytes)
 	if err != nil {
 		return errors.Wrap(err, "can't generate benchmarks data corpus for data stream")
 	}

go.mod

@@ -5,41 +5,41 @@ go 1.20
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.6
 	github.com/Masterminds/semver/v3 v3.2.1
-	github.com/ProtonMail/gopenpgp/v2 v2.7.0
+	github.com/ProtonMail/gopenpgp/v2 v2.7.1
 	github.com/aymerick/raymond v2.0.2+incompatible
 	github.com/boumenot/gocover-cobertura v1.2.0
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/cloud-sdk-go v1.11.0
 	github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0
-	github.com/elastic/go-elasticsearch/v7 v7.17.7
+	github.com/elastic/go-elasticsearch/v7 v7.17.10
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/go-resource v0.1.1
 	github.com/elastic/go-ucfg v0.8.6
-	github.com/elastic/package-spec/v2 v2.7.0
+	github.com/elastic/package-spec/v2 v2.8.0
 	github.com/fatih/color v1.15.0
 	github.com/go-git/go-billy/v5 v5.4.1
 	github.com/go-git/go-git/v5 v5.6.1
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.1.0
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
-	github.com/magefile/mage v1.14.0
+	github.com/magefile/mage v1.15.0
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/shirou/gopsutil/v3 v3.23.3
+	github.com/shirou/gopsutil/v3 v3.23.4
 	github.com/spf13/cobra v1.7.0
-	github.com/stretchr/testify v1.8.2
-	golang.org/x/tools v0.8.0
+	github.com/stretchr/testify v1.8.3
+	golang.org/x/tools v0.9.1
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/gotestsum v1.10.0
-	helm.sh/helm/v3 v3.11.3
+	helm.sh/helm/v3 v3.12.0
 	honnef.co/go/tools v0.4.3
-	k8s.io/apimachinery v0.27.1
-	k8s.io/cli-runtime v0.27.1
-	k8s.io/client-go v0.27.1
+	k8s.io/apimachinery v0.27.2
+	k8s.io/cli-runtime v0.27.2
+	k8s.io/client-go v0.27.2
 )
 
 require (
@@ -59,7 +59,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
-	github.com/cloudflare/circl v1.3.1 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dnephin/pflag v1.0.7 // indirect
@@ -135,7 +135,7 @@ require (
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
-	github.com/shoenig/go-m1cpu v0.1.4 // indirect
+	github.com/shoenig/go-m1cpu v0.1.5 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
@@ -154,25 +154,24 @@ require (
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/term v0.7.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gotest.tools/v3 v3.4.0 // indirect
-	k8s.io/api v0.27.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.0 // indirect
-	k8s.io/component-base v0.26.2 // indirect
+	k8s.io/api v0.27.2 // indirect
+	k8s.io/apiextensions-apiserver v0.27.1 // indirect
+	k8s.io/component-base v0.27.1 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
-	k8s.io/kubectl v0.26.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/kubectl v0.27.1 // indirect
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.2 // indirect