From 0f985779e0f65ef0240ff3ce96d304cc37f8e79d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 21 Apr 2023 12:51:35 +0200 Subject: [PATCH 01/16] Add stack provider for Elastic Cloud --- go.mod | 8 + go.sum | 242 ++++++++++++++++++++++++ internal/stack/cloud.go | 356 ++++++++++++++++++++++++++++++++++++ internal/stack/providers.go | 4 + 4 files changed, 610 insertions(+) create mode 100644 internal/stack/cloud.go diff --git a/go.mod b/go.mod index 875400160..1e527abcb 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/boumenot/gocover-cobertura v1.2.0 github.com/cespare/xxhash/v2 v2.2.0 github.com/dustin/go-humanize v1.0.1 + github.com/elastic/cloud-sdk-go v1.11.0 github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0 github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 @@ -56,6 +57,7 @@ require ( github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudflare/circl v1.3.1 // indirect github.com/creasty/defaults v1.7.0 // indirect @@ -73,11 +75,16 @@ require ( github.com/go-git/gcfg v1.5.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/analysis v0.21.1 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/loads v0.21.0 // indirect + github.com/go-openapi/runtime v0.23.0 // indirect + github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/validate v0.20.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -120,6 +127,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect diff --git a/go.sum b/go.sum index e7441a639..0dcc5b89d 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,13 @@ github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ek github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/gopenpgp/v2 v2.7.0 h1:TUH6csU8OpRtA5EKU8yoAXn8M5Inh2T4Y086lKFFyS8= github.com/ProtonMail/gopenpgp/v2 v2.7.0/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -47,11 +52,18 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/boumenot/gocover-cobertura v1.2.0 h1:g+VROIASoEHBrEilIyaCmgo7HGm+AV5yKEPLk0qIY+s= github.com/boumenot/gocover-cobertura v1.2.0/go.mod h1:fz7ly8dslE42VRR5ZWLt2OHGDHjkTiA2oNvKgJEjLT0= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -83,12 +95,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elastic/cloud-sdk-go v1.11.0 h1:rJRL6EScg4uJXRFlvWB6GdEBvJ7hvbZZ3NjMdH7ak/k= +github.com/elastic/cloud-sdk-go v1.11.0/go.mod h1:YJfcOSFF/MS+o9dWZAUs+JYWfoeRWzAKmtUm27cbUzA= github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0 h1:Me2T3/O4nASmdjmfaKYaiJaGq8zVhasjfZi3il5p/gs= github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM= github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73Els0nbaWOmYachs= @@ -130,6 +146,8 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= @@ -146,18 +164,141 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= +github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.21.1 h1:krcNCEvCttpSUFBPOrfvn7nniejvrOkoNYRlZwQFpEs= +github.com/go-openapi/analysis v0.21.1/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= +github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= +github.com/go-openapi/loads v0.21.0 h1:jYtUO4wwP7psAweisP/MDoOpdzsYEESdoPcsWjHDR68= +github.com/go-openapi/loads v0.21.0/go.mod h1:rHYve9nZrQ4CJhyeIIFJINGCg1tQpx2yJrrNo8sf1ws= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= +github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.23.0 h1:HX6ET2sHCIvaKeDDQoU01CtO1ekg5EkekHSkLTtWXH0= +github.com/go-openapi/runtime v0.23.0/go.mod h1:aQg+kaIQEn+A2CRSY1TxbM8+sT9g2V3aLc1FbIAnbbs= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= +github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= +github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= +github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= +github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= +github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -209,6 +350,7 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -231,6 +373,8 @@ github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -239,19 +383,26 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PH github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= @@ -259,11 +410,14 @@ github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -275,8 +429,16 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -303,7 +465,11 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -334,6 +500,11 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -341,6 +512,7 @@ github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -353,9 +525,13 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -368,6 +544,9 @@ github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= @@ -375,6 +554,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -382,6 +563,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -406,11 +589,16 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -429,6 +617,17 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.2/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= @@ -436,9 +635,17 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.starlark.net v0.0.0-20221205180719-3fd0dac74452 h1:JZtNuL6LPB+scU5yaQ6hqRlJFRiddZm2FwRt2AQqtHA= go.starlark.net v0.0.0-20221205180719-3fd0dac74452/go.mod h1:kIVgS18CjmEC3PqMd5kaJSGEifyV/CeB9x506ZJ1Vbk= golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -467,20 +674,32 @@ golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= @@ -497,6 +716,8 @@ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -506,9 +727,16 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -522,6 +750,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -548,6 +777,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -562,6 +792,7 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -573,10 +804,18 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -630,11 +869,13 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -645,6 +886,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go new file mode 100644 index 000000000..244711490 --- /dev/null +++ b/internal/stack/cloud.go @@ -0,0 +1,356 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + + "github.com/elastic/cloud-sdk-go/pkg/api" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" + "github.com/elastic/cloud-sdk-go/pkg/auth" + "github.com/elastic/cloud-sdk-go/pkg/models" + "github.com/elastic/cloud-sdk-go/pkg/plan" + "github.com/elastic/cloud-sdk-go/pkg/plan/planutil" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" +) + +const ( + paramCloudDeploymentID = "cloud_deployment_id" + paramCloudDeploymentAlias = "cloud_deployment_alias" +) + +var ( + errDeploymentNotExist = errors.New("deployment does not exist") +) + +type cloudProvider struct { + api *api.API + profile *profile.Profile +} + +func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { + apiKey := os.Getenv("EC_API_KEY") + if apiKey == "" { + return nil, errors.New("unable to obtain value from EC_API_KEY environment variable") + } + api, err := api.NewAPI(api.Config{ + Client: new(http.Client), + AuthWriter: auth.APIKey(apiKey), + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize API client: %w", err) + } + + return &cloudProvider{ + api: api, + profile: profile, + }, nil +} + +func (cp *cloudProvider) BootUp(options Options) error { + logger.Warn("Elastic Cloud provider is in technical preview") + + _, err := cp.currentDeployment() + if err == nil { + // Do nothing, deployment already exists. + // TODO: Migrate configuration if changed. + config, err := LoadConfig(cp.profile) + if err != nil { + return err + } + printUserConfig(options.Printer, config) + return nil + } else if err != nil && err != errDeploymentNotExist { + return err + } + + // TODO: Parameterize this. + name := "elastic-package-test" + region := "gcp-europe-west3" + templateID := "gcp-io-optimized" + + logger.Debugf("Getting deployment template %q", templateID) + template, err := deptemplateapi.Get(deptemplateapi.GetParams{ + API: cp.api, + TemplateID: templateID, + Region: region, + StackVersion: options.StackVersion, + }) + if err != nil { + return fmt.Errorf("failed to get deployment template %q: %w", templateID, err) + } + + payload := template.DeploymentTemplate + + // Remove the resources that we don't need. + payload.Resources.Apm = nil + payload.Resources.Appsearch = nil + payload.Resources.EnterpriseSearch = nil + + // Initialize the plan with the id of the template, otherwise the create request fails. + if es := payload.Resources.Elasticsearch; len(es) > 0 { + plan := es[0].Plan + if plan.DeploymentTemplate == nil { + plan.DeploymentTemplate = &models.DeploymentTemplateReference{} + } + plan.DeploymentTemplate.ID = &templateID + } + + logger.Debugf("Creating deployment %q", name) + res, err := deploymentapi.Create(deploymentapi.CreateParams{ + API: cp.api, + Request: payload, + Overrides: &deploymentapi.PayloadOverrides{ + Name: name, + }, + }) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + if created := res.Created; created == nil || !*created { + return fmt.Errorf("request succeeded, but deployment was not created, check in the console UI") + } + + var config Config + config.Provider = ProviderCloud + config.Parameters = map[string]string{ + "cloud_deployment_alias": res.Alias, + } + deploymentID := res.ID + if deploymentID == nil { + return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") + } + config.Parameters["cloud_deployment_id"] = *deploymentID + + for _, resource := range res.Resources { + kind := resource.Kind + if kind == nil { + continue + } + if *kind == "elasticsearch" { + if creds := resource.Credentials; creds != nil { + if creds.Username != nil { + config.ElasticsearchUsername = *creds.Username + } + if creds.Password != nil { + config.ElasticsearchPassword = *creds.Password + } + } + } + } + + deployment, err := deploymentapi.Get(deploymentapi.GetParams{ + API: cp.api, + DeploymentID: *deploymentID, + }) + if err != nil { + return fmt.Errorf("couldn't check deployment health: %w", err) + } + + config.ElasticsearchHost, err = cp.getServiceURL(deployment.Resources.Elasticsearch) + if err != nil { + return fmt.Errorf("failed to get elasticsearch host: %w", err) + } + config.KibanaHost, err = cp.getServiceURL(deployment.Resources.Kibana) + if err != nil { + return fmt.Errorf("failed to get kibana host: %w", err) + } + config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) + if err != nil { + return fmt.Errorf("failed to get fleet host: %w", err) + } + + printUserConfig(options.Printer, config) + + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + + logger.Debug("Waiting for creation plan to be completed") + err = planutil.TrackChange(planutil.TrackChangeParams{ + TrackChangeParams: plan.TrackChangeParams{ + API: cp.api, + DeploymentID: *deploymentID, + }, + Writer: &cloudTrackWriter{}, + Format: "text", + }) + if err != nil { + return fmt.Errorf("failed to track cluster creation: %w", err) + } + + return nil +} + +func (cp *cloudProvider) TearDown(options Options) error { + deployment, err := cp.currentDeployment() + if err != nil { + return fmt.Errorf("failed to find current deployment: %w", err) + } + if deployment.ID == nil { + return fmt.Errorf("deployment doesn't have id?") + } + + logger.Debugf("Deleting deployment %q", *deployment.ID) + + _, err = deploymentapi.Shutdown(deploymentapi.ShutdownParams{ + API: cp.api, + DeploymentID: *deployment.ID, + SkipSnapshot: true, + }) + if err != nil { + return fmt.Errorf("failed to shutdown deployment: %w", err) + } + + return nil +} + +func (*cloudProvider) Update(options Options) error { + fmt.Println("Nothing to do.") + return nil +} + +func (*cloudProvider) Dump(options DumpOptions) (string, error) { + return "", fmt.Errorf("not implemented") +} + +func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { + deployment, err := cp.currentDeployment() + if err != nil { + return nil, err + } + + status, _ := cp.deploymentStatus(deployment) + return status, nil +} + +func (*cloudProvider) deploymentStatus(deployment *models.DeploymentGetResponse) ([]ServiceStatus, bool) { + allHealthy := true + healthStatus := func(healthy *bool) string { + if healthy != nil && *healthy { + return "healthy" + } + allHealthy = false + return "unhealthy" + } + if healthy := deployment.Healthy; healthy == nil || !*healthy { + allHealthy = false + } + + var status []ServiceStatus + for _, resource := range deployment.Resources.Elasticsearch { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("elasticsearch-%d", i) + } else { + name = fmt.Sprintf("elasticsearch-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + for _, resource := range deployment.Resources.Kibana { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("kibana-%d", i) + } else { + name = fmt.Sprintf("kibana-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + for _, resource := range deployment.Resources.IntegrationsServer { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("integrations-server-%d", i) + } else { + name = fmt.Sprintf("integrations-server-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + return status, allHealthy +} + +func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { + config, err := LoadConfig(cp.profile) + if err != nil { + return nil, err + } + deploymentID, found := config.Parameters[paramCloudDeploymentID] + if !found { + return nil, errDeploymentNotExist + } + deployment, err := deploymentapi.Get(deploymentapi.GetParams{ + API: cp.api, + DeploymentID: deploymentID, + }) + if err != nil { + return nil, fmt.Errorf("couldn't check deployment health: %w", err) + } + + // It seems that terminated deployments still exist, but hidden. + if hidden := deployment.Metadata.Hidden; hidden != nil && *hidden { + return nil, errDeploymentNotExist + } + + return deployment, nil +} + +func (*cloudProvider) getServiceURL(resourcesResponse any) (string, error) { + // Converting back and forth for easier access. + var resources []struct { + Info struct { + Metadata struct { + ServiceURL string `json:"service_url"` + } `json:"metadata"` + } `json:"info"` + } + + d, err := json.Marshal(resourcesResponse) + if err != nil { + return "", fmt.Errorf("failed to marshal resources: %w", err) + } + err = json.Unmarshal(d, &resources) + if err != nil { + return "", fmt.Errorf("failed to unmarshal resources back: %w", err) + } + + for _, resource := range resources { + if serviceURL := resource.Info.Metadata.ServiceURL; serviceURL != "" { + return serviceURL, nil + } + } + return "", fmt.Errorf("url not found") +} + +type cloudTrackWriter struct{} + +func (*cloudTrackWriter) Write(p []byte) (n int, err error) { + logger.Debug(string(p)) + return len(p), nil +} diff --git a/internal/stack/providers.go b/internal/stack/providers.go index 7d672cd81..81d4090ae 100644 --- a/internal/stack/providers.go +++ b/internal/stack/providers.go @@ -13,12 +13,14 @@ import ( const ( ProviderCompose = "compose" + ProviderCloud = "cloud" ) var ( DefaultProvider = ProviderCompose SupportedProviders = []string{ ProviderCompose, + ProviderCloud, } ) @@ -52,6 +54,8 @@ func BuildProvider(name string, profile *profile.Profile) (Provider, error) { switch name { case "compose": return &composeProvider{}, nil + case "cloud": + return newCloudProvider(profile) } return nil, fmt.Errorf("unknown provider %q, supported providers: %s", name, strings.Join(SupportedProviders, ", ")) } From 8315709a960069b38318d594c293fdce56298da4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 21 Apr 2023 13:43:07 +0200 Subject: [PATCH 02/16] Fix version --- internal/stack/cloud.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 244711490..390bfb982 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -110,7 +110,8 @@ func (cp *cloudProvider) BootUp(options Options) error { API: cp.api, Request: payload, Overrides: &deploymentapi.PayloadOverrides{ - Name: name, + Name: name, + Version: options.StackVersion, }, }) if err != nil { From ff03886ce2be78455ce38fc699526e1a9e753d04 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 22 May 2023 20:33:12 +0200 Subject: [PATCH 03/16] Upload GeoIP databases --- go.mod | 1 + go.sum | 2 + internal/stack/cloud.go | 128 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 129 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 551ed5611..d08e511a0 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 + github.com/sethvargo/go-retry v0.2.4 github.com/shirou/gopsutil/v3 v3.23.4 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 diff --git a/go.sum b/go.sum index 03e2b9ac2..06664b1f3 100644 --- a/go.sum +++ b/go.sum @@ -536,6 +536,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= +github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= github.com/shirou/gopsutil/v3 v3.23.4 h1:hZwmDxZs7Ewt75DV81r4pFMqbq+di2cbt9FsQBqLD2o= github.com/shirou/gopsutil/v3 v3.23.4/go.mod h1:ZcGxyfzAMRevhUR2+cfhXDH6gQdFYE/t8j1nsU4mPI8= github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 390bfb982..5ab29cc64 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -5,15 +5,24 @@ package stack import ( + "archive/zip" + "bytes" + "context" "encoding/json" "errors" "fmt" + "io" "net/http" "os" + "path" + "time" + + "github.com/sethvargo/go-retry" "github.com/elastic/cloud-sdk-go/pkg/api" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/extensionapi" "github.com/elastic/cloud-sdk-go/pkg/auth" "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/elastic/cloud-sdk-go/pkg/plan" @@ -26,6 +35,7 @@ import ( const ( paramCloudDeploymentID = "cloud_deployment_id" paramCloudDeploymentAlias = "cloud_deployment_alias" + paramGeoIPExtensionID = "geoip_extension_id" ) var ( @@ -78,6 +88,12 @@ func (cp *cloudProvider) BootUp(options Options) error { region := "gcp-europe-west3" templateID := "gcp-io-optimized" + logger.Debugf("Uploading GeoIP databases") + geoIPExtension, err := cp.createGeoIPExtension() + if err != nil { + return fmt.Errorf("failed to create GeoIP extension: %w", err) + } + logger.Debugf("Getting deployment template %q", templateID) template, err := deptemplateapi.Get(deptemplateapi.GetParams{ API: cp.api, @@ -103,6 +119,15 @@ func (cp *cloudProvider) BootUp(options Options) error { plan.DeploymentTemplate = &models.DeploymentTemplateReference{} } plan.DeploymentTemplate.ID = &templateID + + // Add the GeoIP bundle. + plan.Elasticsearch.UserBundles = []*models.ElasticsearchUserBundle{ + &models.ElasticsearchUserBundle{ + ElasticsearchVersion: &options.StackVersion, + Name: geoIPExtension.Name, + URL: geoIPExtension.URL, + }, + } } logger.Debugf("Creating deployment %q", name) @@ -124,13 +149,14 @@ func (cp *cloudProvider) BootUp(options Options) error { var config Config config.Provider = ProviderCloud config.Parameters = map[string]string{ - "cloud_deployment_alias": res.Alias, + paramCloudDeploymentAlias: res.Alias, + paramGeoIPExtensionID: *geoIPExtension.ID, } deploymentID := res.ID if deploymentID == nil { return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") } - config.Parameters["cloud_deployment_id"] = *deploymentID + config.Parameters[paramCloudDeploymentID] = *deploymentID for _, resource := range res.Resources { kind := resource.Kind @@ -193,6 +219,98 @@ func (cp *cloudProvider) BootUp(options Options) error { return nil } +func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { + // From https://www.elastic.co/guide/en/cloud/current/ec-custom-bundles.html + const baseDir = "ingest-geoip" + + files := []struct { + source string + target string + }{ + // These files cannot have the default prefix, we will rename them. + {"GeoLite2-ASN.mmdb", "ElasticPackage-ASN.mmdb"}, + {"GeoLite2-City.mmdb", "ElasticPackage-City.mmdb"}, + {"GeoLite2-Country.mmdb", "ElasticPackage-Country.mmdb"}, + } + + var bundle bytes.Buffer + w := zip.NewWriter(&bundle) + for _, f := range files { + fw, err := w.Create(path.Join(baseDir, f.target)) + if err != nil { + return nil, fmt.Errorf("failed to create file %q in bundle: %w", f.target, err) + } + + fr, err := static.Open(path.Join("_static", f.source)) + if err != nil { + return nil, fmt.Errorf("failed to open static file %q: %w", f.source, err) + } + + _, err = io.Copy(fw, fr) + if err != nil { + fr.Close() + return nil, fmt.Errorf("failed to copy contents of file %q: %w", f.source, err) + } + fr.Close() + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to close bundle: %w", err) + } + + // TODO: Parameterize extension Name. + extensionName := "geoip-extension" + extension, err := extensionapi.Create(extensionapi.CreateParams{ + API: cp.api, + Name: extensionName, + Description: "GeoIP extension for elastic-package tests", + Type: "bundle", + Version: "*", + }) + if err != nil { + return nil, fmt.Errorf("failed to create extension: %w", err) + } + if extension.ID == nil { + return nil, fmt.Errorf("missing identifier in extension") + } + + extension, err = extensionapi.Upload(extensionapi.UploadParams{ + API: cp.api, + ExtensionID: *extension.ID, + File: &bundle, + }) + if err != nil { + return nil, fmt.Errorf("failed to upload bundle: %w", err) + } + + return extension, nil +} + +func (cp *cloudProvider) deleteGeoIPExtension() error { + config, err := LoadConfig(cp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + extensionID, found := config.Parameters[paramGeoIPExtensionID] + if !found { + return nil + } + + backoff := retry.NewFibonacci(1 * time.Second) + backoff = retry.WithMaxDuration(180*time.Second, backoff) + retry.Do(context.TODO(), backoff, func(ctx context.Context) error { + err = extensionapi.Delete(extensionapi.DeleteParams{ + API: cp.api, + ExtensionID: extensionID, + }) + // Actually, we should only retry on extensions.extension_in_use errors. + return retry.RetryableError(err) + }) + if err != nil { + return fmt.Errorf("delete API call failed: %w", err) + } + return nil +} + func (cp *cloudProvider) TearDown(options Options) error { deployment, err := cp.currentDeployment() if err != nil { @@ -213,6 +331,12 @@ func (cp *cloudProvider) TearDown(options Options) error { return fmt.Errorf("failed to shutdown deployment: %w", err) } + logger.Debugf("Deleting GeoIP bundle.") + err = cp.deleteGeoIPExtension() + if err != nil { + return fmt.Errorf("failed to delete GeoIP extension: %w", err) + } + return nil } From 4cede7e09dd416ab1ab0c2c0370eb9b4e1df5749 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 23 May 2023 16:09:43 +0200 Subject: [PATCH 04/16] Replace GeoIP database in cloud deployment --- internal/stack/cloud.go | 181 ++++++++++++++++++++++++++++------------ 1 file changed, 128 insertions(+), 53 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 5ab29cc64..c43e74163 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -15,8 +15,10 @@ import ( "net/http" "os" "path" + "strings" "time" + "github.com/elastic/go-elasticsearch/v7" "github.com/sethvargo/go-retry" "github.com/elastic/cloud-sdk-go/pkg/api" @@ -88,12 +90,6 @@ func (cp *cloudProvider) BootUp(options Options) error { region := "gcp-europe-west3" templateID := "gcp-io-optimized" - logger.Debugf("Uploading GeoIP databases") - geoIPExtension, err := cp.createGeoIPExtension() - if err != nil { - return fmt.Errorf("failed to create GeoIP extension: %w", err) - } - logger.Debugf("Getting deployment template %q", templateID) template, err := deptemplateapi.Get(deptemplateapi.GetParams{ API: cp.api, @@ -119,15 +115,6 @@ func (cp *cloudProvider) BootUp(options Options) error { plan.DeploymentTemplate = &models.DeploymentTemplateReference{} } plan.DeploymentTemplate.ID = &templateID - - // Add the GeoIP bundle. - plan.Elasticsearch.UserBundles = []*models.ElasticsearchUserBundle{ - &models.ElasticsearchUserBundle{ - ElasticsearchVersion: &options.StackVersion, - Name: geoIPExtension.Name, - URL: geoIPExtension.URL, - }, - } } logger.Debugf("Creating deployment %q", name) @@ -150,7 +137,6 @@ func (cp *cloudProvider) BootUp(options Options) error { config.Provider = ProviderCloud config.Parameters = map[string]string{ paramCloudDeploymentAlias: res.Alias, - paramGeoIPExtensionID: *geoIPExtension.ID, } deploymentID := res.ID if deploymentID == nil { @@ -198,12 +184,96 @@ func (cp *cloudProvider) BootUp(options Options) error { printUserConfig(options.Printer, config) + logger.Debug("Waiting for creation plan to be completed") + err = planutil.TrackChange(planutil.TrackChangeParams{ + TrackChangeParams: plan.TrackChangeParams{ + API: cp.api, + DeploymentID: *deploymentID, + }, + Writer: &cloudTrackWriter{}, + Format: "text", + }) + if err != nil { + return fmt.Errorf("failed to track cluster creation: %w", err) + } + + // Storing configuration now, so if something fails with the extension, we still + // keep track of the deployment id. err = storeConfig(cp.profile, config) if err != nil { return fmt.Errorf("failed to store config: %w", err) } - logger.Debug("Waiting for creation plan to be completed") + logger.Debugf("Replacing GeoIP databases") + + client, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{config.ElasticsearchHost}, + Username: config.ElasticsearchUsername, + Password: config.ElasticsearchPassword, + }) + if err != nil { + return fmt.Errorf("failed to initialize Elasticsearch client: %w", err) + } + + settingsPayload := `{"persistent": {"ingest.geoip.downloader.enabled":false}}` + resp, err := client.Cluster.PutSettings(strings.NewReader(settingsPayload)) + if err != nil { + return fmt.Errorf("failed to disable geoip automatic downloader: %w", err) + } + if resp.IsError() { + return fmt.Errorf("failed to disable geoup automatic downloader (status: %v): %v:", resp.StatusCode, resp.String()) + } + + geoIPExtension, err := cp.createGeoIPExtension() + if err != nil { + return fmt.Errorf("failed to create GeoIP extension: %w", err) + } + + config.Parameters[paramGeoIPExtensionID] = *geoIPExtension.ID + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + + // Add the GeoIP bundle. + updatePlan := models.ElasticsearchClusterPlan{ + // If no cluster topology is included, cluster is terminated. + ClusterTopology: payload.Resources.Elasticsearch[0].Plan.ClusterTopology, + Elasticsearch: &models.ElasticsearchConfiguration{ + UserBundles: []*models.ElasticsearchUserBundle{ + &models.ElasticsearchUserBundle{ + ElasticsearchVersion: &options.StackVersion, + Name: geoIPExtension.Name, + URL: geoIPExtension.URL, + }, + }, + Version: options.StackVersion, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: &templateID, + }, + } + pruneOrphans := false + _, err = deploymentapi.Update(deploymentapi.UpdateParams{ + API: cp.api, + DeploymentID: *deploymentID, + Request: &models.DeploymentUpdateRequest{ + PruneOrphans: &pruneOrphans, + Resources: &models.DeploymentUpdateResources{ + Elasticsearch: []*models.ElasticsearchPayload{ + &models.ElasticsearchPayload{ + RefID: res.Resources[0].RefID, + Region: res.Resources[0].Region, + Plan: &updatePlan, + }, + }, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to add extension to deployment: %w", err) + } + err = planutil.TrackChange(planutil.TrackChangeParams{ TrackChangeParams: plan.TrackChangeParams{ API: cp.api, @@ -220,41 +290,9 @@ func (cp *cloudProvider) BootUp(options Options) error { } func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { - // From https://www.elastic.co/guide/en/cloud/current/ec-custom-bundles.html - const baseDir = "ingest-geoip" - - files := []struct { - source string - target string - }{ - // These files cannot have the default prefix, we will rename them. - {"GeoLite2-ASN.mmdb", "ElasticPackage-ASN.mmdb"}, - {"GeoLite2-City.mmdb", "ElasticPackage-City.mmdb"}, - {"GeoLite2-Country.mmdb", "ElasticPackage-Country.mmdb"}, - } - - var bundle bytes.Buffer - w := zip.NewWriter(&bundle) - for _, f := range files { - fw, err := w.Create(path.Join(baseDir, f.target)) - if err != nil { - return nil, fmt.Errorf("failed to create file %q in bundle: %w", f.target, err) - } - - fr, err := static.Open(path.Join("_static", f.source)) - if err != nil { - return nil, fmt.Errorf("failed to open static file %q: %w", f.source, err) - } - - _, err = io.Copy(fw, fr) - if err != nil { - fr.Close() - return nil, fmt.Errorf("failed to copy contents of file %q: %w", f.source, err) - } - fr.Close() - } - if err := w.Close(); err != nil { - return nil, fmt.Errorf("failed to close bundle: %w", err) + bundle, err := zipGeoIPBundle() + if err != nil { + return nil, fmt.Errorf("failed to create GeoIP bundle: %w", err) } // TODO: Parameterize extension Name. @@ -276,7 +314,7 @@ func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { extension, err = extensionapi.Upload(extensionapi.UploadParams{ API: cp.api, ExtensionID: *extension.ID, - File: &bundle, + File: bundle, }) if err != nil { return nil, fmt.Errorf("failed to upload bundle: %w", err) @@ -285,6 +323,43 @@ func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { return extension, nil } +func zipGeoIPBundle() (*bytes.Buffer, error) { + // From https://www.elastic.co/guide/en/cloud/current/ec-custom-bundles.html + const baseDir = "ingest-geoip" + + files := []string{ + "GeoLite2-ASN.mmdb", + "GeoLite2-City.mmdb", + "GeoLite2-Country.mmdb", + } + + var bundle bytes.Buffer + w := zip.NewWriter(&bundle) + for _, fileName := range files { + fw, err := w.Create(path.Join(baseDir, fileName)) + if err != nil { + return nil, fmt.Errorf("failed to create file %q in bundle: %w", fileName, err) + } + + fr, err := static.Open(path.Join("_static", fileName)) + if err != nil { + return nil, fmt.Errorf("failed to open static file %q: %w", fileName, err) + } + + _, err = io.Copy(fw, fr) + if err != nil { + fr.Close() + return nil, fmt.Errorf("failed to copy contents of file %q: %w", fileName, err) + } + fr.Close() + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to close bundle: %w", err) + } + + return &bundle, nil +} + func (cp *cloudProvider) deleteGeoIPExtension() error { config, err := LoadConfig(cp.profile) if err != nil { From 8298102050930a56d85e286a3aae7d46638a2a5c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 01:03:38 +0200 Subject: [PATCH 05/16] Start local agent connected to cloud --- .../_static/cloud-elastic-agent.env.tmpl | 14 ++ .../_static/cloud-elastic-agent.yml.tmpl | 27 ++++ internal/stack/cloud.go | 140 +++++++++++++++--- internal/stack/cloudresources.go | 76 ++++++++++ 4 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 internal/stack/_static/cloud-elastic-agent.env.tmpl create mode 100644 internal/stack/_static/cloud-elastic-agent.yml.tmpl create mode 100644 internal/stack/cloudresources.go diff --git a/internal/stack/_static/cloud-elastic-agent.env.tmpl b/internal/stack/_static/cloud-elastic-agent.env.tmpl new file mode 100644 index 000000000..d8142ff42 --- /dev/null +++ b/internal/stack/_static/cloud-elastic-agent.env.tmpl @@ -0,0 +1,14 @@ +{{ $version := fact "agent_version" }} +{{ $fleetURL := fact "fleet_url" }} +{{ $kibanaHost := fact "kibana_host" }} +{{ $username := fact "username" }} +{{ $password := fact "password" }} +FLEET_ENROLL=1 +FLEET_URL={{ $fleetURL }} +KIBANA_FLEET_HOST={{ $kibanaHost }} +KIBANA_HOST={{ $kibanaHost }} +ELASTICSEARCH_USERNAME={{ $username }} +ELASTICSEARCH_PASSWORD={{ $password }} +{{ if not (semverLessThan $version "8.0.0") }} +FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) +{{ end }} diff --git a/internal/stack/_static/cloud-elastic-agent.yml.tmpl b/internal/stack/_static/cloud-elastic-agent.yml.tmpl new file mode 100644 index 000000000..de2fb0bad --- /dev/null +++ b/internal/stack/_static/cloud-elastic-agent.yml.tmpl @@ -0,0 +1,27 @@ +{{ $image := fact "agent_image" }} +version: '2.3' +services: + elastic-agent: + image: "{{ $image }}" + healthcheck: + test: "elastic-agent status" + timeout: 2s + start_period: 360s + retries: 180 + interval: 5s + hostname: docker-fleet-agent + env_file: "./elastic-agent.env" + volumes: + - type: bind + source: ../../../tmp/service_logs/ + target: /tmp/service_logs/ + # Mount service_logs under /run too as a testing workaround for the journald input (see elastic-package#1235). + - type: bind + source: ../../../tmp/service_logs/ + target: /run/service_logs/ + + elastic-agent_is_ready: + image: tianon/true + depends_on: + elastic-agent: + condition: service_healthy diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index c43e74163..211baed08 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -30,6 +30,8 @@ import ( "github.com/elastic/cloud-sdk-go/pkg/plan" "github.com/elastic/cloud-sdk-go/pkg/plan/planutil" + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" ) @@ -72,7 +74,8 @@ func (cp *cloudProvider) BootUp(options Options) error { logger.Warn("Elastic Cloud provider is in technical preview") _, err := cp.currentDeployment() - if err == nil { + switch err { + case nil: // Do nothing, deployment already exists. // TODO: Migrate configuration if changed. config, err := LoadConfig(cp.profile) @@ -81,7 +84,10 @@ func (cp *cloudProvider) BootUp(options Options) error { } printUserConfig(options.Printer, config) return nil - } else if err != nil && err != errDeploymentNotExist { + case errDeploymentNotExist: + // Deployment doesn't exist, let's continue. + break + default: return err } @@ -221,7 +227,7 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to disable geoip automatic downloader: %w", err) } if resp.IsError() { - return fmt.Errorf("failed to disable geoup automatic downloader (status: %v): %v:", resp.StatusCode, resp.String()) + return fmt.Errorf("failed to disable geoip automatic downloader (status: %v): %v", resp.StatusCode, resp.String()) } geoIPExtension, err := cp.createGeoIPExtension() @@ -286,6 +292,15 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to track cluster creation: %w", err) } + // FIXME: Create initial agent policy. + + logger.Debugf("Starting local agent") + + err = cp.startLocalAgent(options, config) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } + return nil } @@ -360,33 +375,41 @@ func zipGeoIPBundle() (*bytes.Buffer, error) { return &bundle, nil } -func (cp *cloudProvider) deleteGeoIPExtension() error { - config, err := LoadConfig(cp.profile) +func (cp *cloudProvider) localAgentComposeProject() (*compose.Project, error) { + composeFile := cp.profile.Path(profileStackPath, CloudComposeFile) + return compose.NewProject(DockerComposeProjectName, composeFile) +} + +func (cp *cloudProvider) startLocalAgent(options Options, config Config) error { + err := applyCloudResources(cp.profile, options.StackVersion, config) if err != nil { - return fmt.Errorf("failed to load configuration: %w", err) + return fmt.Errorf("could not initialize compose files for local agent: %w", err) } - extensionID, found := config.Parameters[paramGeoIPExtensionID] - if !found { - return nil + + project, err := cp.localAgentComposeProject() + if err != nil { + return fmt.Errorf("could not initialize local agent compose project") } - backoff := retry.NewFibonacci(1 * time.Second) - backoff = retry.WithMaxDuration(180*time.Second, backoff) - retry.Do(context.TODO(), backoff, func(ctx context.Context) error { - err = extensionapi.Delete(extensionapi.DeleteParams{ - API: cp.api, - ExtensionID: extensionID, - }) - // Actually, we should only retry on extensions.extension_in_use errors. - return retry.RetryableError(err) - }) + err = project.Build(compose.CommandOptions{}) if err != nil { - return fmt.Errorf("delete API call failed: %w", err) + return fmt.Errorf("failed to build images for local agent: %w", err) + } + + err = project.Up(compose.CommandOptions{}) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) } + return nil } func (cp *cloudProvider) TearDown(options Options) error { + err := cp.destroyLocalAgent() + if err != nil { + return fmt.Errorf("failed to destroy local agent: %w", err) + } + deployment, err := cp.currentDeployment() if err != nil { return fmt.Errorf("failed to find current deployment: %w", err) @@ -415,6 +438,46 @@ func (cp *cloudProvider) TearDown(options Options) error { return nil } +func (cp *cloudProvider) deleteGeoIPExtension() error { + config, err := LoadConfig(cp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + extensionID, found := config.Parameters[paramGeoIPExtensionID] + if !found { + return nil + } + + backoff := retry.NewFibonacci(1 * time.Second) + backoff = retry.WithMaxDuration(180*time.Second, backoff) + retry.Do(context.TODO(), backoff, func(ctx context.Context) error { + err = extensionapi.Delete(extensionapi.DeleteParams{ + API: cp.api, + ExtensionID: extensionID, + }) + // Actually, we should only retry on extensions.extension_in_use errors. + return retry.RetryableError(err) + }) + if err != nil { + return fmt.Errorf("delete API call failed: %w", err) + } + return nil +} + +func (cp *cloudProvider) destroyLocalAgent() error { + project, err := cp.localAgentComposeProject() + if err != nil { + return fmt.Errorf("could not initialize local agent compose project") + } + + err = project.Down(compose.CommandOptions{}) + if err != nil { + return fmt.Errorf("failed to destroy local agent: %w", err) + } + + return nil +} + func (*cloudProvider) Update(options Options) error { fmt.Println("Nothing to do.") return nil @@ -431,6 +494,14 @@ func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { } status, _ := cp.deploymentStatus(deployment) + + agentStatus, err := cp.localAgentStatus() + if err != nil { + return nil, fmt.Errorf("failed to get local agent status: %w", err) + } + + status = append(status, agentStatus...) + return status, nil } @@ -496,6 +567,35 @@ func (*cloudProvider) deploymentStatus(deployment *models.DeploymentGetResponse) return status, allHealthy } +func (cp *cloudProvider) localAgentStatus() ([]ServiceStatus, error) { + var services []ServiceStatus + // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles + containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName) + if err != nil { + return nil, err + } + + if len(containerIDs) == 0 { + return services, nil + } + + containerDescriptions, err := docker.InspectContainers(containerIDs...) + if err != nil { + return nil, err + } + + for _, containerDescription := range containerDescriptions { + service, err := newServiceStatus(&containerDescription) + if err != nil { + return nil, err + } + logger.Debugf("Adding Service: \"%v\"", service.Name) + services = append(services, *service) + } + + return services, nil +} + func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { config, err := LoadConfig(cp.profile) if err != nil { diff --git a/internal/stack/cloudresources.go b/internal/stack/cloudresources.go new file mode 100644 index 000000000..d0a921096 --- /dev/null +++ b/internal/stack/cloudresources.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/elastic/go-resource" + + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/profile" +) + +const ( + // CloudElasticAgentEnvFile is the elastic agent environment variables file for the + // cloud provider. + CloudElasticAgentEnvFile = "cloud-elastic-agent.env" + + // CloudComposeFile is the docker-compose snapshot.yml file name. + CloudComposeFile = "cloud-elastic-agent.yml" +) + +var ( + cloudStackResources = []resource.Resource{ + &resource.File{ + Path: CloudComposeFile, + Content: staticSource.Template("_static/cloud-elastic-agent.yml.tmpl"), + }, + &resource.File{ + Path: CloudElasticAgentEnvFile, + Content: staticSource.Template("_static/cloud-elastic-agent.env.tmpl"), + }, + } +) + +func applyCloudResources(profile *profile.Profile, stackVersion string, config Config) error { + appConfig, err := install.Configuration() + if err != nil { + return fmt.Errorf("can't read application configuration: %w", err) + } + + stackDir := filepath.Join(profile.ProfilePath, profileStackPath) + + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "agent_version": stackVersion, + "agent_image": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "username": config.ElasticsearchUsername, + "password": config.ElasticsearchPassword, + "kibana_host": config.KibanaHost, + "fleet_url": config.Parameters["fleet_url"], + }) + + os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: stackDir, + }) + + results, err := resourceManager.Apply(cloudStackResources) + if err != nil { + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) + } + + return nil +} From 1931b772c57df800faa5e5af303921d0f2e0d713 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 11:45:19 +0200 Subject: [PATCH 06/16] Create Agent policy --- internal/stack/cloud.go | 103 +++++++++++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 11 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 211baed08..f8fad5b78 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path" "strings" @@ -123,7 +124,7 @@ func (cp *cloudProvider) BootUp(options Options) error { plan.DeploymentTemplate.ID = &templateID } - logger.Debugf("Creating deployment %q", name) + logger.Infof("Creating deployment %q", name) res, err := deploymentapi.Create(deploymentapi.CreateParams{ API: cp.api, Request: payload, @@ -210,7 +211,21 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to store config: %w", err) } - logger.Debugf("Replacing GeoIP databases") + logger.Infof("Creating agent policy") + + err = cp.createAgentPolicy(config) + if err != nil { + return fmt.Errorf("failed to create agent policy: %w", err) + } + + logger.Infof("Starting local agent") + + err = cp.startLocalAgent(options, config) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } + + logger.Infof("Replacing GeoIP databases") client, err := elasticsearch.NewClient(elasticsearch.Config{ Addresses: []string{config.ElasticsearchHost}, @@ -292,15 +307,6 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to track cluster creation: %w", err) } - // FIXME: Create initial agent policy. - - logger.Debugf("Starting local agent") - - err = cp.startLocalAgent(options, config) - if err != nil { - return fmt.Errorf("failed to start local agent: %w", err) - } - return nil } @@ -375,6 +381,81 @@ func zipGeoIPBundle() (*bytes.Buffer, error) { return &bundle, nil } +const cloudKibanaAgentPolicy = `{ + "name": "Elastic-Agent (elastic-package)", + "id": "elastic-agent-managed-ep", + "description": "Policy created by elastic-package", + "namespace": "default", + "monitoring_enabled": [ + "logs", + "metrics" + ] +}` + +// TODO: Avoid hard-coding the package version here. +const cloudKibanaPackagePolicy = `{ + "name": "system-1", + "policy_id": "elastic-agent-managed-ep", + "package": { + "name": "system", + "version": "1.26.0" + } +}` + +func doKibanaRequest(config Config, req *http.Request) error { + req.SetBasicAuth(config.ElasticsearchUsername, config.ElasticsearchPassword) + req.Header.Add("content-type", "application/json") + req.Header.Add("kbn-xsrf", "elastic-package") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("performing request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + // Already created, go on. + // TODO: We could try to update the policy. + return nil + } + if resp.StatusCode >= 300 { + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("request failed with status %v and could not read body: %w", resp.StatusCode, err) + } + return fmt.Errorf("request failed with status %v and response %v", resp.StatusCode, string(body)) + } + return nil +} + +func (cp *cloudProvider) createAgentPolicy(config Config) error { + agentPoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/agent_policies") + if err != nil { + return fmt.Errorf("failed to build url for agent policies: %w", err) + } + req, err := http.NewRequest(http.MethodPost, agentPoliciesURL, strings.NewReader(cloudKibanaAgentPolicy)) + if err != nil { + return fmt.Errorf("failed to initialize request to create agent policy: %w", err) + } + err = doKibanaRequest(config, req) + if err != nil { + return fmt.Errorf("error while creating agent policy: %w", err) + } + + packagePoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/package_policies") + if err != nil { + return fmt.Errorf("failed to build url for package policies: %w", err) + } + req, err = http.NewRequest(http.MethodPost, packagePoliciesURL, strings.NewReader(cloudKibanaPackagePolicy)) + if err != nil { + return fmt.Errorf("failed to initialize request to create package policy: %w", err) + } + err = doKibanaRequest(config, req) + if err != nil { + return fmt.Errorf("error while creating package policy: %w", err) + } + + return nil +} + func (cp *cloudProvider) localAgentComposeProject() (*compose.Project, error) { composeFile := cp.profile.Path(profileStackPath, CloudComposeFile) return compose.NewProject(DockerComposeProjectName, composeFile) From 0938e7a6276eb04283ba8374ef0fcecad13115d4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 12:59:57 +0200 Subject: [PATCH 07/16] Fix path to env file --- internal/stack/_static/cloud-elastic-agent.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stack/_static/cloud-elastic-agent.yml.tmpl b/internal/stack/_static/cloud-elastic-agent.yml.tmpl index de2fb0bad..d70534061 100644 --- a/internal/stack/_static/cloud-elastic-agent.yml.tmpl +++ b/internal/stack/_static/cloud-elastic-agent.yml.tmpl @@ -10,7 +10,7 @@ services: retries: 180 interval: 5s hostname: docker-fleet-agent - env_file: "./elastic-agent.env" + env_file: "./cloud-elastic-agent.env" volumes: - type: bind source: ../../../tmp/service_logs/ From 3a1643f90222586823d32f4ee912aa43464c7ea4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 15:21:43 +0200 Subject: [PATCH 08/16] Retrieve Fleet Server url from Fleet API --- internal/stack/cloud.go | 74 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index f8fad5b78..f49051167 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -184,13 +184,22 @@ func (cp *cloudProvider) BootUp(options Options) error { if err != nil { return fmt.Errorf("failed to get kibana host: %w", err) } - config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) - if err != nil { - return fmt.Errorf("failed to get fleet host: %w", err) - } + + // FIXME: Why this URL is not the good one? + //config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) + //if err != nil { + // return fmt.Errorf("failed to get fleet host: %w", err) + //} printUserConfig(options.Printer, config) + // Storing configuration now, so if something fails from now on, we still + // keep track of the deployment id. + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + logger.Debug("Waiting for creation plan to be completed") err = planutil.TrackChange(planutil.TrackChangeParams{ TrackChangeParams: plan.TrackChangeParams{ @@ -204,11 +213,10 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to track cluster creation: %w", err) } - // Storing configuration now, so if something fails with the extension, we still - // keep track of the deployment id. - err = storeConfig(cp.profile, config) + // FIXME: See comment above, why the Integrations Server URL cannot be used? + config.Parameters["fleet_url"], err = getDefaultFleetServerURL(config) if err != nil { - return fmt.Errorf("failed to store config: %w", err) + return fmt.Errorf("failed to get fleet URL: %w", err) } logger.Infof("Creating agent policy") @@ -426,6 +434,51 @@ func doKibanaRequest(config Config, req *http.Request) error { return nil } +func getDefaultFleetServerURL(config Config) (string, error) { + fleetServersURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/fleet_server_hosts") + if err != nil { + return "", fmt.Errorf("failed to build url for fleet server hosts: %w", err) + } + req, err := http.NewRequest(http.MethodGet, fleetServersURL, nil) + if err != nil { + return "", fmt.Errorf("failed to initialize request: %w", err) + } + req.SetBasicAuth(config.ElasticsearchUsername, config.ElasticsearchPassword) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("performing request failed: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + logger.Debugf("Fleet Server hosts response: %q", string(body)) + + if err != nil { + return "", fmt.Errorf("could not read response body (status %s): %w", resp.StatusCode, err) + } + if resp.StatusCode >= 300 { + return "", fmt.Errorf("request failed with status %v and response %v", resp.StatusCode, string(body)) + } + + var hosts struct { + Items []struct { + IsDefault bool `json:"is_default"` + HostURLs []string `json:"host_urls"` + } `json:"items"` + } + err = json.Unmarshal(body, &hosts) + if err != nil { + return "", fmt.Errorf("failed to decode response: %w", err) + } + + for _, server := range hosts.Items { + if server.IsDefault && len(server.HostURLs) > 0 { + return server.HostURLs[0], nil + } + } + + return "", errors.New("could not find the fleet server URL for this deployment") +} + func (cp *cloudProvider) createAgentPolicy(config Config) error { agentPoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/agent_policies") if err != nil { @@ -477,7 +530,7 @@ func (cp *cloudProvider) startLocalAgent(options Options, config Config) error { return fmt.Errorf("failed to build images for local agent: %w", err) } - err = project.Up(compose.CommandOptions{}) + err = project.Up(compose.CommandOptions{ExtraArgs: []string{"-d"}}) if err != nil { return fmt.Errorf("failed to start local agent: %w", err) } @@ -670,6 +723,9 @@ func (cp *cloudProvider) localAgentStatus() ([]ServiceStatus, error) { if err != nil { return nil, err } + if strings.HasSuffix(service.Name, readyServicesSuffix) { + continue + } logger.Debugf("Adding Service: \"%v\"", service.Name) services = append(services, *service) } From f71f8cbe7ca290baead21ab2677636ac6d845f98 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 15:34:04 +0200 Subject: [PATCH 09/16] Use facts directly in templates where possible --- .../stack/_static/cloud-elastic-agent.env.tmpl | 14 +++++--------- .../stack/_static/cloud-elastic-agent.yml.tmpl | 3 +-- internal/stack/cloud.go | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/internal/stack/_static/cloud-elastic-agent.env.tmpl b/internal/stack/_static/cloud-elastic-agent.env.tmpl index d8142ff42..d8888122e 100644 --- a/internal/stack/_static/cloud-elastic-agent.env.tmpl +++ b/internal/stack/_static/cloud-elastic-agent.env.tmpl @@ -1,14 +1,10 @@ {{ $version := fact "agent_version" }} -{{ $fleetURL := fact "fleet_url" }} -{{ $kibanaHost := fact "kibana_host" }} -{{ $username := fact "username" }} -{{ $password := fact "password" }} FLEET_ENROLL=1 -FLEET_URL={{ $fleetURL }} -KIBANA_FLEET_HOST={{ $kibanaHost }} -KIBANA_HOST={{ $kibanaHost }} -ELASTICSEARCH_USERNAME={{ $username }} -ELASTICSEARCH_PASSWORD={{ $password }} +FLEET_URL={{ fact "fleet_url" }} +KIBANA_FLEET_HOST={{ fact "kibana_host" }} +KIBANA_HOST={{ fact "kibana_host" }} +ELASTICSEARCH_USERNAME={{ fact "username" }} +ELASTICSEARCH_PASSWORD={{ fact "password" }} {{ if not (semverLessThan $version "8.0.0") }} FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) {{ end }} diff --git a/internal/stack/_static/cloud-elastic-agent.yml.tmpl b/internal/stack/_static/cloud-elastic-agent.yml.tmpl index d70534061..63cd82497 100644 --- a/internal/stack/_static/cloud-elastic-agent.yml.tmpl +++ b/internal/stack/_static/cloud-elastic-agent.yml.tmpl @@ -1,8 +1,7 @@ -{{ $image := fact "agent_image" }} version: '2.3' services: elastic-agent: - image: "{{ $image }}" + image: "{{ fact "agent_image" }}" healthcheck: test: "elastic-agent status" timeout: 2s diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index f49051167..35c426aff 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -453,7 +453,7 @@ func getDefaultFleetServerURL(config Config) (string, error) { logger.Debugf("Fleet Server hosts response: %q", string(body)) if err != nil { - return "", fmt.Errorf("could not read response body (status %s): %w", resp.StatusCode, err) + return "", fmt.Errorf("could not read response body (status %v): %w", resp.StatusCode, err) } if resp.StatusCode >= 300 { return "", fmt.Errorf("request failed with status %v and response %v", resp.StatusCode, string(body)) From ee3a31b2bc2dbb98c81729a3d962726df443d53b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 May 2023 18:26:08 +0200 Subject: [PATCH 10/16] Some refactors --- internal/stack/cloud.go | 307 +++++++++++++++++++------------ internal/stack/cloudresources.go | 2 +- 2 files changed, 190 insertions(+), 119 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 35c426aff..193f878a5 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -19,7 +19,6 @@ import ( "strings" "time" - "github.com/elastic/go-elasticsearch/v7" "github.com/sethvargo/go-retry" "github.com/elastic/cloud-sdk-go/pkg/api" @@ -30,6 +29,7 @@ import ( "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/elastic/cloud-sdk-go/pkg/plan" "github.com/elastic/cloud-sdk-go/pkg/plan/planutil" + "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" @@ -38,9 +38,11 @@ import ( ) const ( - paramCloudDeploymentID = "cloud_deployment_id" - paramCloudDeploymentAlias = "cloud_deployment_alias" - paramGeoIPExtensionID = "geoip_extension_id" + paramCloudDeploymentID = "cloud_deployment_id" + paramCloudClusterRefID = "cloud_ref_id" + paramCloudDeploymentAlias = "cloud_deployment_alias" + paramCloudGeoIPExtensionID = "cloud_geoip_extension_id" + paramCloudFleetURL = "cloud_fleet_url" ) var ( @@ -98,6 +100,39 @@ func (cp *cloudProvider) BootUp(options Options) error { templateID := "gcp-io-optimized" logger.Debugf("Getting deployment template %q", templateID) + payload, err := cp.getDeploymentRequest(options, region, templateID) + if err != nil { + return fmt.Errorf("failed to get deployment template: %w", err) + } + + logger.Infof("Creating deployment %q", name) + config, err := cp.createDeployment(name, options, payload) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + + logger.Infof("Creating agent policy") + err = cp.createAgentPolicy(config, options.StackVersion) + if err != nil { + return fmt.Errorf("failed to create agent policy: %w", err) + } + + logger.Infof("Starting local agent") + err = cp.startLocalAgent(options, config) + if err != nil { + return fmt.Errorf("failed to start local agent: %w", err) + } + + logger.Infof("Replacing GeoIP databases") + err = cp.replaceGeoIPDatabases(config, options, templateID, region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) + if err != nil { + return fmt.Errorf("failed to replace GeoIP databases: %w", err) + } + + return nil +} + +func (cp *cloudProvider) getDeploymentRequest(options Options, region, templateID string) (*models.DeploymentCreateRequest, error) { template, err := deptemplateapi.Get(deptemplateapi.GetParams{ API: cp.api, TemplateID: templateID, @@ -105,7 +140,7 @@ func (cp *cloudProvider) BootUp(options Options) error { StackVersion: options.StackVersion, }) if err != nil { - return fmt.Errorf("failed to get deployment template %q: %w", templateID, err) + return nil, fmt.Errorf("failed to get deployment template %q: %w", templateID, err) } payload := template.DeploymentTemplate @@ -124,7 +159,10 @@ func (cp *cloudProvider) BootUp(options Options) error { plan.DeploymentTemplate.ID = &templateID } - logger.Infof("Creating deployment %q", name) + return payload, nil +} + +func (cp *cloudProvider) createDeployment(name string, options Options, payload *models.DeploymentCreateRequest) (Config, error) { res, err := deploymentapi.Create(deploymentapi.CreateParams{ API: cp.api, Request: payload, @@ -134,10 +172,10 @@ func (cp *cloudProvider) BootUp(options Options) error { }, }) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + return Config{}, fmt.Errorf("failed to create deployment: %w", err) } if created := res.Created; created == nil || !*created { - return fmt.Errorf("request succeeded, but deployment was not created, check in the console UI") + return Config{}, fmt.Errorf("request succeeded, but deployment was not created, check in the console UI") } var config Config @@ -147,9 +185,10 @@ func (cp *cloudProvider) BootUp(options Options) error { } deploymentID := res.ID if deploymentID == nil { - return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") + return Config{}, fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") } config.Parameters[paramCloudDeploymentID] = *deploymentID + config.Parameters[paramCloudClusterRefID] = *res.Resources[0].RefID for _, resource := range res.Resources { kind := resource.Kind @@ -173,20 +212,20 @@ func (cp *cloudProvider) BootUp(options Options) error { DeploymentID: *deploymentID, }) if err != nil { - return fmt.Errorf("couldn't check deployment health: %w", err) + return Config{}, fmt.Errorf("couldn't check deployment health: %w", err) } config.ElasticsearchHost, err = cp.getServiceURL(deployment.Resources.Elasticsearch) if err != nil { - return fmt.Errorf("failed to get elasticsearch host: %w", err) + return Config{}, fmt.Errorf("failed to get elasticsearch host: %w", err) } config.KibanaHost, err = cp.getServiceURL(deployment.Resources.Kibana) if err != nil { - return fmt.Errorf("failed to get kibana host: %w", err) + return Config{}, fmt.Errorf("failed to get kibana host: %w", err) } // FIXME: Why this URL is not the good one? - //config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) + //config.Parameters[paramCloudFleetURL], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) //if err != nil { // return fmt.Errorf("failed to get fleet host: %w", err) //} @@ -197,7 +236,7 @@ func (cp *cloudProvider) BootUp(options Options) error { // keep track of the deployment id. err = storeConfig(cp.profile, config) if err != nil { - return fmt.Errorf("failed to store config: %w", err) + return Config{}, fmt.Errorf("failed to store config: %w", err) } logger.Debug("Waiting for creation plan to be completed") @@ -210,112 +249,16 @@ func (cp *cloudProvider) BootUp(options Options) error { Format: "text", }) if err != nil { - return fmt.Errorf("failed to track cluster creation: %w", err) + return Config{}, fmt.Errorf("failed to track cluster creation: %w", err) } // FIXME: See comment above, why the Integrations Server URL cannot be used? - config.Parameters["fleet_url"], err = getDefaultFleetServerURL(config) - if err != nil { - return fmt.Errorf("failed to get fleet URL: %w", err) - } - - logger.Infof("Creating agent policy") - - err = cp.createAgentPolicy(config) - if err != nil { - return fmt.Errorf("failed to create agent policy: %w", err) - } - - logger.Infof("Starting local agent") - - err = cp.startLocalAgent(options, config) - if err != nil { - return fmt.Errorf("failed to start local agent: %w", err) - } - - logger.Infof("Replacing GeoIP databases") - - client, err := elasticsearch.NewClient(elasticsearch.Config{ - Addresses: []string{config.ElasticsearchHost}, - Username: config.ElasticsearchUsername, - Password: config.ElasticsearchPassword, - }) + config.Parameters[paramCloudFleetURL], err = getDefaultFleetServerURL(config) if err != nil { - return fmt.Errorf("failed to initialize Elasticsearch client: %w", err) - } - - settingsPayload := `{"persistent": {"ingest.geoip.downloader.enabled":false}}` - resp, err := client.Cluster.PutSettings(strings.NewReader(settingsPayload)) - if err != nil { - return fmt.Errorf("failed to disable geoip automatic downloader: %w", err) - } - if resp.IsError() { - return fmt.Errorf("failed to disable geoip automatic downloader (status: %v): %v", resp.StatusCode, resp.String()) - } - - geoIPExtension, err := cp.createGeoIPExtension() - if err != nil { - return fmt.Errorf("failed to create GeoIP extension: %w", err) - } - - config.Parameters[paramGeoIPExtensionID] = *geoIPExtension.ID - err = storeConfig(cp.profile, config) - if err != nil { - return fmt.Errorf("failed to store config: %w", err) - } - - // Add the GeoIP bundle. - updatePlan := models.ElasticsearchClusterPlan{ - // If no cluster topology is included, cluster is terminated. - ClusterTopology: payload.Resources.Elasticsearch[0].Plan.ClusterTopology, - Elasticsearch: &models.ElasticsearchConfiguration{ - UserBundles: []*models.ElasticsearchUserBundle{ - &models.ElasticsearchUserBundle{ - ElasticsearchVersion: &options.StackVersion, - Name: geoIPExtension.Name, - URL: geoIPExtension.URL, - }, - }, - Version: options.StackVersion, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: &templateID, - }, - } - pruneOrphans := false - _, err = deploymentapi.Update(deploymentapi.UpdateParams{ - API: cp.api, - DeploymentID: *deploymentID, - Request: &models.DeploymentUpdateRequest{ - PruneOrphans: &pruneOrphans, - Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - &models.ElasticsearchPayload{ - RefID: res.Resources[0].RefID, - Region: res.Resources[0].Region, - Plan: &updatePlan, - }, - }, - }, - }, - }) - if err != nil { - return fmt.Errorf("failed to add extension to deployment: %w", err) + return Config{}, fmt.Errorf("failed to get fleet URL: %w", err) } - err = planutil.TrackChange(planutil.TrackChangeParams{ - TrackChangeParams: plan.TrackChangeParams{ - API: cp.api, - DeploymentID: *deploymentID, - }, - Writer: &cloudTrackWriter{}, - Format: "text", - }) - if err != nil { - return fmt.Errorf("failed to track cluster creation: %w", err) - } - - return nil + return config, nil } func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { @@ -406,7 +349,7 @@ const cloudKibanaPackagePolicy = `{ "policy_id": "elastic-agent-managed-ep", "package": { "name": "system", - "version": "1.26.0" + "version": "%s" } }` @@ -479,7 +422,7 @@ func getDefaultFleetServerURL(config Config) (string, error) { return "", errors.New("could not find the fleet server URL for this deployment") } -func (cp *cloudProvider) createAgentPolicy(config Config) error { +func (cp *cloudProvider) createAgentPolicy(config Config, stackVersion string) error { agentPoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/agent_policies") if err != nil { return fmt.Errorf("failed to build url for agent policies: %w", err) @@ -493,11 +436,17 @@ func (cp *cloudProvider) createAgentPolicy(config Config) error { return fmt.Errorf("error while creating agent policy: %w", err) } + systemVersion, err := getPackageVersion("https://epr.elastic.co", "system", stackVersion) + if err != nil { + return fmt.Errorf("could not get the system package version for kibana %v: %w", stackVersion, err) + } + packagePoliciesURL, err := url.JoinPath(config.KibanaHost, "/api/fleet/package_policies") if err != nil { return fmt.Errorf("failed to build url for package policies: %w", err) } - req, err = http.NewRequest(http.MethodPost, packagePoliciesURL, strings.NewReader(cloudKibanaPackagePolicy)) + packagePolicy := fmt.Sprintf(cloudKibanaPackagePolicy, systemVersion) + req, err = http.NewRequest(http.MethodPost, packagePoliciesURL, strings.NewReader(packagePolicy)) if err != nil { return fmt.Errorf("failed to initialize request to create package policy: %w", err) } @@ -509,6 +458,42 @@ func (cp *cloudProvider) createAgentPolicy(config Config) error { return nil } +func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { + searchURL, err := url.JoinPath(registryURL, "search") + if err != nil { + return "", fmt.Errorf("could not build URL: %w", err) + } + searchURL = fmt.Sprintf("%s?package=%s&kibana.version=%s", searchURL, packageName, stackVersion) + resp, err := http.Get(searchURL) + if err != nil { + return "", fmt.Errorf("request failed (url: %s): %w", searchURL, err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return "", fmt.Errorf("unexpected status code %v", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + var packages []struct { + Name string `json:"name"` + Version string `json:"version"` + } + err = json.Unmarshal(body, &packages) + if err != nil { + return "", fmt.Errorf("failed to parse response body: %w", err) + } + if len(packages) != 1 { + return "", fmt.Errorf("expected 1 package, obtained %v", len(packages)) + } + if found := packages[0].Name; found != packageName { + return "", fmt.Errorf("expected package %s, found %s", packageName, found) + } + + return packages[0].Version, nil +} + func (cp *cloudProvider) localAgentComposeProject() (*compose.Project, error) { composeFile := cp.profile.Path(profileStackPath, CloudComposeFile) return compose.NewProject(DockerComposeProjectName, composeFile) @@ -538,6 +523,92 @@ func (cp *cloudProvider) startLocalAgent(options Options, config Config) error { return nil } +func (cp *cloudProvider) replaceGeoIPDatabases(config Config, options Options, templateID string, region string, topology []*models.ElasticsearchClusterTopologyElement) error { + client, err := elasticsearch.NewClient(elasticsearch.Config{ + Addresses: []string{config.ElasticsearchHost}, + Username: config.ElasticsearchUsername, + Password: config.ElasticsearchPassword, + }) + if err != nil { + return fmt.Errorf("failed to initialize Elasticsearch client: %w", err) + } + + settingsPayload := `{"persistent": {"ingest.geoip.downloader.enabled":false}}` + resp, err := client.Cluster.PutSettings(strings.NewReader(settingsPayload)) + if err != nil { + return fmt.Errorf("failed to disable geoip automatic downloader: %w", err) + } + if resp.IsError() { + return fmt.Errorf("failed to disable geoip automatic downloader (status: %v): %v", resp.StatusCode, resp.String()) + } + + geoIPExtension, err := cp.createGeoIPExtension() + if err != nil { + return fmt.Errorf("failed to create GeoIP extension: %w", err) + } + + config.Parameters[paramCloudGeoIPExtensionID] = *geoIPExtension.ID + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + + // Add the GeoIP bundle. + updatePlan := models.ElasticsearchClusterPlan{ + // If no cluster topology is included, cluster is terminated. + ClusterTopology: topology, + Elasticsearch: &models.ElasticsearchConfiguration{ + UserBundles: []*models.ElasticsearchUserBundle{ + &models.ElasticsearchUserBundle{ + ElasticsearchVersion: &options.StackVersion, + Name: geoIPExtension.Name, + URL: geoIPExtension.URL, + }, + }, + Version: options.StackVersion, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: &templateID, + }, + } + deploymentID := config.Parameters[paramCloudDeploymentID] + refID := config.Parameters[paramCloudClusterRefID] + pruneOrphans := false + _, err = deploymentapi.Update(deploymentapi.UpdateParams{ + API: cp.api, + DeploymentID: deploymentID, + Request: &models.DeploymentUpdateRequest{ + PruneOrphans: &pruneOrphans, + Resources: &models.DeploymentUpdateResources{ + Elasticsearch: []*models.ElasticsearchPayload{ + &models.ElasticsearchPayload{ + RefID: &refID, + Region: ®ion, + Plan: &updatePlan, + }, + }, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to add extension to deployment: %w", err) + } + + err = planutil.TrackChange(planutil.TrackChangeParams{ + TrackChangeParams: plan.TrackChangeParams{ + API: cp.api, + DeploymentID: deploymentID, + }, + Writer: &cloudTrackWriter{}, + Format: "text", + }) + if err != nil { + return fmt.Errorf("failed to track cluster creation: %w", err) + } + + return nil +} + func (cp *cloudProvider) TearDown(options Options) error { err := cp.destroyLocalAgent() if err != nil { @@ -577,7 +648,7 @@ func (cp *cloudProvider) deleteGeoIPExtension() error { if err != nil { return fmt.Errorf("failed to load configuration: %w", err) } - extensionID, found := config.Parameters[paramGeoIPExtensionID] + extensionID, found := config.Parameters[paramCloudGeoIPExtensionID] if !found { return nil } diff --git a/internal/stack/cloudresources.go b/internal/stack/cloudresources.go index d0a921096..f07f4961c 100644 --- a/internal/stack/cloudresources.go +++ b/internal/stack/cloudresources.go @@ -53,7 +53,7 @@ func applyCloudResources(profile *profile.Profile, stackVersion string, config C "username": config.ElasticsearchUsername, "password": config.ElasticsearchPassword, "kibana_host": config.KibanaHost, - "fleet_url": config.Parameters["fleet_url"], + "fleet_url": config.Parameters[paramCloudFleetURL], }) os.MkdirAll(stackDir, 0755) From 6255827086fd1a39efde99b7f533c303cb2543a1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 25 May 2023 13:54:50 +0200 Subject: [PATCH 11/16] Reduce size of deployment --- internal/stack/cloud.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 193f878a5..ca7530620 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -97,7 +97,7 @@ func (cp *cloudProvider) BootUp(options Options) error { // TODO: Parameterize this. name := "elastic-package-test" region := "gcp-europe-west3" - templateID := "gcp-io-optimized" + templateID := "gcp-general-purpose-v5" logger.Debugf("Getting deployment template %q", templateID) payload, err := cp.getDeploymentRequest(options, region, templateID) @@ -111,6 +111,12 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to create deployment: %w", err) } + logger.Infof("Replacing GeoIP databases") + err = cp.replaceGeoIPDatabases(config, options, templateID, region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) + if err != nil { + return fmt.Errorf("failed to replace GeoIP databases: %w", err) + } + logger.Infof("Creating agent policy") err = cp.createAgentPolicy(config, options.StackVersion) if err != nil { @@ -123,12 +129,6 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to start local agent: %w", err) } - logger.Infof("Replacing GeoIP databases") - err = cp.replaceGeoIPDatabases(config, options, templateID, region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) - if err != nil { - return fmt.Errorf("failed to replace GeoIP databases: %w", err) - } - return nil } @@ -157,6 +157,14 @@ func (cp *cloudProvider) getDeploymentRequest(options Options, region, templateI plan.DeploymentTemplate = &models.DeploymentTemplateReference{} } plan.DeploymentTemplate.ID = &templateID + + for _, tier := range plan.ClusterTopology { + if tier.ID == "hot_content" { + memory := int32(4096) + tier.Size.Value = &memory + tier.ZoneCount = 1 + } + } } return payload, nil @@ -188,6 +196,9 @@ func (cp *cloudProvider) createDeployment(name string, options Options, payload return Config{}, fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") } config.Parameters[paramCloudDeploymentID] = *deploymentID + + // We need the ref id to make update requests, otherwise we need to make a get deployment + // request using the deployment ID. config.Parameters[paramCloudClusterRefID] = *res.Resources[0].RefID for _, resource := range res.Resources { @@ -393,8 +404,6 @@ func getDefaultFleetServerURL(config Config) (string, error) { } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) - logger.Debugf("Fleet Server hosts response: %q", string(body)) - if err != nil { return "", fmt.Errorf("could not read response body (status %v): %w", resp.StatusCode, err) } From 9b22f2cb4e21073fd85f49dc5b378494677aa719 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 25 May 2023 14:43:41 +0200 Subject: [PATCH 12/16] Parameterize deployment --- internal/profile/profile.go | 10 ++++ internal/stack/cloud.go | 93 ++++++++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 250107289..e94e64e5a 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/elastic/go-resource" @@ -154,6 +155,15 @@ func (profile Profile) Config(name string, def string) string { return v } +// Config returns a configuration setting as integer, or its default if setting not found +func (profile Profile) ConfigInt(name string, def int) (int, error) { + v, found := profile.config.get(name) + if !found { + return def, nil + } + return strconv.Atoi(v) +} + // ErrNotAProfile is returned in cases where we don't have a valid profile directory var ErrNotAProfile = errors.New("not a profile") diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index ca7530620..86417b7b3 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -94,25 +94,25 @@ func (cp *cloudProvider) BootUp(options Options) error { return err } - // TODO: Parameterize this. - name := "elastic-package-test" - region := "gcp-europe-west3" - templateID := "gcp-general-purpose-v5" + settings, err := getDeploymentSettings(options) + if err != nil { + return err + } - logger.Debugf("Getting deployment template %q", templateID) - payload, err := cp.getDeploymentRequest(options, region, templateID) + logger.Debugf("Getting deployment template %q", settings.TemplateID) + payload, err := cp.getDeploymentRequest(settings) if err != nil { return fmt.Errorf("failed to get deployment template: %w", err) } - logger.Infof("Creating deployment %q", name) - config, err := cp.createDeployment(name, options, payload) + logger.Infof("Creating deployment %q", settings.Name) + config, err := cp.createDeployment(settings.Name, options, payload) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) } logger.Infof("Replacing GeoIP databases") - err = cp.replaceGeoIPDatabases(config, options, templateID, region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) + err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) if err != nil { return fmt.Errorf("failed to replace GeoIP databases: %w", err) } @@ -132,15 +132,60 @@ func (cp *cloudProvider) BootUp(options Options) error { return nil } -func (cp *cloudProvider) getDeploymentRequest(options Options, region, templateID string) (*models.DeploymentCreateRequest, error) { +type deploymentSettings struct { + Name string + Region string + TemplateID string + + StackVersion string + + ZoneCount int + MemorySize int +} + +func getDeploymentSettings(options Options) (deploymentSettings, error) { + const ( + configMemorySize = "stack.cloud.memory_size" + configRegion = "stack.cloud.region" + configTemplate = "stack.cloud.template" + configZoneCount = "stack.cloud.zone_count" + ) + s := deploymentSettings{ + Name: fmt.Sprintf("elastic-package-test-%s", options.Profile.ProfileName), + Region: options.Profile.Config(configRegion, "gcp-europe-west3"), + TemplateID: options.Profile.Config(configTemplate, "gcp-general-purpose-v5"), + StackVersion: options.StackVersion, + } + + var err error + s.ZoneCount, err = options.Profile.ConfigInt(configZoneCount, 1) + if err != nil { + return s, fmt.Errorf("invalid value for %q: %w", configZoneCount, err) + } + if zones := s.ZoneCount; zones < 0 || zones > 3 { + return s, fmt.Errorf("%s should have a value between 1 and 3, found %d", configZoneCount, zones) + } + + s.MemorySize, err = options.Profile.ConfigInt(configMemorySize, 4096) + if err != nil { + return s, fmt.Errorf("invalid value for %s: %w", configMemorySize, err) + } + if size := s.MemorySize; size <= 0 { + return s, fmt.Errorf("%s should have a value greater than 0, found %d", configMemorySize, size) + } + + return s, nil +} + +func (cp *cloudProvider) getDeploymentRequest(settings deploymentSettings) (*models.DeploymentCreateRequest, error) { template, err := deptemplateapi.Get(deptemplateapi.GetParams{ API: cp.api, - TemplateID: templateID, - Region: region, - StackVersion: options.StackVersion, + TemplateID: settings.TemplateID, + Region: settings.Region, + StackVersion: settings.StackVersion, }) if err != nil { - return nil, fmt.Errorf("failed to get deployment template %q: %w", templateID, err) + return nil, fmt.Errorf("failed to get deployment template %q: %w", settings.TemplateID, err) } payload := template.DeploymentTemplate @@ -156,13 +201,13 @@ func (cp *cloudProvider) getDeploymentRequest(options Options, region, templateI if plan.DeploymentTemplate == nil { plan.DeploymentTemplate = &models.DeploymentTemplateReference{} } - plan.DeploymentTemplate.ID = &templateID + plan.DeploymentTemplate.ID = &settings.TemplateID for _, tier := range plan.ClusterTopology { if tier.ID == "hot_content" { - memory := int32(4096) + memory := int32(settings.MemorySize) tier.Size.Value = &memory - tier.ZoneCount = 1 + tier.ZoneCount = int32(settings.ZoneCount) } } } @@ -273,17 +318,17 @@ func (cp *cloudProvider) createDeployment(name string, options Options, payload } func (cp *cloudProvider) createGeoIPExtension() (*models.Extension, error) { + // TODO: Add support for stack.geoip_dir. bundle, err := zipGeoIPBundle() if err != nil { return nil, fmt.Errorf("failed to create GeoIP bundle: %w", err) } - // TODO: Parameterize extension Name. - extensionName := "geoip-extension" + extensionName := fmt.Sprintf("elastic-package-%s-geoip", cp.profile.ProfileName) extension, err := extensionapi.Create(extensionapi.CreateParams{ API: cp.api, Name: extensionName, - Description: "GeoIP extension for elastic-package tests", + Description: fmt.Sprintf("GeoIP extension for elastic-package tests (%s profile)", cp.profile.ProfileName), Type: "bundle", Version: "*", }) @@ -503,9 +548,13 @@ func getPackageVersion(registryURL, packageName, stackVersion string) (string, e return packages[0].Version, nil } +func (cp *cloudProvider) composeProjectName() string { + return DockerComposeProjectName + "-" + cp.profile.ProfileName +} + func (cp *cloudProvider) localAgentComposeProject() (*compose.Project, error) { composeFile := cp.profile.Path(profileStackPath, CloudComposeFile) - return compose.NewProject(DockerComposeProjectName, composeFile) + return compose.NewProject(cp.composeProjectName(), composeFile) } func (cp *cloudProvider) startLocalAgent(options Options, config Config) error { @@ -784,7 +833,7 @@ func (*cloudProvider) deploymentStatus(deployment *models.DeploymentGetResponse) func (cp *cloudProvider) localAgentStatus() ([]ServiceStatus, error) { var services []ServiceStatus // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles - containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName) + containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, cp.composeProjectName()) if err != nil { return nil, err } From 456559214914c534de793b809508467287b6b0fd Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 25 May 2023 19:10:01 +0200 Subject: [PATCH 13/16] Update deployments on stack up --- internal/stack/cloud.go | 154 +++++++++++++++++++++++++++------------- 1 file changed, 103 insertions(+), 51 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 86417b7b3..3d4434d43 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/cloud-sdk-go/pkg/api" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deputil" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/extensionapi" "github.com/elastic/cloud-sdk-go/pkg/auth" "github.com/elastic/cloud-sdk-go/pkg/models" @@ -38,11 +39,11 @@ import ( ) const ( - paramCloudDeploymentID = "cloud_deployment_id" paramCloudClusterRefID = "cloud_ref_id" paramCloudDeploymentAlias = "cloud_deployment_alias" - paramCloudGeoIPExtensionID = "cloud_geoip_extension_id" + paramCloudDeploymentID = "cloud_deployment_id" paramCloudFleetURL = "cloud_fleet_url" + paramCloudGeoIPExtensionID = "cloud_geoip_extension_id" ) var ( @@ -76,22 +77,9 @@ func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { func (cp *cloudProvider) BootUp(options Options) error { logger.Warn("Elastic Cloud provider is in technical preview") - _, err := cp.currentDeployment() - switch err { - case nil: - // Do nothing, deployment already exists. - // TODO: Migrate configuration if changed. - config, err := LoadConfig(cp.profile) - if err != nil { - return err - } - printUserConfig(options.Printer, config) - return nil - case errDeploymentNotExist: - // Deployment doesn't exist, let's continue. - break - default: - return err + config, err := LoadConfig(cp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) } settings, err := getDeploymentSettings(options) @@ -99,28 +87,41 @@ func (cp *cloudProvider) BootUp(options Options) error { return err } - logger.Debugf("Getting deployment template %q", settings.TemplateID) - payload, err := cp.getDeploymentRequest(settings) - if err != nil { - return fmt.Errorf("failed to get deployment template: %w", err) - } + deployment, err := cp.currentDeployment(config) + switch err { + default: + return err + case errDeploymentNotExist: + logger.Debugf("Getting deployment template %q", settings.TemplateID) + payload, err := cp.getDeploymentRequest(settings) + if err != nil { + return fmt.Errorf("failed to get deployment template: %w", err) + } - logger.Infof("Creating deployment %q", settings.Name) - config, err := cp.createDeployment(settings.Name, options, payload) - if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) - } + logger.Infof("Creating deployment %q", settings.Name) + config, err = cp.createDeployment(settings.Name, options, payload) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } - logger.Infof("Replacing GeoIP databases") - err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) - if err != nil { - return fmt.Errorf("failed to replace GeoIP databases: %w", err) - } + logger.Infof("Creating agent policy") + err = cp.createAgentPolicy(config, options.StackVersion) + if err != nil { + return fmt.Errorf("failed to create agent policy: %w", err) + } - logger.Infof("Creating agent policy") - err = cp.createAgentPolicy(config, options.StackVersion) - if err != nil { - return fmt.Errorf("failed to create agent policy: %w", err) + logger.Infof("Replacing GeoIP databases") + err = cp.replaceGeoIPDatabases(config, options, settings.TemplateID, settings.Region, payload.Resources.Elasticsearch[0].Plan.ClusterTopology) + if err != nil { + return fmt.Errorf("failed to replace GeoIP databases: %w", err) + } + case nil: + printUserConfig(options.Printer, config) + logger.Infof("Updating deployment") + err = cp.updateDeployment(deployment, settings) + if err != nil { + return fmt.Errorf("failed to update deployment: %w", err) + } } logger.Infof("Starting local agent") @@ -144,6 +145,7 @@ type deploymentSettings struct { } func getDeploymentSettings(options Options) (deploymentSettings, error) { + // TODO: Implement a config unpacker in options.Profile. const ( configMemorySize = "stack.cloud.memory_size" configRegion = "stack.cloud.region" @@ -196,14 +198,13 @@ func (cp *cloudProvider) getDeploymentRequest(settings deploymentSettings) (*mod payload.Resources.EnterpriseSearch = nil // Initialize the plan with the id of the template, otherwise the create request fails. - if es := payload.Resources.Elasticsearch; len(es) > 0 { - plan := es[0].Plan - if plan.DeploymentTemplate == nil { - plan.DeploymentTemplate = &models.DeploymentTemplateReference{} + for _, es := range payload.Resources.Elasticsearch { + if es.Plan.DeploymentTemplate == nil { + es.Plan.DeploymentTemplate = &models.DeploymentTemplateReference{} } - plan.DeploymentTemplate.ID = &settings.TemplateID + es.Plan.DeploymentTemplate.ID = &settings.TemplateID - for _, tier := range plan.ClusterTopology { + for _, tier := range es.Plan.ClusterTopology { if tier.ID == "hot_content" { memory := int32(settings.MemorySize) tier.Size.Value = &memory @@ -512,6 +513,48 @@ func (cp *cloudProvider) createAgentPolicy(config Config, stackVersion string) e return nil } +func (cp *cloudProvider) updateDeployment(current *models.DeploymentGetResponse, settings deploymentSettings) error { + updateRequest := deploymentapi.NewUpdateRequest(current) + + // If any, we only want to update Elasticsearch. + updateRequest.Resources.Apm = nil + updateRequest.Resources.Kibana = nil + updateRequest.Resources.IntegrationsServer = nil + + // Try to update only what is configurable. + for _, es := range updateRequest.Resources.Elasticsearch { + for _, tier := range es.Plan.ClusterTopology { + if tier.ID == "hot_content" { + memory := int32(settings.MemorySize) + tier.Size.Value = &memory + tier.ZoneCount = int32(settings.ZoneCount) + } + } + } + _, err := deploymentapi.Update(deploymentapi.UpdateParams{ + API: cp.api, + DeploymentID: *current.ID, + Request: updateRequest, + }) + if err != nil { + return fmt.Errorf("update request failed: %w", err) + } + + err = planutil.TrackChange(planutil.TrackChangeParams{ + TrackChangeParams: plan.TrackChangeParams{ + API: cp.api, + DeploymentID: *current.ID, + }, + Writer: &cloudTrackWriter{}, + Format: "text", + }) + if err != nil { + return fmt.Errorf("failed to track cluster creation: %w", err) + } + + return nil +} + func getPackageVersion(registryURL, packageName, stackVersion string) (string, error) { searchURL, err := url.JoinPath(registryURL, "search") if err != nil { @@ -668,12 +711,17 @@ func (cp *cloudProvider) replaceGeoIPDatabases(config Config, options Options, t } func (cp *cloudProvider) TearDown(options Options) error { - err := cp.destroyLocalAgent() + config, err := LoadConfig(cp.profile) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + err = cp.destroyLocalAgent() if err != nil { return fmt.Errorf("failed to destroy local agent: %w", err) } - deployment, err := cp.currentDeployment() + deployment, err := cp.currentDeployment(config) if err != nil { return fmt.Errorf("failed to find current deployment: %w", err) } @@ -751,7 +799,12 @@ func (*cloudProvider) Dump(options DumpOptions) (string, error) { } func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { - deployment, err := cp.currentDeployment() + config, err := LoadConfig(cp.profile) + if err != nil { + return nil, fmt.Errorf("failed to load configuration: %w", err) + } + + deployment, err := cp.currentDeployment(config) if err != nil { return nil, err } @@ -862,11 +915,7 @@ func (cp *cloudProvider) localAgentStatus() ([]ServiceStatus, error) { return services, nil } -func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { - config, err := LoadConfig(cp.profile) - if err != nil { - return nil, err - } +func (cp *cloudProvider) currentDeployment(config Config) (*models.DeploymentGetResponse, error) { deploymentID, found := config.Parameters[paramCloudDeploymentID] if !found { return nil, errDeploymentNotExist @@ -874,6 +923,9 @@ func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, err deployment, err := deploymentapi.Get(deploymentapi.GetParams{ API: cp.api, DeploymentID: deploymentID, + QueryParams: deputil.QueryParams{ + ShowPlans: true, + }, }) if err != nil { return nil, fmt.Errorf("couldn't check deployment health: %w", err) From 4864a3d0a7d09836c7410fe294513619beed514e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 25 May 2023 20:32:35 +0200 Subject: [PATCH 14/16] Use diferent compose project names per project --- cmd/service.go | 9 +++++++++ cmd/testrunner.go | 8 ++++++++ internal/kind/kind.go | 5 +++-- internal/service/boot.go | 4 ++++ internal/stack/boot.go | 16 +++++++++++----- internal/stack/cloud.go | 2 +- internal/stack/compose.go | 12 ++++++------ internal/stack/dump.go | 2 +- internal/stack/logs.go | 6 +++--- internal/stack/network.go | 9 +++++---- internal/stack/providers.go | 2 +- internal/stack/status.go | 4 ++-- internal/testrunner/runners/system/runner.go | 2 ++ .../runners/system/servicedeployer/compose.go | 9 ++++++--- .../system/servicedeployer/custom_agent.go | 9 ++++++--- .../runners/system/servicedeployer/factory.go | 10 +++++++--- .../runners/system/servicedeployer/kubernetes.go | 7 +++++-- internal/testrunner/testrunner.go | 2 ++ 18 files changed, 82 insertions(+), 36 deletions(-) diff --git a/cmd/service.go b/cmd/service.go index fb5206b6a..3501ac76c 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -5,12 +5,14 @@ package cmd import ( + "fmt" "path/filepath" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/service" ) @@ -34,6 +36,7 @@ func setupServiceCommand() *cobraext.Command { Long: serviceLongDescription, } cmd.AddCommand(upCommand) + cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -57,8 +60,14 @@ func upCommandAction(cmd *cobra.Command, args []string) error { variantFlag, _ := cmd.Flags().GetString(cobraext.VariantFlagName) + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + _, serviceName := filepath.Split(packageRoot) err = service.BootUp(service.Options{ + Profile: profile, ServiceName: serviceName, PackageRootPath: packageRoot, DataStreamRootPath: dataStreamPath, diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 2be5235f8..2e98fdb8f 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/signal" "github.com/elastic/elastic-package/internal/testrunner" @@ -71,6 +72,7 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().BoolP(cobraext.TestCoverageFlagName, "", false, cobraext.TestCoverageFlagDescription) cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) + cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) for testType, runner := range testrunner.TestRunners() { action := testTypeCommandActionFactory(runner) @@ -216,9 +218,15 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return err } + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + var results []testrunner.TestResult for _, folder := range testFolders { r, err := testrunner.Run(testType, testrunner.TestOptions{ + Profile: profile, TestFolder: folder, PackageRootPath: packageRootPath, GenerateTestResult: generateTestResult, diff --git a/internal/kind/kind.go b/internal/kind/kind.go index 578970203..6f203c038 100644 --- a/internal/kind/kind.go +++ b/internal/kind/kind.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" ) @@ -35,13 +36,13 @@ func VerifyContext() error { } // ConnectToElasticStackNetwork function ensures that the control plane node is connected to the Elastic stack network. -func ConnectToElasticStackNetwork() error { +func ConnectToElasticStackNetwork(profile *profile.Profile) error { containerID, err := controlPlaneContainerID() if err != nil { return errors.Wrap(err, "can't find kind-control plane node") } - stackNetwork := stack.Network() + stackNetwork := stack.Network(profile) logger.Debugf("check network connectivity between service container %s (ID: %s) and the stack network %s", ControlPlaneContainerName, containerID, stackNetwork) networkDescriptions, err := docker.InspectNetwork(stackNetwork) diff --git a/internal/service/boot.go b/internal/service/boot.go index 561541a2d..d7121c48a 100644 --- a/internal/service/boot.go +++ b/internal/service/boot.go @@ -11,6 +11,7 @@ import ( "syscall" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" "github.com/pkg/errors" @@ -21,6 +22,8 @@ import ( // Options define the details of the service which should be booted up. type Options struct { + Profile *profile.Profile + ServiceName string PackageRootPath string DataStreamRootPath string @@ -32,6 +35,7 @@ type Options struct { func BootUp(options Options) error { logger.Debugf("Create new instance of the service deployer") serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{ + Profile: options.Profile, PackageRootPath: options.DataStreamRootPath, DataStreamRootPath: options.DataStreamRootPath, Variant: options.Variant, diff --git a/internal/stack/boot.go b/internal/stack/boot.go index f499a8331..92b5048cb 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -15,11 +15,17 @@ import ( "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/profile" ) -// DockerComposeProjectName is the name of the Docker Compose project used to boot up +// baseComposeProjectName is the base name of the Docker Compose project used to boot up // Elastic Stack containers. -const DockerComposeProjectName = "elastic-package-stack" +const baseComposeProjectName = "elastic-package-stack" + +// DockerComposeProjectName returns the docker compose project name for a given profile. +func DockerComposeProjectName(profile *profile.Profile) string { + return baseComposeProjectName + "-" + profile.ProfileName +} // BootUp function boots up the Elastic stack. func BootUp(options Options) error { @@ -83,7 +89,7 @@ func BootUp(options Options) error { // to fail too. // As a workaround, try to give another chance to docker-compose if only // elastic-agent failed. - if onlyElasticAgentFailed() { + if onlyElasticAgentFailed(options) { fmt.Println("Elastic Agent failed to start, trying again.") err = dockerComposeUp(options) } @@ -98,8 +104,8 @@ func BootUp(options Options) error { return nil } -func onlyElasticAgentFailed() bool { - status, err := Status() +func onlyElasticAgentFailed(options Options) bool { + status, err := Status(options) if err != nil { fmt.Printf("Failed to check status of the stack after failure: %v\n", err) return false diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 3d4434d43..f7d93fb47 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -592,7 +592,7 @@ func getPackageVersion(registryURL, packageName, stackVersion string) (string, e } func (cp *cloudProvider) composeProjectName() string { - return DockerComposeProjectName + "-" + cp.profile.ProfileName + return DockerComposeProjectName(cp.profile) } func (cp *cloudProvider) localAgentComposeProject() (*compose.Project, error) { diff --git a/internal/stack/compose.go b/internal/stack/compose.go index a1e0bb00e..b40b31b1c 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -55,7 +55,7 @@ func (eb *envBuilder) build() []string { } func dockerComposeBuild(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -81,7 +81,7 @@ func dockerComposeBuild(options Options) error { } func dockerComposePull(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -107,7 +107,7 @@ func dockerComposePull(options Options) error { } func dockerComposeUp(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -139,7 +139,7 @@ func dockerComposeUp(options Options) error { } func dockerComposeDown(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -185,10 +185,10 @@ func withIsReadyServices(services []string) []string { return allServices } -func dockerComposeStatus() ([]ServiceStatus, error) { +func dockerComposeStatus(options Options) ([]ServiceStatus, error) { var services []ServiceStatus // query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles - containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName) + containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName(options.Profile)) if err != nil { return nil, err } diff --git a/internal/stack/dump.go b/internal/stack/dump.go index abe088a80..60671f8c6 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -62,7 +62,7 @@ func dumpStackLogs(options DumpOptions) error { writeLogFiles(logsPath, serviceName, content) } - err = copyDockerInternalLogs(serviceName, logsPath) + err = copyDockerInternalLogs(serviceName, logsPath, options.Profile) if err != nil { logger.Errorf("can't copy internal logs (service: %s): %v", serviceName, err) } diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 41f10c456..4e72966b7 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -23,7 +23,7 @@ func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, er snapshotFile := profile.Path(profileStackPath, SnapshotFile) - p, err := compose.NewProject(DockerComposeProjectName, snapshotFile) + p, err := compose.NewProject(DockerComposeProjectName(profile), snapshotFile) if err != nil { return nil, errors.Wrap(err, "could not create docker compose project") } @@ -44,14 +44,14 @@ func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, er return out, nil } -func copyDockerInternalLogs(serviceName, outputPath string) error { +func copyDockerInternalLogs(serviceName, outputPath string, profile *profile.Profile) error { switch serviceName { case elasticAgentService, fleetServerService: default: return nil // we need to pull internal logs only from Elastic-Agent and Fleets Server container } - p, err := compose.NewProject(DockerComposeProjectName) + p, err := compose.NewProject(DockerComposeProjectName(profile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } diff --git a/internal/stack/network.go b/internal/stack/network.go index 7e0877b00..7b45b9d3e 100644 --- a/internal/stack/network.go +++ b/internal/stack/network.go @@ -10,15 +10,16 @@ import ( "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/profile" ) // EnsureStackNetworkUp function verifies if stack network is up and running. -func EnsureStackNetworkUp() error { - _, err := docker.InspectNetwork(Network()) +func EnsureStackNetworkUp(profile *profile.Profile) error { + _, err := docker.InspectNetwork(Network(profile)) return errors.Wrap(err, "network not available") } // Network function returns the stack network name. -func Network() string { - return fmt.Sprintf("%s_default", DockerComposeProjectName) +func Network(profile *profile.Profile) string { + return fmt.Sprintf("%s_default", DockerComposeProjectName(profile)) } diff --git a/internal/stack/providers.go b/internal/stack/providers.go index 81d4090ae..8e58a5060 100644 --- a/internal/stack/providers.go +++ b/internal/stack/providers.go @@ -79,5 +79,5 @@ func (*composeProvider) Dump(options DumpOptions) (string, error) { } func (*composeProvider) Status(options Options) ([]ServiceStatus, error) { - return Status() + return Status(options) } diff --git a/internal/stack/status.go b/internal/stack/status.go index ecde8a1e0..1c4e9d5f3 100644 --- a/internal/stack/status.go +++ b/internal/stack/status.go @@ -12,8 +12,8 @@ import ( ) // Status shows the status for each service -func Status() ([]ServiceStatus, error) { - servicesStatus, err := dockerComposeStatus() +func Status(options Options) ([]ServiceStatus, error) { + servicesStatus, err := dockerComposeStatus(options) if err != nil { return nil, err } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ba019459c..71b24b2e9 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -156,6 +156,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { } devDeployPath, err := servicedeployer.FindDevDeployPath(servicedeployer.FactoryOptions{ + Profile: r.options.Profile, PackageRootPath: r.options.PackageRootPath, DataStreamRootPath: dataStreamPath, }) @@ -187,6 +188,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { func (r *runner) runTestPerVariant(result *testrunner.ResultComposer, locationManager *locations.LocationManager, cfgFile, dataStreamPath, variantName string) ([]testrunner.TestResult, error) { serviceOptions := servicedeployer.FactoryOptions{ + Profile: r.options.Profile, PackageRootPath: r.options.PackageRootPath, DataStreamRootPath: dataStreamPath, Variant: variantName, diff --git a/internal/testrunner/runners/system/servicedeployer/compose.go b/internal/testrunner/runners/system/servicedeployer/compose.go index b62eb7789..aa167a025 100644 --- a/internal/testrunner/runners/system/servicedeployer/compose.go +++ b/internal/testrunner/runners/system/servicedeployer/compose.go @@ -17,12 +17,14 @@ import ( "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" ) // DockerComposeServiceDeployer knows how to deploy a service defined via // a Docker Compose file. type DockerComposeServiceDeployer struct { + profile *profile.Profile ymlPaths []string variant ServiceVariant } @@ -37,8 +39,9 @@ type dockerComposeDeployedService struct { } // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. -func NewDockerComposeServiceDeployer(ymlPaths []string, sv ServiceVariant) (*DockerComposeServiceDeployer, error) { +func NewDockerComposeServiceDeployer(profile *profile.Profile, ymlPaths []string, sv ServiceVariant) (*DockerComposeServiceDeployer, error) { return &DockerComposeServiceDeployer{ + profile: profile, ymlPaths: ymlPaths, variant: sv, }, nil @@ -61,7 +64,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer } // Verify the Elastic stack network - err = stack.EnsureStackNetworkUp() + err = stack.EnsureStackNetworkUp(d.profile) if err != nil { return nil, errors.Wrap(err, "Elastic stack network is not ready") } @@ -101,7 +104,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer outCtxt.Hostname = p.ContainerName(serviceName) // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network()) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) if err != nil { return nil, errors.Wrapf(err, "can't attach service container to the stack network") } diff --git a/internal/testrunner/runners/system/servicedeployer/custom_agent.go b/internal/testrunner/runners/system/servicedeployer/custom_agent.go index 02165d0e0..e07245de6 100644 --- a/internal/testrunner/runners/system/servicedeployer/custom_agent.go +++ b/internal/testrunner/runners/system/servicedeployer/custom_agent.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" ) @@ -34,12 +35,14 @@ var dockerCustomAgentDockerfileContent []byte // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via // a Docker Compose file. type CustomAgentDeployer struct { + profile *profile.Profile dockerComposeFile string } // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. -func NewCustomAgentDeployer(dockerComposeFile string) (*CustomAgentDeployer, error) { +func NewCustomAgentDeployer(profile *profile.Profile, dockerComposeFile string) (*CustomAgentDeployer, error) { return &CustomAgentDeployer{ + profile: profile, dockerComposeFile: dockerComposeFile, }, nil } @@ -101,7 +104,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err } // Verify the Elastic stack network - err = stack.EnsureStackNetworkUp() + err = stack.EnsureStackNetworkUp(d.profile) if err != nil { return nil, errors.Wrap(err, "Elastic stack network is not ready") } @@ -124,7 +127,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err } // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network()) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) if err != nil { return nil, errors.Wrapf(err, "can't attach service container to the stack network") } diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index bbd4ae02d..70b4f7436 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -10,12 +10,16 @@ import ( "path/filepath" "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/profile" ) const devDeployDir = "_dev/deploy" // FactoryOptions defines options used to create an instance of a service deployer. type FactoryOptions struct { + Profile *profile.Profile + PackageRootPath string DataStreamRootPath string @@ -40,7 +44,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { switch serviceDeployerName { case "k8s": if _, err := os.Stat(serviceDeployerPath); err == nil { - return NewKubernetesServiceDeployer(serviceDeployerPath) + return NewKubernetesServiceDeployer(options.Profile, serviceDeployerPath) } case "docker": dockerComposeYMLPath := filepath.Join(serviceDeployerPath, "docker-compose.yml") @@ -49,14 +53,14 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if err != nil { return nil, errors.Wrap(err, "can't use service variant") } - return NewDockerComposeServiceDeployer([]string{dockerComposeYMLPath}, sv) + return NewDockerComposeServiceDeployer(options.Profile, []string{dockerComposeYMLPath}, sv) } case "agent": customAgentCfgYMLPath := filepath.Join(serviceDeployerPath, "custom-agent.yml") if _, err := os.Stat(customAgentCfgYMLPath); err != nil { return nil, errors.Wrap(err, "can't find expected file custom-agent.yml") } - return NewCustomAgentDeployer(customAgentCfgYMLPath) + return NewCustomAgentDeployer(options.Profile, customAgentCfgYMLPath) case "tf": if _, err := os.Stat(serviceDeployerPath); err == nil { diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index fcaf61e28..3616badb1 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -20,11 +20,13 @@ import ( "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" ) // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster. type KubernetesServiceDeployer struct { + profile *profile.Profile definitionsDir string } @@ -70,8 +72,9 @@ func (s *kubernetesDeployedService) SetContext(sc ServiceContext) error { var _ DeployedService = new(kubernetesDeployedService) // NewKubernetesServiceDeployer function creates a new instance of KubernetesServiceDeployer. -func NewKubernetesServiceDeployer(definitionsPath string) (*KubernetesServiceDeployer, error) { +func NewKubernetesServiceDeployer(profile *profile.Profile, definitionsPath string) (*KubernetesServiceDeployer, error) { return &KubernetesServiceDeployer{ + profile: profile, definitionsDir: definitionsPath, }, nil } @@ -84,7 +87,7 @@ func (ksd KubernetesServiceDeployer) SetUp(ctxt ServiceContext) (DeployedService return nil, errors.Wrap(err, "kind context verification failed") } - err = kind.ConnectToElasticStackNetwork() + err = kind.ConnectToElasticStackNetwork(ksd.profile) if err != nil { return nil, errors.Wrap(err, "can't connect control plane to Elastic stack network") } diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index 4c2a4e912..c3756daa5 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/profile" ) // TestType represents the various supported test types @@ -22,6 +23,7 @@ type TestType string // TestOptions contains test runner options. type TestOptions struct { + Profile *profile.Profile TestFolder TestFolder PackageRootPath string GenerateTestResult bool From 42b95a7d8ba48cd2bb9a20b44d1be01fbbd0a244 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 25 May 2023 21:19:15 +0200 Subject: [PATCH 15/16] Install zipped package --- internal/testrunner/runners/system/runner.go | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 71b24b2e9..e02ba8de5 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -13,8 +13,10 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/pkg/errors" + "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -338,6 +340,14 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(errors.Wrap(err, "can't create Kibana client")) } + // Install the package before creating the policy, so we control exactly what is being + // installed. + logger.Debug("Installing package...") + err = installPackage(kib, pkgManifest, r.options.PackageRootPath) + if err != nil { + return result.WithError(errors.Wrap(err, "unable to install package")) + } + // Configure package (single data stream) via Ingest Manager APIs. logger.Debug("creating test policy...") testTime := time.Now().Format("20060102T15:04:05Z") @@ -529,6 +539,42 @@ func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceCont return agents, nil } +func installPackage(client *kibana.Client, manifest *packages.PackageManifest, rootPath string) error { + kibVersion, err := client.Version() + if err != nil { + return errors.Wrap(err, "could not obtain kibana version") + } + version, err := semver.NewVersion(kibVersion.Number) + if err != nil { + return errors.Wrap(err, "could not parse kibana semantic version") + } + + if version.LessThan(semver.MustParse("8.7.0")) { + _, err := client.InstallPackage(manifest.Name, manifest.Version) + if err != nil { + return errors.Wrap(err, "could not install package through package registry") + } + return nil + } + + target, err := builder.BuildPackage(builder.BuildOptions{ + PackageRoot: rootPath, + CreateZip: true, + SignPackage: false, + SkipValidation: true, + }) + if err != nil { + return errors.Wrap(err, "failed to build zip package for installation") + } + + _, err = client.InstallZipPackage(target) + if err != nil { + return errors.Wrapf(err, "failed to install zip package \"%s\"", target) + } + + return nil +} + func createPackageDatastream( kibanaPolicy kibana.Policy, pkg packages.PackageManifest, From ed9e89de2d9f87ae4f6e6afe9d8e474474123077 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 26 May 2023 18:33:48 +0200 Subject: [PATCH 16/16] Remove TODO --- internal/stack/cloud.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index f7d93fb47..023fdd75c 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -400,7 +400,6 @@ const cloudKibanaAgentPolicy = `{ ] }` -// TODO: Avoid hard-coding the package version here. const cloudKibanaPackagePolicy = `{ "name": "system-1", "policy_id": "elastic-agent-managed-ep",