From d0ad188da5cf1fe95f7921a255e96788b43c8bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20LAURENT?= Date: Tue, 30 Mar 2021 10:35:25 +0200 Subject: [PATCH 01/24] Some ES Store improvements - Allow shards and replicas configuration at index creation - Fix a bug where the Store is still querying ES after the context has been cancelled - Some logging improvements when ES is returning error messages --- doc/configuration.rst | 4 + storage/internal/elastic/config.go | 66 ++++++++++- storage/internal/elastic/es_utils.go | 115 +++++++++++++++++-- storage/internal/elastic/queries.go | 162 +++++++++++++-------------- storage/internal/elastic/store.go | 14 ++- 5 files changed, 259 insertions(+), 102 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 774e5b8b9..3774c9855 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1488,6 +1488,10 @@ This store ables you to store ``Log`` s and ``Event`` s in elasticsearch. +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ | ``trace_events`` | to trace events & logs when sent (for debug only) | bool | no | false | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ +| ``initial_shards`` | number of shards used to initilize indices | int64 | no | | ++-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ +| ``initial_replicas`` | number of replicas used to initializ indices | int64 | no | | ++-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ Vault configuration diff --git a/storage/internal/elastic/config.go b/storage/internal/elastic/config.go index a7c756cf1..f1bae6540 100755 --- a/storage/internal/elastic/config.go +++ b/storage/internal/elastic/config.go @@ -15,12 +15,13 @@ package elastic import ( + "reflect" + "time" + "github.com/pkg/errors" "github.com/spf13/cast" "github.com/ystia/yorc/v4/config" "github.com/ystia/yorc/v4/log" - "reflect" - "time" ) var elasticStoreConfType = reflect.TypeOf(elasticStoreConf{}) @@ -54,6 +55,10 @@ type elasticStoreConf struct { traceRequests bool `json:"trace_requests" default:"false"` // Set to true if you want to trace events & logs when sent (for debug only) traceEvents bool `json:"trace_events" default:"false"` + // Inital shards at index creation + InitialShards int `json:"initial_shards" default:"-1"` + // Initial replicas at index creation + InitialReplicas int `json:"initial_replicas" default:"-1"` } // Get the tag for this field (for internal usage only: fatal if not found !). @@ -108,31 +113,86 @@ func getElasticStoreConfig(yorcConfig config.Configuration, storeConfig config.S } // Define store optional / default configuration t, e = getElasticStorageConfigPropertyTag("caCertPath", "json") + if e != nil { + return + } + if storeProperties.IsSet(t) { cfg.caCertPath = storeProperties.GetString(t) } t, e = getElasticStorageConfigPropertyTag("certPath", "json") + if e != nil { + return + } + if storeProperties.IsSet(t) { cfg.certPath = storeProperties.GetString(t) } t, e = getElasticStorageConfigPropertyTag("keyPath", "json") + if e != nil { + return + } + if storeProperties.IsSet(t) { cfg.keyPath = storeProperties.GetString(t) } cfg.esForceRefresh, e = getBoolFromSettingsOrDefaults("esForceRefresh", storeProperties) + if e != nil { + return + } + t, e = getElasticStorageConfigPropertyTag("indicePrefix", "json") + if e != nil { + return + } + if storeProperties.IsSet(t) { cfg.indicePrefix = storeProperties.GetString(t) } else { cfg.indicePrefix, e = getElasticStorageConfigPropertyTag("indicePrefix", "default") + if e != nil { + return + } } cfg.esQueryPeriod, e = getDurationFromSettingsOrDefaults("esQueryPeriod", storeProperties) + if e != nil { + return + } + cfg.esRefreshWaitTimeout, e = getDurationFromSettingsOrDefaults("esRefreshWaitTimeout", storeProperties) + if e != nil { + return + } + cfg.maxBulkSize, e = getIntFromSettingsOrDefaults("maxBulkSize", storeProperties) + if e != nil { + return + } cfg.maxBulkCount, e = getIntFromSettingsOrDefaults("maxBulkCount", storeProperties) + if e != nil { + return + } + cfg.traceRequests, e = getBoolFromSettingsOrDefaults("traceRequests", storeProperties) + if e != nil { + return + } + cfg.traceEvents, e = getBoolFromSettingsOrDefaults("traceEvents", storeProperties) - // If any error have been encountered, it will be returned + if e != nil { + return + } + + cfg.InitialShards, e = getIntFromSettingsOrDefaults("InitialShards", storeProperties) + if e != nil { + return + } + + cfg.InitialReplicas, e = getIntFromSettingsOrDefaults("InitialReplicas", storeProperties) + if e != nil { + return + } + return } diff --git a/storage/internal/elastic/es_utils.go b/storage/internal/elastic/es_utils.go index eccea77de..c60be83a2 100755 --- a/storage/internal/elastic/es_utils.go +++ b/storage/internal/elastic/es_utils.go @@ -20,16 +20,17 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" - elasticsearch6 "github.com/elastic/go-elasticsearch/v6" - "github.com/elastic/go-elasticsearch/v6/esapi" - "github.com/pkg/errors" - "github.com/ystia/yorc/v4/log" - "github.com/ystia/yorc/v4/storage/store" "io" "io/ioutil" "net/http" "strings" "time" + + elasticsearch6 "github.com/elastic/go-elasticsearch/v6" + "github.com/elastic/go-elasticsearch/v6/esapi" + "github.com/pkg/errors" + "github.com/ystia/yorc/v4/log" + "github.com/ystia/yorc/v4/storage/store" ) var pfalse = false @@ -37,8 +38,7 @@ var pfalse = false func prepareEsClient(elasticStoreConfig elasticStoreConf) (*elasticsearch6.Client, error) { log.Printf("Elastic storage will run using this configuration: %+v", elasticStoreConfig) - var esConfig elasticsearch6.Config - esConfig = elasticsearch6.Config{Addresses: elasticStoreConfig.esUrls} + esConfig := elasticsearch6.Config{Addresses: elasticStoreConfig.esUrls} if len(elasticStoreConfig.caCertPath) > 0 { log.Printf("Reading CACert file from %s", elasticStoreConfig.caCertPath) @@ -71,6 +71,9 @@ func prepareEsClient(elasticStoreConfig elasticStoreConf) (*elasticsearch6.Clien // In debug mode or when traceRequests option is activated, we add a custom logger that print requests & responses log.Printf("\t- Tracing ES requests & response can be expensive and verbose !") esConfig.Logger = &debugLogger{} + } else { + // otherwise log only failure are logger + esConfig.Logger = &defaultLogger{} } log.Printf("\t- Index prefix will be %s", elasticStoreConfig.indicePrefix+elasticStoreConfig.clusterID+"_") @@ -124,7 +127,7 @@ func initStorageIndex(c *elasticsearch6.Client, elasticStoreConfig elasticStoreC } else if res.StatusCode == 404 { log.Printf("Indice %s was not found, let's create it !", indexName) - requestBodyData := buildInitStorageIndexQuery() + requestBodyData := buildInitStorageIndexQuery(elasticStoreConfig) // indice doest not exist, let's create it req := esapi.IndicesCreateRequest{ @@ -158,7 +161,7 @@ func refreshIndex(c *elasticsearch6.Client, indexName string) { } // Query ES for events or logs specifying the expected results 'size' and the sort 'order'. -func doQueryEs(c *elasticsearch6.Client, conf elasticStoreConf, +func doQueryEs(ctx context.Context, c *elasticsearch6.Client, conf elasticStoreConf, index string, query string, waitIndex uint64, @@ -170,7 +173,7 @@ func doQueryEs(c *elasticsearch6.Client, conf elasticStoreConf, lastIndex = waitIndex res, e := c.Search( - c.Search.WithContext(context.Background()), + c.Search.WithContext(ctx), c.Search.WithIndex(index), c.Search.WithSize(size), c.Search.WithBody(strings.NewReader(query)), @@ -197,6 +200,8 @@ func doQueryEs(c *elasticsearch6.Client, conf elasticStoreConf, return } + logShardsInfos(r) + hits = int(r["hits"].(map[string]interface{})["total"].(float64)) duration := int(r["took"].(float64)) log.Debugf("Search ES request on index %s took %dms, hits=%d, response code was %d (%s)", index, duration, hits, res.StatusCode, res.Status()) @@ -307,6 +312,20 @@ func closeResponseBody(requestDescription string, res *esapi.Response) { } } +// Log shards stats +func logShardsInfos(r map[string]interface{}) { + si := r["_shards"].(map[string]interface{}) + + duration := int(r["took"].(float64)) + + tt := int(si["total"].(float64)) + ts := int(si["successful"].(float64)) + + if ts < tt { + log.Printf("[Warn] ES Uncomplete response: %d/%d shards (%dms)", ts, tt, duration) + } +} + type debugLogger struct{} // RequestBodyEnabled makes the client pass request body to logger @@ -354,3 +373,79 @@ func (l *debugLogger) LogRoundTrip( return nil } + +type defaultLogger struct{} + +// RequestBodyEnabled makes the client pass request body to logger +func (l *defaultLogger) RequestBodyEnabled() bool { return true } + +// ResponseBodyEnabled makes the client pass response body to logger +func (l *defaultLogger) ResponseBodyEnabled() bool { return true } + +// LogRoundTrip will use log to debug ES request and response (when debug is activated) +func (l *defaultLogger) LogRoundTrip( + req *http.Request, + res *http.Response, + err error, + start time.Time, + dur time.Duration, +) error { + + var level string + var errType string + var errReason string + + switch { + case err != nil: + level = "Exception" + case res != nil && res.StatusCode > 0 && res.StatusCode < 300: + return nil + case res != nil && res.StatusCode > 299 && res.StatusCode < 500: + level = "Warn" + case res != nil && res.StatusCode > 499: + errType, errReason = extractEsError(res) + level = "Error" + default: + level = "Unknown" + } + + if errType == "" && errReason == "" { + log.Printf("ES Request [%s][%v][%s][%s][%d][%v]", + level, start, req.Method, req.URL.String(), res.StatusCode, dur) + } else { + log.Printf("ES Request [%s][%v][%s][%s][%d][%v][%s][%s]", + level, start, req.Method, req.URL.String(), res.StatusCode, dur, errType, errReason) + } + return nil +} + +func extractEsError(response *http.Response) (errType, errReason string) { + var rb map[string]interface{} + var rv interface{} + var ok bool + + errType = "N/A" + errReason = "N/A" + + if err := json.NewDecoder(response.Body).Decode(&rb); err != nil { + return + } + + if rv, ok = rb["error"]; !ok { + return + } + + if rb, ok = rv.(map[string]interface{}); !ok { + return + } + + if rv, ok = rb["type"]; ok { + errType, _ = rv.(string) + } + + if rv, ok = rb["reason"]; ok { + errReason, _ = rv.(string) + } + + return +} diff --git a/storage/internal/elastic/queries.go b/storage/internal/elastic/queries.go index c818a041c..7a23f84ec 100755 --- a/storage/internal/elastic/queries.go +++ b/storage/internal/elastic/queries.go @@ -14,67 +14,48 @@ package elastic -import "strconv" +import ( + "bytes" + "strconv" + "text/template" +) -// Return the query that is used to create indexes for event and log storage. -// We only index the needed fields to optimize ES indexing performance (no dynamic mapping). -func buildInitStorageIndexQuery() (query string) { - query = ` +// Index creation request +const initStorageTemplateText = ` { "settings": { - "refresh_interval": "1s" + {{ if ne .InitialReplicas -1}}"number_of_replicas": {{ .InitialReplicas}},{{end}} + {{ if ne .InitialShards -1 }}"number_of_shards": {{ .InitialShards}},{{end}} + "refresh_interval": "1s" }, "mappings": { "_doc": { "_all": {"enabled": false}, "dynamic": "false", "properties": { - "deploymentId": { - "type": "keyword", - "index": true - }, - "iid": { - "type": "long", - "index": true - }, - "iidStr": { - "type": "keyword", - "index": false - } + "deploymentId": { "type": "keyword", "index": true }, + "iid": { "type": "long", "index": true }, + "iidStr": { "type": "keyword","index": false } } } } }` - return -} -// This ES aggregation query is built using clusterId and eventually deploymentId. -func buildLastModifiedIndexQuery(deploymentID string) (query string) { - if len(deploymentID) == 0 { - query = ` -{ - "aggs" : { - "max_iid" : { - "filter" : { - "match_all": {} - }, - "aggs" : { - "last_index" : { "max" : { "field" : "iid" } } - } - } - } -}` - } else { - query = ` +// Get last Modified index +const lastModifiedIndexTemplateText = ` { "aggs" : { "max_iid" : { "filter" : { +{{if .}} "bool": { "must": [ - { "term": { "deploymentId": "` + deploymentID + `" } } + { "term": { "deploymentId": "{{ . }}" } } ] } +{{else}} + "match_all": {} +{{end}} }, "aggs" : { "last_index" : { "max" : { "field" : "iid" } } @@ -82,57 +63,68 @@ func buildLastModifiedIndexQuery(deploymentID string) (query string) { } } }` - } - return + +// Range Query +const rangeQueryTemplateText = `{ "range":{ "iid":{ "gt": "{{ conv .WaitIndex }}"{{if gt .MaxIndex 0}},"lte": "{{ conv .MaxIndex }}"{{end}}}}}` + +const listQueryTemplateText = ` +{ + "query":{{if .DeploymentID}}{ + "bool":{ + "must": [ + { + "term":{ + "deploymentId": "{{ .DeploymentID }}" + } + }, + {{template "rangeQuery" .}} + ] + } + }{{else}}{{template "rangeQuery" .}}{{end}} } +` -func getRangeQuery(waitIndex uint64, maxIndex uint64) (rangeQuery string) { - if maxIndex > 0 { - rangeQuery = ` - { - "range":{ - "iid":{ - "gt": "` + strconv.FormatUint(waitIndex, 10) + `", - "lte": "` + strconv.FormatUint(maxIndex, 10) + `" - } - } - }` - } else { - rangeQuery = ` - { - "range":{ - "iid":{ - "gt": "` + strconv.FormatUint(waitIndex, 10) + `" - } - } - }` - } - return +var templates *template.Template + +func init() { + funcMap := template.FuncMap{"conv": func(value uint64) string { return strconv.FormatUint(value, 10) }} + + templates = template.Must(template.New("initStorage").Parse(initStorageTemplateText)) + templates = template.Must(templates.New("lastModifiedIndex").Parse(lastModifiedIndexTemplateText)) + + templates = template.Must(templates.New("rangeQuery").Funcs(funcMap).Parse(rangeQueryTemplateText)) + templates = template.Must(templates.New("listQuery").Parse(listQueryTemplateText)) +} + +// Return the query that is used to create indexes for event and log storage. +// We only index the needed fields to optimize ES indexing performance (no dynamic mapping). +func buildInitStorageIndexQuery(elasticStoreConfig elasticStoreConf) string { + var buffer bytes.Buffer + templates.ExecuteTemplate(&buffer, "initStorage", elasticStoreConfig) + return buffer.String() +} + +// This ES aggregation query is built using clusterId and eventually deploymentId. +func buildLastModifiedIndexQuery(deploymentID string) (query string) { + var buffer bytes.Buffer + templates.ExecuteTemplate(&buffer, "lastModifiedIndex", deploymentID) + return buffer.String() } // This ES range query is built using 'waitIndex' and eventually 'maxIndex' and filtered using 'clusterId' and eventually 'deploymentId'. func getListQuery(deploymentID string, waitIndex uint64, maxIndex uint64) (query string) { - rangeQuery := getRangeQuery(waitIndex, maxIndex) - if len(deploymentID) == 0 { - query = ` -{ - "query":` + rangeQuery + ` -}` - } else { - query = ` -{ - "query":{ - "bool":{ - "must":[ - { - "term":{ - "deploymentId":"` + deploymentID + `" - } - },` + rangeQuery + ` - ] - } - } -}` + var buffer bytes.Buffer + + data := struct { + WaitIndex uint64 + MaxIndex uint64 + DeploymentID string + }{ + WaitIndex: waitIndex, + MaxIndex: maxIndex, + DeploymentID: deploymentID, } - return + + templates.ExecuteTemplate(&buffer, "listQuery", data) + return buffer.String() } diff --git a/storage/internal/elastic/store.go b/storage/internal/elastic/store.go index 08dc0365e..20b5f5a80 100755 --- a/storage/internal/elastic/store.go +++ b/storage/internal/elastic/store.go @@ -255,7 +255,7 @@ func (s *elasticStore) GetLastModifyIndex(k string) (lastIndex uint64, e error) func (s *elasticStore) verifyLastIndex(indexName string, deploymentID string, estimatedLastIndex uint64) uint64 { query := getListQuery(deploymentID, estimatedLastIndex, 0) // size = 1 no need for the documents - hits, _, lastIndex, err := doQueryEs(s.esClient, s.cfg, indexName, query, estimatedLastIndex, 1, "desc") + hits, _, lastIndex, err := doQueryEs(context.Background(), s.esClient, s.cfg, indexName, query, estimatedLastIndex, 1, "desc") if err != nil { log.Printf("An error occurred while verifying lastIndex, returning the initial value %d, error was : %+v", estimatedLastIndex, err) @@ -293,7 +293,7 @@ func (s *elasticStore) List(ctx context.Context, k string, waitIndex uint64, tim var err error for { // first just query to know if they is something to fetch, we just want the max iid (so order desc, size 1) - hits, values, lastIndex, err = doQueryEs(s.esClient, s.cfg, indexName, query, waitIndex, 1, "desc") + hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 1, "desc") if err != nil { return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events, error was: %+v", err) } @@ -301,8 +301,14 @@ func (s *elasticStore) List(ctx context.Context, k string, waitIndex uint64, tim if hits > 0 || now.After(end) { break } + log.Debugf("hits is %d and timeout not reached, sleeping %v ...", hits, s.cfg.esQueryPeriod) - time.Sleep(s.cfg.esQueryPeriod) + select { + case <-time.After(s.cfg.esQueryPeriod): + continue + case <-ctx.Done(): + return values, lastIndex, nil + } } if hits > 0 { // we do have something to retrieve, we will just wait esRefreshWaitTimeout to let any document that has just been stored to be indexed @@ -314,7 +320,7 @@ func (s *elasticStore) List(ctx context.Context, k string, waitIndex uint64, tim } time.Sleep(s.cfg.esRefreshWaitTimeout) oldHits := hits - hits, values, lastIndex, err = doQueryEs(s.esClient, s.cfg, indexName, query, waitIndex, 10000, "asc") + hits, values, lastIndex, err = doQueryEs(ctx, s.esClient, s.cfg, indexName, query, waitIndex, 10000, "asc") if err != nil { return values, waitIndex, errors.Wrapf(err, "Failed to request ES logs or events (after waiting for refresh)") } From 817b88ab520735633bac2adfe572d8ceee56bdc8 Mon Sep 17 00:00:00 2001 From: Laurent Ganne <33217305+laurentganne@users.noreply.github.com> Date: Fri, 2 Apr 2021 12:08:00 +0200 Subject: [PATCH 02/24] Fixed typo --- doc/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 3774c9855..aa0f50418 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1488,9 +1488,9 @@ This store ables you to store ``Log`` s and ``Event`` s in elasticsearch. +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ | ``trace_events`` | to trace events & logs when sent (for debug only) | bool | no | false | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ -| ``initial_shards`` | number of shards used to initilize indices | int64 | no | | +| ``initial_shards`` | number of shards used to initialize indices | int64 | no | | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ -| ``initial_replicas`` | number of replicas used to initializ indices | int64 | no | | +| ``initial_replicas`` | number of replicas used to initialize indices | int64 | no | | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ From 7e72dacbd93f7f59818d45731d775834ae6914b4 Mon Sep 17 00:00:00 2001 From: Laurent Ganne <33217305+laurentganne@users.noreply.github.com> Date: Thu, 8 Apr 2021 19:44:51 +0200 Subject: [PATCH 03/24] Fixed changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b18520a44..10696ae11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ ### ENHANCEMENTS +* Alllow shards and replicas configuration for Elastic storage ([GH-722](https://github.com/ystia/yorc/issues/722)) * Add a new synchronous purge API endpoint ([GH-707](https://github.com/ystia/yorc/issues/707)) * Should be able to specify the type of volume when creating an openstack instance ([GH-703](https://github.com/ystia/yorc/issues/703)) * Support ssh connection retries ([GH-688](https://github.com/ystia/yorc/issues/688)) From 00089fee22279e34d7afa0f334d6f1c15f47dfee Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Sun, 11 Apr 2021 18:04:02 +0000 Subject: [PATCH 04/24] Prepare for next development cycle 4.2.0-SNAPSHOT --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 81da7cc9a..1bd43a147 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,4 +1,4 @@ -yorc_version: 4.1.0-SNAPSHOT +yorc_version: 4.2.0-SNAPSHOT # Alien4Cloud version used by the bootstrap as the default version to download/install alien4cloud_version: 3.0.0-M8 consul_version: 1.2.3 From e2cc1713face2217f19e8a238f16157f2f483ea1 Mon Sep 17 00:00:00 2001 From: Laurent Ganne <33217305+laurentganne@users.noreply.github.com> Date: Sun, 11 Apr 2021 20:34:59 +0200 Subject: [PATCH 05/24] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10696ae11..ad611da19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## UNRELEASED +## 4.1.0 (April 11, 2021) + ### DEPENDENCIES * The orchestrator requires now at least Ansible 2.10.0 (upgrade from 2.7.9 introduced in [GH-648](https://github.com/ystia/yorc/issues/648)) From a7b5805aef118ec2750950754891f75d9f9c78f5 Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Mon, 19 Apr 2021 18:10:51 +0000 Subject: [PATCH 06/24] Support of Alien4Cloud 3.2.0 in bootstrap --- CHANGELOG.md | 5 +++++ commands/bootstrap/inputs.go | 6 +++--- versions.yaml | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad611da19..b49475382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## UNRELEASED +### ENHANCEMENTS + +* Support Alien4Cloud 3.2.0 ([GH-723](https://github.com/ystia/yorc/issues/723)) + + ## 4.1.0 (April 11, 2021) ### DEPENDENCIES diff --git a/commands/bootstrap/inputs.go b/commands/bootstrap/inputs.go index 96c2b8112..52275dd13 100644 --- a/commands/bootstrap/inputs.go +++ b/commands/bootstrap/inputs.go @@ -178,11 +178,11 @@ var ( jdkDefaultInputs = map[string]defaultInputType{ "jdk.download_url": defaultInputType{ description: "Java Development Kit download URL", - value: "https://api.adoptopenjdk.net/v2/binary/releases/openjdk8?openjdk_impl=hotspot&os=linux&arch=x64&release=jdk8u212-b03&type=jdk", + value: "https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk-15.0.2%2B7/OpenJDK15U-jdk_x64_linux_hotspot_15.0.2_7.tar.gz", }, "jdk.version": defaultInputType{ - description: "Java Development Kit version", - value: "1.8.0-212-b03", + description: "OpenJDK version", + value: "15.0.2", }, } diff --git a/versions.yaml b/versions.yaml index 1bd43a147..782b341f1 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,6 +1,6 @@ yorc_version: 4.2.0-SNAPSHOT # Alien4Cloud version used by the bootstrap as the default version to download/install -alien4cloud_version: 3.0.0-M8 +alien4cloud_version: 3.2.0 consul_version: 1.2.3 terraform_version: 0.11.8 ansible_version: 2.10.0 From 7628d0e70a32434227b2d63fbd71b40c9871eb7e Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Mon, 19 Apr 2021 19:45:59 +0000 Subject: [PATCH 07/24] Fixed a python-pip install issue on CentOS7 --- .../resources/topology/tosca_types.zip | Bin 1079709 -> 1079814 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/commands/bootstrap/resources/topology/tosca_types.zip b/commands/bootstrap/resources/topology/tosca_types.zip index e2089ab180762637b18629737f31c91936c2ba4d..71620a04c170a8b59e02193f2a40ef6c3f78a6a3 100644 GIT binary patch delta 15326 zcmaJ|2UHZ<(w^y=o&ZdDBOoXsK~PZ1AW4#BktCo$T0H!3alt~J ze*8%b2gUG9cK&ri!Y^GsRyv4Zy6d8J>Ii=6Z#n^d_z-fP0Kt_4i9TpZcw0+%ZI=(J z1fiIsgir|w)S(bPs}T(3PeOfa1qlnQMZ}U4`M-MwV@du5RCrL3!Ji<%c2Iy~h^h%_ zd95IW=v5@|wo7n_Pq-DyTXzb2^NBu1a{eB{M?Mi$gnrZtniSB(^HH8mIGkUAuF8b@ z=%|@c%r8Z&7-0!wCa`~ZF%aJ2m&$`X3$^(wBjukBguy}u#@13~qAQ#RdeyqZ#r&Db zwS#auf2RCW2VoRJ#1+UxDPaOZKvRjv!U?j{628gQ-MfO#cP>f<4X}$lBarp>g34nYYA&jL_{Tot*yK6&+@HUGPT_PL|7*2g9 zmdNoDZGp-&gk6+AVr1k$PN;`!*GqI3Z;@I8kkJI;#p&?<{D-ig8nDAS;Sv_mgCbM- zxo9ax=7JEPCI=7%WDh_U^0Vc%kthfvk9x(C|I`Gr0$U@A^t+ z5(+es+8j$(0JX;eGQR_SFUurgPvN3~gffg4md$+A0bMt78~@3AGt*xtUPmaBB*?~CYMSSi47`E=A=wHvcN1T4ECrZknDoWKGO`!j38ma z$QvWcD1ln%_Xsi!_r{I>q@Y~{&+3v>z$F;g3J6C%(i$D?z(PFjK)%5`Q;f({?JT~j zPc{%b5VdI^m#=h%Ja9#8B7|Kz6F{Frv=KQ{?F1xQ5HfG3DO7FD;<}lV4r14CHH{odf84+Qoyg-V-_T%3vgN-uqrvj6LF9ZauNO)_ zRm;~0l6QgV9YWTt)nI@Gp`N6uC#Ve~MOj+#eS{W$B;k9nf#{L4Ct>iK{UA@n{E&}q z?Zra{xLnbM)EqVHFx}BrdoeqFMFPkxG^|x(fy|x6aR4&LNgSy38bChVi-qifJ>n{k z2W#}*#3^{(vza#Npx6>sdx|Yk+jt?VOkyHW{3crEhJ3Q8c&allCZ%rTg0L*fe7 zu>1!S%PIqgMh9MryA$9|-2;gQ5W1ikH?g^@f%-yeN3_~rti!T!p;Fr0`J;GW2p%{P zTa09bmbQxVFg?&J?jhtC$e+IzpJ1E1-ba!IhCBF3a)7+tN3xvH&qpqPk^w-S=_g52 zwFZSU8xU6XmFzHt@9-qawh%=~AQDY}60S6Vua?B|r=r+3l4MTx+f5 z3D981utXr_fNnOkHhycCWC)<*npz0l>;{wn};tM7odM z=%b_$Nu0`+3*Sfv@^Nx>`6hWq;^d&crGxpND92kmj0*|11rs}9(kCegy;$j5CkMg7mu&i>=m!@*2ET4cNjS8Ue{feXx0{AW*E&VEh zHCOI9>E8r=cbgzBR~i6b{h2RyX1i(oBxwXd{X0pT#Gmk6e3=wJTJX_^kB&UPOjcqJ zXy;eTV11I~Z+oO-y zxc*7LM#mpKJ2#`}l!p6DZg$N(b!Da9zgEi^4EXVBN2c{o(|6h4Wk+=au8#a9FU-Gr z>*k}xr!Hp~Rn9!L@`1pk$5s81$&K?TNp{B+1+0@AZVod_sLj9ZT{I-7G(!5DeEov> zv5jZSUmx*kI(mLu`S-$cUPZHa*V!(Q-LpMt&EEOKz_o)GwaxmH+hX%^N$`@Modb_$ z^azNy5qeg(ome@hv3m~#J(G{$OmDjiS9!nPdpMACiS-W|adgJazOk9d^nV}RZ%RXm z>#d_6L6+y^Myx-+QpPLeA zMw6$TiB3!w+pN$!Revv~^Eb(& zPA*q(-_8rlksVE3QV}23wf6|!=2MHOTIqKv4zvy(Yre@jw9Ia$Ud-Z z?UKp`d-mivTCMZZ|LcmwpV!VL`CGhN7BoTjz+%ZjAM>2q+PnEzT$g`H`(1QO7I{7E zMsm5!Ictr!E$DmVKjMZ5rs;7ysTt$?c0BR@ko=@mW50Da(}E3N zEPh;LD>55$?LUe3Cw_0PoHem}cA~3ORl0qR_73e7=kD~%w60-Ja|iDUU#ssC(!W!5 zK;`Y}(YD9-9ckR3>Ap-_>2ka3+KQ(Jg2duEOZgSs%5z(${yhEmZq(ShO|FmKocit` zl4m&n_pie;udWj(^6yu!m!96d({-(6_;9mnXszj|2cj*PuaZlpMF~N=qP5kH4)OcdRK?aL>i*@r{7X z>$YD`2)S-JW?i@y@zr_epJ6@%+X2lU856WBpC4a2v;2|X$kdqdZ~HQz&MTZ1A2DED z$IeGB4A;zgImdZSa<=0}M|pzJq{*V1iQ{hhO*GV*`RUVx2ZzuPs{BXm4zq)|@4RV5 z1_L)N*Evy_xio%Rb^gW=cVDL+IIjJ@@#%v*^U;nAxL^7`nwcU|2!#Pt2h z52O3oz4>$3_9=EmdFY0>OM=FXeo{WWa7@gXss|BG`TIQn9s91i$Bz-`f8J$MQeRol z-M%Ve$vT5|cUu=a%sw~GW1-arKjYO^#$WI4+x7mC=e+;iHrX}lNNZc~$VWZje_8d3 z*>~uO_G{fSmwDq7r*2*2z3Gkd+McOXp1VG%a@`Pg@!g1lne!7qRT-E6J0nY9kLlLaPT3#Y5WD8$oPFE;hX1FR+t<;9zd82v)^410?O@f*UXIZp>mHRK zk+;15HqC3d^?^=jpSNw9`}|Q|zoqAH$s;nKtvfmW(%Lt&OM}XLzPq1CYj0n^X+p#4 zL-$TT@i`Ii=D%=#cKYt=<8uQogPo#9U&}8qGW~l~>8lY;UB}q(Wrgv*=V!VM58B%Q zm8AatOOR@9$wio=S0G-Gqn?gKBk{8@?G_4v1Rgw4sL<2Vcm}pdo4Dc zQXHCWc=}e+SgT!UkJa@$7F#}Q-Z`DCOMOG%pB!~+;hP(cAO0-t+Ow@UmDBf9M{)YO zi*bKV*=ahyTl%zv9!sv>i%S1FJ8=A0llO^_&s@8`wd23^f=>&F_Xz7lPsue2=}72J zO44^uUE5n*`^(D{ZSQ+E{f7S8l+ajkw|;Pg-IdQ9hhFb`Y01KDi#(?xx}73#%ks-F zdCLaAJ5g6>8k89Ry~k1Kh8sO9^&an8b~&l$M(Kw~_pGP?>pT5ouIY}^aaG16J?CCH zdfc+P{BCZKv`t!dm&;08(A@=5+gHo3it6<#$@@EIU{c!f*=BA-LyT^nIXTH;qh;5#Z?if%ZA*D~WBsYBSE~x{=elJ* zcvsHDhepaBpS*lAbzV8k!))$2g*r^&X+1!jI)JyM;fp9ZAVSv`QSJ65Kukex*^~&m z+fn#XVyGRp9ItN=g;1N+>+)s&s1!>5vu-RUX}4RuoBIeMZO6`%tU@N+cM8B0k~ zW*NqmFCRm7Q!bSNIxm!34PrWB)N5dV3#0zPJF;67C_RFxES8J0yk zW9cQ?)O+|T89^mtkTruT6HI3QB;vzn4*ewX^uUjG7D7p=MMj-oc-= z01M$eLZ3Q;zvr7!m+^OPC#nH|&!MP$`1@Kns)+4^Oe?AY6KF$FB(gK*!M4;a<;DqI z&C;bFvkRg%UuA1a?zkvrGc2fvcu<{C^D_!|aCn{F5%qdb4PcjfLcGUEPHUAyms_X+ zXhHWSRUf5Zq|SU!bzoP@FTCi1{H4g>n;ycg5~o(6v>>{G=(kk9V<>%{Af{HxbEeaG zNTNrEe0n+^DkR)T$sf$8hXZAleBC(OMu>YQXesS0AWTQeFHWRGNkVs&d`CXrony_W zO$5Y^f$}$D^gM$2UM!!!jJ70+?-lZkBj|@BB4~;{DuzxKsuQW>6gpNwOe>O4iJ;F5 z3AYM)eGqLXB$~_QL&ng70^H(XE9nKwSY?;UZxz!s*r2A*rAI5*r;t^!CGUurj@rn- z5H_vmmXsD6KbXzGwMldjK9SK+ZkIsU;ePlqh~7#Nw@c)glIaEk-coGzruRvR?Qqny zhBlGFvO0A$-Jf5Kwrr-8VX-aRLPttLF@6`VFM(6foC6TfN$BVSI)z^X8%??hI%wWO zIs?Am9;ES+O6pQ%+|7<)?i!8Q6$K7ak8Sr)%kJkhr7oL&oVDsk` z(}%?3VMUIMTje6;d6@2M4$_e&4O?15c)XRNJf4S!^rUP7NPDBBhX8#6?Cl9S^exMR ztzgq(Is`0qI6@C&n}xGq^!gZd(A-)9syHaHLiTcjC5k^zcZI!%In9^|1bbPI9eQ?z zw!~+`O~*k#i=*~L^dW%(3O&KG&e?+kPb}o1{ExvI(UT*z6`UZW??*vvcsrJhg(mgHufP!zGW%O*2XofAltHNtep;Sw)AVEiu z(RTQFT8i5((rM4~-YL*QX+TF6-5D|TYVHZT7`pGl33@zgJV`^HKusrU6T+K9JI~M| zXw@yc3mYzT+)HUI=!$SKaWH%aX;`pdA%FiKO{1JMv@HPHodux9GzU~MM5v4Yy07SS z7R^mpXmUirDXKj~hoZ*Y zv;mrUn(k&cGdLEEQ~gU*;WE6R0ooloV_5Ok9fVS3RR>5jQCJfw^N*dnRflPkMlLe zmw2=je_aoG5nE4NV;$F|`+K8E9&f54k2hAK!;K$rPfZ|@ZxRIiir>p8?LZ%jpheq@p7C+ z`8VhpfU^GsWDb3kE@qqW@=FgTYPea0n48KxQKguks8SKE$i5Bno0ZUx3rQF9Ye!Gk0P`BCDXz%5lXuvhYrX@BZ~vDFgBIPzWr(YG zc(fAssT`D|hjSkQ?U;fFJxrmd2XrsCOQBZIx(VEJI0wO@RMr8L20D*lS|x#rWEY1eBRXHv7Tfm1N3;uW%34)%3Lis*CF79pW7-3yO%QfO_a3vw^CxBmed8GR zKZQD|1Z-28(>2eeqZOEgp!1beEN4!YHrn|TW(Byj>VoP?#u8^ROsghn%@bH6xrL6^! z`>5~+-I*wc@hfBk(C4pE+<_`F?&a_Y)T`(4iPZ>Gi$ZeX586@XzS%eMQrD2fnEZ|Q zRQAlgAHWDV;ux=f&>r~qYv^|%eo_z*f2ZB?!v6?{IMIod?DC!VGsi2<_O&xyBQNiC z#-b6ZP{3GYi>(t=Cf*0}MG(GJ4U1EZvE+0twCnAu4^|*h3n5nybS~VNO*l9|k`bYn z&!je@g$!H1edqmYau0-hK~2+~jc0UG>GVSWFW70lJ_BO>#vI2u99@0}9ato0#JKIl zJJyBX*X8l{!99;=+oLS7csX+kq7Ue%L6&CK&gV#l9jWg+uy5ZX)MN;7zOW=4z z!HEbX4N;GeY$SoevQ;mQoSbl@`(yMJbhgHh3YHVm^r0kP~WOYDz6e4_1e!&iXcjd{*;Ry+sJ z%XS=JbQ640_DXdVEJYg76>Y|iyJ=@p?-`LMXi^im@q4ASCoULD#%qY1a%s=Sc&k9n zrDyFwbOc!O?H_1swn#x#-dG!9B(5`6Q;cAsSRu9c1&xP$OEaj}D~Z7^FrDC`2AoST zfFssch}$&Nu0Tx1;%AgZc{3!FB3bZGN-#zTR#$M6Qz+GKXetMl^RrXqZJuHyB%|1xt)&Z& z+-k?^h|*rd5#=yl#!}f86%5GK&roYCWL{K5>?*WinFf2<1^8GaexF(hSc;wO99&EFdlA!taaB5+tp`ro974Js7#Dp1E#_to1b~JNbCH_iN zm}d0!nK1A>MW4YZLY3Ws^irR3R>EHDh!tB=aYu-in=#W9)?aLTWV=a_zmLQaQSTrs zY37U!^)z7cuDQPfcywEX&A!rEYrweRj&{qN?-dUf!xS>cU8AD~UTVy)s)cNfAp>s; z(O>xWz%@f=Ft}r9#7u?+*k;5G0giSjW`r_xA3^rt_hVr?qM{^8XXMwJVJCybl5gL< zV4sOU%KU=S08@CdzgpfH-R~=rp)BZLwj1#Xd_8wwauA$twS@6_y%h#?n(qgxHM=5T zV^C5Ae0(Pe!e6V@Z1p}N-Iy5)&kc?nGg+wG1gh9n);Hp>H7LN6F~ZLz{D9U4dWYz+ zRz>TK3QXb9P=W6R(IBets<QH3SrO62ZAwKB$7Ft9rd<%jgT!q|leArq~kfeRef z2i>w}TrGlpa-*i1=g)G5Q?v_E4mB(G5*rr52W@#P(MNSwj3w)h*y-D)&0zk1P|~c< zF})NDR<9+xXgzK>-3PT;LA#Uf6|7Ke#u?t8pfg>8deB9wV8uA$`XYl7EzoyksSdcb zsFMUXtR`rEcP5*wgRwQ=0M|!@ zy?jeg#vLBSpn@KNWE<2D2{;tGf#KEwoayGYn#Tb5^}pcehqO~F>jlQKtJL**RXqXn zQ0rm?uv^gc*B1}| z8|E)-Qy$Mz6)~)ICK4;@qZ&I#54qdJETtI8Wp-f4%aJOsxAGh~+nyN)$KG&?ox?4K z;k6DPY{Y_m^u`IQv9=>bz5qEn0_~@QR-FS|(%&5!M_?Sn42NP)m}MLsEtMf_Cz!w{ zpl~PP_8iM`+g8E|Tj#_$m`zBIGFZ4UQ3_tiK{f_yjQ{83uz04@bZ-Zkh4N9G6Jw7U z4|r#$IKfVJhI%sr?Qw>>Jpq}xF!^B3Y!{{ogul8lIglE|U72FeD|iYt%neM5sYc~) zOs08mNbI!z^M`mt<90B9LNyGitCk`EwK87>?$9WvH#^@H_XhQ8>(Tt)P*4?7ZE%IG zx7{SGs+PHG*!H70gP$U$xHGx0_*IfGRf2^fVw~ zrC@??dN4uo%E;K0DWFV;evC@mXI_hw`HTo5PsSI$>ciNZ<5x#`qEP-*U!&kbbfF13 z^?`+Tq8DQYkCCdp!41zBnARJWw8(VPaBs%aJm$CH*roxY|J1^4>DF1j`{Z)b@g0B+d<%OG7eIE%65o52uDVr z`=K|(4mABPryCIkizLjuvttC>c87 z2TtuML#;!B{XSmBW>Cs>$^ez6LCp4)1;?(ua``3=Ni-YcHc}&r9w(}le7RZgzg}Ev z$S+M%$^ZHQOX)1efUqS@)t!hAKu2a8{!=95ha`vHy%zH;(F+N z(l1~+^EobijUSlA`jZ1fglvI|p@$Bqs?~hJ4Z?dN$9)(IyR%a<>}vi6X53RS%EKsQ zG$WQ}Y>tJLKaX_NnMnKu!4MUOQ#6{>pF29L>OS<3Q7{mnLf0wpf4Fg(wFEaZ-@Lj+ z^YTmc9%WJ$XNqGaRBe+8F8Ve7!4^}b;<&RQJ-dw%uYv41pz~{xHmp#IiqP2zDE+1A zO9WuNUDb|prjC|XCA0N1ClZ(^tJ^W#A{iIZlEndI@LG=1h$r`DacoroO$5ddHKPNf zMnVTB#j`P*91q-)+f{1%NH2j^^GpCnryVMWCAy!0HL3;zefb`ao|ZvzbbQR_>Y+7l zCREFP@ENU<3cL5IV6;#}7T9K-$hPi>ZTn|G$32?Jcwn3>huzCI0>&8V4dpc&*D?41 zZih2t1jS%4RJnZ^N5Ji1JCQC)P|XyMvgtG65XT)_2I-@f$|b_S)xjRHLExpsr%1!@ zw+iu^a=^)kfu$Tq%Z5Nxj}LQTt1E!pV<_v8JtQpg{0PUWNrm&&|8+X5JZRufeX5RO z@&4S40!62_1OtxMaTP-c)tOQHs4@)dCk*oCA;7LYsbZUy7s0c7lw?tI=W& zXKd?K;*cAGOkYTvKDe1zXWKDqgBdT}3G2?Up4S_2b^=bJ2F{@i?Zh4XGE!vL8BD_l z0OG*_K>YKvrbqxr^w_#&$7$%tt$?aoDN3(uic3&&7~{p(*7OT`cQ1g_UQp5;q|Mhk zrP@%K%S+L{KxlXHEsl273Q}XDAw}VUb>(NG?P0+Ea;IH0TWu*l3*pm|>0j42OFfcHd2m zR*>K^jUMuU3hTBSeRf#m-45|Y%{5UTRkVx%M$$hV5~y{?Cl%}z&Fxn4p8V9|gB~m3 z1n*C{bnw@(+_^W6_#>F%c)5#TZ3sfVX%y=XmlXv*P_XouqDD+z$x$7UeKrJyJ-UVG zJD1>xK+uOq#X)e-(=7+)=6*~2MK8S5xFHVW)Ch6XylNQT2UQm^h49h`Ef@`Ehx?(U zg>Y^0dNk~7dt@NpLLgcNV`5)aTnH-#_fg1kED-aQNCly~%*V*v>(!Nz*gug9{# z!T;X?7$B~XHWzUts?!+2sU6CJJC1~`UOgMoW{m;%je)2>2iTX=RcudqeS~65fH(wR z#bbkRE`$G%=vWB^>3hY3=ym>a{z=fVjTw&@snI1TJ8 z^FcLzUVBb?0{f_44iJ|a0#eUQXWOHY=eqHHDw zUU#7)h>~FC8W?12;Oh0i%uS(VZ!=9w=s5; delta 15829 zcmZ{L30zI-`}n=*p1Y;aIi)PEw-kjo6={zmqD_&qgpeg7OVK975VH4-ZNxCP5QA&W zzGo?e!Pv(#wqzM=_&@JCclG`K`+V+P_u04id7ty1cV@l|U-dTJG%i{~n~2EY;(~>y z-Nkbk_K6eE+_CYzLR_?XqOy;;=#LA^;$h+<^{fm@Qdfl~QYu7Siqvj*sR6UaMbqQ5 zevZo$tK+h$+A#`2l0_jr+AirWo(n;hk|e12kjtpDVlb7{L#T)%$TOw?)yVNHQuk2O z?Zs5vBK0^qeM3wI6{&wRq2G(CB{S8_DkT%8;$mR;NQQ`u)!BO_5JwHohtX#AP_u$= z@x*H34_^7nn}DhhZ39QFXbx&T(!oSPZ*dtkZ9*>-m#KF&p<{T?lMI?yq~6KVk+ePvt692{SmQQY zmTW^$K{@85nI>TUK-LVxF0#IYHLoV=tjUd}gL#?#ZRt5W8wP%pc@X+Y7?VW1pcGRt zdM6rk*o)qahU9qDGi3<<-hnPhXt_WAJ3_m5r2muZ&3WuV&tUaG>uu>+F&Z6aPyed5 z61_z?7bEm~C><-oZ`aOrH_Ajv-O&7Q@la?bZ3}5pv@@7Q(XCKt&{Z@I0lhTgoRda( zMVIItXh@_1iI*hNJJ7%`y=gTX_^mhXE7Mz4)PtU&(EOC7t9y2%PZ9$gN7F+M z48*uZLHTvIId}}9O`tNASfReqpWZ|1V{QK!gZykT-5qrnl7Anzls7A5XgQ1=Li=be zotRBu=Pi9jC|0#lnvyDM3^#|+;V9BJosMVW$Q7B4)K%;xx-SabG>IOhVPD%wx`b!f zDS)uabhuejY7E^oWOetSM51>;!KGZ8le%&;z4|8<8P`B}*NgOLr2WKGVJRaWg*Y)Q zO`xcXA~AkEs@`$5L_mcpU{M=qGsn?i6kbb@+oX-9%%A%x~kAghsl2ui!sNWM-dO?^u)_mpU2 ziQ=X`l}$y>%iqY%wAEBmr2hI;wp0OyEtE21GcKKcr1@ql4?`p#h-RfrCwsa z*h|f&mSWPQB+aGg@LT=CN@}kS3xboHNh_ENG*i>_@C&A8(@s!+L}~|z+e_V1=F9fd zYgmPQ4ogRhErni9SP}<4KlGCr87O#dEe%z(2c-u{sa01Ym!*)m|Bq-~Uh3yZ&<`~H zk+rlKfBtPN-A%YNY|wN3UT7~(r!aM2??)Br@EZAun!6)c)fqe;uoD zQC|+oQ;`#?oPe|B93jq6r*!`_`9)sov4iq)q@jdQ8sjf<7m0k`NsgKb48(IaGezdB zbq4LpysdD81(uvSRK1kDP(`V5`nJN6Kv)P1P_RS`SsB`xni`<^nm*{KAd!6Nr)W)! z3)F0Y;y7}neyw7rKYVMBJxFjpS)2!_ZYzeP0KJ28cN9I5_~czh2knY#C{h1< zN8zBYF_f(*=WLWMA;wyX3Sg48ax${kSu1~0;7{#7k%q8uEtEqf)YRSZ$wTf4Jd?^* zDTB&Z%9d!Xx?fvmTZ%e4Rei=s*;`CSM5@bOmHQ>s;7E0ehcZY??U@SKTPaVG2}I?m zd?_U!VvwBx?X?%Gnz(6Jg4xeZ`^Uy#Bx)Z7wdR<89fdk=ts+7n0X9(CSKJLo1S++A z)aYoXT1+wf;c}pIJdR0`K}vI6VbCXBIf?QvRQp9J(Xnx~9LL7(!JDdOUv6Z4h$F~LtG{Ftf}^UWryP)PMEGaMiCVc}}C{^;?m z)n-FAYW+5t)rx8F1zqCd^<%~wwyrj_2BtmN9MYm0DO8r5If&$*N zP_Ecy7B9gS2`UY%2^4KI>qy+UdSJFf+v;$=AJ}A;;IDTdJ4F+?_|Pmwyadc1nGGf! zVeieR(6nSAv4j{g?~xg941ckWz~QNxGc`X0vfh|op(F#Elqo=fv! zVH4V(%3Pp67{hduQFC+Ev!a=Hq-&{@Mlc7+KD72IGY`Grc!~KNf2zxFGY`lzR{Nc1 zjuXJpznIlL;Ou#ZOeE@0XPHan+v^yUPQI6(Kmz=Je}x%CzJI#RWKmSc1hwoYGfUe+ zu`l2}+z+^}oC3CFGGw%^&19C*#16JHX`}QttBoYPK!h*|89B|QGYxTbA%z;KoGk-|60VXAmVMtGW{4m<%Q>p+moQ~w(p1rb)u`ajVP>OV1bnOFU8y_7NQgB{ zhcSmJYTLX$Nz4n0d3|YQeC+%`<~YhkA|DHps6W&WVG<#^Gsj4kBFVdO7}uAvD!h<< zckJKoxq?dr$Hm#-qDoh`>}-=$J>6o%)&nQ%oQ6kwn^Nwz?NVnac57YQ^kL!M__ag3 z^h+Xz<=KC3Prm#0-li{8zhrsr+@1b9;dt%+Esg!F7t?uvW#=~ya<@F6vG<{q zx%Y1-Ov&4jeorhz;*Q;yoOMac_s9_VpZl`OBcCHJzO!qk& z_wQEcpmlY3LY7yhJ3cDuvf%Bk?lsXS1G;o-7!>=tK5%J?lIdFKc+iU~4IV$>+g$5w zPCqZHnAXT_-`b7EZpoEDQB{pb{61}*B67LQ`Q#Z^e^wuxn^f#|?~f$M$IHuo!#|n! znX})P>o00wSXh=a{P@wd-6{LliJqG@XBQ>JnR%Rk(zNmS2O@FPK4Wq^Ev330e$->_ zmbZQjOke+$Uh3L-C*?Tp#G(hwn#f*tZ+9W*mfWSult|}Dck^B;J9D?$q-WNwxinF{ zuF-D~azf_j%P$5MzW5M-dc*4%U*8Rrtv>jC)Yx`513!9I&3yZ`Fx~gpc^ew{tCEg= z)4aegwo86_l-2Szlb88^GKw041e@~nc_{;j-cB;QInTWfb_ zV~=I+jwUKRJGm>Pm$lq~{l>U+>1XcT5oayB>}Y+ZS<{OnlD*2~CWUvZ`uRTBeT(A) z=cCf`VZQVJ>YS`vvvz2oB zECa=J?%h9n<3Y3R^|ecmu6^^c@7wH>vp@f}>%6*JwSJ1*TE~q+7MXL`FI$m&abjuo zu$P0l64QhpLz1dggD6{{-_N(&9x?jzhFxjLR=@4BGqvKU>lW`XPwBbST0CRl$f2V0 z!e^%fo)qq=*wJfXi-37KJxk-ZZFw0qCo5FZGuKocSQNa__f(Kyj}|3i*WP>Qy1mI6 z-JvGGEHRYO&(jljd5jQkD5UJ#{w9Uu6HkKLJhBB}e84yM1kIa8TlDS!o^c#|EGJC@P zm^O#%rBxRV)NFG!HrDs70o#^?0rR#+f4KGZWWnVd#nyU()`xl`M>wM zlk=wDD`(=nQ^6+=7jAzaJHWv{p#Dke=0z=kt9MIXZ`mZH>F$^hc6r?vIA0O`TG*Y&&(T>EOMKANRHTC_S1ox7ea+`j=DBBC+_)e&4GKS-XAmB zH#X|est3{=iiWh~d7CdS{kH3KzZ(s|j(fAL)xGZz(}(-qeRq<+>SpH9_nJps+3u<3 zf1YulQ<)g|@^YZ{f&O<{8y>^4R3`#fgUjyzA(RY|ed z1CN)UoOgV_O=eEN&{vm3GX5D?u%tLHVI8x~YY2BD;o%C}t-ztl;0=?ykG%D6?Z&Vx zwpDYhhZT5B()MoI^~=SZ;+KgD%KHm?=3dEL)oAINsHWR~@7m>d<2_qmUpkdD?{>)6 zQwL?{?zQVH3$J&*bKd9v^)*dwS~IQx8`1ZdvCR&pdB=RRNqlUx;acqM0ZmfIEiVi# zP}lx_ZqmVdZ~s{E{8?DV-rV&m%?>Qw;MDHS+g|(jKRYv`| zSl;HHJ+~j9IP7L-wfpmh%UoU`xb$`Wl}Y8zK30T$$*wtgWyXZ06E{?u;;MTmC#&b} z7(4Fs`rj`cOsEU#_0Bx}@dYkp(eG!2JG#%^)jqRz8}~||$k@wZamuf?_xGV|I-TA5 z%E^0U|Lm(fr&rfSsp}?QzA?AqE;%R^cC6j=)OE?s7UTULj{;%EXl9+kuJ)w)xJ99q zNOTs;hcU_Ur8Cz^?Ul~-*G^ivx8Xx`5$;}24QBUZ$asTiI%y~fC-n@?coWi|wUCl+ z5C`)~NIL8ZQxl_~xDo={-s1JJDUcnYVR-cmyH`OKF90@(9mDv|ijRB0`*r08iAbbY zfMYQm1Pg+ctU9`YJtLv64pZM8%$8Er^c;1oT(+8~B4(?{j$`}Nn!^lyCgp&frDt7Jfh)Q_8j$Q`hlXYAJh1LUmuSE(~I?P*m$|wO1tD zm8QPVQkRZo@5(6iV)f!wwiQif6sX_KVrNj)vl(iuVz#q{YF(_p7sI;JlwGm9dvCTY zaa(<5A{#G3chpIt>9P1n{U^`*K6Do?L_Y2y08d+yf3r*pc`cQ-VJwLk{wk?r`l0+X8`~3GlQ4Yy;c(NUWgm zQPzp(k6@!SIBTds&L&|Zx}0Ehfy?FO4tVArUuKth1Ff#5MWP5JCjuF8ywBR>38+Ka zIWz-<{+Z@%sYM(Q7f!v~oMi2VU?SS_G+K$r2!u^QSN5i5Fy}N&X7n8dyW|I~6|_FX zw#4Jz_%n!gwJ_$jf~>Q6Qd4_|b#W+*j32~sJGP;%f8pZGFtYX7|HbG-OpZT^axJV3 zAX0UKbq5v8 z$#D^Qo?~aCw<9AtTey6V9j}Y@vjIcLRiO2@80o%7*2milvdH1SHq(9U>TX zp;m1{!})5{{AZxfa8C3+yoG~~DFY4JbP)v|)WbT1mp5$5pd&mT_Z|XoOA!$>Tc@orYi- zdxQ1RwUFl|#M43y(5ZH`11vryl*W@~x1Z9Oua)>QuD04x10Fv2d^^Ev!y2~}hwHGu z!utm9Hr6lZ?8?&@k5}Rb=42%jO);u~MxBg#7>Ej^?RBO!9IU>%0?m+O>EszPrbZew zHiG&qY$#ug{P|?!RE2=%5L2^C5L zt9toE2^AO#zw&>vH--&=^VM7ZH)@#M^Z(FbE|hTgEyFtMSU;)NO}R+4AJ4G0Nhj6& zecG8`#xj%!#Ux3G`G29Fh<-oP@DoR4vdMh*aT_SOhvqCB!n5(O zxi)8G$<>&1PNZ*+xr^BBVR|gS;j)DbcX>}<6XY{vq&0;7!@6p_;Okb{{v7VJ-B9z0 z`)sCGX%_Cllx#8)>jahmut7u@F^yW8s{^vaQD(gvCxx23_|{Il>8IcXK$%YYk+AXs z+Y$9VBiNhYjIky#CLQy6^~ zN=2f#6wV3d-#IGXUd+37Jd@E|W2=YJ(>5H#NXjPpvGN<&lQgr|A!T64}M7oWZ$ z>Xs3z?hU^DyM_vR2T}R(FRUHqqUUZ3H)u|U2r{HdN^tWW<^SG z>K2;15KWCUGSw|a!`0^_?qmF>gSvZ=K+X@5Q^1ifQ zFz6oIFda9uVMbxgi{fSS^l_xV$e=1|=&phCuVha>6Qp!~(rnE^4F1n3sh5$IqG&z6 zIoN+=@%Xj7>XW^7ZbEi8M=ZxvNf z5q&DdkkRfL8{$Z2*j2mJ>cTMAM&zi7+FQF8_j&$?*Kn>S&~dma-2 zRY#->IPjW}<6RUgd8@_vzhQk)Q2%EL{PICV?qwsjfHTi{^7+pZQ}9ub;o#$QzWux) zJiea|F-gz){nMRP*M={Km>2d^4t__tcK>c`ifgp-H=afq3~@Q}`}f~8)P80CAlrm< zMblQA;G-h07#{Y@glk6z;OiqL6Q*Io9l>@KWK?v2usTX9Olicq5%MkpE6!zNjXuM* z)5(Z@vz0(zL`_n@ho4#Lg|(v4tY6{2juY{i%b=;nO>o?2oQEQc#=z zHb6}K4+mzlLnD?$HgVa~dd^Q6ckr!zW5`3G;0xX|*lNm+;-~n06+XTDnQ?viG}z%| zd(npMRizl3ENpfmMxlX9Gp-{U*(Kzx$=;c>k-YJeL!qZU7>J8x@rQgm*Z(XA^%8#c z^9-r!3~!z^8j2XwbcIJ+SC%8EXynuZ*0Wp>Z81zW$6^@mCx>x+cbgWkGy~b$TD+|t#c&ReJ zW`f5?nrMhbwIlQMROw37E zTsz~5kRM0o_87)E>F7yY@$euI9~O+yPPRj>73W>nlncZ`JE$poc=HXOV>IDB`G+Zt z(o_`!SN|sf^-&_(c{Q&zH3AR8o=53diO_MF_ZoVY4cC^B5MLfDA1sfnO*y<;1FB}+ zMATW(=O6a0(CX2wpb&LaMgGRFps) zTW+{Ene*+Cc;gy8C396ag&JF~F{Jq^P2fT+d=A0u4d{=pWW+i>J`@FTgy~NMpf#k~ zajmp9gUPf7CwndmuXc>G=dzK5fBN}i&#Qmsfau8%TmYEeQ&>PB%mi7JGNLXOagy4* zygEp506*R)I$?(?fhkU? zz^StY=CnZadIO9cdO9o3;S)hm1v_U%C)V+d!I10B zIXgC>O>*ysi<>aZ`>;Nvjk0`Bpbdd7ZIv86L}PgG9Zyc(9TRn8$G)rRN_*t&X2kWe zUZbIjuabp$lAVZ1@ODAvac_83JgjlyoE!&7j*9s<@XO`>ZYTi>T;8POY75I(*RrMm5gk@r~}$adx8wdGpg7A-v$sDyjr zO0O~>yf#=lnQKKe`|-NOm^ZAoq%fGr1%QhiCc7A--H_pp7KpD3@D$licqGx54EZ)L z8@e+uBoun5-~*9&Av|l1Qt||76qH6N<&fM4W4A637-G?bNDTwt!D)xLZy=d4L`U(^ zSUA-hY1iaIUK@n&=&q+#!ar@e7TR^=;LfGu`TbOPZUo*mfqouX8k7g;ruEa2-@3@o zTzzG6-~F~Ck&B&3%k4ec#s2ki1p-pY9kum z4z(Qd#LlRVrg#}lQE_(EiLnsx$+-j93H!k(PYh=PICxQm>Ndtx?Qo+g274cF7=F$0;WE*Zw?14Jw%&AKjvTsO^yT#DZR5Ynd-cPm z<&A42*r=f6$|265YXaZ=I0s!Ig!@KWcUfzy#abQS4c7T#ET+OaKQwT;9+6BkGiP{R zTNiY6EIQiNNJa184Mf~oXJ!gV+Us2z-hm@mo=$e)Mv>fC_#-ODA9tDhw%Az7$U>yc zK3}eX(E*JQ#e0QDc~7Vy(i64W#*x7mLOXI1k|okun0o|IAiNx_uCsJOj93;nmCl3*5ce26HLY z)V(ln0?r@JL%6~G@*tDmqgJ)=RcPPL7C-hyPDF1QltBC}ea93veC8k#jj!IwDfdbqEt^xQ$onwxwe%QPaW! zhz_+hK(9E>IKZqlT!lq&b}(WkO%1T6gE<*=9>RI?qu+1Ut4rHsq<3Jwk2T8sy5>42 z3+Qlxv4W66yvV45C~~^34r>Af&oNDTEUEFy!6&94$7F3msoEo8A$PqT1fKH#TxUYr zdz)YCcSJd$?%_td_d4n*P2uYarmd0FY zIZmpK33FoxpSbbjHJaEMYh$PpeMdJPy$OuIiOL&>kcBQByO9@D)Y-O#BmtUq~b z6=A*H32X>db>*CeyL1gT*m>$S;3jcgYrInj4RJ_2-$S6ykLTJG<9lvDQ|gAsA3%eQ zvv8}OKdJTE^#S9Ap z=#s9f)Lp!LZD&%i#o?zi{i z^UV>@mi9G396HFDLq%V~Xq_bd$3%?*g0P(_20W@Hj+|!1C84lEsm2HssMy9Zu)nt; z@k-a4W@(u96fD+hMlRhS{6CDnIr4xWpNwJ~h8kdRC3ED^<4$ixBn{UgIuQRGzFG~% zI|HIv{1}(jtPC9`{=X~xF>vn5f>FB~#n$dsibNGS$QY+Ge}n-~S|Ue|ZTYHunk^vk zLMC~@XL_J1^Ck$e`8_yKT@Dl8z=V;w%Wp#Yqm3LsFi{{s*$rnS?Y^qJu!%9!zP`kY z!Tq9oiagXtP6T`S_2>1h4bZOk3lNwjEL;TI$JD2 zhK_4{06B4`x|9)YC;d}8vT>nAo(7Q>$_1#%JqwJf?yr@`TVdsFdPHQn!1|$_8?y7q zR3?uFSm`i)>dk6~OS1sdhNIjm1_euv`z_GEkClAOU(1qolAr1FyN0`yyyw|H+75 zE?~8mwW5p3ZSni|IC>;T5JrD6AA!>RW)-coNj9y|NhVGutCF^{4M zc_(1fSS9Qq!+GJwDF`P#6Qc!8ZZ7gvjKfVYV)laAIV>wrq zd2wLunWN!Jt59GKPNT->>}xXx{+XG$H)a$JiizFx`#lEr4@L4&BmLFjmBn?0s%)&N zXEWmCYU}@eUX7G(QD~SEWx@o(QvMlZFG+XnV#pkYmce8}-Gx!;t{qwy^SI5Xlh`I> z@#=>0LBY|f0_n=$Bn_FuG_WlTg_ukeFw8j4TSY2w(6N5q24I_r!n+^FqksRO2x&OD zjO1L&$~lyY{Qe~Zzx^1_n*d*ABe3^80`!MY6S$V7Gsvh8#c+>=xOTok9yW?=!)N^4 z#lxk0(Q!*u8Dr$Q|5AapUoVy;UBn45&A|&&=Q+O!m>y&C@f>yD8jrx0s|20xFr>nn z@75x?K^BJNG??7;B)@30%@vcm|I`S`e+iveW}CR`mY^b|e?@ From 7bb34fe1a4c2fa69935c762b1072da6c5d842e99 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Wed, 21 Apr 2021 09:46:09 +0200 Subject: [PATCH 08/24] Update pyyaml dependency --- doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 0a7abe077..c0822dc38 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1 +1 @@ -PyYAML>=4.2b1,<5.0 +PyYAML>=5.4 From 3a57738b7a17456207e64fea285e03db11ff2c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Albertin?= Date: Thu, 22 Apr 2021 10:24:08 +0200 Subject: [PATCH 09/24] Update SECURITY.md Update released versions --- SECURITY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 03bf4c004..1184de535 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,8 +6,9 @@ This section describes which versions of Yorc are currently being supported with | Version | Supported | | ------- | ------------------ | +| 4.1.x | :white_check_mark: | | 4.0.x | :white_check_mark: | -| 3.2.x | :white_check_mark: | +| 3.2.x | :x: | | < 3.2 | :x: | ## Vulnerabilities in Yorc From 4bf0d06520c16a5a35bc0791bd30dfe545a7a0c0 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Mon, 26 Apr 2021 14:38:35 +0200 Subject: [PATCH 10/24] Add an artifactory cleanup action --- .github/workflows/actions-test.yml | 18 ----------- .github/workflows/cleanup_artifactory.yml | 39 +++++++++++++++++++++++ build/gh-action-cleanup-artifactory.sh | 39 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 18 deletions(-) delete mode 100644 .github/workflows/actions-test.yml create mode 100644 .github/workflows/cleanup_artifactory.yml create mode 100755 build/gh-action-cleanup-artifactory.sh diff --git a/.github/workflows/actions-test.yml b/.github/workflows/actions-test.yml deleted file mode 100644 index e54264279..000000000 --- a/.github/workflows/actions-test.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: actions-test - -on: [repository_dispatch] - - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - name: Run a one-line script - run: echo '${{ toJson(github.event) }}' - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. diff --git a/.github/workflows/cleanup_artifactory.yml b/.github/workflows/cleanup_artifactory.yml new file mode 100644 index 000000000..c5faa98f2 --- /dev/null +++ b/.github/workflows/cleanup_artifactory.yml @@ -0,0 +1,39 @@ +name: Artifactory Cleanup +on: + workflow_dispatch: + inputs: + from_date: + description: '' + required: false + default: '30 days ago' + schedule: + - cron: '0 12 7,14,21,28 * *' + +defaults: + run: + shell: bash + +jobs: + cleanup: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup jfrog CLI + uses: jfrog/setup-jfrog-cli@v1 + env: + JF_ARTIFACTORY_1: ${{ secrets.JF_ARTIFACTORY_SERVER_1 }} + + - name: Ping Artifactory with jfrog CLI + run: | + # Ping the server + jfrog rt ping + + - name: Run Cleanup + run: | + ./build/gh-action-cleanup-artifactory.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FROM_DATE: ${{ github.event.inputs.from_date || '30 days ago' }} diff --git a/build/gh-action-cleanup-artifactory.sh b/build/gh-action-cleanup-artifactory.sh new file mode 100755 index 000000000..c5c38aab0 --- /dev/null +++ b/build/gh-action-cleanup-artifactory.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +comp_date=$(date --date="${FROM_DATE:=14 days ago}" +"%Y-%m-%dT%H:%M:%S.000Z") + +local_dist_path="yorc-bin-dev-local/ystia/yorc/dist" + +all_paths=$(jfrog rt s "${local_dist_path}/*/yorc-*.tgz" --limit 0 | jq -r ".[]| [.modified, .path] | @tsv" | sed -e "s@${local_dist_path}/\(.*\)/yorc-.*\.tgz@\1@g" | sort) + +function get_pr_state() { + gh pr view "${1}" --json state | jq -r ".state" +} + +function does_branch_exit() { + gh api --silent "/repos/:owner/:repo/branches/${1}" 2> /dev/null + return $? +} + +function delete_artifactory_path() { + jfrog rt del --quiet "${local_dist_path}/${1}" || echo "failed to delete ${local_dist_path}/${1}" +} + +echo "${all_paths}" | while read line ; do + item_date=$(echo "$line" | awk '{print $1}') + if [[ "${item_date}" > "${comp_date}" ]] ; then + continue + fi + ref=$(echo "${line}" | awk -F '\t' '{print $2}') + if [[ "${ref}" == PR-* ]] ; then + if [[ "$(get_pr_state "${ref##*PR-}")" != "OPEN" ]] ; then + delete_artifactory_path "${ref}" + fi + else + if ! does_branch_exit "${ref}" ; then + delete_artifactory_path "${ref}" + fi + fi +done From 91684f0ca6d720cfa32652899c4e46dfdc1c3055 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Mon, 26 Apr 2021 15:31:25 +0200 Subject: [PATCH 11/24] Also delete docker artifacts --- build/gh-action-cleanup-artifactory.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/build/gh-action-cleanup-artifactory.sh b/build/gh-action-cleanup-artifactory.sh index c5c38aab0..4dfe7857c 100755 --- a/build/gh-action-cleanup-artifactory.sh +++ b/build/gh-action-cleanup-artifactory.sh @@ -4,9 +4,13 @@ set -euo pipefail comp_date=$(date --date="${FROM_DATE:=14 days ago}" +"%Y-%m-%dT%H:%M:%S.000Z") -local_dist_path="yorc-bin-dev-local/ystia/yorc/dist" +local_bin_dist_path="yorc-bin-dev-local/ystia/yorc/dist" +local_docker_path="yorc-docker-dev-local/ystia/yorc" -all_paths=$(jfrog rt s "${local_dist_path}/*/yorc-*.tgz" --limit 0 | jq -r ".[]| [.modified, .path] | @tsv" | sed -e "s@${local_dist_path}/\(.*\)/yorc-.*\.tgz@\1@g" | sort) +bin_paths=$(jfrog rt s "${local_bin_dist_path}/*/yorc-*.tgz" --limit 0 | jq -r ".[]| [.modified, .path] | @tsv" | sed -e "s@\(${local_bin_dist_path}/\(.*\)\)/yorc-.*\.tgz@\2\t\1@g") +docker_paths=$(jfrog rt s "${local_docker_path}/PR-*/manifest.json" --limit 0 | jq -r ".[]| [.modified, .path] | @tsv" | sed -e "s@\(${local_docker_path}/\(PR-.*\)\)/manifest.json@\2\t\1@g") + +all_paths=$(echo -e "${bin_paths}\n${docker_paths}" | sort) function get_pr_state() { gh pr view "${1}" --json state | jq -r ".state" @@ -18,7 +22,7 @@ function does_branch_exit() { } function delete_artifactory_path() { - jfrog rt del --quiet "${local_dist_path}/${1}" || echo "failed to delete ${local_dist_path}/${1}" + jfrog rt del --quiet "${1}" || echo "failed to delete ${1}" } echo "${all_paths}" | while read line ; do @@ -27,13 +31,14 @@ echo "${all_paths}" | while read line ; do continue fi ref=$(echo "${line}" | awk -F '\t' '{print $2}') + artifact_path=$(echo "${line}" | awk -F '\t' '{print $3}') if [[ "${ref}" == PR-* ]] ; then if [[ "$(get_pr_state "${ref##*PR-}")" != "OPEN" ]] ; then - delete_artifactory_path "${ref}" + delete_artifactory_path "${artifact_path}" fi else if ! does_branch_exit "${ref}" ; then - delete_artifactory_path "${ref}" + delete_artifactory_path "${artifact_path}" fi fi done From b005225ee1ed22d3cb30b52ab5c12f83c0e81d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Albertin?= Date: Fri, 30 Apr 2021 18:29:28 +0200 Subject: [PATCH 12/24] Move from Travis to GH Actions and from BinTray to GH Releases (#726) * Adapt build to GH actions first steps * Remove monitor cmd for snyk * Publish snyk reports on snyk.io * Add tests & sonar to GH actions * Fix typo * Check installed go version * Add org name for sonar * Add build of Yorc distrib * remove travis * Add missing make tools * Add missing doc requirements * Update for python3 * Fix some build issues * Fix jfrog cli path * Allow unbound vars in build_docker.sh * Fix deprecation on jfrog cli * Prepare release workflow Also remove travis scripts * Fix copy/paste typo * Describe release process --- .github/workflows/build.yml | 119 +++++++++++++++++++++++++ .github/workflows/release.yml | 103 +++++++++++++++++++++ .travis.yml | 65 -------------- CONTRIBUTING.md | 14 +++ README.md | 6 +- build/bintray_release.json.tpl | 36 -------- build/deploy_artifactory.sh | 41 ++++----- build/gh-action-cleanup-artifactory.sh | 14 +++ build/pre_bintray_release.sh | 33 ------- build/release.sh | 20 +++-- build/travis-sonar.sh | 29 ------ doc/conf.py | 35 ++++---- doc/configuration.rst | 2 +- docker_build.sh | 64 +++++++------ sonar-project.properties | 1 + 15 files changed, 332 insertions(+), 250 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .travis.yml delete mode 100644 build/bintray_release.json.tpl delete mode 100755 build/pre_bintray_release.sh delete mode 100755 build/travis-sonar.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..e96f85899 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,119 @@ +name: "Yorc GH Actions Build" + +on: [push, pull_request] + + +defaults: + run: + shell: bash + +jobs: + + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/golang@master + continue-on-error: true # To make sure that SARIF upload gets called + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --sarif-file-output=snyk.sarif + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: snyk.sarif + - name: Run Snyk to check for vulnerabilities and send it to Snyk.io + uses: snyk/actions/golang@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + command: monitor + + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting (for sonar) + fetch-depth: 0 + - name: Setup go + uses: actions/setup-go@v1 + with: + go-version: "1" + - name: Test + run: | + go version + go env + echo "YORC_VERSION=$(grep "yorc_version" versions.yaml | awk '{print $2}')" >> $GITHUB_ENV + make tools + TESTARGS='-coverprofile coverage-sonar.out -coverpkg=./...' make json-test + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@master + # Do this only on push commit do not need to be re-analyzed on PR + if: github.event_name == 'push' + with: + args: > + -Dsonar.projectVersion=${{ env.YORC_VERSION }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup jfrog CLI + uses: jfrog/setup-jfrog-cli@v1 + env: + JF_ARTIFACTORY_1: ${{ secrets.JF_ARTIFACTORY_SERVER_1 }} + + - name: Ping Artifactory with jfrog CLI + run: | + # Ping the server + jfrog rt ping + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install sphinx dependencies + run: | + pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure]==1.23 Pygments>=2.7.1 + pip install -r doc/requirements.txt + sudo apt-get install -y jq \ + latexmk \ + texlive-binaries \ + texlive-fonts-recommended \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-latex-recommended + + - name: Setup go + uses: actions/setup-go@v1 + with: + go-version: "1" + + - name: Make distribution + run: | + set -euo pipefail + make tools + SKIP_TESTS=1 make dist + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Make Docker container + run: | + ./docker_build.sh + + - name: Deploy artifacts to Artifactory + run: | + ./build/deploy_artifactory.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..a5214a4ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,103 @@ +name: Release +on: + workflow_dispatch: + inputs: + release_version: + description: 'version to be released' + required: true + default: '' + + +defaults: + run: + shell: bash + +jobs: + release: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Configure Git user + run: | + git config user.email "actions@github.com" + git config user.name "GitHub Actions" + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install semantic_version + + - name: Tag and push a release + id: release + run: | + ./build/release.sh -v "${{ github.event.inputs.release_version }}" + read -r major minor patch prerelease build <<< $(python -c "import semantic_version; v = semantic_version.Version('${{ github.event.inputs.release_version }}'); print(v.major, v.minor, v.patch, '.'.join(v.prerelease), '.'.join(v.build));") + if [[ -z "${prerelease}" ]] ; then + echo "PRERELEASE=false" >> $GITHUB_ENV + else + echo "PRERELEASE=true" >> $GITHUB_ENV + fi + tagName="v${{ github.event.inputs.release_version }}" + echo "TAG_NAME=${tagName}" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout tag + uses: actions/checkout@v2 + with: + ref: ${{ env.TAG_NAME }} + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install sphinx dependencies + run: | + pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure]==1.23 Pygments>=2.7.1 + pip install -r doc/requirements.txt + sudo apt-get install -y jq \ + latexmk \ + texlive-binaries \ + texlive-fonts-recommended \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-latex-recommended + + - name: Setup go + uses: actions/setup-go@v1 + with: + go-version: "1" + + - name: Generate distribution and changelog + run: | + set -euo pipefail + make tools + SKIP_TESTS=1 make dist + # Generate changelog + awk '{f=1} f{ if (/^## / && i++>=1) exit; else print $0}' CHANGELOG.md | tee CHANGELOG-for-version.md + + + - name: Create or Update Github Release draft + id: update_release + uses: loicalbertin/action-gh-release@080e2e752ac77817dcfd2e8809873bdc24817584 + with: + tag_name: ${{ env.TAG_NAME }} + body_path: CHANGELOG-for-version.md + name: ${{ env.TAG_NAME }} + prerelease: ${{ env.PRERELEASE }} + draft: true + files: build/csars/*.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish Github Release + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + release_id: ${{ steps.update_release.outputs.id }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6b1e3fd36..000000000 --- a/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ -language: go -go: - - stable - -dist: xenial -sudo: required - -services: - - docker - -env: - global: - # Make GO111MODULE=on globally for codecov - - GO111MODULE=on - # Docker hub key (DOCKER_HUB_USER) - - secure: j6S26JA04B+6bwr6detZauFu/UVPwmPhWvEcbVfNhvnNX3YKma8D2X9hvE6jDUOKeapiE/UsCHVgg8GFyPGu4M0ixUXUmu7HPNHFQ2x4a0tDgTyu0p9im5fPeJv4qbV+ORuE4Kvg54ZaZec3iBN94MegPVISpme86pMKdJui0cMEy/YPMUd1sh13h95WkESAshJd0n5AHO4xwD0NJjCK2waoA6ygvcwXhIpfTc9HsNgs6S8WVpRAfjkTj7+VGjoXTqov5g9d6SxBYvcJI/iBa0KgY7LhBRiC6AsE2WowVZeGDpnvL1nFsA2DQKcCC6Tv4VlP7jcyZMWcnELn8n6ucsHzT52bEhGu60KtZc5eohqW/1Ejb0riHvEJQMy875keBwIjuzERgxRaKEVGPQ3nZFu/rjEjHZSm0qi9+usb/vBcdKeu9fgjBeEDIkL5bE632P379VLI1bzzQS+dt+sPw1gqqDP+FJ1nED5r3g4zSPh4WqXAz4ohrX8CCqbxpy+Wjirer/yc7S+Bqx/iI2gKjdb6kwy1xePobFJyAMlOyHrwh5Tb6K23wJqVMoHL5mqG0Ent/3iXmvRS6PxsxwNPscyATUbazyGwXMj4oMWzcfs7yf0ADy/nbZLYUPVfHuysRZ6LAjptn+V8bdJ9A0rQmFUFDcP6VmMidxRlTcHR8yA= - # Docker hub key (DOCKER_HUB_PASS) - - secure: "ZNF+PdDD1D3RJN6/ENFt9jZL+QQp1s9sdOMyQBRPPmc/oBOh0j6ELBpvQTgR7JnhzlUyYvfAk1IXPTJPEwoQ++WmAumdGUu6mEqemN//CggpYNtDRc3vnQfRrfWK5M9pVtODSwtMLT0WENkMURCUPBmlGcISjdXrHAAsLiiJxDalM6rBiPDnDX/yawvfJh2mNVkqi+zwJWGdEZibcNdU8Bdz5vU5eJJFxlR+ZYRm2ccLE/sgvArofovcv32MSy9DnvI15Q8kolTS8+3k6eOTvckijgJizYOhRyAZQRrLeF3taYw5GtTRwjs2UFAkxgbx/NrgSHEe/e+31k5Q4sUFJJr3iIilMC+VCVpIDnFbDXJJR8VTXLG3Xjb4NZycGU5wo2PUGXedR+vR+NUVw2E2LH6L1aJBse4lHba1yRh+255jaiebg5iocOnsiMfRtboLKDy0uNoXaAsI/wGNSlxGncXS9euHYPTXp595TKUPyfEUwF5NVvihHAUJOagr4c/bbLe6EHbmY7XFJHdqX/jxul13RQjpEHmVC6ZlpPlRqWIZVrfG1B4SF/m6NVVLc+8dz1xPb2FWmkjiijtElQhAqbzImW1GPf1QiX3jlHWDxE3DdUdy4kbh10zyh66gqWRnfkS3pVJaB2wLHlfZc7GU+eFgs78b3yk9KZivPOaTAtU=" - # ARTIFACTORY_API_KEY - - secure: "Zt5jCb10K4mnjqL0Bx5pR+qvexrLDz3nIk/IaOjL+QfHQI5w3uXv9qc+ultgsYpdT4ORmN235kJia294UTa+N/F/aJwCS4y+gBlssDkABz0D2/dSSK9iMjdUxyv2lg4NT6yUzefYI+nWziJqhVDg3tgcxy61ENn2c+AWHeEYrRyfgbuvehuyl56ZKggJe1WJkaHSRd1UVc/7i/pPo2nomaaWF8jjDxcqlIiV1onqVzb5p/79psuFZyHKH9Xr59lNsLDPEunyXQu0U8Y3ivV4gCL2+GyiZz7INpPemu5IQtA2luViepJcyRWAZOitL+vTScJgeV4k/OKZWHtNAspH4aiRFc3xVV0xe8itoHvnRN4IMJtEAqD4ODZ1XEyGovUkozoLLTfM20fXfLujLT+g9EjtmSWFap5CdvELs2foPYyiKjx2tB35Km/Pg9DgTMn5lsjMNPOm9N6QhHYCrMm095CceYHFZsfqIfONLJSd5RssnNrScsqGbS5BqKuqza8ffH6BvdoBwJauJwGvSBakaOpEtkCs+Sl4RVCtajTPMG/KgatCUk0HJELIUJyj83LpTUA8K85WcTh8dYb/0K2Lo5TU4Hd9pa8u3KbHtf3CVEmxIaece7rDiwuzQK0DTB4E599u5FJiG7tPaMG+Hvll98cn1czcqlYSU6jgn/nTeC8=" - -install: - - make tools - - pip install --user --upgrade sphinx==1.8.1 semantic-version requests urllib3[secure]==1.23 - -script: - # Test and compute coverage - - TESTARGS="-coverprofile coverage-sonar.out -coverpkg=./..." make json-test - - ./build/travis-sonar.sh - # Generate distribution - - SKIP_TESTS=1 make dist - - "./docker_build.sh" - - echo "Deploying artifacts" - - bash "./build/deploy_artifactory.sh" - -before_deploy: - - "bash ./build/pre_bintray_release.sh" - -deploy: - - provider: bintray - file: "build/bintray_release.json" - user: "loicalbertin" - key: - secure: "qC48VD6cU2jxx1Px+jsu3s3D7Qz9dAMSrNwLEoV4h+qG0TYR/2qLuHQIWj4amO4aLXjDQKxg8YcUb5Z63a8AKDoaXH5kiWoYt3+zNj0Sv5C9kv+/DqA4T/G63mOZXxbDkD/3WYxslZhnB4R4/qMhK+yyZNRp7BmuO1IDj320fTyZqBd4ZoHM29ihOHIr/+GRENXY+VSHFvyiZ7JMOiUwWVyR/8miBaNLblQqU5vTy0HdJmuJD4jlNaS68pnvuhSnIGuVHuYbdo9BOHemw1XYCt7T3te8C1CkMk9eGhuBlhxlFDZeKInqioaquoD7dcz7kw1tvfD5kM/XrZ4fw+E2yOP3ZY9bIkHzh9kFh+mknT3VHQ7K8BWT5OPHLoFmTtdld9q96PRVvBQMiBssckBqnxD/MFiym/498L4nN7R6E4yydkHeH9RWkPn7LMjfGJl/GbkThGXg4aViNbs0a9XpVGl+TcKKY7zZdh+Wj/OvEHZZbpmm44EcnMcyE04AMyhgVqEipB61FhIMDXWwlQRJX0wF+YKMJo0BfDjU2YEeNYL87bhslQQf4z46ZHL9EAAaqq74r5KI6ivvLK8hYpYRSkS0l3DOmNunbnfw38MxHNTZUMer7pD8quRhdHCBiSwPbj/FIKeY4/Ujt66evkASqEkKR20y1MYmE3N1VI0DusE=" - skip_cleanup: true - on: - tags: true - -addons: - apt: - packages: - - jq - - latexmk - - texlive-binaries - - texlive-fonts-recommended - - texlive-latex-base - - texlive-latex-extra - - texlive-latex-recommended - - sonarcloud: - organization: "ystia" - -cache: - directories: - - '$HOME/.sonar/cache' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e008829c..a4da98ca7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,3 +96,17 @@ If you are having trouble getting into the mood of idiomatic Go, we recommend reading through [Effective Go](https://golang.org/doc/effective_go.html). The [Go Blog](https://blog.golang.org) is also a great resource. Drinking the kool-aid is a lot easier than going thirsty. + +## Release Yorc + +Releases are now handled by a [GitHub Action Workflow](https://github.com/ystia/yorc/actions/workflows/release.yml). +Contributors with `members` role on this project can trigger this workflow. it requires as an input the release version +in semver format (leading 'v' should be omitted). + +This workflow will: + +1. call the `build/release.sh` script +2. checkout the generated tag +3. generate the distribution and a changelog for this version +4. create a GH Release and upload assets +5. publish the GH Release diff --git a/README.md b/README.md index 517e359cb..4c7e054f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Ystia Orchestrator -[![Download](https://api.bintray.com/packages/ystia/yorc-engine/distributions/images/download.svg?version=4.0.0)](https://bintray.com/ystia/yorc-engine/distributions/4.0.0/link) [![Build Status](https://travis-ci.org/ystia/yorc.svg?branch=release/4.0)](https://travis-ci.org/ystia/yorc) [![Documentation Status](https://readthedocs.org/projects/yorc/badge/?version=latest)](http://yorc.readthedocs.io/en/latest/?badge=latest) [![Go Report Card](https://goreportcard.com/badge/github.com/ystia/yorc)](https://goreportcard.com/report/github.com/ystia/yorc) [![license](https://img.shields.io/github/license/ystia/yorc.svg)](https://github.com/ystia/yorc/blob/develop/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) [![Docker Pulls](https://img.shields.io/docker/pulls/ystia/yorc.svg?style=flat)](https://hub.docker.com/r/ystia/yorc) [![Join the chat at https://gitter.im/ystia/yorc](https://badges.gitter.im/ystia/yorc.svg)](https://gitter.im/ystia/yorc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Download](https://img.shields.io/badge/download-v4.1.0-blue)](https://github.com/ystia/yorc/releases/tag/v4.1.0) [![Build Status](https://github.com/ystia/yorc/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/ystia/yorc/actions) [![Documentation Status](https://readthedocs.org/projects/yorc/badge/?version=latest)](http://yorc.readthedocs.io/en/latest/?badge=latest) [![Go Report Card](https://goreportcard.com/badge/github.com/ystia/yorc)](https://goreportcard.com/report/github.com/ystia/yorc) [![license](https://img.shields.io/github/license/ystia/yorc.svg)](https://github.com/ystia/yorc/blob/develop/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) [![Docker Pulls](https://img.shields.io/docker/pulls/ystia/yorc.svg?style=flat)](https://hub.docker.com/r/ystia/yorc) [![Join the chat at https://gitter.im/ystia/yorc](https://badges.gitter.im/ystia/yorc.svg)](https://gitter.im/ystia/yorc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

@@ -24,9 +24,9 @@ Yorc is now the official orchestrator for Alien4Cloud and Alien4Cloud distributi ## How to download the Ystia Orchestrator -Yorc releases can be downloaded from our [BinTray account](https://bintray.com/ystia/yorc-engine/distributions). +Yorc releases can be downloaded from our [GitHub Release](https://github.com/ystia/yorc/releases). -Grab the [latest release here](https://bintray.com/ystia/yorc-engine/distributions/_latestVersion). +Grab the [latest release here](https://github.com/ystia/yorc/releases/latest). Docker images could be found on [Docker Hub](https://hub.docker.com/r/ystia/yorc). diff --git a/build/bintray_release.json.tpl b/build/bintray_release.json.tpl deleted file mode 100644 index 6f8db016a..000000000 --- a/build/bintray_release.json.tpl +++ /dev/null @@ -1,36 +0,0 @@ -{ - "package": { - "name": "distributions", - "repo": "yorc-engine", - "subject": "ystia", - "desc": "Ystia Orchestrator distributions and documentations", - "website_url": "https://ystia.github.io/", - "issue_tracker_url": "https://github.com/ystia/yorc/issues", - "vcs_url": "https://github.com/ystia/yorc", - "github_use_tag_release_notes": false, - "github_release_notes_file": "CHANGELOG.md", - "licenses": ["Apache-2.0"], - "labels": [], - "public_download_numbers": true, - "public_stats": false, - "attributes": [] - }, - - "version": { - "name": "${VERSION_NAME}", - "desc": "Ystia Orchestrator distributions and documentations ${VERSION_NAME}", - "released": "${RELEASE_DATE}", - "vcs_tag": "${TAG_NAME}", - "attributes": [], - "gpgSign": false - }, - - "files": - [ - {"includePattern": "dist/(yorc-.*\\.tgz)", "uploadPattern": "${VERSION_NAME}/$1"}, - {"includePattern": "dist/(yorc-server-.*-distrib\\.zip)", "uploadPattern": "${VERSION_NAME}/$1"}, - {"includePattern": "pkg/(docker-ystia-yorc-.*\\.tgz)", "uploadPattern": "${VERSION_NAME}/$1"} - ], - "publish": true -} - diff --git a/build/deploy_artifactory.sh b/build/deploy_artifactory.sh index d368d40ab..7826dc157 100755 --- a/build/deploy_artifactory.sh +++ b/build/deploy_artifactory.sh @@ -13,43 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -euo pipefail scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" rootDir=$(readlink -f "${scriptDir}/..") -if [[ "${TRAVIS}" != "true" ]] ; then +if [[ "${GITHUB_ACTIONS}" != "true" ]] ; then echo "This script is designed to publish CI build artifacts" exit 0 fi -if [[ "${DISABLE_ARTIFACTORY}" == "true" ]] ; then +if [[ "${DISABLE_ARTIFACTORY:=false}" == "true" ]] ; then echo "Skipping Artifactory publication" exit 0 fi -if [[ "${TRAVIS_PULL_REQUEST}" != "false" ]] && [[ -z "${ARTIFACTORY_API_KEY}" ]] ; then - echo "Building an external pull request, artifactory publication is disabled" - exit 0 -fi - -if [[ -n "${TRAVIS_TAG}" ]] ; then - deploy_path="yorc-engine-product-ystia-dist/ystia/yorc/dist/${TRAVIS_TAG}/{1}" -elif [[ "${TRAVIS_PULL_REQUEST}" != "false" ]]; then - deploy_path="yorc-bin-dev-local/ystia/yorc/dist/PR-${TRAVIS_PULL_REQUEST}/{1}" +ref="${GITHUB_REF#refs/*/}" +if [[ "${GITHUB_REF}" == refs/tags/* ]] ; then + deploy_path="yorc-engine-product-ystia-dist/ystia/yorc/dist/${ref}/{1}" +elif [[ "${GITHUB_REF}" == refs/pull/* ]] ; then + # For PRs ref is different + ref=$(echo "${GITHUB_REF}" | awk -F / '{print $3;}') + deploy_path="yorc-bin-dev-local/ystia/yorc/dist/PR-${ref}/{1}" else - deploy_path="yorc-bin-dev-local/ystia/yorc/dist/${TRAVIS_BRANCH}/{1}" + deploy_path="yorc-bin-dev-local/ystia/yorc/dist/${ref}/{1}" fi -curl -fL https://getcli.jfrog.io | sh - -build_name="yorc-travis-ci" - -# Disabling interactive mode as config ask for a question about client certificates we do not use -./jfrog rt c --interactive=false --apikey="${ARTIFACTORY_API_KEY}" --user=travis --url=https://ystia.jfrog.io/ystia ystia +cd "${rootDir}" -./jfrog rt u --build-name="${build_name}" --build-number="${TRAVIS_BUILD_NUMBER}" --props="artifactory.licenses=Apache-2.0" --regexp "dist/(yorc-.*.tgz)" "${deploy_path}" -./jfrog rt u --build-name="${build_name}" --build-number="${TRAVIS_BUILD_NUMBER}" --props="artifactory.licenses=Apache-2.0" --regexp "dist/(yorc-server.*-distrib.zip)" "${deploy_path}" -# Do not publish environment variables as it may expose some secrets -#./jfrog rt bce "${build_name}" "${TRAVIS_BUILD_NUMBER}" -./jfrog rt bag "${build_name}" "${TRAVIS_BUILD_NUMBER}" "${rootDir}" -./jfrog rt bp "${build_name}" "${TRAVIS_BUILD_NUMBER}" +jfrog rt u --target-props="artifactory.licenses=Apache-2.0" --regexp "dist/(yorc-.*.tgz)" "${deploy_path}" +jfrog rt u --target-props="artifactory.licenses=Apache-2.0" --regexp "dist/(yorc-server.*-distrib.zip)" "${deploy_path}" +jfrog rt bce +jfrog rt bag +jfrog rt bp diff --git a/build/gh-action-cleanup-artifactory.sh b/build/gh-action-cleanup-artifactory.sh index 4dfe7857c..ca59e9411 100755 --- a/build/gh-action-cleanup-artifactory.sh +++ b/build/gh-action-cleanup-artifactory.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -euo pipefail diff --git a/build/pre_bintray_release.sh b/build/pre_bintray_release.sh deleted file mode 100755 index f6ce4ba04..000000000 --- a/build/pre_bintray_release.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [[ -z "${TRAVIS_TAG}" ]]; then - echo "not a travis release build, no need to publish it on bintray. Exiting." - exit 0 -fi - -TAG_NAME="${TRAVIS_TAG}" -VERSION_NAME="${TAG_NAME#v*}" -RELEASE_DATE="$(git tag -l --format='%(creatordate:short)' "${TAG_NAME}")" - -export TAG_NAME VERSION_NAME RELEASE_DATE -envsubst < "${scriptDir}/bintray_release.json.tpl" > "${scriptDir}/bintray_release.json" - -echo "Resulting bintray release spec" -cat "${scriptDir}/bintray_release.json" diff --git a/build/release.sh b/build/release.sh index 3d7fd8b02..d1c0c519b 100755 --- a/build/release.sh +++ b/build/release.sh @@ -56,13 +56,13 @@ if [[ -z "${version}" ]]; then exit 1 fi -if [[ "$(python -c "import semantic_version; print semantic_version.validate('${version}')" )" != "True" ]]; then +if [[ "$(python -c "import semantic_version; print(semantic_version.validate('${version}'))" )" != "True" ]]; then echo "Parameter -v should be a semver 2.0 compatible version (http://semver.org/)" >&2 exit 1 fi # read version -read -r major minor patch prerelease build <<< $(python -c "import semantic_version; v = semantic_version.Version('${version}'); print v.major, v.minor, v.patch, '.'.join(v.prerelease), '.'.join(v.build);") +read -r major minor patch prerelease build <<< $(python -c "import semantic_version; v = semantic_version.Version('${version}'); print(v.major, v.minor, v.patch, '.'.join(v.prerelease), '.'.join(v.build));") # Detect correct supporting branch branch=$(git branch --list -r "*/release/${major}.${minor}") @@ -79,7 +79,7 @@ if [[ -e versions.yaml ]]; then currentVersion=$(grep "${componentVersionName}:" versions.yaml | head -1 | sed -e 's/^[^:]\+:\s*\(.*\)\s*$/\1/') # Change -SNAPSHOT into -0 for comparaison as a snapshot is never revelant checkVers=$(echo ${currentVersion} | sed -e "s/-SNAPSHOT/-0/") - if [[ "True" != "$(python -c "import semantic_version; print semantic_version.Version('${version}') >= semantic_version.Version('${checkVers}')" )" ]]; then + if [[ "True" != "$(python -c "import semantic_version; print(semantic_version.Version('${version}') >= semantic_version.Version('${checkVers}'))" )" ]]; then echo "Warning: releasing version ${version} on top of branch ${branch} while its current version is ${currentVersion}" >&2 read -p "Are you sure? [y/N]" CONFIRM if [[ "${CONFIRM}" != "y" && "${CONFIRM}" != "Y" ]] ; then @@ -94,7 +94,7 @@ branchTag=$(git describe --abbrev=0 --tags ${branch}) || { } branchTag=$(echo $branchTag | sed -e 's/^v\(.*\)$/\1/') -if [[ "True" != "$(python -c "import semantic_version; print semantic_version.Version('${version}') > semantic_version.Version('${branchTag}')" )" ]]; then +if [[ "True" != "$(python -c "import semantic_version; print(semantic_version.Version('${version}') > semantic_version.Version('${branchTag}'))" )" ]]; then echo "Warning: releasing version ${version} on top of branch ${branch} while it contains a newer tag: ${branchTag}" >&2 read -p "Are you sure? [y/N]" CONFIRM if [[ "${CONFIRM}" != "y" && "${CONFIRM}" != "Y" ]] ; then @@ -107,7 +107,7 @@ if [[ "develop" == "${branch}" ]] && [[ -z "${prerelease}" ]]; then releaseBranch="release/${major}.${minor}" git checkout -b "${releaseBranch}" sed -i -e "s@svg?branch=[^)]*@svg?branch=${releaseBranch}@g" README.md - git commit -m "Update travis links in readme for release ${version}" README.md + git commit -m "Update CI links in readme for release ${version}" README.md fi # Now checks are passed then tag, build, release and cleanup :) @@ -115,7 +115,9 @@ cherries=() # Update changelog Release date sed -i -e "s/^## UNRELEASED.*$/## ${version} ($(LC_ALL=C date +'%B %d, %Y'))/g" CHANGELOG.md # Update readme for Release number -sed -i -e "s@download.svg?version=[^)]*@download.svg?version=${version}@g" -e "s@distributions/[^/]*/link@distributions/${version}/link@g" README.md +versionShield=$(echo "${version}" | sed -e 's/-/--/g') +sed -i -e "s@https://img.shields.io/badge/download-[^-]*-blue@https://img.shields.io/badge/download-v${versionShield}-blue@g" \ + -e "s@releases/tag/[^)]*@releases/tag/v${version}@g" README.md git commit -m "Update changelog and readme for release ${version}" CHANGELOG.md README.md cherries+=("$(git log -1 --pretty=format:"%h")") @@ -137,7 +139,7 @@ if [[ -e versions.yaml ]]; then nextDevelopmentVersion="" if [[ -z "${prerelease}" ]]; then # We are releasing a final version - nextDevelopmentVersion=$(python -c "import semantic_version; v=semantic_version.Version('${version}'); print v.next_patch()" ) + nextDevelopmentVersion=$(python -c "import semantic_version; v=semantic_version.Version('${version}'); print(v.next_patch())" ) nextDevelopmentVersion="${nextDevelopmentVersion}-SNAPSHOT" else # in prerelease revert to version minus prerelease plus -SNAPSHOT @@ -159,7 +161,7 @@ if [[ "develop" == "${branch}" ]] && [[ -z "${prerelease}" ]]; then if [[ -e versions.yaml ]]; then # Update version - nextDevelopmentVersion=$(python -c "import semantic_version; v=semantic_version.Version('${version}'); print v.next_minor()" ) + nextDevelopmentVersion=$(python -c "import semantic_version; v=semantic_version.Version('${version}'); print(v.next_minor())" ) nextDevelopmentVersion="${nextDevelopmentVersion}-SNAPSHOT" sed -i -e "/${componentVersionName}: /c${componentVersionName}: ${nextDevelopmentVersion}" versions.yaml git commit -m "Prepare for next development cycle ${nextDevelopmentVersion}" versions.yaml @@ -173,7 +175,7 @@ if [[ -z "${prerelease}" ]]; then } masterTag=$(echo ${masterTag} | sed -e 's/^v\(.*\)$/\1/') - if [[ "True" == "$(python -c "import semantic_version; print semantic_version.Version('${version}') > semantic_version.Version('${masterTag}')" )" ]]; then + if [[ "True" == "$(python -c "import semantic_version; print(semantic_version.Version('${version}') > semantic_version.Version('${masterTag}'))" )" ]]; then # We should merge the tag to master as it is our highest release git checkout master git merge --no-ff "v${version}" -X theirs -m "merging latest tag v${version} into master" || { diff --git a/build/travis-sonar.sh b/build/travis-sonar.sh deleted file mode 100755 index 766d14e72..000000000 --- a/build/travis-sonar.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [[ -z "${SONAR_TOKEN}" ]] ; then - echo "No sonar token detected, we are probably building an external PR, lets skip sonar publication..." - exit 0 -fi - -cd "${scriptDir}/.." || { echo "failed to move to yorc directory ${scriptDir}/.."; exit 1; } -sed -i -e "s@$(go list)@github.com/ystia/yorc@g" coverage-sonar.out -git fetch --no-tags origin "+refs/heads/develop:refs/remotes/origin/develop" "+refs/heads/release/*:refs/remotes/origin/release/*" -git fetch --unshallow --quiet -sonar-scanner --define "sonar.projectVersion=$(grep "yorc_version" versions.yaml | awk '{print $2}')" diff --git a/doc/conf.py b/doc/conf.py index a81914d26..441814b37 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,19 +53,19 @@ master_doc = 'index' # General information about the project. -project = u'Yorc' +project = 'Yorc' copyright=str(date.today().year) -copyright += u', Atos BDS R&D' -author = u'Atos BDS R&D' +copyright += ', Atos BDS R&D' +author = 'Atos BDS R&D' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'T.B.D' +version = 'T.B.D' # The full version, including alpha/beta/rc tags. -release = u'T.B.D' +release = 'T.B.D' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -242,8 +242,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Yorc.tex', u'Yorc Documentation', - u'Atos BDS R\\&D', 'manual'), + (master_doc, 'Yorc.tex', 'Yorc Documentation', + 'Atos BDS R\\&D', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -272,7 +272,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'yorc', u'Yorc Documentation', + (master_doc, 'yorc', 'Yorc Documentation', [author], 1) ] @@ -286,7 +286,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Yorc', u'Yorc Documentation', + (master_doc, 'Yorc', 'Yorc Documentation', author, 'Yorc', 'One line description of project.', 'Miscellaneous'), ] @@ -309,14 +309,15 @@ if os.path.exists('../versions.yaml'): - versions_file = yaml.load(file('../versions.yaml', 'r')) - rst_epilog = '' - for product in versions_file: - rst_epilog += ".. |" + product + "| replace:: " + versions_file[product] + "\n" - m = re.search(r"^(([0-9.]+)[^.]+)$", versions_file['yorc_version']) - if m: - release = m.group(1) - version = m.group(2) + with open('../versions.yaml', 'r') as file: + versions_file = yaml.load(file, Loader=yaml.SafeLoader) + rst_epilog = '' + for product in versions_file: + rst_epilog += ".. |" + product + "| replace:: " + versions_file[product] + "\n" + m = re.search(r"^(([0-9.]+)[^.]+)$", versions_file['yorc_version']) + if m: + release = m.group(1) + version = m.group(2) # print rst_epilog # print release diff --git a/doc/configuration.rst b/doc/configuration.rst index aa0f50418..71ba55982 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1459,7 +1459,7 @@ This store ables you to store ``Log`` s and ``Event`` s in elasticsearch. | ``key_path`` | path to a PEM encoded private key file when TLS | string | no | | | | is activated for ES | | | | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ -| ``index_prefix`` | indexes used by yorc can be prefixed | string | no | yorc_ | +| ``index_prefix`` | indexes used by yorc can be prefixed | string | no | yorc\_ | +-----------------------------+----------------------------------------------------+-----------+------------------+-----------------+ | ``es_query_period`` | when querying logs and event, we wait this timeout | duration | no | 4s | | | before each request when it returns nothing (until | | | | diff --git a/docker_build.sh b/docker_build.sh index f767a15ee..3fb8b749b 100755 --- a/docker_build.sh +++ b/docker_build.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,22 +38,28 @@ tf_aws_plugin_version=$(grep tf_aws_plugin_version ${script_dir}/versions.yaml | tf_openstack_plugin_version=$(grep tf_openstack_plugin_version ${script_dir}/versions.yaml | awk '{print $2}') tf_google_plugin_version=$(grep tf_google_plugin_version ${script_dir}/versions.yaml | awk '{print $2}') -if [[ "${TRAVIS}" == "true" ]]; then - if [[ "${TRAVIS_PULL_REQUEST}" == "false" ]] ; then - if [[ -n "${TRAVIS_TAG}" ]] ; then - DOCKER_TAG="$(echo "${TRAVIS_TAG}" | sed -e 's/^v\(.*\)$/\1/')" - else - case ${TRAVIS_BRANCH} in - develop) - DOCKER_TAG="latest";; - *) - # Do not build a container for other branches - echo "No container is built for other branches than develop." - exit 0;; - esac - fi - else - DOCKER_TAG="PR-${TRAVIS_PULL_REQUEST}" +CI_TAG="" +CI_PULL_REQUEST="" +CI_BRANCH="" + +if [[ "${GITHUB_ACTIONS}" == "true" ]] ; then + ref="${GITHUB_REF#refs/*/}" + if [[ "${GITHUB_REF}" == refs/tags/* ]] ; then + CI_TAG="${ref}" + DOCKER_TAG="$(echo "${CI_TAG}" | sed -e 's/^v\(.*\)$/\1/')" + elif [[ "${GITHUB_REF}" == refs/pull/* ]] ; then + CI_PULL_REQUEST="$(echo "${GITHUB_REF}" | awk -F / '{print $3;}')" + DOCKER_TAG="PR-${CI_PULL_REQUEST}" + else + CI_BRANCH="${ref}" + case ${CI_BRANCH} in + develop) + DOCKER_TAG="latest";; + *) + # Do not build a container for other branches + echo "No container is built for other branches than develop." + exit 0;; + esac fi fi @@ -71,18 +77,12 @@ docker build ${BUILD_ARGS} \ --build-arg "TF_GOOGLE_PLUGIN_VERSION=${tf_google_plugin_version}" \ -t "ystia/yorc:${DOCKER_TAG:-latest}" . -if [[ "${TRAVIS}" == "true" ]]; then +if [[ "${GITHUB_ACTIONS}" == "true" ]]; then docker save "ystia/yorc:${DOCKER_TAG:-latest}" | gzip > docker-ystia-yorc-${DOCKER_TAG:-latest}.tgz ls -lh docker-ystia-yorc-${DOCKER_TAG:-latest}.tgz - if [[ "${TRAVIS_PULL_REQUEST}" != "false" ]] && [[ -z "${ARTIFACTORY_API_KEY}" ]] ; then - echo "Building an external pull request, artifactory publication is disabled" - exit 0 - fi - - if [[ -n "${TRAVIS_TAG}" ]] && [[ "${DOCKER_TAG}" != *"-"* ]] ; then + if [[ -n "${CI_TAG}" ]] && [[ "${DOCKER_TAG}" != *"-"* ]] ; then ## Push Image to the Docker hub - docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASS} docker push "ystia/yorc:${DOCKER_TAG:-latest}" else ## Push Image on Artifact Docker Registry @@ -91,11 +91,9 @@ if [[ "${TRAVIS}" == "true" ]]; then exit 0 fi docker tag "ystia/yorc:${DOCKER_TAG:-latest}" "${artifactory_docker_registry}/${artifactory_docker_repo}:${DOCKER_TAG:-latest}" - curl -fL https://getcli.jfrog.io | sh - build_name="yorc-travis-ci" - ./jfrog rt c --interactive=false --user=travis --apikey="${ARTIFACTORY_API_KEY}" --url=https://ystia.jfrog.io/ystia ystia - ./jfrog rt docker-push --build-name="${build_name}" --build-number="${TRAVIS_BUILD_NUMBER}" "${artifactory_docker_registry}/${artifactory_docker_repo}:${DOCKER_TAG:-latest}" yorc-docker-dev-local - ./jfrog rt bag "${build_name}" "${TRAVIS_BUILD_NUMBER}" "${script_dir}" - ./jfrog rt bp "${build_name}" "${TRAVIS_BUILD_NUMBER}" + jfrog rt docker-push "${artifactory_docker_registry}/${artifactory_docker_repo}:${DOCKER_TAG:-latest}" yorc-docker-dev-local + jfrog rt bce + jfrog rt bag + jfrog rt bp fi fi diff --git a/sonar-project.properties b/sonar-project.properties index 65b9b8b60..32826e779 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,3 +1,4 @@ +sonar.organization=ystia sonar.projectKey=ystia_yorc sonar.projectName=Ystia Orchestrator From 3c9397d5dd78b93d03efd1c200f15c51228d778b Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Wed, 5 May 2021 21:07:21 +0200 Subject: [PATCH 13/24] Fix Bintray sunsetting --- CHANGELOG.md | 3 +++ commands/bootstrap/inputs.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b49475382..9b0b243aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ * Support Alien4Cloud 3.2.0 ([GH-723](https://github.com/ystia/yorc/issues/723)) +### BUG FIXES + +* Can't bootstrap Yorc as BinTray is now unavailable ([GH-727](https://github.com/ystia/yorc/issues/727)) ## 4.1.0 (April 11, 2021) diff --git a/commands/bootstrap/inputs.go b/commands/bootstrap/inputs.go index 52275dd13..afb51238b 100644 --- a/commands/bootstrap/inputs.go +++ b/commands/bootstrap/inputs.go @@ -1552,7 +1552,7 @@ func getYorcDownloadURL() string { yorcVersion) } else { downloadURL = fmt.Sprintf( - "https://dl.bintray.com/ystia/yorc-engine/%s/yorc-%s.tgz", + "https://github.com/ystia/yorc/releases/download/v%s/yorc-%s.tgz", yorcVersion, yorcVersion) } return downloadURL From 2182e4694c90320929216ca1aac2cd7332965aae Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Wed, 5 May 2021 22:41:14 +0200 Subject: [PATCH 14/24] Change checkout behavior for releases --- .github/workflows/build.yml | 4 ++-- .github/workflows/release.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e96f85899..a878566cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: security: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Run Snyk to check for vulnerabilities uses: snyk/actions/golang@master continue-on-error: true # To make sure that SARIF upload gets called @@ -34,7 +34,7 @@ jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: # Disabling shallow clone is recommended for improving relevancy of reporting (for sonar) fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a5214a4ac..d35449472 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,6 +19,9 @@ jobs: steps: - uses: actions/checkout@v2 + with: + # Disabling shallow clone to access git history (specially tags for comparing) + fetch-depth: 0 - name: Configure Git user run: | git config user.email "actions@github.com" From d1dd3ef0abae0cf8ab36b122e02508985c49eff1 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Thu, 6 May 2021 11:22:19 +0200 Subject: [PATCH 15/24] Use YstiaBot user to perform releases Fix the way we retrieve tags on master branch --- .github/workflows/release.yml | 6 ++++-- build/release.sh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d35449472..f3bba0acc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,10 +22,11 @@ jobs: with: # Disabling shallow clone to access git history (specially tags for comparing) fetch-depth: 0 + token: ${{ secrets.YSTIA_BOT_TOKEN }} - name: Configure Git user run: | - git config user.email "actions@github.com" - git config user.name "GitHub Actions" + git config user.email "ystiabot@users.noreply.github.com" + git config user.name "@YstiaBot" - name: Setup Python uses: actions/setup-python@v2 with: @@ -53,6 +54,7 @@ jobs: uses: actions/checkout@v2 with: ref: ${{ env.TAG_NAME }} + token: ${{ secrets.YSTIA_BOT_TOKEN }} - name: Setup Python uses: actions/setup-python@v2 diff --git a/build/release.sh b/build/release.sh index d1c0c519b..e88a4d2af 100755 --- a/build/release.sh +++ b/build/release.sh @@ -14,7 +14,7 @@ # limitations under the License. #set -x -set -e +set -eo pipefail scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" componentVersionName="yorc_version" @@ -170,7 +170,7 @@ fi if [[ -z "${prerelease}" ]]; then # Merge on master only final version - masterTag=$(git describe --abbrev=0 --tags master) || { + masterTag=$(git describe --abbrev=0 --tags origin/master) || { masterTag="v0.0.0" } masterTag=$(echo ${masterTag} | sed -e 's/^v\(.*\)$/\1/') From f6986d4f34fabce4dcfffad1bc278a2d382e180b Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Thu, 6 May 2021 11:30:44 +0200 Subject: [PATCH 16/24] Fix release workflow artifacts --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3bba0acc..a7ea21871 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,7 +96,9 @@ jobs: name: ${{ env.TAG_NAME }} prerelease: ${{ env.PRERELEASE }} draft: true - files: build/csars/*.zip + files: | + dist/yorc-.*.tgz + dist/yorc-server.*-distrib.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4b63b8ac190585d920207c2700a01b36f12013fc Mon Sep 17 00:00:00 2001 From: "@YstiaBot" Date: Thu, 6 May 2021 12:53:14 +0000 Subject: [PATCH 17/24] Update changelog and readme for release 4.2.0-milestone.1 --- CHANGELOG.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b0b243aa..39bd921b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Yorc Changelog -## UNRELEASED +## 4.2.0-milestone.1 (May 06, 2021) ### ENHANCEMENTS diff --git a/README.md b/README.md index 4c7e054f5..1d5386548 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Ystia Orchestrator -[![Download](https://img.shields.io/badge/download-v4.1.0-blue)](https://github.com/ystia/yorc/releases/tag/v4.1.0) [![Build Status](https://github.com/ystia/yorc/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/ystia/yorc/actions) [![Documentation Status](https://readthedocs.org/projects/yorc/badge/?version=latest)](http://yorc.readthedocs.io/en/latest/?badge=latest) [![Go Report Card](https://goreportcard.com/badge/github.com/ystia/yorc)](https://goreportcard.com/report/github.com/ystia/yorc) [![license](https://img.shields.io/github/license/ystia/yorc.svg)](https://github.com/ystia/yorc/blob/develop/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) [![Docker Pulls](https://img.shields.io/docker/pulls/ystia/yorc.svg?style=flat)](https://hub.docker.com/r/ystia/yorc) [![Join the chat at https://gitter.im/ystia/yorc](https://badges.gitter.im/ystia/yorc.svg)](https://gitter.im/ystia/yorc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Download](https://img.shields.io/badge/download-v4.2.0--milestone.1-blue)](https://github.com/ystia/yorc/releases/tag/v4.2.0-milestone.1) [![Build Status](https://github.com/ystia/yorc/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/ystia/yorc/actions) [![Documentation Status](https://readthedocs.org/projects/yorc/badge/?version=latest)](http://yorc.readthedocs.io/en/latest/?badge=latest) [![Go Report Card](https://goreportcard.com/badge/github.com/ystia/yorc)](https://goreportcard.com/report/github.com/ystia/yorc) [![license](https://img.shields.io/github/license/ystia/yorc.svg)](https://github.com/ystia/yorc/blob/develop/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) [![Docker Pulls](https://img.shields.io/docker/pulls/ystia/yorc.svg?style=flat)](https://hub.docker.com/r/ystia/yorc) [![Join the chat at https://gitter.im/ystia/yorc](https://badges.gitter.im/ystia/yorc.svg)](https://gitter.im/ystia/yorc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

From e1fd5bc47f281201e14e36ec5e38205f9c5f3230 Mon Sep 17 00:00:00 2001 From: "@YstiaBot" Date: Thu, 6 May 2021 12:53:14 +0000 Subject: [PATCH 18/24] Prepare release 4.2.0-milestone.1 --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 782b341f1..949908bf9 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,4 +1,4 @@ -yorc_version: 4.2.0-SNAPSHOT +yorc_version: 4.2.0-milestone.1 # Alien4Cloud version used by the bootstrap as the default version to download/install alien4cloud_version: 3.2.0 consul_version: 1.2.3 From d9ed3c1371b5827c24afc0813c124cb5c04ec2c5 Mon Sep 17 00:00:00 2001 From: "@YstiaBot" Date: Thu, 6 May 2021 12:53:14 +0000 Subject: [PATCH 19/24] Update changelog for future release --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39bd921b8..2b4fe703b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Yorc Changelog +## UNRELEASED + ## 4.2.0-milestone.1 (May 06, 2021) ### ENHANCEMENTS From 0730853cc66e1b024e45e55d995166103a0f385e Mon Sep 17 00:00:00 2001 From: "@YstiaBot" Date: Thu, 6 May 2021 12:53:14 +0000 Subject: [PATCH 20/24] Prepare for next development cycle 4.2.0-SNAPSHOT --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 949908bf9..782b341f1 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,4 +1,4 @@ -yorc_version: 4.2.0-milestone.1 +yorc_version: 4.2.0-SNAPSHOT # Alien4Cloud version used by the bootstrap as the default version to download/install alien4cloud_version: 3.2.0 consul_version: 1.2.3 From 3716cdafa1ffb8d13b3b52a6e9499ec682302bf0 Mon Sep 17 00:00:00 2001 From: Albertin Loic Date: Thu, 6 May 2021 14:59:49 +0200 Subject: [PATCH 21/24] Fix release workflow artifacts pattern --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7ea21871..910b2ac47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,8 +97,8 @@ jobs: prerelease: ${{ env.PRERELEASE }} draft: true files: | - dist/yorc-.*.tgz - dist/yorc-server.*-distrib.zip + dist/yorc-*.tgz + dist/yorc-server*-distrib.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d6cee0f6715df642653b916c42e7b7df4a4f057f Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Thu, 6 May 2021 20:33:27 +0000 Subject: [PATCH 22/24] Fixed async action status when another step failed --- prov/scheduling/scheduler/consul_test.go | 3 +++ prov/scheduling/scheduler/scheduler_test.go | 28 +++++++++++++++++++++ prov/scheduling/scheduling.go | 15 ++++++++--- tasks/workflow/worker.go | 5 ++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/prov/scheduling/scheduler/consul_test.go b/prov/scheduling/scheduler/consul_test.go index 3c77cd919..81069d889 100644 --- a/prov/scheduling/scheduler/consul_test.go +++ b/prov/scheduling/scheduler/consul_test.go @@ -71,5 +71,8 @@ func TestRunConsulSchedulingPackageTests(t *testing.T) { t.Run("testUnregisterAction", func(t *testing.T) { testUnregisterAction(t, client) }) + t.Run("testUpdateActionData", func(t *testing.T) { + testUpdateActionData(t, client) + }) }) } diff --git a/prov/scheduling/scheduler/scheduler_test.go b/prov/scheduling/scheduler/scheduler_test.go index f7629db87..d52fce253 100644 --- a/prov/scheduling/scheduler/scheduler_test.go +++ b/prov/scheduling/scheduler/scheduler_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" + "gotest.tools/v3/assert" "github.com/ystia/yorc/v4/events" "github.com/ystia/yorc/v4/helper/consulutil" @@ -368,3 +369,30 @@ func testUnregisterAction(t *testing.T, client *api.Client) { require.NotNil(t, kvp, "kvp is nil") require.Equal(t, "true", string(kvp.Value), "unregisterFlag is not set to true") } + +func testUpdateActionData(t *testing.T, client *api.Client) { + t.Parallel() + deploymentID := "dep-" + t.Name() + ti := 1 * time.Second + actionType := "test-action" + action := &prov.Action{ActionType: actionType, Data: map[string]string{"key1": "val1", "key2": "val2", "key3": "val3"}} + id, err := scheduling.RegisterAction(client, deploymentID, ti, action) + assert.NilError(t, err, "Failed to register action") + + err = scheduling.UpdateActionData(client, id, "key2", "newVal") + assert.NilError(t, err, "Failed to update action data") + + testSched := scheduler{cc: client} + newAction, err := testSched.buildScheduledAction(id) + assert.NilError(t, err, "Failed to build action") + + val := newAction.Data["key2"] + assert.Equal(t, val, "newVal", "Unexpected value for action key updated") + + // Check the update of an unregistered action, should fail + err = testSched.unregisterAction(id) + assert.NilError(t, err, "Failed to unregister action") + + err = scheduling.UpdateActionData(client, id, "key3", "newVal") + assert.ErrorContains(t, err, "unregistered") +} diff --git a/prov/scheduling/scheduling.go b/prov/scheduling/scheduling.go index 770fbc37c..a5a3a26d5 100644 --- a/prov/scheduling/scheduling.go +++ b/prov/scheduling/scheduling.go @@ -22,7 +22,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/pkg/errors" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" "github.com/ystia/yorc/v4/helper/consulutil" "github.com/ystia/yorc/v4/log" @@ -104,7 +104,16 @@ func UnregisterAction(client *api.Client, id string) error { // UpdateActionData updates the value of a given data within an action func UpdateActionData(client *api.Client, id, key, value string) error { - //TODO check if action exists - scaKeyPath := path.Join(consulutil.SchedulingKVPrefix, "actions", id, "data", key) + // check if action still exists + actionIdPrefix := path.Join(consulutil.SchedulingKVPrefix, "actions", id) + kvp, _, err := client.KV().Get(path.Join(actionIdPrefix, "deploymentID"), nil) + if err != nil { + return err + } + if kvp == nil { + return errors.Errorf("Action with ID %s is unregistered", id) + } + + scaKeyPath := path.Join(actionIdPrefix, "data", key) return errors.Wrapf(consulutil.StoreConsulKeyAsString(scaKeyPath, value), "Failed to update data %q for action %q", key, id) } diff --git a/tasks/workflow/worker.go b/tasks/workflow/worker.go index 078d0a00d..542c42a63 100644 --- a/tasks/workflow/worker.go +++ b/tasks/workflow/worker.go @@ -447,6 +447,7 @@ func (w *worker) runAction(ctx context.Context, t *taskExecution) error { } } wasCancelled := new(bool) + taskFailure := new(bool) if action.AsyncOperation.TaskID != "" { ctx = operations.SetOperationLogFields(ctx, action.AsyncOperation.Operation) ctx = events.AddLogOptionalFields(ctx, events.LogOptionalFields{ @@ -467,6 +468,7 @@ func (w *worker) runAction(ctx context.Context, t *taskExecution) error { tasks.UpdateTaskStepWithStatus(action.AsyncOperation.TaskID, action.AsyncOperation.StepName, tasks.TaskStepStatusCANCELED) }) tasks.MonitorTaskFailure(ctx, action.AsyncOperation.TaskID, func() { + *taskFailure = true // Unregister this action asap to prevent new schedulings scheduling.UnregisterAction(w.consulClient, action.ID) @@ -495,6 +497,9 @@ func (w *worker) runAction(ctx context.Context, t *taskExecution) error { if deregister || *wasCancelled { scheduling.UnregisterAction(w.consulClient, action.ID) w.endAction(ctx, t, action, *wasCancelled, err) + } else if *taskFailure { + err = errors.Errorf("Stopped on task failure") + w.endAction(ctx, t, action, *wasCancelled, err) } if err != nil { return err From 6ce2383ed3a7c2e210526d4e32b4441c81e40b53 Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Thu, 6 May 2021 21:01:14 +0000 Subject: [PATCH 23/24] Updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4fe703b..44360fa90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## UNRELEASED +### BUG FIXES + +* Workflow with asynchronous action never stops after another step failure ([GH-733](https://github.com/ystia/yorc/issues/733)) + ## 4.2.0-milestone.1 (May 06, 2021) ### ENHANCEMENTS From ef8e3614559029872d58d4ceb32e82d8a9a68b96 Mon Sep 17 00:00:00 2001 From: Laurent Ganne Date: Mon, 10 May 2021 12:55:23 +0000 Subject: [PATCH 24/24] Added openstack compute instance user_data property --- CHANGELOG.md | 4 ++++ data/tosca/yorc-openstack-types.yml | 6 +++++- prov/terraform/openstack/osinstance.go | 5 ++++- prov/terraform/openstack/osinstance_test.go | 2 +- prov/terraform/openstack/resources.go | 1 + prov/terraform/openstack/testdata/simpleOSInstance.yaml | 1 + 6 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44360fa90..ad89d3a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## UNRELEASED +### ENHANCEMENTS + +* Add the ability to define OpenStack Compute Instance user_data ([GH-735](https://github.com/ystia/yorc/issues/735)) + ### BUG FIXES * Workflow with asynchronous action never stops after another step failure ([GH-733](https://github.com/ystia/yorc/issues/733)) diff --git a/data/tosca/yorc-openstack-types.yml b/data/tosca/yorc-openstack-types.yml index 0c0f0ef13..eda6cd3f7 100644 --- a/data/tosca/yorc-openstack-types.yml +++ b/data/tosca/yorc-openstack-types.yml @@ -3,7 +3,7 @@ tosca_definitions_version: yorc_tosca_simple_yaml_1_0 metadata: template_name: yorc-openstack-types template_author: yorc - template_version: 1.2.1 + template_version: 1.3.0 imports: - yorc: @@ -144,6 +144,10 @@ node_types: entry_schema: type: string required: false + user_data: + type: string + description: User data to provide when launching the instance + required: false requirements: - group: capability: yorc.capabilities.Group diff --git a/prov/terraform/openstack/osinstance.go b/prov/terraform/openstack/osinstance.go index e89e10139..592c01fe0 100644 --- a/prov/terraform/openstack/osinstance.go +++ b/prov/terraform/openstack/osinstance.go @@ -172,11 +172,14 @@ func generateComputeInstance(ctx context.Context, opts osInstanceOptions) (Compu if toscaVal != nil && toscaVal.RawString() != "" { err = json.Unmarshal([]byte(toscaVal.RawString()), &instance.Metadata) if err != nil { - err = errors.Wrapf(err, "Expected a map of strings for the metadata value of node %s instance %s, got: %s", + return instance, errors.Wrapf(err, "Expected a map of strings for the metadata value of node %s instance %s, got: %s", opts.nodeName, opts.instanceName, toscaVal.RawString()) } } + instance.UserData, err = deployments.GetStringNodeProperty(ctx, opts.deploymentID, + opts.nodeName, "user_data", false) + return instance, err } diff --git a/prov/terraform/openstack/osinstance_test.go b/prov/terraform/openstack/osinstance_test.go index ec2e668a1..276ce47c8 100644 --- a/prov/terraform/openstack/osinstance_test.go +++ b/prov/terraform/openstack/osinstance_test.go @@ -97,7 +97,7 @@ func testSimpleOSInstance(t *testing.T) { require.Len(t, compute.Metadata, 2) require.Equal(t, "firstValue", compute.Metadata["firstKey"]) require.Equal(t, "secondValue", compute.Metadata["secondKey"]) - + require.Contains(t, compute.UserData, "cloud-config") require.Len(t, compute.Provisioners, 0) require.Contains(t, infrastructure.Resource, "null_resource") require.Len(t, infrastructure.Resource["null_resource"], 1) diff --git a/prov/terraform/openstack/resources.go b/prov/terraform/openstack/resources.go index 09469c465..4e6a6371d 100644 --- a/prov/terraform/openstack/resources.go +++ b/prov/terraform/openstack/resources.go @@ -46,6 +46,7 @@ type ComputeInstance struct { KeyPair string `json:"key_pair,omitempty"` SchedulerHints SchedulerHints `json:"scheduler_hints,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` + UserData string `json:"user_data,omitempty"` commons.Resource } diff --git a/prov/terraform/openstack/testdata/simpleOSInstance.yaml b/prov/terraform/openstack/testdata/simpleOSInstance.yaml index 115b361b5..b60a5cf96 100644 --- a/prov/terraform/openstack/testdata/simpleOSInstance.yaml +++ b/prov/terraform/openstack/testdata/simpleOSInstance.yaml @@ -26,6 +26,7 @@ topology_template: key_pair: yorc security_groups: openbar,default metadata: {get_input: vm_metadata} + user_data: "#cloud-config\nwrite_files:\n- content: |\n test\n owner: root:root\n path: /etc/test.txt\n permissions: '0644'" capabilities: endpoint: properties: