feat: support publiccode.yml Standard v0.3.0 (#274)

crawler/crawler/crawler.go

@@ -23,7 +23,7 @@ import (
 	"github.com/italia/developers-italia-backend/crawler/jekyll"
 	"github.com/italia/developers-italia-backend/crawler/metrics"
 	httpclient "github.com/italia/httpclient-lib-go"
-	publiccode "github.com/italia/publiccode-parser-go/v2"
+	publiccode "github.com/italia/publiccode-parser-go/v3"
 	es "github.com/olivere/elastic/v7"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
@@ -401,11 +401,23 @@ func (c *Crawler) ProcessRepo(repository Repository) {
 		}
 		err = parser.ParseInDomain(resp.Body, repository.Domain.Host, repository.Domain.UseTokenFor, repository.Domain.BasicAuth)
 		if err != nil {
-			message = fmt.Sprintf("[%s] BAD publiccode.yml: %+v\n", repository.Name, err)
-			log.Errorf(message)
-			addLogEntry(&logEntries, message)
+			valid := true
+		out:
+			for _, res := range err.(publiccode.ValidationResults) {
+				switch res.(type) {
+				case publiccode.ValidationError:
+					valid = false
+					break out
+				}
+			}
 
-			return
+			if !valid {
+				message = fmt.Sprintf("[%s] BAD publiccode.yml: %+v\n", repository.Name, err)
+				log.Errorf(message)
+				addLogEntry(&logEntries, message)
+
+				return
+			}
 		}
 
 		// HACK: Publishers named "_"" are special and get to skip the additional checks.

crawler/crawler/crawler_test.go

@@ -5,7 +5,7 @@ import (
     "net/url"
 	"testing"
 
-	publiccode "github.com/italia/publiccode-parser-go/v2"
+	publiccode "github.com/italia/publiccode-parser-go/v3"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	yaml "gopkg.in/yaml.v2"

crawler/crawler/saveToElasticsearch.go

@@ -14,7 +14,7 @@ import (
 	"github.com/italia/developers-italia-backend/crawler/ipa"
 	"github.com/italia/developers-italia-backend/crawler/metrics"
 	"github.com/alranel/go-vcsurl/v2"
-	publiccode "github.com/italia/publiccode-parser-go/v2"
+	publiccode "github.com/italia/publiccode-parser-go/v3"
 	elastic "github.com/olivere/elastic/v7"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"

crawler/go.mod

@@ -5,14 +5,14 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/icza/dyno v0.0.0-20200205103839-49cb13720835
 	github.com/italia/httpclient-lib-go v0.0.0-20201009133728-9044482688d7
-	github.com/italia/publiccode-parser-go/v2 v2.0.0
+	github.com/italia/publiccode-parser-go/v3 v3.0.0
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/olivere/elastic/v7 v7.0.27
 	github.com/prometheus/client_golang v1.7.1
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/thoas/go-funk v0.7.0
 	github.com/xanzy/go-gitlab v0.57.0
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -21,17 +21,16 @@ require (
 
 require (
 	github.com/Jeffail/gabs v1.4.0 // indirect
-	github.com/alranel/go-spdx v0.0.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deadcheat/goblet v1.3.1 // indirect
 	github.com/dyatlov/go-oembed v0.0.0-20180429203341-4bc5ab7a42e9 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.7 // indirect
-	github.com/go-playground/locales v0.13.0 // indirect
-	github.com/go-playground/universal-translator v0.17.0 // indirect
-	github.com/go-playground/validator/v10 v10.4.1 // indirect
+	github.com/go-playground/locales v0.14.0 // indirect
+	github.com/go-playground/universal-translator v0.18.0 // indirect
+	github.com/go-playground/validator/v10 v10.10.1 // indirect
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
@@ -41,7 +40,8 @@ require (
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
-	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/kyoh86/go-spdx v0.0.5-0.20220421143955-2f42f2d4c410 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
@@ -64,18 +64,18 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	github.com/xanzy/ssh-agent v0.2.1 // indirect
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
-	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
 go 1.17