add sonar metrics in scanning job

Signed-off-by: Patrick Zhao <[email protected]>

pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go

@@ -615,6 +615,7 @@ func (e *Events) Error(message string) {
 type StepTask struct {
 	Name      string          `bson:"name"           json:"name"         yaml:"name"`
 	JobName   string          `bson:"job_name"       json:"job_name"     yaml:"job_name"`
+	JobKey    string          `bson:"job_key"        json:"job_key"      yaml:"job_key"`
 	Error     string          `bson:"error"          json:"error"        yaml:"error"`
 	StepType  config.StepType `bson:"type"           json:"type"         yaml:"type"`
 	Onfailure bool            `bson:"on_failure"     json:"on_failure"   yaml:"on_failure"`

pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go

@@ -92,7 +92,7 @@ func instantiateStepCtl(step *commonmodels.StepTask, workflowCtx *commonmodels.W
 	case config.StepTarArchive:
 		stepCtl, err = NewTarArchiveCtl(step, logger)
 	case config.StepSonarCheck:
-		stepCtl, err = NewSonarCheckCtl(step, logger)
+		stepCtl, err = NewSonarCheckCtl(step, workflowCtx, logger)
 	case config.StepDistributeImage:
 		stepCtl, err = NewDistributeCtl(step, workflowCtx, jobName, logger)
 	case config.StepDebugBefore, config.StepDebugAfter:

pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_sonar_check.go

@@ -24,16 +24,21 @@ import (
 	"gopkg.in/yaml.v2"
 
 	commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models"
+	"github.com/koderover/zadig/v2/pkg/setting"
+	"github.com/koderover/zadig/v2/pkg/tool/log"
+	"github.com/koderover/zadig/v2/pkg/tool/sonar"
+	"github.com/koderover/zadig/v2/pkg/types/job"
 	"github.com/koderover/zadig/v2/pkg/types/step"
 )
 
 type sonarCheckCtl struct {
 	step           *commonmodels.StepTask
 	sonarCheckSpec *step.StepSonarCheckSpec
 	log            *zap.SugaredLogger
+	workflowCtx    *commonmodels.WorkflowTaskCtx
 }
 
-func NewSonarCheckCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*sonarCheckCtl, error) {
+func NewSonarCheckCtl(stepTask *commonmodels.StepTask, workflowCtx *commonmodels.WorkflowTaskCtx, log *zap.SugaredLogger) (*sonarCheckCtl, error) {
 	yamlString, err := yaml.Marshal(stepTask.Spec)
 	if err != nil {
 		return nil, fmt.Errorf("marshal sonar check spec error: %v", err)
@@ -43,13 +48,38 @@ func NewSonarCheckCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (
 		return nil, fmt.Errorf("unmarshal sonar check error: %v", err)
 	}
 	stepTask.Spec = sonarCheckSpec
-	return &sonarCheckCtl{sonarCheckSpec: sonarCheckSpec, log: log, step: stepTask}, nil
+	return &sonarCheckCtl{sonarCheckSpec: sonarCheckSpec, log: log, step: stepTask, workflowCtx: workflowCtx}, nil
 }
 
 func (s *sonarCheckCtl) PreRun(ctx context.Context) error {
 	return nil
 }
 
 func (s *sonarCheckCtl) AfterRun(ctx context.Context) error {
+	key := job.GetJobOutputKey(s.step.JobKey, setting.WorkflowScanningJobOutputKey)
+	id, ok := s.workflowCtx.GlobalContextGet(key)
+	if !ok {
+		err := fmt.Errorf("sonar check job output %s not found", key)
+		log.Error(err)
+		return err
+	}
+
+	client := sonar.NewSonarClient(s.sonarCheckSpec.SonarServer, s.sonarCheckSpec.SonarToken)
+	taskInfo, err := client.GetCETaskInfo(id)
+	if err != nil {
+		err = fmt.Errorf("get ce task %s info error: %v", id, err)
+		log.Error(err)
+		return err
+	}
+	resp, err := client.GetComponentMeasures(taskInfo.Task.ComponentKey)
+	if err != nil {
+		err = fmt.Errorf("get component measures error: %v", err)
+		log.Error(err)
+		return err
+	}
+	for _, mesures := range resp.Component.Measures {
+		_ = mesures
+	}
+
 	return nil
 }

pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go

@@ -338,7 +338,7 @@ func (j *ScanningJob) GetOutPuts(log *zap.SugaredLogger) []string {
 				}
 			}
 		}
-		resp = append(resp, getOutputKey(jobKey, scanningInfo.Outputs)...)
+		resp = append(resp, getOutputKey(jobKey, ensureScanningOutputs(scanningInfo.Outputs))...)
 	}
 	return resp
 }
@@ -390,7 +390,7 @@ func (j *ScanningJob) toJobTask(scanning *commonmodels.ScanningModule, taskID in
 		JobType:        string(config.JobZadigScanning),
 		Spec:           jobTaskSpec,
 		Timeout:        timeout,
-		Outputs:        scanningInfo.Outputs,
+		Outputs:        ensureScanningOutputs(scanningInfo.Outputs),
 		Infrastructure: scanningInfo.Infrastructure,
 		VMLabels:       scanningInfo.VMLabels,
 	}
@@ -635,20 +635,20 @@ func (j *ScanningJob) toJobTask(scanning *commonmodels.ScanningModule, taskID in
 			jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarScriptStep)
 		}
 
-		if scanningInfo.CheckQualityGate {
-			sonarChekStep := &commonmodels.StepTask{
-				Name:     scanning.Name + "-sonar-check",
-				JobName:  jobTask.Name,
-				StepType: config.StepSonarCheck,
-				Spec: &step.StepSonarCheckSpec{
-					Parameter:   scanningInfo.Parameter,
-					CheckDir:    repoName,
-					SonarToken:  sonarInfo.Token,
-					SonarServer: sonarInfo.ServerAddress,
-				},
-			}
-			jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarChekStep)
+		sonarChekStep := &commonmodels.StepTask{
+			Name:     scanning.Name + "-sonar-check",
+			JobName:  jobTask.Name,
+			JobKey:   jobTask.Key,
+			StepType: config.StepSonarCheck,
+			Spec: &step.StepSonarCheckSpec{
+				Parameter:        scanningInfo.Parameter,
+				CheckDir:         repoName,
+				SonarToken:       sonarInfo.Token,
+				SonarServer:      sonarInfo.ServerAddress,
+				CheckQualityGate: scanningInfo.CheckQualityGate,
+			},
 		}
+		jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarChekStep)
 	} else {
 		scriptStep := &commonmodels.StepTask{
 			JobName: jobTask.Name,
@@ -818,3 +818,16 @@ func fillScanningDetail(moduleScanning *commonmodels.Scanning) error {
 func getScanningJobCacheObjectPath(workflowName, scanningName string) string {
 	return fmt.Sprintf("%s/cache/%s", workflowName, scanningName)
 }
+
+func ensureScanningOutputs(outputs []*commonmodels.Output) []*commonmodels.Output {
+	keyMap := map[string]struct{}{}
+	for _, output := range outputs {
+		keyMap[output.Name] = struct{}{}
+	}
+	if _, ok := keyMap[setting.WorkflowScanningJobOutputKey]; !ok {
+		outputs = append(outputs, &commonmodels.Output{
+			Name: setting.WorkflowScanningJobOutputKey,
+		})
+	}
+	return outputs
+}

pkg/microservice/jobexecutor/core/service/job.go

@@ -179,7 +179,7 @@ func (j *Job) collectJobResult(ctx context.Context) error {
 func (j *Job) getJobOutputVars(ctx context.Context) ([]*job.JobOutput, error) {
 	outputs := []*job.JobOutput{}
 	for _, outputName := range j.Ctx.Outputs {
-		fileContents, err := ioutil.ReadFile(filepath.Join(job.JobOutputDir, outputName))
+		fileContents, err := os.ReadFile(filepath.Join(job.JobOutputDir, outputName))
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {

pkg/microservice/jobexecutor/core/service/step/step_sonar_check.go

@@ -20,16 +20,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/koderover/zadig/v2/pkg/setting"
 	"gopkg.in/yaml.v2"
 
+	"github.com/koderover/zadig/v2/pkg/setting"
 	"github.com/koderover/zadig/v2/pkg/tool/log"
 	"github.com/koderover/zadig/v2/pkg/tool/sonar"
+	"github.com/koderover/zadig/v2/pkg/types/job"
 	"github.com/koderover/zadig/v2/pkg/types/step"
+	"github.com/koderover/zadig/v2/pkg/util"
 )
 
 type SonarCheckStep struct {
@@ -62,7 +64,7 @@ func (s *SonarCheckStep) Run(ctx context.Context) error {
 		sonarWorkDir = filepath.Join(s.workspace, s.spec.CheckDir, sonarWorkDir)
 	}
 	taskReportDir := filepath.Join(sonarWorkDir, "report-task.txt")
-	bytes, err := ioutil.ReadFile(taskReportDir)
+	bytes, err := os.ReadFile(taskReportDir)
 	if err != nil {
 		log.Errorf("read sonar task report file: %s error :%v", time.Now().Format(setting.WorkflowTimeFormat), taskReportDir, err)
 		return err
@@ -73,20 +75,31 @@ func (s *SonarCheckStep) Run(ctx context.Context) error {
 		log.Error("can not get sonar ce task ID")
 		return errors.New("can not get sonar ce task ID")
 	}
-	analysisID, err := client.WaitForCETaskTobeDone(ceTaskID, time.Minute*10)
-	if err != nil {
-		log.Error(err)
-		return err
-	}
-	gateInfo, err := client.GetQualityGateInfo(analysisID)
+
+	outputFileName := filepath.Join(job.JobOutputDir, setting.WorkflowScanningJobOutputKey)
+	err = util.AppendToFile(outputFileName, ceTaskID)
 	if err != nil {
+		err = fmt.Errorf("append sonar ce task ID %s to output file %s error: %v", ceTaskID, outputFileName, err)
 		log.Error(err)
 		return err
 	}
-	log.Infof("Sonar quality gate status: %s", gateInfo.ProjectStatus.Status)
-	sonar.PrintSonarConditionTables(gateInfo.ProjectStatus.Conditions)
-	if gateInfo.ProjectStatus.Status != sonar.QualityGateOK && gateInfo.ProjectStatus.Status != sonar.QualityGateNone {
-		return fmt.Errorf("sonar quality gate status was: %s", gateInfo.ProjectStatus.Status)
+
+	if s.spec.CheckQualityGate {
+		analysisID, err := client.WaitForCETaskTobeDone(ceTaskID, time.Minute*10)
+		if err != nil {
+			log.Error(err)
+			return err
+		}
+		gateInfo, err := client.GetQualityGateInfo(analysisID)
+		if err != nil {
+			log.Error(err)
+			return err
+		}
+		log.Infof("Sonar quality gate status: %s", gateInfo.ProjectStatus.Status)
+		sonar.PrintSonarConditionTables(gateInfo.ProjectStatus.Conditions)
+		if gateInfo.ProjectStatus.Status != sonar.QualityGateOK && gateInfo.ProjectStatus.Status != sonar.QualityGateNone {
+			return fmt.Errorf("sonar quality gate status was: %s", gateInfo.ProjectStatus.Status)
+		}
 	}
 	return nil
 }

pkg/setting/consts.go

@@ -915,3 +915,7 @@ const (
 	SQLExecStatusFailed  SQLExecStatus = "failed"
 	SQLExecStatusNotExec SQLExecStatus = "not_exec"
 )
+
+const (
+	WorkflowScanningJobOutputKey = "SonarCETaskID"
+)

pkg/tool/sonar/sonar.go

@@ -81,6 +81,22 @@ type CETask struct {
 	Status         CETaskStatus `json:"status"`
 	SubmitterLogin string       `json:"submitterLogin"`
 	WarningCount   int          `json:"warningCount"`
+	SubmittedAt    string       `json:"submittedAt"`
+	StartedAt      string       `json:"startedAt"`
+	ExecutedAt     string       `json:"executedAt"`
+}
+
+type MeasuresComponentResponse struct {
+	Component Component `json:"component"`
+}
+
+type Component struct {
+	Measures []Measure `json:"measures"`
+}
+
+type Measure struct {
+	Metric string `json:"metric"`
+	Value  string `json:"value"`
 }
 
 func (c *Client) GetCETaskInfo(taskID string) (*CETaskInfo, error) {
@@ -92,6 +108,15 @@ func (c *Client) GetCETaskInfo(taskID string) (*CETaskInfo, error) {
 	return res, nil
 }
 
+func (c *Client) GetComponentMeasures(componentKey string) (*MeasuresComponentResponse, error) {
+	url := "/api/measures/component"
+	resp := &MeasuresComponentResponse{}
+	if _, err := c.Client.Get(url, httpclient.SetQueryParam("component", componentKey), httpclient.SetQueryParam("metricKeys", "ncloc,bugs,vulnerabilities,code_smells,coverage"), httpclient.SetResult(resp)); err != nil {
+		return nil, fmt.Errorf("search sonar component measures: component %s, error: %v", componentKey, err)
+	}
+	return resp, nil
+}
+
 type QualityGateStatus string
 
 const (

pkg/types/step/step_sonar_check.go

@@ -17,8 +17,9 @@ limitations under the License.
 package step
 
 type StepSonarCheckSpec struct {
-	Parameter   string `bson:"parameter"       json:"parameter"         yaml:"parameter"`
-	SonarToken  string `bson:"sonar_token"     json:"sonar_token"       yaml:"sonar_token"`
-	SonarServer string `bson:"sonar_server"    json:"sonar_server"      yaml:"sonar_server"`
-	CheckDir    string `bson:"check_dir"       json:"check_dir"         yaml:"check_dir"`
+	Parameter        string `bson:"parameter"          json:"parameter"          yaml:"parameter"`
+	SonarToken       string `bson:"sonar_token"        json:"sonar_token"        yaml:"sonar_token"`
+	SonarServer      string `bson:"sonar_server"       json:"sonar_server"       yaml:"sonar_server"`
+	CheckDir         string `bson:"check_dir"          json:"check_dir"          yaml:"check_dir"`
+	CheckQualityGate bool   `bson:"check_quality_gate" json:"check_quality_gate" yaml:"check_quality_gate"`
 }

pkg/util/file.go

@@ -95,3 +95,18 @@ func PathExists(path string) (bool, error) {
 	}
 	return false, err
 }
+
+func AppendToFile(filename string, data string) error {
+	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}