diff --git a/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go index 8e3b0cc7d2..72a6fae61b 100644 --- a/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go +++ b/pkg/microservice/aslan/core/common/repository/models/wokflow_task_v4.go @@ -615,6 +615,7 @@ func (e *Events) Error(message string) { type StepTask struct { Name string `bson:"name" json:"name" yaml:"name"` JobName string `bson:"job_name" json:"job_name" yaml:"job_name"` + JobKey string `bson:"job_key" json:"job_key" yaml:"job_key"` Error string `bson:"error" json:"error" yaml:"error"` StepType config.StepType `bson:"type" json:"type" yaml:"type"` Onfailure bool `bson:"on_failure" json:"on_failure" yaml:"on_failure"` diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go index 25b2189cb5..8fd61d9491 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step.go @@ -92,7 +92,7 @@ func instantiateStepCtl(step *commonmodels.StepTask, workflowCtx *commonmodels.W case config.StepTarArchive: stepCtl, err = NewTarArchiveCtl(step, logger) case config.StepSonarCheck: - stepCtl, err = NewSonarCheckCtl(step, logger) + stepCtl, err = NewSonarCheckCtl(step, workflowCtx, logger) case config.StepDistributeImage: stepCtl, err = NewDistributeCtl(step, workflowCtx, jobName, logger) case config.StepDebugBefore, config.StepDebugAfter: diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_sonar_check.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_sonar_check.go index 489465b235..c1e9221a8d 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_sonar_check.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/stepcontroller/step_sonar_check.go @@ -24,6 +24,10 @@ import ( "gopkg.in/yaml.v2" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/v2/pkg/setting" + "github.com/koderover/zadig/v2/pkg/tool/log" + "github.com/koderover/zadig/v2/pkg/tool/sonar" + "github.com/koderover/zadig/v2/pkg/types/job" "github.com/koderover/zadig/v2/pkg/types/step" ) @@ -31,9 +35,10 @@ type sonarCheckCtl struct { step *commonmodels.StepTask sonarCheckSpec *step.StepSonarCheckSpec log *zap.SugaredLogger + workflowCtx *commonmodels.WorkflowTaskCtx } -func NewSonarCheckCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) (*sonarCheckCtl, error) { +func NewSonarCheckCtl(stepTask *commonmodels.StepTask, workflowCtx *commonmodels.WorkflowTaskCtx, log *zap.SugaredLogger) (*sonarCheckCtl, error) { yamlString, err := yaml.Marshal(stepTask.Spec) if err != nil { return nil, fmt.Errorf("marshal sonar check spec error: %v", err) @@ -43,7 +48,7 @@ func NewSonarCheckCtl(stepTask *commonmodels.StepTask, log *zap.SugaredLogger) ( return nil, fmt.Errorf("unmarshal sonar check error: %v", err) } stepTask.Spec = sonarCheckSpec - return &sonarCheckCtl{sonarCheckSpec: sonarCheckSpec, log: log, step: stepTask}, nil + return &sonarCheckCtl{sonarCheckSpec: sonarCheckSpec, log: log, step: stepTask, workflowCtx: workflowCtx}, nil } func (s *sonarCheckCtl) PreRun(ctx context.Context) error { @@ -51,5 +56,30 @@ func (s *sonarCheckCtl) PreRun(ctx context.Context) error { } func (s *sonarCheckCtl) AfterRun(ctx context.Context) error { + key := job.GetJobOutputKey(s.step.JobKey, setting.WorkflowScanningJobOutputKey) + id, ok := s.workflowCtx.GlobalContextGet(key) + if !ok { + err := fmt.Errorf("sonar check job output %s not found", key) + log.Error(err) + return err + } + + client := sonar.NewSonarClient(s.sonarCheckSpec.SonarServer, s.sonarCheckSpec.SonarToken) + taskInfo, err := client.GetCETaskInfo(id) + if err != nil { + err = fmt.Errorf("get ce task %s info error: %v", id, err) + log.Error(err) + return err + } + resp, err := client.GetComponentMeasures(taskInfo.Task.ComponentKey) + if err != nil { + err = fmt.Errorf("get component measures error: %v", err) + log.Error(err) + return err + } + for _, mesures := range resp.Component.Measures { + _ = mesures + } + return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go index 12922fc244..2f18e35929 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go @@ -338,7 +338,7 @@ func (j *ScanningJob) GetOutPuts(log *zap.SugaredLogger) []string { } } } - resp = append(resp, getOutputKey(jobKey, scanningInfo.Outputs)...) + resp = append(resp, getOutputKey(jobKey, ensureScanningOutputs(scanningInfo.Outputs))...) } return resp } @@ -390,7 +390,7 @@ func (j *ScanningJob) toJobTask(scanning *commonmodels.ScanningModule, taskID in JobType: string(config.JobZadigScanning), Spec: jobTaskSpec, Timeout: timeout, - Outputs: scanningInfo.Outputs, + Outputs: ensureScanningOutputs(scanningInfo.Outputs), Infrastructure: scanningInfo.Infrastructure, VMLabels: scanningInfo.VMLabels, } @@ -635,20 +635,20 @@ func (j *ScanningJob) toJobTask(scanning *commonmodels.ScanningModule, taskID in jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarScriptStep) } - if scanningInfo.CheckQualityGate { - sonarChekStep := &commonmodels.StepTask{ - Name: scanning.Name + "-sonar-check", - JobName: jobTask.Name, - StepType: config.StepSonarCheck, - Spec: &step.StepSonarCheckSpec{ - Parameter: scanningInfo.Parameter, - CheckDir: repoName, - SonarToken: sonarInfo.Token, - SonarServer: sonarInfo.ServerAddress, - }, - } - jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarChekStep) + sonarChekStep := &commonmodels.StepTask{ + Name: scanning.Name + "-sonar-check", + JobName: jobTask.Name, + JobKey: jobTask.Key, + StepType: config.StepSonarCheck, + Spec: &step.StepSonarCheckSpec{ + Parameter: scanningInfo.Parameter, + CheckDir: repoName, + SonarToken: sonarInfo.Token, + SonarServer: sonarInfo.ServerAddress, + CheckQualityGate: scanningInfo.CheckQualityGate, + }, } + jobTaskSpec.Steps = append(jobTaskSpec.Steps, sonarChekStep) } else { scriptStep := &commonmodels.StepTask{ JobName: jobTask.Name, @@ -818,3 +818,16 @@ func fillScanningDetail(moduleScanning *commonmodels.Scanning) error { func getScanningJobCacheObjectPath(workflowName, scanningName string) string { return fmt.Sprintf("%s/cache/%s", workflowName, scanningName) } + +func ensureScanningOutputs(outputs []*commonmodels.Output) []*commonmodels.Output { + keyMap := map[string]struct{}{} + for _, output := range outputs { + keyMap[output.Name] = struct{}{} + } + if _, ok := keyMap[setting.WorkflowScanningJobOutputKey]; !ok { + outputs = append(outputs, &commonmodels.Output{ + Name: setting.WorkflowScanningJobOutputKey, + }) + } + return outputs +} diff --git a/pkg/microservice/jobexecutor/core/service/job.go b/pkg/microservice/jobexecutor/core/service/job.go index 675eb6fc41..f555185933 100644 --- a/pkg/microservice/jobexecutor/core/service/job.go +++ b/pkg/microservice/jobexecutor/core/service/job.go @@ -179,7 +179,7 @@ func (j *Job) collectJobResult(ctx context.Context) error { func (j *Job) getJobOutputVars(ctx context.Context) ([]*job.JobOutput, error) { outputs := []*job.JobOutput{} for _, outputName := range j.Ctx.Outputs { - fileContents, err := ioutil.ReadFile(filepath.Join(job.JobOutputDir, outputName)) + fileContents, err := os.ReadFile(filepath.Join(job.JobOutputDir, outputName)) if os.IsNotExist(err) { continue } else if err != nil { diff --git a/pkg/microservice/jobexecutor/core/service/step/step_sonar_check.go b/pkg/microservice/jobexecutor/core/service/step/step_sonar_check.go index 233df1002f..e591700ab4 100644 --- a/pkg/microservice/jobexecutor/core/service/step/step_sonar_check.go +++ b/pkg/microservice/jobexecutor/core/service/step/step_sonar_check.go @@ -20,16 +20,18 @@ import ( "context" "errors" "fmt" - "io/ioutil" + "os" "path/filepath" "time" - "github.com/koderover/zadig/v2/pkg/setting" "gopkg.in/yaml.v2" + "github.com/koderover/zadig/v2/pkg/setting" "github.com/koderover/zadig/v2/pkg/tool/log" "github.com/koderover/zadig/v2/pkg/tool/sonar" + "github.com/koderover/zadig/v2/pkg/types/job" "github.com/koderover/zadig/v2/pkg/types/step" + "github.com/koderover/zadig/v2/pkg/util" ) type SonarCheckStep struct { @@ -62,7 +64,7 @@ func (s *SonarCheckStep) Run(ctx context.Context) error { sonarWorkDir = filepath.Join(s.workspace, s.spec.CheckDir, sonarWorkDir) } taskReportDir := filepath.Join(sonarWorkDir, "report-task.txt") - bytes, err := ioutil.ReadFile(taskReportDir) + bytes, err := os.ReadFile(taskReportDir) if err != nil { log.Errorf("read sonar task report file: %s error :%v", time.Now().Format(setting.WorkflowTimeFormat), taskReportDir, err) return err @@ -73,20 +75,31 @@ func (s *SonarCheckStep) Run(ctx context.Context) error { log.Error("can not get sonar ce task ID") return errors.New("can not get sonar ce task ID") } - analysisID, err := client.WaitForCETaskTobeDone(ceTaskID, time.Minute*10) - if err != nil { - log.Error(err) - return err - } - gateInfo, err := client.GetQualityGateInfo(analysisID) + + outputFileName := filepath.Join(job.JobOutputDir, setting.WorkflowScanningJobOutputKey) + err = util.AppendToFile(outputFileName, ceTaskID) if err != nil { + err = fmt.Errorf("append sonar ce task ID %s to output file %s error: %v", ceTaskID, outputFileName, err) log.Error(err) return err } - log.Infof("Sonar quality gate status: %s", gateInfo.ProjectStatus.Status) - sonar.PrintSonarConditionTables(gateInfo.ProjectStatus.Conditions) - if gateInfo.ProjectStatus.Status != sonar.QualityGateOK && gateInfo.ProjectStatus.Status != sonar.QualityGateNone { - return fmt.Errorf("sonar quality gate status was: %s", gateInfo.ProjectStatus.Status) + + if s.spec.CheckQualityGate { + analysisID, err := client.WaitForCETaskTobeDone(ceTaskID, time.Minute*10) + if err != nil { + log.Error(err) + return err + } + gateInfo, err := client.GetQualityGateInfo(analysisID) + if err != nil { + log.Error(err) + return err + } + log.Infof("Sonar quality gate status: %s", gateInfo.ProjectStatus.Status) + sonar.PrintSonarConditionTables(gateInfo.ProjectStatus.Conditions) + if gateInfo.ProjectStatus.Status != sonar.QualityGateOK && gateInfo.ProjectStatus.Status != sonar.QualityGateNone { + return fmt.Errorf("sonar quality gate status was: %s", gateInfo.ProjectStatus.Status) + } } return nil } diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 39ab4f0aa1..e45135ed36 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -915,3 +915,7 @@ const ( SQLExecStatusFailed SQLExecStatus = "failed" SQLExecStatusNotExec SQLExecStatus = "not_exec" ) + +const ( + WorkflowScanningJobOutputKey = "SonarCETaskID" +) diff --git a/pkg/tool/sonar/sonar.go b/pkg/tool/sonar/sonar.go index 162d79a8af..b280c132f0 100644 --- a/pkg/tool/sonar/sonar.go +++ b/pkg/tool/sonar/sonar.go @@ -81,6 +81,22 @@ type CETask struct { Status CETaskStatus `json:"status"` SubmitterLogin string `json:"submitterLogin"` WarningCount int `json:"warningCount"` + SubmittedAt string `json:"submittedAt"` + StartedAt string `json:"startedAt"` + ExecutedAt string `json:"executedAt"` +} + +type MeasuresComponentResponse struct { + Component Component `json:"component"` +} + +type Component struct { + Measures []Measure `json:"measures"` +} + +type Measure struct { + Metric string `json:"metric"` + Value string `json:"value"` } func (c *Client) GetCETaskInfo(taskID string) (*CETaskInfo, error) { @@ -92,6 +108,15 @@ func (c *Client) GetCETaskInfo(taskID string) (*CETaskInfo, error) { return res, nil } +func (c *Client) GetComponentMeasures(componentKey string) (*MeasuresComponentResponse, error) { + url := "/api/measures/component" + resp := &MeasuresComponentResponse{} + if _, err := c.Client.Get(url, httpclient.SetQueryParam("component", componentKey), httpclient.SetQueryParam("metricKeys", "ncloc,bugs,vulnerabilities,code_smells,coverage"), httpclient.SetResult(resp)); err != nil { + return nil, fmt.Errorf("search sonar component measures: component %s, error: %v", componentKey, err) + } + return resp, nil +} + type QualityGateStatus string const ( diff --git a/pkg/types/step/step_sonar_check.go b/pkg/types/step/step_sonar_check.go index 11358128d6..a429bd2a6d 100644 --- a/pkg/types/step/step_sonar_check.go +++ b/pkg/types/step/step_sonar_check.go @@ -17,8 +17,9 @@ limitations under the License. package step type StepSonarCheckSpec struct { - Parameter string `bson:"parameter" json:"parameter" yaml:"parameter"` - SonarToken string `bson:"sonar_token" json:"sonar_token" yaml:"sonar_token"` - SonarServer string `bson:"sonar_server" json:"sonar_server" yaml:"sonar_server"` - CheckDir string `bson:"check_dir" json:"check_dir" yaml:"check_dir"` + Parameter string `bson:"parameter" json:"parameter" yaml:"parameter"` + SonarToken string `bson:"sonar_token" json:"sonar_token" yaml:"sonar_token"` + SonarServer string `bson:"sonar_server" json:"sonar_server" yaml:"sonar_server"` + CheckDir string `bson:"check_dir" json:"check_dir" yaml:"check_dir"` + CheckQualityGate bool `bson:"check_quality_gate" json:"check_quality_gate" yaml:"check_quality_gate"` } diff --git a/pkg/util/file.go b/pkg/util/file.go index b9c71bf6f3..95468993df 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -95,3 +95,18 @@ func PathExists(path string) (bool, error) { } return false, err } + +func AppendToFile(filename string, data string) error { + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + _, err = file.WriteString(data) + if err != nil { + return err + } + + return nil +}