Allow policies to be skipped

lint/lint.go

@@ -20,6 +20,9 @@ func printTestsuite(ts Testsuite) {
 		if tc.Failure != nil {
 			result = "FAIL"
 		}
+		if tc.Skipped != nil {
+			result = "SKIP"
+		}
 		fmt.Printf("%s (%.5fs) %s\n", result, tc.Time, tc.Name)
 	}
 	fmt.Println("")
@@ -84,6 +87,7 @@ func evalTestsuite(policyPath string, modelSourcePath string) (*Testsuite, error
 	var packageName string = ""
 	var pattern string = ""
 	var policy_canonical_name string = ""
+	var skipReason string = ""
 
 	lines := strings.Split(string(policyContent), "\n")
 
@@ -96,6 +100,10 @@ func evalTestsuite(policyPath string, modelSourcePath string) (*Testsuite, error
 				return nil, err
 			}
 		}
+		tokens = strings.Split(line, "#  skip: ")
+		if len(tokens) > 1 && skipReason == "" {
+			skipReason = tokens[1]
+		}
 		tokens = strings.Split(line, "package ")
 		if len(tokens) > 1 && packageName == "" {
 			packageName = tokens[1]
@@ -111,15 +119,33 @@ func evalTestsuite(policyPath string, modelSourcePath string) (*Testsuite, error
 	log.Debugf("input pattern: %s", pattern)
 	log.Debugf("expanded input files %v", inputFiles)
 
+	var skipped *Skipped = nil
+	if skipReason != "" {
+		skipped = &Skipped{
+			Message: skipReason,
+		}
+	}
+
 	queryString := "data." + packageName + "." + policy_canonical_name + " == true"
 	testcases := make([]Testcase, 0)
 	failuresCount := 0
+	skippedCount := 0
 	totalTime := 0.0
+	testcase := &Testcase{}
 
 	for _, inputFile := range inputFiles {
-		testcase, err := evalTestcase(policyPath, queryString, inputFile)
-		if err != nil {
-			return nil, err
+		if skipped != nil {
+			testcase = &Testcase{
+				Name:    inputFile,
+				Time:    0,
+				Skipped: skipped,
+			}
+			skippedCount++
+		} else {
+			testcase, err = evalTestcase(policyPath, queryString, inputFile)
+			if err != nil {
+				return nil, err
+			}
 		}
 		if testcase.Failure != nil {
 			failuresCount++
@@ -133,6 +159,7 @@ func evalTestsuite(policyPath string, modelSourcePath string) (*Testsuite, error
 		Name:      policyPath,
 		Tests:     len(testcases),
 		Failures:  failuresCount,
+		Skipped:   skippedCount,
 		Time:      totalTime,
 		Testcases: testcases,
 	}
@@ -191,6 +218,7 @@ func evalTestcase(policyPath string, queryString string, inputFilePath string) (
 		Name:    inputFilePath,
 		Time:    float64(duration.Nanoseconds()) / 1e9, // convert to seconds
 		Failure: failure,
+		Skipped: nil,
 	}
 	return testcase, nil
 }

lint/lint_test.go

@@ -6,15 +6,15 @@ import (
 
 // TestAdd tests the Add function to ensure it returns correct results.
 func TestLintSingle(t *testing.T) {
-	t.Run("single policy fails", func(t *testing.T) {
+	t.Run("single policy skipped", func(t *testing.T) {
 		result, err := evalTestsuite("./../policies/001-projectsettings/001-0004-strong_password.rego", "./../modelsource")
 
 		if err != nil {
 			t.Errorf("Failed to evaluate")
 		}
 
-		if result.Failures != 1 {
-			t.Errorf("Failed policy")
+		if result.Skipped != 1 {
+			t.Errorf("Policy not skipped")
 		}
 	})
 	t.Run("single policy passes", func(t *testing.T) {
@@ -25,7 +25,7 @@ func TestLintSingle(t *testing.T) {
 		}
 
 		if result.Failures != 0 {
-			t.Errorf("Failed policy")
+			t.Errorf("Policy passes")
 		}
 	})
 }
@@ -34,8 +34,8 @@ func TestLintBundle(t *testing.T) {
 	t.Run("all-policy", func(t *testing.T) {
 		err := EvalAll("./../policies", "./../modelsource", "")
 
-		if err == nil {
-			t.Errorf("Failed to evaluate")
+		if err != nil {
+			t.Errorf("Failed to evaluate: %v", err)
 		}
 	})
 }

lint/types.go

@@ -12,6 +12,7 @@ type Testsuite struct {
 	Name      string   `xml:"name,attr"`
 	Tests     int      `xml:"tests,attr"`
 	Failures  int      `xml:"failures,attr"`
+	Skipped   int      `xml:"skipped,attr"`
 	Time      float64  `xml:"time,attr"`
 	Testcases []Testcase
 }
@@ -21,10 +22,14 @@ type Testcase struct {
 	Name    string   `xml:"name,attr"`
 	Time    float64  `xml:"time,attr"`
 	Failure *Failure `xml:"failure,omitempty"`
+	Skipped *Skipped `xml:"skipped,omitempty"`
 }
 
 type Failure struct {
 	Message string `xml:"message,attr"`
 	Type    string `xml:"type,attr"`
 	Data    string `xml:",chardata"`
 }
+type Skipped struct {
+	Message string `xml:"message,attr"`
+}

policies/001-projectsettings/001-0002-demo_users_disabled.rego

@@ -7,6 +7,7 @@
 #  category: security
 #  rulename: DemoUsersDisabled
 #  priority: 5
+#  skip: FIXME
 #  rulenumber: 001-0002
 #  remediation: Disable demo users in Project Security
 #  input: Security$ProjectSecurity.yaml

policies/001-projectsettings/001-0004-strong_password.rego

@@ -7,6 +7,7 @@
 #  category: security
 #  rulename: StrongPasswordPolicy
 #  priority: 5
+#  skip: FIXME
 #  rulenumber: 001-0004
 #  remediation: Ensure minimum password length of at least 8 characters and must use all character classes.
 #  input: Security$ProjectSecurity.yaml