Merge branch 'main' into use_request_body

Author: M4tteoP

Diff for: .github/workflows/codeql-analysis.yml

@@ -13,12 +13,12 @@ jobs:
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@396bb3e45325a47dd9ef434068033c6d5bb0d11a # v3
+      uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3
       with:
         languages: go
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@396bb3e45325a47dd9ef434068033c6d5bb0d11a # v3
+      uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@396bb3e45325a47dd9ef434068033c6d5bb0d11a # v3
+      uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3

Diff for: .github/workflows/regression.yml

@@ -11,13 +11,29 @@ on:
     paths-ignore:
       - "**/*.md"
       - "LICENSE"
-
 jobs:
+  # Generate matrix of tags for all permutations of the tests
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.generate.outputs.tags }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      
+      - name: Generate tag combinations
+        id: generate
+        run: |
+          go run mage.go tagsmatrix > tags.json
+          echo "::set-output name=tags::$(cat tags.json)"
+        shell: bash
   test:
+    needs: generate-matrix
     strategy:
       matrix:
         go-version: [1.22.x, 1.23.x]
         os: [ubuntu-latest]
+        build-flag: ${{ fromJson(needs.generate-matrix.outputs.tags) }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
@@ -26,36 +42,37 @@ jobs:
         uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version: ${{ matrix.go-version }}
-          cache: true
+          cache: true  
       - name: Tests and coverage
-        run: go run mage.go coverage
+        run: |
+          export BUILD_TAGS=${{ matrix.build-flag }}
+          go run mage.go coverage
       - name: "Codecov: General"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
+        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage.txt
-          flags: default
+          flags: default,${{ matrix.build-flag }}
+          token: ${{ secrets.CODECOV_TOKEN }}   
       - name: "Codecov: Examples"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
+        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage-examples.txt
-          flags: examples
+          flags: examples+${{ matrix.build-flag }}
+          token: ${{ secrets.CODECOV_TOKEN }}
       - name: "Codecov: FTW"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
+        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage-ftw.txt
-          flags: ftw
-      - name: "Codecov: FTW Multiphase tag"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
-        if: ${{ matrix.go-version == '1.22.x' }}
-        with:
-          files: build/coverage-ftw-multiphase.txt
-          flags: ftw-multiphase
+          flags: ftw,${{ matrix.build-flag }}
+          token: ${{ secrets.CODECOV_TOKEN }}
       - name: "Codecov: Tinygo"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
-        if: ${{ matrix.go-version == '1.22.x' }}
+        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5
+        # only if coverage-tinygo.txt exists
+        if: ${{ matrix.go-version == '1.22.x' && hashFiles('build/coverage-tinygo.txt') != '' }}
         with:
           files: build/coverage-tinygo.txt
-          flags: tinygo
+          flags: tinygo,${{ matrix.build-flag }}
+          token: ${{ secrets.CODECOV_TOKEN }}

Diff for: README.md

@@ -104,6 +104,7 @@ dictionaries to reduce memory consumption in deployments that launch several cor
 instances. For more context check [this issue](https://github.com/corazawaf/coraza-caddy/issues/76)
 * `no_fs_access` - indicates that the target environment has no access to FS in order to not leverage OS' filesystem related functionality e.g. file body buffers.
 * `coraza.rule.case_sensitive_args_keys` - enables case-sensitive matching for ARGS keys, aligning Coraza behavior with RFC 3986 specification. It will be enabled by default in the next major version.
+* `coraza.rule.no_regex_multiline` - disables enabling by default regexes multiline modifiers in `@rx` operator. It aligns with CRS expected behavior, reduces false positives and might improve performances. No multiline regexes by default will be enabled in the next major version. For more context check [this PR](https://github.com/corazawaf/coraza/pull/876)
 
 ## E2E Testing
 

Diff for: experimental/plugins/macro/macro.go

@@ -166,7 +166,7 @@ func (m *macro) compile(input string) error {
 }
 
 func isValidMacroChar(c byte) bool {
-	return c == '.' || c == '_' || c == '-' || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
+	return c == '[' || c == ']' || c == '.' || c == '_' || c == '-' || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
 }
 
 // String returns the original string

Diff for: experimental/plugins/macro/macro_test.go

@@ -69,7 +69,10 @@ func TestCompile(t *testing.T) {
 	})
 
 	t.Run("malformed macros", func(t *testing.T) {
-		for _, test := range []string{"%{tx.count", "%{{tx.count}", "%{{tx.{count}", "something %{tx.count"} {
+		for _, test := range []string{
+			"%{tx.count", "%{{tx.count}", "%{{tx.{count}", "something %{tx.count",
+			"%{ARG_NAMES:/exec/", // Wildcard variable names are not supported
+		} {
 			t.Run(test, func(t *testing.T) {
 				m := &macro{}
 				err := m.compile(test)
@@ -118,19 +121,28 @@ func TestCompile(t *testing.T) {
 	})
 
 	t.Run("valid macro", func(t *testing.T) {
-		m := &macro{}
-		err := m.compile("%{tx.count}")
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err.Error())
-		}
-
-		if want, have := 1, len(m.tokens); want != have {
-			t.Fatalf("unexpected number of tokens: want %d, have %d", want, have)
-		}
-
-		expectedMacro := macroToken{"tx.count", variables.TX, "count"}
-		if want, have := m.tokens[0], expectedMacro; want != have {
-			t.Errorf("unexpected token: want %v, have %v", want, have)
+		type testCase struct {
+			input         string
+			expectedMacro macroToken
+		}
+		for _, tc := range []testCase{
+			{"%{tx.count}", macroToken{"tx.count", variables.TX, "count"}},
+			{"%{ARGS.exec}", macroToken{"ARGS.exec", variables.Args, "exec"}},
+			{"%{ARGS_GET.db[]}", macroToken{"ARGS_GET.db[]", variables.ArgsGet, "db[]"}},
+		} {
+			m := &macro{}
+			err := m.compile(tc.input)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err.Error())
+			}
+
+			if len(m.tokens) != 1 {
+				t.Fatalf("unexpected number of tokens: want %d, have %d", 1, len(m.tokens))
+			}
+
+			if m.tokens[0] != tc.expectedMacro {
+				t.Errorf("unexpected token: want %v, have %v", tc.expectedMacro, m.tokens[0])
+			}
 		}
 	})
 

Diff for: go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/corazawaf/libinjection-go v0.2.2
 	github.com/foxcpp/go-mockdns v1.1.0
 	github.com/jcchavezs/mergefs v0.1.0
-	github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f
+	github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516
 	github.com/mccutchen/go-httpbin/v2 v2.15.0
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4
 	github.com/tidwall/gjson v1.18.0

Diff for: go.sum

@@ -14,6 +14,10 @@ github.com/jcchavezs/mergefs v0.1.0 h1:7oteO7Ocl/fnfFMkoVLJxTveCjrsd//UB0j89xmnp
 github.com/jcchavezs/mergefs v0.1.0/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
 github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f h1:iiLWLoibjCL0XND6inF7bs2nc20lU/FYkiR//VIOLUc=
 github.com/magefile/mage v1.15.1-0.20231118170541-2385abb49a1f/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa h1:RAKYgtUC3OPrwUYKQcwB0wWXCHo09ZWh5TFlnF0nVfA=
+github.com/magefile/mage v1.15.1-0.20241124190125-32e01077f0aa/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516 h1:aAO0L0ulox6m/CLRYvJff+jWXYYCKGpEm3os7dM/Z+M=
+github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mccutchen/go-httpbin/v2 v2.15.0 h1:3b2s8LMRR2aFd+8U+1Bx2kdgHNQ5ZQkQOiW8e52Jj9A=
 github.com/mccutchen/go-httpbin/v2 v2.15.0/go.mod h1:GBy5I7XwZ4ZLhT3hcq39I4ikwN9x4QUt6EAxNiR8Jus=
 github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=

Diff for: internal/collections/map.go

@@ -47,11 +47,15 @@ func (c *Map) Get(key string) []string {
 	if !c.isCaseSensitive {
 		key = strings.ToLower(key)
 	}
-	var values []string
-	for _, a := range c.data[key] {
-		values = append(values, a.value)
+	values := c.data[key]
+	if len(values) == 0 {
+		return nil
+	}
+	result := make([]string, len(values))
+	for i, v := range values {
+		result[i] = v.value
 	}
-	return values
+	return result
 }
 
 // FindRegex returns all map elements whose key matches the regular expression.
@@ -120,16 +124,22 @@ func (c *Map) Add(key string, value string) {
 	c.data[key] = append(c.data[key], aVal)
 }
 
-// Set sets the value of a key with the array of strings passed. If the key already exists, it will be overwritten.
+// Sets the value of a key with the array of strings passed. If the key already exists, it will be overwritten.
 func (c *Map) Set(key string, values []string) {
 	originalKey := key
 	if !c.isCaseSensitive {
 		key = strings.ToLower(key)
 	}
-	c.data[key] = make([]keyValue, 0, len(values))
-	for _, v := range values {
-		c.data[key] = append(c.data[key], keyValue{key: originalKey, value: v})
+	dataSlice, exists := c.data[key]
+	if !exists || cap(dataSlice) < len(values) {
+		dataSlice = make([]keyValue, len(values))
+	} else {
+		dataSlice = dataSlice[:len(values)] // Reuse existing slice with the same length
+	}
+	for i, v := range values {
+		dataSlice[i] = keyValue{key: originalKey, value: v}
 	}
+	c.data[key] = dataSlice
 }
 
 // SetIndex sets the value of a key at the specified index. If the key already exists, it will be overwritten.

Diff for: internal/collections/map_test.go

@@ -106,3 +106,25 @@ func TestNewCaseSensitiveKeyMap(t *testing.T) {
 	}
 
 }
+
+func BenchmarkTxSetGet(b *testing.B) {
+	keys := make(map[int]string, b.N)
+	for i := 0; i < b.N; i++ {
+		keys[i] = fmt.Sprintf("key%d", i)
+	}
+	c := NewCaseSensitiveKeyMap(variables.RequestHeaders)
+
+	b.Run("Set", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			c.Set(keys[i], []string{"value2"})
+		}
+	})
+	b.Run("Get", func(b *testing.B) {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			c.Get(keys[i])
+		}
+	})
+	b.ReportAllocs()
+}

Diff for: internal/collections/named.go

@@ -61,11 +61,11 @@ func (c *NamedCollection) Len() int {
 
 // Data is an internal method used for serializing to JSON
 func (c *NamedCollection) Data() map[string][]string {
-	result := map[string][]string{}
+	result := make(map[string][]string, len(c.data))
 	for k, v := range c.data {
-		result[k] = make([]string, 0, len(v))
-		for _, a := range v {
-			result[k] = append(result[k], a.value)
+		result[k] = make([]string, len(v))
+		for i, a := range v {
+			result[k][i] = a.value
 		}
 	}
 	return result

Diff for: internal/corazarules/rule_match.go

@@ -33,27 +33,27 @@ type MatchData struct {
 
 var _ types.MatchData = (*MatchData)(nil)
 
-func (m *MatchData) Variable() variables.RuleVariable {
+func (m MatchData) Variable() variables.RuleVariable {
 	return m.Variable_
 }
 
-func (m *MatchData) Key() string {
+func (m MatchData) Key() string {
 	return m.Key_
 }
 
-func (m *MatchData) Value() string {
+func (m MatchData) Value() string {
 	return m.Value_
 }
 
-func (m *MatchData) Message() string {
+func (m MatchData) Message() string {
 	return m.Message_
 }
 
-func (m *MatchData) Data() string {
+func (m MatchData) Data() string {
 	return m.Data_
 }
 
-func (m *MatchData) ChainLevel() int {
+func (m MatchData) ChainLevel() int {
 	return m.ChainLevel_
 }