[Enhancement] PowerShell - Optimize Entropy Calculation, Add Normalized Entropy, Add Pipeline Benchmark

packages/windows/changelog.yml

@@ -1,4 +1,11 @@
 # newer versions go on top
+- version: "3.4.0"
+  changes:
+    - description: |
+        Optimize PowerShell script block entropy processing (faster calculation, skip truncated signature fragments)
+        and add `powershell.file.script_block_entropy_normalized` scaled by script length for a 0-1 range.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16707
 - version: "3.3.0"
   changes:
     - description: Makes Sysmon Library Load events compatible with DLL ECS fields.

packages/windows/data_stream/forwarded/_dev/test/pipeline/test-powershell-operational-events.json-expected.json

@@ -304,11 +304,12 @@
             },
             "powershell": {
                 "file": {
-                    "script_block_entropy_bits": 4.947922042402605,
+                    "script_block_entropy_bits": 4.947922042402607,
+                    "script_block_entropy_normalized": 0.4225401470065925,
                     "script_block_hash": "r0sdjfD0qsH7ckPwQpUfLLA0Slo=",
                     "script_block_id": "50d2dbda-7361-4926-a94d-d9eadfdb43fa",
                     "script_block_length": 3350,
-                    "script_block_surprisal_stdev": 1.760352963786286,
+                    "script_block_surprisal_stdev": 1.7603529637862858,
                     "script_block_text": "###\n# ==++==\n#\n# Copyright (c) Microsoft Corporation. All rights reserved.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n###\n@{\n    GUID = \"4ae9fd46-338a-459c-8186-07f910774cb8\"\n    Author = \"Microsoft Corporation\"\n    CompanyName = \"Microsoft Corporation\"\n    Copyright = \"(C) Microsoft Corporation. All rights reserved.\"\n    HelpInfoUri = \"https://go.microsoft.com/fwlink/?linkid=2113634\"\n    ModuleVersion = \"1.4.8.1\"\n    PowerShellVersion = \"3.0\"\n    ClrVersion = \"4.0\"\n    RootModule = \"PackageManagement.psm1\"\n\tDescription = 'PackageManagement (a.k.a. OneGet) is a new way to discover and install software packages from around the web.\n It is a manager or multiplexor of existing package managers (also called package providers) that unifies Windows package management with a single Windows PowerShell interface. With PackageManagement, you can do the following.\n  - Manage a list of software repositories in which packages can be searched, acquired and installed\n  - Discover software packages\n  - Seamlessly install, uninstall, and inventory packages from one or more software repositories'\n\n    CmdletsToExport = @(\n        'Find-Package',\n        'Get-Package',\n        'Get-PackageProvider',\n        'Get-PackageSource',\n        'Install-Package',\n        'Import-PackageProvider'\n        'Find-PackageProvider'\n        'Install-PackageProvider'\n        'Register-PackageSource',\n        'Set-PackageSource',\n        'Unregister-PackageSource',\n        'Uninstall-Package'\n        'Save-Package'\n\t)\n\n\tFormatsToProcess  = @('PackageManagement.format.ps1xml')\n\n\tPrivateData = @{\n        PSData = @{\n            Tags = @('PackageManagement', 'PSEdition_Core', 'PSEdition_Desktop', 'Linux', 'Mac')\n            ProjectUri = 'https://oneget.org'\n            ReleaseNotes = @'\n## 1.4.8.1\n- Update PackageManagement's strong name signing\n\n## 1.4.8\n- Add NuGet as a source when generating nuget.config file for user in the NuGet Provider\n\n## 1.4.7\n- Update security protocol to use TLS 1.2\n- Remove catalog file\n\n## 1.4.6\n- Update `HelpInfoUri` to point to the latest content\n\n## 1.4.5\n- Bug fix for deadlock when getting parameters in an event\n\n## 1.4.4\n- Bug fix when installing modules from private feeds\n\n ## 1.4.3\n- Another bug fix when registering repositories with PowerShellGet\n\n## 1.4.2\n- Bug fix for passing credentials from PowerShellGet when registering repositories\n\n## 1.4.1\n- Bug fix for using credential provider installed in Visual Studio\n\n## 1.4\n- Allow credential persistance for registering private repositories and finding or installing packages from those repositories\n\n## 1.3.2\n- Enable bootstrap on PSCore\n- Bug fix to run on .NET Core 3.0\n\n## 1.3.1\n- Targets net452 and netstandard2.0 instead of net451, netcoreapp2.0, and netstandard1.6\n            \n## Previous releases are not included in this Changelog\n'@\n        }\n    }\n}\n\n# SIG # Begin signature block\n# MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5Bgor\n# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG\n# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCANw97w1D+bi5LY\n# 8ZEuubcA0tI0Z0h+CImFRYop+IIqQaCCDYEwggX/MIID56ADAgECAhMzAAACUosz\n# qviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD\n# +nC4D7IMA1+6smM7fbSJa7o4BHfyje8PHB3w9GF223mZTG0EhBlultQkMSpV/c88\n# 9hsbwx16Cr5sY9M/lSRt4oC3qzSuTmYd6VYJ/ILt9ptrpOkaYCiXXRx8Cfz7w53w\n# Au/J8xJjNWvrKxkcc8XiUXPfGGTXujyiS2MqvztBkg6wCduFKqogmvOtQiiwQQxE\n# G6lU/rss27omoTUc41EawOr1km5y+fUS9aoYX9K8NNhFH6TSni3dp/+Hiyif1T7X\n# g0cBy4yHuYxMmRrFcmGeplW3KhXHfkJjbHaVs1QgnRfkgFuypwF5YoFWrW7Xgj+a\n# ZCDKSoYq45E4v0ryIvyu0shBoHQXREAzpBv3L9h5A9vEFQG4alCI57oSbdqJ1YIa\n# ggkTQHR2CWdB7FnQilCqqZjSnAtXYZh/RD+PX6fg1UyUUQf5ohnw951pQeKYTYHm\n# Fwut+RibzdbHEF/kLZr6SZsDupCv\n# SIG # End signature block",
                     "script_block_unique_symbols": 79
                 },
@@ -385,11 +386,12 @@
             },
             "powershell": {
                 "file": {
-                    "script_block_entropy_bits": 4.999389346653313,
+                    "script_block_entropy_bits": 4.9993893466533095,
+                    "script_block_entropy_normalized": 0.46973699610806635,
                     "script_block_hash": "UqkTOrIKrALdr6uz7n5JWUnSHs8=",
                     "script_block_id": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48",
                     "script_block_length": 1599,
-                    "script_block_surprisal_stdev": 1.7385527153601164,
+                    "script_block_surprisal_stdev": 1.738552715360117,
                     "script_block_text": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
                     "script_block_unique_symbols": 66
                 },

packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/powershell_operational.yml

@@ -329,26 +329,99 @@ processors:
       source: |-
         // Entropy Variance from: https://github.com/elastic/toutoumomoma/blob/be287c9c0d0e435572e3889a6584199983c688f0/toutoumomoma.go#L326-L363.
         String script = ctx._temp.script_block_no_signature;
+
+        int length = script.length();
+        if (length == 0) {
+          return;
+        }
+
+        // Skip signature only scripts:
+        // - Inspect only line 2 (line 1 can be truncated mid-signature).
+        // - Match "# " + base64-ish content at the fixed signature line length (64 chars).
+        int lf = 10;    // '\n'
+        int cr = 13;    // '\r'
+        int hash = 35;  // '#'
+        int space = 32; // ' '
+        int sigLineLen = 64; // Content length, excluding "# " prefix.
+
+        int firstLineEnd = -1;
+        for (int idx = 0; idx < length; idx++) {
+          if (script.charAt(idx) == lf) {
+            firstLineEnd = idx;
+            break;
+          }
+        }
+
+        if (firstLineEnd > 0) {
+          int secondLineStart = firstLineEnd + 1;
+          if (secondLineStart < length) {
+            int secondLineEnd = length;
+            for (int idx = secondLineStart; idx < length; idx++) {
+              if (script.charAt(idx) == lf) {
+                secondLineEnd = idx;
+                break;
+              }
+            }
+
+            if (secondLineEnd > secondLineStart && script.charAt(secondLineEnd - 1) == cr) {
+              secondLineEnd--;
+            }
+
+            if (secondLineStart < secondLineEnd && script.charAt(secondLineStart) == hash) {
+              int contentStart = secondLineStart + 1;
+              if (contentStart < secondLineEnd && script.charAt(contentStart) == space) {
+                contentStart++;
+              }
+              int lineLen = secondLineEnd - contentStart;
+              if (lineLen == sigLineLen) {
+                boolean base64Line = true;
+                for (int i = contentStart; i < secondLineEnd; i++) {
+                  int c = (int) script.charAt(i);
+                  if (!((c >= 65 && c <= 90) || (c >= 97 && c <= 122) ||
+                        (c >= 48 && c <= 57) || c == 43 || c == 47 || c == 61)) {
+                    base64Line = false;
+                    break;
+                  }
+                }
+                if (base64Line) {
+                  return;
+                }
+              }
+            }
+          }
+        }
 
         script = java.text.Normalizer.normalize(script, java.text.Normalizer.Form.NFC);
 
-        int n = script.codePointCount(0, script.length());
-
-        if (n == 0) {
+        length = script.length();
+        if (length == 0) {
           return;
         }
 
-        Map counts = script.codePoints().boxed().collect(Collectors.groupingBy(c -> c, Collectors.counting()));
-        int uniqueSymbols = counts.size();
+        int[] counts = new int[65536];
+        int[] seen = new int[length];
+        int seenCount = 0;
+        int uniqueSymbols = 0;
+        for (int i = 0; i < length; i++) {
+            int ch = script.charAt(i);
+            if (counts[ch] == 0) {
+                counts[ch] = 1;
+                seen[seenCount++] = ch;
+                uniqueSymbols++;
+            } else {
+                counts[ch]++;
+            }
+        }
 
         double invLog2 = 1.0 / Math.log(2.0);
         double entropy = 0.0;
         double surprisalVar = 0.0;
         double pSum = 0.0;
 
-        for (def value : counts.values()) {
-            double cnt = ((Number) value).doubleValue();
-            double p = cnt / (double) n;
+        for (int i = 0; i < seenCount; i++) {
+            int code = seen[i];
+            double cnt = (double) counts[code];
+            double p = cnt / (double) length;
             double l2p = Math.log(p) * invLog2;
 
             pSum += p;
@@ -363,10 +436,19 @@ processors:
         if (entropyBits == -0.0) {
             entropyBits = 0.0;
         }
+
+        double normalizedEntropy = 0.0;
+        if (length > 1) {
+            double maxEntropy = Math.log((double) length) * invLog2; // max bits if every character is unique
+            normalizedEntropy = entropyBits / maxEntropy;           // scale 0..1 against script length
+            if (normalizedEntropy < 0.0) normalizedEntropy = 0.0;
+            else if (normalizedEntropy > 1.0) normalizedEntropy = 1.0;
+        }
 
         ctx.powershell.file.script_block_entropy_bits = entropyBits;
+        ctx.powershell.file.script_block_entropy_normalized = normalizedEntropy;
         ctx.powershell.file.script_block_surprisal_stdev = surprisalSd;
-        ctx.powershell.file.script_block_length = n;
+        ctx.powershell.file.script_block_length = length;
         ctx.powershell.file.script_block_unique_symbols = uniqueSymbols;
 
   - split:

packages/windows/data_stream/forwarded/fields/fields.yml

@@ -153,6 +153,10 @@
       description: >
         Randomness measure of the script using Shannon entropy over Unicode characters (0-20 bits).
         Entropy values outside the expected range may indicate random or obfuscated code.
+    - name: script_block_entropy_normalized
+      type: float
+      description: >
+        Normalized entropy in the range [0,1], computed as entropy_bits divided by log2(script_block_length).
     - name: script_block_surprisal_stdev
       type: float
       description: >

packages/windows/data_stream/powershell_operational/_dev/benchmark/pipeline/config.yml

@@ -0,0 +1 @@
+num_docs: 2500