Merge QS into master

.gitattributes

@@ -0,0 +1,4 @@
+floss/tags/data/**/*.bin filter=lfs diff=lfs merge=lfs -text
+floss/tags/data/**/*.gz filter=lfs diff=lfs merge=lfs -text
+floss/tags/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text
+floss/sigs/*.sig filter=lfs diff=lfs merge=lfs -text

.github/mypy/mypy.ini

@@ -1,5 +1,6 @@
 [mypy]
 plugins = pydantic.mypy
+mypy_path = .
 
 [mypy-viv_utils.*]
 ignore_missing_imports = True
@@ -34,6 +35,21 @@ ignore_missing_imports = True
 [mypy-pefile.*]
 ignore_missing_imports = True
 
+[mypy-intervaltree.*]
+ignore_missing_imports = True
+
+[mypy-lancelot.*]
+ignore_missing_imports = True
+
+[mypy-capa.*]
+ignore_missing_imports = True
+
+[mypy-virustotal3.*]
+ignore_missing_imports = True
+
+[mypy-colorama.*]
+ignore_missing_imports = True
+
 [mypy-requests.*]
 ignore_missing_imports = True
 

.github/pyinstaller/floss.spec

@@ -1,21 +1,38 @@
 # -*- mode: python -*-
-# Copyright 2017 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import subprocess
 
+from PyInstaller.utils.hooks import collect_submodules
+
+# layout/tags are imported lazily from floss.pipeline; collect them so the
+# standalone binary still bundles the full layout-aware static path.
+layout_tags_hiddenimports = (
+    collect_submodules("floss.layout")
+    + collect_submodules("floss.tags")
+    + [
+        "floss.ranges",
+        "elftools",
+        "lancelot",
+        "machofile",
+        "dnfile",
+        "msgspec",
+    ]
+)
+
 # when invoking pyinstaller from the project root,
 # this gets run from the project root.
 with open("./floss/version.py", "wb") as f:
@@ -33,55 +50,66 @@ with open("./floss/version.py", "wb") as f:
     )
     f.write(("__version__ = '%s'" % version).encode("utf-8"))
 
+datas = [
+    # when invoking pyinstaller from the project root,
+    # this gets invoked from the directory of the spec file,
+    # i.e. ./.github/pyinstaller
+    ('../../floss/sigs', 'sigs'),
+    # tag databases
+    ('../../floss/tags/data/crt/*.jsonl.gz', 'floss/tags/data/crt'),
+    ('../../floss/tags/data/expert/*.jsonl', 'floss/tags/data/expert'),
+    ('../../floss/tags/data/gp/*.jsonl.gz', 'floss/tags/data/gp'),
+    ('../../floss/tags/data/gp/*.bin', 'floss/tags/data/gp'),
+    ('../../floss/tags/data/oss/*.jsonl.gz', 'floss/tags/data/oss'),
+    ('../../floss/tags/data/winapi/*.txt.gz', 'floss/tags/data/winapi'),
+]
+
+excludes = [
+    # ignore packages that would otherwise be bundled with the .exe.
+    # review: build/pyinstaller/xref-pyinstaller.html
+    # we don't do any GUI stuff, so ignore these modules
+    "tkinter",
+    "_tkinter",
+    "Tkinter",
+
+    # tqdm provides renderers for ipython,
+    # however, this drags in a lot of dependencies.
+    # since we don't spawn a notebook, we can safely remove these.
+    "IPython",
+    "ipywidgets",
+
+    # these are pulled in by networkx
+    # but we don't need to compute the strongly connected components.
+    "numpy",
+    "scipy",
+    "matplotlib",
+    "pandas",
+    "pytest",
+
+    # deps from viv that we don't use.
+    # this duplicates the entries in `hook-vivisect`,
+    # but works better this way.
+    "vqt",
+    "vdb.qt",
+    "envi.qt",
+    "PyQt5",
+    "qt5",
+    "pyqtwebengine",
+    "pyasn1",
+]
+
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
     ["../../floss/main.py"],
     pathex=["floss"],
     binaries=[],
-    datas=[
-        # when invoking pyinstaller from the project root,
-        # this gets invoked from the directory of the spec file,
-        # i.e. ./.github/pyinstaller
-        ('../../floss/sigs', 'sigs'),
-    ],
-    hiddenimports=[],
+    datas=datas,
+    hiddenimports=layout_tags_hiddenimports,
     hookspath=[".github/pyinstaller/hooks"],
     runtime_hooks=[],
-    excludes=[
-        # ignore packages that would otherwise be bundled with the .exe.
-        # review: build/pyinstaller/xref-pyinstaller.html
-        # we don't do any GUI stuff, so ignore these modules
-        "tkinter",
-        "_tkinter",
-        "Tkinter",
-
-        # tqdm provides renderers for ipython,
-        # however, this drags in a lot of dependencies.
-        # since we don't spawn a notebook, we can safely remove these.
-        "IPython",
-        "ipywidgets",
-
-        # these are pulled in by networkx
-        # but we don't need to compute the strongly connected components.
-        "numpy",
-        "scipy",
-        "matplotlib",
-        "pandas",
-        "pytest",
-
-        # deps from viv that we don't use.
-        # this duplicates the entries in `hook-vivisect`,
-        # but works better this way.
-        "vqt",
-        "vdb.qt",
-        "envi.qt",
-        "PyQt5",
-        "qt5",
-        "pyqtwebengine",
-        "pyasn1",
-    ],
+    excludes=excludes,
     win_no_prefer_redirects=False,
     win_private_assemblies=False,
     noarchive=False,

.github/workflows/build-oss-db.yml

@@ -0,0 +1,127 @@
+name: Build OSS String Databases
+
+on:
+  # Rebuild the databases bi-weekly (1st and 15th of each month at 00:00 UTC).
+  schedule:
+    - cron: '0 0 1,15 * *'
+
+  # Allow manual runs from the Actions tab.
+  workflow_dispatch:
+
+# Cancel any in-progress run when a newer one starts.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  build-databases:
+    name: Build OSS string databases
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout flare-floss
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          path: flare-floss
+          lfs: true
+
+      - name: Checkout lancelot
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          repository: williballenthin/lancelot
+          path: lancelot
+          lfs: true
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.12'
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # v1
+        with:
+          toolchain: stable
+
+      - name: Build jh
+        run: |
+          cargo build --release -p lancelot-bin
+        working-directory: lancelot
+
+      - name: Build OSS databases
+        run: |
+          python scripts/tags/build_oss_db.py `
+            --config floss\tags\data\oss\libraries.json `
+            --jh-path ..\lancelot\target\release\jh.exe `
+            --output-dir floss\tags\data\oss `
+            --continue-on-error
+        working-directory: flare-floss
+
+      - name: Show metrics summary
+        if: success() || failure()
+        run: |
+          Get-Content floss\tags\data\oss\build_metrics.json
+        working-directory: flare-floss
+
+      - name: Show entry-level diff
+        if: success() || failure()
+        run: |
+          if (Test-Path floss\tags\data\oss\build_diff.txt) {
+            Get-Content floss\tags\data\oss\build_diff.txt
+          } else {
+            Write-Host "build_diff.txt not produced"
+          }
+        working-directory: flare-floss
+
+      - name: Prepare PR body
+        if: success() || failure()
+        shell: pwsh
+        run: |
+          # Markdown already has per-library ## headings and ```diff fences.
+          $diffPath = "flare-floss/floss/tags/data/oss/build_diff_pr.txt"
+          $diff = if (Test-Path $diffPath) {
+            Get-Content $diffPath -Raw
+          } else {
+            "(no entry-level diff produced)"
+          }
+          $librariesConfig = Get-Content "flare-floss/floss/tags/data/oss/libraries.json" -Raw | ConvertFrom-Json
+          $body = @(
+            "Automated bi-weekly rebuild of the OSS string databases.",
+            "",
+            "- Triplet: ``$($librariesConfig.triplet)``",
+            "- Compiler: ``$($librariesConfig.compiler)``",
+            "- Library list: see ``floss/tags/data/oss/libraries.json``",
+            "",
+            "Per-library entry counts and timing are printed in the workflow logs (``build_metrics.json``).",
+            "",
+            "Entry-level diff (up to 20 lines per library):",
+            "",
+            $diff.TrimEnd(),
+            ""
+          ) -join "`n"
+          # GitHub rejects PR bodies over 65536 characters ("Body is too long").
+          # The Python script already caps the diff; this is a final safety net.
+          $maxBodyChars = 65000
+          if ($body.Length -gt $maxBodyChars) {
+            $notice = "`n`n... truncated to stay under GitHub's 65536-character PR body limit."
+            $keep = [Math]::Max(0, $maxBodyChars - $notice.Length)
+            $body = $body.Substring(0, $keep) + $notice
+          }
+          # Write outside the flare-floss checkout so create-pull-request does not commit it.
+          Set-Content -Path pr-body.md -Value $body -Encoding utf8
+      - name: Create Pull Request if databases changed
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
+        with:
+          path: flare-floss
+          base: master
+          commit-message: 'chore(oss-db): update string databases'
+          title: 'Update OSS string databases'
+          # Relative to $GITHUB_WORKSPACE (not the path: checkout).
+          body-path: pr-body.md
+          branch: update-oss-string-databases
+          # Always open a fresh PR (e.g. update-oss-string-databases-6qj97jr).
+          branch-suffix: random
+          delete-branch: true

.github/workflows/build.yml

@@ -2,9 +2,13 @@
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [ master, quantumstrand ]
   release:
     types: [edited, published]
+    branches: [ master ]
+
+permissions:
+  contents: read
 
 jobs:
   build:
@@ -29,6 +33,7 @@
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: false
+          lfs: true
       - name: Set up Python 3.10
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
@@ -39,6 +44,18 @@
           pip install -e .[build]
       - name: Build standalone executable
         run: pyinstaller .github/pyinstaller/floss.spec
+      # build job uses submodules: false; smoke test needs a real PE sample
+      - name: Checkout test sample
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: mandiant/flare-floss-testfiles
+          path: tests/data
+      - name: Smoke test layout/tags via standalone floss
+        run: |
+          chmod +x dist/${{ matrix.artifact_name }}
+          ./dist/${{ matrix.artifact_name }} --help
+          # static-only layout path (default) should include tag annotations in JSON
+          ./dist/${{ matrix.artifact_name }} tests/data/test-decode-to-stack.exe --string-type static -j | grep -q '#common'
       - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.asset_name }}
@@ -111,9 +128,9 @@
       - name: Set zip name
         run: echo "zip_name=floss-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
-        run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
+        run: zip "$zip_name" "${{ matrix.artifact_name }}"
       - name: Upload ${{ env.zip_name }} to GH Release
         uses: svenstaro/upload-release-action@29e53e917877a24fad85510ded594ab3c9ca12de # 2.11.5
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: ${{ env.zip_name }}