diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..c4fad3df5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +floss/tags/data/**/*.bin filter=lfs diff=lfs merge=lfs -text +floss/tags/data/**/*.gz filter=lfs diff=lfs merge=lfs -text +floss/tags/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text +floss/sigs/*.sig filter=lfs diff=lfs merge=lfs -text diff --git a/.github/mypy/mypy.ini b/.github/mypy/mypy.ini index d9c2e92ce..1145a9f0d 100644 --- a/.github/mypy/mypy.ini +++ b/.github/mypy/mypy.ini @@ -1,5 +1,6 @@ [mypy] plugins = pydantic.mypy +mypy_path = . [mypy-viv_utils.*] ignore_missing_imports = True @@ -34,6 +35,21 @@ ignore_missing_imports = True [mypy-pefile.*] ignore_missing_imports = True +[mypy-intervaltree.*] +ignore_missing_imports = True + +[mypy-lancelot.*] +ignore_missing_imports = True + +[mypy-capa.*] +ignore_missing_imports = True + +[mypy-virustotal3.*] +ignore_missing_imports = True + +[mypy-colorama.*] +ignore_missing_imports = True + [mypy-requests.*] ignore_missing_imports = True diff --git a/.github/pyinstaller/floss.spec b/.github/pyinstaller/floss.spec index 9ec067a92..0e08561ea 100644 --- a/.github/pyinstaller/floss.spec +++ b/.github/pyinstaller/floss.spec @@ -1,21 +1,38 @@ # -*- mode: python -*- -# Copyright 2017 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import subprocess +from PyInstaller.utils.hooks import collect_submodules + +# layout/tags are imported lazily from floss.pipeline; collect them so the +# standalone binary still bundles the full layout-aware static path. +layout_tags_hiddenimports = ( + collect_submodules("floss.layout") + + collect_submodules("floss.tags") + + [ + "floss.ranges", + "elftools", + "lancelot", + "machofile", + "dnfile", + "msgspec", + ] +) + # when invoking pyinstaller from the project root, # this gets run from the project root. with open("./floss/version.py", "wb") as f: @@ -33,6 +50,54 @@ with open("./floss/version.py", "wb") as f: ) f.write(("__version__ = '%s'" % version).encode("utf-8")) +datas = [ + # when invoking pyinstaller from the project root, + # this gets invoked from the directory of the spec file, + # i.e. ./.github/pyinstaller + ('../../floss/sigs', 'sigs'), + # tag databases + ('../../floss/tags/data/crt/*.jsonl.gz', 'floss/tags/data/crt'), + ('../../floss/tags/data/expert/*.jsonl', 'floss/tags/data/expert'), + ('../../floss/tags/data/gp/*.jsonl.gz', 'floss/tags/data/gp'), + ('../../floss/tags/data/gp/*.bin', 'floss/tags/data/gp'), + ('../../floss/tags/data/oss/*.jsonl.gz', 'floss/tags/data/oss'), + ('../../floss/tags/data/winapi/*.txt.gz', 'floss/tags/data/winapi'), +] + +excludes = [ + # ignore packages that would otherwise be bundled with the .exe. + # review: build/pyinstaller/xref-pyinstaller.html + # we don't do any GUI stuff, so ignore these modules + "tkinter", + "_tkinter", + "Tkinter", + + # tqdm provides renderers for ipython, + # however, this drags in a lot of dependencies. + # since we don't spawn a notebook, we can safely remove these. + "IPython", + "ipywidgets", + + # these are pulled in by networkx + # but we don't need to compute the strongly connected components. + "numpy", + "scipy", + "matplotlib", + "pandas", + "pytest", + + # deps from viv that we don't use. + # this duplicates the entries in `hook-vivisect`, + # but works better this way. + "vqt", + "vdb.qt", + "envi.qt", + "PyQt5", + "qt5", + "pyqtwebengine", + "pyasn1", +] + a = Analysis( # when invoking pyinstaller from the project root, # this gets invoked from the directory of the spec file, @@ -40,48 +105,11 @@ a = Analysis( ["../../floss/main.py"], pathex=["floss"], binaries=[], - datas=[ - # when invoking pyinstaller from the project root, - # this gets invoked from the directory of the spec file, - # i.e. ./.github/pyinstaller - ('../../floss/sigs', 'sigs'), - ], - hiddenimports=[], + datas=datas, + hiddenimports=layout_tags_hiddenimports, hookspath=[".github/pyinstaller/hooks"], runtime_hooks=[], - excludes=[ - # ignore packages that would otherwise be bundled with the .exe. - # review: build/pyinstaller/xref-pyinstaller.html - # we don't do any GUI stuff, so ignore these modules - "tkinter", - "_tkinter", - "Tkinter", - - # tqdm provides renderers for ipython, - # however, this drags in a lot of dependencies. - # since we don't spawn a notebook, we can safely remove these. - "IPython", - "ipywidgets", - - # these are pulled in by networkx - # but we don't need to compute the strongly connected components. - "numpy", - "scipy", - "matplotlib", - "pandas", - "pytest", - - # deps from viv that we don't use. - # this duplicates the entries in `hook-vivisect`, - # but works better this way. - "vqt", - "vdb.qt", - "envi.qt", - "PyQt5", - "qt5", - "pyqtwebengine", - "pyasn1", - ], + excludes=excludes, win_no_prefer_redirects=False, win_private_assemblies=False, noarchive=False, diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml new file mode 100644 index 000000000..8be8631a1 --- /dev/null +++ b/.github/workflows/build-oss-db.yml @@ -0,0 +1,127 @@ +name: Build OSS String Databases + +on: + # Rebuild the databases bi-weekly (1st and 15th of each month at 00:00 UTC). + schedule: + - cron: '0 0 1,15 * *' + + # Allow manual runs from the Actions tab. + workflow_dispatch: + +# Cancel any in-progress run when a newer one starts. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +jobs: + build-databases: + name: Build OSS string databases + runs-on: windows-latest + + steps: + - name: Checkout flare-floss + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + path: flare-floss + lfs: true + + - name: Checkout lancelot + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: williballenthin/lancelot + path: lancelot + lfs: true + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Set up Rust + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # v1 + with: + toolchain: stable + + - name: Build jh + run: | + cargo build --release -p lancelot-bin + working-directory: lancelot + + - name: Build OSS databases + run: | + python scripts/tags/build_oss_db.py ` + --config floss\tags\data\oss\libraries.json ` + --jh-path ..\lancelot\target\release\jh.exe ` + --output-dir floss\tags\data\oss ` + --continue-on-error + working-directory: flare-floss + + - name: Show metrics summary + if: success() || failure() + run: | + Get-Content floss\tags\data\oss\build_metrics.json + working-directory: flare-floss + + - name: Show entry-level diff + if: success() || failure() + run: | + if (Test-Path floss\tags\data\oss\build_diff.txt) { + Get-Content floss\tags\data\oss\build_diff.txt + } else { + Write-Host "build_diff.txt not produced" + } + working-directory: flare-floss + + - name: Prepare PR body + if: success() || failure() + shell: pwsh + run: | + # Markdown already has per-library ## headings and ```diff fences. + $diffPath = "flare-floss/floss/tags/data/oss/build_diff_pr.txt" + $diff = if (Test-Path $diffPath) { + Get-Content $diffPath -Raw + } else { + "(no entry-level diff produced)" + } + $librariesConfig = Get-Content "flare-floss/floss/tags/data/oss/libraries.json" -Raw | ConvertFrom-Json + $body = @( + "Automated bi-weekly rebuild of the OSS string databases.", + "", + "- Triplet: ``$($librariesConfig.triplet)``", + "- Compiler: ``$($librariesConfig.compiler)``", + "- Library list: see ``floss/tags/data/oss/libraries.json``", + "", + "Per-library entry counts and timing are printed in the workflow logs (``build_metrics.json``).", + "", + "Entry-level diff (up to 20 lines per library):", + "", + $diff.TrimEnd(), + "" + ) -join "`n" + # GitHub rejects PR bodies over 65536 characters ("Body is too long"). + # The Python script already caps the diff; this is a final safety net. + $maxBodyChars = 65000 + if ($body.Length -gt $maxBodyChars) { + $notice = "`n`n... truncated to stay under GitHub's 65536-character PR body limit." + $keep = [Math]::Max(0, $maxBodyChars - $notice.Length) + $body = $body.Substring(0, $keep) + $notice + } + # Write outside the flare-floss checkout so create-pull-request does not commit it. + Set-Content -Path pr-body.md -Value $body -Encoding utf8 + - name: Create Pull Request if databases changed + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 + with: + path: flare-floss + base: master + commit-message: 'chore(oss-db): update string databases' + title: 'Update OSS string databases' + # Relative to $GITHUB_WORKSPACE (not the path: checkout). + body-path: pr-body.md + branch: update-oss-string-databases + # Always open a fresh PR (e.g. update-oss-string-databases-6qj97jr). + branch-suffix: random + delete-branch: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f428e6cb..28b531336 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,9 +2,13 @@ name: build on: pull_request: - branches: [ master ] + branches: [ master, quantumstrand ] release: types: [edited, published] + branches: [ master ] + +permissions: + contents: read jobs: build: @@ -29,6 +33,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: false + lfs: true - name: Set up Python 3.10 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: @@ -39,6 +44,18 @@ jobs: pip install -e .[build] - name: Build standalone executable run: pyinstaller .github/pyinstaller/floss.spec + # build job uses submodules: false; smoke test needs a real PE sample + - name: Checkout test sample + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: mandiant/flare-floss-testfiles + path: tests/data + - name: Smoke test layout/tags via standalone floss + run: | + chmod +x dist/${{ matrix.artifact_name }} + ./dist/${{ matrix.artifact_name }} --help + # static-only layout path (default) should include tag annotations in JSON + ./dist/${{ matrix.artifact_name }} tests/data/test-decode-to-stack.exe --string-type static -j | grep -q '#common' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: ${{ matrix.asset_name }} @@ -111,7 +128,7 @@ jobs: - name: Set zip name run: echo "zip_name=floss-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} - run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} + run: zip "$zip_name" "${{ matrix.artifact_name }}" - name: Upload ${{ env.zip_name }} to GH Release uses: svenstaro/upload-release-action@29e53e917877a24fad85510ded594ab3c9ca12de # 2.11.5 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 553d326e3..1f159e35b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,9 +2,17 @@ name: CI on: push: - branches: [ master ] + branches: [ master, quantumstrand ] pull_request: - branches: [ master ] + branches: [ master, quantumstrand ] + +permissions: + contents: read + +# Cancel outdated runs when a new commit is pushed to the same branch/PR. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true # save workspaces to speed up testing env: @@ -17,6 +25,9 @@ jobs: steps: - name: Checkout FLOSS uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + # tag DBs and FLIRT sigs are LFS; mypy does not need them but keep checkout consistent + lfs: true - name: Set up Python 3.10 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: @@ -51,6 +62,8 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: true + # layout/tag DBs and FLIRT signatures are stored in Git LFS + lfs: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: diff --git a/.github/workflows/web-release.yml b/.github/workflows/web-release.yml new file mode 100644 index 000000000..bee6e77cb --- /dev/null +++ b/.github/workflows/web-release.yml @@ -0,0 +1,55 @@ +name: web-release + +on: + push: + branches: [ master ] + paths: + - 'viewer/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: 'npm' + cache-dependency-path: 'viewer/package-lock.json' + + - name: Install dependencies + run: npm install + working-directory: ./viewer + + - name: Build + run: npm run build + working-directory: ./viewer + + - name: Upload artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + # vite-plugin-singlefile puts everything into index.html in the dist folder + path: ./viewer/dist + + deploy: + name: Deploy site to GitHub Pages + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + permissions: + contents: read + pages: write + id-token: write + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.gitignore b/.gitignore index 32cf7f990..dcc4db1f9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,15 +21,14 @@ lib/ # Test executables bin/ -# PyCharm .idea venv flare_floss.egg-info .eggs - - -# vscode +.direnv/ +.envrc .vscode .direnv/ .env/ .envrc +bun.lock diff --git a/AGENTS.md b/AGENTS.md index 1e7404678..383b00051 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,3 +3,4 @@ - To set up the development environment, make sure that a venv is created and the pre-commit and pre-push hooks are installed, see `.pre-commit-config.yaml` - All lints, formatters and tests in `.github/workflows` **must** pass before making a PR. Enforce this strictly. - The `floss/` folder has the main functionality, while `scripts/` has auxiliary plugins and scripts. Docs are in `doc/`. +- Result caching is on by default. When reproducing or diffing output across commits, disable it with `FLOSS_CACHE_ENABLE=0` (or `FLOSS_CACHE_DIR` to point it at a scratch dir) — otherwise a run may serve a cached document written by earlier code. diff --git a/MANIFEST.in b/MANIFEST.in index a67565cf7..a8a28fda0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE.txt graft floss/sigs +graft floss/tags/data diff --git a/README.md b/README.md index 189795ab0..42f1fe6e7 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,41 @@ Not all compilers use string formats that the classic `strings.exe` algorithm su 1. Go 2. Rust -The strings FLOSS extracts specific to a compiler are much easier to inspect by humans. +The strings FLOSS extracts specific to a compiler are much easier to inspect by humans. Please consult the documentation to learn more about the [language-specific string extraction](doc/language_specific_strings.md). +### Layout-aware static strings + +FLOSS enriches static strings by default with +file structure context and tags (global prevalence, open-source libraries, +expert rules, and more). Stack, tight, and decoded strings still appear after +the layout-aware static listing when deobfuscation is enabled. + +```console +$ floss sample.exe +$ floss sample.exe -j +``` + +Features: + +- extract ASCII and UTF-16LE strings +- show strings next to right-aligned, colored context, including tags and file offset +- render strings within PE section range delimiters +- annotate strings from known PE structures, like the import table +- don't show junk strings that overlap with instructions +- mute strings known to be globally prevalent, via an embedded database +- mute strings from popular open source libraries, via embedded databases +- highlight strings that match expert rules, via embedded databases + +![screenshot 1](https://github.com/mandiant/flare-floss/assets/156560/f2d471a3-2624-498c-aaa9-928e2909c338) +![screenshot 2](https://github.com/mandiant/flare-floss/assets/156560/23bd20a1-7dff-46b5-be65-12582cb90d64) + +Tag databases and FLIRT signature files are tracked with Git LFS; contributors +cloning the repo may need Git LFS installed to fetch those files. Maintenance of +tag databases is documented in [scripts/tags/README.md](scripts/tags/README.md) +and the per-database notes under `floss/tags/data/`. + ## Installation To use FLOSS, download a standalone executable file from the releases page: https://github.com/mandiant/flare-floss/releases @@ -56,22 +87,21 @@ Extract obfuscated strings from a malware binary: Only extract stack and tight strings: - $ floss --only stack tight -- suspicious.exe + $ floss --string-type stack tight -- suspicious.exe Do not extract static strings: - $ floss --no static -- backdoor.exe + $ floss --no-string-type static -- backdoor.exe Display the help/usage screens: - $ floss -h # show core arguments - $ floss -H # show all supported arguments + $ floss -h # show all supported arguments For a detailed description of using FLOSS, review the documentation [here](doc/usage.md). ## Scripts -FLOSS also contains additional Python scripts in the [scripts](scripts) directory +FLOSS also contains additional Python scripts in the [scripts](scripts) directory which can be used to load its output into other tools such as Binary Ninja or IDA Pro. For detailed description of these scripts review the documentation [here](scripts/README.md). diff --git a/doc/usage.md b/doc/usage.md index 41ec45f23..c824d92ba 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -18,9 +18,7 @@ Since FLOSS also extracts static strings (like `strings.exe`), Here's a summary of the command line flags and options you can provide to FLOSS to modify its behavior. -See `floss -h` for all supported arguments and usage examples. This displays the most used arguments only. - -To see all supported arguments run `floss -H`. +See `floss -h` for all supported arguments and usage examples. ### Extract static, obfuscated, and stack strings (default mode) @@ -42,33 +40,33 @@ FLOSS can identify programs compiled from selected programming languages and ext By default, this process is automatic. However, you can use the `--language` argument to manually select or disable this feature. -### Disable string type extraction (`--no {static,decoded,stack,tight}`) +### Disable string type extraction (`--no-string-type {static,decoded,stack,tight}`) When FLOSS searches for static strings, it looks for human-readable ASCII and UTF-16 strings across the entire binary contents of the file. This means you may be able to replace `strings.exe` with FLOSS in your analysis workflow. However, you may disable - the extraction of static strings via the `--no static` switch. + the extraction of static strings via the `--no-string-type static` switch. - floss.exe --no static -- malware.exe + floss.exe --no-string-type static -- malware.exe -Since `--no` supports multiple arguments, end the command options with a double dash `--`. +Since `--no-string-type` supports multiple arguments, end the command options with a double dash `--`. Analogous, you can disable the extraction of obfuscated strings, stackstrings or any combination. - floss.exe --no decoded -- malware.exe - floss.exe --no stack tight -- malware.exe + floss.exe --no-string-type decoded -- malware.exe + floss.exe --no-string-type stack tight -- malware.exe -### Enable string type extraction (`--only {static,decoded,stack,tight}`) +### Enable string type extraction (`--string-type {static,decoded,stack,tight}`) Sometimes it's easier to specify only the string type(s) you want to extract. -Use the `--only` option for that. +Use the `--string-type` option for that. - floss.exe --only decoded -- malware.exe + floss.exe --string-type decoded -- malware.exe -Please note that `--no` and `--only` cannot be used at the same time. +Please note that `--string-type` and `--no-string-type` cannot be used at the same time. ### Write output as JSON (`-j/--json`) @@ -76,11 +74,12 @@ Write FLOSS results to `stdout` structured in JSON to make it easy to ingest by floss.exe -j malware.exe > malware_strings.json -### Load FLOSS results (`-l/--load`) +### Load FLOSS results (automatic) -Load a FLOSS results JSON document. This allows to explore FLOSS results without re-running the analysis. +Loading a saved FLOSS results JSON document is automatic and detected from the file +content, so you can explore results without re-running the analysis. - floss.exe -l malware_floss_results.json + floss.exe malware_floss_results.json ### Verbose results (`-v`) @@ -120,10 +119,10 @@ Supplying a larger minimum length reduces the chances floss.exe -n 10 malware.exe -### Decoding function specification (`--functions`) +### Decoding function specification (`--analyze-functions`) You can instruct FLOSS to decode the strings provided - to specific functions by using the `--functions` + to specific functions by using the `--analyze-functions` option. By default, FLOSS uses heuristics to identify decoding routines in malware. @@ -136,8 +135,9 @@ This can improve performance as FLOSS by perhaps one-third (on the order of seconds, so it is usually _not_ worth it to always manually identify decoding routines). Specify functions by using their hex-encoded virtual address. +Since `--analyze-functions` accepts multiple arguments, end the command options with a double dash `--`. - floss.exe --functions 0x401000 0x402000 malware.exe + floss.exe --analyze-functions 0x401000 0x402000 -- malware.exe ### Install/Uninstall right click menu option for Windows (`--install-right-click-menu/--uninstall-right-click-menu`) diff --git a/floss/cache.py b/floss/cache.py new file mode 100644 index 000000000..4b04dae81 --- /dev/null +++ b/floss/cache.py @@ -0,0 +1,340 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Result-document caching for repeated FLOSS analyses. + +The full ResultDocument is cached on first execution and reused on later runs, +so repeated analyses of the same sample are fast. The cache is keyed by the +SHA-256 of the sample bytes plus the FLOSS version, and stores the same JSON +schema emitted by ``--json``. + +Caching applies to binary sample analysis only: loading a user-supplied JSON +document never writes to the cache. + +The cache directory defaults to the platform cache directory and can be +overridden with ``FLOSS_CACHE_DIR``. Caching can be disabled entirely by +setting ``FLOSS_CACHE_ENABLE=0``, and a single run can be forced to +re-analyze and overwrite its entry with ``FLOSS_CACHE_REFRESH=1``. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +from typing import Optional +from pathlib import Path + +from platformdirs import user_cache_dir + +import floss.logging_ +import floss.render.json +from floss.results import ( + STRING_TYPE_FIELDS, + Analysis, + ResultLayout, + ResultDocument, + filter_string_len, + check_set_string_types, +) + +logger = floss.logging_.getLogger(__name__) + +ENV_CACHE_DIR = "FLOSS_CACHE_DIR" +ENV_CACHE_ENABLE = "FLOSS_CACHE_ENABLE" +ENV_CACHE_REFRESH = "FLOSS_CACHE_REFRESH" + +# values that disable caching, so FLOSS_CACHE_ENABLE=0 behaves as documented +_DISABLED_VALUES = ("0", "false", "no", "n", "") +# values that force a re-analysis, so FLOSS_CACHE_REFRESH=1 behaves as documented +_ENABLED_VALUES = ("1", "true", "yes", "y") + + +def cache_enabled() -> bool: + """Whether result caching is enabled. + + ``FLOSS_CACHE_ENABLE`` disables caching when set to 0 (or false/no/n, + case-insensitive); caching is enabled by default. + """ + return os.environ.get(ENV_CACHE_ENABLE, "1").lower() not in _DISABLED_VALUES + + +def cache_refresh() -> bool: + """Whether the current run should bypass the cache and overwrite its entry. + + ``FLOSS_CACHE_REFRESH=1`` (or true/yes/y, case-insensitive) forces a + re-analysis: the cached document is ignored and the fresh result is stored + on top of it. Unset by default. + """ + return os.environ.get(ENV_CACHE_REFRESH, "").lower() in _ENABLED_VALUES + + +def get_cache_dir() -> Path: + """Resolve the analysis cache directory. + + ``FLOSS_CACHE_DIR`` overrides the platform default cache directory: + - Linux: ``$XDG_CACHE_HOME/floss`` (or ``~/.cache/floss``) + - macOS: ``~/Library/Caches/floss`` + - Windows: ``%LOCALAPPDATA%\\floss\\Cache`` + """ + override = os.environ.get(ENV_CACHE_DIR) + if override: + return Path(override) + return Path(user_cache_dir("floss")) + + +def compute_key(sha256: str, version: str, format: str = "auto") -> str: + """The cache key: content-addressed sample hash + analysis format + version. + + The analysis format is part of the key so interpreting the same sample bytes + under a different ``--format`` (e.g. sc32 vs sc64) never serves a stale + document. + """ + return f"{sha256}-{format}-{version}" + + +def cache_file_path(cache_dir: Path, key: str) -> Path: + """The on-disk location of a cache entry: ``{cache_dir}/{key}.json``.""" + return cache_dir / f"{key}.json" + + +def load(cache_dir: Path, key: str, sha256: str, version: str) -> Optional[ResultDocument]: + """Load and validate a cached document, or None on a miss. + + On a parse failure or a checksum or version mismatch the entry is dropped + so the caller re-analyzes and stores a fresh document. + """ + path = cache_file_path(cache_dir, key) + try: + if not path.is_file(): + return None + except OSError: + # e.g. a locked parent directory makes stat() raise PermissionError; + # treat it as a miss so the analysis runs instead of crashing + return None + + try: + doc = ResultDocument.parse_file(path) + except (OSError, UnicodeDecodeError, ValueError) as e: + logger.warning("dropping invalid cache entry %s: %s", path.name, e) + _drop_cache_entry(path) + return None + + if doc.metadata.sha256 != sha256 or doc.metadata.version != version: + logger.warning("dropping stale cache entry %s (checksum/version mismatch)", path.name) + _drop_cache_entry(path) + return None + + return doc + + +def _drop_cache_entry(path: Path) -> None: + """Best-effort removal of a stale cache entry; never raises. + + The entry may be held open by another reader or an antivirus scanner (e.g. + on Windows), so removal can fail with a PermissionError. Leave it and let + the caller continue. + """ + try: + path.unlink(missing_ok=True) + except OSError as e: + logger.warning("could not remove cache entry %s: %s", path.name, e) + + +def store(cache_dir: Path, key: str, doc: ResultDocument) -> bool: + """Atomically write a result document to the cache. + + The write is guarded by a lock file so concurrent first runs cannot corrupt + the entry. When the lock cannot be acquired, caching is skipped and False is + returned (the caller decides how to warn). + """ + try: + cache_dir.mkdir(parents=True, exist_ok=True) + except OSError as e: + logger.warning("could not create cache directory %s: %s; skipping cache write", cache_dir, e) + return False + + lock_path = cache_dir / f"{key}.lock" + lock_fd = _acquire_lock(lock_path) + if lock_fd is None: + logger.warning("could not acquire cache lock %s; skipping cache write", lock_path) + return False + + try: + return _write_atomic(cache_dir, key, floss.render.json.render(doc)) + finally: + _release_lock(lock_fd, lock_path) + + +def _write_atomic(cache_dir: Path, key: str, payload: str) -> bool: + """Atomically write ``{key}.json`` into the cache, or False on any OS failure. + + The payload is written to a temporary file in the cache directory and then + renamed into place. Cache writes are best-effort: a failure (disk full, an + unwritable cache directory, the destination held open by an antivirus + scanner or another reader, etc.) must not crash the analysis, so it is + logged and caching is skipped. + """ + try: + fd, tmp_name = tempfile.mkstemp(dir=str(cache_dir), prefix=f"{key}.", suffix=".tmp") + except OSError as e: + logger.warning("could not create temporary cache file in %s: %s; skipping cache write", cache_dir, e) + return False + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(payload) + os.replace(tmp_name, cache_file_path(cache_dir, key)) + return True + except OSError as e: + logger.warning("could not store cache entry %s: %s; skipping cache write", key, e) + return False + finally: + try: + os.unlink(tmp_name) + except OSError: + pass + + +def covers(cached: ResultDocument, wanted: Analysis, min_length: int) -> bool: + """Whether a cached document satisfies the requested analysis. + + Every string type the user wants enabled must be present in the cached + document, and the requested ``--minimum-length`` must not be below what the + document was built with (shorter strings were dropped at extraction time + and cannot be recovered). Layout and tags are not part of the match: a + cached layout/tags document can satisfy a request without them because + `materialize()` drops the layout and redacts the tags when they are not + wanted. The reverse (requesting layout/tags the cache was not built with) is + a miss. + """ + for field in STRING_TYPE_FIELDS: + if getattr(wanted, field) and not getattr(cached.analysis, field): + return False + + if wanted.enable_layout and not cached.analysis.enable_layout: + return False + if wanted.enable_tags and not cached.analysis.enable_tags: + return False + + if min_length < cached.metadata.min_length: + return False + + return True + + +def materialize(doc: ResultDocument, sample: Path, analysis: Analysis, min_length: int) -> ResultDocument: + """Apply the same post-load filtering as loading a user-supplied JSON document. + + Rendering flags (--query, --columns, --max-strings, and the tag and section + filters) apply at render time, so they work unchanged on cached results. + """ + doc.metadata.file_path = str(sample) + check_set_string_types(doc, analysis) + doc.analysis.enable_layout = analysis.enable_layout + doc.analysis.enable_tags = analysis.enable_tags + # mirror a fresh run: a disabled string type is absent from the document, + # not merely flagged. a cache hit must not emit data the user excluded. + if not analysis.enable_static_strings: + doc.strings.static_strings = [] + doc.layout = None + if not analysis.enable_stack_strings: + doc.strings.stack_strings = [] + if not analysis.enable_tight_strings: + doc.strings.tight_strings = [] + if not analysis.enable_decoded_strings: + doc.strings.decoded_strings = [] + if not analysis.enable_language_strings: + doc.strings.language_strings = [] + doc.strings.language_strings_missed = [] + if not analysis.enable_layout: + doc.layout = None + if not analysis.enable_tags: + _clear_tags(doc) + filter_string_len(doc, min_length) + # mirror a fresh run: the requested -n is what the document reports, so + # --json does not advertise a looser extraction threshold than it holds + doc.metadata.min_length = min_length + return doc + + +def _clear_tags(doc: ResultDocument) -> None: + """Redact tag classifications from a document (tags-off requests).""" + for s in doc.strings.static_strings: + s.tags.clear() + for s in doc.strings.language_strings: + s.tags.clear() + for s in doc.strings.language_strings_missed: + s.tags.clear() + _clear_layout_tags(doc.layout) + + +def _clear_layout_tags(layout: Optional[ResultLayout]) -> None: + if layout is None: + return + for s in layout.strings: + s.tags.clear() + for child in layout.children: + _clear_layout_tags(child) + + +def _acquire_lock(lock_path: Path) -> Optional[int]: + """Acquire an exclusive lock on the given lock file, non-blocking. + + Returns the open file descriptor when the lock is held, or None when + another process holds it. + """ + try: + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600) + except OSError: + return None + + try: + if sys.platform == "win32": + import msvcrt + + # msvcrt.locking cannot lock a byte range past EOF, so ensure the + # lock file has at least one byte before taking the lock. + if os.fstat(fd).st_size == 0: + os.write(fd, b"\0") + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + return None + + return fd + + +def _release_lock(fd: int, lock_path: Path) -> None: + """Release a lock previously acquired by `_acquire_lock()`.""" + if sys.platform == "win32": + import msvcrt + + try: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + except OSError: + pass + os.close(fd) + # the lock is never contended: acquire is non-blocking and skips on failure, + # so no other process can be waiting on the file we just released. unlink it + # to keep the cache directory clean. + try: + lock_path.unlink(missing_ok=True) + except OSError: + pass diff --git a/floss/cli.py b/floss/cli.py new file mode 100644 index 000000000..503acdf27 --- /dev/null +++ b/floss/cli.py @@ -0,0 +1,430 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI argument parsing for FLOSS.""" + +from __future__ import annotations + +import sys +import logging +import argparse +import textwrap +from enum import Enum +from typing import List, Optional +from pathlib import Path + +import floss.utils +import floss.logging_ +from floss.const import ( + MEGABYTE, + MAX_FILE_SIZE, + MIN_STRING_LENGTH, +) +from floss.utils import set_vivisect_log_level +from floss.render import Verbosity +from floss.version import __version__ +from floss.logging_ import TRACE, DebugLevel +from floss.render.filter import NOISY_TAGS, TAG_FAMILIES, KNOWN_STRUCTURE_SLUGS +from floss.render.layout import COLUMN_CHOICES, DEFAULT_COLUMNS +from floss.language.identify import Language + +logger = floss.logging_.getLogger("floss") + +SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)" + + +EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") +EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") + + +class StringType(str, Enum): + STATIC = "static" + STACK = "stack" + TIGHT = "tight" + DECODED = "decoded" + LANGUAGE = "language" + ALL = "all" + + +# concrete string types; `all` is a convenience alias for the full set +CONCRETE_STRING_TYPES = (StringType.STATIC, StringType.STACK, StringType.TIGHT, StringType.DECODED, StringType.LANGUAGE) + +# string types selectable via --string-type / --no-string-type +STRING_TYPE_CHOICES = [t.value for t in StringType] + + +class WorkspaceLoadError(ValueError): + pass + + +class ArgumentValueError(ValueError): + pass + + +class ArgumentParser(argparse.ArgumentParser): + """ + argparse will call sys.exit upon parsing invalid arguments. + we don't want that, because we might be parsing args within test cases, run as a module, etc. + so, we override the behavior to raise a ArgumentValueError instead. + + this strategy is originally described here: https://stackoverflow.com/a/16942165/87207 + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # when a JSON output mode is active, parsing errors emit a single JSON + # object on STDERR, without the usage text + self.json_mode = False + + def error(self, message): + if not self.json_mode: + self.print_usage(sys.stderr) + args = {"prog": self.prog, "message": message} + raise ArgumentValueError("%(prog)s: error: %(message)s" % args) + + +def make_parser(): + desc = ( + "The FLARE team's open-source tool to extract ALL strings from malware.\n" + f" %(prog)s {__version__} - https://github.com/mandiant/flare-floss/\n\n" + "FLOSS extracts the following string types:\n" + ' 1. static strings: "regular" ASCII and UTF-16LE strings\n' + " 2. stack strings: strings constructed on the stack at run-time\n" + " 3. tight strings: special form of stack strings, decoded on the stack\n" + " 4. decoded strings: strings decoded in a function\n\n" + "Language-specific strings:\n" + " 1. Go: strings from binaries written in Go\n" + " 2. Rust: strings from binaries written in Rust\n\n" + "By default, static strings are layout-aware with tags for PE/ELF/Mach-O\n" + "(section context, prevalence/library/expert tags).\n" + ) + epilog = textwrap.dedent(""" + examples: + extract all strings from an executable + floss suspicious.exe + + classic flat list of strings without layout and tags + floss --plain suspicious.exe + + do not extract static strings + floss --no-string-type static -- suspicious.exe + + only extract stack and tight strings + floss --string-type stack tight -- suspicious.exe + + extract strings from 32-bit shellcode + floss -f sc32 shellcode.bin + + only decode strings from the specified functions + floss --analyze-functions 0x401000 0x401100 -- suspicious.exe + + only show static strings from the .rdata section + floss --section .rdata -- suspicious.exe + + only show strings tagged winapi or openssl + floss --tag winapi openssl -- suspicious.exe + + hide noisy strings and search for a pattern in the layout tree + floss --interesting --query "http://" -- suspicious.exe + + emit a concise summary instead of the full listing + floss --summary suspicious.exe + + extract strings from a binary written in Go (if automatic language identification fails) + floss --language go program.exe + + environment variables: + FLOSS_CACHE_DIR directory for the analysis result cache (default: platform cache directory) + FLOSS_CACHE_ENABLE set to 0 to disable result caching (default: enabled) + FLOSS_CACHE_REFRESH set to 1 to ignore cached results and overwrite the entry (default: disabled) + """) + + parser = ArgumentParser( + description=desc, + epilog=epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-n", + "--minimum-length", + dest="min_length", + type=int, + default=MIN_STRING_LENGTH, + help="minimum string length", + ) + + parser.add_argument( + "sample", + type=argparse.FileType("rb"), + help="path to sample to analyze", + ) + + analysis_group = parser.add_argument_group("analysis arguments") + analysis_group.add_argument( + "--string-type", + action="extend", + dest="enabled_string_types", + nargs="+", + choices=STRING_TYPE_CHOICES, + default=[], + help="only extract specified string type(s); valid values: %s" % ", ".join(STRING_TYPE_CHOICES), + ) + analysis_group.add_argument( + "--no-string-type", + action="extend", + dest="disabled_string_types", + nargs="+", + choices=STRING_TYPE_CHOICES, + default=[], + help="do not extract specified string type(s); valid values: %s" % ", ".join(STRING_TYPE_CHOICES), + ) + + filter_group = parser.add_argument_group("filtering arguments") + structure_examples = ", ".join(KNOWN_STRUCTURE_SLUGS) + for flag, dest, metavar, example, help_ in ( + ( + "--section", + "include_sections", + "NAME", + "e.g. .rdata", + "only show static strings in the given binary section(s)", + ), + ( + "--no-section", + "exclude_sections", + "NAME", + None, + "do not show static strings in the given binary section(s)", + ), + ( + "--structure", + "include_structures", + "NAME", + f"e.g. {structure_examples}; names are slugs and match regardless of separators; run --summary " + "to see the structures present in a specific sample", + "only show static strings in the given binary structure(s)", + ), + ( + "--no-structure", + "exclude_structures", + "NAME", + None, + "do not show static strings in the given binary structure(s)", + ), + ( + "--tag", + "include_tags", + "TAG", + "e.g. winapi, crypto, or a tag family: %s; run --summary to see the tags present in a specific sample" + % ", ".join(sorted(TAG_FAMILIES)), + "only show strings with the given semantic tag(s)", + ), + ( + "--no-tag", + "exclude_tags", + "TAG", + None, + "do not show strings with the given semantic tag(s)", + ), + ): + help_text = help_ + (f"; {example}" if example else "") + filter_group.add_argument( + flag, + action="extend", + dest=dest, + nargs="+", + metavar=metavar, + default=[], + help=help_text, + ) + filter_group.add_argument( + "--interesting", + action="store_true", + dest="interesting", + default=False, + help="exclude strings with noisy tags: %s" % ", ".join(sorted(NOISY_TAGS)), + ) + filter_group.add_argument( + "--query", + action="extend", + dest="queries", + nargs="+", + metavar="REGEX", + default=[], + help="only show strings matching the given regular expression(s); repeatable, patterns are ORed", + ) + filter_group.add_argument( + "--max-strings", + dest="max_strings", + type=int, + default=None, + metavar="N", + help="cap the emitted strings per section to the top N by relevance (highlighted, then " + "untagged, then tagged, ascending by offset)", + ) + + advanced_group = parser.add_argument_group("advanced arguments") + formats = [ + ("auto", "(default) detect file type automatically"), + ("pe", "Windows PE file"), + ("sc32", "32-bit shellcode"), + ("sc64", "64-bit shellcode"), + ] + format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats]) + advanced_group.add_argument( + "-f", + "--format", + choices=[f[0] for f in formats], + default="auto", + help="select sample format, %s" % format_help, + ) + advanced_group.add_argument( + "--language", + type=str, + choices=[Language.AUTO.value, Language.GO.value, Language.RUST.value, Language.DISABLED.value], + default=Language.AUTO.value, + help="use language-specific string extraction, auto-detect language by default, disable using 'none'", + ) + advanced_group.add_argument( + "--analyze-functions", + dest="analyze_functions", + type=lambda x: int(x, 0x10), + default=None, + nargs="+", + help="only analyze the specified functions, hex-encoded like 0x401000, space-separate multiple functions", + ) + advanced_group.add_argument( + "--signatures", + type=str, + default=SIGNATURES_PATH_DEFAULT_STRING, + help="path to .sig/.pat file or directory used to identify library functions, use embedded signatures by default", + ) + advanced_group.add_argument( + "-L", + "--large-file", + action="store_true", + help="allow processing files larger than {} MB".format(int(MAX_FILE_SIZE / MEGABYTE)), + ) + advanced_group.add_argument( + "--version", + action="version", + version="%(prog)s {:s}".format(__version__), + help="show program's version number and exit", + ) + if sys.platform == "win32": + advanced_group.add_argument( + "--install-right-click-menu", + action=floss.utils.InstallContextMenu, + help="install FLOSS to the right-click context menu for Windows Explorer and exit", + ) + + advanced_group.add_argument( + "--uninstall-right-click-menu", + action=floss.utils.UninstallContextMenu, + help="uninstall FLOSS from the right-click context menu for Windows Explorer and exit", + ) + + output_group = parser.add_argument_group("rendering arguments") + output_group.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text") + output_group.add_argument( + "--summary", + action="store_true", + default=False, + help="emit a concise summary (metadata, counts, tag histogram, high-value strings); " + "static-only by default unless string types are explicitly selected", + ) + output_group.add_argument( + "-v", + "--verbose", + action="count", + default=Verbosity.DEFAULT, + help="enable verbose results, e.g. including function offsets (does not affect JSON output)", + ) + output_group.add_argument( + "--plain", + action="store_true", + default=False, + help="render the classic flat list of strings without layout and tags", + ) + output_group.add_argument( + "--columns", + dest="columns", + action="extend", + nargs="+", + choices=COLUMN_CHOICES, + default=[], + help="columns to show in the layout view; valid values: tags, offset, structure, encoding. Default: tags, offset.", + ) + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument( + "-d", + "--debug", + action="count", + default=DebugLevel.NONE, + help="enable debugging output on STDERR, specify multiple times to increase verbosity", + ) + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output on STDOUT except fatal errors" + ) + logging_group.add_argument( + "--color", + type=str, + choices=("auto", "always", "never"), + default="auto", + help="enable ANSI color codes in results, default: only during interactive session", + ) + + return parser + + +def set_log_config(debug, quiet): + if quiet: + log_level = logging.WARNING + elif debug >= DebugLevel.TRACE: + log_level = TRACE + elif debug >= DebugLevel.DEFAULT: + log_level = logging.DEBUG + else: + log_level = logging.INFO + + logging.basicConfig(level=log_level) + logging.getLogger().setLevel(log_level) + + if debug < DebugLevel.SUPERTRACE: + # these loggers are too verbose even for the TRACE level, enable via `-ddd` + logging.getLogger("floss.api_hooks").setLevel(logging.WARNING) + logging.getLogger("floss.function_argument_getter").setLevel(logging.WARNING) + + # configure vivisect-related logging, it's verbose and not relevant for regular FLOSS users + # enable to do more vigorous testing + if debug < DebugLevel.TRACE: + set_vivisect_log_level(logging.CRITICAL) + else: + set_vivisect_log_level(logging.DEBUG) + + # configure viv-utils logging + if debug == DebugLevel.DEFAULT: + logging.getLogger("viv_utils.emulator_drivers").setLevel(logging.DEBUG) + elif debug <= DebugLevel.TRACE: + logging.getLogger("viv_utils.emulator_drivers").setLevel(logging.ERROR) + + # install the log message colorizer to the default handler. + # because basicConfig is just above this, + # handlers[0] is a StreamHandler to STDERR. + # + # calling this code from outside script main may do something unexpected. + root_handlers = logging.getLogger().handlers + if root_handlers: + root_handlers[0].setFormatter(floss.logging_.ColorFormatter()) diff --git a/floss/enrich.py b/floss/enrich.py new file mode 100644 index 000000000..ee9039755 --- /dev/null +++ b/floss/enrich.py @@ -0,0 +1,115 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Project layout analysis onto FLOSS static/language string results.""" + +from __future__ import annotations + +from typing import Dict, List, Tuple, Optional + +from floss.results import ResultLayout, ResultString, StaticString, StringEncoding + + +def layout_encoding_to_string_encoding(encoding: str) -> StringEncoding: + # layout ExtractedString.encoding is only "ascii" | "unicode" + if encoding == "unicode": + return StringEncoding.UTF16LE + return StringEncoding.ASCII + + +def is_structured_layout(layout_name: str) -> bool: + """True when compute_layout produced PE/ELF/Mach-O (not the binary fallback). + + An XOR-obfuscated PE/ELF header appends `` (XOR decoded with key: 0x...)`` + to the layout name, so strip any parenthetical suffix before matching. + """ + name = layout_name.lower() + if " (" in name: + name = name.split(" (", 1)[0] + return name in ("pe", "elf") or name.startswith("macho") + + +def _walk_offset_index( + layout: ResultLayout, + index: Dict[int, Tuple[ResultString, str]], +) -> None: + for s in layout.strings: + index[s.offset] = (s, layout.name) + for child in layout.children: + _walk_offset_index(child, index) + + +def build_offset_index(layout: ResultLayout) -> Dict[int, Tuple[ResultString, str]]: + """Map file offset → (ResultString, containing layout node name).""" + index: Dict[int, Tuple[ResultString, str]] = {} + _walk_offset_index(layout, index) + return index + + +def static_strings_from_layout(layout: ResultLayout) -> List[StaticString]: + """Flatten a serializable layout tree into enriched StaticString values.""" + index = build_offset_index(layout) + # stable order by offset + items = sorted(index.items(), key=lambda kv: kv[0]) + out: List[StaticString] = [] + for offset, (rs, section) in items: + out.append( + StaticString( + string=rs.string, + offset=offset, + encoding=layout_encoding_to_string_encoding(rs.encoding), + tags=list(rs.tags), + section=section, + structure=rs.structure or "", + ) + ) + return out + + +def enrich_static_string( + s: StaticString, + offset_index: Dict[int, Tuple[ResultString, str]], +) -> StaticString: + """Copy tags/section/structure from layout for a string with a file offset.""" + hit = offset_index.get(s.offset) + if not hit: + return s + rs, section = hit + return StaticString( + string=s.string, + offset=s.offset, + encoding=s.encoding, + tags=list(rs.tags), + section=section, + structure=rs.structure or "", + ) + + +def enrich_static_strings( + strings: List[StaticString], + layout: Optional[ResultLayout] = None, + *, + offset_index: Optional[Dict[int, Tuple[ResultString, str]]] = None, +) -> List[StaticString]: + """ + Project layout tags/section/structure onto static strings by file offset. + + Pass a prebuilt ``offset_index`` when enriching multiple string lists so the + layout tree is walked only once. + """ + if offset_index is None: + if layout is None: + return strings + offset_index = build_offset_index(layout) + return [enrich_static_string(s, offset_index) for s in strings] diff --git a/floss/language/cli_common.py b/floss/language/cli_common.py new file mode 100644 index 000000000..4929b1263 --- /dev/null +++ b/floss/language/cli_common.py @@ -0,0 +1,46 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import argparse + + +def add_common_args(parser: argparse.ArgumentParser, default_min_length: int): + parser.add_argument("path", help="file or path to analyze") + parser.add_argument( + "-n", + "--minimum-length", + dest="min_length", + type=int, + default=default_min_length, + help="minimum string length", + ) + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", + "--quiet", + action="store_true", + help="disable all status output except fatal errors", + ) + + +def configure_logging(args: argparse.Namespace): + if args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) diff --git a/floss/language/go/coverage.py b/floss/language/go/coverage.py index 5081dfc59..7598121e0 100644 --- a/floss/language/go/coverage.py +++ b/floss/language/go/coverage.py @@ -23,6 +23,7 @@ from floss.utils import get_static_strings from floss.results import StaticString, StringEncoding from floss.language.utils import get_extract_stats +from floss.language.cli_common import add_common_args, configure_logging from floss.language.go.extract import extract_go_strings logger = logging.getLogger(__name__) @@ -32,31 +33,10 @@ def main(): parser = argparse.ArgumentParser(description="Get Go strings") - parser.add_argument("path", help="file or path to analyze") - parser.add_argument( - "-n", - "--minimum-length", - dest="min_length", - type=int, - default=MIN_STR_LEN, - help="minimum string length", - ) - logging_group = parser.add_argument_group("logging arguments") - logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") - logging_group.add_argument( - "-q", - "--quiet", - action="store_true", - help="disable all status output except fatal errors", - ) + add_common_args(parser, MIN_STR_LEN) args = parser.parse_args() - if args.debug: - logging.basicConfig(level=logging.DEBUG) - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - logging.getLogger().setLevel(logging.INFO) + configure_logging(args) try: pe = pefile.PE(args.path) diff --git a/floss/language/identify.py b/floss/language/identify.py index bc22cfc5b..0c9f59534 100644 --- a/floss/language/identify.py +++ b/floss/language/identify.py @@ -32,6 +32,7 @@ class Language(Enum): + AUTO = "auto" GO = "go" RUST = "rust" DOTNET = "dotnet" diff --git a/floss/language/rust/coverage.py b/floss/language/rust/coverage.py index 0596a1ec1..1e3f8492d 100644 --- a/floss/language/rust/coverage.py +++ b/floss/language/rust/coverage.py @@ -23,6 +23,7 @@ from floss.strings import extract_ascii_unicode_strings from floss.language.utils import get_extract_stats +from floss.language.cli_common import add_common_args, configure_logging from floss.language.rust.extract import extract_rust_strings logger = logging.getLogger(__name__) @@ -32,31 +33,10 @@ def main(): parser = argparse.ArgumentParser(description="Get Rust strings") - parser.add_argument("path", help="file or path to analyze") - parser.add_argument( - "-n", - "--minimum-length", - dest="min_length", - type=int, - default=MIN_STR_LEN, - help="minimum string length", - ) - logging_group = parser.add_argument_group("logging arguments") - logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") - logging_group.add_argument( - "-q", - "--quiet", - action="store_true", - help="disable all status output except fatal errors", - ) + add_common_args(parser, MIN_STR_LEN) args = parser.parse_args() - if args.debug: - logging.basicConfig(level=logging.DEBUG) - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - logging.getLogger().setLevel(logging.INFO) + configure_logging(args) try: pe = pefile.PE(args.path) diff --git a/scripts/extract_rust_hashes.py b/floss/language/rust/extract_rust_hashes.py similarity index 82% rename from scripts/extract_rust_hashes.py rename to floss/language/rust/extract_rust_hashes.py index 4c822f0be..588738006 100644 --- a/scripts/extract_rust_hashes.py +++ b/floss/language/rust/extract_rust_hashes.py @@ -19,9 +19,9 @@ Description: Generates a database of Rust hashes from the Rust repository. Repo: https://github.com/rust-lang/rust/releases -Usage: +Usage (from repository root): - $ python3 extract_rust_hashes.py + $ python3 floss/language/rust/extract_rust_hashes.py """ import subprocess @@ -48,8 +48,12 @@ # for each table, get the hash and version for table in tables: - hash = str(table.find("a", attrs={"class": "Link Link--muted mb-2"})["href"]).split("/")[-1] - version = table.find("span").text.strip() + link = table.find("a", attrs={"class": "Link Link--muted mb-2"}) + span = table.find("span") + if link is None or span is None or not link.get("href"): + continue + hash = str(link["href"]).split("/")[-1] + version = span.text.strip() rust_hashes[hash] = version page_number += 1 @@ -85,21 +89,19 @@ # # Regeneration Instructions: # -# To regenerate or update this file, you can follow these steps: -# 1. Navigate to the script directory. -# 2. Execute the script 'extract_rust_hashes.py'. -# Example command: python extract_rust_hashes.py +# To regenerate or update this file, run from the repository root: +# python floss/language/rust/extract_rust_hashes.py ############################################################################################# """ -# write the hashes to a file -file_path = Path("rust_version_database.py") +# write the hashes next to this script +file_path = Path(__file__).resolve().parent / "rust_version_database.py" with file_path.open(mode="w") as f: f.write(header) f.write("rust_commit_hash = ") f.write(str(rust_hashes)) # format the file -subprocess.call(["black", "-l", "120", "rust_version_database.py"]) +subprocess.call(["black", "-l", "120", str(file_path)]) diff --git a/floss/language/rust/rust_version_database.py b/floss/language/rust/rust_version_database.py index e27f08590..9ef9c2caa 100644 --- a/floss/language/rust/rust_version_database.py +++ b/floss/language/rust/rust_version_database.py @@ -25,8 +25,8 @@ # # To regenerate or update this file, you can follow these steps: # 1. Navigate to the script directory. -# 2. Execute the script 'extract_rust_hashes.py'. -# Example command: python extract_rust_hashes.py +# 2. Run from the repository root: +# python floss/language/rust/extract_rust_hashes.py ############################################################################################# diff --git a/floss/layout/__init__.py b/floss/layout/__init__.py new file mode 100644 index 000000000..d475a95fc --- /dev/null +++ b/floss/layout/__init__.py @@ -0,0 +1,101 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Binary layout analysis (PE / ELF / Mach-O). + +Public entrypoint: ``compute_layout``. Import types, extract helpers, and +format builders from the submodules that define them (``base``, ``extract``, +``types``, ``pe``, ``elf``, ``macho``). +""" + +from __future__ import annotations + +import logging + +from elftools.common.exceptions import ELFError + +from floss.ranges import Range, Slice +from floss.layout.pe import compute_pe_layout +from floss.layout.elf import compute_elf_layout +from floss.layout.base import Layout, SegmentLayout +from floss.layout.macho import _get_u32_be, _is_macho_magic, compute_macho_layout + +logger = logging.getLogger("floss.layout") + + +def xor_static(data: bytes, i: int) -> bytes: + return bytes(c ^ i for c in data) + + +def compute_layout(slice_: Slice) -> Layout: + + # TODO don't do this for text or other obvious non-xored data + + mz_xor = [ + ( + xor_static(b"MZ", key), + key, + ) + for key in range(1, 256) + ] + + xor_key = None + decoded_slice = slice_ + + # Try to find the XOR key. Read only the first two bytes from buf; slice_.data + # copies the entire underlying buffer on each call. + rel_start = slice_.range.offset - slice_.base_offset + start_bytes = slice_.buf[rel_start : rel_start + 2] + for mz, key in mz_xor: + if start_bytes == mz: + xor_key = key + break + + # If XOR key is found, apply XOR decoding + if xor_key is not None: + decoded_data = xor_static(slice_.data, xor_key) + # Use base_offset to match the absolute offset, + # so that Slice/Range logic based on absolute offsets still works + # without requiring a large NULL-padded buffer. + decoded_slice = Slice( + buf=decoded_data, + range=Range(offset=slice_.offset, length=len(decoded_data)), + base_offset=slice_.offset, + ) + + # Try to parse as PE file + if decoded_slice.data.startswith(b"MZ"): + try: + # lancelot may panic here, which we can't currently catch from Python + return compute_pe_layout(decoded_slice, xor_key) + except ValueError as e: + logger.debug("failed to parse as PE file: %s", e) + elif _is_macho_magic(_get_u32_be(slice_.data, 0)): + try: + return compute_macho_layout(slice_) + except Exception as e: + # TODO: narrow exception handling once machofile error types are clearer. + logger.debug("failed to parse as Mach-O file: %s", e) + elif decoded_slice.data.startswith(b"\x7fELF"): + try: + return compute_elf_layout(decoded_slice, xor_key) + except ELFError as e: + logger.debug("failed to parse as ELF file: %s", e) + else: + logger.debug("unrecognized file format, falling back to binary layout") + + return SegmentLayout( + slice=slice_, + name="binary", + ) diff --git a/floss/layout/base.py b/floss/layout/base.py new file mode 100644 index 000000000..9e8bda975 --- /dev/null +++ b/floss/layout/base.py @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recursive layout tree nodes for binary structure.""" + +from __future__ import annotations + +import abc +import bisect +from typing import Any, Set, Dict, List, Tuple, Callable, Iterable, Optional, Sequence +from collections import defaultdict + +import pefile +from pydantic import Field, BaseModel, ConfigDict + +from floss.ranges import Range, Slice, OffsetRanges +from floss.tags.engine import check_is_xor, check_is_code, check_is_reloc +from floss.layout.types import Tag, TaggedString, ExtractedString + +Tagger = Callable[[ExtractedString], Sequence[Tag]] + + +class Structure(BaseModel): + slice: Slice + name: str + + +class Layout(BaseModel, abc.ABC): + """ + recursively describe a region of a data, as a tree. + the compute_layout routines construct this tree. + + each node in the tree (Layout), describes a range of the data. + it may have children, which describes sub-ranges of the data. + children don't overlap nor extend before/beyond the parent range. + children are ordered by their offset in the data. + children don't have to be contiguous - there can be gaps, or none at all. + there are routines for traversing to the prior/next sibling, if any, + and accessor properties for the parent and children. + + each node has a nice human readable name. + each node has a list of strings that are contained by the node; + these strings don't overlap with any children strings, they're only found in the gaps. + + note that `Layout` is the abstract base class for nodes in the tree. + subclasses are used to represent different types of regions, + such as a PE file, a section, a segment, or a resource. + subclasses can provide more specific behavior when it comes to tagging strings. + """ + + slice: Slice + + # human readable name + name: str + + parent: Optional["Layout"] = Field(default=None, init=False) + + # ordered by address + # non-overlapping + # may not cover the entire range (non-contiguous) + children: Sequence["Layout"] = Field(default_factory=list, init=False) + + # this is populated by the call to extract_strings. + # only strings not contained by the children are in this list. + # so they come from before/between/after the children ranges. + strings: List[TaggedString] = Field(default_factory=list, init=False) + + @property + def predecessors(self) -> Iterable["Layout"]: + """traverse to the prior siblings`""" + if self.parent is None: + return + + index = self.parent.children.index(self) + if index == 0: + return + + for i in range(index - 1, -1, -1): + yield self.parent.children[i] + + @property + def predecessor(self) -> Optional["Layout"]: + """traverse to the prior sibling""" + return next(iter(self.predecessors), None) + + @property + def successors(self) -> Iterable["Layout"]: + """traverse to the next siblings""" + if self.parent is None: + return + + index = self.parent.children.index(self) + if index == len(self.parent.children) - 1: + return + + for i in range(index + 1, len(self.parent.children)): + yield self.parent.children[i] + + @property + def successor(self) -> Optional["Layout"]: + """traverse to the next sibling""" + return next(iter(self.successors), None) + + def add_child(self, child: "Layout"): + # this works in py3.11, though mypy gets confused, + # maybe due to the use of the key function. + bisect.insort(self.children, child, key=lambda c: c.slice.range.offset) # type: ignore + child.parent = self + + @property + def offset(self) -> int: + "convenience" + return self.slice.range.offset + + @property + def end(self) -> int: + "convenience" + return self.slice.range.end + + def extract_strings(self, min_len: int) -> None: + """ + find the strings in this layout and its children, recursively. + + this finds strings in the gaps between the children (and before the + first and after the last child), so this method must run before + ``tag_strings``. + """ + # imported here to avoid a circular import with floss.layout.extract + from floss.layout.extract import extract_strings as extract_gap_strings + + if not self.children: + # at this moment, self.strings contains only ExtractedStrings + # after tag_strings, it will contain TaggedStrings. + self.strings = extract_gap_strings(self.slice, min_len) # type: ignore + return + + # we have children, so we need to recurse to find their strings, + # and also find strings in the gaps between children. + # lets find the gap strings first: + for i, child in enumerate(self.children): + if i == 0: + # find the strings before the first child + offset = 0 + size = self.children[0].offset - self.offset + + else: + # find strings between children + prior = self.children[i - 1] + offset = prior.end - self.offset + size = child.offset - prior.end + + if size == 0: + # there is no gap here. + continue + + gap = self.slice.slice(offset, size) + self.strings.extend(extract_gap_strings(gap, min_len)) # type: ignore + + # finally, find strings after the last child + last_child = self.children[-1] + offset = last_child.end - self.offset + size = self.end - last_child.end + + if size > 0: + gap = self.slice.slice(offset, size) + self.strings.extend(extract_gap_strings(gap, min_len)) # type: ignore + + # now recurse to find the strings in the children. + for child in self.children: + child.extract_strings(min_len) + + if self.strings: + child_ranges = [(child.offset, child.end) for child in self.children] + filtered = [] + for string in self.strings: + if isinstance(string, TaggedString): + offset = string.offset + else: + offset = string.slice.range.offset + if any(start <= offset < end for start, end in child_ranges): + continue + filtered.append(string) + self.strings = filtered + + def tag_strings(self, taggers: Sequence[Tagger]): + """ + tag the strings in this layout and its children, recursively. + this means that the .strings field will contain TaggedStrings now + (it used to contain ExtractedStrings). + + this can be overridden, if a subclass has more ways of tagging strings, + such as a PE file and code/reloc regions. + """ + string_counts: Dict[str, int] = defaultdict(int) + + tagged_strings: List[TaggedString] = [] + + for string in self.strings: + # at this moment, the list of strings contains only ExtractedStrings. + # this routine will transform them into TaggedStrings. + assert isinstance(string, ExtractedString) + tags: Set[Tag] = set() + + string_counts[string.string] += 1 + + if string_counts[string.string] > 1: + tags.add("#duplicate") + + for tagger in taggers: + tags.update(tagger(string)) + + tagged_strings.append(TaggedString(string=string, tags=tags)) + self.strings = tagged_strings + + for child in self.children: + child.tag_strings(taggers) + + def mark_structures(self, structures: Optional[Tuple[Dict[int, Structure], ...]] = (), **kwargs): + """ + mark the structures that might be associated with each string, recursively. + this means that the TaggedStrings may now have a non-empty .structure field. + + this can be overridden, if a subclass has a way of parsing structures, + such as a PE file and all its data. + """ + if structures: + self._mark_string_structures(structures) + + for child in self.children: + child.mark_structures(structures=structures, **kwargs) + + def _mark_string_structures(self, structures) -> None: + """attach the first matching structure name to this node's own strings.""" + for string in self.strings: + for structures_by_address in structures: + structure = structures_by_address.get(string.offset) + if structure: + string.structure = structure.name + break + + +class SectionLayout(Layout): + model_config = ConfigDict(arbitrary_types_allowed=True) + + section: Optional[pefile.SectionStructure] = None + + +class SegmentLayout(Layout): + """region not covered by any section, such as PE header or overlay""" + + pass + + +class PELayout(Layout): + model_config = ConfigDict(arbitrary_types_allowed=True) + + # xor key if the file was xor decoded + xor_key: Optional[int] + + # file offsets of bytes that are part of the relocation table + reloc_offsets: OffsetRanges + + # file offsets of bytes that are recognized as code + code_offsets: OffsetRanges + + structures_by_address: Dict[int, Structure] + + def tag_strings(self, taggers: Sequence[Tagger]): + def check_is_xor_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_xor(self.xor_key) + + def check_is_reloc_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_reloc(self.reloc_offsets, s) + + def check_is_code_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_code(self.code_offsets, s) + + taggers = tuple(taggers) + ( + check_is_xor_tagger, + check_is_reloc_tagger, + check_is_code_tagger, + ) + + super().tag_strings(taggers) + + def mark_structures(self, structures=(), **kwargs): + # apply the PE structures to this node's own strings too (e.g. strings + # in the header gap that are attached to the root layout node) + self._mark_string_structures((structures or ()) + (self.structures_by_address,)) + for child in self.children: + if isinstance(child, (SectionLayout, SegmentLayout)): + # expected child of a PE + child.mark_structures(structures=structures + (self.structures_by_address,), **kwargs) + else: + # unexpected child of a PE + # maybe like a resource or overlay, etc. + # which is fine - but we don't expect it to know about the PE structures. + child.mark_structures(structures=structures, **kwargs) + + +class ELFLayout(Layout): + xor_key: Optional[int] + + # file offsets of bytes that are part of relocation sections + relocation_offsets: OffsetRanges + + # file offsets of bytes that are recognized as code + code_offsets: OffsetRanges + + structures_by_address: Dict[int, Structure] + + def tag_strings(self, taggers: Sequence[Tagger]): + def check_is_xor_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_xor(self.xor_key) + + def check_is_reloc_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_reloc(self.relocation_offsets, s) + + def check_is_code_tagger(s: ExtractedString) -> Sequence[Tag]: + return check_is_code(self.code_offsets, s) + + taggers = tuple(taggers) + ( + check_is_xor_tagger, + check_is_reloc_tagger, + check_is_code_tagger, + ) + + super().tag_strings(taggers) + + def mark_structures(self, structures: Optional[Tuple[Dict[int, Structure], ...]] = (), **kwargs): + # apply the ELF structures to this node's own strings too (the ELF + # header/program-header gap strings attach to the root layout node) + self._mark_string_structures((structures or ()) + (self.structures_by_address,)) + for child in self.children: + if isinstance(child, (SectionLayout, SegmentLayout)): + child.mark_structures(structures=(structures or ()) + (self.structures_by_address,), **kwargs) + else: + child.mark_structures(structures=structures, **kwargs) + + +class ResourceLayout(Layout): + pass + + +class MachOLayout(Layout): + arch: str + structures_by_address: Dict[int, Structure] = Field(default_factory=dict) + + def mark_structures(self, structures=(), **kwargs): + if self.structures_by_address: + structures = structures + (self.structures_by_address,) + super().mark_structures(structures=structures, **kwargs) + + def tag_strings(self, taggers: Sequence[Tagger]): + super().tag_strings(taggers) + + +class MachOFatLayout(Layout): + pass diff --git a/floss/layout/elf.py b/floss/layout/elf.py new file mode 100644 index 000000000..5c2f01206 --- /dev/null +++ b/floss/layout/elf.py @@ -0,0 +1,274 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ELF layout construction.""" + +from __future__ import annotations + +import io +import logging +from typing import Any, Dict, List, Tuple, Iterable, Optional, Sequence + +from elftools.elf.elffile import ELFFile +from elftools.elf.constants import P_FLAGS, SH_FLAGS +from elftools.elf.relocation import RelocationSection +from elftools.common.exceptions import ELFError + +from floss.ranges import Range, Slice, OffsetRanges, merge_overlapping_ranges +from floss.layout.base import Layout, ELFLayout, Structure, SectionLayout, SegmentLayout +from floss.layout.types import Tag, ExtractedString + +logger = logging.getLogger("floss.layout.elf") + + +def elf_has_valid_sections(elf: ELFFile, limit: int) -> bool: + shoff = elf.header.get("e_shoff", 0) + shnum = elf.header.get("e_shnum", 0) + shentsize = elf.header.get("e_shentsize", 0) + if shoff == 0 or shnum == 0 or shentsize == 0: + return False + + try: + expected_shentsize = elf.structs.Elf_Shdr.sizeof() + except Exception: + return False + + if shentsize < expected_shentsize: + return False + + sh_end = shoff + shnum * shentsize + return sh_end <= limit + + +def elf_has_valid_segments(elf: ELFFile, limit: int) -> bool: + phoff = elf.header.get("e_phoff", 0) + phnum = elf.header.get("e_phnum", 0) + phentsize = elf.header.get("e_phentsize", 0) + if phnum == 0 or phnum >= 0xFFFF: + return False + if phoff == 0 or phentsize == 0: + return False + + try: + expected_phentsize = elf.structs.Elf_Phdr.sizeof() + except Exception: + return False + + if phentsize < expected_phentsize: + return False + + ph_end = phoff + phnum * phentsize + return ph_end <= limit + + +def iter_sections_robust(elf: ELFFile) -> Iterable[Any]: + try: + num_sections = elf.num_sections() + except Exception as e: + logger.warning("failed to get number of sections: %s", e) + return + + for i in range(num_sections): + try: + yield elf.get_section(i) + except Exception as e: + logger.warning("failed to parse section %d: %s", i, e) + continue + + +def get_relocations_elf(slice_: Slice, elf: ELFFile) -> List[Tuple[int, int]]: + if not elf_has_valid_sections(elf, slice_.range.length): + return [] + + ranges: List[Tuple[int, int]] = [] + + for section in iter_sections_robust(elf): + if isinstance(section, RelocationSection): + offset = section["sh_offset"] + size = section["sh_size"] + + if not slice_.contains_range(offset, size): + logger.warning("relocation directory points to an invalid location, skipping") + continue + + ranges.append((slice_.offset + offset, slice_.offset + offset + size - 1)) + return merge_overlapping_ranges(ranges) + + +def collect_elf_structures(slice_: Slice, elf: ELFFile) -> Sequence[Structure]: + structures: List[Structure] = [] + + # ELF file header: 52 bytes (32-bit) or 64 bytes (64-bit) + header_size = 52 if elf.elfclass == 32 else 64 + if slice_.contains_range(0, header_size): + structures.append(Structure(slice=slice_.slice(0, header_size), name="elf header")) + + # Program header table + phoff = elf.header["e_phoff"] + phentsize = elf.header["e_phentsize"] + phnum = elf.header["e_phnum"] + if phnum > 0 and phentsize > 0: + ph_total = phentsize * phnum + if slice_.contains_range(phoff, ph_total): + structures.append(Structure(slice=slice_.slice(phoff, ph_total), name="program header")) + + # Section header table + shoff = elf.header["e_shoff"] + shentsize = elf.header["e_shentsize"] + shnum = elf.header["e_shnum"] + if shnum > 0 and shentsize > 0: + sh_total = shentsize * shnum + if slice_.contains_range(shoff, sh_total) and elf_has_valid_sections(elf, slice_.range.length): + structures.append(Structure(slice=slice_.slice(shoff, sh_total), name="section header")) + + # String tables (.shstrtab, .strtab, .dynstr) and symbol tables (.symtab, .dynsym) + if elf_has_valid_sections(elf, slice_.range.length): + for section in iter_sections_robust(elf): + if section["sh_size"] == 0: + continue + if section["sh_type"] == "SHT_NOBITS": + continue + + offset = section["sh_offset"] + size = section["sh_size"] + + if not slice_.contains_range(offset, size): + continue + + if section["sh_type"] == "SHT_STRTAB": + structures.append(Structure(slice=slice_.slice(offset, size), name="string table")) + elif section["sh_type"] in {"SHT_SYMTAB", "SHT_DYNSYM"}: + structures.append(Structure(slice=slice_.slice(offset, size), name="symbol table")) + + return structures + + +def compute_elf_layout(slice_: Slice, xor_key: int | None) -> Layout: + data = slice_.data + + elf = ELFFile(io.BytesIO(data)) + + structures = collect_elf_structures(slice_, elf) + relocation_offsets = OffsetRanges.from_merged_ranges(get_relocations_elf(slice_, elf)) + + structures_by_address: Dict[int, Structure] = {} + for structure in structures: + for offset in structure.slice.range: + structures_by_address[offset] = structure + + # Collect valid file-backed sections/segments, sorted by offset, deduplicating overlaps. + # For sections: SHT_NOBITS sections (.bss, .noptrbss) have no file content; skip them. + # For segments: PT_LOAD segments are main focus. + # Also track executable parts (SHF_EXECINSTR or PF_X) for #code tagging. + layout_elements: List[Tuple[int, int, str, bool]] = [] # (offset, size, name, is_exec) + + use_sections = elf_has_valid_sections(elf, slice_.range.length) + if use_sections: + for idx, section in enumerate(iter_sections_robust(elf)): + if section["sh_size"] == 0: + continue + if section["sh_type"] == "SHT_NOBITS": + continue + + try: + name = section.name + except (ELFError, IndexError, UnicodeDecodeError) as e: + name = f"unnamed_section_{idx}" + logger.warning("failed to get section name for section %d: %s", idx, e) + + offset = section["sh_offset"] + size = section["sh_size"] + is_exec = bool(section["sh_flags"] & SH_FLAGS.SHF_EXECINSTR) + + if offset >= slice_.range.length: + logger.warning("section %s out of range", name) + continue + + if offset + size > slice_.range.length: + size_orig = size + size = slice_.range.length - offset + logger.warning( + "section size %s out of range, truncating from 0x%x to 0x%x bytes", name, size_orig, size + ) + + layout_elements.append((offset, size, name, is_exec)) + else: + logger.debug("ELF section headers missing or invalid, using segments for layout") + if not elf_has_valid_segments(elf, slice_.range.length): + raise ELFError("ELF program headers missing or invalid") + num_segments = elf.num_segments() + + for i in range(num_segments): + try: + segment_header = elf.get_segment(i).header + + if segment_header["p_type"] not in ("PT_LOAD", 1): + continue + + if segment_header["p_filesz"] == 0: + continue + + offset = segment_header["p_offset"] + size = segment_header["p_filesz"] + is_exec = bool(segment_header["p_flags"] & P_FLAGS.PF_X) + name = f"segment_{i}_{segment_header['p_type']}" + + if offset >= slice_.range.length: + logger.warning("segment %s out of range", name) + continue + + if offset + size > slice_.range.length: + size_orig = size + size = slice_.range.length - offset + logger.warning( + "segment size %s out of range, truncating from 0x%x to 0x%x bytes", name, size_orig, size + ) + + layout_elements.append((offset, size, name, is_exec)) + except Exception as e: + logger.warning("failed to parse segment %d: %s", i, e) + continue + + # Build code_offsets from executable parts before constructing the layout. + layout_elements.sort(key=lambda t: t[0]) + exec_ranges: List[Tuple[int, int]] = [ + (offset, offset + size) for offset, size, _name, is_exec in layout_elements if is_exec + ] + code_offsets = OffsetRanges.from_merged_ranges(merge_overlapping_ranges(exec_ranges)) + + layout = ELFLayout( + slice=slice_, + name="elf", + xor_key=xor_key, + relocation_offsets=relocation_offsets, + code_offsets=code_offsets, + structures_by_address=structures_by_address, + ) + + if xor_key: + layout.name += f" (XOR decoded with key: 0x{xor_key:x})" + + # Sort by offset, then skip any element that overlaps a previously added one. + cursor = 0 + for offset, size, name, _is_exec in layout_elements: + if offset < cursor: + logger.debug("element %s overlaps previous element, skipping", name) + continue + if use_sections: + layout.add_child(SectionLayout(slice=slice_.slice(offset, size), name=name)) + else: + layout.add_child(SegmentLayout(slice=slice_.slice(offset, size), name=name)) + cursor = offset + size + + return layout diff --git a/floss/layout/extract.py b/floss/layout/extract.py new file mode 100644 index 000000000..449655fb9 --- /dev/null +++ b/floss/layout/extract.py @@ -0,0 +1,79 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Extract and collect strings within a layout tree.""" + +from __future__ import annotations + +import itertools +from typing import List, Literal, Iterable + +from floss import strings as floss_strings +from floss.ranges import Slice +from floss.results import StaticString, StringEncoding +from floss.layout.base import Layout +from floss.layout.types import TaggedString, ExtractedString + +MIN_STR_LEN = floss_strings.MIN_LENGTH + + +def _to_extracted(s: StaticString, slice: Slice) -> ExtractedString: + encoding: Literal["ascii", "unicode"] + if s.encoding == StringEncoding.UTF16LE: + encoding = "unicode" + byte_len = len(s.string) * 2 + else: + encoding = "ascii" + byte_len = len(s.string) + + return ExtractedString(string=s.string, slice=slice.slice(s.offset, byte_len), encoding=encoding) + + +def extract_ascii_strings(slice: Slice, n: int = MIN_STR_LEN) -> Iterable[ExtractedString]: + "enumerate ASCII strings in the given binary data" + if not slice.range.length: + return + + for s in floss_strings.extract_ascii_strings(slice.data, n): + yield _to_extracted(s, slice) + + +def extract_unicode_strings(slice: Slice, n: int = MIN_STR_LEN) -> Iterable[ExtractedString]: + "enumerate naive UTF-16 strings in the given binary data" + if not slice.range.length: + return + + for s in floss_strings.extract_unicode_strings(slice.data, n): + yield _to_extracted(s, slice) + + +def extract_strings(slice: Slice, n: int = MIN_STR_LEN) -> Iterable[ExtractedString]: + "enumerate ASCII and naive UTF-16 strings in the given binary data" + return list( + sorted( + itertools.chain(extract_ascii_strings(slice, n), extract_unicode_strings(slice, n)), + key=lambda s: s.slice.range.offset, + ) + ) + + +def collect_strings(layout: Layout) -> List[TaggedString]: + ret = [] + + ret.extend(layout.strings) + + for child in layout.children: + ret.extend(collect_strings(child)) + + return ret diff --git a/floss/layout/macho.py b/floss/layout/macho.py new file mode 100644 index 000000000..efa2a8de3 --- /dev/null +++ b/floss/layout/macho.py @@ -0,0 +1,472 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Mach-O (thin and fat) layout construction.""" + +from __future__ import annotations + +import struct +import logging +from typing import Any, Dict, List, Tuple, Optional, Sequence + +import machofile # type: ignore[import-untyped] + +from floss.ranges import Range, Slice +from floss.layout.base import Layout, Structure, MachOLayout, SegmentLayout, MachOFatLayout +from floss.layout.types import Tag + +logger = logging.getLogger("floss.layout.macho") + + +MACHO_MAGIC = 0xFEEDFACE +MACHO_CIGAM = 0xCEFAEDFE +MACHO_MAGIC_64 = 0xFEEDFACF +MACHO_CIGAM_64 = 0xCFFAEDFE +FAT_MAGIC = 0xCAFEBABE +FAT_CIGAM = 0xBEBAFECA +FAT_MAGIC_64 = 0xCAFEBABF +FAT_CIGAM_64 = 0xBFBAFECA + +MACHO_MAGICS = {MACHO_MAGIC, MACHO_CIGAM, MACHO_MAGIC_64, MACHO_CIGAM_64} +FAT_MAGICS = {FAT_MAGIC, FAT_CIGAM, FAT_MAGIC_64, FAT_CIGAM_64} + +CPU_TYPE_X86 = 0x7 +CPU_TYPE_X86_64 = 0x1000007 +CPU_TYPE_ARM = 0xC +CPU_TYPE_ARM64 = 0x100000C +CPU_TYPE_PPC = 0x12 +CPU_TYPE_PPC64 = 0x10000012 + +CPU_TYPE_MAP = { + CPU_TYPE_X86: "x86", + CPU_TYPE_X86_64: "x86_64", + CPU_TYPE_ARM: "arm", + CPU_TYPE_ARM64: "arm64", + CPU_TYPE_PPC: "ppc", + CPU_TYPE_PPC64: "ppc64", +} + +LC_SEGMENT = 0x1 +LC_SEGMENT_64 = 0x19 +LC_CODE_SIGNATURE = 0x1D + +CSMAGIC_EMBEDDED_SIGNATURE = 0xFADE0CC0 +CSMAGIC_EMBEDDED_ENTITLEMENTS = 0xFADE7171 +CSMAGIC_EMBEDDED_DER_ENTITLEMENTS = 0xFADE7172 +CSMAGIC_BLOBWRAPPER = 0xFADE0B01 + + +def _get_u32_be(data: bytes, offset: int) -> Optional[int]: + if offset + 4 > len(data): + return None + return struct.unpack(">I", data[offset : offset + 4])[0] + + +def _is_macho_magic(magic: Optional[int]) -> bool: + if magic is None: + return False + return magic in MACHO_MAGICS or magic in FAT_MAGICS + + +def _format_macho_arch(cputype: int, cpusubtype: int) -> str: + base = CPU_TYPE_MAP.get(cputype, f"cpu_{cputype}") + clean_subtype = cpusubtype & 0x00FFFFFF + if cputype == CPU_TYPE_ARM64: + if clean_subtype == 0: + return "arm64" + if clean_subtype == 2: + return "arm64e" + return f"arm64_{clean_subtype}" + return base + + +def _parse_fat_arches(data: bytes) -> List[Tuple[str, int, int]]: + """ + Parse the Mach-O fat header to extract architecture information. + Returns: + List of (arch_name, offset, size) tuples: + - arch_name (str): The name of the architecture (e.g., 'x86_64', 'arm64'). + - offset (int): The file offset to the architecture-specific binary. + - size (int): The size of the architecture-specific binary in bytes. + """ + arches: List[Tuple[str, int, int]] = [] + if len(data) < 8: + return arches + + magic = _get_u32_be(data, 0) + if magic not in FAT_MAGICS: + return arches + + swap = magic in {FAT_CIGAM, FAT_CIGAM_64} + endian = "<" if swap else ">" + nfat_arch = struct.unpack(endian + "I", data[4:8])[0] + + is_64 = magic in {FAT_MAGIC_64, FAT_CIGAM_64} + offset = 8 + + for _ in range(nfat_arch): + if is_64: + if offset + 32 > len(data): + break + cputype, cpusubtype, arch_offset, size, align, _reserved = struct.unpack( + endian + "IIQQII", data[offset : offset + 32] + ) + offset += 32 + else: + if offset + 20 > len(data): + break + cputype, cpusubtype, arch_offset, size, _align = struct.unpack(endian + "IIIII", data[offset : offset + 20]) + offset += 20 + + arch_name = _format_macho_arch(cputype, cpusubtype) + arches.append((arch_name, arch_offset, size)) + + return arches + + +def _parse_macho_endian_and_cmds(data: bytes) -> Tuple[str, bool, int, int]: + if len(data) < 4: + raise ValueError("insufficient data for Mach-O header") + + magic = struct.unpack(">I", data[:4])[0] + if magic not in MACHO_MAGICS: + raise ValueError("not a Mach-O header") + + big_endian = magic in {MACHO_MAGIC, MACHO_MAGIC_64} + endian = ">" if big_endian else "<" + is_64 = magic in {MACHO_MAGIC_64, MACHO_CIGAM_64} + + header_size = 32 if is_64 else 28 + if len(data) < header_size: + raise ValueError("insufficient data for Mach-O header") + + ncmds = struct.unpack(endian + "I", data[16:20])[0] + sizeofcmds = struct.unpack(endian + "I", data[20:24])[0] + return endian, is_64, ncmds, sizeofcmds + + +def _parse_macho_load_commands( + slice_: Slice, endian: str, is_64: bool, ncmds: int +) -> Tuple[List[Structure], Sequence[Dict[str, int]], Optional[Tuple[int, int]]]: + structures: List[Structure] = [] + segments: List[Dict[str, int]] = [] + code_sig: Optional[Tuple[int, int]] = None + + data = slice_.data + header_size = 32 if is_64 else 28 + if slice_.range.length >= header_size: + structures.append(Structure(slice=slice_.slice(0, header_size), name="macho header")) + offset = header_size + cmd_header_size = 8 + seg_fmt = "II16sQQQQIIII" if is_64 else "II16sIIIIIIII" + seg_header_size = struct.calcsize(endian + seg_fmt) + + for _ in range(ncmds): + if offset + cmd_header_size > slice_.range.length: + break + + cmd, cmdsize = struct.unpack(endian + "II", data[offset : offset + cmd_header_size]) + if cmdsize < cmd_header_size: + break + + cmd_offset = offset + cmd_end = offset + cmdsize + if cmd_end > slice_.range.length: + break + + structures.append(Structure(slice=slice_.slice(cmd_offset, cmdsize), name="load command")) + + if cmd == LC_CODE_SIGNATURE: + if cmdsize >= 16: + dataoff = struct.unpack(endian + "I", data[cmd_offset + 8 : cmd_offset + 12])[0] + datasize = struct.unpack(endian + "I", data[cmd_offset + 12 : cmd_offset + 16])[0] + code_sig = (int(dataoff), int(datasize)) + + if cmd in {LC_SEGMENT, LC_SEGMENT_64}: + if cmdsize >= seg_header_size: + seg_data = data[cmd_offset : cmd_offset + seg_header_size] + seg_values = struct.unpack(endian + seg_fmt, seg_data) + segname = seg_values[2].split(b"\x00", 1)[0].decode("utf-8", errors="replace") + fileoff = seg_values[5] + filesize = seg_values[6] + nsects = seg_values[9] + + segments.append({"segname": segname, "offset": int(fileoff), "size": int(filesize)}) + + structures.append(Structure(slice=slice_.slice(cmd_offset, seg_header_size), name="segment header")) + + section_offset = cmd_offset + seg_header_size + section_size = 80 if is_64 else 68 + for _section_index in range(nsects): + if section_offset + section_size > cmd_end: + break + structures.append( + Structure(slice=slice_.slice(section_offset, section_size), name="section header") + ) + section_offset += section_size + + offset += cmdsize + + return structures, segments, code_sig + + +def _add_macho_segments(parent: Layout, slice_: Slice, segments: Sequence[Dict[str, int]]): + for segment in segments: + offset = segment.get("offset", 0) + size = segment.get("size", 0) + raw_name = segment.get("segname", "segment") + if isinstance(raw_name, bytes): + name = raw_name.decode("utf-8", errors="replace") + else: + name = str(raw_name) + name = name.replace("\x00", "").strip() + if not name: + name = f"segment@0x{offset:x}" + + if size <= 0: + continue + + if not slice_.contains_range(offset, size): + if offset >= slice_.range.length: + logger.warning("Mach-O segment %s out of range", name) + continue + size = slice_.range.length - offset + if size <= 0: + continue + logger.warning("Mach-O segment %s size out of range, truncating", name) + + parent.add_child(SegmentLayout(slice=slice_.slice(offset, size), name=name)) + + +def _attach_nested_layout(parent: Layout, child: Layout): + container = next( + (candidate for candidate in parent.children if candidate.offset <= child.offset < candidate.end), + None, + ) + if container and child.end <= container.end: + container.add_child(child) + else: + parent.add_child(child) + + +def _parse_superblob_blobs(slice_: Slice, cs_offset: int, cs_size: int) -> Sequence[Tuple[int, int, int]]: + blobs: List[Tuple[int, int, int]] = [] + if cs_size <= 0: + return blobs + + if not slice_.contains_range(cs_offset, cs_size): + return blobs + + cs_data = slice_.data[cs_offset : cs_offset + cs_size] + if len(cs_data) < 12: + return blobs + + magic, length, count = struct.unpack(">III", cs_data[:12]) + if magic != CSMAGIC_EMBEDDED_SIGNATURE: + return blobs + + if length > cs_size: + length = cs_size + + index_offset = 12 + for _ in range(count): + if index_offset + 8 > length: + break + _blob_type, blob_offset = struct.unpack(">II", cs_data[index_offset : index_offset + 8]) + index_offset += 8 + + if blob_offset + 8 > length: + continue + + blob_magic, blob_length = struct.unpack(">II", cs_data[blob_offset : blob_offset + 8]) + if blob_length < 8: + continue + + if blob_offset + blob_length > length: + blob_length = length - blob_offset + if blob_length < 8: + continue + + blobs.append((blob_magic, cs_offset + blob_offset, blob_length)) + + return blobs + + +def _scan_entitlements_plist(slice_: Slice, cs_offset: int, cs_size: int) -> Sequence[Tuple[int, int]]: + entitlements: List[Tuple[int, int]] = [] + if cs_size <= 0: + return entitlements + + if not slice_.contains_range(cs_offset, cs_size): + return entitlements + + cs_data = slice_.data[cs_offset : cs_offset + cs_size] + + xml_marker = b"" + start = 0 + while True: + index = cs_data.find(xml_marker, start) + if index == -1: + break + end_index = cs_data.find(plist_end, index) + if end_index != -1: + end_index += len(plist_end) + entitlements.append((cs_offset + index, end_index - index)) + start = end_index + else: + break + + bplist_marker = b"bplist00" + index = cs_data.find(bplist_marker) + if index != -1: + bplist_len = _find_bplist_length(cs_data, index) + if bplist_len: + entitlements.append((cs_offset + index, bplist_len)) + + return entitlements + + +def _find_bplist_length(data: bytes, start: int) -> Optional[int]: + bplist_marker = b"bplist00" + if start < 0 or start + 8 > len(data): + return None + + trailer_size = 32 + min_len = 8 + trailer_size + max_len = len(data) - start + if max_len < min_len: + return None + + for end in range(start + max_len, start + min_len - 1, -1): + trailer_offset = end - trailer_size + trailer = data[trailer_offset:end] + + offset_size = trailer[6] + object_ref_size = trailer[7] + num_objects = int.from_bytes(trailer[8:16], "big") + top_object = int.from_bytes(trailer[16:24], "big") + offset_table_offset = int.from_bytes(trailer[24:32], "big") + + if offset_size == 0 or offset_size > 8: + continue + if object_ref_size == 0 or object_ref_size > 8: + continue + if num_objects == 0: + continue + if top_object >= num_objects: + continue + + length = end - start + if offset_table_offset < 8 or offset_table_offset >= length: + continue + + offset_table_size = num_objects * offset_size + if offset_table_offset + offset_table_size > length - trailer_size: + continue + + if data[start : start + 8] == bplist_marker: + return length + + return None + + +def _populate_thin_macho_layout(layout: MachOLayout, slice_: Slice): + try: + endian, is_64, ncmds, _sizeofcmds = _parse_macho_endian_and_cmds(slice_.data) + structures, segments, code_sig = _parse_macho_load_commands(slice_, endian, is_64, ncmds) + except ValueError: + structures = [] + segments = [] + code_sig = None + + if segments: + _add_macho_segments(layout, slice_, segments) + + if code_sig: + cs_offset, cs_size = code_sig + if slice_.contains_range(cs_offset, cs_size): + cs_layout = SegmentLayout(slice=slice_.slice(cs_offset, cs_size), name="code signature") + blobs = _parse_superblob_blobs(slice_, cs_offset, cs_size) + entitlements: List[Tuple[int, int]] = [] + for blob_magic, blob_offset, blob_length in blobs: + if not slice_.contains_range(blob_offset, blob_length): + continue + if blob_magic in {CSMAGIC_EMBEDDED_ENTITLEMENTS, CSMAGIC_EMBEDDED_DER_ENTITLEMENTS}: + entitlements.append((blob_offset, blob_length)) + elif blob_magic == CSMAGIC_BLOBWRAPPER: + cs_layout.add_child( + SegmentLayout( + slice=slice_.slice(blob_offset, blob_length), + name="certificates", + ) + ) + + if not entitlements: + entitlements = list(_scan_entitlements_plist(slice_, cs_offset, cs_size)) + + for ent_offset, ent_size in entitlements: + if slice_.contains_range(ent_offset, ent_size): + plist_layout = SegmentLayout( + slice=slice_.slice(ent_offset, ent_size), + name="plist: entitlements", + ) + _attach_nested_layout(cs_layout, plist_layout) + _attach_nested_layout(layout, cs_layout) + + if structures: + for structure in structures: + for offset_value in structure.slice.range: + layout.structures_by_address[offset_value] = structure + + +def compute_macho_layout(slice_: Slice) -> Layout: + data = slice_.data + magic = _get_u32_be(data, 0) + + if magic in FAT_MAGICS: + layout = MachOFatLayout(slice=slice_, name="macho (fat)") + arches = _parse_fat_arches(data) + for arch_name, offset, size in arches: + if not slice_.contains_range(offset, size): + logger.warning("fat arch %s out of range, skipping", arch_name) + continue + + arch_slice = slice_.slice(offset, size) + arch_layout = MachOLayout(slice=arch_slice, name=f"macho: {arch_name}", arch=arch_name) + + _populate_thin_macho_layout(arch_layout, arch_slice) + + layout.add_child(arch_layout) + + return layout + + arch_name = "macho" + try: + macho = machofile.UniversalMachO(data=data) + macho.parse() + header = macho.get_macho_header() + if isinstance(header, dict): + cputype = header.get("cputype") + cpusubtype = header.get("cpusubtype") + if isinstance(cputype, int) and isinstance(cpusubtype, int): + arch_name = _format_macho_arch(cputype, cpusubtype) + except Exception as e: + logger.debug("failed to parse Mach-O header via machofile: %s", e) + + thin_layout = MachOLayout(slice=slice_, name=f"macho: {arch_name}", arch=arch_name) + + _populate_thin_macho_layout(thin_layout, slice_) + + return thin_layout diff --git a/floss/layout/pe.py b/floss/layout/pe.py new file mode 100644 index 000000000..894b0234f --- /dev/null +++ b/floss/layout/pe.py @@ -0,0 +1,483 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PE layout construction.""" + +from __future__ import annotations + +import struct +import logging +import functools +from typing import Any, Set, Dict, List, Tuple, Optional, Sequence + +import pefile +import lancelot + +from floss.ranges import Range, Slice, OffsetRanges, timing, merge_overlapping_ranges +from floss.layout.base import ( + Layout, + PELayout, + Structure, + SectionLayout, + SegmentLayout, + ResourceLayout, +) +from floss.layout.types import Tag, ExtractedString + +logger = logging.getLogger("floss.layout.pe") + + +PE_RESOURCE_TYPES = { + 1: "Cursors", + 2: "Bitmaps", + 3: "Icons", + 4: "Menus", + 5: "Dialogs", + 6: "String Tables", + 7: "Font Directories", + 8: "Fonts", + 9: "Accelerators", + 10: "RCData", + 11: "Message Tables", + 12: "Cursor Groups", + 14: "Icon Groups", + 16: "Version Info", + 17: "DLGInclude", + 19: "Plug and Play", + 20: "VXD", + 21: "Animated Cursors", + 22: "Animated Icons", + 23: "HTML", + 24: "Manifest", + 240: "DLGInit", # MFC specific + 241: "Toolbars", # MFC specific +} + + +def get_reloc_offsets(slice: Slice, pe: pefile.PE) -> Set[int]: + ret: Set[int] = set() + + directory_index = pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_BASERELOC"] + + if pe.OPTIONAL_HEADER is None or pe.OPTIONAL_HEADER.DATA_DIRECTORY is None: + return ret + + try: + dir_entry = pe.OPTIONAL_HEADER.DATA_DIRECTORY[directory_index] + except IndexError: + return ret + + rva = dir_entry.VirtualAddress + try: + offset = pe.get_offset_from_rva(rva) + except pefile.PEFormatError as e: + logger.warning("failed to get offset for relocation directory RVA 0x%x: %s", rva, e) + return ret + + size = dir_entry.Size + + if not slice.contains_range(offset, size): + logger.warning("relocation directory points to an invalid location, skipping") + return ret + + for fo in slice.range.slice(offset, size): + ret.add(fo) + + return ret + + +def _get_code_ranges( + be2: "lancelot.BinExport2", + idx: "lancelot.be2utils.BinExport2Index", + base_address: int, + pe: pefile.PE, + slice_: Slice, +) -> List[Tuple[int, int]]: + """ + Extract and return the raw, unmerged code ranges from a PE file. + """ + + # cache because getting the offset is slow + @functools.lru_cache(maxsize=None) + def get_offset_from_rva_cached(rva): + try: + return pe.get_offset_from_rva(rva) + except pefile.PEFormatError as e: + logger.warning("%s", str(e)) + return None + + code_ranges: List[Tuple[int, int]] = [] + for flow_graph in be2.flow_graph: + for basic_block_index in flow_graph.basic_block_index: + try: + basic_block = be2.basic_block[basic_block_index] + except IndexError: + logger.warning("lancelot basic block index %d out of range, skipping", basic_block_index) + continue + + current_range: Optional[Tuple[int, int]] = None + for _instruction_index, instruction, instruction_address in idx.basic_block_instructions(basic_block): + va = instruction_address + rva = va - base_address + offset = get_offset_from_rva_cached(rva) + if offset is None: + if current_range is not None: + code_ranges.append(current_range) + current_range = None + continue + + size = len(instruction.raw_bytes) + if size == 0: + continue + + if not slice_.contains_range(offset, size): + logger.warning("lancelot identified code at an invalid location, skipping instruction at 0x%x", rva) + if current_range is not None: + code_ranges.append(current_range) + current_range = None + continue + + start = slice_.offset + offset + end = slice_.offset + offset + size - 1 + if current_range is None: + current_range = (start, end) + elif start == current_range[1] + 1: + current_range = (current_range[0], end) + else: + code_ranges.append(current_range) + current_range = (start, end) + if current_range is not None: + code_ranges.append(current_range) + return code_ranges + + +def collect_pe_structures(slice_: Slice, pe: pefile.PE) -> Sequence[Structure]: + structures = [] + + for section in sorted(pe.sections, key=lambda s: s.PointerToRawData): + offset = section.get_file_offset() + size = section.sizeof() + + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="section header", + ) + ) + + if hasattr(pe, "DIRECTORY_ENTRY_IMPORT"): + for dll in pe.DIRECTORY_ENTRY_IMPORT: + try: + dll_name = dll.dll.decode("ascii") + except UnicodeDecodeError: + continue + + rva = dll.struct.Name + size = len(dll_name) + try: + offset = pe.get_offset_from_rva(rva) + except pefile.PEFormatError as e: + logger.warning("failed to get offset for import DLL name RVA 0x%x: %s", rva, e) + continue + + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="import table", + ) + ) + + for entry in dll.imports: + if entry.name is None: + continue + + if entry.name_offset is None: + continue + + try: + symbol_name = entry.name.decode("ascii") + except UnicodeDecodeError: + continue + + offset = entry.name_offset + size = len(symbol_name) + + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="import table", + ) + ) + + if hasattr(pe, "DIRECTORY_ENTRY_EXPORT"): + exp = pe.DIRECTORY_ENTRY_EXPORT + if hasattr(exp, "name") and exp.name: + try: + dll_name = exp.name.decode("ascii") + rva = exp.struct.Name + size = len(dll_name) + offset = pe.get_offset_from_rva(rva) + + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="export table", + ) + ) + except (UnicodeDecodeError, pefile.PEFormatError) as e: + logger.warning("failed to parse export table DLL name: %s", e) + + if hasattr(exp, "symbols"): + for entry in exp.symbols: + if entry.name is None: + continue + + if entry.name_offset is None: + continue + + try: + symbol_name = entry.name.decode("ascii") + except UnicodeDecodeError: + continue + + offset = entry.name_offset + size = len(symbol_name) + + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="export table", + ) + ) + + if entry.forwarder: + try: + forwarder_name = entry.forwarder.decode("ascii") + except UnicodeDecodeError: + continue + offset = entry.forwarder_offset + size = len(forwarder_name) + structures.append( + Structure( + slice=slice_.slice(offset, size), + name="export table", + ) + ) + + if hasattr(pe, "RICH_HEADER") and pe.RICH_HEADER: + key_bytes = pe.RICH_HEADER.key + + rich_sig_offset = pe.__data__.find(b"Rich", 0x40, pe.DOS_HEADER.e_lfanew) + # The structure end is 'Rich' (4) + key (4) = 8 bytes + rich_end = rich_sig_offset + 8 + + # Find the start of rich header by looking for 'DanS' XORed with the key + xor_dans = bytes(a ^ b for a, b in zip(b"DanS", key_bytes)) + rich_start = pe.__data__.rfind(xor_dans, 0x40, rich_sig_offset) + + if rich_sig_offset != -1 and rich_start != -1: + structures.append(Structure(slice=slice_.slice(rich_start, rich_end - rich_start), name="rich header")) + + return structures + + +def compute_pe_layout(slice_: Slice, xor_key: int | None) -> Layout: + data = slice_.data + + try: + pe = pefile.PE(data=data) + except pefile.PEFormatError as e: + raise ValueError("pefile failed to load workspace") from e + + structures = collect_pe_structures(slice_, pe) + reloc_offsets = OffsetRanges.from_offsets(get_reloc_offsets(slice_, pe)) + + structures_by_address = {} + for structure in structures: + for offset in structure.slice.range: + structures_by_address[offset] = structure + + be2: Optional[lancelot.BinExport2] = None + with timing("lancelot: load workspace"): + try: + be2 = lancelot.get_binexport2_from_bytes(data) + except ValueError as e: + logger.warning("lancelot failed to load workspace: %s", e) + except BaseException as e: + if isinstance(e, (KeyboardInterrupt, SystemExit)): + raise + logger.warning("lancelot failed critically (panic): %s", e) + + # contains the file offsets of bytes that are part of recognized instructions. + code_offsets = OffsetRanges() + if be2: + with timing("lancelot: find code"): + base_address = lancelot.be2utils.find_be2_base_address(be2) + idx = lancelot.be2utils.BinExport2Index(be2) + code_ranges = _get_code_ranges(be2, idx, base_address, pe, slice_) + merged_code_ranges = merge_overlapping_ranges(code_ranges) + code_offsets = OffsetRanges.from_merged_ranges(merged_code_ranges) + + layout = PELayout( + slice=slice_, + name="pe", + xor_key=xor_key, + reloc_offsets=reloc_offsets, + code_offsets=code_offsets, + structures_by_address=structures_by_address, + ) + + if xor_key: + layout.name += f" (XOR decoded with key: 0x{xor_key:x})" + + for section in pe.sections: + if section.SizeOfRawData == 0: + continue + + try: + name = section.Name.partition(b"\x00")[0].decode("utf-8") + except UnicodeDecodeError: + name = "(invalid)" + + offset = section.get_PointerToRawData_adj() + size = section.SizeOfRawData + + if offset > slice_.range.end: + logger.warning("section %s out of range", name) + continue + + if offset + size > slice_.range.length: + size_orig = size + size = slice_.range.length - offset + assert size >= 0 + logger.warning("section size %s out of range, truncating from 0x%x to 0x%x bytes", name, size_orig, size) + + layout.add_child(SectionLayout(slice=slice_.slice(offset, size), name=name, section=section)) + + # segment that contains all data until the first section + offset = 0 + size = layout.children[0].offset - slice_.range.offset + layout.add_child( + SegmentLayout( + slice=slice_.slice(offset, size), + name="header", + ) + ) + + # segment that contains all data after the last section + # aka. "overlay" + last_section: Layout = layout.children[-1] + if last_section.end < layout.end: + offset = last_section.end - layout.offset + size = layout.end - last_section.end + layout.add_child( + SegmentLayout( + slice=slice_.slice(offset, size), + name="overlay", + ) + ) + + # the "overlay" may contain Authenticode digital signatures + security = pe.OPTIONAL_HEADER.DATA_DIRECTORY[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_SECURITY"]] + if security.VirtualAddress and security.Size - 1 > 0: + overlay: Layout = layout.children[-1] + if overlay.name != "overlay": + logger.debug("expected overlay to be present") + # tread with caution + + if overlay.end < (security.VirtualAddress + security.Size - 1): + logger.debug("overlay ends before authenticode digital signature") + else: + overlay.add_child( + SegmentLayout( + slice=slice_.slice(security.VirtualAddress, security.Size - 1), + name="Authenticode digital signature", + ) + ) + + # add segments for any gaps between sections. + # note that we append new items to the end of the list and then resort, + # to avoid mutating the list while we're iterating over it. + for i in range(1, len(layout.children)): + prior: Layout = layout.children[i - 1] + current: Layout = layout.children[i] + + if prior.end != current.offset: + offset = prior.end + size = current.offset - prior.end + layout.add_child( + SegmentLayout( + slice=slice_.slice(offset, size), + name="gap", + ) + ) + + if hasattr(pe, "DIRECTORY_ENTRY_RESOURCE"): + + def collect_pe_resources(dir_data: pefile.ResourceDirData, path: Tuple[str, ...] = ()) -> Sequence[Layout]: + resources: List[Layout] = [] + for entry in dir_data.entries: + if entry.name: + name = str(entry.name) + else: + name = str(entry.id) + if not path and entry.id in PE_RESOURCE_TYPES: + name = PE_RESOURCE_TYPES[entry.id] + + epath = path + (name,) + + if hasattr(entry, "directory"): + resources.extend(collect_pe_resources(entry.directory, epath)) + + else: + rva = entry.data.struct.OffsetToData + try: + offset = pe.get_offset_from_rva(rva) + except pefile.PEFormatError as e: + logger.warning("failed to get offset for resource RVA 0x%x: %s", rva, e) + continue + + size = entry.data.struct.Size + + if not slice_.contains_range(offset, size): + logger.warning("resource '%s' points to an invalid location, skipping", "/".join(epath)) + continue + + logger.debug("resource: %s, size: 0x%x", "/".join(epath), size) + + resources.append( + ResourceLayout( + slice=slice_.slice(offset, size), + name="rsrc: " + "/".join(epath), + ) + ) + + return resources + + resources = collect_pe_resources(pe.DIRECTORY_ENTRY_RESOURCE) + + for resource in resources: + # parse content of resources, such as embedded PE files + from floss.layout import compute_layout + + resource.add_child(compute_layout(resource.slice)) + + for resource in resources: + # place resources into their parent section, usually .rsrc + container = next( + filter(lambda candidate: candidate.offset <= resource.offset < candidate.end, layout.children) + ) + container.add_child(resource) + + return layout diff --git a/floss/layout/types.py b/floss/layout/types.py new file mode 100644 index 000000000..e66a85f3c --- /dev/null +++ b/floss/layout/types.py @@ -0,0 +1,42 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Layout string types used while walking binary layouts.""" + +from __future__ import annotations + +from typing import Set, List, Literal, TypeAlias + +from pydantic import BaseModel + +from floss.ranges import Slice + +Tag: TypeAlias = str + + +class ExtractedString(BaseModel): + string: str + slice: Slice + encoding: Literal["ascii", "unicode"] + + +class TaggedString(BaseModel): + string: ExtractedString + tags: Set[Tag] + structure: str = "" + + @property + def offset(self) -> int: + "convenience" + return self.string.slice.range.offset diff --git a/floss/logging_.py b/floss/logging_.py index 36509e76c..e8bb59633 100644 --- a/floss/logging_.py +++ b/floss/logging_.py @@ -38,7 +38,7 @@ class DebugLevel(int, Enum): def make_format(color): - return f"{color}%(levelname)s{RESET}: %(name)s: %(message)s" + return f"%(asctime)s {color}%(levelname)s{RESET}: %(name)s: %(message)s" FORMATS = { @@ -50,7 +50,7 @@ def make_format(color): logging.CRITICAL: make_format(BOLD_RED), } -FORMATTERS = {level: logging.Formatter(FORMATS[level]) for level in FORMATS.keys()} +FORMATTERS = {level: logging.Formatter(FORMATS[level], "%Y-%m-%d %H:%M:%S") for level in FORMATS.keys()} class ColorFormatter(logging.Formatter): diff --git a/floss/main.py b/floss/main.py index 6a56f28b2..269d1d0b2 100644 --- a/floss/main.py +++ b/floss/main.py @@ -13,445 +13,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os +import re import sys -import codecs -import logging -import argparse -import textwrap -from enum import Enum -from time import time -from typing import Set, List, Optional +import json from pathlib import Path -import halo -import viv_utils import rich.traceback -import viv_utils.flirt -from vivisect import VivWorkspace -import floss.utils +import floss.cache import floss.results -import floss.version import floss.logging_ import floss.render.json -import floss.language.utils import floss.render.default -import floss.language.go.extract -import floss.language.go.coverage -import floss.language.rust.extract -import floss.language.rust.coverage -from floss.const import ( - MEGABYTE, - MAX_FILE_SIZE, - MIN_STRING_LENGTH, - UNSUPPORTED_FILE_MAGIC, - SUPPORTED_FILE_MAGIC_PE, - SUPPORTED_FILE_MAGIC_ELF, +import floss.render.summary +from floss.cli import ( + SIGNATURES_PATH_DEFAULT_STRING, + StringType, + ArgumentValueError, + make_parser, + set_log_config, ) -from floss.utils import ( - hex, - get_imagebase, - get_runtime_diff, - get_static_strings, - get_vivisect_meta_info, - is_string_type_enabled, - set_vivisect_log_level, -) -from floss.render import Verbosity -from floss.results import Analysis, Metadata, ResultDocument, load -from floss.version import __version__ -from floss.identify import ( - append_unique, - get_function_fvas, - get_top_functions, - get_tight_function_fvas, - get_functions_with_tightloops, - find_decoding_function_features, - get_functions_without_tightloops, -) -from floss.logging_ import TRACE, DebugLevel -from floss.stackstrings import extract_stackstrings -from floss.tightstrings import extract_tightstrings -from floss.string_decoder import decode_strings -from floss.language.identify import Language, identify_language_and_version - -SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)" -EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") -EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") +from floss.utils import FileType, detect_file_type, expand_string_types, is_string_type_enabled +from floss.results import Analysis, load +from floss.pipeline import Options, PipelineError, analyze +from floss.render.filter import LayoutFilter +from floss.language.identify import Language logger = floss.logging_.getLogger("floss") -class StringType(str, Enum): - STATIC = "static" - STACK = "stack" - TIGHT = "tight" - DECODED = "decoded" - - -class WorkspaceLoadError(ValueError): - pass - - -class ArgumentValueError(ValueError): - pass - - -class ArgumentParser(argparse.ArgumentParser): - """ - argparse will call sys.exit upon parsing invalid arguments. - we don't want that, because we might be parsing args within test cases, run as a module, etc. - so, we override the behavior to raise a ArgumentValueError instead. - - this strategy is originally described here: https://stackoverflow.com/a/16942165/87207 - """ - - def error(self, message): - self.print_usage(sys.stderr) - args = {"prog": self.prog, "message": message} - raise ArgumentValueError("%(prog)s: error: %(message)s" % args) - - -def make_parser(argv): - desc = ( - "The FLARE team's open-source tool to extract ALL strings from malware.\n" - f" %(prog)s {__version__} - https://github.com/mandiant/flare-floss/\n\n" - "FLOSS extracts the following string types:\n" - ' 1. static strings: "regular" ASCII and UTF-16LE strings\n' - " 2. stack strings: strings constructed on the stack at run-time\n" - " 3. tight strings: special form of stack strings, decoded on the stack\n" - " 4. decoded strings: strings decoded in a function\n\n" - "Language-specific strings:\n" - " 1. Go: strings from binaries written in Go\n" - " 2. Rust: strings from binaries written in Rust\n" - ) - epilog = textwrap.dedent(""" - only displaying core arguments, run `floss -H` to see all supported options - - examples: - extract all strings from an executable - floss suspicious.exe - - do not extract static strings - floss --no static -- suspicious.exe - - only extract stack and tight strings - floss --only stack tight -- suspicious.exe - """) - epilog_advanced = textwrap.dedent(""" - examples: - extract all strings from 32-bit shellcode - floss -f sc32 shellcode.bin - - only decode strings from the specified functions - floss --functions 0x401000 0x401100 suspicious.exe - - extract strings from a binary written in Go (if automatic language identification fails) - floss --language go program.exe - """) - - show_all_options = "-H" in argv - - parser = ArgumentParser( - description=desc, - epilog=epilog_advanced if show_all_options else epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("-H", action="help", help="show advanced options and exit") - parser.add_argument( - "-n", - "--minimum-length", - dest="min_length", - type=int, - default=MIN_STRING_LENGTH, - help="minimum string length", - ) - - parser.add_argument( - "sample", - type=argparse.FileType("rb"), - help="path to sample to analyze", - ) - - analysis_group = parser.add_argument_group("analysis arguments") - analysis_group.add_argument( - "--no", - action="extend", - dest="disabled_types", - nargs="+", - choices=[t.value for t in StringType], - default=[], - help="do not extract specified string type(s)", - ) - analysis_group.add_argument( - "--only", - action="extend", - dest="enabled_types", - nargs="+", - choices=[t.value for t in StringType], - default=[], - help="only extract specified string type(s)", - ) - - advanced_group = parser.add_argument_group("advanced arguments") - formats = [ - ("auto", "(default) detect file type automatically"), - ("pe", "Windows PE file"), - ("sc32", "32-bit shellcode"), - ("sc64", "64-bit shellcode"), - ] - format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats]) - advanced_group.add_argument( - "-f", - "--format", - choices=[f[0] for f in formats], - default="auto", - help="select sample format, %s" % format_help if show_all_options else argparse.SUPPRESS, - ) - advanced_group.add_argument( - "--language", - type=str, - choices=[l.value for l in Language if l != Language.UNKNOWN], - default=Language.UNKNOWN.value, - help=( - "use language-specific string extraction, auto-detect language by default, disable using 'none'" - if show_all_options - else argparse.SUPPRESS - ), - ) - advanced_group.add_argument( - "-l", - "--load", - action="store_true", - help="load from existing FLOSS results document" if show_all_options else argparse.SUPPRESS, - ) - advanced_group.add_argument( - "--functions", - type=lambda x: int(x, 0x10), - default=None, - nargs="+", - help=( - "only analyze the specified functions, hex-encoded like 0x401000, space-separate multiple functions" - if show_all_options - else argparse.SUPPRESS - ), - ) - advanced_group.add_argument( - "--disable-progress", - action="store_true", - help="disable all progress bars" if show_all_options else argparse.SUPPRESS, - ) - advanced_group.add_argument( - "--signatures", - type=str, - default=SIGNATURES_PATH_DEFAULT_STRING, - help=( - "path to .sig/.pat file or directory used to identify library functions, use embedded signatures by default" - if show_all_options - else argparse.SUPPRESS - ), - ) - advanced_group.add_argument( - "-L", - "--large-file", - action="store_true", - help=( - "allow processing files larger than {} MB".format(int(MAX_FILE_SIZE / MEGABYTE)) - if show_all_options - else argparse.SUPPRESS - ), - ) - advanced_group.add_argument( - "--version", - action="version", - version="%(prog)s {:s}".format(__version__), - help="show program's version number and exit" if show_all_options else argparse.SUPPRESS, - ) - if sys.platform == "win32": - advanced_group.add_argument( - "--install-right-click-menu", - action=floss.utils.InstallContextMenu, - help=( - "install FLOSS to the right-click context menu for Windows Explorer and exit" - if show_all_options - else argparse.SUPPRESS - ), - ) - - advanced_group.add_argument( - "--uninstall-right-click-menu", - action=floss.utils.UninstallContextMenu, - help=( - "uninstall FLOSS from the right-click context menu for Windows Explorer and exit" - if show_all_options - else argparse.SUPPRESS - ), - ) - - output_group = parser.add_argument_group("rendering arguments") - output_group.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text") - output_group.add_argument( - "-v", - "--verbose", - action="count", - default=Verbosity.DEFAULT, - help="enable verbose results, e.g. including function offsets (does not affect JSON output)", - ) - - logging_group = parser.add_argument_group("logging arguments") - logging_group.add_argument( - "-d", - "--debug", - action="count", - default=DebugLevel.NONE, - help="enable debugging output on STDERR, specify multiple times to increase verbosity", - ) - logging_group.add_argument( - "-q", "--quiet", action="store_true", help="disable all status output on STDOUT except fatal errors" - ) - logging_group.add_argument( - "--color", - type=str, - choices=("auto", "always", "never"), - default="auto", - help="enable ANSI color codes in results, default: only during interactive session", - ) - - return parser - - -def set_log_config(debug, quiet): - if quiet: - log_level = logging.WARNING - elif debug >= DebugLevel.TRACE: - log_level = TRACE - elif debug >= DebugLevel.DEFAULT: - log_level = logging.DEBUG - else: - log_level = logging.INFO - - logging.basicConfig(level=log_level) - logging.getLogger().setLevel(log_level) - - if debug < DebugLevel.SUPERTRACE: - # these loggers are too verbose even for the TRACE level, enable via `-ddd` - logging.getLogger("floss.api_hooks").setLevel(logging.WARNING) - logging.getLogger("floss.function_argument_getter").setLevel(logging.WARNING) - - # configure vivisect-related logging, it's verbose and not relevant for regular FLOSS users - # enable to do more vigorous testing - if debug < DebugLevel.TRACE: - set_vivisect_log_level(logging.CRITICAL) - else: - set_vivisect_log_level(logging.DEBUG) - - # configure viv-utils logging - if debug == DebugLevel.DEFAULT: - logging.getLogger("viv_utils.emulator_drivers").setLevel(logging.DEBUG) - elif debug <= DebugLevel.TRACE: - logging.getLogger("viv_utils.emulator_drivers").setLevel(logging.ERROR) - - # install the log message colorizer to the default handler. - # because basicConfig is just above this, - # handlers[0] is a StreamHandler to STDERR. - # - # calling this code from outside script main may do something unexpected. - logging.getLogger().handlers[0].setFormatter(floss.logging_.ColorFormatter()) - - -def select_functions(vw, asked_functions: Optional[List[int]]) -> Set[int]: - """ - Given a workspace and an optional list of function addresses, - collect the set of valid functions, - or all valid function addresses. - - arguments: - asked_functions: the functions a user wants, or None. - - raises: - ValueError: if an asked for function does not exist in the workspace. - """ - functions = set(vw.getFunctions()) - if not asked_functions: - # user didn't specify anything, so return them all. - logger.debug("selected ALL functions") - return functions - - asked_functions_ = set(asked_functions or []) - - # validate that all functions requested by the user exist. - missing_functions = sorted(asked_functions_ - functions) - if missing_functions: - raise ValueError("failed to find functions: %s" % (", ".join(map(hex, sorted(missing_functions))))) - - logger.debug("selected %d functions", len(asked_functions_)) - logger.trace("selected the following functions: %s", ", ".join(map(hex, sorted(asked_functions_)))) - - return asked_functions_ - - -def get_file_type(sample_file_path: Path) -> bytes: - """ - Returns input file type, based on header bytes - :param sample_file_path: - :return: file type - """ - with sample_file_path.open("rb") as f: - magic = f.read(4) - - if magic == SUPPORTED_FILE_MAGIC_ELF: - return SUPPORTED_FILE_MAGIC_ELF - elif magic[:2] == SUPPORTED_FILE_MAGIC_PE: - return SUPPORTED_FILE_MAGIC_PE - else: - return UNSUPPORTED_FILE_MAGIC - - -def load_vw( - sample_path: Path, - format: str, - sigpaths: List[Path], - should_save_workspace: bool = False, -) -> VivWorkspace: - file_type = get_file_type(sample_path) - if format not in ("sc32", "sc64"): - if file_type is UNSUPPORTED_FILE_MAGIC: - raise WorkspaceLoadError( - "FLOSS currently supports the following formats for string decoding and stackstrings: PE and ELF\n" - "You can analyze shellcode using the --format sc32|sc64 switch. See the help (-h) for more information." - ) - - # get shellcode type based on sample file extension - if format == "auto" and sample_path.suffix.lower() in EXTENSIONS_SHELLCODE_32: - format = "sc32" - elif format == "auto" and sample_path.suffix.lower() in EXTENSIONS_SHELLCODE_64: - format = "sc64" - - if format == "sc32": - vw = viv_utils.getShellcodeWorkspaceFromFile(str(sample_path), arch="i386", analyze=False) - elif format == "sc64": - vw = viv_utils.getShellcodeWorkspaceFromFile(str(sample_path), arch="amd64", analyze=False) - else: - vw = viv_utils.getWorkspace(str(sample_path), analyze=False, should_save=False) - - if file_type == SUPPORTED_FILE_MAGIC_PE: - viv_utils.flirt.register_flirt_signature_analyzers(vw, list(map(str, sigpaths))) - - vw.analyze() - - if should_save_workspace: - logger.debug("saving workspace") - try: - vw.saveWorkspace() - except IOError: - logger.info("source directory is not writable, won't save intermediate workspace") - else: - logger.debug("not saving workspace") - - return vw - - def is_running_standalone() -> bool: """ are we running from a PyInstaller'd executable? @@ -475,32 +65,55 @@ def get_default_root() -> Path: return Path(__file__).resolve().parent -def get_signatures(sigs_path: Path) -> List[Path]: - if not sigs_path.exists(): - raise IOError("signatures path %s does not exist or cannot be accessed" % str(sigs_path)) - - paths = [] - if sigs_path.is_file(): - paths.append(sigs_path) - elif sigs_path.is_dir(): - logger.debug("reading signatures from directory %s", str(sigs_path.resolve().absolute())) - for item in sigs_path.iterdir(): - if item.is_file(): - if item.suffix in [".pat", ".pat.gz", ".sig"]: - sig_path = item - paths.append(sig_path) +def emit_json_error(message: str, code: int = 1) -> None: + """emit a structured JSON error on STDERR for the JSON output modes.""" + sys.stderr.write(json.dumps({"error": message, "code": code}) + "\n") - # nicely normalize and format path so that debugging messages are clearer - paths = [path.resolve().absolute() for path in paths] - # load signatures in deterministic order: the alphabetic sorting of filename. - # this means that `0_sigs.pat` loads before `1_sigs.pat`. - paths = sorted(paths, key=lambda p: p.name) +def report_error(parser, message: str) -> None: + """emit an error in the active output mode: JSON object on STDERR for + JSON modes, otherwise a plain message on STDERR.""" + if parser.json_mode: + emit_json_error(message) + else: + print(message, file=sys.stderr) + + +def json_requested(argv) -> bool: + """best-effort detection of a JSON output mode before argument parsing completes.""" + argv = argv or [] + return "--json" in argv or any(a == "-j" for a in argv) + + +def build_layout_filter(args) -> LayoutFilter: + return LayoutFilter( + include_sections=args.include_sections, + exclude_sections=args.exclude_sections, + include_structures=args.include_structures, + exclude_structures=args.exclude_structures, + include_tags=args.include_tags, + exclude_tags=args.exclude_tags, + interesting=args.interesting, + queries=args.queries, + max_strings=args.max_strings, + tag_rules=floss.render.default.DEFAULT_TAG_RULES, + ) - for path in paths: - logger.debug("found signature file: %s", str(path)) - return paths +def render_text(args, results: floss.results.ResultDocument) -> str: + """render results as text, honoring --summary, --plain, --columns, and the filters.""" + if args.summary: + # --summary is its own output; it needs the layout tree intact + return floss.render.summary.render_summary(results, args.color) + return floss.render.default.render( + results, + args.verbose, + args.quiet, + args.color, + columns=args.columns, + layout_filter=build_layout_filter(args), + plain=args.plain, + ) def main(argv=None) -> int: @@ -508,24 +121,68 @@ def main(argv=None) -> int: arguments: argv: the command line arguments """ - # use rich as default Traceback handler rich.traceback.install(show_locals=True) if argv is None: argv = sys.argv[1:] - parser = make_parser(argv) + parser = make_parser() + parser.json_mode = json_requested(argv) try: + if not argv: + # no arguments: print the full option list and exit with code 1 + parser.print_help() + return 1 args = parser.parse_args(args=argv) - # manual check here, because add_mutually_exclusive_group() on argument_group("...") appears wrong - if args.enabled_types and args.disabled_types: - parser.error("--no and --only arguments are not allowed together") + for flag, include, exclude in ( + ("--string-type", args.enabled_string_types, args.disabled_string_types), + ("--section", args.include_sections, args.exclude_sections), + ("--structure", args.include_structures, args.exclude_structures), + ("--tag", args.include_tags, args.exclude_tags), + ): + if include and exclude: + parser.error("%s and --no-%s arguments are not allowed together" % (flag, flag[2:])) + for flag, values in ( + ("--string-type", args.enabled_string_types), + ("--no-string-type", args.disabled_string_types), + ): + if len(values) > 1 and StringType.ALL.value in values: + parser.error("%s: 'all' cannot be combined with other string types" % flag) + if args.summary: + if args.analyze_functions: + parser.error( + "--summary only covers static strings, which --analyze-functions does not show; " + "these flags cannot be combined" + ) + if args.enabled_string_types or args.disabled_string_types: + # --summary is its own static-only view; reject any string-type + # selection rather than accepting shadow args + parser.error("--summary only covers static strings and does not take --string-type/--no-string-type") + if args.max_strings is not None and args.max_strings <= 0: + parser.error("--max-strings must be a positive integer") + for pattern in args.queries: + try: + re.compile(pattern) + except re.error as e: + parser.error("invalid --query regular expression %r: %s" % (pattern, e)) except ArgumentValueError as e: - print(e) + report_error(parser, str(e)) return -1 set_log_config(args.debug, args.quiet) + # caching applies to the default analysis variant only: an explicit format, + # language, or custom signatures change the analysis, so the content- + # addressed cache could return a mismatched document. disable it for those. + cache_dir = None + if ( + not args.analyze_functions + and args.format == "auto" + and args.language == Language.AUTO.value + and args.signatures == SIGNATURES_PATH_DEFAULT_STRING + ): + cache_dir = floss.cache.get_cache_dir() + if hasattr(args, "signatures"): if args.signatures == SIGNATURES_PATH_DEFAULT_STRING: logger.debug("-" * 80) @@ -542,300 +199,111 @@ def main(argv=None) -> int: args.signatures = sigs_path - # alternatively: pass buffer along instead of file path, also should work for stdin sample = Path(args.sample.name) args.sample.close() - if args.functions: - if is_string_type_enabled(StringType.STATIC, args.disabled_types, args.enabled_types): + disabled_string_types = expand_string_types(list(args.disabled_string_types or [])) + enabled_string_types = expand_string_types(list(args.enabled_string_types or [])) + + if args.summary and not disabled_string_types and not enabled_string_types: + # the summary's layout-derived sections cover static strings only, so + # don't spin up the slow deobfuscation for stack/tight/decoded + logger.info("--summary is static-only; skipping stack/tight/decoded extraction") + disabled_string_types.extend([StringType.STACK.value, StringType.TIGHT.value, StringType.DECODED.value]) + + if args.analyze_functions: + static_was_enabled = is_string_type_enabled(StringType.STATIC, disabled_string_types, enabled_string_types) + try: + if enabled_string_types and StringType.STATIC.value in enabled_string_types: + # --string-type explicitly selected static, but --analyze-functions cannot show it: + # drop it from the include list instead of forcing the exclude list. + enabled_string_types.remove(StringType.STATIC.value) + if not enabled_string_types: + parser.error( + "--string-type static cannot be combined with --analyze-functions, " + "which does not show static strings" + ) + elif not enabled_string_types and StringType.STATIC.value not in disabled_string_types: + disabled_string_types.append(StringType.STATIC.value) + except ArgumentValueError as e: + report_error(parser, str(e)) + return -1 + if static_was_enabled: logger.warning("analyzing specified functions, not showing static strings") - args.disabled_types.append(StringType.STATIC) + # layout/tags are always on: automatic and detected from the sample content analysis = Analysis( - enable_static_strings=is_string_type_enabled(StringType.STATIC, args.disabled_types, args.enabled_types), - enable_stack_strings=is_string_type_enabled(StringType.STACK, args.disabled_types, args.enabled_types), - enable_tight_strings=is_string_type_enabled(StringType.TIGHT, args.disabled_types, args.enabled_types), - enable_decoded_strings=is_string_type_enabled(StringType.DECODED, args.disabled_types, args.enabled_types), + enable_static_strings=is_string_type_enabled(StringType.STATIC, disabled_string_types, enabled_string_types), + enable_stack_strings=is_string_type_enabled(StringType.STACK, disabled_string_types, enabled_string_types), + enable_tight_strings=is_string_type_enabled(StringType.TIGHT, disabled_string_types, enabled_string_types), + enable_decoded_strings=is_string_type_enabled(StringType.DECODED, disabled_string_types, enabled_string_types), + enable_language_strings=is_string_type_enabled( + StringType.LANGUAGE, disabled_string_types, enabled_string_types + ), ) - if args.load: + if detect_file_type(sample) is FileType.RESULTS: try: - results = load(sample, analysis, args.functions, args.min_length) + results = load(sample, analysis, args.analyze_functions, args.min_length) except floss.results.InvalidResultsFile as e: - logger.error("cannot load JSON results file: %s", e) + if args.json: + emit_json_error(f"cannot load JSON results file: {e}") + else: + logger.error("cannot load JSON results file: %s", e) return -1 except floss.results.InvalidLoadConfig as e: - logger.error("%s", e) + if args.json: + emit_json_error(str(e)) + else: + logger.error("%s", e) return -1 if args.json: r = floss.render.json.render(results) else: - r = floss.render.default.render(results, args.verbose, args.quiet, args.color) + r = render_text(args, results) print(r) - return 0 - results = ResultDocument(metadata=Metadata(file_path=str(sample), min_length=args.min_length), analysis=analysis) - - sample_size = sample.stat().st_size - if sample_size > sys.maxsize: - logger.warning("file is very large, strings listings may be truncated") + options = Options( + sample=sample, + min_length=args.min_length, + analysis=analysis, + format=args.format, + language=args.language, + enabled_string_types=enabled_string_types, + disabled_string_types=disabled_string_types, + analyze_functions=args.analyze_functions, + signatures=args.signatures, + large_file=args.large_file, + quiet=args.quiet, + verbose=args.verbose, + cache_dir=cache_dir, + ) - # always extract static strings, it's fast and we use them for language identification - # can throw away result later if not desired in output - time0 = time() - interim = time0 + try: + analysis_results = analyze(options) + except PipelineError as e: + if args.json: + emit_json_error(str(e), code=e.exit_code if e.exit_code >= 0 else 1) + elif e.exit_code in (1, 130): + logger.info("%s", e) + else: + logger.error("%s", e) + return e.exit_code - static_strings = get_static_strings(sample, args.min_length) - if not static_strings: + if analysis_results is None: return 0 - static_runtime = get_runtime_diff(interim) - # set language configurations - selected_lang = Language(args.language) - if selected_lang == Language.DISABLED: - results.metadata.language = "" - results.metadata.language_version = "" - results.metadata.language_selected = "" - else: - lang_id, lang_version = identify_language_and_version(sample, static_strings) - - if selected_lang == Language.UNKNOWN: - pass - elif selected_lang != lang_id: - logger.warning( - "the selected language '%s' differs to the automatically identified language '%s (%s)' - extracted " - "strings may be incomplete or inaccurate", - selected_lang.value, - lang_id.value, - lang_version, - ) - results.metadata.language_selected = selected_lang.value - - results.metadata.language = lang_id.value - results.metadata.language_version = lang_version - - if results.metadata.language == Language.GO.value: - if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: - logger.warning( - "FLOSS handles Go static strings, but string deobfuscation may be inaccurate and take a long time" - ) - - elif results.metadata.language == Language.RUST.value: - if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: - logger.warning( - "FLOSS handles Rust static strings, but string deobfuscation may be inaccurate and take a long time" - ) - - elif results.metadata.language == Language.DOTNET.value: - logger.warning(".NET language-specific string extraction is not supported yet") - logger.warning("FLOSS does NOT attempt to deobfuscate any strings from .NET binaries") - - # enable .NET strings once we can extract them - # results.metadata.language = Language.DOTNET.value - - # TODO for pure .NET binaries our deobfuscation algorithms do nothing, but for mixed-mode assemblies they may - analysis.enable_stack_strings = False - analysis.enable_tight_strings = False - analysis.enable_decoded_strings = False - - if results.metadata.language not in ("", "unknown"): - if args.enabled_types == [] and args.disabled_types == []: - # when stdout is redirected, such as in 'floss foo.exe | less' use default prompt values - if sys.stdout.isatty(): - try: - prompt = input("Do you want to enable string deobfuscation? (this could take a long time) [y/N] ") - except KeyboardInterrupt: - logger.info("aborted by user") - return 130 - except EOFError: - logger.info("aborted by user") - return 1 - else: - prompt = "n" - - if prompt.lower() == "y": - logger.info("enabled string deobfuscation") - analysis.enable_stack_strings = True - analysis.enable_tight_strings = True - analysis.enable_decoded_strings = True - - else: - logger.info("disabled string deobfuscation") - analysis.enable_stack_strings = False - analysis.enable_tight_strings = False - analysis.enable_decoded_strings = False - - # in order of expected run time, fast to slow - # 1. static strings (done above) - # a) includes language-specific strings, if applicable - # 2. stack strings - # 3. tight strings - # 4. decoded strings - - if results.analysis.enable_static_strings: - logger.info("extracting static strings") - results.strings.static_strings = static_strings - results.metadata.runtime.static_strings = static_runtime - - if results.metadata.language == Language.GO.value: - logger.info("extracting language-specific Go strings") - - interim = time() - results.strings.language_strings = floss.language.go.extract.extract_go_strings(sample, args.min_length) - results.metadata.runtime.language_strings = get_runtime_diff(interim) - - # missed strings only includes non-identified strings in searched range - # here currently only focus on strings in string blob range - string_blob_strings = floss.language.go.extract.get_static_strings_from_blob_range(sample, static_strings) - results.strings.language_strings_missed = floss.language.utils.get_missed_strings( - string_blob_strings, results.strings.language_strings, args.min_length - ) - - elif results.metadata.language == Language.RUST.value: - logger.info("extracting language-specific Rust strings") - - interim = time() - results.strings.language_strings = floss.language.rust.extract.extract_rust_strings(sample, args.min_length) - results.metadata.runtime.language_strings = get_runtime_diff(interim) - - # currently Rust strings are only extracted from the .rdata section - rdata_strings = floss.language.rust.extract.get_static_strings_from_rdata(sample, static_strings) - results.strings.language_strings_missed = floss.language.utils.get_missed_strings( - rdata_strings, results.strings.language_strings, args.min_length - ) - if ( - results.analysis.enable_decoded_strings - or results.analysis.enable_stack_strings - or results.analysis.enable_tight_strings - ): - if sample_size > MAX_FILE_SIZE: - if not args.large_file: - logger.error( - "cannot deobfuscate strings from files larger than 0x%x bytes", - MAX_FILE_SIZE, - ) - return -1 - else: - logger.warning( - "a large file was provided with a size of %i bytes, this may take much more time and system resource to process", - sample_size, - ) - - sigpaths = get_signatures(args.signatures) - - should_save_workspace = os.environ.get("FLOSS_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) - try: - with halo.Halo( - text="analyzing program", - spinner="simpleDots", - stream=sys.stderr, - enabled=not (args.quiet or args.disable_progress), - ): - interim = time() - vw = load_vw(sample, args.format, sigpaths, should_save_workspace) - results.metadata.runtime.vivisect = get_runtime_diff(interim) - interim = time() - except WorkspaceLoadError as e: - logger.error("failed to analyze sample: %s", e) - return -1 - - results.metadata.imagebase = get_imagebase(vw) - - try: - selected_functions = select_functions(vw, args.functions) - results.analysis.functions.discovered = len(vw.getFunctions()) - except ValueError as e: - # failed to find functions in workspace - logger.error(e.args[0]) - return -1 - - decoding_function_features, library_functions = find_decoding_function_features( - vw, selected_functions, disable_progress=args.quiet or args.disable_progress - ) - results.analysis.functions.library = len(library_functions) - results.metadata.runtime.find_features = get_runtime_diff(interim) - interim = time() - - logger.trace("analysis summary:") - for k, v in get_vivisect_meta_info(vw, selected_functions, decoding_function_features).items(): - logger.trace(" %s: %s", k, v or "N/A") - - if results.analysis.enable_stack_strings: - if results.analysis.enable_tight_strings: - # don't run this on functions with tight loops as this will likely result in FPs - # and should be caught by the tightstrings extraction below - selected_functions = get_functions_without_tightloops(decoding_function_features) - - results.strings.stack_strings = extract_stackstrings( - vw, - selected_functions, - args.min_length, - verbosity=args.verbose, - disable_progress=args.quiet or args.disable_progress, - ) - results.analysis.functions.analyzed_stack_strings = len(selected_functions) - results.metadata.runtime.stack_strings = get_runtime_diff(interim) - interim = time() - - if results.analysis.enable_tight_strings: - tightloop_functions = get_functions_with_tightloops(decoding_function_features) - results.strings.tight_strings = extract_tightstrings( - vw, - tightloop_functions, - min_length=args.min_length, - verbosity=args.verbose, - disable_progress=args.quiet or args.disable_progress, - ) - results.analysis.functions.analyzed_tight_strings = len(tightloop_functions) - results.metadata.runtime.tight_strings = get_runtime_diff(interim) - interim = time() - - if results.analysis.enable_decoded_strings: - # TODO select more based on score rather than absolute count?! - top_functions = get_top_functions(decoding_function_features, 20) - - fvas_to_emulate = get_function_fvas(top_functions) - fvas_tight_functions = get_tight_function_fvas( - decoding_function_features - ) # TODO exclude tight functions from stackstrings analysis?! - fvas_to_emulate = append_unique(fvas_to_emulate, fvas_tight_functions) - - if len(fvas_to_emulate) == 0: - logger.info("no candidate decoding functions found.") - else: - logger.debug("identified %d candidate decoding functions", len(fvas_to_emulate)) - for fva in fvas_to_emulate: - score = decoding_function_features[fva]["score"] - xrefs_to = decoding_function_features[fva]["xrefs_to"] - results.analysis.functions.decoding_function_scores[fva] = {"score": score, "xrefs_to": xrefs_to} - logger.debug(" - 0x%x: score: %.3f, xrefs to: %d", fva, score, xrefs_to) - - # TODO filter out strings decoded in library function or function only called by library function(s) - results.strings.decoded_strings = decode_strings( - vw, - fvas_to_emulate, - args.min_length, - verbosity=args.verbose, - disable_progress=args.quiet or args.disable_progress, - ) - results.analysis.functions.analyzed_decoded_strings = len(fvas_to_emulate) - results.metadata.runtime.decoded_strings = get_runtime_diff(interim) - - results.metadata.runtime.total = get_runtime_diff(time0) - logger.info("finished execution after %.2f seconds", results.metadata.runtime.total) - if args.json: - r = floss.render.json.render(results) + r = floss.render.json.render(analysis_results) else: # this may be slow when there's many strings, so informing users what's happening logger.info("rendering results") - r = floss.render.default.render(results, args.verbose, args.quiet, args.color) + r = render_text(args, analysis_results) print(r) - return 0 diff --git a/floss/pipeline.py b/floss/pipeline.py new file mode 100644 index 000000000..42150f9a8 --- /dev/null +++ b/floss/pipeline.py @@ -0,0 +1,592 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Full FLOSS analysis orchestration. + +Unified pipeline: static/language strings, optional layout+tags, +then vivisect deobfuscation (stack/tight/decoded) when enabled. +""" + +from __future__ import annotations + +import os +import sys +import hashlib +from time import time +from typing import Set, List, Optional +from pathlib import Path +from dataclasses import dataclass + +import halo +import viv_utils +import viv_utils.flirt +from vivisect import VivWorkspace + +import floss.cache +import floss.utils +import floss.results +import floss.logging_ +import floss.language.utils +import floss.language.go.extract +import floss.language.rust.extract +from floss.cli import WorkspaceLoadError +from floss.const import ( + MAX_FILE_SIZE, +) +from floss.utils import ( + FileType, + hex, + get_imagebase, + detect_file_type, + get_runtime_diff, + get_vivisect_meta_info, +) +from floss.enrich import ( + build_offset_index, + is_structured_layout, + enrich_static_strings, + static_strings_from_layout, +) +from floss.layout import Layout +from floss.render import Verbosity +from floss.results import Runtime, Analysis, Metadata, ResultLayout, ResultDocument +from floss.strings import extract_ascii_unicode_strings +from floss.version import __version__ +from floss.identify import ( + append_unique, + get_function_fvas, + get_top_functions, + get_tight_function_fvas, + get_functions_with_tightloops, + find_decoding_function_features, + get_functions_without_tightloops, +) +from floss.stackstrings import extract_stackstrings +from floss.tightstrings import extract_tightstrings +from floss.string_decoder import decode_strings +from floss.language.identify import Language, identify_language_and_version + +logger = floss.logging_.getLogger("floss.pipeline") + +EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") +EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") + + +class PipelineError(Exception): + """Analysis failed; ``exit_code`` is for the CLI.""" + + def __init__(self, message: str, exit_code: int = -1): + super().__init__(message) + self.exit_code = exit_code + + +@dataclass +class Options: + sample: Path + min_length: int + analysis: Analysis + format: str = "auto" + language: str = Language.UNKNOWN.value + enabled_string_types: Optional[List[str]] = None + disabled_string_types: Optional[List[str]] = None + analyze_functions: Optional[List[int]] = None + signatures: Optional[Path] = None + large_file: bool = False + quiet: bool = False + verbose: int = Verbosity.DEFAULT + # analysis cache directory; None disables caching (default in the CLI is + # the platform cache directory via floss.cache.get_cache_dir()) + cache_dir: Optional[Path] = None + + +def select_functions(vw, asked_functions: Optional[List[int]]) -> Set[int]: + """ + Given a workspace and an optional list of function addresses, + collect the set of valid functions, or all valid function addresses. + + raises: + ValueError: if an asked for function does not exist in the workspace. + """ + functions = set(vw.getFunctions()) + if not asked_functions: + # user didn't specify anything, so return them all. + logger.debug("selected ALL functions") + return functions + + asked_functions_ = set(asked_functions or []) + + # validate that all functions requested by the user exist. + missing_functions = sorted(asked_functions_ - functions) + if missing_functions: + raise ValueError("failed to find functions: %s" % (", ".join(map(hex, sorted(missing_functions))))) + + logger.debug("selected %d functions", len(asked_functions_)) + logger.trace("selected the following functions: %s", ", ".join(map(hex, sorted(asked_functions_)))) + + return asked_functions_ + + +def load_vw( + sample_path: Path, + format: str, + sigpaths: List[Path], + should_save_workspace: bool = False, +) -> VivWorkspace: + file_type = detect_file_type(sample_path) + if format not in ("sc32", "sc64"): + if file_type in (FileType.UNSUPPORTED, FileType.RESULTS): + raise WorkspaceLoadError( + "FLOSS currently supports the following formats for string decoding and stackstrings: PE and ELF\n" + "You can analyze shellcode using the --format sc32|sc64 switch. See the help (-h) for more information." + ) + + if format == "auto" and sample_path.suffix.lower() in EXTENSIONS_SHELLCODE_32: + format = "sc32" + elif format == "auto" and sample_path.suffix.lower() in EXTENSIONS_SHELLCODE_64: + format = "sc64" + + if format == "sc32": + vw = viv_utils.getShellcodeWorkspaceFromFile(str(sample_path), arch="i386", analyze=False) + elif format == "sc64": + vw = viv_utils.getShellcodeWorkspaceFromFile(str(sample_path), arch="amd64", analyze=False) + else: + vw = viv_utils.getWorkspace(str(sample_path), analyze=False, should_save=False) + + if file_type is FileType.PE: + viv_utils.flirt.register_flirt_signature_analyzers(vw, list(map(str, sigpaths))) + + vw.analyze() + + if should_save_workspace: + logger.debug("saving workspace") + try: + vw.saveWorkspace() + except IOError: + logger.info("source directory is not writable, won't save intermediate workspace") + else: + logger.debug("not saving workspace") + + return vw + + +def get_signatures(sigs_path: Path) -> List[Path]: + if not sigs_path.exists(): + raise IOError("signatures path %s does not exist or cannot be accessed" % str(sigs_path)) + + paths = [] + if sigs_path.is_file(): + paths.append(sigs_path) + elif sigs_path.is_dir(): + logger.debug("reading signatures from directory %s", str(sigs_path.resolve().absolute())) + for item in sigs_path.iterdir(): + if item.is_file(): + if item.suffix in [".pat", ".pat.gz", ".sig"]: + paths.append(item) + + # load signatures in deterministic order: the alphabetic sorting of filename. + # this means that `0_sigs.pat` loads before `1_sigs.pat`. + paths = [path.resolve().absolute() for path in paths] + paths = sorted(paths, key=lambda p: p.name) + + for path in paths: + logger.debug("found signature file: %s", str(path)) + + return paths + + +def compute_layout( + buf: bytes, + min_length: int, +) -> Optional[Layout]: + """ + Compute a structured layout and extract static strings. + + Returns the populated layout tree, or None to fall back to classic statics when + the layout does not parse or any step fails. Default-on layout must not + crash the whole run. + """ + from floss.layout import compute_layout as layout_compute + from floss.ranges import Slice + + try: + file_slice = Slice.from_bytes(buf=buf) + parsed_layout = layout_compute(file_slice) + + if not is_structured_layout(parsed_layout.name): + logger.debug("no structured layout (got %r); using classic static strings", parsed_layout.name) + return None + + parsed_layout.extract_strings(min_length) + return parsed_layout + except Exception as e: + logger.warning("layout-aware static analysis failed; using classic statics: %s", e) + return None + + +def tag_layout( + layout: Layout, + enable_tags: bool, +) -> None: + """ + Tag the layout strings and drop false positives. + """ + from floss.tags import load_databases, remove_false_positive_lib_strings + + # tag_strings always converts ExtractedString → TaggedString (needed for mark_structures) + taggers = load_databases() if enable_tags else [] + layout.tag_strings(taggers) + layout.mark_structures() + if enable_tags: + remove_false_positive_lib_strings(layout) + + +def try_layout_static( + buf: bytes, + min_length: int, + enable_tags: bool, + runtime: Runtime, +) -> Optional[ResultLayout]: + """ + Layout-aware static extraction with timing. + + Runs the layout computation and the tag matching steps, and records + the elapsed time of each phase separately: ``runtime.layout`` covers + the layout computation only, ``runtime.tags`` covers the tag matching + step only. + + Returns a serializable ResultLayout, or None to fall back to classic + statics. Default-on layout must not crash the whole run. + """ + try: + with runtime.measure_and_set_time("layout"): + layout = compute_layout(buf, min_length) + if layout is None: + return None + + with runtime.measure_and_set_time("tags"): + tag_layout(layout, enable_tags) + + return ResultLayout.from_layout(layout) + except Exception as e: + logger.warning("layout-aware static analysis failed; using classic statics: %s", e) + return None + + +def analyze(options: Options) -> Optional[ResultDocument]: + """ + Run full analysis. Returns None when there are no static strings to start from + (matches historical CLI early-exit with code 0). + """ + sample = options.sample + analysis = options.analysis + + results = ResultDocument( + metadata=Metadata(file_path=str(sample), min_length=options.min_length), + analysis=analysis, + ) + + sample_size = sample.stat().st_size + if sample_size > sys.maxsize: + logger.warning("file is very large, strings listings may be truncated") + + time0 = time() + + # one read for classic statics + layout (layout is default and needs a full buffer) + # TODO: mmap-only classic path if layout is ever optional-only again + sample_buf = sample.read_bytes() + if not sample_buf: + logger.warning("file is empty") + return None + + results.metadata.md5 = hashlib.md5(sample_buf).hexdigest() + results.metadata.sha1 = hashlib.sha1(sample_buf).hexdigest() + results.metadata.sha256 = hashlib.sha256(sample_buf).hexdigest() + + # result caching: keyed on the sample bytes + FLOSS version. on a valid hit + # we skip extraction, layout, and tagging entirely and render from the cache. + # FLOSS_CACHE_REFRESH=1 forces a miss so the fresh result overwrites the entry. + cache_key: Optional[str] = None + if options.cache_dir is not None and not options.analyze_functions and floss.cache.cache_enabled(): + # --analyze-functions changes which functions are analyzed, so it is a miss: + # a hit would wrongly return a full-function document. + cache_key = floss.cache.compute_key(results.metadata.sha256, __version__, options.format) + if not floss.cache.cache_refresh(): + cached = floss.cache.load(options.cache_dir, cache_key, results.metadata.sha256, __version__) + if cached is not None and floss.cache.covers(cached, analysis, options.min_length): + logger.debug("using cached results: %s", cache_key) + return floss.cache.materialize(cached, sample, analysis, options.min_length) + else: + logger.debug("FLOSS_CACHE_REFRESH set; re-analyzing and overwriting cache entry %s", cache_key) + + static_strings = list(extract_ascii_unicode_strings(sample_buf, options.min_length)) + if not static_strings: + return None + + static_runtime = get_runtime_diff(time0) + + # set language configurations + selected_lang = Language(options.language) + if selected_lang == Language.DISABLED: + results.metadata.language = "" + results.metadata.language_version = "" + results.metadata.language_selected = "" + else: + lang_id, lang_version = identify_language_and_version(sample, static_strings) + + if selected_lang in (Language.UNKNOWN, Language.AUTO): + pass + elif selected_lang != lang_id: + logger.warning( + "the selected language '%s' differs to the automatically identified language '%s (%s)' - extracted " + "strings may be incomplete or inaccurate", + selected_lang.value, + lang_id.value, + lang_version, + ) + results.metadata.language_selected = selected_lang.value + + if selected_lang in (Language.GO, Language.RUST, Language.DOTNET): + # a concrete manual selection unilaterally wins over auto-detection, + # so the selected language drives extraction and rendering below + results.metadata.language = selected_lang.value + results.metadata.language_version = lang_version if lang_id == selected_lang else "" + else: + results.metadata.language = lang_id.value + results.metadata.language_version = lang_version + + if results.metadata.language == Language.GO.value: + if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: + logger.warning( + "FLOSS handles Go static strings, but string deobfuscation may be inaccurate and take a long time" + ) + + elif results.metadata.language == Language.RUST.value: + if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: + logger.warning( + "FLOSS handles Rust static strings, but string deobfuscation may be inaccurate and take a long time" + ) + + elif results.metadata.language == Language.DOTNET.value: + logger.warning(".NET language-specific string extraction is not supported yet") + logger.warning("FLOSS does NOT attempt to deobfuscate any strings from .NET binaries") + # enable .NET strings once we can extract them + # results.metadata.language = Language.DOTNET.value + # TODO for pure .NET binaries our deobfuscation algorithms do nothing, but for mixed-mode assemblies they may + analysis.enable_stack_strings = False + analysis.enable_tight_strings = False + analysis.enable_decoded_strings = False + + # in order of expected run time, fast to slow + # 1. static strings (done above for language ID; layout-aware replace below when enabled) + # a) includes language-specific strings, if applicable + # 2. stack strings + # 3. tight strings + # 4. decoded strings + + layout_doc: Optional[ResultLayout] = None + if results.analysis.enable_static_strings: + logger.info("extracting static strings") + if analysis.enable_layout: + # only layout/tag work for static_strings runtime — not language ID or the TTY prompt above + with results.metadata.runtime.measure_and_set_time("static_strings"): + layout_doc = try_layout_static( + sample_buf, options.min_length, analysis.enable_tags, results.metadata.runtime + ) + + if layout_doc is not None: + results.layout = layout_doc + results.strings.static_strings = static_strings_from_layout(layout_doc) + # add the classic extraction time (done above for language ID) + results.metadata.runtime.static_strings += static_runtime + else: + results.strings.static_strings = static_strings + # add the elapsed time of the failed/skipped layout attempt, which + # measure_and_set_time("static_strings") already recorded above + results.metadata.runtime.static_strings += static_runtime + + # language-specific strings are independent of static strings: extract them + # whenever enabled, reusing the classic extraction buffer (and layout, when + # static+layout actually ran) for missed-string/enrichment. + if analysis.enable_language_strings and results.metadata.language in (Language.GO.value, Language.RUST.value): + # one offset index for both language_strings and language_strings_missed + layout_offset_index = None + if layout_doc is not None: + layout_offset_index = build_offset_index(layout_doc) + + if results.metadata.language == Language.GO.value: + logger.info("extracting language-specific Go strings") + with results.metadata.runtime.measure_and_set_time("language_strings"): + results.strings.language_strings = floss.language.go.extract.extract_go_strings( + sample, options.min_length + ) + + # missed strings only includes non-identified strings in searched range + # here currently only focus on strings in string blob range + base_statics = results.strings.static_strings if layout_doc is not None else static_strings + string_blob_strings = floss.language.go.extract.get_static_strings_from_blob_range(sample, base_statics) + results.strings.language_strings_missed = floss.language.utils.get_missed_strings( + string_blob_strings, results.strings.language_strings, options.min_length + ) + if layout_offset_index is not None: + results.strings.language_strings = enrich_static_strings( + results.strings.language_strings, offset_index=layout_offset_index + ) + results.strings.language_strings_missed = enrich_static_strings( + results.strings.language_strings_missed, offset_index=layout_offset_index + ) + + elif results.metadata.language == Language.RUST.value: + logger.info("extracting language-specific Rust strings") + with results.metadata.runtime.measure_and_set_time("language_strings"): + results.strings.language_strings = floss.language.rust.extract.extract_rust_strings( + sample, options.min_length + ) + + # currently Rust strings are only extracted from the .rdata section + base_statics = results.strings.static_strings if layout_doc is not None else static_strings + rdata_strings = floss.language.rust.extract.get_static_strings_from_rdata(sample, base_statics) + results.strings.language_strings_missed = floss.language.utils.get_missed_strings( + rdata_strings, results.strings.language_strings, options.min_length + ) + if layout_offset_index is not None: + results.strings.language_strings = enrich_static_strings( + results.strings.language_strings, offset_index=layout_offset_index + ) + results.strings.language_strings_missed = enrich_static_strings( + results.strings.language_strings_missed, offset_index=layout_offset_index + ) + + if ( + results.analysis.enable_decoded_strings + or results.analysis.enable_stack_strings + or results.analysis.enable_tight_strings + ): + if sample_size > MAX_FILE_SIZE: + if not options.large_file: + raise PipelineError( + "cannot deobfuscate strings from files larger than 0x%x bytes" % MAX_FILE_SIZE, + exit_code=-1, + ) + else: + logger.warning( + "a large file was provided with a size of %i bytes, this may take much more time and system resource to process", + sample_size, + ) + + if options.signatures is None: + raise PipelineError("signatures path required for deobfuscation", exit_code=-1) + + sigpaths = get_signatures(options.signatures) + + should_save_workspace = os.environ.get("FLOSS_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) + try: + with halo.Halo( + text="analyzing program", + spinner="simpleDots", + stream=sys.stderr, + enabled=not options.quiet, + ): + with results.metadata.runtime.measure_and_set_time("vivisect"): + vw = load_vw(sample, options.format, sigpaths, should_save_workspace) + except WorkspaceLoadError as e: + raise PipelineError("failed to analyze sample: %s" % e, exit_code=-1) + + results.metadata.imagebase = get_imagebase(vw) + + with results.metadata.runtime.measure_and_set_time("find_features"): + try: + selected_functions = select_functions(vw, options.analyze_functions) + results.analysis.functions.discovered = len(vw.getFunctions()) + except ValueError as e: + # failed to find functions in workspace + raise PipelineError(e.args[0], exit_code=-1) + + decoding_function_features, library_functions = find_decoding_function_features( + vw, selected_functions, disable_progress=options.quiet + ) + results.analysis.functions.library = len(library_functions) + + logger.trace("analysis summary:") + for k, v in get_vivisect_meta_info(vw, selected_functions, decoding_function_features).items(): + logger.trace(" %s: %s", k, v or "N/A") + + if results.analysis.enable_stack_strings: + with results.metadata.runtime.measure_and_set_time("stack_strings"): + funcs = selected_functions + if results.analysis.enable_tight_strings: + # don't run stack-string extraction on functions with tight loops as this will likely + # result in FPs and should be caught by the tightstrings extraction below + funcs = get_functions_without_tightloops(decoding_function_features) + + results.strings.stack_strings = extract_stackstrings( + vw, + funcs, + options.min_length, + verbosity=options.verbose, + disable_progress=options.quiet, + ) + results.analysis.functions.analyzed_stack_strings = len(funcs) + + if results.analysis.enable_tight_strings: + with results.metadata.runtime.measure_and_set_time("tight_strings"): + tightloop_functions = get_functions_with_tightloops(decoding_function_features) + results.strings.tight_strings = extract_tightstrings( + vw, + tightloop_functions, + min_length=options.min_length, + verbosity=options.verbose, + disable_progress=options.quiet, + ) + results.analysis.functions.analyzed_tight_strings = len(tightloop_functions) + + if results.analysis.enable_decoded_strings: + with results.metadata.runtime.measure_and_set_time("decoded_strings"): + # TODO select more based on score rather than absolute count?! + top_functions = get_top_functions(decoding_function_features, 20) + + fvas_to_emulate = get_function_fvas(top_functions) + fvas_tight_functions = get_tight_function_fvas(decoding_function_features) + fvas_to_emulate = append_unique(fvas_to_emulate, fvas_tight_functions) + + if len(fvas_to_emulate) == 0: + logger.info("no candidate decoding functions found.") + else: + logger.debug("identified %d candidate decoding functions", len(fvas_to_emulate)) + for fva in fvas_to_emulate: + score = decoding_function_features[fva]["score"] + xrefs_to = decoding_function_features[fva]["xrefs_to"] + results.analysis.functions.decoding_function_scores[fva] = { + "score": score, + "xrefs_to": xrefs_to, + } + logger.debug(" - 0x%x: score: %.3f, xrefs to: %d", fva, score, xrefs_to) + + # TODO filter out strings decoded in library function or function only called by library function(s) + results.strings.decoded_strings = decode_strings( + vw, + fvas_to_emulate, + options.min_length, + verbosity=options.verbose, + disable_progress=options.quiet, + ) + results.analysis.functions.analyzed_decoded_strings = len(fvas_to_emulate) + + results.metadata.runtime.total = get_runtime_diff(time0) + logger.info("finished execution after %.2f seconds", results.metadata.runtime.total) + + if cache_key is not None and options.cache_dir is not None: + logger.debug("storing results in cache: %s", cache_key) + floss.cache.store(options.cache_dir, cache_key, results) + + return results diff --git a/floss/ranges.py b/floss/ranges.py new file mode 100644 index 000000000..d8cc75195 --- /dev/null +++ b/floss/ranges.py @@ -0,0 +1,224 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contiguous offset ranges and byte slices used by layout analysis.""" + +from __future__ import annotations + +import time +import bisect +import logging +import contextlib +from typing import Set, List, Tuple, Iterable, Optional + +from pydantic import Field, BaseModel + +logger = logging.getLogger("floss.ranges") + + +@contextlib.contextmanager +def timing(msg: str): + t0 = time.time() + yield + t1 = time.time() + logger.debug("perf: %s: %0.2fs", msg, t1 - t0) + + +class Range(BaseModel): + "a range of contiguous integer values, such as offsets within a byte sequence" + + offset: int + length: int + + @property + def end(self) -> int: + return self.offset + self.length + + def slice(self, offset, size) -> "Range": + "create a new range thats a sub-range of this one, using relative offsets" + assert 0 <= offset <= self.length + assert 0 <= size + assert offset + size <= self.length + return Range(offset=self.offset + offset, length=size) + + def __iter__(self): + "iterate over the values in this range" + yield from range(self.offset, self.end) + + def __repr__(self): + return f"Range(start: 0x{self.offset:x}, size: 0x{self.length:x}, end: 0x{self.end:x})" + + def __str__(self): + return repr(self) + + +class Slice(BaseModel): + """ + a contiguous range within a sequence of bytes. + notably, it can be further sliced without copying the underlying bytes. + a bit like a memoryview. + """ + + buf: bytes + range: Range + base_offset: int = 0 + + @property + def offset(self) -> int: + return self.range.offset + + @property + def data(self) -> bytes: + "get the bytes in this slice, copying the data out" + return self.buf[self.range.offset - self.base_offset : self.range.end - self.base_offset] + + def slice(self, offset, size) -> "Slice": + "create a new slice thats a sub-slice of this one, using relative offsets" + return Slice(buf=self.buf, range=self.range.slice(offset, size), base_offset=self.base_offset) + + def contains_range(self, offset: int, size: int) -> bool: + """ + checks if this slice contains the given range, + where offset is relative to the start of this slice. + """ + if not (0 <= offset <= self.range.length): + return False + + if size < 0: + return False + + if (offset + size) > self.range.length: + return False + + return True + + @classmethod + def from_bytes(cls, buf: bytes) -> "Slice": + return cls(buf=buf, range=Range(offset=0, length=len(buf))) + + def __repr__(self): + buf_len = len(self.buf) if self.buf is not None else 0 + return f"Slice({repr(self.range)} of bytes of size 0x{buf_len:x})" + + def __str__(self): + return repr(self) + + +class OffsetRanges(BaseModel): + ranges: list[tuple[int, int]] = Field(default_factory=list) + + @classmethod + def from_offsets(cls, offsets: Set[int]) -> "OffsetRanges": + """given a bunch of number, return the contiguous spans (start, end). + + example: + + {1, 2, 3, 5, 6, 9} -> [(1, 3), (5, 6), (9, 9)] + """ + if not offsets: + return cls(ranges=[]) + + if len(offsets) == 1: + v = next(iter(offsets)) + return cls(ranges=[(v, v)]) + + sorted_offsets = list(sorted(offsets)) + ranges: List[Tuple[int, int]] = [] + start = sorted_offsets[0] + end = start + for offset in sorted_offsets[1:]: + if offset == end + 1: + end = offset + else: + ranges.append((start, end)) + start = offset + end = offset + ranges.append((start, end)) + + return cls(ranges=ranges) + + @classmethod + def from_merged_ranges(cls, merged_ranges: List[Tuple[int, int]]) -> "OffsetRanges": + return cls(ranges=merged_ranges) + + def __contains__(self, offset: int) -> bool: + if not self.ranges: + return False + + # Find the index where the offset would be inserted to maintain order. + index = bisect.bisect_left(self.ranges, (offset, 0)) + + # Check the range at the insertion index. + # This handles cases where the offset is the start of a range. + if index < len(self.ranges): + start, end = self.ranges[index] + if start == offset: + return True + + # Check the range just before the insertion index. + # This handles cases where the offset is within or at the end of a range. + if index > 0: + start, end = self.ranges[index - 1] + if start <= offset <= end: + return True + + return False + + def overlaps(self, start: int, end: int) -> bool: + if not self.ranges: + return False + + # Find the index where the start of the given range would be inserted + index = bisect.bisect_right(self.ranges, (start, 0)) + + # Check the range at index-1 for overlap + if index > 0: + prev_start, prev_end = self.ranges[index - 1] + if max(start, prev_start) <= min(end, prev_end): + return True + + # Check the range at index for overlap + if index < len(self.ranges): + next_start, next_end = self.ranges[index] + if max(start, next_start) <= min(end, next_end): + return True + + return False + + +def merge_overlapping_ranges(ranges: List[Tuple[int, int]]) -> List[Tuple[int, int]]: + """ + Merge a list of (start, end) tuples into a list of contiguous ranges. + """ + if not ranges: + return [] + + sorted_ranges = sorted(ranges) + merged_ranges: List[Tuple[int, int]] = [] + for higher in sorted_ranges: + if not merged_ranges: + merged_ranges.append(higher) + else: + lower = merged_ranges[-1] + lower_start, lower_end = lower + higher_start, higher_end = higher + + # test for intersection between lower and higher: + # we know via sorting that lower_start <= higher_start + if higher_start <= lower_end + 1: + upper_bound = max(lower_end, higher_end) + merged_ranges[-1] = (lower_start, upper_bound) + else: + merged_ranges.append(higher) + return merged_ranges diff --git a/floss/render/default.py b/floss/render/default.py index f5b3337a3..c4be7baa7 100644 --- a/floss/render/default.py +++ b/floss/render/default.py @@ -17,9 +17,10 @@ import sys import textwrap import collections -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Tuple, Union, Callable, Optional, Sequence from rich import box +from rich.text import Text from rich.table import Table from rich.markup import escape from rich.console import Console @@ -27,8 +28,20 @@ import floss.utils as util import floss.logging_ import floss.language.identify +from floss.enrich import static_strings_from_layout from floss.render import Verbosity -from floss.results import AddressType, StackString, TightString, DecodedString, ResultDocument, StringEncoding +from floss.results import ( + AddressType, + StackString, + TightString, + ResultLayout, + DecodedString, + ResultDocument, + StringEncoding, +) +from floss.tags.filter import TagRules, hide_strings_by_rules +from floss.render.filter import LayoutFilter +from floss.render.layout import DEFAULT_COLUMNS, render_strings from floss.render.sanitize import sanitize MIN_WIDTH_LEFT_COL = 22 @@ -38,6 +51,14 @@ logger = floss.logging_.getLogger(__name__) +DEFAULT_TAG_RULES: TagRules = { + "#capa": "highlight", + "#common": "mute", + "#duplicate": "mute", + "#code": "hide", + "#reloc": "hide", +} + def heading_style(s: str): colored_string = "[cyan]" + escape(s) + "[/cyan]" @@ -57,9 +78,8 @@ def width(s: str, character_count: int) -> str: return s -def render_meta(results: ResultDocument, console, verbose): - rows: List[Tuple[str, str]] = list() - +def language_value(results: ResultDocument) -> str: + """compose the human-readable identified-language string.""" lang = f"{results.metadata.language}" if results.metadata.language else "" lang_v = ( f" ({results.metadata.language_version})" @@ -67,19 +87,37 @@ def render_meta(results: ResultDocument, console, verbose): else "" ) lang_s = f" - selected: {results.metadata.language_selected}" if results.metadata.language_selected else "" - language_value = f"{lang}{lang_v}{lang_s}" + return f"{lang}{lang_v}{lang_s}" + + +def render_meta(results: ResultDocument, console, verbose): + rows: List[Tuple[str, str]] = list() + + language_value_ = language_value(results) if verbose == Verbosity.DEFAULT: rows.append((width("file path", MIN_WIDTH_LEFT_COL), width(results.metadata.file_path, MIN_WIDTH_RIGHT_COL))) - rows.append(("identified language", language_value)) + if results.metadata.sha256: + rows.append(("sha256", results.metadata.sha256)) + rows.append(("identified language", language_value_)) else: rows.extend( [ (width("file path", MIN_WIDTH_LEFT_COL), width(results.metadata.file_path, MIN_WIDTH_RIGHT_COL)), + ] + ) + if results.metadata.md5: + rows.append(("md5", results.metadata.md5)) + if results.metadata.sha1: + rows.append(("sha1", results.metadata.sha1)) + if results.metadata.sha256: + rows.append(("sha256", results.metadata.sha256)) + rows.extend( + [ ("start date", results.metadata.runtime.start_date.strftime("%Y-%m-%d %H:%M:%S")), ("runtime", strtime(results.metadata.runtime.total)), ("version", results.metadata.version), - ("identified language", language_value), + ("identified language", language_value_), ("imagebase", f"0x{results.metadata.imagebase:x}"), ("min string length", f"{results.metadata.min_length}"), ] @@ -114,7 +152,7 @@ def render_string_type_rows(results: ResultDocument) -> List[Tuple[str, str]]: " language strings", ( f"{len_ls:>{len(str(len_ss))}} ({len_chars_ls:>{len(str(len_chars_ss))}d} characters)" - if results.metadata.language + if results.analysis.enable_language_strings and results.metadata.language else DISABLED ), ), @@ -290,15 +328,30 @@ def render_heading(heading, console, verbose, disable_headers): """ if disable_headers: return - style = "" - if verbose != Verbosity.DEFAULT: - style = "cyan" - table = Table(box=box.HORIZONTALS, style=style, show_header=False) - table.add_row(heading, style=style) - if verbose == Verbosity.DEFAULT: - console.print(table) - else: - console.print(table) + table = Table(box=box.HORIZONTALS, show_header=False) + table.add_row(heading) + console.print(table) + console.print() + + +def render_section_heading(name, console, verbose, disable_headers): + """centered lowercase heading for recovered-string sections, in the same + style as the layout section headings: a horizontal line above and below. + + example:: + + ───────────────────── + stack strings + ───────────────────── + """ + if disable_headers: + return + + line = Text("─" * console.width) + heading = Text(name.center(console.width)) + console.print(line) + console.print(heading) + console.print(line) console.print() @@ -331,25 +384,85 @@ def get_color(color): return color_system -def render(results: floss.results.ResultDocument, verbose, disable_headers, color): +def effective_tag_rules(layout_filter: Optional[LayoutFilter]) -> TagRules: + """tag rules for the layout view. + + When the user expressed tag intent (--tag or --interesting), the default + hide rules (e.g. #code, #reloc) must not drop strings the filter + deliberately kept — the filter already narrowed the set, so re-hiding would + undo it. Without a tag filter the default hide behavior is unchanged. + """ + if layout_filter is not None and (layout_filter.include_tags or layout_filter.interesting): + return {tag: "default" if rule == "hide" else rule for tag, rule in DEFAULT_TAG_RULES.items()} + return DEFAULT_TAG_RULES + + +def render( + results: floss.results.ResultDocument, + verbose, + disable_headers, + color, + columns: Sequence[str] = DEFAULT_COLUMNS, + layout_filter: Optional[LayoutFilter] = None, + plain: bool = False, +): sys.__stdout__.reconfigure(encoding="utf-8") # type: ignore [union-attr] console = Console(file=io.StringIO(), color_system=get_color(color), highlight=False, soft_wrap=True) - if not disable_headers: - console.print("\n") - if verbose == Verbosity.DEFAULT: - console.print(f"FLARE FLOSS RESULTS (version {results.metadata.version})\n") - else: - colored_str = heading_style(f"FLARE FLOSS RESULTS (version {results.metadata.version})\n") - console.print(colored_str) - render_meta(results, console, verbose) - console.print("\n") + if not columns: + columns = DEFAULT_COLUMNS - if results.analysis.enable_static_strings: - render_staticstrings(results.strings.static_strings, console, verbose, disable_headers) - console.print("\n") + if layout_filter is not None and layout_filter.active and results.layout is None: + logger.warning( + "layout-aware filters (--section, --structure, --tag, --query, --max-strings, --interesting) " + "have no layout tree to apply to and are ignored" + ) + + # layout-aware path: no classic meta table (spec 1.2/2.4). the layout tree + # is the static string view, so it is only rendered when static strings are + # enabled; otherwise fall back to the classic metadata view. + if not plain and results.layout is not None and results.analysis.enable_static_strings: + layout = results.layout + if layout_filter is not None and layout_filter.active: + filtered = layout_filter.apply(layout) + if filtered is None: + layout = ResultLayout(name=layout.name, offset=layout.offset, length=layout.length) + else: + layout = filtered + + # when the user expressed tag intent (--tag or --interesting), don't let + # the default hide rules (e.g. #code, #reloc) drop strings the filter + # deliberately kept. tag-aware filtering already narrowed the set. + tag_rules = effective_tag_rules(layout_filter) + + layout_view = hide_strings_by_rules(layout, tag_rules) + render_strings(console, layout_view, tag_rules, columns=columns) + console.print() + else: + if not disable_headers: + console.print("\n") + if verbose == Verbosity.DEFAULT: + console.print(f"FLARE FLOSS RESULTS (version {results.metadata.version})\n") + else: + colored_str = heading_style(f"FLARE FLOSS RESULTS (version {results.metadata.version})\n") + console.print(colored_str) + render_meta(results, console, verbose) + console.print("\n") + + if results.analysis.enable_static_strings: + static_strings = results.strings.static_strings + # --plain is filter-aware: apply the render-time filters to the + # layout tree (when present) and flatten the filtered result + if plain and layout_filter is not None and layout_filter.active and results.layout is not None: + filtered = layout_filter.apply(results.layout) + if filtered is not None: + static_strings = static_strings_from_layout(filtered) + else: + static_strings = [] + render_staticstrings(static_strings, console, verbose, disable_headers) + console.print("\n") - if results.metadata.language in ( + if results.analysis.enable_language_strings and results.metadata.language in ( floss.language.identify.Language.GO.value, floss.language.identify.Language.RUST.value, ): @@ -363,21 +476,26 @@ def render(results: floss.results.ResultDocument, verbose, disable_headers, colo ) console.print("\n") - if results.analysis.enable_stack_strings: - render_heading(f"FLOSS STACK STRINGS ({len(results.strings.stack_strings)})", console, verbose, disable_headers) - render_stackstrings(results.strings.stack_strings, console, verbose, disable_headers) - console.print("\n") - - if results.analysis.enable_tight_strings: - render_heading(f"FLOSS TIGHT STRINGS ({len(results.strings.tight_strings)})", console, verbose, disable_headers) - render_stackstrings(results.strings.tight_strings, console, verbose, disable_headers) + # recovered strings always after static/language (classic blocks). + # show the section whenever the mode is enabled, including count 0. + recovered_sections: Tuple[ + Tuple[str, bool, Union[List[StackString], List[TightString], List[DecodedString]], Callable], ... + ] = ( + ("stack strings", results.analysis.enable_stack_strings, results.strings.stack_strings, render_stackstrings), + ("tight strings", results.analysis.enable_tight_strings, results.strings.tight_strings, render_stackstrings), + ( + "decoded strings", + results.analysis.enable_decoded_strings, + results.strings.decoded_strings, + render_decoded_strings, + ), + ) + for name, enabled, strings, renderer in recovered_sections: + if not enabled: + continue + render_section_heading(name, console, verbose, disable_headers) + renderer(strings, console, verbose, disable_headers) console.print("\n") - if results.analysis.enable_decoded_strings: - render_heading( - f"FLOSS DECODED STRINGS ({len(results.strings.decoded_strings)})", console, verbose, disable_headers - ) - render_decoded_strings(results.strings.decoded_strings, console, verbose, disable_headers) - console.file.seek(0) return console.file.read() diff --git a/floss/render/filter.py b/floss/render/filter.py new file mode 100644 index 000000000..180ba641e --- /dev/null +++ b/floss/render/filter.py @@ -0,0 +1,372 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render-time filters for layout-aware static strings. + +Implements the ``--section``/``--no-section``, ``--structure``/``--no-structure``, +``--tag``/``--no-tag``, ``--interesting``, ``--query``, and ``--max-strings`` +options. Filters operate on the serializable ``ResultLayout`` tree and return a +new pruned tree, so the original result document is never mutated. +""" + +from __future__ import annotations + +import re +from typing import Set, List, Tuple, Optional, Sequence + +from floss.results import ResultLayout, ResultString +from floss.tags.oss import DEFAULT_FILENAMES +from floss.tags.filter import TagRules + +# noisy tags that the --interesting shortcut excludes +NOISY_TAGS: Set[str] = {"#common", "#duplicate", "#code", "#reloc", "#code-junk"} + + +def is_interesting(tags: Sequence[str], tag_rules: TagRules) -> bool: + """ + Determine if a string is considered 'interesting' for human analysis. + Uses Option A (Strict): Drops all strings containing a noisy tag, + UNLESS the string has an active 'highlight' tag (e.g., #capa rules). + """ + if any(tag_rules.get(tag) == "highlight" for tag in tags): + return True + return not any(tag in NOISY_TAGS for tag in tags) + + +# tag families, one per tag-source directory under floss/tags/data. a meta tag +# matches any tag in its family, so --tag winapi / --tag oss / --tag gp etc. +# work as selectors. +TAG_FAMILIES: dict = { + "winapi": {"#winapi"}, + "crt": {"#msvc"}, + "expert": {"#capa"}, + "gp": {"#common", "#code-junk"}, + "oss": {f"#{name.partition('.')[0]}" for name in DEFAULT_FILENAMES}, +} + + +def normalize_tag(tag: str) -> str: + """normalize a user-supplied tag so it can be compared with stored tags. + + strips a leading ``#`` and lowercases, so ``winapi`` and ``#WinAPI`` both + match the stored tag ``#winapi``. + """ + return tag.lstrip("#").lower() + + +# normalized (no leading #, lowercase) family names for lookup +TAG_FAMILIES_NORMALIZED: dict = {name: {normalize_tag(t) for t in tags} for name, tags in TAG_FAMILIES.items()} + + +def normalize_structure(name: str) -> str: + """normalize a structure name so slugs match the stored names. + + ``import-table``, ``import_table``, and ``import table`` all compare equal. + """ + return name.strip().lower().replace("_", "-").replace(" ", "-") + + +# known structure slugs, produced by the layout parsers (floss/layout/*.py) +KNOWN_STRUCTURE_SLUGS = ( + "import-table", + "export-table", + "rich-header", + "section-header", + "elf-header", + "program-header", + "string-table", + "symbol-table", + "macho-header", + "load-command", + "segment-header", +) + + +def tag_matches(user_tag: str, string_tags: Sequence[str]) -> bool: + """true if ``user_tag`` (already normalized) matches any of ``string_tags``. + + a tag family (e.g. ``oss``, ``winapi``, ``gp``) matches any tag in that + family; otherwise the tag is compared directly. + """ + normalized = user_tag + family = TAG_FAMILIES_NORMALIZED.get(normalized) + if family is not None: + return any(normalize_tag(tag) in family for tag in string_tags) + return any(normalize_tag(tag) == normalized for tag in string_tags) + + +def relevance_key(s: ResultString, tag_rules: TagRules) -> Tuple[int, int, int, int]: + """sort key for --max-strings and high-value string ordering. + + relevance order within a section: + 1. strings with a highlighted tag first + 2. then untagged strings + 3. then strings with any non-noisy tag + 4. then the rest (only noisy tags) + within each group, strings up to 256 chars are sorted descending by length, + then strings over 256 chars are sorted ascending by length, + and finally ascending by offset. + """ + has_highlight = any(tag_rules.get(tag) == "highlight" for tag in s.tags) + has_non_noisy = any(tag not in NOISY_TAGS for tag in s.tags) + if has_highlight: + group = 0 + elif not s.tags: + group = 1 + elif has_non_noisy: + group = 2 + else: + group = 3 + + length = len(s.string) + len_tier = 0 if length <= 256 else 1 + len_score = -length if length <= 256 else length + + return (group, len_tier, len_score, s.offset) + + +def is_macho_arch_wrapper(layout: ResultLayout) -> bool: + """true when a node is a Mach-O fat-arch wrapper rather than a section. + + On a fat Mach-O the root's children are arch wrappers (``macho: x86_64``); + the binary segments (``__TEXT``) live one level deeper. Section filters + must descend through this layer so ``--section __TEXT`` works on a + universal binary. + """ + return layout.name.startswith("macho:") + + +def is_section_child(parent: ResultLayout, child: ResultLayout, depth: int) -> bool: + """true when ``child`` is a binary section of ``parent``. + + a child becomes a section when it is not a format wrapper (e.g. a Mach-O + fat-arch layer) and its parent is the root or an arch wrapper; otherwise it + inherits its containing section. + """ + return not is_macho_arch_wrapper(child) and (depth == 0 or is_macho_arch_wrapper(parent)) + + +class LayoutFilter: + """Build and apply render-time filters to a ``ResultLayout`` tree. + + Attributes: + include_sections: keep strings whose containing layout node name is in + this list. Empty list means no section include filter. + exclude_sections: drop strings whose containing layout node name is in + this list. Empty list means no section exclude filter. + include_structures: keep strings whose structure field is in this list. + exclude_structures: drop strings whose structure field is in this list. + include_tags: keep strings with any matching tag. + exclude_tags: drop strings with any matching tag. + interesting: drop any string carrying a noisy tag, even when it also has + a non-noisy tag (e.g. #winapi #common is dropped). + queries: regex patterns ORed against string content. + max_strings: cap emitted strings per top-level section to the top N by + relevance. + tag_rules: tag rules used to compute the relevance order. + """ + + def __init__( + self, + *, + include_sections: Optional[Sequence[str]] = None, + exclude_sections: Optional[Sequence[str]] = None, + include_structures: Optional[Sequence[str]] = None, + exclude_structures: Optional[Sequence[str]] = None, + include_tags: Optional[Sequence[str]] = None, + exclude_tags: Optional[Sequence[str]] = None, + interesting: bool = False, + queries: Optional[Sequence[str]] = None, + max_strings: Optional[int] = None, + tag_rules: Optional[TagRules] = None, + ): + self.include_sections = set(include_sections or []) + self.exclude_sections = set(exclude_sections or []) + self.include_structures = {normalize_structure(name) for name in (include_structures or [])} + self.exclude_structures = {normalize_structure(name) for name in (exclude_structures or [])} + self.include_tags = [normalize_tag(tag) for tag in (include_tags or [])] + self.exclude_tags = [normalize_tag(tag) for tag in (exclude_tags or [])] + self.interesting = interesting + self.queries = [re.compile(q) for q in (queries or [])] + self.max_strings = max_strings + self.tag_rules = tag_rules or {} + + @property + def active(self) -> bool: + return bool( + self.include_sections + or self.exclude_sections + or self.include_structures + or self.exclude_structures + or self.include_tags + or self.exclude_tags + or self.interesting + or self.queries + or self.max_strings is not None + ) + + def string_matches(self, section: str, s: ResultString) -> bool: + if self.include_sections and section not in self.include_sections: + return False + if self.exclude_sections and section in self.exclude_sections: + return False + + structure = normalize_structure(s.structure) + if self.include_structures and structure not in self.include_structures: + return False + if self.exclude_structures and structure in self.exclude_structures: + return False + + if self.include_tags and not any(tag_matches(t, s.tags) for t in self.include_tags): + return False + if self.exclude_tags and any(tag_matches(t, s.tags) for t in self.exclude_tags): + return False + if self.interesting and not is_interesting(s.tags, self.tag_rules): + return False + + if self.queries and not any(pattern.search(s.string) for pattern in self.queries): + return False + + return True + + @staticmethod + def relevance_key(s: ResultString, tag_rules: TagRules) -> Tuple[int, int, int, int]: + """sort key for --max-strings, see the module-level relevance_key.""" + return relevance_key(s, tag_rules) + + def apply_node(self, layout: ResultLayout, section: str, depth: int) -> Optional[ResultLayout]: + """filter one layout node, recursing into children. + + The ``section`` name is threaded down the tree: the root node is the + file itself (its own strings carry the root name), each child of the + root is a binary section, and deeper nodes inherit their containing + section. This way ``--section .rdata`` also matches strings that live + in nested structure nodes under ``.rdata``. + + returns None when the node has no matching strings and no matching + children, so empty branches are pruned but headers that still contain + matches are kept. + """ + if depth == 0: + section = layout.name + + strings = [s for s in layout.strings if self.string_matches(section, s)] + + children: List[ResultLayout] = [] + for child in layout.children: + if is_section_child(layout, child, depth): + child_section = child.name + else: + child_section = section + filtered = self.apply_node(child, child_section, depth + 1) + if filtered is not None: + children.append(filtered) + + if not strings and not children: + return None + + return ResultLayout( + name=layout.name, + offset=layout.offset, + length=layout.length, + strings=strings, + children=children, + ) + + def collect_strings(self, layout: ResultLayout) -> List[ResultString]: + """flatten all strings in a layout subtree, in render order.""" + strings = list(layout.strings) + for child in layout.children: + strings.extend(self.collect_strings(child)) + return strings + + def prune(self, layout: ResultLayout, keep: set) -> Optional[ResultLayout]: + """rebuild a layout subtree keeping only strings in ``keep`` (by id). + + kept strings within a node are emitted in relevance order. + """ + strings = [s for s in layout.strings if id(s) in keep] + strings.sort(key=lambda s: self.relevance_key(s, self.tag_rules)) + children: List[ResultLayout] = [] + for child in layout.children: + pruned = self.prune(child, keep) + if pruned is not None: + children.append(pruned) + if not strings and not children: + return None + return ResultLayout( + name=layout.name, + offset=layout.offset, + length=layout.length, + strings=strings, + children=children, + ) + + def cap_node(self, layout: ResultLayout) -> Optional[ResultLayout]: + """cap a top-level section subtree to the top N strings by relevance.""" + all_strings = self.collect_strings(layout) + if self.max_strings is None or len(all_strings) <= self.max_strings: + return layout + ranked = sorted(all_strings, key=lambda s: self.relevance_key(s, self.tag_rules)) + keep = {id(s) for s in ranked[: self.max_strings]} + return self.prune(layout, keep) + + def apply(self, layout: ResultLayout) -> Optional[ResultLayout]: + """return a filtered copy of ``layout``, or None when nothing matches.""" + filtered = self.apply_node(layout, "", 0) + if filtered is None: + return None + + if self.max_strings is None: + return filtered + + # cap each top-level section (child of the root) as a unit, plus the + # root's own strings, so a section never emits more than N strings. + # descend through Mach-O fat-arch wrappers so each segment is capped + # independently rather than the whole architecture. + children: List[ResultLayout] = [] + for child in filtered.children: + if is_macho_arch_wrapper(child): + arch_children = [] + for section in child.children: + capped = self.cap_node(section) + if capped is not None: + arch_children.append(capped) + ranked_wrapper = sorted(child.strings, key=lambda s: self.relevance_key(s, self.tag_rules)) + children.append( + ResultLayout( + name=child.name, + offset=child.offset, + length=child.length, + strings=ranked_wrapper[: self.max_strings], + children=arch_children, + ) + ) + else: + capped = self.cap_node(child) + if capped is not None: + children.append(capped) + ranked_root = sorted(filtered.strings, key=lambda s: self.relevance_key(s, self.tag_rules)) + strings = ranked_root[: self.max_strings] + + if not strings and not children: + return None + + return ResultLayout( + name=filtered.name, + offset=filtered.offset, + length=filtered.length, + strings=strings, + children=children, + ) diff --git a/floss/render/json.py b/floss/render/json.py index a6474dfdf..ecc620055 100644 --- a/floss/render/json.py +++ b/floss/render/json.py @@ -32,13 +32,43 @@ def default(self, o): if dataclasses.is_dataclass(o): return dataclasses.asdict(o) # type: ignore [arg-type] if isinstance(o, datetime.datetime): + if o.tzinfo is not None: + o = o.astimezone(datetime.timezone.utc) + return o.isoformat("T").replace("+00:00", "Z") return o.isoformat("T") + "Z" return super().default(o) +def sort_nested(obj): + """recursively sort dict keys, preserving the given top-level ordering. + + the top level of a results document is kept in a fixed field order so the + small ``metadata`` block always appears near the start of the file; nested + dict keys are still emitted in sorted order. + """ + if isinstance(obj, dict): + return {key: sort_nested(value) for key, value in sorted(obj.items())} + if isinstance(obj, list): + return [sort_nested(value) for value in obj] + return obj + + +# top-level key order of a results document. +# metadata is deliberately first: it is small, so a results document is always +# recognizable from its leading bytes by detect_file_type. +TOP_LEVEL_KEYS = ("metadata", "analysis", "strings", "layout") + + +def _render_dict(data: dict) -> str: + """serialize a results-document dict with fixed top-level ordering.""" + # keep the fixed top-level ordering (metadata first so the document stays + # detectable from its leading bytes), then append any future/extra keys so + # they are never silently dropped + extra_keys = [key for key in data if key not in TOP_LEVEL_KEYS] + ordered_keys = list(TOP_LEVEL_KEYS) + extra_keys + top = {key: sort_nested(data[key]) for key in ordered_keys if key in data} + return json.dumps(top, cls=FlossJSONEncoder, sort_keys=False) + + def render(doc: ResultDocument) -> str: - return json.dumps( - doc, - cls=FlossJSONEncoder, - sort_keys=True, - ) + return _render_dict(dataclasses.asdict(doc)) diff --git a/floss/render/layout.py b/floss/render/layout.py new file mode 100644 index 000000000..3d4e0a2d4 --- /dev/null +++ b/floss/render/layout.py @@ -0,0 +1,417 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rich text rendering of layout-aware string results. + +This module renders the serializable ``ResultLayout`` tree produced by the +layout-aware static analysis, including tags, offsets, structures, and the +tree headers/footers. +""" + +from __future__ import annotations + +import json +from typing import Optional, Sequence + +from rich.text import Text +from rich.style import Style +from rich.console import Console + +from floss.results import ResultLayout, ResultString +from floss.tags.filter import TagRules + +# columns available in the layout view; controlled by --columns +COLUMN_CHOICES = ("tags", "offset", "structure", "encoding") +DEFAULT_COLUMNS = ("tags", "offset") + +MUTED_STYLE = Style(color="gray50") +DEFAULT_STYLE = Style() +HIGHLIGHT_STYLE = Style(color="yellow") + +PADDING_WIDTH = 2 +STRUCTURE_WIDTH = 20 + + +def make_span(text: str, style: Style = DEFAULT_STYLE) -> Text: + """convenience function for single-line, styled text region""" + return Text(text, style=style, no_wrap=True, overflow="ellipsis", end="") + + +def render_string_padding(): + return make_span(" " * PADDING_WIDTH) + + +def compute_string_style(s: ResultString, tag_rules: TagRules) -> Optional[Style]: + """compute the style for a string based on its tags + + returns: Style, or None if the string should be hidden. + """ + styles = set(tag_rules.get(tag, "mute") for tag in s.tags) + + # precedence: + # + # 1. highlight + # 2. hide + # 3. mute + # 4. default + if "highlight" in styles: + return HIGHLIGHT_STYLE + elif "hide" in styles: + return None + elif "mute" in styles: + return MUTED_STYLE + else: + return DEFAULT_STYLE + + +def render_string_string(s: ResultString, tag_rules: TagRules) -> Text: + string_style = compute_string_style(s, tag_rules) + if string_style is None: + raise ValueError("string should be hidden") + + # render like json, but strip the leading/trailing quote marks. + # this means that whitespace characters like \t, \n, and \r are rendered as + # literal escape sequences, which keeps the rendered string on a single line + # and matches the escaping done by sanitize() in the classic views. + rendered_string = json.dumps(s.string)[1:-1] + return make_span(rendered_string, style=string_style) + + +def get_visible_tags(s: ResultString) -> tuple: + """compute the tuple of visible tag names for a string, in sorted order. + + this applies the same filtering as render_string_tags + (e.g. removing #common when there are other tags). + the result can be compared across strings to detect tag groups. + """ + tags = list(s.tags) + if len(tags) != 1 and "#common" in tags: + tags.remove("#common") + return tuple(sorted(tags)) + + +def render_string_tags(s: ResultString, tag_rules: TagRules, is_group_start: bool = False): + ret = Text() + + # don't show #common if there are other tags, + # because the other tags will be more specific (like library names). + tags = list(get_visible_tags(s)) + + for i, tag in enumerate(tags): + tag_style = DEFAULT_STYLE + rule = tag_rules.get(tag, "mute") + if rule == "highlight": + tag_style = HIGHLIGHT_STYLE + elif rule == "mute": + tag_style = MUTED_STYLE + elif rule == "default": + tag_style = DEFAULT_STYLE + else: + raise ValueError(f"unknown tag rule: {rule}") + + ret.append_text(make_span(tag, style=tag_style)) + if i < len(tags) - 1: + ret.append_text(make_span(" ")) + + if is_group_start: + ret.append_text(make_span(" ┓", style=MUTED_STYLE)) + else: + # reserve same width as " ┓" so tags stay aligned + ret.append_text(make_span(" ")) + + return ret + + +def render_string_tags_continuation(tags_width: int, is_group_end: bool = False) -> Text: + """render a continuation indicator instead of repeating tag text. + + the character is right-aligned in the given width to line up with the ┓. + on the last line of a group, render ┛ as a terminator. + """ + if tags_width == 0: + return make_span("") + if is_group_end: + left_pad = tags_width - 1 + bar = make_span(" " * left_pad + "┛", style=MUTED_STYLE) + else: + left_pad = tags_width - 1 + bar = make_span(" " * left_pad + "┃", style=MUTED_STYLE) + return bar + + +def render_string_offset(s: ResultString): + # render the 000 prefix of the 8-digit offset in muted gray + # and the non-zero suffix as blue. + offset_chars = f"{s.offset:08x}" + unpadded = offset_chars.lstrip("0") + padding_width = len(offset_chars) - len(unpadded) + + offset = make_span("") + offset.append_text(make_span("0" * padding_width, style=MUTED_STYLE)) + offset.append_text(make_span(unpadded, style=DEFAULT_STYLE)) + + return offset + + +def render_string_structure(s: ResultString): + ret = Text() + + if s.structure: + structure = make_span(s.structure, style=Style(color="blue")) + structure.align("left", STRUCTURE_WIDTH - 1) + ret.append(make_span("/", style=MUTED_STYLE)) + ret.append(structure) + else: + ret.append_text(make_span(" " * STRUCTURE_WIDTH)) + + return ret + + +def render_string( + line_width: int, + s: ResultString, + tag_rules: TagRules, + columns: Sequence[str] = DEFAULT_COLUMNS, + prev_tags: Optional[tuple] = None, + prev_tags_width: int = 0, + is_group_end: bool = False, + is_group_start: bool = False, +) -> Text: + # + # | stringstringstring #tag #tag #tag 00000001 | + # | stringstring #tag 0000004A | + # | string │ 00000050 | + # | stringstringstringstringstringst... #tag #tag 0000005E | + # ^ ^ ^ ^ ^ + # | | | | offset + # | | | padding + # | | tags (or │ continuation) + # | padding + # string + # + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^ + # left column right column + # + # fields are basically laid out from right to left, + # which means that the metadata may cause a string to be clipped. + # + # field sizes: + # structure: 8 + # padding: 2 + # offset: 8 + # padding: 2 + # tags: variable, or 0 + # padding: 2 + # string: variable + + left = render_string_string(s, tag_rules) + + visible_tags = get_visible_tags(s) + use_continuation = ( + "tags" in columns and prev_tags is not None and visible_tags == prev_tags and len(visible_tags) > 0 + ) + + right = make_span("") + if "tags" in columns: + right.append_text(render_string_padding()) + if use_continuation: + right.append_text(render_string_tags_continuation(prev_tags_width, is_group_end=is_group_end)) + else: + right.append_text(render_string_tags(s, tag_rules, is_group_start=is_group_start)) + if "offset" in columns: + right.append_text(render_string_padding()) + right.append_text(render_string_offset(s)) + if "encoding" in columns: + right.append_text(render_string_padding()) + # indicate encoding: ascii is the implicit default + right.append_text(make_span("U " if s.encoding == "unicode" else " ")) + if "structure" in columns: + right.append_text(render_string_structure(s)) + + # this alignment clips the string if it's too long, + # leaving an ellipsis at the end when it would collide with a tag/offset. + # this is bad for showing all data verbatim, + # but is good for the common case of triage analysis. + left.align("left", line_width - len(right)) + + line = Text() + line.append_text(left) + line.append_text(right) + + return line + + +def is_visible(layout: ResultLayout) -> bool: + "a layout is visible if it has any strings (or its children do)" + return bool(layout.strings) or any(map(is_visible, layout.children)) + + +def has_visible_predecessors(parent: ResultLayout | None, child_index: int | None) -> bool: + if parent is None or child_index is None: + # root node + return False + + for i in range(child_index): + if is_visible(parent.children[i]): + return True + return False + + +def has_visible_successors(parent: ResultLayout | None, child_index: int | None) -> bool: + if parent is None or child_index is None: + # root node + return False + + for i in range(child_index + 1, len(parent.children)): + if is_visible(parent.children[i]): + return True + return False + + +def render_strings( + console: Console, + layout: ResultLayout, + tag_rules: TagRules, + depth: int = 0, + name_hint: Optional[str] = None, + parent: Optional[ResultLayout] = None, + child_index: Optional[int] = None, + columns: Sequence[str] = DEFAULT_COLUMNS, +): + if not is_visible(layout): + return + + if ( + len(layout.children) == 1 + and not layout.strings + and layout.offset == layout.children[0].offset + and layout.length == layout.children[0].length + ): + # when a layout is completely dominated by its single child + # then we can directly render the child, + # retaining just a hint of the parent's name. + # + # for example: + # + # rsrc: BINARY/102/0 (pe) + return render_strings( + console, + layout.children[0], + tag_rules, + depth, + name_hint=layout.name, + parent=parent, + child_index=child_index, + columns=columns, + ) + + name = layout.name + if name_hint: + name = f"{name_hint} ({name})" + + header = make_span(name, style=MUTED_STYLE) + header.pad(1) + header.align("center", width=console.width, character="─") + + # box is muted color + # name of section is blue + name_offset = header.plain.index(" ") + 1 + header.stylize(Style(color="blue"), name_offset, name_offset + len(name)) + + if not has_visible_predecessors(parent, child_index): + header_shape = "┐" + else: + header_shape = "┤" + + header.remove_suffix("─" * (depth + 1)) + header.append_text(make_span(header_shape, style=MUTED_STYLE)) + header.append_text(make_span("│" * depth, style=MUTED_STYLE)) + + console.print(header) + + def render_string_lines(console: Console, tag_rules: TagRules, strings: list, depth: int): + """render a batch of strings, grouping consecutive strings with the same tags.""" + visible_tags_by_index = [get_visible_tags(string) for string in strings] + prev_tags = None + prev_tags_width = 0 + for idx, string in enumerate(strings): + visible_tags = visible_tags_by_index[idx] + next_tags = visible_tags_by_index[idx + 1] if idx + 1 < len(strings) else None + + # lookahead: is this the last line in a continuation group? + is_group_end = False + if prev_tags is not None and visible_tags == prev_tags and len(visible_tags) > 0: + # we are in a continuation — check if the next string breaks the group + if next_tags is None or next_tags != visible_tags: + is_group_end = True + + # lookahead: is this the first line of a continuation group? + is_group_start = False + if (prev_tags is None or visible_tags != prev_tags) and len(visible_tags) > 0: + if next_tags is not None and next_tags == visible_tags: + is_group_start = True + + line = render_string( + console.width - (depth + 1), + string, + tag_rules, + columns=columns, + prev_tags=prev_tags, + prev_tags_width=prev_tags_width, + is_group_end=is_group_end, + is_group_start=is_group_start, + ) + line.append_text(make_span("│" * (depth + 1), style=MUTED_STYLE)) + console.print(line) + + # track for next iteration + if visible_tags != prev_tags: + # tags changed — compute the rendered width for continuation bars + prev_tags = visible_tags + prev_tags_width = ( + len(render_string_tags(string, tag_rules, is_group_start=is_group_start)) + if "tags" in columns + else 0 + ) + + if not layout.children: + render_string_lines(console, tag_rules, layout.strings, depth) + + else: + for i, child in enumerate(layout.children): + if i == 0: + # render strings before first child + strings_before_child = list(filter(lambda s: layout.offset <= s.offset < child.offset, layout.strings)) + else: + # render strings between children + last_child = layout.children[i - 1] + strings_before_child = list(filter(lambda s: last_child.end <= s.offset < child.offset, layout.strings)) + + render_string_lines(console, tag_rules, strings_before_child, depth) + + render_strings(console, child, tag_rules, depth + 1, parent=layout, child_index=i, columns=columns) + + # render strings after last child + strings_after_children = list(filter(lambda s: child.end <= s.offset < layout.end, layout.strings)) + render_string_lines(console, tag_rules, strings_after_children, depth) + + if not has_visible_successors(parent, child_index): + footer = make_span("", style=MUTED_STYLE) + footer.align("center", width=console.width, character="─") + + footer.remove_suffix("─" * (depth + 1)) + footer.append_text(make_span("┘", style=MUTED_STYLE)) + footer.append_text(make_span("│" * depth, style=MUTED_STYLE)) + + console.print(footer) diff --git a/floss/render/summary.py b/floss/render/summary.py new file mode 100644 index 000000000..bb81e9737 --- /dev/null +++ b/floss/render/summary.py @@ -0,0 +1,268 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Concise, token-efficient summary of a FLOSS result document. + +The summary is built from the same ``ResultDocument`` as the ``--json`` output, +so it is programmatic and consistent. It reports sample metadata, per-type +string counts, section counts, tag histograms, and the strings that carry +interesting (non-noisy) tags. Intended for human consumers: agents should use +``-j/--json`` for machine-readable output instead of parsing the formatted +tables. +""" + +from __future__ import annotations + +import io +import sys +from typing import Dict, List, Tuple, Sequence +from collections import Counter + +from rich.markup import escape +from rich.console import Console + +from floss.results import ResultLayout, ResultString, ResultDocument +from floss.render.filter import ( + NOISY_TAGS, + relevance_key, + is_interesting, + is_section_child, + is_macho_arch_wrapper, +) +from floss.render.layout import get_visible_tags +from floss.render.default import ( + DEFAULT_TAG_RULES, + get_color, + heading_style, + language_value, +) +from floss.render.sanitize import sanitize + +INTERESTING_MAX_STRINGS = 25 +INTERESTING_MAX_STRINGS_PER_SECTION = 5 + + +def analyze_layout(layout: ResultLayout): + """walk the layout tree once, returning the derived summary values. + + returns a 3-tuple of (section_counts, tag_histogram, interesting_strings). + The section of a string is its containing top-level section: children of + the root (or of a Mach-O fat-arch wrapper) are the sections, and deeper + nodes inherit their section (so strings under a nested ``import table`` + node are counted under ``.rdata``). + """ + section_counts: Dict[str, int] = {} + tag_histogram: Counter = Counter() + interesting_map: Dict[Tuple[str, str], Tuple[ResultString, int, set]] = {} + + def walk(node: ResultLayout, section: str, depth: int) -> None: + if section not in section_counts: + section_counts[section] = 0 + section_counts[section] += len(node.strings) + for s in node.strings: + tag_histogram.update(s.tags) + if is_interesting(s.tags, DEFAULT_TAG_RULES): + key = (s.string, section) + if key not in interesting_map: + interesting_map[key] = (s, 1, set(s.tags)) + else: + existing_s, count, tags = interesting_map[key] + tags.update(s.tags) + best_s = s if s.offset < existing_s.offset else existing_s + interesting_map[key] = (best_s, count + 1, tags) + for child in node.children: + if is_section_child(node, child, depth): + child_section = child.name + elif is_macho_arch_wrapper(child): + child_section = child.name + else: + child_section = section + walk(child, child_section, depth + 1) + + # walk the whole tree from the root: root-attached strings count under the + # root name, children of the root (or of an arch wrapper) are sections + walk(layout, layout.name, 0) + + hist = sorted(tag_histogram.items(), key=lambda kv: (-kv[1], kv[0])) + + interesting = [] + for (string_val, section), (best_s, count, tags) in interesting_map.items(): + s_copy = ResultString( + string=best_s.string, + offset=best_s.offset, + size=best_s.size, + encoding=best_s.encoding, + tags=list(tags), + structure=best_s.structure, + ) + interesting.append((s_copy, count, section)) + + interesting.sort(key=lambda item: relevance_key(item[0], DEFAULT_TAG_RULES)) + return dict(section_counts), hist, interesting + + +def render_summary(results: ResultDocument, color: str = "auto") -> str: + """render a token-efficient summary of ``results`` as text. + + The summary is a human- and agent-consumable view over the same + ``ResultDocument`` that ``--json`` emits. + """ + sys.__stdout__.reconfigure(encoding="utf-8") # type: ignore [union-attr] + console = Console( + file=io.StringIO(), + color_system=get_color(color), + highlight=False, + soft_wrap=True, + ) + + console.print(f"FLOSS SUMMARY (version {results.metadata.version})") + + # 1a. Metadata + meta = results.metadata + meta_pairs = [("file_path", meta.file_path)] + if meta.md5: + meta_pairs.append(("md5", meta.md5)) + if meta.sha256: + meta_pairs.append(("sha256", meta.sha256)) + if meta.language: + meta_pairs.append(("language", language_value(results))) + + meta_pairs.extend( + [ + ("imagebase", f"0x{meta.imagebase:x}"), + ("min_length", f"{meta.min_length}"), + ] + ) + console.print(heading_style("file details")) + for km, vm in meta_pairs: + console.print(f"{km:<15} {vm}") + console.print() + + # 1b. Counts + strings = results.strings + a = results.analysis + count_pairs = [ + ("static", len(strings.static_strings) if a.enable_static_strings else 0), + ("language", len(strings.language_strings) if a.enable_language_strings else 0), + ("stack", len(strings.stack_strings) if a.enable_stack_strings else 0), + ("tight", len(strings.tight_strings) if a.enable_tight_strings else 0), + ("decoded", len(strings.decoded_strings) if a.enable_decoded_strings else 0), + ] + console.print(heading_style("extracted strings")) + for kc, vc in count_pairs: + console.print(f"{kc:<15} {vc}") + console.print() + + if results.layout is not None and results.analysis.enable_static_strings: + section_counts, tag_hist, interesting = analyze_layout(results.layout) + + if tag_hist: + console.print(heading_style("strings by tag")) + for kt, vt in tag_hist: + console.print(f"{kt:<15} {vt}") + console.print() + + # 1c. High Value Strings (Integrated Physical Map) + if interesting or section_counts: + console.print(heading_style("preview")) + + # Map interesting strings by section + sec_to_interesting: Dict[str, List[Tuple[ResultString, int]]] = {} + for s, c_val, sec in interesting: + if sec not in sec_to_interesting: + sec_to_interesting[sec] = [] + sec_to_interesting[sec].append((s, c_val)) + + # Pick strings reflecting per-section cap and global cap + picked_strings_by_sec: Dict[str, List[Tuple[ResultString, int]]] = {} + total_picked = 0 + for s, c_val, sec in interesting: + if sec not in picked_strings_by_sec: + picked_strings_by_sec[sec] = [] + if len(picked_strings_by_sec[sec]) < INTERESTING_MAX_STRINGS_PER_SECTION: + if total_picked < INTERESTING_MAX_STRINGS: + picked_strings_by_sec[sec].append((s, c_val)) + total_picked += 1 + if total_picked >= INTERESTING_MAX_STRINGS: + break + + # Sort within sections by physical offset + for sec in picked_strings_by_sec: + picked_strings_by_sec[sec].sort(key=lambda item: item[0].offset) + + is_first_section = True + for section, total_count in section_counts.items(): + if section == results.layout.name and total_count == 0: + continue + + if not is_first_section: + console.print() + is_first_section = False + + all_interesting_in_sec = sec_to_interesting.get(section, []) + num_interesting = len(all_interesting_in_sec) + + # Format section header without dashes + picked_count = len(picked_strings_by_sec.get(section, [])) + + if num_interesting > 0: + if picked_count < num_interesting: + stats = f"showing {picked_count} of {num_interesting} interesting strings | {total_count} total" + else: + stats = f"{num_interesting} interesting strings | {total_count} total" + else: + stats = f"{total_count} total strings" + + if section == "macho (fat)": + hdr = rf"[cyan bold]fat wrapper[/cyan bold] [dim]({stats})[/dim]" + else: + hdr = rf"[cyan bold]\[{escape(section)}][/cyan bold] [dim]({stats})[/dim]" + + # Render header + console.print(hdr) + + # Render picked interesting strings + picked = picked_strings_by_sec.get(section, []) + for s, count in picked: + from rich.text import Text + + raw_string = sanitize(s.string) + count_str = f" (count: {count})" if count > 1 else "" + + string_text = Text(raw_string) + if count > 1: + string_text.append(count_str, style="dim") + string_text.truncate(80, overflow="ellipsis", pad=True) + + visible_tags = get_visible_tags(s) + tags = f"[{', '.join(visible_tags)}]" if visible_tags else "" + + tags_text = Text(tags) + tags_text.truncate(25, overflow="ellipsis", pad=True) + + offset_text = Text(f"0x{s.offset:x}", style="dim") + + line = Text() + line.append_text(string_text) + line.append(" ") + line.append_text(tags_text) + line.append(" ") + line.append_text(offset_text) + + console.print(line) + + console.print() + + console.file.seek(0) + return console.file.read() diff --git a/floss/results.py b/floss/results.py index 4630aa739..683226b33 100644 --- a/floss/results.py +++ b/floss/results.py @@ -15,9 +15,11 @@ import re import json +import time import datetime +import contextlib from enum import Enum -from typing import Dict, List +from typing import TYPE_CHECKING, Dict, List, Iterator, Optional from pathlib import Path from dataclasses import field @@ -37,6 +39,9 @@ from floss.version import __version__ from floss.render.sanitize import sanitize +if TYPE_CHECKING: + from floss.layout.base import Layout + logger = floss.logging_.getLogger(__name__) @@ -58,10 +63,10 @@ class StringEncoding(str, Enum): class StackString: """ here's what the following members represent: - - + + [smaller addresses] - + +---------------+ <- stack_pointer (top of stack) | | \ +---------------+ | offset @@ -75,7 +80,7 @@ class StackString: +---------------+ | | | / +---------------+ <- original_stack_pointer (bottom of stack, probably bp) - + [bigger addresses] @@ -141,11 +146,17 @@ class StaticString: string: the string offset: the offset into the input where the string is found encoding: the string encoding, like ASCII or unicode + tags: classification tags (layout/content), when enriched + section: containing layout node name, when known + structure: PE/ELF/Mach-O structure name, when known """ string: str offset: int encoding: StringEncoding + tags: List[str] = field(default_factory=list) + section: str = "" + structure: str = "" @classmethod def from_utf8(cls, buf, addr, min_length): @@ -162,17 +173,98 @@ def from_utf8(cls, buf, addr, min_length): return cls(string=decoded_string, offset=addr, encoding=StringEncoding.UTF8) +@dataclass +class ResultString: + """Serializable layout-tree string (used for section-aware static render).""" + + string: str + offset: int + size: int + encoding: str + tags: List[str] = field(default_factory=list) + structure: str = "" + + +@dataclass +class ResultLayout: + """Serializable binary layout tree for static string context.""" + + name: str + offset: int + length: int + strings: List[ResultString] = field(default_factory=list) + children: List["ResultLayout"] = field(default_factory=list) + + @property + def end(self) -> int: + return self.offset + self.length + + @classmethod + def from_layout(cls, layout: "Layout") -> "ResultLayout": + """Recursively convert a layout tree to the serializable form.""" + from floss.layout.types import TaggedString, ExtractedString + + result_strings: List[ResultString] = [] + for s in layout.strings: + # after tagging, strings are TaggedString; before, ExtractedString + if isinstance(s, TaggedString): + extracted: ExtractedString = s.string + tags = sorted(list(s.tags)) + structure = s.structure or "" + else: + assert isinstance(s, ExtractedString) + extracted = s + tags = [] + structure = "" + + result_strings.append( + ResultString( + string=extracted.string, + offset=extracted.slice.range.offset, + size=extracted.slice.range.length, + encoding=extracted.encoding, + tags=tags, + structure=structure, + ) + ) + + result_children = [cls.from_layout(child) for child in (layout.children or [])] + + return cls( + name=layout.name, + offset=layout.slice.range.offset, + length=layout.slice.range.length, + strings=result_strings, + children=result_children, + ) + + @dataclass class Runtime: - start_date: datetime.datetime = datetime.datetime.now() - total: float = 0 - vivisect: float = 0 - find_features: float = 0 - static_strings: float = 0 - language_strings: float = 0 - stack_strings: float = 0 - decoded_strings: float = 0 - tight_strings: float = 0 + start_date: datetime.datetime = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc)) + total: float = 0.0 + vivisect: float = 0.0 + find_features: float = 0.0 + static_strings: float = 0.0 + layout: float = 0.0 + tags: float = 0.0 + language_strings: float = 0.0 + stack_strings: float = 0.0 + decoded_strings: float = 0.0 + tight_strings: float = 0.0 + + @contextlib.contextmanager + def measure_and_set_time(self, field: str) -> Iterator[None]: + """ + Record the elapsed time of the wrapped block into the given runtime field. + """ + if not hasattr(self, field): + raise AttributeError(f"Runtime has no field {field!r}") + t0 = time.time() + try: + yield + finally: + setattr(self, field, round(time.time() - t0, 4)) @dataclass @@ -191,15 +283,28 @@ class Analysis: enable_stack_strings: bool = True enable_tight_strings: bool = True enable_decoded_strings: bool = True + enable_language_strings: bool = True + enable_layout: bool = True + enable_tags: bool = True functions: Functions = field(default_factory=Functions) -STRING_TYPE_FIELDS = set([field for field in Analysis.__annotations__ if field.startswith("enable_")]) +# string-type enable flags only (layout/tags are separate product toggles) +STRING_TYPE_FIELDS = { + "enable_static_strings", + "enable_stack_strings", + "enable_tight_strings", + "enable_decoded_strings", + "enable_language_strings", +} @dataclass class Metadata: file_path: str + md5: str = "" + sha1: str = "" + sha256: str = "" version: str = __version__ imagebase: int = 0 min_length: int = 0 @@ -224,6 +329,7 @@ class ResultDocument: metadata: Metadata analysis: Analysis = field(default_factory=Analysis) strings: Strings = field(default_factory=Strings) + layout: Optional[ResultLayout] = None @classmethod def parse_file(cls, path: Path) -> "ResultDocument": @@ -286,17 +392,30 @@ def read(sample: Path) -> ResultDocument: def check_set_string_types(results: ResultDocument, wanted_analysis: Analysis) -> None: for string_type in STRING_TYPE_FIELDS: if getattr(wanted_analysis, string_type) and not getattr(results.analysis, string_type): - logger.warning(f"{string_type} not in loaded data, use --only/--no to enable/disable type(s)") + logger.warning( + f"{string_type} not in loaded data, use --string-type/--no-string-type to enable/disable type(s)" + ) setattr(results.analysis, string_type, getattr(wanted_analysis, string_type)) def filter_functions(results: ResultDocument, functions: List[int]) -> None: - filtered_scores = dict() + # a function is valid if it appears in any string category; decoding + # functions additionally have a score. don't require a decoding score for + # stack/tight-only functions. + stack_fvas = {f.function for f in results.strings.stack_strings} + tight_fvas = {f.function for f in results.strings.tight_strings} + decoded_fvas = {f.decoding_routine for f in results.strings.decoded_strings} + known_fvas = stack_fvas | tight_fvas | decoded_fvas | set(results.analysis.functions.decoding_function_scores) + for fva in functions: - try: - filtered_scores[fva] = results.analysis.functions.decoding_function_scores[fva] - except KeyError: + if fva not in known_fvas: raise InvalidLoadConfig(f"function 0x{fva:x} not found in loaded data") + + filtered_scores = { + fva: results.analysis.functions.decoding_function_scores[fva] + for fva in functions + if fva in results.analysis.functions.decoding_function_scores + } results.analysis.functions.decoding_function_scores = filtered_scores results.strings.stack_strings = list(filter(lambda f: f.function in functions, results.strings.stack_strings)) @@ -311,9 +430,47 @@ def filter_functions(results: ResultDocument, functions: List[int]) -> None: def filter_string_len(results: ResultDocument, min_length: int) -> None: + """filter strings below min_length. + + Applied when loading a saved results document (and future cache hits): + extraction already respects min_length, but a document loaded with a + different -n may contain shorter strings, so re-filter here. + + Strings shorter than the stored extraction min_length were dropped at + extraction time and cannot be recovered, so abort when the requested + threshold is lower than what the document was built with. + """ + stored_min_length = results.metadata.min_length + if min_length < stored_min_length: + raise InvalidLoadConfig( + "requested --minimum-length %d is below the %d used to build this " + "document, so strings between %d and %d were already dropped and " + "cannot be recovered" % (min_length, stored_min_length, min_length, stored_min_length) + ) results.strings.static_strings = list(filter(lambda s: len(s.string) >= min_length, results.strings.static_strings)) results.strings.stack_strings = list(filter(lambda s: len(s.string) >= min_length, results.strings.stack_strings)) results.strings.tight_strings = list(filter(lambda s: len(s.string) >= min_length, results.strings.tight_strings)) results.strings.decoded_strings = list( filter(lambda s: len(s.string) >= min_length, results.strings.decoded_strings) ) + results.strings.language_strings = list( + filter(lambda s: len(s.string) >= min_length, results.strings.language_strings) + ) + results.strings.language_strings_missed = list( + filter(lambda s: len(s.string) >= min_length, results.strings.language_strings_missed) + ) + if results.layout is not None: + results.layout = _filter_layout_string_len(results.layout, min_length) + + +def _filter_layout_string_len(layout: ResultLayout, min_length: int) -> ResultLayout: + """recursively drop layout-tree strings shorter than min_length.""" + strings = [s for s in layout.strings if len(s.string) >= min_length] + children = [_filter_layout_string_len(child, min_length) for child in layout.children] + return ResultLayout( + name=layout.name, + offset=layout.offset, + length=layout.length, + strings=strings, + children=children, + ) diff --git a/floss/sigs/1_flare_msvc_rtf_32_64.sig b/floss/sigs/1_flare_msvc_rtf_32_64.sig index 0edf8f7d4..7a8c0b54f 100644 Binary files a/floss/sigs/1_flare_msvc_rtf_32_64.sig and b/floss/sigs/1_flare_msvc_rtf_32_64.sig differ diff --git a/floss/sigs/2_flare_msvc_atlmfc_32_64.sig b/floss/sigs/2_flare_msvc_atlmfc_32_64.sig index a00aaafa5..ca1dff5ac 100644 Binary files a/floss/sigs/2_flare_msvc_atlmfc_32_64.sig and b/floss/sigs/2_flare_msvc_atlmfc_32_64.sig differ diff --git a/floss/sigs/3_flare_common_libs.sig b/floss/sigs/3_flare_common_libs.sig index 436b2046a..521d69dc6 100644 Binary files a/floss/sigs/3_flare_common_libs.sig and b/floss/sigs/3_flare_common_libs.sig differ diff --git a/floss/tags/__init__.py b/floss/tags/__init__.py new file mode 100644 index 000000000..7a22ef198 --- /dev/null +++ b/floss/tags/__init__.py @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""String tagging: tag sources, layout-derived checks, and visibility filters. + +Modules like ``expert``, ``gp``, ``oss``, and ``winapi`` are *tag sources* — they +load on-disk classification databases and expose query interfaces. ``engine`` wires +those into ``Tagger`` callables, including layout-derived tags (#code, etc.). +""" + +from __future__ import annotations + +import pathlib + + +def data_root() -> pathlib.Path: + """Shipped tag databases under floss/tags/data.""" + return pathlib.Path(__file__).resolve().parent / "data" + + +# the first line of a Git LFS pointer file; used to detect unpulled databases +LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/" + + +def ensure_not_lfs_pointer(path: pathlib.Path) -> None: + """Raise a clear error when a tag database file is an unpulled Git LFS pointer. + + Without ``git lfs pull`` the LFS-tracked database files are tiny text + pointers, which the loaders otherwise fail on with confusing gzip/msgspec + errors. + """ + try: + with path.open("rb") as f: + head = f.read(len(LFS_POINTER_PREFIX)) + except OSError: + return + if head == LFS_POINTER_PREFIX: + raise ValueError(f"Git LFS pointer detected in {path.name}; please run `git lfs pull`") + + +from floss.tags.engine import ( + Tagger, + load_databases, + query_code_string_database, + query_winapi_name_database, + query_expert_string_database, + query_library_string_database, + query_global_prevalence_database, + query_global_prevalence_hash_database, +) +from floss.tags.filter import ( + TagRules, + should_hide_string, + hide_strings_by_rules, + remove_false_positive_lib_strings, +) + +load_taggers = load_databases diff --git a/floss/tags/data/crt/msvc_v143.jsonl.gz b/floss/tags/data/crt/msvc_v143.jsonl.gz new file mode 100644 index 000000000..0e9154fa4 --- /dev/null +++ b/floss/tags/data/crt/msvc_v143.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0737082a90bf489d393c7ed1b3fe08a08120c5ac204a297895c3c290a21a505a +size 913851 diff --git a/floss/tags/data/expert/capa.jsonl b/floss/tags/data/expert/capa.jsonl new file mode 100644 index 000000000..953c6f8d9 --- /dev/null +++ b/floss/tags/data/expert/capa.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e34ed954107bb4b2c98e8b5d74734bf86c5cd97174daa7d9ff905820879224 +size 302075 diff --git a/floss/tags/data/expert/import_from_capa.py b/floss/tags/data/expert/import_from_capa.py new file mode 100644 index 000000000..b94165143 --- /dev/null +++ b/floss/tags/data/expert/import_from_capa.py @@ -0,0 +1,88 @@ +import sys + +import msgspec +import capa.main +import capa.rules +import capa.engine +import capa.features.file +import capa.features.insn +import capa.features.common +import capa.features.basicblock + +from floss.tags.expert import ExpertRule + + +def walk_rule_logic(rule: capa.rules.Rule, node: capa.engine.Statement | capa.engine.Feature): + match node: + case ( + capa.features.common.Regex(name=type, value=value) + | capa.features.common.Substring(name=type, value=value) + | capa.features.common.String(name=type, value=value) + ): + # mypy doesn't seem to be very good at narrowing types here, + # maybe due to the use of `match` above? + assert type in ("regex", "substring", "string") # type: ignore + assert isinstance(value, str) # type: ignore + + yield ExpertRule( + type=type, # type: ignore + value=value, # type: ignore + tag="#capa", + action="highlight", + note=rule.name[:-33] if rule.is_subscope_rule() else rule.name, + description=rule.meta.get("description", ""), + authors=rule.meta.get("authors", []), + references=rule.meta.get("references", []), + ) + case ( + capa.engine.And(children=[*children]) + | capa.engine.Or(children=[*children]) + | capa.engine.Some(children=[*children]) + ): + # children: List[Statement | Feature] + for child in children: # type: ignore + yield from walk_rule_logic(rule, child) + case capa.engine.Not(child=child) | capa.engine.Range(child=child): + yield from walk_rule_logic(rule, child) + case ( + capa.features.insn.Mnemonic() + | capa.features.insn.Number() + | capa.features.insn.Offset() + | capa.features.insn.OperandNumber() + | capa.features.insn.OperandOffset() + | capa.features.insn.API() + | capa.features.insn.Property() + ): + pass + case ( + capa.features.common.MatchedRule() + | capa.features.common.Arch() + | capa.features.common.OS() + | capa.features.common.Format() + | capa.features.common.Namespace() + | capa.features.common.Class() + | capa.features.common.Characteristic() + | capa.features.common.Bytes() + ): + pass + case ( + capa.features.file.Section() + | capa.features.file.Export() + | capa.features.file.Import() + | capa.features.file.FunctionName() + ): + pass + case capa.features.basicblock.BasicBlock(): + pass + case _: + raise ValueError(f"unknown node type: {node}") + + +def walk_rule(rule: capa.rules.Rule): + yield from walk_rule_logic(rule, rule.statement) + + +rules = capa.main.get_rules([sys.argv[1]]) +for rule in rules.rules.values(): + for er in walk_rule(rule): + print(msgspec.json.encode(er).decode("utf-8")) diff --git a/floss/tags/data/expert/readme.md b/floss/tags/data/expert/readme.md new file mode 100644 index 000000000..f08f230dc --- /dev/null +++ b/floss/tags/data/expert/readme.md @@ -0,0 +1,23 @@ +# Expert String Database + +This directory contains databases of strings manually curated by experts. + +The format of the database is a gzip-compressed JSONL file (one JSON document per line). +Each document looks like: + +```json +{ + "type":"string", + "value":"This program cannot be run in DOS mode.", + "tag":"#capa", + "action":"highlight", + "note":"contain an embedded PE file", + "description":"", + "authors":["moritz.raabe@mandiant.com"], + "references":[] +} +``` + +The expert databases are: + + - `capa.jsonl.gz`: strings extracted from [capa](https://github.com/mandiant/capa) rules using the `import_from_capa.py` script: `$ python import_from_capa.py ~/code/capa/rules/ > capa.jsonl`. \ No newline at end of file diff --git a/floss/tags/data/gp/cwindb-dotnet.jsonl.gz b/floss/tags/data/gp/cwindb-dotnet.jsonl.gz new file mode 100644 index 000000000..d68e3e5e3 --- /dev/null +++ b/floss/tags/data/gp/cwindb-dotnet.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e25b8f0d28606ba9fec0ba12573db042c775b0a0bda06d840475b089b396f75 +size 19592 diff --git a/floss/tags/data/gp/cwindb-native.jsonl.gz b/floss/tags/data/gp/cwindb-native.jsonl.gz new file mode 100644 index 000000000..1f4d892a0 --- /dev/null +++ b/floss/tags/data/gp/cwindb-native.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f858d1c0905096e8aa95158c9d2a3adda9c5ee99875d1ee3627811637a9c214 +size 49958 diff --git a/floss/tags/data/gp/gp-2026-hashes.bin b/floss/tags/data/gp/gp-2026-hashes.bin new file mode 100644 index 000000000..93d1093b1 --- /dev/null +++ b/floss/tags/data/gp/gp-2026-hashes.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72861f9b322dea0b441c8968328e740f80c0fdac1b1146522925a67f826dee4f +size 658152 diff --git a/floss/tags/data/gp/gp-go-specific.bin b/floss/tags/data/gp/gp-go-specific.bin new file mode 100644 index 000000000..39a959357 --- /dev/null +++ b/floss/tags/data/gp/gp-go-specific.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f989aff19b40d4ff850fadc57b8da1d29820c2bf1fbc933ea402c018422aa7a4 +size 332680 diff --git a/floss/tags/data/gp/gp-pyinstaller-specific.bin b/floss/tags/data/gp/gp-pyinstaller-specific.bin new file mode 100644 index 000000000..f0f33ab6b --- /dev/null +++ b/floss/tags/data/gp/gp-pyinstaller-specific.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:485bdd252371ef63f7f69607f0b81e88b903d64a359037af0fe88fc3fddab197 +size 109616 diff --git a/floss/tags/data/gp/gp-rust-specific.bin b/floss/tags/data/gp/gp-rust-specific.bin new file mode 100644 index 000000000..065eb5b77 --- /dev/null +++ b/floss/tags/data/gp/gp-rust-specific.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f272ba86e88fa48f8454565a07ee591a0cd183c04708732c642cc818a9d3f8b9 +size 31624 diff --git a/floss/tags/data/gp/gp.jsonl.gz b/floss/tags/data/gp/gp.jsonl.gz new file mode 100644 index 000000000..06ef0dfb9 --- /dev/null +++ b/floss/tags/data/gp/gp.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46e28e4623734d94b8ebfc11386b95b1b967d5cb84d78449018029eca29339f8 +size 4057 diff --git a/floss/tags/data/gp/junk-code.jsonl.gz b/floss/tags/data/gp/junk-code.jsonl.gz new file mode 100644 index 000000000..cbe8ef066 --- /dev/null +++ b/floss/tags/data/gp/junk-code.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a6dbd9ad0f77378d834f99ab9372d6db28727e6b1391862e6d90ad54d947eaf +size 57990 diff --git a/floss/tags/data/gp/raw/gp-2026-hashes.csv.gz b/floss/tags/data/gp/raw/gp-2026-hashes.csv.gz new file mode 100644 index 000000000..76ae8aa1d --- /dev/null +++ b/floss/tags/data/gp/raw/gp-2026-hashes.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf44449eedde8f2a733d446d6ef710eb1037674046859b6f5731d9a0bb496c30 +size 1102625 diff --git a/floss/tags/data/gp/raw/gp-go-specific.csv.gz b/floss/tags/data/gp/raw/gp-go-specific.csv.gz new file mode 100644 index 000000000..4b0d727a9 --- /dev/null +++ b/floss/tags/data/gp/raw/gp-go-specific.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:910e63e0d9aaa69e4465faa8a2a3eb6195fac224d26a30a00014e9d72be9ed95 +size 674252 diff --git a/floss/tags/data/gp/raw/gp-pyinstaller-specific.csv.gz b/floss/tags/data/gp/raw/gp-pyinstaller-specific.csv.gz new file mode 100644 index 000000000..5d3badbdf --- /dev/null +++ b/floss/tags/data/gp/raw/gp-pyinstaller-specific.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574d8427c7f0c8080cf3dd81dd8d8dd17814214bdc0b1b9d5939fdc4cbd80813 +size 118316 diff --git a/floss/tags/data/gp/raw/gp-rust-specific.csv.gz b/floss/tags/data/gp/raw/gp-rust-specific.csv.gz new file mode 100644 index 000000000..ec8fc0c8f --- /dev/null +++ b/floss/tags/data/gp/raw/gp-rust-specific.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9161585b66d0045112ab06773bc2e09da77de17f6209efa99e759502a39ac449 +size 51544 diff --git a/floss/tags/data/gp/readme.md b/floss/tags/data/gp/readme.md new file mode 100644 index 000000000..52b4918a3 --- /dev/null +++ b/floss/tags/data/gp/readme.md @@ -0,0 +1,92 @@ +# Globally Prevalent Strings + +This directory contains databases of strings that are globally prevalent. +In other words, they are seen widely, and may be difficult to attribute to a specific library. + +There are two types of databases here: + - jsonl.gz files that contain strings and metadata + - hash databases that contain hashes of strings + +## JSONL files + +These databases are gzip-compressed JSONL files (one JSON document per line). +The first line contains metadata about the database, such as: + +```json +{ + "type":"metadata", + "version":"1.0", + "timestamp":"2023-05-11T12:49:35.328896", + "note":null +} +``` + +The subsequent lines look like: + +```json +{ + "string":"!This program cannot be run in DOS mode.", + "encoding":"ascii", + "global_count":424466, + "location":null +} +``` + +JSONL databases: + + - gp.jsonl.gz: a proof-of-concept GP database derived from an internal string database. All strings were seen at least 100,000 times across millions of files. This database doesn't provide much value. + - cwindb-dotnet.jsonl.gz: strings seen in .NET modules found on a Windows 10 system during May 2023. + - cwindb-native.jsonl.gz: strings seen in native PE files found on a Windows 10 system during May 2023. + - junk-code.jsonl.gz: junk strings from .text section of native PE files found on a Windows 10 system during May 2023. These strings are likely instruction sequences and we use them to supplement our code analysis recovery solution. + + +## Hash databases + +When collecting strings from a large number of files, we encounter a huge number of strings. For example, 100,000 files results in more than 3 million strings seen more than 100 times (and almost 600 million distinct strings). + +The hash database format is a sorted list of eight byte truncated MD5 hashes of strings found in a large corpus like this. FLOSS can quickly check if a string is in the database by computing the hash of the string and performing a binary search in the database; however, it can't recover any additional metadata about the string. + +Hash databases: + + - xaa-hashes.bin: strings seen more than 100 times in 100,000 files uploaded to VirusTotal on May 1, 2023. There's probably a substantial bias in this collection. See issue #722 for the history. + - yaa-hashes.bin: strings seen more than 100 times in 100,000 files uploaded to VirusTotal between May 18 and 24, 2023. The samples are randomly selected from more than 3 million total candidates in this time range. There's probably less bias in this collection (I hope). Also see issue #722 for the history. + - gp-2026-hashes.bin: Refined global prevalence database (82,269 hashes). Compiled from a 153k sample corpus (Sep 2023 - Jun 2026) to filter out common string noise. + - gp-go-specific.bin: Go-specific runtime noise (41,585 hashes). Contains Go-specific runtime and type descriptor strings. + - gp-rust-specific.bin: Rust-specific runtime noise (3,953 hashes). Contains Rust panic-handling and standard library strings. + - gp-pyinstaller-specific.bin: PyInstaller-specific noise (13,702 hashes). Contains PyInstaller bootloader and standard library Python bytecode strings. + +## Source Data (CSVs) + +For transparency and future reference, the raw string lists (with counts and locations) used to generate the 2026 databases are included as compressed CSV files in the `raw/` directory: + + - raw/gp-2026-hashes.csv.gz: Raw strings and metadata for `gp-2026-hashes.bin`. + - raw/gp-go-specific.csv.gz: Raw strings and metadata for `gp-go-specific.bin`. + - raw/gp-rust-specific.csv.gz: Raw strings and metadata for `gp-rust-specific.bin`. + - raw/gp-pyinstaller-specific.csv.gz: Raw strings and metadata for `gp-pyinstaller-specific.bin`. + +## 2026 Database Generation + +The 2026 prevalence databases were generated from a temporally balanced corpus to represent the modern threat landscape while filtering out campaign-specific noise. + +### 1. Corpus Selection & Extraction +* **Source**: 247,485 unique PE binaries collected from VirusTotal (balanced at ~250/day over 1,000 days from Sep 2023 to Jun 2026). +* **Clustering**: Clustered by Vhash and TLSH to select at most 3 representatives per cluster, resulting in **153,913 representative samples** used for string extraction. + +### 2. Global Database (`gp-2026-hashes.bin`) +To filter out general PE noise while avoiding malware campaign flooding: +* **Imphash Grouping**: Grouped the 153k files into **54,996 unique imphash groups** (counting only 1 file per group) to normalize the weight of malware families. +* **Section Filtering**: Restricted the database to strings from **14 standard PE sections** (e.g., `.text`, `.rdata`), removing packer and overlay noise. +* **Campaign Pruning**: Removed low-frequency campaign noise by requiring strings to appear in >= 11 groups (or >= 1,000 raw samples). +* *Result*: **82,269 unique hashes** (a clean, high-confidence global noise database). + +### 3. Specialized Sub-Databases (Go, Rust, PyInstaller) +Statically linked runtimes share identical IATs and collapse into single imphash groups, hiding their runtime strings. To capture this noise: +* **Classification**: Identified Go (1,574 samples), Rust (1,038 samples), and PyInstaller (1,229 samples) subsets using metadata. +* **Raw Samples Approach**: Compiled strings using raw sample counts (no grouping) at a 5% threshold (10% for PyInstaller) to capture the runtime. +* **Global Subtraction**: Subtracted the Global DB from each subset to remove redundant common APIs and keep the sub-databases lightweight. +* *Results*: + * **`gp-go-specific.bin`** (41,585 hashes) + * **`gp-rust-specific.bin`** (3,953 hashes) + * **`gp-pyinstaller-specific.bin`** (13,702 hashes) + + diff --git a/floss/tags/data/gp/xaa-hashes.bin b/floss/tags/data/gp/xaa-hashes.bin new file mode 100644 index 000000000..c5cdfacba --- /dev/null +++ b/floss/tags/data/gp/xaa-hashes.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40e2b5b8485a5dca0a8178d5de1611cbc3a7943ef6fc677e615c10afaa14d005 +size 3082616 diff --git a/floss/tags/data/gp/yaa-hashes.bin b/floss/tags/data/gp/yaa-hashes.bin new file mode 100644 index 000000000..532883690 --- /dev/null +++ b/floss/tags/data/gp/yaa-hashes.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cfbc2d8e8eb6d258d26b989e8287eec5cd95a746831a1051c03dbc2b3f52794 +size 3081184 diff --git a/floss/tags/data/oss/.gitignore b/floss/tags/data/oss/.gitignore new file mode 100644 index 000000000..498c1fb77 --- /dev/null +++ b/floss/tags/data/oss/.gitignore @@ -0,0 +1,6 @@ +*.csv +*.jsonl +# Ephemeral CI outputs (surfaced in the workflow PR body / logs, not committed). +build_metrics.json +build_diff.txt +build_diff_pr.txt diff --git a/floss/tags/data/oss/brotli.jsonl.gz b/floss/tags/data/oss/brotli.jsonl.gz new file mode 100644 index 000000000..c495a28de --- /dev/null +++ b/floss/tags/data/oss/brotli.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf805032b66a7acf26af3da9f1c15bb8066a64e9693b9caaef5826ef69947c7d +size 1472 diff --git a/floss/tags/data/oss/bzip2.jsonl.gz b/floss/tags/data/oss/bzip2.jsonl.gz new file mode 100644 index 000000000..5574e9527 --- /dev/null +++ b/floss/tags/data/oss/bzip2.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b19ac5fd564620478a17f48be28e619f556ff33a52c113e6a430ad75d2951b +size 1431 diff --git a/floss/tags/data/oss/cryptopp.jsonl.gz b/floss/tags/data/oss/cryptopp.jsonl.gz new file mode 100644 index 000000000..fb955a993 --- /dev/null +++ b/floss/tags/data/oss/cryptopp.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5eaae3db6cf8a644937658a43bcc072f028e00be9634edaa27c884a510f103 +size 76504 diff --git a/floss/tags/data/oss/curl.jsonl.gz b/floss/tags/data/oss/curl.jsonl.gz new file mode 100644 index 000000000..e92ee26cc --- /dev/null +++ b/floss/tags/data/oss/curl.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1129256dbedc87b3097b48fcd8e0921b52864198aeb9aa753a4c6c08c5016204 +size 23189 diff --git a/floss/tags/data/oss/detours.jsonl.gz b/floss/tags/data/oss/detours.jsonl.gz new file mode 100644 index 000000000..77185cf91 --- /dev/null +++ b/floss/tags/data/oss/detours.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc84ea7e67b2d088c38cea7618b7e80fb431a7b101b5f5efba61dc10c714eb21 +size 496 diff --git a/floss/tags/data/oss/jemalloc.jsonl.gz b/floss/tags/data/oss/jemalloc.jsonl.gz new file mode 100644 index 000000000..36c6400f9 --- /dev/null +++ b/floss/tags/data/oss/jemalloc.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d170e38e1f02a918e046a32571b1be4e8ce0f3dd9dfc22bd4f9e578d927bc5bc +size 8004 diff --git a/floss/tags/data/oss/jh_to_oss.py b/floss/tags/data/oss/jh_to_oss.py new file mode 100644 index 000000000..d9553df5f --- /dev/null +++ b/floss/tags/data/oss/jh_to_oss.py @@ -0,0 +1,53 @@ +""" +convert from a jh CSV file to a .jsonl.gz OpenSourceString database. + +the jh file looks like: + + # triplet,compiler,library,version,profile,path,function,type,value + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,number,0x00000100 + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,number,0xfffffff8 + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,number,0xfffffffe + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,api,BZ2_bzCompressInit + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,api,handle_compress + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffDecompress,number,0x0000fa90 + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffDecompress,number,0xfffffff8 + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffDecompress,number,0xfffffff9 + x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffDecompress,number,0xfffffffd + +jh is found here: https://github.com/williballenthin/lancelot/blob/master/bin/src/bin/jh.rs +""" + +import sys +import json +import pathlib + +import msgspec + +import floss.tags.oss + +p = pathlib.Path(sys.argv[1]) +for line in p.read_text().split("\n"): + if not line: + continue + + if line.startswith("#"): + continue + + triplet, compiler, library, version, profile, path, function, rest = line.split(",", 7) + type, _, value = rest.partition(",") + if type != "string": + continue + + if value.startswith('"'): + value = json.loads(value) + + s = floss.tags.oss.OpenSourceString( + string=value, + library_name=library, + library_version=version, + file_path=path, + function_name=function, + ) + + sys.stdout.buffer.write(msgspec.json.encode(s)) + sys.stdout.buffer.write(b"\n") diff --git a/floss/tags/data/oss/jsoncpp.jsonl.gz b/floss/tags/data/oss/jsoncpp.jsonl.gz new file mode 100644 index 000000000..b82ba0b8d --- /dev/null +++ b/floss/tags/data/oss/jsoncpp.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6099fdae2a201a1fafb48df804ee476f121face8bb35a940bcb374b2e46aa1d5 +size 7082 diff --git a/floss/tags/data/oss/kcp.jsonl.gz b/floss/tags/data/oss/kcp.jsonl.gz new file mode 100644 index 000000000..2c4eae01f --- /dev/null +++ b/floss/tags/data/oss/kcp.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9a9512d519dcf06691d7cbc7d31853b076536f050731418effd5192364d8aa1 +size 254 diff --git a/floss/tags/data/oss/liblzma.jsonl.gz b/floss/tags/data/oss/liblzma.jsonl.gz new file mode 100644 index 000000000..2daa29581 --- /dev/null +++ b/floss/tags/data/oss/liblzma.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4ff165ad8a84f9f3da123141550002b215e9f9a51d7ad6d46930f7e139e1d4c +size 349 diff --git a/floss/tags/data/oss/libpcap.jsonl.gz b/floss/tags/data/oss/libpcap.jsonl.gz new file mode 100644 index 000000000..73a365653 --- /dev/null +++ b/floss/tags/data/oss/libpcap.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:742e9082289d6cd51f1c2b4d0fd622261b5529df2c90344254ed8ae538bfb2d1 +size 5649 diff --git a/floss/tags/data/oss/libraries.json b/floss/tags/data/oss/libraries.json new file mode 100644 index 000000000..3fc696be1 --- /dev/null +++ b/floss/tags/data/oss/libraries.json @@ -0,0 +1,69 @@ +{ + "triplet": "x64-windows-static", + "compiler": "msvc143", + "profile": "release", + "libraries": [ + "apr", + "avro-c", + "binn", + "boost-chrono", + "boost-container", + "boost-filesystem", + "boost-graph", + "boost-iostreams", + "boost-json", + "boost-log", + "boost-serialization", + "boost-test", + "boost-thread", + "boost-wave", + "capnproto", + "cello", + "cunit", + "curl", + "czmq", + "duktape", + "expat", + "flatcc", + "hiredis", + "jansson", + "json-c", + "libarchive", + "libbson", + "libcoap", + "libev", + "libevent", + "libgit2", + "libhv", + "libimobiledevice", + "liblzma", + "libmysql", + "libpcap", + "libsndfile", + "libsrt", + "libuv", + "libxml2", + "libyaml", + "libzip", + "llhttp", + "lmdb", + "lz4", + "lzo", + "mbedtls", + "mongoose", + "nng", + "oniguruma", + "openssl", + "parson", + "poco", + "raylib", + "sail", + "sdl2", + "sqlite3", + "tinyfiledialogs", + "tre", + "wolfssl", + "zlib", + "zyre" + ] +} diff --git a/floss/tags/data/oss/libsodium.jsonl.gz b/floss/tags/data/oss/libsodium.jsonl.gz new file mode 100644 index 000000000..c3daef4b5 --- /dev/null +++ b/floss/tags/data/oss/libsodium.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2372322cf68cd8c3db49042bff7b8cd4c71d82d4c01943da3db9fd3f66ef57d +size 864 diff --git a/floss/tags/data/oss/mbedtls.jsonl.gz b/floss/tags/data/oss/mbedtls.jsonl.gz new file mode 100644 index 000000000..08dcfb84c --- /dev/null +++ b/floss/tags/data/oss/mbedtls.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb5bb8d3dc03ff04c4be5edb91d317ef4e7317ef5f4a8e0c5b8ce09fa0481a3 +size 19571 diff --git a/floss/tags/data/oss/openssl.jsonl.gz b/floss/tags/data/oss/openssl.jsonl.gz new file mode 100644 index 000000000..8ddda37c8 --- /dev/null +++ b/floss/tags/data/oss/openssl.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:158ecfc2abf2ea10fb93951c466e6b8ce8b536f47fd2289f7ac8b8d2069e859f +size 96896 diff --git a/floss/tags/data/oss/readme.md b/floss/tags/data/oss/readme.md new file mode 100644 index 000000000..9c2fa2552 --- /dev/null +++ b/floss/tags/data/oss/readme.md @@ -0,0 +1,110 @@ +# Strings from Open Source libraries + +This directory contains databases of strings extracted from open soure software. FLOSS uses these databases to show the user when a string is likely from a library. + +There is one file for each database. Each database is a gzip-compressed, JSONL (one JSON document per line) file. +The JSON document looks like this: + + string: "1.0.8, 13-Jul-2019" + library_name: "bzip2" + library_version: "1.0.8#3" + file_path: "CMakeFiles/bz2.dir/bzlib.c.obj" + function_name: "BZ2_bzlibVersion" + line_number: null + +The following databases were extracted via the vkpkg & jh technique: + + - brotli 1.0.9#5 + - bzip2 1.0.8#3 + - cryptopp 8.7.0 + - curl 7.86.0#1 + - detours 4.0.1#7 + - jemalloc 5.3.0#1 + - jsoncpp 1.9.5 + - kcp 1.7 + - liblzma 5.2.5#6 + - libsodium 1.0.18#8 + - libpcap 1.10.1#3 + - mbedtls 2.28.1 + - openssl 3.0.7#1 + - sqlite3 3.40.0#1 + - tomcrypt 1.18.2#2 + - wolfssl 5.5.0 + - zlib 1.2.13 + +## The vkpkg & jh technique + +Major steps: + + 1. build static libraries via vcpkg + 2. extract features via jh + 3. convert to JSONL format with `jh_to_oss.py` + 4. compress with gzip + +### Build static libraries via vcpkg + +[vcpkg](https://vcpkg.io/en/) is a free C/C++ package manager for acquiring and managing libraries. +We use it to easily build common open source libraries, like zlib. +Use the triplet `x64-windows-static` to build static archives (.lib files that are AR archives containing COFF object files): + +```console +PS > C:\vcpkg\vcpkg.exe install --triplet x64-windows-static zlib +``` + +### Why these build parameters + +We use `triplet=x64-windows-static`, `compiler=msvc143`, `profile=release` to match FLOSS's primary target (PE binaries on Windows). MSVC v143 was the current Visual Studio toolchain at the time the databases were first generated; switching to a newer toolchain would be expected to shift extracted string counts only marginally, since most strings come from source-level literals and symbol names. + +A cross-platform check (macOS arm64 vs. MinGW x86_64) showed total counts stay similar but per-string overlap is low: most non-overlap is random printable byte runs in compiled code (which differ per ISA), section/segment names (`.rdata` vs. `__TEXT,__cstring`), and symbol conventions (Mach-O prefixes C symbols with `_`, COFF does not). We keep the databases Windows-only since FLOSS targets PE. + +### Extract features via jh + +[jh](https://github.com/williballenthin/lancelot/blob/master/bin/src/bin/jh.rs) +is a lancelot-based utility that parses AR archives containing COFF object files, +reconstructs their control flow, finds functions, and extracts features. +jh extracts numbers, API calls, and strings; we are only interested in the string features. + +For each feature, jh emits a CSV line with the fields + - target triplet + - compiler + - library + - version + - build profile + - path + - function + - feature type + - feature value + +For example: + +```csv +x64-windows-static,msvc143,bzip2,1.0.8#3,release,CMakeFiles/bz2.dir/bzlib.c.obj,BZ2_bzBuffToBuffCompress,number,0x00000100 +``` + +For example, to invoke jh: + +```console +$ ~/lancelot/target/release/jh x64-windows-static msvc143 zlib 1.2.13 release /mnt/c/vcpkg/installed/x64-windows-static/lib/zlib.lib > ~/flare-floss/floss/tags/data/oss/zlib.csv +``` + +### Convert to OSS database format + +We use the script `jh_to_oss.py` to convert these CSV lines into JSONL file prepared for FLOSS: + +```console +$ python3 jh_to_oss.py zlib.csv > zlib.jsonl +``` + +These files are then gzip'd: + +```console +$ gzip -c zlib.jsonl > zlib.jsonl.gz +``` + +The `scripts/tags/build_oss_db.py` script automates the steps above and merges into +any `.jsonl.gz` already present in the output directory rather than +replacing them. For each library being rebuilt, the newly-extracted +entries are combined with the existing entries (new entries win on +string collisions, which is the right behavior when the library version +changes). + diff --git a/floss/tags/data/oss/sqlite3.jsonl.gz b/floss/tags/data/oss/sqlite3.jsonl.gz new file mode 100644 index 000000000..4edf3b5f9 --- /dev/null +++ b/floss/tags/data/oss/sqlite3.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d678d8a17bd78f81663fb060f03c7591231bc6dd40663a15cab0da83c82b8e3 +size 26184 diff --git a/floss/tags/data/oss/tomcrypt.jsonl.gz b/floss/tags/data/oss/tomcrypt.jsonl.gz new file mode 100644 index 000000000..32b8f411e --- /dev/null +++ b/floss/tags/data/oss/tomcrypt.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f5d6e7337c936a19b15bb54ac397b35a351bcfa2cfac3f9c80888fa574d0e1 +size 6519 diff --git a/floss/tags/data/oss/wolfssl.jsonl.gz b/floss/tags/data/oss/wolfssl.jsonl.gz new file mode 100644 index 000000000..39dc1ebe5 --- /dev/null +++ b/floss/tags/data/oss/wolfssl.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d7fe69bc335cdb6f1e7be13911cad36667a55f65f9f0ce0ac4be7cd59056586 +size 7727 diff --git a/floss/tags/data/oss/zlib.jsonl.gz b/floss/tags/data/oss/zlib.jsonl.gz new file mode 100644 index 000000000..2a65f7fcf --- /dev/null +++ b/floss/tags/data/oss/zlib.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e36bfc3be1c60a66851eea41305f9bd19ba4c74a946a8c60bc461fad576b08 +size 869 diff --git a/floss/tags/data/winapi/apis.txt.gz b/floss/tags/data/winapi/apis.txt.gz new file mode 100644 index 000000000..a23d11e5a --- /dev/null +++ b/floss/tags/data/winapi/apis.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72f64bbc78a06fb3aa08baa592296035337047f07ca38880ff31b18db405f5da +size 544325 diff --git a/floss/tags/data/winapi/dlls.txt.gz b/floss/tags/data/winapi/dlls.txt.gz new file mode 100644 index 000000000..c191bf067 --- /dev/null +++ b/floss/tags/data/winapi/dlls.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb348eeeabb79e3577b65823bba02b990b7c137c3267b9af5aadd464928aabf9 +size 17098 diff --git a/floss/tags/engine.py b/floss/tags/engine.py new file mode 100644 index 000000000..3ae2abc95 --- /dev/null +++ b/floss/tags/engine.py @@ -0,0 +1,131 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tagger callables and database loading.""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Callable, Iterable, Optional, Sequence + +import floss.tags.gp +import floss.tags.oss +import floss.tags.expert +import floss.tags.winapi +from floss.tags import data_root +from floss.ranges import OffsetRanges +from floss.tags.gp import StringHashDatabase, StringGlobalPrevalenceDatabase +from floss.tags.oss import OpenSourceStringDatabase +from floss.tags.expert import ExpertStringDatabase +from floss.tags.winapi import WindowsApiStringDatabase + +if TYPE_CHECKING: + from floss.layout.types import ExtractedString + +Tag = str +Tagger = Callable[["ExtractedString"], Sequence[Tag]] + + +def check_is_xor(xor_key: int | None) -> Sequence[Tag]: + if isinstance(xor_key, int): + return ("#decoded",) + return () + + +def check_is_reloc(reloc_offsets: OffsetRanges, string: ExtractedString) -> Sequence[Tag]: + if reloc_offsets.overlaps(string.slice.range.offset, string.slice.range.end - 1): + return ("#reloc",) + return () + + +def check_is_code(code_offsets: OffsetRanges, string: ExtractedString) -> Sequence[Tag]: + if code_offsets.overlaps(string.slice.range.offset, string.slice.range.end - 1): + return ("#code",) + return () + + +def query_code_string_database(db: StringGlobalPrevalenceDatabase, string: str): + if db.query(string): + return ("#code-junk",) + + return () + + +def query_global_prevalence_database(db: StringGlobalPrevalenceDatabase, string: str): + if db.query(string): + return ("#common",) + + return () + + +def query_global_prevalence_hash_database(db: StringHashDatabase, string: str): + if string in db: + return ("#common",) + + return () + + +def query_library_string_database(db: OpenSourceStringDatabase, string: str) -> Sequence[Tag]: + meta = db.metadata_by_string.get(string) + if not meta: + return () + + return (f"#{meta.library_name}",) + + +def query_expert_string_database(db: ExpertStringDatabase, string: str) -> Sequence[Tag]: + return tuple(db.query(string)) + + +def query_winapi_name_database(db: WindowsApiStringDatabase, string: str) -> Sequence[Tag]: + if string.lower() in db.dll_names: + return ("#winapi",) + + if string in db.api_names: + return ("#winapi",) + + return () + + +def load_databases() -> Sequence[Tagger]: + ret = [] + + def query_database(db, queryfn, string: ExtractedString): + return queryfn(db, string.string) + + def make_tagger(db, queryfn) -> Tagger: + return functools.partial(query_database, db, queryfn) + + for db in floss.tags.winapi.get_default_databases(): + ret.append(make_tagger(db, query_winapi_name_database)) + + for db_expert in floss.tags.expert.get_default_databases(): + ret.append(make_tagger(db_expert, query_expert_string_database)) + + for db_oss in floss.tags.oss.get_default_databases(): + ret.append(make_tagger(db_oss, query_library_string_database)) + + for db_gp in floss.tags.gp.get_default_databases(): + if isinstance(db_gp, StringGlobalPrevalenceDatabase): + ret.append(make_tagger(db_gp, query_global_prevalence_database)) + elif isinstance(db_gp, StringHashDatabase): + ret.append(make_tagger(db_gp, query_global_prevalence_hash_database)) + else: + raise ValueError(f"unexpected database type: {type(db_gp)}") + + # supplement code analysis with a database of junk code strings + junk_db = StringGlobalPrevalenceDatabase.from_file(data_root() / "gp" / "junk-code.jsonl.gz") + ret.append(make_tagger(junk_db, query_code_string_database)) + + return ret diff --git a/floss/tags/expert.py b/floss/tags/expert.py new file mode 100644 index 000000000..8efe79025 --- /dev/null +++ b/floss/tags/expert.py @@ -0,0 +1,112 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expert-curated tag source: rules authored by analysts (CAPA-derived, etc.). + +The ``floss/tags`` package is named for the user-visible outcome (tags on strings), +not for the on-disk JSONL databases. Each module here is a *tag source*: it loads +serialized classification data and exposes a query interface. ``floss.tags.engine`` +wraps those queries into ``Tagger`` callables applied during analysis. +""" + +import re +import pathlib +from typing import Set, Dict, List, Tuple, Literal, Sequence +from dataclasses import dataclass + +import msgspec + +from floss.tags import data_root, ensure_not_lfs_pointer + + +class ExpertRule(msgspec.Struct): + type: Literal["string", "substring", "regex"] + value: str + + tag: str + action: Literal["mute", "highlight", "hide"] + note: str + description: str + + authors: List[str] + references: List[str] + + +@dataclass +class ExpertStringDatabase: + string_rules: Dict[str, ExpertRule] + substring_rules: List[ExpertRule] + regex_rules: List[Tuple[ExpertRule, re.Pattern]] + + def __len__(self) -> int: + return len(self.string_rules) + len(self.substring_rules) + len(self.regex_rules) + + def query(self, s: str) -> Set[str]: + ret = set() + + if s in self.string_rules: + ret.add(self.string_rules[s].tag) + + # note that this is O(m * n) + # #strings * #rules + for rule in self.substring_rules: + if rule.value in s: + ret.add(rule.tag) + + # note that this is O(m * n) + # #strings * #rules + for rule, regex in self.regex_rules: + if regex.search(s): + ret.add(rule.tag) + + return ret + + @classmethod + def from_file(cls, path: pathlib.Path) -> "ExpertStringDatabase": + string_rules: Dict[str, ExpertRule] = {} + substring_rules: List[ExpertRule] = [] + regex_rules: List[Tuple[ExpertRule, re.Pattern]] = [] + + ensure_not_lfs_pointer(path) + decoder = msgspec.json.Decoder(type=ExpertRule) + buf = path.read_bytes() + for line in buf.split(b"\n"): + if not line: + continue + + rule = decoder.decode(line) + match rule: + case ExpertRule(type="string"): + # no duplicates today + string_rules[rule.value] = rule + case ExpertRule(type="substring"): + substring_rules.append(rule) + case ExpertRule(type="regex"): + # TODO: may have to cleanup the //gi from the regex + regex_rules.append((rule, re.compile(rule.value))) + case _: + raise ValueError(f"unexpected rule type: {rule.type}") + + return cls( + string_rules=string_rules, + substring_rules=substring_rules, + regex_rules=regex_rules, + ) + + +DEFAULT_PATHS = (data_root() / "expert" / "capa.jsonl",) + + +def get_default_databases() -> Sequence[ExpertStringDatabase]: + return [ExpertStringDatabase.from_file(path) for path in DEFAULT_PATHS] diff --git a/floss/tags/filter.py b/floss/tags/filter.py new file mode 100644 index 000000000..49bd85997 --- /dev/null +++ b/floss/tags/filter.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Visibility filters and false-positive library tag cleanup.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Literal + +from floss.results import ResultLayout, ResultString +from floss.tags.oss import DEFAULT_FILENAMES + +if TYPE_CHECKING: + from floss.layout.base import Layout + +Tag = str +TagRules = Dict[Tag, Literal["mute"] | Literal["highlight"] | Literal["default"] | Literal["hide"]] + + +def remove_false_positive_lib_strings(layout: "Layout"): + from floss.layout.extract import collect_strings + + # list of references to all the tagged strings across the layout. + # we can (carefully) manipulate the tags here. + tagged_strings = collect_strings(layout) + + # open source libraries should have at least 5 strings, + # or don't show their tag, since the couple hits are probably false positives. + # + # hack: assume the libname is embedded in the filename. + # otherwise, we don't have an easy way to recover the library tag names. + for filename in DEFAULT_FILENAMES: + libname = filename.partition(".")[0] + tagname = f"#{libname}" + + count = 0 + for string in tagged_strings: + if tagname in string.tags: + count += 1 + + if 0 < count < 5: + # I picked 5 as a reasonable threshold. + # we could research what a better value is. + # + # also note that large binaries with many strings have + # a higher chance of false positives, even with this threshold. + # this is still a useful filter, though. + for string in tagged_strings: + if tagname in string.tags: + string.tags.remove(tagname) + + +def should_hide_string(s: ResultString, tag_rules: TagRules) -> bool: + return any(map(lambda tag: tag_rules.get(tag) == "hide", s.tags)) + + +def hide_strings_by_rules(layout: ResultLayout, tag_rules: TagRules) -> ResultLayout: + """Return a new layout tree with hide-rule strings removed. + + Does not mutate ``layout`` or its children, so callers can render a + filtered view without deep-copying the original result document. + """ + return ResultLayout( + name=layout.name, + offset=layout.offset, + length=layout.length, + strings=[s for s in layout.strings if not should_hide_string(s, tag_rules)], + children=[hide_strings_by_rules(child, tag_rules) for child in layout.children], + ) diff --git a/floss/tags/gp.py b/floss/tags/gp.py new file mode 100644 index 000000000..a145dffe6 --- /dev/null +++ b/floss/tags/gp.py @@ -0,0 +1,176 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Global-prevalence tag source: strings common across many binaries (#common, #code-junk). + +Tag sources load on-disk databases and answer whether a string should receive a tag. +See ``floss.tags.engine`` for wiring into the analysis pipeline. +""" + +import gzip +import hashlib +import pathlib +import datetime +from typing import Set, Dict, List, Literal, Optional, Sequence +from collections import defaultdict +from dataclasses import dataclass + +import msgspec + +from floss.tags import data_root, ensure_not_lfs_pointer + +Encoding = Literal["ascii"] | Literal["utf-16le"] | Literal["unknown"] +# header | gap | overlay +# or section name +Location = Literal["header"] | Literal["gap"] | Literal["overlay"] | str + + +class Metadata(msgspec.Struct): + note: str | None + timestamp: str | None + type: str = "global_prevalence" + version: str = "1.0" + + +class StringGlobalPrevalence(msgspec.Struct): + string: str + encoding: Encoding + global_count: int + location: Location | None + + +@dataclass +class StringGlobalPrevalenceDatabase: + meta: Metadata + metadata_by_string: Dict[str, List[StringGlobalPrevalence]] + + def __len__(self) -> int: + return len(self.metadata_by_string) + + def insert(self, str_gp: StringGlobalPrevalence): + # TODO combine if existing data + self.metadata_by_string[str_gp.string].append(str_gp) + + def query(self, string): + return self.metadata_by_string.get(string, []) + + def update(self, other: "StringGlobalPrevalenceDatabase"): + # TODO combine if existing data + self.metadata_by_string.update(other.metadata_by_string) + + @classmethod + def new_db(cls, note: Optional[str] = None): + return cls( + meta=Metadata(timestamp=datetime.datetime.now().isoformat(), note=note), + metadata_by_string=defaultdict(list), + ) + + @classmethod + def from_file(cls, path: pathlib.Path, compress: bool = True) -> "StringGlobalPrevalenceDatabase": + metadata_by_string: Dict[str, List[StringGlobalPrevalence]] = defaultdict(list) + + if compress: + lines = gzip.decompress(path.read_bytes()).split(b"\n") + else: + lines = path.read_bytes().split(b"\n") + + decoder = msgspec.json.Decoder(type=StringGlobalPrevalence) + for line in lines[1:]: + if not line: + continue + s = decoder.decode(line) + + metadata_by_string[s.string].append(s) + + return cls( + meta=msgspec.json.Decoder(type=Metadata).decode(lines[0]), + metadata_by_string=metadata_by_string, + ) + + def to_file(self, outfile: str, compress: bool = True): + if compress: + with gzip.open(outfile, "w") as f: + f.write(msgspec.json.encode(self.meta) + b"\n") + for k, v in sorted(self.metadata_by_string.items(), key=lambda x: x[1][0].global_count, reverse=True): + # TODO needs fixing to write most common to least common + for e in v: + f.write(msgspec.json.encode(e) + b"\n") + else: + with open(outfile, "w", encoding="utf-8") as f: + f.write(msgspec.json.encode(self.meta).decode("utf-8") + "\n") + for k, v in sorted(self.metadata_by_string.items(), key=lambda x: x[1][0].global_count, reverse=True): + for e in v: + f.write(msgspec.json.encode(e).decode("utf-8") + "\n") + + +@dataclass +class StringHashDatabase: + string_hashes: Set[bytes] + + def __len__(self) -> int: + return len(self.string_hashes) + + def __contains__(self, other: bytes | str) -> bool: + if isinstance(other, bytes): + return other in self.string_hashes + elif isinstance(other, str): + m = hashlib.md5() + m.update(other.encode("utf-8")) + return m.digest()[:8] in self.string_hashes + else: + raise ValueError("other must be bytes or str") + + @classmethod + def from_file(cls, path: pathlib.Path) -> "StringHashDatabase": + string_hashes: Set[bytes] = set() + + ensure_not_lfs_pointer(path) + buf = path.read_bytes() + + for i in range(0, len(buf), 8): + string_hashes.add(buf[i : i + 8]) + + return cls( + string_hashes=string_hashes, + ) + + +DEFAULT_PATHS = ( + data_root() / "gp" / "gp.jsonl.gz", + data_root() / "gp" / "cwindb-native.jsonl.gz", + data_root() / "gp" / "cwindb-dotnet.jsonl.gz", + data_root() / "gp" / "xaa-hashes.bin", + data_root() / "gp" / "yaa-hashes.bin", + data_root() / "gp" / "gp-2026-hashes.bin", + data_root() / "gp" / "gp-go-specific.bin", + data_root() / "gp" / "gp-rust-specific.bin", + data_root() / "gp" / "gp-pyinstaller-specific.bin", +) + + +def get_default_databases() -> Sequence[StringGlobalPrevalenceDatabase | StringHashDatabase]: + merged_hash_db = StringHashDatabase(string_hashes=set()) + results: List[StringGlobalPrevalenceDatabase | StringHashDatabase] = [] + + for path in DEFAULT_PATHS: + if path.name.endswith(".jsonl.gz"): + results.append(StringGlobalPrevalenceDatabase.from_file(path)) + else: + db = StringHashDatabase.from_file(path) + merged_hash_db.string_hashes.update(db.string_hashes) + + if merged_hash_db.string_hashes: + results.append(merged_hash_db) + + return results diff --git a/floss/tags/oss.py b/floss/tags/oss.py new file mode 100644 index 000000000..7f75b98f3 --- /dev/null +++ b/floss/tags/oss.py @@ -0,0 +1,87 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Open-source library tag source: strings from rebuilt OSS libs (#openssl, #zlib, …). + +Tag sources load on-disk databases and answer whether a string should receive a tag. +See ``floss.tags.engine`` for wiring into the analysis pipeline. +""" + +import gzip +import pathlib +from typing import Dict, Sequence +from dataclasses import dataclass + +import msgspec + +from floss.tags import data_root, ensure_not_lfs_pointer + + +class OpenSourceString(msgspec.Struct): + string: str + library_name: str + library_version: str + file_path: str | None = None + function_name: str | None = None + line_number: int | None = None + + +@dataclass +class OpenSourceStringDatabase: + metadata_by_string: Dict[str, OpenSourceString] + + def __len__(self) -> int: + return len(self.metadata_by_string) + + @classmethod + def from_file(cls, path: pathlib.Path) -> "OpenSourceStringDatabase": + metadata_by_string: Dict[str, OpenSourceString] = {} + ensure_not_lfs_pointer(path) + decoder = msgspec.json.Decoder(type=OpenSourceString) + for line in gzip.decompress(path.read_bytes()).split(b"\n"): + if not line: + continue + s = decoder.decode(line) + metadata_by_string[s.string] = s + + return cls(metadata_by_string=metadata_by_string) + + +DEFAULT_FILENAMES = ( + "brotli.jsonl.gz", + "bzip2.jsonl.gz", + "cryptopp.jsonl.gz", + "curl.jsonl.gz", + "detours.jsonl.gz", + "jemalloc.jsonl.gz", + "jsoncpp.jsonl.gz", + "kcp.jsonl.gz", + "liblzma.jsonl.gz", + "libsodium.jsonl.gz", + "libpcap.jsonl.gz", + "mbedtls.jsonl.gz", + "openssl.jsonl.gz", + "sqlite3.jsonl.gz", + "tomcrypt.jsonl.gz", + "wolfssl.jsonl.gz", + "zlib.jsonl.gz", +) + +DEFAULT_PATHS = tuple(data_root() / "oss" / filename for filename in DEFAULT_FILENAMES) + ( + data_root() / "crt" / "msvc_v143.jsonl.gz", +) + + +def get_default_databases() -> Sequence[OpenSourceStringDatabase]: + return [OpenSourceStringDatabase.from_file(path) for path in DEFAULT_PATHS] diff --git a/floss/tags/winapi.py b/floss/tags/winapi.py new file mode 100644 index 000000000..20d9d2f74 --- /dev/null +++ b/floss/tags/winapi.py @@ -0,0 +1,61 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Windows API tag source: known DLL and API name strings (#winapi). + +Tag sources load on-disk databases and answer whether a string should receive a tag. +See ``floss.tags.engine`` for wiring into the analysis pipeline. +""" + +import gzip +import pathlib +from typing import Set, Sequence +from dataclasses import dataclass + +from floss.tags import data_root, ensure_not_lfs_pointer + + +@dataclass +class WindowsApiStringDatabase: + dll_names: Set[str] + api_names: Set[str] + + def __len__(self) -> int: + return len(self.dll_names) + len(self.api_names) + + @classmethod + def from_dir(cls, path: pathlib.Path) -> "WindowsApiStringDatabase": + dll_names: Set[str] = set() + api_names: Set[str] = set() + + ensure_not_lfs_pointer(path / "dlls.txt.gz") + for line in gzip.decompress((path / "dlls.txt.gz").read_bytes()).decode("utf-8").splitlines(): + if not line: + continue + dll_names.add(line) + + ensure_not_lfs_pointer(path / "apis.txt.gz") + for line in gzip.decompress((path / "apis.txt.gz").read_bytes()).decode("utf-8").splitlines(): + if not line: + continue + api_names.add(line) + + return cls(dll_names=dll_names, api_names=api_names) + + +DEFAULT_PATHS = (data_root() / "winapi",) + + +def get_default_databases() -> Sequence[WindowsApiStringDatabase]: + return [WindowsApiStringDatabase.from_dir(path) for path in DEFAULT_PATHS] diff --git a/floss/utils.py b/floss/utils.py index c17d08a40..984d76e97 100644 --- a/floss/utils.py +++ b/floss/utils.py @@ -22,7 +22,8 @@ import argparse import builtins import contextlib -from typing import Set, Tuple, Iterable, Optional +from enum import Enum +from typing import Set, List, Tuple, Iterable, Optional from pathlib import Path from collections import OrderedDict @@ -37,7 +38,14 @@ import floss.strings import floss.logging_ -from .const import MEGABYTE, MOD_NAME, MAX_STRING_LENGTH +from .const import ( + MEGABYTE, + MOD_NAME, + MAX_STRING_LENGTH, + UNSUPPORTED_FILE_MAGIC, + SUPPORTED_FILE_MAGIC_PE, + SUPPORTED_FILE_MAGIC_ELF, +) from .results import StaticString from .strings import extract_ascii_unicode_strings from .api_hooks import ENABLED_VIV_DEFAULT_HOOKS @@ -47,6 +55,57 @@ logger = floss.logging_.getLogger(__name__) +class FileType(str, Enum): + """ + The kind of a file, detected from its content. + """ + + PE = "pe" + ELF = "elf" + UNSUPPORTED = "unsupported" + RESULTS = "results" # a saved FLOSS results document + + +# how many leading bytes to sniff when detecting a results document +_RESULTS_SNIFF_SIZE = 8192 + + +def detect_file_type(sample: Path) -> FileType: + """ + Detect the kind of a file from its content. + + A saved FLOSS results document is a JSON object with the top-level + ResultDocument fields metadata, analysis, and strings. The top-level + schema is stable across the migration iterations, so this check does not + need a version. + + Binary samples are rejected cheaply by peeking the first non-whitespace + byte. Results documents are recognized by a fast byte-sequence check over + the leading chunk for the top-level keys, so large binaries that happen to + start with '{' are not read or parsed in full. Strict validation of the + document structure happens when the file is loaded. + """ + try: + with sample.open("rb") as f: + chunk = f.read(_RESULTS_SNIFF_SIZE) + except OSError: + return FileType.UNSUPPORTED + + if chunk[:4] == SUPPORTED_FILE_MAGIC_ELF: + return FileType.ELF + elif chunk[:2] == SUPPORTED_FILE_MAGIC_PE: + return FileType.PE + + stripped = chunk.lstrip() + if not stripped or stripped[:1] != b"{": + return FileType.UNSUPPORTED + + if all(key in chunk for key in (b'"metadata"', b'"analysis"', b'"strings"')): + return FileType.RESULTS + + return FileType.UNSUPPORTED + + class InstallContextMenu(argparse.Action): def __init__(self, option_strings, dest, nargs=None, **kwargs): super(InstallContextMenu, self).__init__(option_strings, dest, nargs=0, **kwargs) @@ -471,15 +530,28 @@ def get_call_funcname(api): return api[3] -def is_string_type_enabled(type_, disabled_types, enabled_types): - if disabled_types: - return type_ not in disabled_types - elif enabled_types: - return type_ in enabled_types +def is_string_type_enabled(type_, disabled_string_types, enabled_string_types): + if disabled_string_types: + return type_ not in disabled_string_types + elif enabled_string_types: + return type_ in enabled_string_types else: return True +def expand_string_types(string_types): + """expand the `all` alias into the full set of concrete string types. + + `all` may only be used on its own (validated at argument-parse time), so a + single value never needs deduplicating. + """ + from floss.cli import CONCRETE_STRING_TYPES, StringType + + if string_types == [StringType.ALL.value]: + return [t.value for t in CONCRETE_STRING_TYPES] + return list(string_types) + + def get_max_size(size: int, max_: int, api: Optional[Tuple] = None, argv: Optional[Tuple] = None) -> int: if size > max_: post = "" @@ -608,7 +680,7 @@ def get_static_strings(sample: Path, min_length: int) -> list: """ if sample.stat().st_size == 0: - logger.warning("File is empty") + logger.warning("file is empty") return [] with sample.open("rb") as f: diff --git a/pyproject.toml b/pyproject.toml index baa9ec354..db87944c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,7 @@ dependencies = [ # These dependencies are often actively influenced by capa, # so we provide a minimum patch version that includes the # latest bug fixes we need here. - "viv-utils[flirt]>=0.7.9", + "viv-utils[flirt]>=0.8.0", "vivisect>=1.1.1", "dncil>=1.0.2", @@ -103,6 +103,13 @@ dependencies = [ # we still support. "networkx>=3", + # layout / tagging + "dnfile==0.13.0", + "colorama==0.4.6", + "machofile==2026.2.4", + "msgspec==0.21.1", + "python-lancelot==0.9.7", + "pyelftools==0.31", ] dynamic = ["version", "readme"] @@ -110,8 +117,15 @@ dynamic = ["version", "readme"] version = {attr = "floss.version.__version__"} readme = {file = "README.md", content-type = "text/markdown"} -[tool.setuptools] -packages = ["floss", "floss.sigs"] +[tool.pytest.ini_options] +# so tests can import scripts.tags.* (scripts is not an installed package) +pythonpath = ["."] + +[tool.setuptools.packages.find] +include = ["floss*"] + +[tool.setuptools.package-data] +"floss" = ["sigs/*", "tags/data/**/*"] [project.optional-dependencies] # Dev and build dependencies are not relaxed because diff --git a/requirements.txt b/requirements.txt index d30b236ed..8e4212050 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ coverage==7.15.0 cxxfilt==0.3.0 distlib==0.4.0 dncil==1.0.2 +dnfile==0.13.0 filelock==3.32.0 funcy==2.0 halo==0.0.31 @@ -16,9 +17,11 @@ iniconfig==2.3.0 intervaltree==3.2.1 isort==8.0.1 log-symbols==0.0.14 +machofile==2026.2.4 markdown-it-py==4.2.0 mdurl==0.1.2 msgpack==1.2.1 +msgspec==0.21.1 mypy==2.3.0 networkx==3.4.2 nodeenv==1.10.0 @@ -38,12 +41,14 @@ pydantic==2.13.1 # but dependabot updates these separately (which is broken) and is annoying, # so we rely on pydantic to pull in the right version of pydantic-core. # pydantic-core==2.27.1 +pyelftools==0.31 pygments==2.20.0 pytest==9.1.0 pytest-cov==7.1.0 pytest-instafail==0.5.0 pytest-sugar==1.1.1 python-flirt==0.10.0 +python-lancelot==0.9.7 pyyaml==6.0.1 rich==15.0.0 setuptools==83.0.0 diff --git a/scripts/README.md b/scripts/README.md index fa5217073..a950f0059 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,33 +1,36 @@ # FLOSS Scripts -FLOSS supports converting its output into scripts for various tools. Please see the render scripts in this directory. - -Additionally, there is another [plugin for IDA](idaplugin.py) to allow FLOSS to automatically -extract obfuscated strings and apply them to the currently loaded module in IDA. `idaplugin.py` is a IDAPython script you can directly run within IDA Pro (File - Script File... [ALT + F7]). -# Installation -These scripts can be downloaded from the FLOSS [GitHub](https://github.com/mandiant/flare-floss) repository -alongside the source, which is required for the scripts to run. -To install FLOSS as source, see the documentation [here](../doc/installation.md). +Auxiliary scripts live under `scripts/`, grouped by purpose: +| Directory | Purpose | +|-----------|---------| +| [`disassemblers/`](disassemblers/) | Convert classic FLOSS JSON output into import scripts for Binary Ninja, Ghidra, IDA Pro, Radare2, and x64dbg; includes the IDA plugin | +| [`tags/`](tags/) | Build and maintain FLOSS tag databases (global prevalence, OSS libraries, VT feeds) | +| [`analysis/`](analysis/) | Batch analysis helpers | -# Usage -## Convert FLOSS output for use by other tools +## disassemblers/ -- Run FLOSS on the desired executable with the `-j` or `--json` argument to emit a JSON result -and redirect it to a JSON file. - `$ floss -j suspicious.exe > floss_results.json` +Turn FLOSS JSON (`floss -j sample.exe > results.json`) into tool-specific artifacts. -For Binary Ninja, IDA Pro, Ghidra or Radare2: -- Run the script for your tool of choice by passing the result json file as an argument and -redirect the output to a Python (.py) file. +1. Run a render script, redirecting stdout to a file. +2. Import or run the generated artifact in the target tool. -Ghidra Example: - `$ python render-ghidra-import-script.py floss_results.json > apply_floss.py` +Example (Ghidra): -- Run the Python script `apply_floss.py` using the desired tool. +```console +$ python render-ghidra-import-script.py results.json > apply_floss.py +``` -For x64dbg: -- Instead of a Python file, redirect the output to a .json file. - `$ python render-x64dbg-database.py floss-results.json > database.json` +See [`disassemblers/`](disassemblers/) for per-tool scripts and the [IDA plugin](disassemblers/idaplugin.py) (`File → Script file…` in IDA Pro). -- Open the JSON file `database.json` in x64dbg. +Install FLOSS from source first; see [installation](../doc/installation.md). + +## tags/ + +Scripts that extract strings, build tag databases, and query them. See [`tags/README.md`](tags/README.md) for the full pipeline. + +## analysis/ + +- [`bulk_analyze.py`](analysis/bulk_analyze.py) — run `floss` over every binary in a directory and write JSON results. + +Language-specific data maintenance (for example regenerating the Rust version hash database) lives alongside the implementation under `floss/language/`. \ No newline at end of file diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/analysis/__init__.py b/scripts/analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/analysis/bulk_analyze.py b/scripts/analysis/bulk_analyze.py new file mode 100644 index 000000000..286810959 --- /dev/null +++ b/scripts/analysis/bulk_analyze.py @@ -0,0 +1,183 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import logging +import pathlib +import argparse +import subprocess + +from floss.cli import set_log_config +from floss.layout.extract import MIN_STR_LEN + +# TODO: full deobf when those strings are first-class in layout output) +logger = logging.getLogger("floss.bulk") + + +def main(): + parser = argparse.ArgumentParser(description="Bulk analyze a directory of binaries with floss.") + parser.add_argument("input_directory", type=pathlib.Path, help="Directory containing binaries to analyze.") + parser.add_argument("output_directory", type=pathlib.Path, help="Directory to write JSON results to.") + parser.add_argument( + "-n", + "--minimum-length", + dest="min_length", + type=int, + default=MIN_STR_LEN, + help="Minimum string length.", + ) + parser.add_argument( + "--save-rendered", + action="store_true", + help="Save the rendered output to a .txt file in the output directory.", + ) + parser.add_argument( + "--reprocess", + action="store_true", + help="Reprocess files even if the output files already exist.", + ) + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="Enable debugging output on STDERR.") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="Disable all status output except fatal errors." + ) + args = parser.parse_args() + + set_log_config(args.debug, args.quiet) + + if not args.input_directory.is_dir(): + logger.error("Input path %s is not a directory.", args.input_directory) + return 1 + + args.output_directory.mkdir(parents=True, exist_ok=True) + + for file_path in args.input_directory.rglob("*"): + if not file_path.is_file(): + continue + + relative_path = file_path.relative_to(args.input_directory) + output_dir_for_file = args.output_directory / relative_path.parent + output_dir_for_file.mkdir(parents=True, exist_ok=True) + + json_output_path = output_dir_for_file / f"{file_path.name}.json" + rendered_output_path = output_dir_for_file / f"{file_path.name}.txt" + + should_analyze = not json_output_path.exists() or args.reprocess + should_render = args.save_rendered and (not rendered_output_path.exists() or args.reprocess) + + if not should_analyze and not should_render: + logger.info("Skipping file, all required outputs already exist: %s", file_path) + continue + + if should_analyze: + logger.info("Analyzing file: %s", file_path) + cmd = [ + sys.executable, + "-m", + "floss.main", + str(file_path), + "--no-string-type", + "stack", + "tight", + "decoded", + "--json", + "-n", + str(args.min_length), + ] + if args.quiet: + cmd.append("--quiet") + if args.debug: + cmd.append("--debug") + + try: + result = subprocess.run(cmd, check=False, capture_output=True, text=True, encoding="utf-8") + if result.returncode == 0: + with json_output_path.open("w", encoding="utf-8") as f: + f.write(result.stdout) + logger.info("Wrote JSON output to %s", json_output_path) + + if should_render: + cmd_render = [ + sys.executable, + "-m", + "floss.main", + str(json_output_path), + ] + if args.quiet: + cmd_render.append("--quiet") + if args.debug: + cmd_render.append("--debug") + + result_render = subprocess.run( + cmd_render, check=False, capture_output=True, text=True, encoding="utf-8" + ) + if result_render.returncode == 0: + with rendered_output_path.open("w", encoding="utf-8") as f: + f.write(result_render.stdout) + logger.info("Wrote rendered output to %s", rendered_output_path) + else: + logger.error( + "Failed to render file %s from JSON, exited with code %d", + file_path, + result_render.returncode, + ) + if result_render.stdout: + logger.error("stdout:\n%s", result_render.stdout) + if result_render.stderr: + logger.error("stderr:\n%s", result_render.stderr) + else: + logger.error("Failed to analyze file %s, exited with code %d", file_path, result.returncode) + if result.stdout: + logger.error("stdout:\n%s", result.stdout) + if result.stderr: + logger.error("stderr:\n%s", result.stderr) + except Exception as e: + logger.error("Failed to run analysis subprocess for file %s: %s", file_path, e, exc_info=True) + + elif should_render: + logger.info("Generating rendered output from existing JSON for: %s", file_path) + cmd = [ + sys.executable, + "-m", + "floss.main", + str(json_output_path), + ] + if args.quiet: + cmd.append("--quiet") + if args.debug: + cmd.append("--debug") + + try: + result = subprocess.run(cmd, check=False, capture_output=True, text=True, encoding="utf-8") + if result.returncode == 0: + with rendered_output_path.open("w", encoding="utf-8") as f: + f.write(result.stdout) + logger.info("Wrote rendered output to %s", rendered_output_path) + else: + logger.error( + "Failed to generate rendered output for %s, exited with code %d", file_path, result.returncode + ) + if result.stdout: + logger.error("stdout:\n%s", result.stdout) + if result.stderr: + logger.error("stderr:\n%s", result.stderr) + except Exception as e: + logger.error("Failed to run rendering subprocess for file %s: %s", file_path, e, exc_info=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/disassemblers/__init__.py b/scripts/disassemblers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/idaplugin.py b/scripts/disassemblers/idaplugin.py similarity index 100% rename from scripts/idaplugin.py rename to scripts/disassemblers/idaplugin.py diff --git a/scripts/render-binja-import-script.py b/scripts/disassemblers/render-binja-import-script.py similarity index 100% rename from scripts/render-binja-import-script.py rename to scripts/disassemblers/render-binja-import-script.py diff --git a/scripts/render-ghidra-import-script.py b/scripts/disassemblers/render-ghidra-import-script.py similarity index 100% rename from scripts/render-ghidra-import-script.py rename to scripts/disassemblers/render-ghidra-import-script.py diff --git a/scripts/render-ida-import-script.py b/scripts/disassemblers/render-ida-import-script.py similarity index 100% rename from scripts/render-ida-import-script.py rename to scripts/disassemblers/render-ida-import-script.py diff --git a/scripts/render-r2-import-script.py b/scripts/disassemblers/render-r2-import-script.py similarity index 100% rename from scripts/render-r2-import-script.py rename to scripts/disassemblers/render-r2-import-script.py diff --git a/scripts/render-x64dbg-database.py b/scripts/disassemblers/render-x64dbg-database.py similarity index 100% rename from scripts/render-x64dbg-database.py rename to scripts/disassemblers/render-x64dbg-database.py diff --git a/scripts/tags/README.md b/scripts/tags/README.md new file mode 100644 index 000000000..cba847d15 --- /dev/null +++ b/scripts/tags/README.md @@ -0,0 +1,55 @@ +# Tag database maintenance + +Scripts for building and inspecting the tag databases shipped under `floss/tags/data/`. + +## Pipeline overview + +``` +extract_strings.py → generate_gp_db.py → gp.jsonl.gz + ↑ + (raw PE string JSON) + +build_oss_db.py → .jsonl.gz (OSS tag databases) + +fetch_vt_hashes.py → hash list (VT feed sampling) + +query_string.py → lookup GP tags (debug / inspection) +``` + +## Global prevalence (GP) + +1. **Extract** raw strings from PEs or `.lib` archives: + + ```console + $ python scripts/tags/extract_strings.py C:\Windows outdir --pes + ``` + +2. **Generate** the global-prevalence database from extracted JSON: + + ```console + $ python scripts/tags/generate_gp_db.py outdir gp.jsonl.gz --type native + ``` + +3. Install the result at `floss/tags/data/gp/gp.jsonl.gz` (and related hash files). + +4. **Query** a string against the installed database: + + ```console + $ python scripts/tags/query_string.py "kernel32.dll" + ``` + +## Open-source library (OSS) databases + +`build_oss_db.py` automates the [vcpkg & jh technique](../../floss/tags/data/oss/readme.md): install static libraries, extract features with jh, emit gzip-compressed JSONL, and merge with any existing databases in the output directory. + +```console +$ python scripts/tags/build_oss_db.py --libraries zlib curl --output-dir floss/tags/data/oss +``` + +## VirusTotal feed sampling + +`fetch_vt_hashes.py` downloads hashes for relevant file types from the VT feed over a time range (requires `virustotal3` and API credentials). + +```console +$ python scripts/tags/fetch_vt_hashes.py 202305010000 202305020000 +``` diff --git a/scripts/tags/__init__.py b/scripts/tags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/tags/build_oss_db.py b/scripts/tags/build_oss_db.py new file mode 100644 index 000000000..8c4017e3a --- /dev/null +++ b/scripts/tags/build_oss_db.py @@ -0,0 +1,1180 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Build OSS string databases from vcpkg static libraries. + +This script automates the "vcpkg & jh" technique described in readme.md: + + 1. install static libraries via vcpkg + 2. extract string features (and function names) via jh + 3. convert to JSONL and compress with gzip + +It is intentionally modular so the underlying extractor (jh today) can be +swapped for a more minimal tool later without rewriting the orchestration. + +Strings are NOT deduped across libraries: a string observed in both zlib +and curl (e.g. when zlib is vendored into curl) stays in both databases. +The query tagger already emits one ``#`` tag per matching +database, so the consumer can see the overlap directly. Within a single +library, the same string still collapses to one entry (when +``--no-deduplicate`` is not passed). +""" + +from __future__ import annotations + +import os +import re +import sys +import gzip +import json +import time +import shutil +import logging +import pathlib +import argparse +import subprocess +from typing import Set, Dict, List, Tuple, Callable, Optional +from dataclasses import dataclass + +from floss.tags import data_root + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger("build_oss_db") + + +class BuildError(Exception): + """Raised when a single library cannot be built; the caller decides whether to abort or continue.""" + + +class UnsupportedPlatformError(BuildError): + """Raised when a library does not support the target triplet/platform.""" + + +@dataclass(frozen=True) +class BuildConfig: + triplet: str + compiler: str + profile: str + libraries: List[str] + output_dir: pathlib.Path + vcpkg_root: Optional[pathlib.Path] + jh_path: Optional[pathlib.Path] + lancelot_dir: Optional[pathlib.Path] + emit_function_names: bool = True + deduplicate: bool = True + continue_on_error: bool = False + + +def make_db_entry( + string: str, + library_name: Optional[str], + library_version: Optional[str], + file_path: Optional[str], + function_name: Optional[str], + line_number: Optional[int] = None, +) -> dict: + """Construct a database entry using the standard OSS schema.""" + return { + "string": string, + "library_name": library_name, + "library_version": library_version, + "file_path": file_path, + "function_name": function_name, + "line_number": line_number, + } + + +@dataclass +class ParseResult: + """Result of parsing jh JSONL output for a single library.""" + + entries: List[dict] + num_objects: int + num_functions: int + + +@dataclass +class LibraryMetrics: + library: str + version: str + triplet: str + num_objects: int = 0 + num_functions: int = 0 + num_string_entries: int = 0 + num_function_name_entries: int = 0 + num_raw_entries: int = 0 + total_entries: int = 0 + duration_seconds: float = 0.0 + error: Optional[str] = None + + def as_dict(self) -> dict: + return { + "library": self.library, + "version": self.version, + "triplet": self.triplet, + "num_objects": self.num_objects, + "num_functions": self.num_functions, + "num_string_entries": self.num_string_entries, + "num_function_name_entries": self.num_function_name_entries, + "num_raw_entries": self.num_raw_entries, + "total_entries": self.total_entries, + "duration_seconds": round(self.duration_seconds, 2), + "error": self.error, + } + + +def run( + cmd: List[str], + cwd: Optional[pathlib.Path] = None, + check: bool = True, +) -> subprocess.CompletedProcess: + """Run a subprocess and return its output.""" + logger.debug("running: %s", " ".join(cmd)) + result = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + text=True, + capture_output=True, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + cmd, + output=result.stdout, + stderr=result.stderr, + ) + return result + + +class Vcpkg: + """Thin wrapper around a vcpkg installation.""" + + def __init__(self, vcpkg_root: Optional[pathlib.Path] = None): + self.exe = self._find_executable(vcpkg_root) + self.root = self._resolve_root(vcpkg_root) + self.installed_dir = self.root / "installed" + self.info_dir = self.installed_dir / "vcpkg" / "info" + + def _find_executable(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path: + # 1. Executable bundled inside the provided root. + if vcpkg_root: + for name in ("vcpkg.exe", "vcpkg"): + candidate = vcpkg_root / name + if candidate.exists(): + return candidate.resolve() + + # 2. Executable on PATH. + exe = shutil.which("vcpkg") + if exe: + return pathlib.Path(exe).resolve() + + # 3. Executable inside VCPKG_ROOT. + env_root = os.environ.get("VCPKG_ROOT") + if env_root: + for name in ("vcpkg.exe", "vcpkg"): + candidate = pathlib.Path(env_root) / name + if candidate.exists(): + return candidate.resolve() + + raise FileNotFoundError("vcpkg not found. Set VCPKG_ROOT or pass --vcpkg-root.") + + def _resolve_root(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path: + if vcpkg_root: + return vcpkg_root.resolve() + + env_root = os.environ.get("VCPKG_ROOT") + if env_root: + return pathlib.Path(env_root).resolve() + + # The executable normally lives at /vcpkg. + return self.exe.parent.resolve() + + def install(self, library: str, triplet: str) -> None: + """Install a library for the given triplet.""" + spec = f"{library}:{triplet}" + logger.info("vcpkg install %s", spec) + try: + run([str(self.exe), "install", spec]) + except subprocess.CalledProcessError as exc: + output = (exc.stdout or "") + (exc.stderr or "") + if "is only supported on" in output: + raise UnsupportedPlatformError(f"{spec} is not supported on this platform") from exc + raise + + def get_installed_version(self, library: str, triplet: str) -> str: + """Return the installed version string (e.g. '3.0.7#1').""" + result = run([str(self.exe), "list", f"{library}:{triplet}"]) + expected_prefix = f"{library}:{triplet}" + + for line in result.stdout.splitlines(): + line = line.strip() + if not line or line.startswith("The following packages are"): + continue + + parts = line.split() + if len(parts) < 2: + continue + + if parts[0] == expected_prefix: + return parts[1] + + raise BuildError(f"could not determine installed version for {library}:{triplet}") + + def find_package_libs(self, library: str, triplet: str) -> List[pathlib.Path]: + """Return static-library files (.lib/.a) owned by the given package.""" + # vcpkg records installed files in /installed/vcpkg/info/__.list + pattern = re.compile(re.escape(library) + r"_.*?_" + re.escape(triplet) + r"\.list$") + list_files = [] + if self.info_dir.exists(): + list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)] + + if not list_files: + # The .list file is the only authoritative way to know which static + # libraries belong to this package. vcpkg installs every package's + # libs into the same shared /lib/ directory, so a blind + # scan of that directory would attribute other packages' + # strings/functions to this library. Refuse to extract instead of + # silently polluting the database, and surface the files we would + # have wrongly included for diagnostics. + candidate_files = self._find_all_static_libs(triplet) + logger.error( + "could not find vcpkg info .list for %s:%s; " + "refusing to extract to avoid misattributing %d other-package " + "library file(s): %s", + library, + triplet, + len(candidate_files), + ", ".join(p.name for p in candidate_files), + ) + return [] + + lib_paths: List[pathlib.Path] = [] + for list_file in list_files: + for line in list_file.read_text().splitlines(): + line = line.strip() + if not line: + continue + if line.startswith(triplet + "/lib/"): + candidate = self.installed_dir / line + if candidate.suffix in (".lib", ".a"): + lib_paths.append(candidate) + + return sorted(set(lib_paths)) + + def _find_all_static_libs(self, triplet: str) -> List[pathlib.Path]: + lib_dir = self.installed_dir / triplet / "lib" + if not lib_dir.exists(): + return [] + return sorted(p for p in lib_dir.iterdir() if p.suffix in (".lib", ".a")) + + +class JHExtractor: + """Wrapper around the jh binary. Builds it from source if needed.""" + + def __init__( + self, + jh_path: Optional[pathlib.Path] = None, + lancelot_dir: Optional[pathlib.Path] = None, + ): + self.jh_path = self._resolve(jh_path, lancelot_dir) + + def _resolve( + self, + jh_path: Optional[pathlib.Path], + lancelot_dir: Optional[pathlib.Path], + ) -> pathlib.Path: + if jh_path: + path = pathlib.Path(jh_path).resolve() + if not path.exists(): + raise FileNotFoundError(f"jh binary not found: {path}") + return path + + env_path = os.environ.get("JH_PATH") + if env_path: + path = pathlib.Path(env_path).resolve() + if path.exists(): + return path + + if lancelot_dir: + return self._build(lancelot_dir) + + env_lancelot = os.environ.get("LANCELOT_DIR") + if env_lancelot: + return self._build(pathlib.Path(env_lancelot)) + + exe = shutil.which("jh") + if exe: + return pathlib.Path(exe).resolve() + + raise FileNotFoundError("jh not found. Provide --jh-path, --lancelot-dir, or set JH_PATH.") + + def _build(self, lancelot_dir: pathlib.Path) -> pathlib.Path: + logger.info("building jh from %s", lancelot_dir) + run( + ["cargo", "build", "--release", "-p", "lancelot-bin"], + cwd=lancelot_dir, + ) + exe = lancelot_dir / "target" / "release" / "jh" + if sys.platform == "win32": + exe = exe.with_suffix(".exe") + if not exe.exists(): + raise FileNotFoundError(f"jh binary not found after build: {exe}") + return exe.resolve() + + def extract( + self, + lib_path: pathlib.Path, + library: str, + version: str, + triplet: str, + compiler: str, + profile: str, + ) -> str: + """Run jh on a single static library and return its JSONL output.""" + cmd = [ + str(self.jh_path), + triplet, + compiler, + library, + version, + profile, + str(lib_path), + ] + try: + result = run(cmd) + except subprocess.CalledProcessError as exc: + logger.error( + "jh failed for %s (%s): stdout=%r stderr=%r", + library, + lib_path.name, + exc.stdout, + exc.stderr, + ) + raise + return result.stdout + + +class Converter: + """Convert jh JSONL output into a gzip-compressed JSONL database.""" + + def __init__(self, emit_function_names: bool = True, deduplicate: bool = True): + self.emit_function_names = emit_function_names + self.deduplicate = deduplicate + + def parse( + self, + jh_text: str, + library: str, + version: str, + ) -> ParseResult: + """Parse jh JSONL into entries and tally object/function counts in one pass. + + Within-library dedup is applied to entries if enabled. Object and + function counts reflect the raw (pre-dedup) input, matching the + behavior of the previous standalone counter. + """ + entries: List[dict] = [] + objects: Set[str] = set() + function_names: Set[str] = set() + explicit_function_names: Set[str] = set() + + for line in jh_text.splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + logger.warning("skipping malformed JSONL line: %s (%s)", line, exc) + continue + + file_path = row.get("path") + function_name = row.get("function") + feat_type = row.get("type") + value = row.get("value") + + if not function_name: + continue + + objects.add(file_path) + function_names.add(function_name) + + if feat_type == "string": + entries.append(make_db_entry(value, library, version, file_path, function_name)) + elif feat_type == "function_name": + # Future-proof: a minimal extractor may emit function names explicitly. + entries.append(make_db_entry(value, library, version, file_path, value)) + explicit_function_names.add(value) + + # Stock jh does not emit function_name rows, so derive them from the + # function column. Functions without any string/number/api features + # will be missed unless the extractor is patched to emit them. + if self.emit_function_names: + for fn in function_names - explicit_function_names: + entries.append(make_db_entry(fn, library, version, None, fn)) + + if self.deduplicate: + # Match loader semantics: one metadata object per unique string. + seen: dict = {} + for entry in entries: + key = entry["string"] + if key not in seen: + seen[key] = entry + entries = list(seen.values()) + + return ParseResult( + entries=entries, + num_objects=len(objects), + num_functions=len(function_names), + ) + + def write( + self, + entries: List[dict], + output_path: pathlib.Path, + ) -> dict: + """Write entries to a gzip-compressed JSONL file. Returns counts.""" + output_path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(output_path, "wt", encoding="utf-8") as f: + for entry in entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + num_string_entries = sum( + 1 for e in entries if e["function_name"] is not None and e["function_name"] != e["string"] + ) + num_function_name_entries = sum( + 1 for e in entries if e["function_name"] is not None and e["function_name"] == e["string"] + ) + + return { + "num_string_entries": num_string_entries, + "num_function_name_entries": num_function_name_entries, + "total_entries": len(entries), + } + + +def build_library( + library: str, + config: BuildConfig, + vcpkg: Vcpkg, + jh: JHExtractor, + converter: Converter, +) -> Tuple[LibraryMetrics, List[dict]]: + """Parse a single library's strings. Returns metrics and the deduped entries. + + The returned entries are not yet written to disk; the caller is responsible + for cross-library deduplication and final file emission. + """ + start = time.time() + metrics = LibraryMetrics( + library=library, + version="unknown", + triplet=config.triplet, + ) + entries: List[dict] = [] + + try: + vcpkg.install(library, config.triplet) + version = vcpkg.get_installed_version(library, config.triplet) + metrics.version = version + + lib_paths = vcpkg.find_package_libs(library, config.triplet) + if not lib_paths: + logger.info( + "%s: no static libraries found for %s:%s (likely header-only); skipping extraction", + library, + library, + config.triplet, + ) + else: + logger.info( + "%s: found %d static library file(s): %s", + library, + len(lib_paths), + ", ".join(str(p.name) for p in lib_paths), + ) + + all_jh_parts: List[str] = [] + for lib_path in lib_paths: + logger.info("%s: extracting strings from %s", library, lib_path.name) + jh_text = jh.extract( + lib_path, + library, + version, + config.triplet, + config.compiler, + config.profile, + ) + all_jh_parts.append(jh_text) + + combined_jh_text = "\n".join(all_jh_parts) + result = converter.parse(combined_jh_text, library, version) + metrics.num_objects = result.num_objects + metrics.num_functions = result.num_functions + entries = result.entries + metrics.num_raw_entries = len(entries) + except UnsupportedPlatformError as exc: + logger.warning("%s: skipping unsupported library (%s)", library, exc) + except Exception as exc: + logger.exception("%s: build failed", library) + metrics.error = f"{type(exc).__name__}: {exc}" + finally: + metrics.duration_seconds = time.time() - start + + return metrics, entries + + +def load_existing_entries(path: pathlib.Path) -> List[dict]: + """Load entries from an existing OSS database .jsonl.gz. + + Returns an empty list if the file is missing, empty, unreadable, or does + not contain entries in the expected schema. New keys are silently dropped; + missing keys are filled with None so the entries can be merged uniformly. + """ + if not path.exists(): + return [] + try: + raw = gzip.decompress(path.read_bytes()) + except (OSError, gzip.BadGzipFile, EOFError) as exc: + logger.warning("could not read existing database %s: %s", path, exc) + return [] + + entries: List[dict] = [] + for line in raw.split(b"\n"): + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + logger.warning("skipping malformed line in %s: %s", path, exc) + continue + if not isinstance(row, dict) or "string" not in row: + continue + string = row.get("string") + if not isinstance(string, str): + continue + library_name = row.get("library_name") + library_version = row.get("library_version") + file_path = row.get("file_path") + function_name = row.get("function_name") + line_number = row.get("line_number") + entries.append( + make_db_entry( + string, + library_name if isinstance(library_name, str) else None, + library_version if isinstance(library_version, str) else None, + file_path if isinstance(file_path, str) else None, + function_name if isinstance(function_name, str) else None, + line_number if isinstance(line_number, int) else None, + ) + ) + return entries + + +def merge_entries( + new_entries: List[dict], + existing_entries: List[dict], + deduplicate: bool, +) -> List[dict]: + """Combine new and existing entries, with new taking precedence on conflict. + + When ``deduplicate`` is true the result contains at most one entry per + unique string value; otherwise the lists are concatenated as-is. + """ + if not new_entries: + return list(existing_entries) + if not existing_entries: + return list(new_entries) + if not deduplicate: + return list(existing_entries) + list(new_entries) + + seen: Dict[str, dict] = {} + # Iterate new first so that freshly-built entries win on string collisions, + # which is the right behavior when the underlying library version changes. + for entry in list(new_entries) + list(existing_entries): + key = entry["string"] + if key not in seen: + seen[key] = entry + return list(seen.values()) + + +# Max +/-/~ lines per library in the CI log summary (build_diff.txt). +DIFF_MAX_LINES_PER_LIBRARY = 100 +# Shorter per-library cap for the PR description body. +DIFF_PR_MAX_LINES_PER_LIBRARY = 20 +# Cap individual string values so one long entry does not dominate the diff. +DIFF_STRING_MAX_LEN = 120 +# GitHub rejects PR bodies over 65536 codepoints ("Body is too long"). Leave +# headroom for the workflow's static header and a truncation footer. +DIFF_PR_MAX_CHARS = 60_000 +# Fields compared when deciding whether an existing string's metadata changed. +_DIFF_META_FIELDS = ("library_version", "file_path", "function_name", "line_number") + + +def _escape_diff_string(value: Optional[str]) -> str: + """Make a string safe/readable for a single-line text diff. + + Backticks are separated so string values cannot close markdown code fences. + """ + if value is None: + return "" + text = ( + value.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + .replace("```", "` ` `") + ) + if len(text) > DIFF_STRING_MAX_LEN: + return text[: DIFF_STRING_MAX_LEN - 3] + "..." + return text + + +def _format_meta_value(value: object) -> str: + if value is None: + return "" + return _escape_diff_string(str(value)) + + +def _format_meta_delta(old: dict, new: dict) -> str: + """Return a short 'field: old -> new' summary for changed metadata fields.""" + parts: List[str] = [] + for field in _DIFF_META_FIELDS: + old_val, new_val = old.get(field), new.get(field) + if old_val != new_val: + parts.append(f"{field}: {_format_meta_value(old_val)} -> {_format_meta_value(new_val)}") + return "; ".join(parts) + + +@dataclass +class LibraryDiff: + """Text-diff summary for one library's database rewrite.""" + + library: str + old_count: int + new_count: int + added: List[dict] + removed: List[dict] + changed: List[Tuple[dict, dict]] # (old_entry, new_entry) + + @property + def has_changes(self) -> bool: + return bool(self.added or self.removed or self.changed) + + def header_line(self) -> str: + return ( + f"## {self.library} " + f"(entries: {self.old_count} -> {self.new_count}; " + f"+{len(self.added)} -{len(self.removed)} ~{len(self.changed)})" + ) + + def body_lines(self) -> List[str]: + lines: List[str] = [] + # Sort for stable, reviewable output. + for entry in sorted(self.added, key=lambda e: e.get("string") or ""): + lines.append(f"+ {_escape_diff_string(entry.get('string'))}") + for entry in sorted(self.removed, key=lambda e: e.get("string") or ""): + lines.append(f"- {_escape_diff_string(entry.get('string'))}") + for old, new in sorted(self.changed, key=lambda pair: pair[0].get("string") or ""): + delta = _format_meta_delta(old, new) + lines.append(f"~ {_escape_diff_string(old.get('string'))} ({delta})") + return lines + + +def diff_library_entries(library: str, old_entries: List[dict], new_entries: List[dict]) -> LibraryDiff: + """Compare two entry lists for one library and return a structured diff. + + Entries are keyed by their ``string`` value (matching merge/dedup + semantics). Added/removed strings get ``+``/``-`` lines; same string with + different metadata gets a ``~`` line. + """ + old_by_string: Dict[str, dict] = {} + for entry in old_entries: + key = entry.get("string") + if key is not None and key not in old_by_string: + old_by_string[key] = entry + + new_by_string: Dict[str, dict] = {} + for entry in new_entries: + key = entry.get("string") + if key is not None and key not in new_by_string: + new_by_string[key] = entry + + old_keys = set(old_by_string) + new_keys = set(new_by_string) + + added = [new_by_string[k] for k in new_keys - old_keys] + removed = [old_by_string[k] for k in old_keys - new_keys] + changed: List[Tuple[dict, dict]] = [] + for key in old_keys & new_keys: + old, new = old_by_string[key], new_by_string[key] + if any(old.get(field) != new.get(field) for field in _DIFF_META_FIELDS): + changed.append((old, new)) + + return LibraryDiff( + library=library, + old_count=len(old_entries), + new_count=len(new_entries), + added=added, + removed=removed, + changed=changed, + ) + + +def _truncated_body_lines(body: List[str], max_lines: int, library: str) -> List[str]: + """Keep at most ``max_lines`` change lines; append a truncation note if needed.""" + if len(body) <= max_lines: + return body + omitted = len(body) - max_lines + return body[:max_lines] + [f"... truncated ({omitted} more line(s) omitted for {library})"] + + +def format_build_diff( + library_diffs: List[LibraryDiff], + max_lines_per_library: int = DIFF_MAX_LINES_PER_LIBRARY, +) -> str: + """Render library diffs as a plain-text report, truncated per library. + + Each library section keeps its header plus at most ``max_lines_per_library`` + change lines (``+``/``-``/``~``). A trailing note is added when a library's + body is cut off. Intended for the capped CI log summary (``build_diff.txt``). + """ + if max_lines_per_library < 1: + return "" + + if not library_diffs: + return "No libraries were rebuilt.\n" + + changed = [d for d in library_diffs if d.has_changes] + if not changed: + return "No entry-level changes detected.\n" + + lines: List[str] = [ + "OSS string database entry-level diff", + f"(libraries with changes: {len(changed)}/{len(library_diffs)})", + "", + ] + for diff in changed: + lines.append(diff.header_line()) + lines.extend(_truncated_body_lines(diff.body_lines(), max_lines_per_library, diff.library)) + lines.append("") + + # Drop trailing blank line. + while lines and lines[-1] == "": + lines.pop() + + return "\n".join(lines) + "\n" + + +def _omission_footer(omitted: List[str]) -> str: + """Short note listing libraries dropped to stay under the PR body size cap.""" + if not omitted: + return "" + if len(omitted) <= 10: + names = ", ".join(omitted) + else: + names = ", ".join(omitted[:8]) + f", ... (+{len(omitted) - 8} more)" + return ( + f"\n... omitted {len(omitted)} more libraries with changes ({names}).\n" + "See `build_diff.txt` in the workflow logs for the capped CI summary.\n" + ) + + +def _clip_to_max_chars(text: str, max_chars: int) -> str: + """Hard-cap ``text`` at ``max_chars``, appending a short notice if clipped.""" + if max_chars < 1: + return "" + if len(text) <= max_chars: + return text + notice = "\n... truncated to stay under GitHub's PR body length limit.\n" + if len(notice) >= max_chars: + return notice[:max_chars] + + limit = max_chars - len(notice) + truncated = text[:limit] + fence = "```" # Markdown fenced code-block delimiter. + if truncated.count(fence) % 2: + closing_fence = "\n" + fence + if limit >= len(closing_fence): + truncated = text[: limit - len(closing_fence)].rstrip() + closing_fence + else: + # A closing fence does not fit; omit the unmatched opening fence. + truncated = truncated[: truncated.rfind(fence)] + return truncated.rstrip() + notice + + +def format_build_diff_markdown( + library_diffs: List[LibraryDiff], + max_lines_per_library: int = DIFF_PR_MAX_LINES_PER_LIBRARY, + max_chars: int = DIFF_PR_MAX_CHARS, +) -> str: + """Render library diffs as markdown for a GitHub PR description. + + Each library gets a ``##`` heading outside its own fenced diff code block + that contains only the ``+``/``-``/``~`` change lines (truncated per library). + + The whole report is also capped at ``max_chars`` so the PR body stays under + GitHub's 65536-character limit (with headroom for the workflow header). + Libraries that do not fit are summarized in a trailing note; the capped CI + log summary remains in ``build_diff.txt`` / workflow logs. + """ + if max_lines_per_library < 1 or max_chars < 1: + return "" + + if not library_diffs: + return "No libraries were rebuilt.\n" + + changed = [d for d in library_diffs if d.has_changes] + if not changed: + return "No entry-level changes detected.\n" + + sections: List[str] = [] + omitted: List[str] = [] + + for i, diff in enumerate(changed): + body = _truncated_body_lines(diff.body_lines(), max_lines_per_library, diff.library) + # Heading outside the fence; only +/-/~ (and optional truncation note) inside. + section = "\n".join( + [ + f"## {diff.library}", + "", + f"entries: {diff.old_count} -> {diff.new_count}; " + f"+{len(diff.added)} -{len(diff.removed)} ~{len(diff.changed)}", + "", + "```diff", + *body, + "```", + "", + ] + ) + remaining = [d.library for d in changed[i + 1 :]] + footer = _omission_footer(remaining) + candidate_text = "\n".join([*sections, section]).rstrip() + "\n" + footer + if len(candidate_text) > max_chars: + if sections: + omitted = [d.library for d in changed[i:]] + break + # First section alone may still exceed the cap; hard-cut it. + section_budget = max_chars - len(footer) if footer else max_chars + cut = _clip_to_max_chars(section, max(1, section_budget)) + sections.append(cut if cut.endswith("\n") else cut + "\n") + omitted = list(remaining) + break + sections.append(section) + + text = "\n".join(sections).rstrip() + "\n" + if omitted: + text += _omission_footer(omitted) + return _clip_to_max_chars(text, max_chars) + + +def write_library_database( + metrics: LibraryMetrics, + entries: List[dict], + output_dir: pathlib.Path, + converter: Converter, +) -> LibraryMetrics: + """Write the per-library JSONL.gz and update metrics. Returns metrics.""" + output_path = output_dir / f"{metrics.library}.jsonl.gz" + + if not entries: + if output_path.exists(): + output_path.unlink() + logger.info("%s: removed empty database %s", metrics.library, output_path) + metrics.num_string_entries = 0 + metrics.num_function_name_entries = 0 + metrics.total_entries = 0 + return metrics + + counts = converter.write(entries, output_path) + metrics.num_string_entries = counts["num_string_entries"] + metrics.num_function_name_entries = counts["num_function_name_entries"] + metrics.total_entries = counts["total_entries"] + logger.info( + "%s: wrote %s (%d entries)", + metrics.library, + output_path, + metrics.total_entries, + ) + return metrics + + +def run_build( + config: BuildConfig, + vcpkg: Vcpkg, + jh: JHExtractor, + converter: Converter, + build_library_fn: Callable[ + [str, BuildConfig, Vcpkg, JHExtractor, Converter], Tuple[LibraryMetrics, List[dict]] + ] = build_library, +) -> int: + """Build and write databases for the configured libraries. + + This is split out from `main()` so tests can inject fakes directly without + monkeypatching module globals. + """ + config.output_dir.mkdir(parents=True, exist_ok=True) + + metrics: List[LibraryMetrics] = [] + per_library_new: Dict[str, List[dict]] = {} + failed = False + for library in config.libraries: + metric, entries = build_library_fn(library, config, vcpkg, jh, converter) + metrics.append(metric) + # Even on error, preserve any partial entries so they aren't lost. + per_library_new[library] = entries + if metric.error: + failed = True + if not config.continue_on_error: + break + + # Discover all existing .jsonl.gz databases in the output directory. These + # include libraries we are rebuilding (whose fresh entries will be merged + # in) and libraries we are leaving alone. Strings are NOT deduped across + # libraries: a string that appears in both zlib and curl (e.g. when zlib + # is vendored into curl) stays in both databases. The query tagger already + # emits one #library tag per matching database, so the consumer can see + # the overlap directly. + existing_files = sorted(config.output_dir.glob("*.jsonl.gz")) + + def _lib_name_from_path(p: pathlib.Path) -> str: + # p.name looks like ".jsonl.gz"; Path.stem would only strip ".gz". + suffix = ".jsonl.gz" + if p.name.endswith(suffix): + return p.name[: -len(suffix)] + return p.stem + + # Snapshot pre-merge contents for rebuilt libraries so we can emit a + # human-readable text diff after writing (gzipped JSONL is opaque in PRs). + existing_by_lib: Dict[str, List[dict]] = {} + merged: Dict[str, List[dict]] = {} + for path in existing_files: + lib = _lib_name_from_path(path) + existing = load_existing_entries(path) + existing_by_lib[lib] = existing + if lib in per_library_new: + merged[lib] = merge_entries(per_library_new[lib], existing, config.deduplicate) + elif existing: + merged[lib] = existing + + # Libraries being built for the first time (no pre-existing file). + for lib, new_entries in per_library_new.items(): + if lib not in merged: + merged[lib] = list(new_entries) + + # Write rebuilt libraries, update their metrics, and collect entry diffs. + library_diffs: List[LibraryDiff] = [] + for metric in metrics: + if metric.error: + logger.warning("%s: skipping database write due to build error", metric.library) + continue + entries = merged.get(metric.library, []) + old_entries = existing_by_lib.get(metric.library, []) + library_diffs.append(diff_library_entries(metric.library, old_entries, entries)) + write_library_database(metric, entries, config.output_dir, converter) + + summary = { + "triplet": config.triplet, + "compiler": config.compiler, + "profile": config.profile, + "libraries": [m.as_dict() for m in metrics], + "successful": sum(1 for m in metrics if not m.error), + "failed": sum(1 for m in metrics if m.error), + } + + metrics_path = config.output_dir / "build_metrics.json" + metrics_path.write_text(json.dumps(summary, indent=2)) + logger.info("wrote metrics to %s", metrics_path) + + # Capped CI log summary (up to 100 change lines per library). + diff_text = format_build_diff(library_diffs, max_lines_per_library=DIFF_MAX_LINES_PER_LIBRARY) + diff_path = config.output_dir / "build_diff.txt" + diff_path.write_text(diff_text, encoding="utf-8") + logger.info("wrote entry-level diff to %s", diff_path) + + # Markdown report for the GitHub PR description: ## heading per library, + # each with its own ```diff fence (20 change lines per library). + pr_diff_text = format_build_diff_markdown(library_diffs, max_lines_per_library=DIFF_PR_MAX_LINES_PER_LIBRARY) + pr_diff_path = config.output_dir / "build_diff_pr.txt" + pr_diff_path.write_text(pr_diff_text, encoding="utf-8") + logger.info("wrote PR entry-level diff to %s", pr_diff_path) + + # Log a short preview so local/CI logs also surface the change. + for line in pr_diff_text.splitlines()[:30]: + logger.info("diff: %s", line) + if pr_diff_text.count("\n") > 30: + logger.info("diff: ... (see %s / %s for full reports)", diff_path, pr_diff_path) + + if failed: + successful = sum(1 for m in metrics if not m.error) + if config.continue_on_error and successful > 0: + # Partial success: --continue-on-error let us build some libraries. + # The CI will pick up the updated databases and a follow-up run can + # retry the failed ones. + logger.warning( + "%d/%d libraries failed; exiting 0 because --continue-on-error is set", + failed, + len(metrics), + ) + return 0 + # Either --continue-on-error was not set, or every library failed. In + # the latter case we cannot let the workflow step go green: nothing was + # produced, so a missing build (broken vcpkg, wrong jh path, etc.) + # would be silent. + logger.error( + "one or more libraries failed to build (failed=%d, total=%d)", + failed, + len(metrics), + ) + return 1 + return 0 + + +def load_config(path: pathlib.Path) -> dict: + """Load build configuration from a JSON file.""" + data = json.loads(path.read_text()) + if not isinstance(data, dict): + raise ValueError(f"config file {path} must contain a JSON object") + return data + + +def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + # First pass: figure out if a config file was provided. + pre_parser = argparse.ArgumentParser(add_help=False) + pre_parser.add_argument("--config", type=pathlib.Path, default=None) + pre_args, _ = pre_parser.parse_known_args(argv) + + config: dict = {} + if pre_args.config: + config = load_config(pre_args.config) + elif "CONFIG" in os.environ: + config = load_config(pathlib.Path(os.environ["CONFIG"])) + + # Defaults are taken from (lowest to highest precedence): + # built-in constants < config file < environment variables < CLI args + defaults = { + "triplet": config.get("triplet"), + "compiler": config.get("compiler"), + "profile": config.get("profile"), + "libraries": config.get("libraries"), + } + + parser = argparse.ArgumentParser( + description="Build OSS string databases from vcpkg libraries.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + parents=[pre_parser], + ) + parser.set_defaults(**defaults) + parser.add_argument( + "--triplet", + help="vcpkg triplet", + ) + parser.add_argument( + "--compiler", + help="compiler label passed to jh", + ) + parser.add_argument( + "--profile", + help="build profile label passed to jh", + ) + parser.add_argument( + "--libraries", + nargs="+", + help="libraries to build", + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=data_root() / "oss", + help="directory for generated .jsonl.gz files and metrics", + ) + parser.add_argument( + "--vcpkg-root", + type=pathlib.Path, + default=os.environ.get("VCPKG_ROOT", None), + help="vcpkg installation root", + ) + parser.add_argument( + "--jh-path", + type=pathlib.Path, + default=os.environ.get("JH_PATH", None), + help="path to an existing jh binary", + ) + parser.add_argument( + "--lancelot-dir", + type=pathlib.Path, + default=os.environ.get("LANCELOT_DIR", None), + help="directory containing lancelot source; jh will be built if --jh-path is not given", + ) + parser.add_argument( + "--no-function-names", + action="store_true", + help="do not emit function-name-as-string entries", + ) + parser.add_argument( + "--no-deduplicate", + action="store_true", + help="emit one JSON object per JSONL row instead of one per unique string", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help="continue building remaining libraries if one fails and exit successfully", + ) + parser.add_argument( + "--log-level", + default=os.environ.get("LOG_LEVEL", "INFO"), + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + help="logging level", + ) + return parser.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> int: + args = parse_args(argv) + logging.getLogger().setLevel(args.log_level.upper()) + + config = BuildConfig( + triplet=args.triplet, + compiler=args.compiler, + profile=args.profile, + libraries=[lib.strip() for lib in args.libraries if lib.strip()], + output_dir=args.output_dir.resolve(), + vcpkg_root=args.vcpkg_root.resolve() if args.vcpkg_root else None, + jh_path=args.jh_path.resolve() if args.jh_path else None, + lancelot_dir=args.lancelot_dir.resolve() if args.lancelot_dir else None, + emit_function_names=not args.no_function_names, + deduplicate=not args.no_deduplicate, + continue_on_error=args.continue_on_error, + ) + + logger.info("configuration: %s", config) + + vcpkg = Vcpkg(config.vcpkg_root) + jh = JHExtractor(config.jh_path, config.lancelot_dir) + converter = Converter( + emit_function_names=config.emit_function_names, + deduplicate=config.deduplicate, + ) + + return run_build(config, vcpkg, jh, converter) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tags/extract_strings.py b/scripts/tags/extract_strings.py new file mode 100644 index 000000000..9bd01da74 --- /dev/null +++ b/scripts/tags/extract_strings.py @@ -0,0 +1,280 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# examples: +# $ extract_strings.py -d --pes C:\Windows cwinpes + +import os +import sys +import json +import hashlib +import logging +import argparse +import datetime +import collections +import dataclasses +from typing import List, Tuple, Mapping +from collections.abc import Iterator + +import dnfile +import pefile + +import floss.strings +from floss.tags.gp import Encoding, Location + +MIN_LEN = 6 +MAX_LEN_PES = 100 +MAX_LEN_LIBS = 64 # TODO check, but these tend to contain long strings, focus on actual string data via better parsing + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class FileString: + offset: int + string: str + encoding: Encoding + location: Location + + +@dataclasses.dataclass +class PeStrings: + path: str + sha256: str + timestamp: str + dotnet: bool + strings: List[FileString] + + +def match(path: str, suffixes: Tuple[str, ...], prefixes: Tuple[str, ...]): + if suffixes and path.endswith(suffixes): + return True + elif prefixes and path.startswith(prefixes): + return True + return False + + +def find_file_paths(path: str, suffixes: Tuple[str, ...] = (), prefixes: Tuple[str, ...] = ()) -> Iterator[str]: + if not os.path.exists(path): + raise IOError(f"path {path} does not exist or cannot be accessed") + + if os.path.isfile(path): + if match(path, suffixes, prefixes): + yield path + elif os.path.isdir(path): + logger.debug("searching directory %s", os.path.abspath(os.path.normpath(path))) + for root, dirs, files in os.walk(path): + if root.startswith((r"C:\Windows\WinSxS",)): # can be large, stores install/backup related files + logger.debug("skip %s", root) + continue + + for file in files: + if match(file, suffixes, prefixes): + file_path = os.path.join(root, file) + logger.debug("found file: %s", os.path.abspath(os.path.normpath(file_path))) + yield file_path + + +# TODO adjust to new JSON format +def extract_libs(dir_path: str, outdir: str, min_len: int, max_len: int): + for file_path in find_file_paths(dir_path, suffixes=(".lib",)): + with open(file_path, "rb") as f: + binary_data = f.read() + + extracted_strings = floss.strings.extract_ascii_unicode_strings(binary_data, min_len) + filtered_strings = filter(lambda s: len(s.string) <= max_len, extracted_strings) + sorted_strings = sorted(filtered_strings, key=lambda s: (s.string, len(s.string))) + + outfile = os.path.join(outdir, f"{file_path.replace(os.sep, '--')}.json") + d: Mapping[str, Mapping[str, List[str]]] = collections.defaultdict(lambda: collections.defaultdict(list)) + for s in sorted_strings: + if s.string not in d[file_path][s.encoding]: + d[file_path][s.encoding.value].append(s.string) + with open(outfile, "w", encoding="utf-8") as f: + json.dump(d, f, indent=2) + + +def get_section(offset: int, sections: List): + sec = None + for sname, (low, high) in sections: + if low <= offset < high: + return sname + if sec is None: + raise ValueError(f"{offset} not in sections:\n {sections}") + + +def extract_pes(dir_path, outdir, min_len: int, max_len: int): + seen_hashes = set() + + for file_path in find_file_paths(dir_path, suffixes=(".exe", ".dll", ".sys", ".exe_", ".dll_", ".sys_")): + outfile = os.path.join(outdir, f"{os.path.basename(file_path)}.json") + if os.path.exists(outfile): + with open(outfile, "r", encoding="utf-8") as f: + existing_data = json.load(f) + if os.path.abspath(file_path) == existing_data["path"]: + logger.info("skipping file with existing data: %s", file_path) + continue + else: + # this doesn't work well for multiple extractions of the same data sources as data gets duplicated + # dedup is possible via the hashes though + # + # ignoring type so that the f can alias over the file handle above + f, ext = os.path.splitext(outfile) # type: ignore + outfile = f"{f}{str(datetime.datetime.now().timestamp()).replace('.', '')}{ext}" + if os.path.exists(outfile): + logger.warning("skipping file with existing data: %s", file_path) + logger.info("updating file name: %s", outfile) + + try: + with open(file_path, "rb") as f: + binary_data = f.read() + except PermissionError as e: + logger.warning("%s", e) + continue + + try: + pe = pefile.PE(data=binary_data) + except pefile.PEFormatError: + continue + + dnpe = dnfile.dnPE(data=binary_data) + sections = get_section_boundaries(pe, len(binary_data)) + + extracted_strings = floss.strings.extract_ascii_unicode_strings(binary_data, min_len) + filtered_strings = filter(lambda es: len(es.string) <= max_len, extracted_strings) + + if os.path.exists(outfile): + raise Exception(f"{outfile} already exists") + + sha256 = hashlib.sha256() + sha256.update(binary_data) + sha256_hash = sha256.hexdigest() + + if sha256_hash in seen_hashes: + logger.info("skipping file with sha256 hash %s: already analyzed", sha256_hash) + continue + else: + seen_hashes.add(sha256_hash) + + filestrings = [] + for s in filtered_strings: + encoding = s.encoding.value.lower() + assert isinstance(encoding, str) + assert encoding in ("ascii", "utf-16le", "unknown") + + filestrings.append( + FileString( + offset=s.offset, + string=s.string, + encoding=encoding, # type: ignore + location=get_section(s.offset, sections), + ) + ) + + pestrings = PeStrings( + path=os.path.abspath(os.path.normpath(file_path)), + sha256=sha256_hash, + timestamp=datetime.datetime.now().isoformat(), + dotnet=bool(dnpe.net), + strings=filestrings, + ) + + with open(outfile, "w", encoding="utf-8") as f: + json.dump(dataclasses.asdict(pestrings), f, indent=2) + + +def get_section_boundaries(pe: pefile.PE, file_size: int): + sections = [("header", (0, len(pe.header)))] + + for section in pe.sections: + try: + # TODO there must be a better way to deal with section names + name = section.Name.decode("utf-8").split("\x00")[0] + except UnicodeDecodeError: + name = section.Name[: section.Name.index(b"\x00")].decode("utf-8").rstrip("\x00") + logger.warning("weird section name: %s - using: %s", section.Name, name) + if section.Misc_PhysicalAddress and section.SizeOfRawData: + # section names may not be unique + sections.append((name, (section.PointerToRawData, section.PointerToRawData + section.SizeOfRawData))) + + if file_size > sections[-1][1][1]: + sections.append(("overlay", (sections[-1][1][1], file_size))) + + return sections + + +def main(): + parser = argparse.ArgumentParser(description="Extract raw strings from select files.") + parser.add_argument("path", help="file or path to analyze") + parser.add_argument("outdir", help="directory to store results to") + parser.add_argument( + "--libs", + action="store_true", + help=r"recursively search and extract string from .lib files under path, e.g., C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.35.32215\crt\src\i386", + ) + parser.add_argument( + "--pes", + action="store_true", + help="recursively search and extract string from PE files under path, e.g., C:\\Windows", + ) + parser.add_argument("--min-len", type=int, default=MIN_LEN, help="minimum string length") + parser.add_argument("--max-len", type=int, default=-1, help="maximum string length") + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output except fatal errors" + ) + args = parser.parse_args() + + if args.quiet: + logging.basicConfig(level=logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) + elif args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) + + # ignore WARNING:dnfile.utils:invalid compressed int: leading byte: 0xec + logging.getLogger("dnfile.utils").setLevel(logging.CRITICAL) + + if os.path.exists(args.outdir): + logger.error("%s already exists", args.outdir) + use = input("use existing dir? y/[n] ") + if use != "y": + return -1 + else: + os.mkdir(args.outdir) + + max_len = args.max_len + if max_len == -1: + if args.libs: + max_len = MAX_LEN_LIBS + elif args.pes: + max_len = MAX_LEN_PES + else: + raise ValueError("unknown extraction type") + + if args.libs: + extract_libs(args.path, args.outdir, args.min_len, max_len) + elif args.pes: + extract_pes(args.path, args.outdir, args.min_len, max_len) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tags/fetch_vt_hashes.py b/scripts/tags/fetch_vt_hashes.py new file mode 100644 index 000000000..f8348190c --- /dev/null +++ b/scripts/tags/fetch_vt_hashes.py @@ -0,0 +1,307 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +fetches hashes of relevent file types published by VT in the given time range. +example: + + $ python fetch_vt_hashes.py 202305010000 202305020000 + INFO:__main__:fetching feed: 2023-05-01T00:00:00 + 3bf67578d120ecc7710e56781e2f9a2fa14b94bbcd45d7ae0aa82098d327e4d7 + 2a2c83bfd4b2e73e452365b972f2e479a6476f8ba6db2eb44d166bb9dfb1d3bd + 6ac76d0fc8bfe86f97301824b8631827aa77ef951d48cfc2c60b648098571fe5 + d27979456d897b1a8ebda208cb413004e48c4894ebabd934d89f6bfaeca2ae25 + 12dd4c2e26a7e686d952addfde5eb8c1ccce99bdd3af2a7661c4368a2b2526c4 + 33831bd454dffa78b850cd4d24903d968d36aaf2141a2c6eb8f29fc7cecd35d9 + 5b641dec81aec1cf7ac0cce9fc067bb642fbd32da138a36e3bdac3bb5b36c37a + b801b4fcb18b341f1d64f89aa631731be22511067bf5b5197e24359ce471184a + f636851dc4ea34e0defae4d54aeb42af8d48d473aced48c4fc6a757db762bc00 + 5fb46dabf5d4e418eaba2b0ccaa8fdf138c4af8502b6b1ba37aef1f97035da90 + 047af1acd89d8d58a60f1894a7bd80184d6954b0407e2f3443eae83ad177945c + ... + INFO:__main__:fetching feed: 2023-05-01T00:01:00 + ... + INFO:__main__:fetching feed: 2023-05-01T00:02:00 + ... + +dependencies: + + virustotal3==1.0.8 + +selected mime types from one hour of VT data: + + 31798 PE32 executable for MS Windows (GUI) Intel 80386 32-bit + 4749 PE32 executable for MS Windows (DLL) (GUI) Intel 80386 32-bit + 3319 PE32 executable for MS Windows (console) Intel 80386 32-bit + 2125 PE32+ executable for MS Windows (GUI) Mono/.Net assembly + 1346 ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, stripped + 1128 MS-DOS executable, MZ for MS-DOS + 1124 PE32+ executable for MS Windows (console) Mono/.Net assembly + 1000 PE32+ executable for MS Windows (DLL) (GUI) Mono/.Net assembly + 943 PE32+ executable for MS Windows (console) + 901 PE32 executable for MS Windows (DLL) (console) Intel 80386 32-bit + 874 PE32 executable for MS Windows (GUI) Intel 80386 32-bit Mono/.Net assembly + 649 PE32+ executable for MS Windows (DLL) (GUI) + 604 PE32+ executable for MS Windows (DLL) (console) + 528 PE32+ executable for MS Windows (DLL) (console) Mono/.Net assembly + 451 PE32 executable for MS Windows (GUI) Intel 80386 Mono/.Net assembly + 406 PE32 executable for MS Windows (DLL) (console) Intel 80386 32-bit Mono/.Net assembly + 300 PE32+ executable for MS Windows (GUI) + 285 ELF 64-bit LSB shared object, version 1 (SYSV), dynamically linked, stripped + 280 PE32 executable for MS Windows (DLL) (console) Intel 80386 Mono/.Net assembly + 271 ELF 32-bit LSB shared object, ARM, version 1 (SYSV), dynamically linked, stripped + 233 PE32 executable for MS Windows (console) Intel 80386 32-bit Mono/.Net assembly + 199 Mach-O 64-bit dynamically linked shared library + 191 ELF 32-bit LSB shared object, Intel 80386, version 1 (SYSV), dynamically linked, stripped + 143 PE32 executable for MS Windows (console) Intel 80386 Mono/.Net assembly + 115 MS-DOS executable + 104 COM executable for DOS + 96 PE32 executable for MS Windows (unknown subsystem) unknown processor 32-bit + 83 ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, stripped + 76 Mach-O 64-bit bundle + 66 PE32 executable for MS Windows (native) Intel 80386 32-bit + 66 Mach-O fat file with 2 architectures + 61 PE32+ executable for MS Windows (native) Mono/.Net assembly + 61 Mach-O 64-bit executable + 50 Mach-O 64-bit filetype=10 + 38 PE32 executable for MS Windows (DLL) Intel 80386 32-bit + 35 ELF 32-bit LSB shared object, ARM, version 1 (SYSV), dynamically linked (uses shared libs), stripped +""" + +import io +import os +import bz2 +import sys +import json +import shelve +import hashlib +import logging +import pathlib +import argparse +import datetime +from typing import Any, List, Iterator + +import requests +import virustotal3.errors + +logger = logging.getLogger(__name__) + + +API_KEY = os.environ["VT_API_KEY"] + + +# TypeAlias. note: using `foo: TypeAlias = bar` is Python 3.10+ +CacheIdentifier = str + + +def get_this_file_hash() -> str: + hash = hashlib.sha256() + hash.update(pathlib.Path(__file__).read_bytes()) + return hash.hexdigest() + + +def compute_cache_identifier(*keys: bytes) -> CacheIdentifier: + hash = hashlib.sha256() + + # so that if we change this file the cache is invalidated. + hash.update(get_this_file_hash().encode("ascii")) + hash.update(b"\x00") + + for key in keys: + hash.update(key) + hash.update(b"\x00") + + return hash.hexdigest() + + +def get_default_cache_directory(app="floss") -> str: + # ref: https://github.com/mandiant/capa/issues/1212#issuecomment-1361259813 + # + # Linux: $XDG_CACHE_HOME/floss/ + # Windows: %LOCALAPPDATA%\flare\floss\cache + # MacOS: ~/Library/Caches/floss + + # ref: https://stackoverflow.com/a/8220141/87207 + if sys.platform == "linux" or sys.platform == "linux2": + directory = os.environ.get("XDG_CACHE_HOME", os.path.join(os.environ["HOME"], ".cache", app)) + elif sys.platform == "darwin": + directory = os.path.join(os.environ["HOME"], "Library", "Caches", app) + elif sys.platform == "win32": + directory = os.path.join(os.environ["LOCALAPPDATA"], "flare", "capa", app) + else: + raise NotImplementedError(f"unsupported platform: {sys.platform}") + + os.makedirs(directory, exist_ok=True) + + return directory + + +def format_timestamp(dt: datetime.datetime) -> str: + return f"{dt.year}{dt.month:02}{dt.day:02}{dt.hour:02}{dt.minute:02}" + + +VirusTotalApiError = virustotal3.errors.VirusTotalApiError + + +def _raise_exception(response): + """Raise Exception + + Function to raise an exception using the error messages returned by the API. + + Parameters: + response (dict) Reponse containing the error returned by the API. + + vendored from: https://github.com/traceflow/virustotal3/blob/58dcfab/virustotal3/enterprise.py + """ + # https://developers.virustotal.com/v3.0/reference#errors + raise VirusTotalApiError(response.text) + + +def _get_feed(api_key, type_, time, timeout=None): + """Get a minute from a feed + + Parameters: + api_key (str): VT key + type_ (str): type of feed to get + time (str): YYYYMMDDhhmm + timeout (float, optional): The amount of time in seconds the request should wait before timing out. + + Returns: + BytesIO: each line is a json string for one report + + vendored from: https://github.com/traceflow/virustotal3/blob/58dcfab/virustotal3/enterprise.py + """ + if api_key is None: + raise Exception("You must provide a valid API key") + + response = requests.get( + "https://www.virustotal.com/api/v3/feeds/{}/{}".format(type_, time), + headers={"x-apikey": api_key, "Content-Type": "application/json"}, + timeout=timeout, + ) + + if response.status_code != 200: + _raise_exception(response) + + return io.BytesIO(bz2.decompress(response.content)) + + +def file_feed(api_key, time, timeout=None): + """Get a file feed batch for a given date, by the minute. + + From the official documentation: + "Time 201912010802 will return the batch corresponding to December 1st, 2019 08:02 UTC. + You can download batches up to 7 days old, and the most recent batch has always a 60 minutes + lag with respect with to the current time." + + Parameters: + api_key (str): VirusTotal key + time (str): YYYYMMDDhhmm + timeout (float, optional): The amount of time in seconds the request should wait before timing out. + + Returns: + BytesIO: each line is a json string for one report + + vendored from: https://github.com/traceflow/virustotal3/blob/58dcfab/virustotal3/enterprise.py + """ + return _get_feed(api_key, "files", time, timeout=timeout) + + +def fetch_feed(api_key: str, ts: datetime.datetime) -> Iterator[Any]: + feed = file_feed(api_key, format_timestamp(ts)).read().decode("utf-8") + for line in feed.split("\n"): + if not line: + continue + yield json.loads(line) + + +def fetch_feed_hashes(api_key: str, ts: datetime.datetime) -> List[str]: + ts_key = format_timestamp(ts) + + dir = pathlib.Path(get_default_cache_directory()) + name = compute_cache_identifier() + ".db" + + p = dir / name + + with shelve.open(str(p)) as db: + if ts_key not in db: + try: + hashes = [] + for line in fetch_feed(API_KEY, ts): + try: + if line.get("type") != "file": + continue + + if "magic" not in line.get("attributes", {}) or "sha256" not in line.get("attributes", {}): + continue + + magic = line["attributes"]["magic"] + if any( + map(lambda prefix: magic.startswith(prefix), ["PE32", "ELF", "MS-DOS", "Mach-O", "COM"]) + ): + hashes.append(line["attributes"]["sha256"]) + except Exception as e: + logger.warning("error: %s", str(e), exc_info=True) + continue + + db[ts_key] = hashes + except Exception as e: + logger.warning("error: %s", str(e), exc_info=True) + return [] + + return db[ts_key] + + +def main(): + parser = argparse.ArgumentParser( + description="fetch the hashes of PE files from the VT feed for the given time range." + ) + parser.add_argument("start", help="timestamp to start, YYYYMMDDhhmm") + parser.add_argument("end", help="timestamp to start, YYYYMMDDhhmm") + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output except fatal errors" + ) + args = parser.parse_args() + + if args.quiet: + logging.basicConfig(level=logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) + elif args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) + + start = datetime.datetime.strptime(args.start, "%Y%m%d%H%M") + end = datetime.datetime.strptime(args.end, "%Y%m%d%H%M") + + logger.info("fetching feed: %s - %s", start.isoformat(), end.isoformat()) + + current = start + while format_timestamp(current) < format_timestamp(end): + logger.info("fetching feed: %s", current.isoformat()) + for hash in fetch_feed_hashes(API_KEY, current): + print(hash) + + current += datetime.timedelta(minutes=1) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tags/generate_gp_db.py b/scripts/tags/generate_gp_db.py new file mode 100644 index 000000000..db6b320e3 --- /dev/null +++ b/scripts/tags/generate_gp_db.py @@ -0,0 +1,132 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# examples: +# $ generate_gp_db.py cwinpes cwindb-native.jsonl.gz --type native +# scanned 24,212 files with 43,918,395 strings +# final db contains 3,631 strings (more than 500 occurrences) +# +# $ generate_gp_db.py cwinpes cwindb-dotnet.jsonl.gz --type dotnet +# scanned 24,212 files with 24,767,670 strings +# final db contains 1,683 strings (more than 500 occurrences) + +import os +import sys +import json +import logging +import argparse +import collections +from typing import Dict, Tuple +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from floss.tags.gp import Encoding, Location, StringGlobalPrevalence, StringGlobalPrevalenceDatabase +from scripts.tags.extract_strings import PeStrings + +MIN_COUNT = 500 + + +logger = logging.getLogger(__name__) + + +def generate_gp_db(path: str, min_count: int, type_: str) -> "StringGlobalPrevalenceDatabase": + if not os.path.exists(path): + raise IOError(f"path {path} does not exist or cannot be accessed") + if not os.path.isdir(path): + raise IOError(f"path {path} is not a directory") + + db: Dict[Tuple[str, Encoding, Location], int] = collections.defaultdict(int) + seen_hashes = set() + nfiles = 0 + nstrings = 0 + for root, dirs, files in os.walk(path): + for file in files: + nfiles += 1 + file_path = os.path.join(root, file) + logger.debug("found file: %s", os.path.abspath(os.path.normpath(file_path))) + + with open(file_path, "r", encoding="utf-8") as f: + d = json.load(f) + pestrings = PeStrings(**d) + + dotnative = "dotnet" if pestrings.dotnet else "native" + if type_ != "all" and dotnative != type_: + logger.debug("skipping unwanted type %s: %s", dotnative, file_path) + continue + + if pestrings.sha256 in seen_hashes: + logger.debug("skipping already indexed file with sha256 hash %s: %s", pestrings.sha256, file_path) + seen_hashes.add(pestrings.sha256) + + nstrings += len(pestrings.strings) + for s in pestrings.strings: + db[(s.string, s.encoding, s.location)] += 1 + + print(f"scanned {nfiles:,} files with {nstrings:,} strings") + + gpdb = StringGlobalPrevalenceDatabase.new_db() + for (string, encoding, location), n in db.items(): + if n < min_count: + continue + gpdb.insert(StringGlobalPrevalence(string, encoding, n, location)) + + print(f"final db contains {len(gpdb):,} strings (more than {MIN_COUNT} occurrences)") + return gpdb + + +def main(): + parser = argparse.ArgumentParser(description="Generate global prevalence database from raw files.") + parser.add_argument("path", help="path containing extracted string data") + parser.add_argument("outfile", help="file to store results to") + parser.add_argument( + "--type", choices=("dotnet", "native", "all"), default="all", help="include strings from dotnet, native or all" + ) + parser.add_argument("--min-count", type=int, default=MIN_COUNT, help="minimum count string needs to occur") + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output except fatal errors" + ) + args = parser.parse_args() + + if args.quiet: + logging.basicConfig(level=logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) + elif args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) + + if os.path.exists(args.outfile): + logger.error("%s already exists", args.outfile) + use = input("overwrite existing file? y/[n] ") + if use != "y": + return -1 + + gp = generate_gp_db(args.path, args.min_count, args.type) + + compress = args.outfile.endswith(".gz") + gp.to_file(args.outfile, compress=compress) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tags/query_string.py b/scripts/tags/query_string.py new file mode 100644 index 000000000..88f7bb6e6 --- /dev/null +++ b/scripts/tags/query_string.py @@ -0,0 +1,64 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import logging +import argparse + +from floss.tags import data_root +from floss.tags.gp import StringGlobalPrevalence, StringGlobalPrevalenceDatabase + +logger = logging.getLogger(__name__) + + +def load_db_gp(): + gpfile = data_root() / "gp" / "gp.jsonl.gz" + compress = gpfile.suffix == ".gz" + return StringGlobalPrevalenceDatabase.from_file(gpfile, compress=compress) + + +def query_string(string) -> StringGlobalPrevalence: + gpdb = load_db_gp() + return gpdb.query(string) + + +def main(): + parser = argparse.ArgumentParser(description="Query string databases.") + parser.add_argument("string", help="string to query for") + + logging_group = parser.add_argument_group("logging arguments") + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output except fatal errors" + ) + args = parser.parse_args() + + if args.quiet: + logging.basicConfig(level=logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) + elif args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) + + result = query_string(args.string) + print(result) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 84ed5eba4..bed2ff2eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,10 +20,7 @@ import pytest import viv_utils -import floss.main as floss_main -import floss.stackstrings as stackstrings -import floss.tightstrings as tightstrings -import floss.string_decoder as string_decoder +import floss.cache from floss.const import MIN_STRING_LENGTH from floss.identify import ( get_function_fvas, @@ -32,12 +29,26 @@ find_decoding_function_features, get_functions_without_tightloops, ) +from floss.pipeline import select_functions + + +@pytest.fixture(autouse=True) +def _isolate_analysis_cache(tmp_path, monkeypatch): + """point the analysis cache at a throwaway directory per test so tests never + touch the real platform cache, and make caching deterministic.""" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(tmp_path / "floss-cache")) + monkeypatch.setenv(floss.cache.ENV_CACHE_ENABLE, "1") def extract_strings(vw): """ Deobfuscate strings from vivisect workspace """ + # import here to avoid circular import with floss.features (identify <-> features) + import floss.stackstrings as stackstrings + import floss.tightstrings as tightstrings + import floss.string_decoder as string_decoder + top_functions, decoding_function_features = identify_decoding_functions(vw) for s_decoded in string_decoder.decode_strings( @@ -57,7 +68,7 @@ def extract_strings(vw): def identify_decoding_functions(vw): - selected_functions = floss_main.select_functions(vw, None) + selected_functions = select_functions(vw, None) decoding_function_features, _ = find_decoding_function_features(vw, selected_functions, disable_progress=True) top_functions = get_top_functions(decoding_function_features, 20) return top_functions, decoding_function_features @@ -98,7 +109,12 @@ def collect(self): filepath = test_dir / filename if filepath.exists(): yield FLOSSTest.from_parent( - self, path=str(filepath), platform=platform, arch=arch, filename=filename, spec=spec + self, + path=str(filepath), + platform=platform, + arch=arch, + filename=filename, + spec=spec, ) diff --git a/tests/data b/tests/data index 53e910192..6eba5b19c 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 53e910192ea6f3f4c825370389393bdd9631580c +Subproject commit 6eba5b19cc8034b7c3128398618366632f84680b diff --git a/tests/test_build_oss_db.py b/tests/test_build_oss_db.py new file mode 100644 index 000000000..7261e23af --- /dev/null +++ b/tests/test_build_oss_db.py @@ -0,0 +1,855 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import logging +import pathlib + +import scripts.tags.build_oss_db as build_oss_db +from floss.tags import data_root + +SAMPLE_DB_PATH = data_root() / "oss" + + +def _capture_warnings(logger_name: str) -> list: + """Attach a recording handler at WARNING level to the named logger. + + Returns a list that the caller can read after invoking the system under + test. Each call installs an independent handler with its own list, so + tests do not interfere with each other. + """ + records: list = [] + handler = logging.Handler(level=logging.WARNING) + + def emit(record): + records.append(record) + + handler.emit = emit # type: ignore[assignment] + logger = logging.getLogger(logger_name) + logger.addHandler(handler) + logger.setLevel(logging.WARNING) + return records + + +def _row(path, function, feat_type, value): + return json.dumps({"path": path, "function": function, "type": feat_type, "value": value}) + + +# --------------------------------------------------------------------------- +# make_db_entry +# --------------------------------------------------------------------------- + + +def test_make_db_entry_uses_standard_schema(): + e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn") + assert e == { + "string": "s", + "library_name": "lib", + "library_version": "1.0", + "file_path": "f.c", + "function_name": "fn", + "line_number": None, + } + + +def test_make_db_entry_line_number_default_is_none(): + e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn") + assert e["line_number"] is None + + +def test_make_db_entry_preserves_explicit_line_number(): + e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn", 42) + assert e["line_number"] == 42 + + +# --------------------------------------------------------------------------- +# Converter.parse: structure +# --------------------------------------------------------------------------- + + +def test_parse_returns_parse_result_with_expected_fields(): + jh = _row("a.c", "foo", "string", "hello") + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert isinstance(result, build_oss_db.ParseResult) + assert hasattr(result, "entries") + assert hasattr(result, "num_objects") + assert hasattr(result, "num_functions") + + +# --------------------------------------------------------------------------- +# Converter.parse: single-pass counts +# --------------------------------------------------------------------------- + + +def test_parse_counts_unique_objects_and_functions(): + # Three rows: two paths, two functions, with one duplicate row. + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("a.c", "foo", "string", "world"), + _row("b.c", "bar", "string", "baz"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert result.num_objects == 2 + assert result.num_functions == 2 + + +def test_parse_counts_include_duplicate_rows(): + # Same row repeated should not inflate the unique counts. + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("a.c", "foo", "string", "hello"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert result.num_objects == 1 + assert result.num_functions == 1 + + +def test_parse_counts_exclude_rows_without_function(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("b.c", None, "string", "ignored"), + _row("c.c", "", "string", "ignored"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert result.num_objects == 1 + assert result.num_functions == 1 + + +def test_parse_counts_none_path_counts_as_an_object(): + # Rows with `path == None` are still attributed to an "object" (a None path), + # matching the previous count_jsonl_rows behavior. + jh = _row(None, "foo", "string", "hello") + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert result.num_objects == 1 + assert result.num_functions == 1 + + +# --------------------------------------------------------------------------- +# Converter.parse: entry list +# --------------------------------------------------------------------------- + + +def test_parse_emits_string_and_synthetic_function_name_entries(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("b.c", "bar", "string", "baz"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + strings = {e["string"] for e in result.entries} + # Two real strings plus the two synthesized function-name entries. + assert strings == {"hello", "baz", "foo", "bar"} + # All entries carry the library/version passed in. + for e in result.entries: + assert e["library_name"] == "lib" + assert e["library_version"] == "1.0" + assert e["line_number"] is None + + +def test_parse_dedup_collapses_identical_strings_by_default(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("a.c", "foo", "string", "hello"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert sum(1 for e in result.entries if e["string"] == "hello") == 1 + + +def test_parse_dedup_false_keeps_all_rows(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("a.c", "foo", "string", "hello"), + ] + ) + converter = build_oss_db.Converter(deduplicate=False) + result = converter.parse(jh, "lib", "1.0") + # 2 string rows + 1 synthetic function-name row for "foo" + # (synthetic rows are deduped by their function_name, not affected by deduplicate). + assert len(result.entries) == 3 + assert sum(1 for e in result.entries if e["string"] == "hello") == 2 + + +def test_parse_dedup_false_keeps_all_function_name_rows(): + # Two distinct function_name rows on different functions, with deduplicate off, + # should all be retained (no dedup of the synthetic entries either). + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + _row("a.c", "bar", "string", "world"), + ] + ) + converter = build_oss_db.Converter(deduplicate=False) + result = converter.parse(jh, "lib", "1.0") + # 2 string rows + 2 synthetic function-name rows (foo, bar) + assert len(result.entries) == 4 + strings = sorted(e["string"] for e in result.entries) + assert strings == ["bar", "foo", "hello", "world"] + + +def test_parse_emit_function_names_false_skips_synthetic_entries(): + jh = _row("a.c", "foo", "string", "hello") + converter = build_oss_db.Converter(emit_function_names=False) + result = converter.parse(jh, "lib", "1.0") + assert len(result.entries) == 1 + assert result.entries[0]["string"] == "hello" + assert result.num_functions == 1 # counts are still tracked + + +def test_parse_explicit_function_name_row_is_not_duplicated(): + jh = _row("a.c", "foo", "function_name", "fn_x") + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + # The explicit function_name row should appear once, and "fn_x" should + # NOT be re-emitted as a synthetic entry. + fn_entries = [e for e in result.entries if e["string"] == "fn_x"] + assert len(fn_entries) == 1 + assert fn_entries[0]["function_name"] == "fn_x" + + +# --------------------------------------------------------------------------- +# Converter.parse: error tolerance +# --------------------------------------------------------------------------- + + +def test_parse_skips_malformed_jsonl_lines(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + "this is not json", + _row("b.c", "bar", "string", "world"), + ] + ) + records = _capture_warnings("build_oss_db") + converter = build_oss_db.Converter() + result = converter.parse(jh, "lib", "1.0") + # Counts still cover the well-formed rows. + assert result.num_objects == 2 + assert result.num_functions == 2 + # A warning was logged for the bad line. + assert any("malformed" in record.message.lower() for record in records) + + +def test_parse_skips_empty_lines(): + jh = "\n".join( + [ + _row("a.c", "foo", "string", "hello"), + "", + " ", + _row("b.c", "bar", "string", "world"), + ] + ) + result = build_oss_db.Converter().parse(jh, "lib", "1.0") + assert result.num_objects == 2 + assert result.num_functions == 2 + + +# --------------------------------------------------------------------------- +# load_existing_entries +# --------------------------------------------------------------------------- + + +def _write_gz_jsonl(path: pathlib.Path, rows) -> None: + """Write each row to a gzipped JSONL file, preserving its raw form. + + Pass dicts to be JSON-serialized, or pass pre-serialized strings to inject + malformed/non-JSON content for negative-path tests. + """ + with gzip.open(path, "wt", encoding="utf-8") as f: + for row in rows: + if isinstance(row, str): + f.write(row + "\n") + else: + f.write(json.dumps(row) + "\n") + + +def test_load_existing_entries_round_trips_make_db_entry(tmp_path): + path = tmp_path / "lib.jsonl.gz" + entries = [ + build_oss_db.make_db_entry("s1", "lib", "1.0", "f.c", "fn"), + build_oss_db.make_db_entry("s2", "lib", "1.0", None, None, 5), + ] + _write_gz_jsonl(path, entries) + assert build_oss_db.load_existing_entries(path) == entries + + +def test_load_existing_entries_missing_file_returns_empty(tmp_path): + path = tmp_path / "missing_lib.jsonl.gz" + assert build_oss_db.load_existing_entries(path) == [] + + +def test_load_existing_entries_skips_malformed_lines(tmp_path): + path = tmp_path / "lib.jsonl.gz" + _write_gz_jsonl( + path, + [ + build_oss_db.make_db_entry("s1", "lib", "1.0", "f.c", "fn"), + "this is not json", + build_oss_db.make_db_entry("s2", "lib", "1.0", "f.c", "fn"), + ], + ) + records = _capture_warnings("build_oss_db") + loaded = build_oss_db.load_existing_entries(path) + assert len(loaded) == 2 + assert [e["string"] for e in loaded] == ["s1", "s2"] + assert any("malformed" in record.message.lower() for record in records) + + +def test_load_existing_entries_skips_non_dict_and_missing_string(tmp_path): + path = tmp_path / "lib.jsonl.gz" + with gzip.open(path, "wt", encoding="utf-8") as f: + f.write(json.dumps([1, 2, 3]) + "\n") # non-dict + f.write(json.dumps({"library_name": "lib"}) + "\n") # missing "string" + f.write(json.dumps({"string": "ok", "library_name": "lib"}) + "\n") + loaded = build_oss_db.load_existing_entries(path) + assert len(loaded) == 1 + assert loaded[0]["string"] == "ok" + + +def test_load_existing_entries_ignores_unknown_keys_and_fills_missing(tmp_path): + path = tmp_path / "lib.jsonl.gz" + _write_gz_jsonl( + path, + [ + {"string": "s", "library_name": "l", "extra_key": "ignored"}, + ], + ) + loaded = build_oss_db.load_existing_entries(path) + assert loaded == [ + { + "string": "s", + "library_name": "l", + "library_version": None, + "file_path": None, + "function_name": None, + "line_number": None, + } + ] + + +def test_load_existing_entries_handles_empty_file(tmp_path): + path = tmp_path / "empty.jsonl.gz" + path.write_bytes(b"") + # Empty file is not valid gzip; load_existing_entries should warn and return []. + assert build_oss_db.load_existing_entries(path) == [] + + +# --------------------------------------------------------------------------- +# merge_entries +# --------------------------------------------------------------------------- + + +def _entry(string, library="lib", version="1.0", function_name="fn"): + return build_oss_db.make_db_entry(string, library, version, "f.c", function_name) + + +def test_merge_entries_new_wins_on_collision_when_dedup(): + new = [_entry("hello", version="2.0"), _entry("world", version="2.0")] + existing = [_entry("hello", version="1.0"), _entry("other", version="1.0")] + merged = build_oss_db.merge_entries(new, existing, deduplicate=True) + + by_string = {e["string"]: e for e in merged} + # New "hello" wins (version 2.0), "world" is new, "other" is from existing. + assert by_string["hello"]["library_version"] == "2.0" + assert by_string["world"]["library_version"] == "2.0" + assert by_string["other"]["library_version"] == "1.0" + assert len(merged) == 3 + + +def test_merge_entries_dedup_false_keeps_duplicates(): + new = [_entry("hello"), _entry("world")] + existing = [_entry("hello"), _entry("other")] + merged = build_oss_db.merge_entries(new, existing, deduplicate=False) + # All four rows preserved. Without dedup the function concatenates + # existing first, then new; the order is purely cosmetic (the loader + # indexes by string). + assert [e["string"] for e in merged] == ["hello", "other", "hello", "world"] + + +def test_merge_entries_both_empty_returns_empty(): + assert build_oss_db.merge_entries([], [], deduplicate=True) == [] + assert build_oss_db.merge_entries([], [], deduplicate=False) == [] + + +def test_merge_entries_only_new_returns_copy_of_new(): + new = [_entry("a"), _entry("b")] + result = build_oss_db.merge_entries(new, [], deduplicate=True) + assert result == new + assert result is not new # callers rely on a fresh list + + +def test_merge_entries_only_existing_returns_copy_of_existing(): + existing = [_entry("a"), _entry("b")] + result = build_oss_db.merge_entries([], existing, deduplicate=True) + assert result == existing + assert result is not existing + + +# --------------------------------------------------------------------------- +# main() orchestration +# --------------------------------------------------------------------------- + + +class _FakeVcpkg: + """Stand-in for Vcpkg: records the install calls but does nothing.""" + + def __init__(self, *args, **kwargs): + self.installed = [] + + def install(self, library, triplet): + self.installed.append((library, triplet)) + + def get_installed_version(self, library, triplet): + return "1.0#1" + + def find_package_libs(self, library, triplet): + return [] + + +class _FakeJH: + def __init__(self, *args, **kwargs): + pass + + +def _stub_build_library(library, config, vcpkg, jh, converter): + """Replacement for build_library that returns the library's name as its only entry.""" + metrics = build_oss_db.LibraryMetrics( + library=library, + version="1.0#1", + triplet=config.triplet, + num_objects=0, + num_functions=0, + num_raw_entries=1, + total_entries=1, + duration_seconds=0.0, + ) + entry = build_oss_db.make_db_entry( + f"hello-from-{library}", + library, + "1.0#1", + "f.c", + f"fn_{library}", + ) + return metrics, [entry] + + +def _make_config(output_dir, libraries, *, continue_on_error=False): + return build_oss_db.BuildConfig( + triplet="x64-windows-static", + compiler="msvc143", + profile="release", + libraries=list(libraries), + output_dir=output_dir, + vcpkg_root=None, + jh_path=None, + lancelot_dir=None, + emit_function_names=True, + deduplicate=True, + continue_on_error=continue_on_error, + ) + + +def _invoke_run_build( + output_dir, libraries, *, continue_on_error=False, existing=None, build_library_fn=_stub_build_library +): + """Invoke run_build() with build_library stubbed to return one entry per library. + + ``existing`` is a dict of {library_name: [entry, ...]} to write to the + output dir as if they were previously built. + """ + config = _make_config(output_dir, libraries, continue_on_error=continue_on_error) + + if existing: + for lib, entries in existing.items(): + path = output_dir / f"{lib}.jsonl.gz" + with gzip.open(path, "wt", encoding="utf-8") as f: + for e in entries: + f.write(json.dumps(e) + "\n") + + return build_oss_db.run_build( + config, + _FakeVcpkg(), # type: ignore[arg-type] + _FakeJH(), # type: ignore[arg-type] + build_oss_db.Converter(), + build_library_fn=build_library_fn, + ) + + +def _read_gz_jsonl(path: pathlib.Path): + with gzip.open(path, "rt", encoding="utf-8") as f: + return [json.loads(line) for line in f if line.strip()] + + +def test_main_writes_one_database_per_library(tmp_path): + rc = _invoke_run_build(tmp_path, ["zlib", "curl"]) + assert rc == 0 + for lib in ("zlib", "curl"): + entries = _read_gz_jsonl(tmp_path / f"{lib}.jsonl.gz") + assert len(entries) == 1 + assert entries[0]["string"] == f"hello-from-{lib}" + assert entries[0]["library_name"] == lib + + +def test_main_merges_existing_database_with_fresh_entries(tmp_path): + existing_entry = build_oss_db.make_db_entry("old-string", "zlib", "0.9#1", "f.c", "old_fn") + rc = _invoke_run_build( + tmp_path, + ["zlib"], + existing={"zlib": [existing_entry]}, + ) + assert rc == 0 + entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz") + strings = {e["string"] for e in entries} + # Old entry from disk is preserved; new entry from the stubbed build is added. + assert strings == {"old-string", "hello-from-zlib"} + + +def test_main_preserves_existing_libraries_not_in_current_run(tmp_path): + # Pre-existing database for a library we are NOT rebuilding this run. + preexisting = [build_oss_db.make_db_entry("preexisting", "other", "1.0", "f.c", "fn")] + rc = _invoke_run_build( + tmp_path, + ["zlib"], + existing={"other": preexisting}, + ) + assert rc == 0 + # "other" database was not rewritten; its content is unchanged. + entries = _read_gz_jsonl(tmp_path / "other.jsonl.gz") + assert entries == preexisting + # "zlib" was rebuilt. + zlib_entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz") + assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"} + + +def test_main_exits_zero_on_partial_success_with_continue_on_error(tmp_path): + def stub_partial(library, config, vcpkg, jh, converter): + if library == "broken": + metrics = build_oss_db.LibraryMetrics( + library=library, + version="unknown", + triplet=config.triplet, + error="boom", + ) + return metrics, [] + return _stub_build_library(library, config, vcpkg, jh, converter) + + rc = _invoke_run_build( + tmp_path, + ["zlib", "broken"], + continue_on_error=True, + build_library_fn=stub_partial, + ) + # At least one library succeeded, so the workflow should see a green step. + assert rc == 0 + # The successful library's database was still written. + zlib_entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz") + assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"} + + +def test_main_exits_nonzero_when_all_libraries_fail_with_continue_on_error(tmp_path): + def stub_all_fail(library, config, vcpkg, jh, converter): + metrics = build_oss_db.LibraryMetrics( + library=library, + version="unknown", + triplet=config.triplet, + error="boom", + ) + return metrics, [] + + rc = _invoke_run_build( + tmp_path, + ["broken1", "broken2"], + continue_on_error=True, + build_library_fn=stub_all_fail, + ) + # Everything failed, even with --continue-on-error: must exit non-zero so + # the CI step doesn't silently go green on a total pipeline failure. + assert rc == 1 + + +def test_main_exits_nonzero_on_any_failure_without_continue_on_error(tmp_path): + def stub_one_fail(library, config, vcpkg, jh, converter): + if library == "broken": + metrics = build_oss_db.LibraryMetrics( + library=library, + version="unknown", + triplet=config.triplet, + error="boom", + ) + return metrics, [] + return _stub_build_library(library, config, vcpkg, jh, converter) + + rc = _invoke_run_build( + tmp_path, + ["zlib", "broken"], + build_library_fn=stub_one_fail, + ) + assert rc == 1 + + +def test_main_writes_build_metrics_summary(tmp_path): + rc = _invoke_run_build(tmp_path, ["zlib", "curl"]) + assert rc == 0 + summary = json.loads((tmp_path / "build_metrics.json").read_text()) + assert summary["triplet"] == "x64-windows-static" + assert summary["compiler"] == "msvc143" + assert summary["profile"] == "release" + assert summary["successful"] == 2 + assert summary["failed"] == 0 + names = {m["library"] for m in summary["libraries"]} + assert names == {"zlib", "curl"} + + +def test_diff_library_entries_added_removed_and_changed(): + old = [ + build_oss_db.make_db_entry("keep", "zlib", "1.0", "a.c", "fn_a"), + build_oss_db.make_db_entry("gone", "zlib", "1.0", "b.c", "fn_b"), + build_oss_db.make_db_entry("meta", "zlib", "1.0", "c.c", "old_fn"), + ] + new = [ + build_oss_db.make_db_entry("keep", "zlib", "1.0", "a.c", "fn_a"), + build_oss_db.make_db_entry("fresh", "zlib", "2.0", "d.c", "fn_d"), + build_oss_db.make_db_entry("meta", "zlib", "2.0", "c.c", "new_fn"), + ] + diff = build_oss_db.diff_library_entries("zlib", old, new) + assert diff.library == "zlib" + assert diff.old_count == 3 + assert diff.new_count == 3 + assert {e["string"] for e in diff.added} == {"fresh"} + assert {e["string"] for e in diff.removed} == {"gone"} + assert len(diff.changed) == 1 + assert diff.changed[0][0]["string"] == "meta" + assert diff.changed[0][1]["function_name"] == "new_fn" + assert diff.has_changes + + +def test_diff_library_entries_no_changes(): + entries = [build_oss_db.make_db_entry("s", "zlib", "1.0", "f.c", "fn")] + diff = build_oss_db.diff_library_entries("zlib", entries, list(entries)) + assert not diff.has_changes + assert diff.added == [] + assert diff.removed == [] + assert diff.changed == [] + + +def test_format_build_diff_truncates_per_library(): + added = [build_oss_db.make_db_entry(f"s{i}", "zlib", "1.0", "f.c", "fn") for i in range(50)] + diff = build_oss_db.LibraryDiff( + library="zlib", + old_count=0, + new_count=50, + added=added, + removed=[], + changed=[], + ) + text = build_oss_db.format_build_diff([diff], max_lines_per_library=10) + lines = text.splitlines() + # Header for the library + 10 body lines + truncation notice. + assert any(line.startswith("## zlib") for line in lines) + body_lines = [line for line in lines if line.startswith(("+ ", "- ", "~ "))] + assert len(body_lines) == 10 + assert any("truncated" in line and "zlib" in line for line in lines) + assert "more line(s) omitted" in text + + +def test_format_build_diff_truncates_each_library_independently(): + text = build_oss_db.format_build_diff( + [_lib_diff("zlib", 30), _lib_diff("curl", 30)], + max_lines_per_library=5, + ) + for lib in ("zlib", "curl"): + body = [line for line in text.splitlines() if line.startswith("+ ") and lib in line] + # Body lines are "+ zlib-0" etc.; count lines for that library. + assert len(body) == 5 + assert f"omitted for {lib}" in text + + +def test_format_build_diff_markdown_reserves_actual_omission_footer_size(): + keep = build_oss_db.LibraryDiff( + library="keep", + old_count=0, + new_count=1, + added=[build_oss_db.make_db_entry("keep-string", "keep", "1.0", "f.c", "fn")], + removed=[], + changed=[], + ) + omitted_name = "lib-" + ("x" * 180) + omitted = build_oss_db.LibraryDiff( + library=omitted_name, + old_count=0, + new_count=1, + added=[build_oss_db.make_db_entry("omit-string", omitted_name, "1.0", "f.c", "fn")], + removed=[], + changed=[], + ) + keep_text = build_oss_db.format_build_diff_markdown([keep], max_lines_per_library=5, max_chars=10_000) + footer = build_oss_db._omission_footer([omitted_name]) + text = build_oss_db.format_build_diff_markdown( + [keep, omitted], + max_lines_per_library=5, + max_chars=len(keep_text) + len(footer), + ) + + assert len(text) <= len(keep_text) + len(footer) + assert "## keep" in text + assert omitted_name in text + assert "```diff" in text + assert text.count("```") % 2 == 0 + + +def _lib_diff(name: str, n: int) -> build_oss_db.LibraryDiff: + added = [build_oss_db.make_db_entry(f"{name}-{i}", name, "1.0", "f.c", "fn") for i in range(n)] + return build_oss_db.LibraryDiff( + library=name, + old_count=0, + new_count=n, + added=added, + removed=[], + changed=[], + ) + + +def test_format_build_diff_markdown_heads_outside_fences(): + text = build_oss_db.format_build_diff_markdown( + [_lib_diff("zlib", 25), _lib_diff("curl", 25)], + max_lines_per_library=5, + ) + # Headings are markdown, not inside the fenced blocks. + assert "## zlib" in text + assert "## curl" in text + assert text.count("```diff") == 2 + assert text.count("```") == 4 # open + close per library + + # Each ```diff ... ``` block should not contain a ## heading. + for part in text.split("```"): + if part.startswith("diff"): + assert "## " not in part + body_lines = [line for line in part.splitlines() if line.startswith(("+ ", "- ", "~ "))] + assert len(body_lines) == 5 + + +def test_format_build_diff_markdown_never_exceeds_github_limit_constant(): + # Stress: 80 libraries x 20 long lines — must still stay under DIFF_PR_MAX_CHARS. + long = "x" * 200 + diffs = [] + for i in range(80): + added = [build_oss_db.make_db_entry(f"{long}-{j}", f"lib{i}", "1.0", "f.c", "fn") for j in range(30)] + diffs.append( + build_oss_db.LibraryDiff( + library=f"lib{i}", + old_count=0, + new_count=30, + added=added, + removed=[], + changed=[], + ) + ) + text = build_oss_db.format_build_diff_markdown(diffs) + assert len(text) <= build_oss_db.DIFF_PR_MAX_CHARS + + +def test_clip_to_max_chars_closes_unclosed_code_fence(): + text = "Before\n```diff\n+ " + ("x" * 100) + clipped = build_oss_db._clip_to_max_chars(text, 80) + + assert len(clipped) <= 80 + assert clipped.count("```") % 2 == 0 + assert clipped.endswith("... truncated to stay under GitHub's PR body length limit.\n") + + +def test_clip_to_max_chars_omits_fence_when_its_closing_fence_does_not_fit(): + notice = "\n... truncated to stay under GitHub's PR body length limit.\n" + clipped = build_oss_db._clip_to_max_chars("```diff" + ("x" * 100), len(notice) + 3) + + assert len(clipped) <= len(notice) + 3 + assert clipped.count("```") % 2 == 0 + + +def test_format_build_diff_empty_and_no_changes(): + assert "No libraries were rebuilt" in build_oss_db.format_build_diff([]) + unchanged = build_oss_db.LibraryDiff( + library="zlib", + old_count=1, + new_count=1, + added=[], + removed=[], + changed=[], + ) + assert "No entry-level changes" in build_oss_db.format_build_diff([unchanged]) + assert "No entry-level changes" in build_oss_db.format_build_diff_markdown([unchanged]) + + +def test_format_build_diff_escapes_control_characters_backticks_and_long_strings(): + entry = build_oss_db.make_db_entry("hello\nworld\rcolumn\tvalue```" + ("x" * 200), "zlib", "1.0", "f.c", "fn") + diff = build_oss_db.diff_library_entries("zlib", [], [entry]) + text = build_oss_db.format_build_diff([diff]) + assert "\\n" in text + assert "\\r" in text + assert "\\t" in text + assert "` ` `" in text + # Long strings are truncated with ellipsis; each +/-/~ line stays single-line. + assert "..." in text + for line in text.splitlines(): + if line.startswith(("+ ", "- ", "~ ")): + assert "\n" not in line[2:] + assert len(line) < 200 + + +def test_main_writes_build_diff_for_new_and_merged_libraries(tmp_path): + existing_entry = build_oss_db.make_db_entry("old-string", "zlib", "0.9#1", "f.c", "old_fn") + rc = _invoke_run_build( + tmp_path, + ["zlib", "curl"], + existing={"zlib": [existing_entry]}, + ) + assert rc == 0 + # Plain-text log report. + log_text = (tmp_path / "build_diff.txt").read_text(encoding="utf-8") + assert "## zlib" in log_text + assert "## curl" in log_text + assert "+ hello-from-zlib" in log_text + assert "+ hello-from-curl" in log_text + assert "- old-string" not in log_text + + # PR markdown: headings outside per-library ```diff fences. + pr_text = (tmp_path / "build_diff_pr.txt").read_text(encoding="utf-8") + assert "## zlib" in pr_text + assert "## curl" in pr_text + assert "```diff" in pr_text + assert "+ hello-from-zlib" in pr_text + assert "+ hello-from-curl" in pr_text + assert "- old-string" not in pr_text + for part in pr_text.split("```"): + if part.startswith("diff"): + assert "## " not in part + + +def test_main_build_diff_reports_no_changes_when_entries_identical(tmp_path): + # Rebuild with the same entry the stub always produces: after merge, the + # only string is still hello-from-zlib with identical metadata... but the + # stub always emits version 1.0#1, so seed with that exact entry. + existing = [build_oss_db.make_db_entry("hello-from-zlib", "zlib", "1.0#1", "f.c", "fn_zlib")] + rc = _invoke_run_build(tmp_path, ["zlib"], existing={"zlib": existing}) + assert rc == 0 + text = (tmp_path / "build_diff.txt").read_text(encoding="utf-8") + assert "No entry-level changes detected" in text diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 000000000..a3727302c --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,471 @@ +import os +import json +import tempfile +from pathlib import Path + +import pytest + +import floss.cache +from floss.results import ( + Strings, + Analysis, + Metadata, + AddressType, + StackString, + TightString, + ResultLayout, + ResultString, + StaticString, + DecodedString, + ResultDocument, + StringEncoding, +) +from floss.version import __version__ + + +def make_doc( + sha256="a" * 64, + version=__version__, + min_length=4, + enable_static=True, + enable_stack=False, + enable_tight=False, + enable_decoded=False, + enable_language=False, + enable_layout=True, + enable_tags=True, +): + layout = ResultLayout(name="pe", offset=0, length=8) if enable_layout else None + return ResultDocument( + metadata=Metadata( + file_path="sample.exe", + md5="0" * 32, + sha1="0" * 40, + sha256=sha256, + version=version, + min_length=min_length, + ), + analysis=Analysis( + enable_static_strings=enable_static, + enable_stack_strings=enable_stack, + enable_tight_strings=enable_tight, + enable_decoded_strings=enable_decoded, + enable_language_strings=enable_language, + enable_layout=enable_layout, + enable_tags=enable_tags, + ), + strings=Strings( + static_strings=[StaticString(string="hello", offset=0, encoding=StringEncoding.ASCII)], + ), + layout=layout, + ) + + +def make_full_doc(): + layout = ResultLayout( + name="pe", + offset=0, + length=8, + strings=[ResultString(string="layout", offset=0, size=6, encoding="ASCII", tags=["#winapi"])], + ) + return ResultDocument( + metadata=Metadata(file_path="sample.exe", sha256="a" * 64, version=__version__, min_length=4), + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=True, + enable_tight_strings=True, + enable_decoded_strings=True, + enable_language_strings=True, + enable_layout=True, + enable_tags=True, + ), + strings=Strings( + static_strings=[StaticString(string="hello", offset=0, encoding=StringEncoding.ASCII, tags=["#common"])], + stack_strings=[ + StackString( + function=0, + string="stack", + encoding=StringEncoding.ASCII, + program_counter=0, + stack_pointer=0, + original_stack_pointer=0, + offset=0, + frame_offset=0, + ) + ], + tight_strings=[ + TightString( + function=0, + string="tight", + encoding=StringEncoding.ASCII, + program_counter=0, + stack_pointer=0, + original_stack_pointer=0, + offset=0, + frame_offset=0, + ) + ], + decoded_strings=[ + DecodedString( + address=0, + address_type=AddressType.STACK, + string="decoded", + encoding=StringEncoding.ASCII, + decoded_at=0, + decoding_routine=0, + ) + ], + language_strings=[StaticString(string="gostring", offset=0, encoding=StringEncoding.UTF8, tags=["#go"])], + language_strings_missed=[StaticString(string="missed", offset=0, encoding=StringEncoding.UTF8)], + ), + layout=layout, + ) + + +def wanted( + enable_static=True, + enable_stack=False, + enable_tight=False, + enable_decoded=False, + enable_language=False, + enable_layout=True, + enable_tags=True, +): + return Analysis( + enable_static_strings=enable_static, + enable_stack_strings=enable_stack, + enable_tight_strings=enable_tight, + enable_decoded_strings=enable_decoded, + enable_language_strings=enable_language, + enable_layout=enable_layout, + enable_tags=enable_tags, + ) + + +def test_cache_enabled_default(monkeypatch): + monkeypatch.delenv(floss.cache.ENV_CACHE_ENABLE, raising=False) + assert floss.cache.cache_enabled() + + +@pytest.mark.parametrize("value", ("0", "false", "no", "n", "")) +def test_cache_enabled_disabled(monkeypatch, value): + monkeypatch.setenv(floss.cache.ENV_CACHE_ENABLE, value) + assert not floss.cache.cache_enabled() + + +@pytest.mark.parametrize("value", ("False", "NO", "N", "FALSE")) +def test_cache_enabled_disabled_case_insensitive(monkeypatch, value): + monkeypatch.setenv(floss.cache.ENV_CACHE_ENABLE, value) + assert not floss.cache.cache_enabled() + + +def test_cache_refresh_default(monkeypatch): + monkeypatch.delenv(floss.cache.ENV_CACHE_REFRESH, raising=False) + assert not floss.cache.cache_refresh() + + +@pytest.mark.parametrize("value", ("1", "true", "yes", "y")) +def test_cache_refresh_enabled(monkeypatch, value): + monkeypatch.setenv(floss.cache.ENV_CACHE_REFRESH, value) + assert floss.cache.cache_refresh() + + +@pytest.mark.parametrize("value", ("True", "YES", "Y", "TRUE")) +def test_cache_refresh_enabled_case_insensitive(monkeypatch, value): + monkeypatch.setenv(floss.cache.ENV_CACHE_REFRESH, value) + assert floss.cache.cache_refresh() + + +def test_get_cache_dir_override(monkeypatch, tmp_path): + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(tmp_path)) + assert floss.cache.get_cache_dir() == tmp_path + + +def test_get_cache_dir_default(monkeypatch): + from platformdirs import user_cache_dir + + monkeypatch.delenv(floss.cache.ENV_CACHE_DIR, raising=False) + assert floss.cache.get_cache_dir() == Path(user_cache_dir("floss")) + + +def test_compute_key(): + sha256 = "a" * 64 + # auto is the default analysis format + assert floss.cache.compute_key(sha256, "1.0") == f"{sha256}-auto-1.0" + assert floss.cache.compute_key(sha256, "1.0") == f"{sha256}-auto-1.0" + # the analysis format is part of the key: sc32 and sc64 never collide + assert floss.cache.compute_key(sha256, "1.0", "sc32") != floss.cache.compute_key(sha256, "1.0", "sc64") + + +def test_cache_file_path(tmp_path): + assert floss.cache.cache_file_path(tmp_path, "key") == tmp_path / "key.json" + + +def test_store_and_load_roundtrip(tmp_path): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + + assert floss.cache.store(tmp_path, key, doc) + assert (tmp_path / f"{key}.json").is_file() + + loaded = floss.cache.load(tmp_path, key, doc.metadata.sha256, __version__) + assert loaded is not None + assert loaded.metadata.sha256 == doc.metadata.sha256 + assert loaded.metadata.version == __version__ + assert loaded.strings.static_strings[0].string == "hello" + + +def test_store_writes_json_schema(tmp_path): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + + floss.cache.store(tmp_path, key, doc) + payload = json.loads((tmp_path / f"{key}.json").read_text()) + assert set(("metadata", "analysis", "strings")) <= set(payload.keys()) + + +def test_load_missing(tmp_path): + key = floss.cache.compute_key("a" * 64, __version__) + assert floss.cache.load(tmp_path, key, "a" * 64, __version__) is None + + +def test_load_drops_invalid_json(tmp_path): + key = floss.cache.compute_key("a" * 64, __version__) + (tmp_path / f"{key}.json").write_text("{not json") + + assert floss.cache.load(tmp_path, key, "a" * 64, __version__) is None + assert not (tmp_path / f"{key}.json").exists() + + +def test_load_drops_checksum_mismatch(tmp_path): + doc = make_doc(sha256="a" * 64) + key = floss.cache.compute_key("a" * 64, __version__) + floss.cache.store(tmp_path, key, doc) + + assert floss.cache.load(tmp_path, key, "b" * 64, __version__) is None + assert not (tmp_path / f"{key}.json").exists() + + +def test_load_ignores_unlink_failure_on_invalid_entry(tmp_path, monkeypatch): + key = floss.cache.compute_key("a" * 64, __version__) + (tmp_path / f"{key}.json").write_text("{not json") + + def raising_unlink(self, *args, **kwargs): + raise PermissionError("held open by another process") + + monkeypatch.setattr(Path, "unlink", raising_unlink) + assert floss.cache.load(tmp_path, key, "a" * 64, __version__) is None + + +def test_load_ignores_unlink_failure_on_stale_entry(tmp_path, monkeypatch): + doc = make_doc(sha256="a" * 64) + key = floss.cache.compute_key("a" * 64, __version__) + floss.cache.store(tmp_path, key, doc) + + def raising_unlink(self, *args, **kwargs): + raise PermissionError("held open by another process") + + monkeypatch.setattr(Path, "unlink", raising_unlink) + assert floss.cache.load(tmp_path, key, "b" * 64, __version__) is None + + +def test_store_skips_when_replace_fails(tmp_path, monkeypatch): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + + def raising_replace(src, dst): + raise PermissionError("destination held open by antivirus") + + monkeypatch.setattr(os, "replace", raising_replace) + assert floss.cache.store(tmp_path, key, doc) is False + assert not (tmp_path / f"{key}.json").exists() + # the temporary file is cleaned up + assert list(tmp_path.glob("*.tmp")) == [] + + +def test_store_skips_when_mkdir_fails(tmp_path, monkeypatch): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + cache_dir = tmp_path / "cache" + + def raising_mkdir(self, *args, **kwargs): + raise PermissionError("no write access to cache directory") + + monkeypatch.setattr(Path, "mkdir", raising_mkdir) + assert floss.cache.store(cache_dir, key, doc) is False + + +def test_store_skips_when_mkstemp_fails(tmp_path, monkeypatch): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + + def raising_mkstemp(*args, **kwargs): + raise OSError("no space left on device") + + monkeypatch.setattr(tempfile, "mkstemp", raising_mkstemp) + assert floss.cache.store(tmp_path, key, doc) is False + assert not (tmp_path / f"{key}.json").exists() + + +def test_load_skips_when_isfile_fails(tmp_path, monkeypatch): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + + def raising_is_file(self, *args, **kwargs): + raise PermissionError("no read access to cache directory") + + monkeypatch.setattr(Path, "is_file", raising_is_file) + assert floss.cache.load(tmp_path, key, doc.metadata.sha256, __version__) is None + + +def test_load_drops_version_mismatch(tmp_path): + doc = make_doc(version="9.9.9") + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + floss.cache.store(tmp_path, key, doc) + + assert floss.cache.load(tmp_path, key, doc.metadata.sha256, __version__) is None + assert not (tmp_path / f"{key}.json").exists() + + +def test_store_skips_when_locked(tmp_path): + doc = make_doc() + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + lock_path = tmp_path / f"{key}.lock" + + fd = floss.cache._acquire_lock(lock_path) + assert fd is not None + try: + assert floss.cache.store(tmp_path, key, doc) is False + assert not (tmp_path / f"{key}.json").exists() + finally: + floss.cache._release_lock(fd, lock_path) + + +def test_covers_matches(): + doc = make_doc() + assert floss.cache.covers(doc, wanted(), 4) + + +def test_covers_missing_requested_type_is_miss(): + doc = make_doc(enable_static=True, enable_stack=False) + assert not floss.cache.covers(doc, wanted(enable_stack=True), 4) + + +def test_covers_min_length_below_stored_is_miss(): + doc = make_doc(min_length=8) + assert not floss.cache.covers(doc, wanted(), 4) + + +def test_covers_layout_enabled_mismatch_is_miss(): + doc = make_doc(enable_layout=False) + assert not floss.cache.covers(doc, wanted(enable_layout=True), 4) + + +def test_covers_layout_disabled_is_hit(): + # a cached layout can satisfy a no-layout request: materialize() drops it + doc = make_doc(enable_layout=True) + assert floss.cache.covers(doc, wanted(enable_layout=False), 4) + + +def test_covers_tags_enabled_mismatch_is_miss(): + doc = make_doc(enable_tags=False) + assert not floss.cache.covers(doc, wanted(enable_tags=True), 4) + + +def test_covers_tags_disabled_is_hit(): + # a tags-enabled document satisfies a no-tags request: materialize redacts + doc = make_doc(enable_tags=True) + assert floss.cache.covers(doc, wanted(enable_tags=False), 4) + + +def test_materialize_sets_file_path_and_filters(tmp_path): + doc = make_doc(min_length=4) + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + floss.cache.store(tmp_path, key, doc) + loaded = floss.cache.load(tmp_path, key, doc.metadata.sha256, __version__) + assert loaded is not None + + sample = tmp_path / "run" / "sample.exe" + mat = floss.cache.materialize(loaded, sample, wanted(), 8) + assert mat.metadata.file_path == str(sample) + # "hello" is 5 chars, below the requested -n 8 + assert mat.strings.static_strings == [] + + +def test_materialize_drops_layout_when_disabled(tmp_path): + doc = make_doc(enable_layout=True) + key = floss.cache.compute_key(doc.metadata.sha256, __version__) + floss.cache.store(tmp_path, key, doc) + loaded = floss.cache.load(tmp_path, key, doc.metadata.sha256, __version__) + assert loaded is not None + assert loaded.layout is not None + + mat = floss.cache.materialize(loaded, tmp_path / "sample.exe", wanted(enable_layout=False), 4) + assert mat.layout is None + + +def test_materialize_clears_disabled_string_types(): + mat = floss.cache.materialize(make_full_doc(), Path("sample.exe"), wanted(), 4) + assert mat.analysis.enable_static_strings is True + assert mat.strings.static_strings + + disabled = Analysis( + enable_static_strings=False, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + enable_language_strings=False, + ) + mat = floss.cache.materialize(make_full_doc(), Path("sample.exe"), disabled, 4) + assert mat.strings.static_strings == [] + assert mat.strings.stack_strings == [] + assert mat.strings.tight_strings == [] + assert mat.strings.decoded_strings == [] + assert mat.strings.language_strings == [] + assert mat.strings.language_strings_missed == [] + # static strings are disabled, so the layout that holds them is gone too + assert mat.layout is None + + +def test_materialize_clears_only_disabled_types(): + disabled_stack = Analysis(enable_stack_strings=False) + mat = floss.cache.materialize(make_full_doc(), Path("sample.exe"), disabled_stack, 4) + assert mat.strings.stack_strings == [] + assert mat.strings.static_strings + assert mat.strings.tight_strings + assert mat.strings.decoded_strings + assert mat.layout is not None + + +def test_materialize_syncs_layout_and_tag_flags(): + doc = make_full_doc() + mat = floss.cache.materialize(doc, Path("sample.exe"), Analysis(enable_layout=False, enable_tags=False), 4) + assert mat.analysis.enable_layout is False + assert mat.analysis.enable_tags is False + + +def test_materialize_updates_metadata_min_length(): + # a -n 6 hit against a -n 4 cache entry must report the requested length + mat = floss.cache.materialize(make_full_doc(), Path("sample.exe"), wanted(), 6) + assert mat.metadata.min_length == 6 + assert all(len(s.string) >= 6 for s in mat.strings.static_strings) + + +def test_materialize_clears_tags_when_disabled(): + doc = make_full_doc() + assert doc.strings.static_strings[0].tags == ["#common"] + assert doc.layout.strings[0].tags == ["#winapi"] + + # Analysis(enable_tags=False) keeps every string type enabled, tags off + mat = floss.cache.materialize(doc, Path("sample.exe"), Analysis(enable_tags=False), 4) + assert mat.layout is not None + assert mat.strings.static_strings[0].tags == [] + assert mat.strings.language_strings[0].tags == [] + assert mat.layout.strings[0].tags == [] + + +def test_materialize_keeps_tags_when_enabled(): + mat = floss.cache.materialize(make_full_doc(), Path("sample.exe"), wanted(), 4) + assert mat.layout is not None + assert mat.strings.static_strings[0].tags == ["#common"] + assert mat.layout.strings[0].tags == ["#winapi"] diff --git a/tests/test_cache_integration.py b/tests/test_cache_integration.py new file mode 100644 index 000000000..36286512a --- /dev/null +++ b/tests/test_cache_integration.py @@ -0,0 +1,87 @@ +import hashlib +import logging +from pathlib import Path + +from fixtures import scfile, exefile + +import floss.main +import floss.cache +import floss.render.json +from floss.results import Strings, Analysis, Metadata, ResultDocument +from floss.version import __version__ + + +def cache_entries(cache_dir): + return list(cache_dir.rglob("*.json")) + + +def test_cache_hit_skips_analysis_and_matches_output(capsys, caplog, tmp_path, monkeypatch, exefile): + caplog.set_level(logging.DEBUG) + cache_dir = tmp_path / "cache" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(cache_dir)) + + assert floss.main.main([exefile, "--summary"]) == 0 + out1 = capsys.readouterr().out + + sha256 = hashlib.sha256(Path(exefile).read_bytes()).hexdigest() + key = floss.cache.compute_key(sha256, __version__) + assert (cache_dir / f"{key}.json").is_file() + assert len(cache_entries(cache_dir)) == 1 + + caplog.clear() + assert floss.main.main([exefile, "--summary", "-d"]) == 0 + assert any("using cached results" in r.getMessage() for r in caplog.records) + assert capsys.readouterr().out == out1 + + +def test_cache_disabled_writes_nothing(capsys, tmp_path, monkeypatch, exefile): + cache_dir = tmp_path / "cache" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(cache_dir)) + monkeypatch.setenv(floss.cache.ENV_CACHE_ENABLE, "0") + + assert floss.main.main([exefile, "--summary"]) == 0 + assert cache_entries(cache_dir) == [] + + +def test_cache_not_written_for_results_document(capsys, tmp_path, monkeypatch): + cache_dir = tmp_path / "cache" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(cache_dir)) + + doc = ResultDocument( + metadata=Metadata(file_path="sample.exe", min_length=4), + analysis=Analysis(enable_static_strings=False), + strings=Strings(), + ) + results_path = tmp_path / "results.json" + results_path.write_text(floss.render.json.render(doc)) + + assert floss.main.main([str(results_path)]) == 0 + assert cache_entries(cache_dir) == [] + + +def test_cache_refresh_reanalyzes_and_overwrites(capsys, caplog, tmp_path, monkeypatch, exefile): + caplog.set_level(logging.DEBUG) + cache_dir = tmp_path / "cache" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(cache_dir)) + + assert floss.main.main([exefile, "--summary"]) == 0 + sha256 = hashlib.sha256(Path(exefile).read_bytes()).hexdigest() + key = floss.cache.compute_key(sha256, __version__) + assert (cache_dir / f"{key}.json").is_file() + + # FLOSS_CACHE_REFRESH=1 forces a miss, re-analyzes, and overwrites the entry + caplog.clear() + monkeypatch.setenv(floss.cache.ENV_CACHE_REFRESH, "1") + assert floss.main.main([exefile, "--summary", "-d"]) == 0 + assert not any("using cached results" in r.getMessage() for r in caplog.records) + assert any("FLOSS_CACHE_REFRESH" in r.getMessage() for r in caplog.records) + assert (cache_dir / f"{key}.json").is_file() + + +def test_analysis_variant_disables_cache(capsys, tmp_path, monkeypatch, scfile): + cache_dir = tmp_path / "cache" + monkeypatch.setenv(floss.cache.ENV_CACHE_DIR, str(cache_dir)) + + # an explicit --format is a non-default analysis variant, so no caching + assert floss.main.main([scfile, "-f", "sc32", "--summary"]) == 0 + assert cache_entries(cache_dir) == [] diff --git a/tests/test_cli_args.py b/tests/test_cli_args.py index 17b415aa6..ee04576ed 100644 --- a/tests/test_cli_args.py +++ b/tests/test_cli_args.py @@ -13,20 +13,25 @@ # limitations under the License. +from pathlib import Path + import pytest from fixtures import scfile, exefile import floss.main +from floss.cli import StringType def test_functions(exefile): # 0x1111111 is not a function - assert floss.main.main([exefile, "--function", "0x1111111"]) == -1 + assert floss.main.main([exefile, "--analyze-functions", "0x1111111"]) == -1 # ok - assert floss.main.main([exefile, "--function", "0x401560"]) == 0 - assert floss.main.main([exefile, "--function", "0x401560"]) == 0 - assert floss.main.main([exefile, "--function", "0x401560", "0x401000"]) == 0 + assert floss.main.main([exefile, "--analyze-functions", "0x401560"]) == 0 + assert floss.main.main([exefile, "--analyze-functions", "0x401560", "0x401000"]) == 0 + + # --string-type static only cannot be combined with --analyze-functions + assert floss.main.main([exefile, "--analyze-functions", "0x401560", "--string-type", "static"]) == -1 def test_shellcode(scfile): @@ -34,12 +39,12 @@ def test_shellcode(scfile): assert floss.main.main([scfile, "-f", "sc32"]) == 0 assert floss.main.main([scfile, "--format", "sc64"]) == 0 - # fail + # fail: forcing the PE format on shellcode errors once vivisect runs assert floss.main.main([scfile, "--format", "pe"]) == -1 -@pytest.mark.parametrize("type_", [t.value for t in floss.main.StringType]) -@pytest.mark.parametrize("analysis", ("--only", "--no")) +@pytest.mark.parametrize("type_", [t.value for t in StringType]) +@pytest.mark.parametrize("analysis", ("--string-type", "--no-string-type")) def test_args_analysis_type(exefile, analysis, type_): assert ( floss.main.main( @@ -51,3 +56,136 @@ def test_args_analysis_type(exefile, analysis, type_): ) == 0 ) + + +def test_args_analysis_type_conflict(exefile): + assert floss.main.main([exefile, "--string-type", "stack", "--no-string-type", "tight"]) == -1 + + +def test_language_extraction_independent_of_static(capsys): + """language strings are extracted even when static strings are disabled. + + uses --string-type language (so only language extraction runs) on a Go sample + whose language is detectable. + """ + import json + + sample = Path(__file__).parent / "data" / "language" / "go" / "go-hello" / "bin" / "go-hello64.exe" + + assert floss.main.main([str(sample), "--string-type", "language", "-j"]) == 0 + doc = json.loads(capsys.readouterr().out) + assert doc["metadata"]["language"] == "go" + assert len(doc["strings"]["language_strings"]) > 0 + assert doc["strings"]["static_strings"] == [] + + +def test_manual_language_override_wins_over_auto_detect(capsys): + """--language go must be honored even when auto-detection returns unknown.""" + import json + + # a C binary, so auto-detection yields unknown; forcing go must stick + sample = Path(__file__).parent / "data" / "src" / "decode-in-place" / "bin" / "test-decode-in-place.exe" + + assert floss.main.main([str(sample), "--language", "go", "--string-type", "language", "-j"]) == 0 + doc = json.loads(capsys.readouterr().out) + assert doc["metadata"]["language"] == "go" + assert doc["metadata"]["language_selected"] == "go" + + +def test_manual_language_override_beats_wrong_auto_detect(monkeypatch, capsys): + """--language go must win even when auto-detection wrongly says rust.""" + import json + + import floss.language.identify + + def fake_identify(sample, static_strings): + from floss.language.identify import Language + + return Language.RUST, "1.75.0" + + monkeypatch.setattr(floss.language.identify, "identify_language_and_version", fake_identify) + + sample = Path(__file__).parent / "data" / "src" / "decode-in-place" / "bin" / "test-decode-in-place.exe" + assert floss.main.main([str(sample), "--language", "go", "--string-type", "language", "-j"]) == 0 + doc = json.loads(capsys.readouterr().out) + assert doc["metadata"]["language"] == "go" + assert doc["metadata"]["language_version"] == "" + assert doc["metadata"]["language_selected"] == "go" + + +def test_expand_string_types(): + from floss.utils import expand_string_types + + assert set(expand_string_types(["all"])) == {"static", "stack", "tight", "decoded", "language"} + assert expand_string_types(["static", "stack"]) == ["static", "stack"] + + +def test_no_layout_yields_classic_static(exefile): + """enable_layout=False: no layout tree; classic static strings still present.""" + from pathlib import Path + + from floss.results import Analysis + from floss.pipeline import Options, analyze + + results = analyze( + Options( + sample=Path(exefile), + min_length=4, + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + enable_layout=False, + enable_tags=True, + ), + ) + ) + assert results is not None + assert results.layout is None + assert len(results.strings.static_strings) > 0 + + +def test_no_tags_skips_tag_databases(exefile): + """enable_tags=False: layout present; no DB tags (#common, #winapi, …). + + Layout-intrinsic tags such as #code / #duplicate may still appear. + """ + from pathlib import Path + + from floss.results import Analysis, ResultLayout + from floss.pipeline import Options, analyze + + results = analyze( + Options( + sample=Path(exefile), + min_length=4, + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + enable_layout=True, + enable_tags=False, + ), + ) + ) + assert results is not None + assert results.layout is not None + assert isinstance(results.layout, ResultLayout) + + def all_tags(layout): + tags = set() + for s in layout.strings: + tags.update(s.tags) + for child in layout.children: + tags.update(all_tags(child)) + return tags + + tags = all_tags(results.layout) + # database-backed tags must be absent when tag DBs are disabled + for db_tag in ("#common", "#winapi", "#capa", "#msvc", "#openssl"): + assert db_tag not in tags + for s in results.strings.static_strings: + for db_tag in ("#common", "#winapi", "#capa", "#msvc", "#openssl"): + assert db_tag not in s.tags diff --git a/tests/test_code_ranges.py b/tests/test_code_ranges.py new file mode 100644 index 000000000..f55606fbb --- /dev/null +++ b/tests/test_code_ranges.py @@ -0,0 +1,183 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock, MagicMock + +import pefile +import pytest + +from floss.ranges import Range, Slice, merge_overlapping_ranges +from floss.layout.pe import _get_code_ranges + + +# Tests for merge_overlapping_ranges +def test_merge_empty_list(): + """Test merging an empty list of ranges.""" + assert merge_overlapping_ranges([]) == [] + + +def test_merge_no_overlap(): + """Test merging ranges that do not overlap.""" + ranges = [(10, 20), (30, 40), (50, 60)] + assert merge_overlapping_ranges(ranges) == [(10, 20), (30, 40), (50, 60)] + + +def test_merge_with_overlap(): + """Test merging ranges that partially overlap.""" + ranges = [(10, 20), (15, 25), (30, 40)] + assert merge_overlapping_ranges(ranges) == [(10, 25), (30, 40)] + + +def test_merge_adjacent(): + """Test merging ranges that are right next to each other.""" + ranges = [(10, 20), (21, 30), (31, 40)] + assert merge_overlapping_ranges(ranges) == [(10, 40)] + + +def test_merge_fully_contained(): + """Test merging ranges where some are fully contained within others.""" + ranges = [(10, 40), (15, 25), (20, 30)] + assert merge_overlapping_ranges(ranges) == [(10, 40)] + + +def test_merge_complex_mix(): + """Test a complex mixture of overlapping, adjacent, and contained ranges.""" + ranges = [(50, 60), (10, 20), (18, 30), (35, 40), (39, 55)] + # After sorting: [(10, 20), (18, 30), (35, 40), (39, 55), (50, 60)] + # (10, 20) and (18, 30) -> (10, 30) + # (35, 40) and (39, 55) -> (35, 55) + # (35, 55) and (50, 60) -> (35, 60) + assert merge_overlapping_ranges(ranges) == [(10, 30), (35, 60)] + + +# Tests for _get_code_ranges +@pytest.fixture +def mock_pe(): + """Fixture for a mocked pefile.PE object.""" + pe = MagicMock(spec=pefile.PE) + + def get_offset_from_rva(rva): + # Simple mapping for testing: offset is just rva + 0x1000 + return rva + 0x1000 + + pe.get_offset_from_rva.side_effect = get_offset_from_rva + return pe + + +def _make_instr(size: int) -> Mock: + instr = Mock() + instr.raw_bytes = b"\x90" * size + return instr + + +def _make_be2_mocks(bb_instructions: list): + """ + Build mock be2 and idx objects. + + bb_instructions: list of lists of (va, size) tuples, one sub-list per basic block. + """ + be2 = MagicMock() + idx = MagicMock() + + basic_blocks = {} + instr_map = {} + for bb_i, instrs in enumerate(bb_instructions): + bb = Mock() + basic_blocks[bb_i] = bb + instr_map[id(bb)] = [(i, _make_instr(sz), va) for i, (va, sz) in enumerate(instrs)] + + fg = Mock() + fg.basic_block_index = list(range(len(basic_blocks))) + be2.flow_graph = [fg] + be2.basic_block = basic_blocks + + def _bb_instructions(bb): + return iter(instr_map.get(id(bb), [])) + + idx.basic_block_instructions.side_effect = _bb_instructions + + return be2, idx + + +def test_get_code_ranges_basic(mock_pe): + """Test basic extraction of code ranges.""" + # base_address = 0x400000, rva = va - base + # bb1: va 0x401000, size 0x10 -> rva 0x1000, offset 0x2000 -> range (0x2000, 0x200F) + # bb2: va 0x401020, size 0x15 -> rva 0x1020, offset 0x2020 -> range (0x2020, 0x2034) + # bb3: va 0x402000, size 0x20 -> rva 0x2000, offset 0x3000 -> range (0x3000, 0x301F) + be2, idx = _make_be2_mocks( + [ + [(0x401000, 0x10)], + [(0x401020, 0x15)], + [(0x402000, 0x20)], + ] + ) + + slice_ = Slice(buf=b"", range=Range(offset=0, length=0x5000)) + + ranges = _get_code_ranges(be2, idx, 0x400000, mock_pe, slice_) + + assert ranges == [ + (0x2000, 0x200F), # bb1: offset 0x2000, size 0x10 + (0x2020, 0x2034), # bb2: offset 0x2020, size 0x15 + (0x3000, 0x301F), # bb3: offset 0x3000, size 0x20 + ] + + +def test_get_code_ranges_skips_invalid_offset(mock_pe): + """Test that it skips instructions that fall outside the slice.""" + be2, idx = _make_be2_mocks( + [ + [(0x401000, 0x10)], # offset 0x2000, fits in slice + [(0x401020, 0x15)], # offset 0x2020, outside slice + [(0x402000, 0x20)], # offset 0x3000, outside slice + ] + ) + + # Slice only covers through offset 0x2010 + slice_ = Slice(buf=b"", range=Range(offset=0, length=0x2010)) + + ranges = _get_code_ranges(be2, idx, 0x400000, mock_pe, slice_) + + # Only bb1 should be included + assert ranges == [(0x2000, 0x200F)] + + +def test_get_code_ranges_handles_pe_error(mock_pe): + """Test that it handles PEFormatError when getting an offset.""" + + def get_offset_from_rva_with_error(rva): + if rva == 0x1020: # Corresponds to bb2 + raise pefile.PEFormatError("Test Error") + return rva + 0x1000 + + mock_pe.get_offset_from_rva.side_effect = get_offset_from_rva_with_error + + be2, idx = _make_be2_mocks( + [ + [(0x401000, 0x10)], + [(0x401020, 0x15)], + [(0x402000, 0x20)], + ] + ) + + slice_ = Slice(buf=b"", range=Range(offset=0, length=0x5000)) + + ranges = _get_code_ranges(be2, idx, 0x400000, mock_pe, slice_) + + # bb2 should be skipped due to PEFormatError + assert ranges == [ + (0x2000, 0x200F), + (0x3000, 0x301F), + ] diff --git a/tests/test_scripts.py b/tests/test_disassembler_scripts.py similarity index 87% rename from tests/test_scripts.py rename to tests/test_disassembler_scripts.py index 407e39c9a..d54ae72e2 100644 --- a/tests/test_scripts.py +++ b/tests/test_disassembler_scripts.py @@ -29,8 +29,8 @@ CD = Path(__file__).resolve().parent -def get_script_path(s) -> Path: - return CD / ".." / "scripts" / s +def get_disassembler_script_path(s) -> Path: + return CD / ".." / "scripts" / "disassemblers" / s def get_file_path() -> Path: @@ -46,7 +46,7 @@ def run_program(script_path: Path, args): @lru_cache() def get_results_file_path(): res_path = Path("results.json") - p = run_program(Path("floss/main.py"), ["--no", "static", "-j", str(get_file_path())]) + p = run_program(Path("floss/main.py"), ["--no-string-type", "static", "-j", str(get_file_path())]) with res_path.open("w") as f: f.write(p.stdout.decode("utf-8")) return str(res_path) @@ -62,7 +62,7 @@ def get_results_file_path(): pytest.param("render-x64dbg-database.py", [get_results_file_path()]), ], ) -def test_scripts(script, args): - script_path = get_script_path(script) +def test_disassembler_scripts(script, args): + script_path = get_disassembler_script_path(script) p = run_program(script_path, args) assert p.returncode == 0 diff --git a/tests/test_gp_db.py b/tests/test_gp_db.py new file mode 100644 index 000000000..1b006e5c8 --- /dev/null +++ b/tests/test_gp_db.py @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import floss.tags.gp +from floss.tags import data_root + + +def test_load_db(): + path = data_root() / "gp" / "gp.jsonl.gz" + db = floss.tags.gp.StringGlobalPrevalenceDatabase.from_file(path) + + assert len(db) > 0 # 21 entries at time of writing + + +def test_query_db(): + path = data_root() / "gp" / "gp.jsonl.gz" + db = floss.tags.gp.StringGlobalPrevalenceDatabase.from_file(path) + res = db.metadata_by_string["!This program cannot be run in DOS mode."] + + assert len(res) == 1 + s = res[0] + + assert s is not None + assert s.string == "!This program cannot be run in DOS mode." + assert s.encoding == "ascii" + assert s.global_count == 424466 + assert s.location == None + + +def test_load_hash_db(): + path = data_root() / "gp" / "xaa-hashes.bin" + db = floss.tags.gp.StringHashDatabase.from_file(path) + + assert len(db) > 0 + + +def test_query_hash_db(): + path = data_root() / "gp" / "xaa-hashes.bin" + db = floss.tags.gp.StringHashDatabase.from_file(path) + + assert "!This program cannot be run in DOS mode." in db + assert "Willi rules" not in db diff --git a/tests/test_layout.py b/tests/test_layout.py new file mode 100644 index 000000000..5d9ffa0e4 --- /dev/null +++ b/tests/test_layout.py @@ -0,0 +1,167 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import tempfile +from pathlib import Path + +import pytest + +import floss.render.json +from floss.tags import load_databases +from floss.enrich import static_strings_from_layout +from floss.layout import compute_layout +from floss.ranges import Slice +from floss.results import Strings, Analysis, Metadata, ResultLayout, ResultDocument +from floss.layout.extract import collect_strings + +CD = Path(__file__).resolve().parent +MIN_STR_LEN = 6 + + +@pytest.fixture +def pma_binary_path(): + return CD / "data" / "pma" / "Practical Malware Analysis Lab 03-03.exe_" + + +@pytest.fixture +def analyzed_layout(pma_binary_path): + slice_buf = pma_binary_path.read_bytes() + file_slice = Slice.from_bytes(slice_buf) + parsed = compute_layout(file_slice) + parsed.extract_strings(6) + taggers = load_databases() + parsed.tag_strings(taggers) + parsed.mark_structures() + return parsed + + +def test_round_trip(analyzed_layout, pma_binary_path): + layout_doc = ResultLayout.from_layout(analyzed_layout) + statics = static_strings_from_layout(layout_doc) + one = ResultDocument( + metadata=Metadata(file_path=str(pma_binary_path.resolve()), min_length=MIN_STR_LEN), + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + enable_layout=True, + enable_tags=True, + ), + strings=Strings(static_strings=statics), + layout=layout_doc, + ) + + doc = floss.render.json.render(one) + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + f.write(doc) + path = Path(f.name) + try: + two = ResultDocument.parse_file(path) + finally: + path.unlink() + + # show the round trip works + assert one == two + assert floss.render.json.render(one) == floss.render.json.render(two) + + # now show that two different versions are not equal. + three = copy.deepcopy(two) + three.metadata.version = "0" + assert two.metadata.version != three.metadata.version + assert floss.render.json.render(two) != floss.render.json.render(three) + + +def test_string_extraction(analyzed_layout): + strings = collect_strings(analyzed_layout) + # Check if a known string is extracted + assert any(s.string.string == "user32.dll" for s in strings) + + +def test_tagging(analyzed_layout): + strings = collect_strings(analyzed_layout) + # Check if a known string is tagged correctly + user32_string = next(s for s in strings if s.string.string == "user32.dll") + assert "#winapi" in user32_string.tags + + +def test_structure_marking(analyzed_layout): + strings = collect_strings(analyzed_layout) + # Check if a string is correctly associated with a structure + data_string = next(s for s in strings if s.string.string == "@.data") + assert data_string.structure == "section header" + + close_string = next(s for s in strings if s.string.string == "CloseHandle") + assert close_string.structure == "import table" + + +def test_analysis_pipeline(pma_binary_path): + # Run the analysis pipeline + slice_buf = pma_binary_path.read_bytes() + file_slice = Slice.from_bytes(slice_buf) + parsed = compute_layout(file_slice) + parsed.extract_strings(6) + + # Check that the layout has been computed correctly + assert parsed.name == "pe" + + +def test_is_structured_layout(): + import floss.enrich + + assert floss.enrich.is_structured_layout("pe") + assert floss.enrich.is_structured_layout("elf") + assert floss.enrich.is_structured_layout("macho") + assert floss.enrich.is_structured_layout("macho (fat)") + # XOR-obfuscated PE/ELF headers append the XOR note to the name + assert floss.enrich.is_structured_layout("pe (XOR decoded with key: 0x41)") + assert floss.enrich.is_structured_layout("elf (XOR decoded with key: 0x42)") + assert not floss.enrich.is_structured_layout("binary") + + +def _make_root_layout(cls, name, **extra): + from floss.ranges import Range, Slice, OffsetRanges + from floss.layout.base import Structure + from floss.layout.types import TaggedString, ExtractedString + + buf = b"\x00" * 32 + sl = Slice(buf=buf, range=Range(offset=0, length=len(buf)), base_offset=0) + layout = cls( + name=name, + slice=sl, + structures_by_address={0: Structure(slice=sl, name="pe/elf header")}, + reloc_offsets=OffsetRanges(ranges=[]), + code_offsets=OffsetRanges(ranges=[]), + **extra, + ) + layout.strings = [TaggedString(string=ExtractedString(string="rootstr", slice=sl, encoding="ascii"), tags=set())] + return layout + + +def test_pe_root_strings_get_structure_annotations(): + from floss.layout.base import PELayout + + layout = _make_root_layout(PELayout, "pe", xor_key=None) + layout.mark_structures() + assert layout.strings[0].structure == "pe/elf header" + + +def test_elf_root_strings_get_structure_annotations(): + from floss.ranges import OffsetRanges + from floss.layout.base import ELFLayout + + layout = _make_root_layout(ELFLayout, "elf", xor_key=None, relocation_offsets=OffsetRanges(ranges=[])) + layout.mark_structures() + assert layout.strings[0].structure == "pe/elf header" diff --git a/tests/test_layout_elf.py b/tests/test_layout_elf.py new file mode 100644 index 000000000..45613b13a --- /dev/null +++ b/tests/test_layout_elf.py @@ -0,0 +1,106 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from floss.layout import compute_layout +from floss.ranges import Slice +from floss.layout.base import ELFLayout, SegmentLayout + +CD = Path(__file__).resolve().parent +ELF_DIR = CD / "data" / "elf" + +# x86-64 Position Independent Executable (PIE), dynamically linked, not stripped +X86_64_PIE = "055da8e6ccfe5a9380231ea04b850e18.elf" +# ARM64 shared object, dynamically linked, stripped, Android linker +ARM64_SO = "687e79cde5b0ced75ac229465835054931f9ec438816f2827a8be5f3bd474929.elf" +# ARM64 PIE, dynamically linked, stripped +ARM64_LS = "ls" + + +def _load_layout(name: str): + path = ELF_DIR / name + data = path.read_bytes() + return compute_layout(Slice.from_bytes(data)) + + +def test_elf_layout(): + layout = _load_layout(X86_64_PIE) + assert isinstance(layout, ELFLayout) + assert layout.name == "elf" + assert layout.children + text_sections = [c for c in layout.children if c.name == ".text"] + assert len(text_sections) == 1 + assert text_sections[0].offset == 0x10A0 + assert text_sections[0].slice.range.length == 0x1C5 + + +def test_elf_section_names(): + # x86-64, not stripped: executable code section, dynamic string table, and symbol table all present + layout = _load_layout(X86_64_PIE) + names = {child.name for child in layout.children} + assert ".text" in names + assert ".dynstr" in names + assert ".symtab" in names + + +def test_elf_structures_present(): + layout = _load_layout(X86_64_PIE) + assert layout.structures_by_address[0x0].name == "elf header" + assert layout.structures_by_address[0x40].name == "program header" + assert layout.structures_by_address[0x39D0].name == "section header" + assert layout.structures_by_address[0x3C8].name == "symbol table" # .dynsym + assert layout.structures_by_address[0x4A0].name == "string table" # .dynstr + + +def test_elf_code_and_reloc_offsets(): + layout = _load_layout(X86_64_PIE) + # .text (offset 0x10a0, size 0x1c5) is fully covered by code ranges; + # it merges with adjacent exec sections (.plt etc.) so we check coverage, not exact range + assert layout.code_offsets.overlaps(0x10A0, 0x10A0 + 0x1C5 - 1) + assert (0x568, 0x66F) in layout.relocation_offsets.ranges # .rela.dyn + .rela.plt merged + + +def test_arm64_so_android_note_section(): + # Android shared object has an Android-specific note section absent from standard Linux ELFs + layout = _load_layout(ARM64_SO) + names = {child.name for child in layout.children} + assert ".note.android.ident" in names + + +def test_arm64_ls_stripped(): + # stripped binary has no symbol table + layout = _load_layout(ARM64_LS) + names = {child.name for child in layout.children} + assert ".symtab" not in names + + +def test_elf_segment_fallback(): + # Test fallback to segments when section headers are missing/corrupted + path = ELF_DIR / X86_64_PIE + data = bytearray(path.read_bytes()) + + # Verify ELFCLASS64 + assert data[4] == 2 + # Corrupt e_shoff (set 8 bytes at offset 40 to 0) + data[40:48] = b"\x00" * 8 + # Corrupt e_shnum (set 2 bytes at offset 60 to 0) + data[60:62] = b"\x00\x00" + + layout = compute_layout(Slice.from_bytes(bytes(data))) + assert isinstance(layout, ELFLayout) + assert layout.children + for child in layout.children: + assert isinstance(child, SegmentLayout) + assert child.name.startswith("segment_") diff --git a/tests/test_layout_macho.py b/tests/test_layout_macho.py new file mode 100644 index 000000000..0e24ad9fb --- /dev/null +++ b/tests/test_layout_macho.py @@ -0,0 +1,91 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from floss.layout import compute_layout +from floss.ranges import Slice + +CD = Path(__file__).resolve().parent +MACHO_DIR = CD / "data" / "macho" + + +def _load_layout(name: str): + path = MACHO_DIR / name + data = path.read_bytes() + return compute_layout(Slice.from_bytes(data)) + + +def test_thin_macho_layout(): + layout = _load_layout("ls") + assert layout.name.startswith("macho:") + assert layout.children + + +def test_thin_macho_segment_names(): + layout = _load_layout("regdmp") + names = {child.name for child in layout.children} + assert "__TEXT" in names + assert "__LINKEDIT" in names + + +def test_macho_structures_present(): + layout = _load_layout("ls") + assert getattr(layout, "structures_by_address", None) + + +def test_fat_macho_layout(): + layout = _load_layout("true") + assert layout.name == "macho (fat)" + assert len(layout.children) == 2 + assert {child.name for child in layout.children} == {"macho: x86_64", "macho: arm64e"} + + +def test_entitlements_plist_layout(): + layout = _load_layout("true") + found = [] + for arch in layout.children: + for child in arch.children: + if child.name == "__LINKEDIT": + for code_sig in child.children: + if code_sig.name != "code signature": + continue + for cert in code_sig.children: + if any(sub.name == "plist: entitlements" for sub in cert.children): + found.append(arch.name) + assert set(found) == {"macho: x86_64", "macho: arm64e"} + + +def test_entitlements_plist_parse(): + import plistlib + + path = MACHO_DIR / "true" + data = path.read_bytes() + layout = compute_layout(Slice.from_bytes(data)) + + parsed = 0 + for arch in layout.children: + for child in arch.children: + if child.name == "__LINKEDIT": + for code_sig in child.children: + if code_sig.name != "code signature": + continue + for cert in code_sig.children: + for subchild in cert.children: + if subchild.name == "plist: entitlements": + blob = data[subchild.offset : subchild.end] + plistlib.loads(blob) + parsed += 1 + + assert parsed == 2 diff --git a/tests/test_layout_pma0101.py b/tests/test_layout_pma0101.py new file mode 100644 index 000000000..825d0b391 --- /dev/null +++ b/tests/test_layout_pma0101.py @@ -0,0 +1,103 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from floss.tags import load_databases +from floss.layout import compute_layout +from floss.ranges import Slice +from floss.layout.extract import collect_strings + + +@pytest.fixture(scope="module") +def pma0101_layout(): + """ + Provides the analyzed layout. + The analysis pipeline (string extraction, tagging, structure marking) + is run once for all tests in this module. + """ + binary_path = Path("tests") / Path("data") / Path("pma") / Path("Practical Malware Analysis Lab 01-01.dll_") + slice_buf = binary_path.read_bytes() + file_slice = Slice.from_bytes(slice_buf) + parsed = compute_layout(file_slice) + parsed.extract_strings(6) + taggers = load_databases() + parsed.tag_strings(taggers) + parsed.mark_structures() + return parsed + + +def find_string(layout, text): + """Helper to find a specific string in the layout.""" + all_strings = collect_strings(layout) + found = [s for s in all_strings if s.string.string == text] + return found[0] if found else None + + +def test_pe_layout(pma0101_layout): + assert pma0101_layout.name == "pe" + + +def test_header_strings(pma0101_layout): + dos_mode_str = find_string(pma0101_layout, "!This program cannot be run in DOS mode.") + assert dos_mode_str is not None + assert "#common" in dos_mode_str.tags + + rdata_str = find_string(pma0101_layout, "@.data") + assert rdata_str is not None + assert "#common" in rdata_str.tags + assert rdata_str.structure == "section header" + + reloc_str = find_string(pma0101_layout, ".reloc") + assert reloc_str is not None + assert "#common" in reloc_str.tags + assert reloc_str.structure == "section header" + + +def test_rdata_strings(pma0101_layout): + kernel32_str = find_string(pma0101_layout, "KERNEL32.dll") + assert kernel32_str is not None + assert "#winapi" in kernel32_str.tags + assert kernel32_str.structure == "import table" + + msvcrt_str = find_string(pma0101_layout, "MSVCRT.dll") + assert msvcrt_str is not None + assert "#winapi" in msvcrt_str.tags + assert msvcrt_str.structure == "import table" + + initterm_str = find_string(pma0101_layout, "_initterm") + assert initterm_str is not None + assert "#winapi" in initterm_str.tags + assert "#code-junk" in initterm_str.tags + assert initterm_str.structure == "import table" + + +def test_data_strings(pma0101_layout): + ip_str = find_string(pma0101_layout, "127.26.152.13") + assert ip_str is not None + + garbage_str = find_string(pma0101_layout, "SADFHUHF") + assert garbage_str is not None + + +def test_strings(pma0101_layout): + all_strings = collect_strings(pma0101_layout) + + assert len(all_strings) == 21 + + # assert count of expected strings not tagged as #code or #reloc + filtered_strings = [s for s in all_strings if not s.tags.intersection({"#code", "#reloc"})] + assert len(filtered_strings) == 17 diff --git a/tests/test_load.py b/tests/test_load.py index 0ccca6f92..ea3280920 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,8 +1,14 @@ +import json import textwrap +from pathlib import Path + +import pytest +from fixtures import exefile import floss.main +import floss.utils -# floss --no static -j tests/data/src/decode-in-place/bin/test-decode-in-place.exe +# floss --no-string-type static -j tests/data/src/decode-in-place/bin/test-decode-in-place.exe RESULTS = textwrap.dedent(""" { "analysis": { @@ -94,9 +100,151 @@ def test_load(tmp_path): assert ( floss.main.main( [ - "-l", str(d.joinpath(p)), ] ) == 0 ) + + +def test_detect_file_type_returns_results_for_results_json(tmp_path): + p = tmp_path / "results.json" + p.write_text(RESULTS) + assert floss.utils.detect_file_type(p) is floss.utils.FileType.RESULTS + + +def test_detect_file_type_not_results_for_binary(exefile): + assert floss.utils.detect_file_type(Path(exefile)) is not floss.utils.FileType.RESULTS + + +def test_detect_file_type_not_results_for_invalid_json(tmp_path): + p = tmp_path / "invalid.json" + p.write_text("{not valid json") + assert floss.utils.detect_file_type(p) is not floss.utils.FileType.RESULTS + + +def test_detect_file_type_not_results_for_non_floss_json(tmp_path): + p = tmp_path / "other.json" + p.write_text(json.dumps({"hello": "world"})) + assert floss.utils.detect_file_type(p) is not floss.utils.FileType.RESULTS + + +def test_filter_string_len_filters_language_and_layout(): + """-n on a loaded document must prune language and layout strings too.""" + from floss.results import ( + Strings, + Analysis, + Metadata, + ResultLayout, + ResultString, + StaticString, + ResultDocument, + StringEncoding, + filter_string_len, + ) + + short = ResultString(string="ab", offset=1, size=2, encoding="ascii") + long = ResultString(string="abcdef", offset=2, size=6, encoding="ascii") + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis( + enable_static_strings=False, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + ), + strings=Strings( + language_strings=[StaticString(string="xy", offset=1, encoding=StringEncoding.ASCII)], + ), + layout=ResultLayout(name="pe", offset=0, length=10, strings=[short, long]), + ) + filter_string_len(doc, 4) + assert doc.layout is not None + assert [s.string for s in doc.layout.strings] == ["abcdef"] + assert doc.strings.language_strings == [] + + +def test_filter_string_len_aborts_when_requested_below_stored(): + """-n below the stored extraction min_length must abort: those strings are gone.""" + import pytest + + from floss.results import ( + Strings, + Analysis, + Metadata, + ResultDocument, + InvalidLoadConfig, + filter_string_len, + ) + + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=6), + analysis=Analysis( + enable_static_strings=False, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + ), + strings=Strings(), + ) + with pytest.raises(InvalidLoadConfig, match="minimum-length 4"): + filter_string_len(doc, 4) + + +def test_filter_string_len_ok_when_at_or_above_stored(): + """-n equal to or above the stored min_length is fine, no abort.""" + from floss.results import ( + Strings, + Analysis, + Metadata, + ResultDocument, + filter_string_len, + ) + + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis( + enable_static_strings=False, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + ), + strings=Strings(), + ) + filter_string_len(doc, 4) + filter_string_len(doc, 6) + + +def test_filter_functions_accepts_stack_only_function(): + """a function with only stack strings (no decoding score) is valid.""" + from floss.results import ( + Strings, + Analysis, + Metadata, + StackString, + ResultDocument, + StringEncoding, + InvalidLoadConfig, + filter_functions, + ) + + ss = StackString( + function=0x401000, + string="hello", + encoding=StringEncoding.ASCII, + program_counter=0x1000, + stack_pointer=0x4000, + original_stack_pointer=0x39A0, + offset=0x10, + frame_offset=0x10, + ) + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=True, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(stack_strings=[ss]), + ) + filter_functions(doc, [0x401000]) + assert len(doc.strings.stack_strings) == 1 + + with pytest.raises(InvalidLoadConfig): + filter_functions(doc, [0x999999]) diff --git a/tests/test_main.py b/tests/test_main.py index fcc4358ad..aaaf17371 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -18,15 +18,34 @@ from fixtures import exefile import floss.main +from floss.version import __version__ -def test_main_help(): - for help_str in ("-h", "-H"): +def test_main_help(capsys): + for help_str in ("-h", "--help"): # via https://medium.com/python-pandemonium/testing-sys-exit-with-pytest-10c6e5f7726f with pytest.raises(SystemExit) as pytest_wrapped_e: floss.main.main([help_str]) assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 0 + out = capsys.readouterr().out + assert "usage:" in out + assert "--json" in out + + # running without arguments prints the same help and exits with code 1 + assert floss.main.main([]) == 1 + out = capsys.readouterr().out + assert "usage:" in out + assert "--json" in out + + +def test_main_version(capsys): + with pytest.raises(SystemExit) as pytest_wrapped_e: + floss.main.main(["--version"]) + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + out = capsys.readouterr().out + assert __version__ in out def test_main(exefile): diff --git a/tests/test_offset_ranges.py b/tests/test_offset_ranges.py new file mode 100644 index 000000000..4b3351957 --- /dev/null +++ b/tests/test_offset_ranges.py @@ -0,0 +1,127 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from floss.ranges import OffsetRanges + + +def test_offset_ranges_init_empty(): + """Test initialization with no offsets.""" + offsets: set[int] = set() + ranges = OffsetRanges.from_offsets(offsets) + assert ranges.ranges == [] + + +def test_offset_ranges_init(): + """Test initialization with a mix of contiguous and non-contiguous offsets.""" + offsets = {0, 1, 2, 5, 6, 8, 10} + ranges = OffsetRanges.from_offsets(offsets) + assert ranges.ranges == [(0, 2), (5, 6), (8, 8), (10, 10)] + + +def test_offset_ranges_init_single_range(): + """Test initialization with a single contiguous block of offsets.""" + offsets = {10, 11, 12, 13, 14} + ranges = OffsetRanges.from_offsets(offsets) + assert ranges.ranges == [(10, 14)] + + +def test_offset_ranges_from_merged_ranges(): + """Test the from_merged_ranges class method.""" + merged = [(10, 20), (30, 40)] + ranges = OffsetRanges.from_merged_ranges(merged) + assert ranges.ranges == [(10, 20), (30, 40)] + + +@pytest.fixture +def sample_ranges(): + """Provides a standard OffsetRanges instance for testing.""" + # Ranges will be: (10, 15), (20, 25), (30, 30) + offsets = {10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 30} + return OffsetRanges.from_offsets(offsets) + + +def test_contains_empty(sample_ranges): + """Test __contains__ on an empty OffsetRanges instance.""" + empty_ranges = OffsetRanges() + assert 10 not in empty_ranges + + +def test_contains_inside(sample_ranges): + """Test __contains__ for an offset well within a range.""" + assert 12 in sample_ranges + assert 23 in sample_ranges + + +def test_contains_edges(sample_ranges): + """Test __contains__ for offsets at the exact start and end of ranges.""" + assert 10 in sample_ranges # Start of first range + assert 15 in sample_ranges # End of first range + assert 20 in sample_ranges # Start of second range + assert 25 in sample_ranges # End of second range + assert 30 in sample_ranges # Single-point range + + +def test_contains_outside(sample_ranges): + """Test __contains__ for offsets outside of any range.""" + assert 9 not in sample_ranges # Before first range + assert 16 not in sample_ranges # Between ranges + assert 29 not in sample_ranges # Between ranges + assert 31 not in sample_ranges # After last range + + +def test_overlaps_empty(sample_ranges): + """Test overlaps on an empty OffsetRanges instance.""" + empty_ranges = OffsetRanges() + assert not empty_ranges.overlaps(10, 20) + + +def test_overlaps_fully_contained(sample_ranges): + """Test overlaps where the query range is fully inside an existing range.""" + assert sample_ranges.overlaps(11, 14) # Fully inside (10, 15) + assert sample_ranges.overlaps(21, 22) # Fully inside (20, 25) + + +def test_overlaps_contains_full_range(sample_ranges): + """Test overlaps where the query range fully contains an existing range.""" + assert sample_ranges.overlaps(9, 16) # Contains (10, 15) + assert sample_ranges.overlaps(19, 26) # Contains (20, 25) + assert sample_ranges.overlaps(29, 31) # Contains (30, 30) + + +def test_overlaps_start(sample_ranges): + """Test overlaps where the query range overlaps the beginning of an existing range.""" + assert sample_ranges.overlaps(8, 12) # Overlaps start of (10, 15) + assert sample_ranges.overlaps(18, 20) # Touches start of (20, 25) + + +def test_overlaps_end(sample_ranges): + """Test overlaps where the query range overlaps the end of an existing range.""" + assert sample_ranges.overlaps(14, 17) # Overlaps end of (10, 15) + assert sample_ranges.overlaps(25, 28) # Touches end of (20, 25) + + +def test_overlaps_multiple_ranges(sample_ranges): + """Test overlaps where the query range spans across multiple existing ranges.""" + assert sample_ranges.overlaps(12, 22) # Spans from first to second range + assert sample_ranges.overlaps(14, 30) # Spans all three ranges + + +def test_no_overlap(sample_ranges): + """Test overlaps for ranges that do not overlap at all.""" + assert not sample_ranges.overlaps(0, 8) # Before all ranges + assert not sample_ranges.overlaps(16, 19) # Between ranges + assert not sample_ranges.overlaps(26, 29) # Between ranges + assert not sample_ranges.overlaps(31, 40) # After all ranges diff --git a/tests/test_oss_db.py b/tests/test_oss_db.py new file mode 100644 index 000000000..5cb634b03 --- /dev/null +++ b/tests/test_oss_db.py @@ -0,0 +1,37 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import floss.tags.oss +from floss.tags import data_root + + +def test_load_db(): + path = data_root() / "oss" / "zlib.jsonl.gz" + db = floss.tags.oss.OpenSourceStringDatabase.from_file(path) + assert len(db) > 0 # 21 entries at time of writing + + +def test_query_db(): + path = data_root() / "oss" / "zlib.jsonl.gz" + db = floss.tags.oss.OpenSourceStringDatabase.from_file(path) + + s = db.metadata_by_string["invalid distance code"] + + assert s is not None + assert s.string == "invalid distance code" + assert s.library_name == "zlib" + assert s.library_version == "1.2.13" + assert s.file_path == "CMakeFiles/zlib.dir/inffast.obj" + assert s.function_name == "inflate_fast" + assert s.line_number is None diff --git a/tests/test_ranges.py b/tests/test_ranges.py new file mode 100644 index 000000000..5fbf800d3 --- /dev/null +++ b/tests/test_ranges.py @@ -0,0 +1,89 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from floss.ranges import Range, Slice + + +def test_range_slice(): + r = Range(offset=10, length=20) + assert r.end == 30 + + # Valid slice + s = r.slice(5, 10) + assert s.offset == 15 + assert s.length == 10 + + # Boundary: offset 0 + s = r.slice(0, 5) + assert s.offset == 10 + assert s.length == 5 + + # Boundary: offset + size == length + s = r.slice(15, 5) + assert s.offset == 25 + assert s.length == 5 + + # Boundary: offset == length, size 0 + s = r.slice(20, 0) + assert s.offset == 30 + assert s.length == 0 + + # Invalid: offset < 0 + with pytest.raises(AssertionError): + r.slice(-1, 5) + + # Invalid: size < 0 + with pytest.raises(AssertionError): + r.slice(5, -1) + + # Invalid: offset > length + with pytest.raises(AssertionError): + r.slice(21, 0) + + # Invalid: offset + size > length + with pytest.raises(AssertionError): + r.slice(15, 6) + + +def test_slice_contains_range(): + buf = b"A" * 100 + s = Slice(buf=buf, range=Range(offset=10, length=20)) + # s covers buf[10:30] + + # Valid sub-ranges (relative to slice start) + assert s.contains_range(0, 20) is True + assert s.contains_range(5, 10) is True + assert s.contains_range(0, 0) is True + assert s.contains_range(20, 0) is True # Boundary at the very end + + # Invalid: offset < 0 + assert s.contains_range(-1, 5) is False + + # Invalid: offset > length + assert s.contains_range(21, 0) is False + + # Invalid: size < 0 + assert s.contains_range(5, -1) is False + + # Invalid: offset + size > length + assert s.contains_range(15, 6) is False + assert s.contains_range(20, 1) is False + + # Edge case: offset == length, size == 0 + assert s.contains_range(20, 0) is True + + # Edge case: offset == length, size == -1 (handled by size < 0) + assert s.contains_range(20, -1) is False diff --git a/tests/test_render_filters.py b/tests/test_render_filters.py new file mode 100644 index 000000000..7c336b102 --- /dev/null +++ b/tests/test_render_filters.py @@ -0,0 +1,826 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render-time filters, columns, and summary output for the layout view.""" + +import json + +from fixtures import exefile + +import floss.main +import floss.render.filter +import floss.render.summary +from floss.results import ( + Strings, + Analysis, + Metadata, + ResultLayout, + ResultString, + StaticString, + ResultDocument, + StringEncoding, +) +from floss.tags.filter import TagRules +from floss.render.layout import render_strings +from floss.render.default import render + + +def make_layout() -> ResultLayout: + """a small PE-like tree with tagged strings in .rdata.""" + return ResultLayout( + name="pe", + offset=0, + length=0x1000, + children=[ + ResultLayout( + name=".text", + offset=0x400, + length=0x400, + strings=[ + ResultString(string="junk code", offset=0x410, size=9, encoding="ascii", tags=["#code"]), + ], + ), + ResultLayout( + name=".rdata", + offset=0x800, + length=0x800, + strings=[ + ResultString( + string="CreateFileA", + offset=0x810, + size=11, + encoding="ascii", + tags=["#winapi"], + structure="import table", + ), + ResultString( + string="hello world", + offset=0x820, + size=11, + encoding="ascii", + tags=["#common"], + ), + ResultString( + string="kcp://url", + offset=0x830, + size=9, + encoding="ascii", + tags=["#kcp"], + ), + ], + ), + ], + ) + + +def make_results() -> ResultDocument: + return ResultDocument( + metadata=Metadata(file_path="test.exe", min_length=4), + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + ), + strings=Strings( + static_strings=[ + StaticString(string="CreateFileA", offset=0x810, encoding=StringEncoding.ASCII), + StaticString(string="hello world", offset=0x820, encoding=StringEncoding.ASCII), + StaticString(string="kcp://url", offset=0x830, encoding=StringEncoding.ASCII), + ] + ), + layout=make_layout(), + ) + + +def collect_layout_strings(layout): + out = [s.string for s in layout.strings] + for child in layout.children: + out.extend(collect_layout_strings(child)) + return out + + +def test_filter_by_section(): + f = floss.render.filter.LayoutFilter(include_sections=[".rdata"]) + filtered = f.apply(make_layout()) + assert filtered is not None + strings = collect_layout_strings(filtered) + assert "CreateFileA" in strings + assert "hello world" in strings + assert "junk code" not in strings + # empty branches are pruned + assert ".text" not in [c.name for c in filtered.children] + + +def test_filter_exclude_section(): + f = floss.render.filter.LayoutFilter(exclude_sections=[".text"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert "junk code" not in strings + assert "CreateFileA" in strings + + +def test_filter_by_section_matches_nested_nodes(): + """--section must match strings living in nested nodes under the section.""" + layout = ResultLayout( + name="pe", + offset=0, + length=0x1000, + children=[ + ResultLayout( + name=".rdata", + offset=0x800, + length=0x800, + children=[ + ResultLayout( + name="import table", + offset=0x810, + length=0x10, + strings=[ + ResultString( + string="CloseHandle", + offset=0x812, + size=11, + encoding="ascii", + tags=["#winapi"], + structure="import table", + ) + ], + ) + ], + ) + ], + ) + f = floss.render.filter.LayoutFilter(include_sections=[".rdata"]) + filtered = f.apply(layout) + strings = collect_layout_strings(filtered) + assert strings == ["CloseHandle"] + + +def test_filter_by_section_fat_macho_descends_arch_wrappers(): + """--section must descend through Mach-O fat-arch wrapper layers.""" + layout = ResultLayout( + name="macho (fat)", + offset=0, + length=0x2000, + children=[ + ResultLayout( + name="macho: x86_64", + offset=0, + length=0x1000, + children=[ + ResultLayout( + name="__TEXT", + offset=0, + length=0x800, + strings=[ + ResultString(string="in text", offset=1, size=7, encoding="ascii"), + ], + ), + ResultLayout( + name="__DATA", + offset=0x800, + length=0x800, + strings=[ + ResultString(string="in data", offset=2, size=7, encoding="ascii"), + ], + ), + ], + ), + ], + ) + f = floss.render.filter.LayoutFilter(include_sections=["__TEXT"]) + filtered = f.apply(layout) + strings = collect_layout_strings(filtered) + assert strings == ["in text"] + + +def test_filter_max_strings_fat_macho_caps_per_section(): + """--max-strings caps each fat Mach-O segment, not the whole architecture.""" + layout = ResultLayout( + name="macho (fat)", + offset=0, + length=0x300, + children=[ + ResultLayout( + name="macho: x86_64", + offset=0, + length=0x200, + children=[ + ResultLayout( + name="__TEXT", + offset=0, + length=0x100, + strings=[ + ResultString(string="a", offset=1, size=1, encoding="ascii", tags=["#winapi"]), + ResultString(string="b", offset=2, size=1, encoding="ascii", tags=["#winapi"]), + ], + ), + ResultLayout( + name="__DATA", + offset=0x100, + length=0x100, + strings=[ + ResultString(string="c", offset=3, size=1, encoding="ascii", tags=["#winapi"]), + ], + ), + ], + ), + ], + ) + f = floss.render.filter.LayoutFilter(max_strings=1, tag_rules={"#winapi": "default"}) + filtered = f.apply(layout) + strings = collect_layout_strings(filtered) + # one string per segment: a from __TEXT and c from __DATA + assert strings == ["a", "c"] + + +def test_filter_by_structure_slug(): + f = floss.render.filter.LayoutFilter(include_structures=["import-table"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["CreateFileA"] + + +def test_filter_by_structure_slug_variants(): + """import-table, import_table, and import table all match the same structure.""" + for slug in ("import-table", "import_table", "import table"): + f = floss.render.filter.LayoutFilter(include_structures=[slug]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["CreateFileA"], slug + + +def test_filter_no_structure(): + """--no-structure drops strings in the given structure and keeps the rest.""" + f = floss.render.filter.LayoutFilter(exclude_structures=["import-table"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert "CreateFileA" not in strings + assert "hello world" in strings + + +def test_filter_by_tag(): + f = floss.render.filter.LayoutFilter(include_tags=["winapi"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["CreateFileA"] + + +def test_filter_exclude_tag(): + f = floss.render.filter.LayoutFilter(exclude_tags=["common"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert "CreateFileA" in strings + assert "hello world" not in strings + + +def test_filter_repeated_tags_accumulate(): + """repeated --tag values are ORed.""" + f = floss.render.filter.LayoutFilter(include_tags=["winapi", "kcp"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert "CreateFileA" in strings + assert "kcp://url" in strings + + +def test_filter_oss_meta_tag(): + """the oss meta tag matches any OSS library tag.""" + f = floss.render.filter.LayoutFilter(include_tags=["oss"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["kcp://url"] + + +def test_filter_tag_family_gp(): + """the gp tag family matches global-prevalence tags like #common.""" + f = floss.render.filter.LayoutFilter(include_tags=["gp"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["hello world"] + + +def test_filter_tag_normalization(): + """--tag input is case- and #-insensitive (#WinAPI and WinAPI match #winapi).""" + for tag in ("#WINAPI", "WinAPI", "#winapi", "winapi"): + f = floss.render.filter.LayoutFilter(include_tags=[tag]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["CreateFileA"], tag + + +def test_filter_interesting(): + f = floss.render.filter.LayoutFilter(interesting=True) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + # #code, #common are noisy; #winapi and #kcp are not + assert "CreateFileA" in strings + assert "hello world" not in strings + assert "junk code" not in strings + assert "kcp://url" in strings + + +def test_filter_interesting_drops_strings_with_any_noisy_tag(): + """--interesting drops a string if it carries any noisy tag, even alongside + a non-noisy tag (e.g. #winapi #common is dropped).""" + layout = ResultLayout( + name=".rdata", + offset=0, + length=0x10, + strings=[ + ResultString(string="CreateFileA", offset=1, size=11, encoding="ascii", tags=["#winapi", "#common"]), + ResultString(string="only winapi", offset=2, size=11, encoding="ascii", tags=["#winapi"]), + ], + ) + f = floss.render.filter.LayoutFilter(interesting=True) + filtered = f.apply(layout) + assert filtered is not None + assert [s.string for s in filtered.strings] == ["only winapi"] + + +def test_filter_query(): + f = floss.render.filter.LayoutFilter(queries=["CreateFile"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert strings == ["CreateFileA"] + + +def test_filter_query_ored(): + f = floss.render.filter.LayoutFilter(queries=["CreateFile", "kcp"]) + filtered = f.apply(make_layout()) + strings = collect_layout_strings(filtered) + assert "CreateFileA" in strings + assert "kcp://url" in strings + + +def test_filter_max_strings_relevance(): + # highlighted tags (#capa) sort first, then untagged, then non-noisy tags, + # then only-noisy tags; within a group, ascending offset. + layout = ResultLayout( + name=".rdata", + offset=0x800, + length=0x400, + strings=[ + ResultString(string="z-only-noisy", offset=0x810, size=1, encoding="ascii", tags=["#common"]), + ResultString(string="a-untagged", offset=0x801, size=1, encoding="ascii", tags=[]), + ResultString(string="m-tagged", offset=0x805, size=1, encoding="ascii", tags=["#winapi"]), + ResultString(string="b-untagged", offset=0x802, size=1, encoding="ascii", tags=[]), + ResultString(string="c-highlighted", offset=0x800, size=1, encoding="ascii", tags=["#capa"]), + ], + ) + tag_rules: TagRules = {"#capa": "highlight", "#common": "mute", "#winapi": "default"} + f = floss.render.filter.LayoutFilter(max_strings=3, tag_rules=tag_rules) + filtered = f.apply(layout) + assert filtered is not None + assert [s.string for s in filtered.strings] == ["c-highlighted", "a-untagged", "b-untagged"] + + +def test_filter_max_strings_caps_per_section(): + """--max-strings caps each top-level section, not each nested node.""" + s1 = ResultString(string="one", offset=1, size=3, encoding="ascii", tags=["#winapi"]) + s2 = ResultString(string="two", offset=2, size=3, encoding="ascii", tags=["#winapi"]) + s3 = ResultString(string="three", offset=3, size=5, encoding="ascii", tags=["#winapi"]) + s4 = ResultString(string="four", offset=4, size=4, encoding="ascii", tags=["#winapi"]) + layout = ResultLayout( + name="pe", + offset=0, + length=0x100, + children=[ + ResultLayout( + name=".rdata", + offset=0, + length=0x80, + children=[ + ResultLayout(name="import table", offset=0, length=0x40, strings=[s1, s2]), + ResultLayout(name="export table", offset=0x40, length=0x40, strings=[s3, s4]), + ], + ), + ResultLayout( + name=".text", + offset=0x80, + length=0x80, + strings=[ResultString(string="five", offset=0x90, size=4, encoding="ascii", tags=["#winapi"])], + ), + ], + ) + f = floss.render.filter.LayoutFilter(max_strings=1, tag_rules={"#winapi": "default"}) + filtered = f.apply(layout) + assert filtered is not None + strings = collect_layout_strings(filtered) + # one per top-level section (.rdata and .text) + assert len(strings) == 2 + + +def test_columns_hide_tags(): + out = render(make_results(), True, False, "auto", columns=["offset"]) + assert "CreateFileA" in out + assert "#winapi" not in out + + +def test_columns_show_structure(): + out = render(make_results(), True, False, "auto", columns=["offset", "structure"]) + assert "import table" in out + + +def test_columns_encoding(): + """--columns encoding renders U for unicode strings and nothing for ascii.""" + unicode_s = ResultString(string="héllo", offset=1, size=6, encoding="unicode") + ascii_s = ResultString(string="ascii", offset=2, size=5, encoding="ascii") + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(), + layout=ResultLayout(name="pe", offset=0, length=10, strings=[unicode_s, ascii_s]), + ) + out = render(doc, True, False, "auto", columns=["encoding"]) + # unicode marker rendered, ascii marker absent (rich trims the trailing space) + assert "U" in out + + +def test_filter_all_removed_renders_empty(): + """a filter that matches nothing must not fall back to the unfiltered layout.""" + f = floss.render.filter.LayoutFilter(include_tags=["#nonexistent"]) + out = render(make_results(), True, False, "auto", layout_filter=f) + assert "#winapi" not in out + assert "CreateFileA" not in out + assert "kcp://url" not in out + + +def test_tag_filter_overrides_default_hide_rules(): + """--tag code / --tag winapi must show strings that carry a hide-rule tag + (#code/#reloc) which the default render would suppress.""" + layout = ResultLayout( + name=".rdata", + offset=0, + length=0x10, + strings=[ + ResultString(string="junk code", offset=1, size=9, encoding="ascii", tags=["#code", "#winapi"]), + ], + ) + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(), + layout=layout, + ) + + for include in ("code", "winapi"): + out = render( + doc, + True, + False, + "auto", + layout_filter=floss.render.filter.LayoutFilter(include_tags=[include]), + ) + assert "junk code" in out, include + + # --interesting drops any string carrying a noisy tag, including #code + out = render(doc, True, False, "auto", layout_filter=floss.render.filter.LayoutFilter(interesting=True)) + assert "junk code" not in out + + # without a tag filter, the default hide rules still suppress #code strings + out = render(doc, True, False, "auto") + assert "junk code" not in out + + +def test_summary_output(): + out = floss.render.summary.render_summary(make_results()) + assert "FLOSS SUMMARY" in out + assert "CreateFileA" in out + assert "#winapi" in out + + +def test_summary_section_counts_thread_top_level(): + """nested structure nodes count under their containing top-level section.""" + root_s = ResultString(string="root", offset=1, size=4, encoding="ascii") + sec_s = ResultString(string="sec", offset=2, size=3, encoding="ascii") + nested_s = ResultString(string="nested", offset=3, size=6, encoding="ascii") + layout = ResultLayout( + name="pe", + offset=0, + length=20, + strings=[root_s], + children=[ + ResultLayout( + name=".rdata", + offset=0, + length=10, + strings=[sec_s], + children=[ + ResultLayout(name="import table", offset=0, length=10, strings=[nested_s]), + ], + ) + ], + ) + counts, _, _ = floss.render.summary.analyze_layout(layout) + assert counts == {"pe": 1, ".rdata": 2} + assert sum(counts.values()) == 3 + + +def test_summary_section_counts_thread_fat_macho(): + """on a fat Mach-O, strings under an arch wrapper's segments count under + their segment (__TEXT), while the wrapper's own strings count under the + arch wrapper (macho: x86_64), and the root strings count under the root.""" + root_s = ResultString(string="root", offset=1, size=4, encoding="ascii") + wrapper_s = ResultString(string="wrapper", offset=2, size=7, encoding="ascii") + text_s = ResultString(string="text", offset=3, size=4, encoding="ascii") + layout = ResultLayout( + name="macho (fat)", + offset=0, + length=20, + strings=[root_s], + children=[ + ResultLayout( + name="macho: x86_64", + offset=0, + length=10, + strings=[wrapper_s], + children=[ + ResultLayout(name="__TEXT", offset=0, length=10, strings=[text_s]), + ], + ) + ], + ) + counts, _, _ = floss.render.summary.analyze_layout(layout) + assert counts == {"macho (fat)": 1, "__TEXT": 1, "macho: x86_64": 1} + assert sum(counts.values()) == 3 + + +def test_main_summary_flag(exefile, capsys): + assert floss.main.main([exefile, "--summary"]) == 0 + out = capsys.readouterr().out + assert "FLOSS SUMMARY" in out + assert "extracted strings" in out + + +def test_main_summary_is_static_only_by_default(exefile, capsys): + """--summary alone skips stack/tight/decoded extraction.""" + assert floss.main.main([exefile, "--summary"]) == 0 + out = capsys.readouterr().out + assert "FLOSS SUMMARY" in out + # counts show recovered strings as 0 because they were not extracted + assert "stack 0" in out + assert "tight 0" in out + assert "decoded 0" in out + + +def test_main_summary_rejects_string_type(exefile, capsys): + """--summary is its own static-only view, so any string-type selection is rejected.""" + assert floss.main.main([exefile, "--summary", "--string-type", "static"]) == -1 + err = capsys.readouterr().err + assert "--summary" in err + + assert floss.main.main([exefile, "--summary", "--no-string-type", "static"]) == -1 + err = capsys.readouterr().err + assert "--summary" in err + + +def test_main_summary_rejects_analyze_functions(exefile, capsys): + """--summary is static-only and --analyze-functions cannot show static, so + combining them must error instead of silently disabling every type.""" + assert floss.main.main([exefile, "--summary", "--analyze-functions", "0x401000"]) == -1 + err = capsys.readouterr().err + assert "--summary" in err + assert "--analyze-functions" in err + + +def test_summary_no_layout_ok(): + """--summary must not crash when there is no layout tree.""" + results = make_results() + results.layout = None + out = floss.render.summary.render_summary(results, "auto") + assert "FLOSS SUMMARY" in out + assert "extracted strings" in out + + +def test_main_json_error(capsys, exefile): + assert floss.main.main([exefile, "-j", "--section", ".text", "--no-section", ".data"]) == -1 + err = capsys.readouterr().err + # in JSON mode the whole STDERR output is a single JSON object, no usage text + obj = json.loads(err) + assert "error" in obj + assert obj["code"] == 1 + + +def test_main_invalid_query_regex_json_error(capsys, exefile): + assert floss.main.main([exefile, "-j", "--query", "["]) == -1 + err = capsys.readouterr().err + obj = json.loads(err) + assert "query" in obj["error"] + assert obj["code"] == 1 + + +def test_main_invalid_query_regex_text_error(capsys, exefile): + assert floss.main.main([exefile, "--query", "["]) == -1 + captured = capsys.readouterr() + # the error message and usage both go to stderr in text mode + assert "invalid --query regular expression" in captured.err + assert "usage:" in captured.err + + +def test_main_columns_flag(exefile, capsys): + assert ( + floss.main.main([exefile, "--columns", "offset", "structure", "--no-string-type", "stack", "tight", "decoded"]) + == 0 + ) + out = capsys.readouterr().out + assert "KERNEL32.dll" in out + assert "import table" in out + + +def test_main_columns_invalid_value(exefile): + assert floss.main.main([exefile, "--columns", "bogus"]) == -1 + + +def test_main_columns_with_plain(exefile): + """--columns is irrelevant with --plain (no layout), but must not crash.""" + assert floss.main.main([exefile, "--plain", "--columns", "offset", "structure"]) == 0 + + +def test_plain_render_does_not_mutate_results(): + """--plain selects the classic view without mutating the result document.""" + layout = ResultLayout( + name="pe", + offset=0, + length=0x10, + strings=[ResultString(string="x", offset=1, size=1, encoding="ascii")], + ) + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(), + layout=layout, + ) + out = render(doc, True, False, "auto", plain=True) + assert "FLARE FLOSS RESULTS" in out + assert doc.layout is not None + + +def test_plain_render_applies_filters(): + """--plain is filter-aware: --query/--tag narrow the flat listing.""" + s = ResultString(string="http://evil", offset=1, size=11, encoding="ascii", tags=["#winapi"]) + s2 = ResultString(string="junk", offset=2, size=4, encoding="ascii", tags=["#common"]) + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings( + static_strings=[ + StaticString(string="http://evil", offset=1, encoding=StringEncoding.ASCII, tags=["#winapi"]), + StaticString(string="junk", offset=2, encoding=StringEncoding.ASCII, tags=["#common"]), + ] + ), + layout=ResultLayout(name="pe", offset=0, length=10, strings=[s, s2]), + ) + out = render(doc, True, False, "auto", plain=True, layout_filter=floss.render.filter.LayoutFilter(queries=["http"])) + assert "http://evil" in out + assert "junk" not in out + + +def test_classic_meta_fallback_when_layout_present_but_static_disabled(): + """with a layout present but static strings disabled, the classic metadata + table renders as a fallback (file path, language, etc.) instead of dropping + all sample context.""" + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis( + enable_static_strings=False, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + ), + strings=Strings(), + layout=ResultLayout(name="pe", offset=0, length=10), + ) + out = render(doc, True, False, "auto") + assert "FLARE FLOSS RESULTS" in out + assert "file path" in out + # the layout tree itself is not shown when static strings are disabled + assert "pe" not in out + + +def test_layout_none_warns_filters_ignored(caplog): + """an active layout filter with no layout tree must warn, not silently no-op.""" + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(), + layout=None, + ) + with caplog.at_level("WARNING", logger="floss.render.default"): + render(doc, True, False, "auto", layout_filter=floss.render.filter.LayoutFilter(include_tags=["winapi"])) + assert "no layout tree" in caplog.text + assert "ignored" in caplog.text + + +def test_summary_escapes_rich_markup(): + """summary must render strings containing Rich markup literally, not crash.""" + s = ResultString(string="[bold red]X[/bold]", offset=1, size=12, encoding="ascii", tags=["#winapi"]) + doc = ResultDocument( + metadata=Metadata(file_path="x", min_length=4), + analysis=Analysis(enable_stack_strings=False, enable_tight_strings=False, enable_decoded_strings=False), + strings=Strings(), + layout=ResultLayout(name="pe", offset=0, length=10, strings=[s]), + ) + out = floss.render.summary.render_summary(doc) + assert "[bold red]X[/bold]" in out + + +def test_summary_ignores_plain(exefile, capsys): + """--summary with --plain still renders the layout-backed summary.""" + assert floss.main.main([exefile, "--summary", "--plain"]) == 0 + out = capsys.readouterr().out + assert "FLOSS SUMMARY" in out + assert "preview" in out + + +def test_main_summary_with_json(exefile, capsys): + """--json takes precedence over --summary.""" + assert floss.main.main([exefile, "-j", "--summary"]) == 0 + out = capsys.readouterr().out + assert "FLOSS SUMMARY" not in out + json.loads(out) + + +def test_main_filter_mutual_exclusion(capsys, exefile): + for args in ( + ["--structure", "import-table", "--no-structure", "export-table"], + ["--tag", "winapi", "--no-tag", "crypto"], + ): + assert floss.main.main([exefile] + args) == -1 + captured = capsys.readouterr() + assert "not allowed together" in captured.out + captured.err + + +def test_main_repeated_tags_accumulate(exefile, capsys): + """repeated --tag flags accumulate (OR semantics).""" + assert ( + floss.main.main([exefile, "--tag", "winapi", "--tag", "msvc", "--no-string-type", "stack", "tight", "decoded"]) + == 0 + ) + out = capsys.readouterr().out + assert "KERNEL32.dll" in out + + +def test_main_max_strings_invalid(exefile, capsys): + for value in ("0", "-1"): + assert floss.main.main([exefile, "--max-strings", value]) == -1 + captured = capsys.readouterr() + assert "positive integer" in captured.out + captured.err + + +def test_main_max_strings_larger_than_section(exefile, capsys): + """a max larger than the section size is fine and returns everything.""" + assert floss.main.main([exefile, "--max-strings", "100000", "--no-string-type", "stack", "tight", "decoded"]) == 0 + out = capsys.readouterr().out + assert "KERNEL32.dll" in out + + +def _render_layout(layout): + import io + + from rich.console import Console + + console = Console(file=io.StringIO(), width=80) + render_strings(console, layout, {}) + return console.file.getvalue() # type: ignore [attr-defined] + + +def test_render_offset_not_truncated_by_depth(): + """rendering at depth must not chop characters off the offset column.""" + s = ResultString(string="hello", offset=0x810, size=5, encoding="ascii", tags=["#winapi"]) + layout = ResultLayout(name=".rdata", offset=0, length=0x900, strings=[s]) + out = _render_layout(layout) + assert "00000810" in out + + +def test_render_boundary_string_not_omitted(): + """a string starting exactly at a child boundary must be rendered.""" + s = ResultString(string="boundary", offset=0xA0, size=8, encoding="ascii") + child = ResultLayout(name=".child", offset=0x50, length=0x50) + layout = ResultLayout(name="pe", offset=0, length=0x150, strings=[s], children=[child]) + out = _render_layout(layout) + assert "boundary" in out + + +def test_render_single_child_collapse_keeps_own_strings(): + """collapsing a node into its sole dominating child must not drop the + parent's own strings.""" + own = ResultString(string="ownstring", offset=0x250, size=9, encoding="ascii") + child_s = ResultString(string="childstr", offset=0x100, size=8, encoding="ascii") + child = ResultLayout(name="inner", offset=0, length=0x200, strings=[child_s]) + # child spans [0, 0x200); parent's own string sits in the gap after it + layout = ResultLayout(name="outer", offset=0, length=0x300, strings=[own], children=[child]) + out = _render_layout(layout) + assert "ownstring" in out + assert "childstr" in out diff --git a/tests/test_results_json_roundtrip.py b/tests/test_results_json_roundtrip.py new file mode 100644 index 000000000..c12a469b4 --- /dev/null +++ b/tests/test_results_json_roundtrip.py @@ -0,0 +1,179 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""JSON round-trip for the unified ResultDocument schema (enrichment fields).""" + +import json +import tempfile +import dataclasses +from pathlib import Path + +import floss.utils +import floss.render.json +from floss.results import ( + Strings, + Analysis, + Metadata, + ResultLayout, + ResultString, + StaticString, + ResultDocument, + StringEncoding, +) + + +def test_enriched_static_json_roundtrip(): + layout = ResultLayout( + name="pe", + offset=0, + length=0x1000, + strings=[ + ResultString( + string="kernel32.dll", + offset=0x100, + size=12, + encoding="ascii", + tags=["#winapi", "#common"], + structure="import table", + ) + ], + children=[], + ) + doc = ResultDocument( + metadata=Metadata( + file_path="sample.exe", + min_length=4, + md5="d" * 32, + sha1="a" * 40, + sha256="b" * 64, + ), + analysis=Analysis( + enable_static_strings=True, + enable_stack_strings=False, + enable_tight_strings=False, + enable_decoded_strings=False, + enable_layout=True, + enable_tags=True, + ), + strings=Strings( + static_strings=[ + StaticString( + string="kernel32.dll", + offset=0x100, + encoding=StringEncoding.ASCII, + tags=["#common", "#winapi"], + section="pe", + structure="import table", + ) + ] + ), + layout=layout, + ) + + raw = floss.render.json.render(doc) + data = json.loads(raw) + assert data["metadata"]["md5"] == "d" * 32 + assert data["metadata"]["sha1"] == "a" * 40 + assert data["metadata"]["sha256"] == "b" * 64 + assert data["layout"]["name"] == "pe" + assert data["strings"]["static_strings"][0]["tags"] == ["#common", "#winapi"] + assert data["strings"]["static_strings"][0]["section"] == "pe" + assert data["strings"]["static_strings"][0]["structure"] == "import table" + assert data["analysis"]["enable_layout"] is True + + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + f.write(raw) + path = Path(f.name) + try: + loaded = ResultDocument.parse_file(path) + finally: + path.unlink() + + assert loaded.layout is not None + assert loaded.layout.name == "pe" + assert loaded.metadata.md5 == "d" * 32 + assert loaded.metadata.sha256 == "b" * 64 + assert loaded.strings.static_strings[0].tags == ["#common", "#winapi"] + assert loaded.strings.static_strings[0].structure == "import table" + + +def test_layout_none_roundtrip(): + doc = ResultDocument( + metadata=Metadata(file_path="blob.bin", min_length=4), + analysis=Analysis(enable_layout=False, enable_tags=False), + strings=Strings( + static_strings=[ + StaticString(string="hello", offset=0, encoding=StringEncoding.ASCII), + ] + ), + layout=None, + ) + raw = floss.render.json.render(doc) + data = json.loads(raw) + assert data["layout"] is None + assert data["strings"]["static_strings"][0]["tags"] == [] + + +def test_top_level_key_order_metadata_first(): + """the top-level keys must keep metadata near the start so results + documents are detectable from their leading bytes (see detect_file_type).""" + doc = ResultDocument( + metadata=Metadata(file_path="sample.exe", min_length=4), + analysis=Analysis(enable_layout=True, enable_tags=True), + strings=Strings(static_strings=[]), + layout=ResultLayout(name="pe", offset=0, length=0x1000), + ) + raw = floss.render.json.render(doc) + # metadata must appear before the (potentially large) layout tree + assert raw.index('"metadata"') < raw.index('"layout"') + data = json.loads(raw) + # nested keys are still sorted + assert list(data["metadata"].keys()) == sorted(data["metadata"].keys()) + + +def test_rendered_document_is_detectable(tmp_path): + """a JSON document produced by the renderer must be recognized as a + results document by detect_file_type (sniffs the leading bytes).""" + doc = ResultDocument( + metadata=Metadata(file_path="sample.exe", min_length=4), + analysis=Analysis(enable_layout=True, enable_tags=True), + strings=Strings(static_strings=[]), + layout=ResultLayout(name="pe", offset=0, length=0x1000), + ) + p = tmp_path / "results.json" + p.write_text(floss.render.json.render(doc)) + + assert floss.utils.detect_file_type(p) is floss.utils.FileType.RESULTS + + +def test_json_render_keeps_extra_top_level_keys(): + """future/extra top-level fields must not be silently dropped.""" + from floss.render import json as render_json + + doc = ResultDocument( + metadata=Metadata(file_path="sample.exe", min_length=4), + analysis=Analysis(), + strings=Strings(), + layout=None, + ) + data = dataclasses.asdict(doc) + # simulate a future document gaining an extra top-level field + data["future_field"] = {"a": 1} + + raw = render_json._render_dict(data) + obj = json.loads(raw) + assert obj["future_field"] == {"a": 1} + # ordering preserved: metadata first, extras appended after known keys + assert list(obj.keys())[0] == "metadata" + assert list(obj.keys())[-1] == "future_field" diff --git a/tests/test_tags_lfs.py b/tests/test_tags_lfs.py new file mode 100644 index 000000000..82e0f72e3 --- /dev/null +++ b/tests/test_tags_lfs.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import floss.tags.gp +import floss.tags.oss +import floss.tags.expert +import floss.tags.winapi +from floss.tags import ensure_not_lfs_pointer + +LFS_POINTER = b"version https://git-lfs.github.com/spec/v1\noid sha256:0" * 4 + b"\nsize 123\n" + + +def test_ensure_not_lfs_pointer_raises(tmp_path): + path = tmp_path / "db.bin" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + ensure_not_lfs_pointer(path) + + +def test_ensure_not_lfs_pointer_ok(tmp_path): + path = tmp_path / "db.bin" + path.write_bytes(b"not an lfs pointer") + ensure_not_lfs_pointer(path) + + +def test_oss_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "oss.jsonl.gz" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.oss.OpenSourceStringDatabase.from_file(path) + + +def test_expert_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "expert.jsonl.gz" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.expert.ExpertStringDatabase.from_file(path) + + +def test_winapi_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "winapi" + path.mkdir() + (path / "dlls.txt.gz").write_bytes(LFS_POINTER) + (path / "apis.txt.gz").write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.winapi.WindowsApiStringDatabase.from_dir(path) + + +def test_gp_loader_rejects_lfs_pointer(tmp_path): + path = tmp_path / "hashes.bin" + path.write_bytes(LFS_POINTER) + with pytest.raises(ValueError, match="Git LFS pointer detected"): + floss.tags.gp.StringHashDatabase.from_file(path) diff --git a/tests/test_winapi_db.py b/tests/test_winapi_db.py new file mode 100644 index 000000000..03a096509 --- /dev/null +++ b/tests/test_winapi_db.py @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import floss.tags.winapi +from floss.tags import data_root + + +def test_load_db(): + path = data_root() / "winapi" + db = floss.tags.winapi.WindowsApiStringDatabase.from_dir(path) + assert len(db) > 0 + + +def test_query_db(): + path = data_root() / "winapi" + db = floss.tags.winapi.WindowsApiStringDatabase.from_dir(path) + + assert "kernel32.dll" in db.dll_names + assert "kernel33.dll" not in db.dll_names + + assert "CreateFileA" in db.api_names + assert "CreateFileB" not in db.api_names diff --git a/vercel.json b/vercel.json new file mode 100644 index 000000000..5b001b652 --- /dev/null +++ b/vercel.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "framework": null, + "installCommand": "cd viewer && npm ci", + "buildCommand": "cd viewer && npm run build", + "outputDirectory": "viewer/dist" +} diff --git a/viewer/.gitignore b/viewer/.gitignore new file mode 100644 index 000000000..fc5ae9f0c --- /dev/null +++ b/viewer/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +.vercel diff --git a/viewer/README.md b/viewer/README.md new file mode 100644 index 000000000..108ad4287 --- /dev/null +++ b/viewer/README.md @@ -0,0 +1,36 @@ +# FLOSS Graphical Viewer + +This is a web-based viewer for analyzing the output of the `floss` tool. It allows for interactive filtering and exploration of extracted strings, tags, and structures from a binary file. + +## Features + +- Upload and parse `floss` JSON output. +- Filter strings by search term, minimum length, tags, and structures. +- Toggle display of columns (tags, encoding, offset/structure). +- Copy filtered strings to the clipboard. + +## Development + +To set up the development environment, first install the dependencies: + +```bash +npm install +``` + +Then, run the development server: + +```bash +npm run dev +``` + +This will start a local server, and you can view the application in your browser. The server supports Hot Module Replacement (HMR), so changes to the source code will be reflected live without a full page reload. + +## Building + +To build the application for production, run the following command: + +```bash +npm run build +``` + +This will create a single, self-contained HTML file in the `dist` directory. This file can be opened directly in a browser or hosted on a web server. \ No newline at end of file diff --git a/viewer/eslint.config.js b/viewer/eslint.config.js new file mode 100644 index 000000000..d94e7deb7 --- /dev/null +++ b/viewer/eslint.config.js @@ -0,0 +1,23 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' +import { globalIgnores } from 'eslint/config' + +export default tseslint.config([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs['recommended-latest'], + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + }, +]) diff --git a/viewer/index.html b/viewer/index.html new file mode 100644 index 000000000..016ce951f --- /dev/null +++ b/viewer/index.html @@ -0,0 +1,30 @@ + + + + + + + FLOSS Graphical Viewer + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/viewer/package-lock.json b/viewer/package-lock.json new file mode 100644 index 000000000..b3ebe6097 --- /dev/null +++ b/viewer/package-lock.json @@ -0,0 +1,3243 @@ +{ + "name": "floss-viewer", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "floss-viewer", + "version": "0.0.0", + "dependencies": { + "react": "^19.1.0", + "react-dom": "^19.1.0", + "react-dropzone": "^14.3.8" + }, + "devDependencies": { + "@eslint/js": "^9.30.1", + "@types/react": "^19.1.8", + "@types/react-dom": "^19.1.6", + "@vitejs/plugin-react": "^4.6.0", + "eslint": "^9.30.1", + "eslint-plugin-react-hooks": "^5.2.0", + "eslint-plugin-react-refresh": "^0.4.20", + "globals": "^16.3.0", + "typescript": "~5.8.3", + "typescript-eslint": "^8.35.1", + "vite": "^7.0.4", + "vite-plugin-singlefile": "^2.3.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", + "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", + "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.27.3", + "@babel/helpers": "^7.27.6", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.0", + "@babel/types": "^7.28.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", + "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.27.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", + "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.27.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.27.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", + "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", + "dev": true, + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.27.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", + "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "dev": true, + "dependencies": { + "@babel/types": "^7.28.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", + "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.0.tgz", + "integrity": "sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==", + "dev": true, + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.6.tgz", + "integrity": "sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.6.tgz", + "integrity": "sha512-S8ToEOVfg++AU/bHwdksHNnyLyVM+eMVAOf6yRKFitnwnbwwPNqKr3srzFRe7nzV69RQKb5DgchIX5pt3L53xg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.6.tgz", + "integrity": "sha512-hd5zdUarsK6strW+3Wxi5qWws+rJhCCbMiC9QZyzoxfk5uHRIE8T287giQxzVpEvCwuJ9Qjg6bEjcRJcgfLqoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.6.tgz", + "integrity": "sha512-0Z7KpHSr3VBIO9A/1wcT3NTy7EB4oNC4upJ5ye3R7taCc2GUdeynSLArnon5G8scPwaU866d3H4BCrE5xLW25A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.6.tgz", + "integrity": "sha512-FFCssz3XBavjxcFxKsGy2DYK5VSvJqa6y5HXljKzhRZ87LvEi13brPrf/wdyl/BbpbMKJNOr1Sd0jtW4Ge1pAA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.6.tgz", + "integrity": "sha512-GfXs5kry/TkGM2vKqK2oyiLFygJRqKVhawu3+DOCk7OxLy/6jYkWXhlHwOoTb0WqGnWGAS7sooxbZowy+pK9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.6.tgz", + "integrity": "sha512-aoLF2c3OvDn2XDTRvn8hN6DRzVVpDlj2B/F66clWd/FHLiHaG3aVZjxQX2DYphA5y/evbdGvC6Us13tvyt4pWg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.6.tgz", + "integrity": "sha512-2SkqTjTSo2dYi/jzFbU9Plt1vk0+nNg8YC8rOXXea+iA3hfNJWebKYPs3xnOUf9+ZWhKAaxnQNUf2X9LOpeiMQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.6.tgz", + "integrity": "sha512-SZHQlzvqv4Du5PrKE2faN0qlbsaW/3QQfUUc6yO2EjFcA83xnwm91UbEEVx4ApZ9Z5oG8Bxz4qPE+HFwtVcfyw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.6.tgz", + "integrity": "sha512-b967hU0gqKd9Drsh/UuAm21Khpoh6mPBSgz8mKRq4P5mVK8bpA+hQzmm/ZwGVULSNBzKdZPQBRT3+WuVavcWsQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.6.tgz", + "integrity": "sha512-aHWdQ2AAltRkLPOsKdi3xv0mZ8fUGPdlKEjIEhxCPm5yKEThcUjHpWB1idN74lfXGnZ5SULQSgtr5Qos5B0bPw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.6.tgz", + "integrity": "sha512-VgKCsHdXRSQ7E1+QXGdRPlQ/e08bN6WMQb27/TMfV+vPjjTImuT9PmLXupRlC90S1JeNNW5lzkAEO/McKeJ2yg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.6.tgz", + "integrity": "sha512-WViNlpivRKT9/py3kCmkHnn44GkGXVdXfdc4drNmRl15zVQ2+D2uFwdlGh6IuK5AAnGTo2qPB1Djppj+t78rzw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.6.tgz", + "integrity": "sha512-wyYKZ9NTdmAMb5730I38lBqVu6cKl4ZfYXIs31Baf8aoOtB4xSGi3THmDYt4BTFHk7/EcVixkOV2uZfwU3Q2Jw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.6.tgz", + "integrity": "sha512-KZh7bAGGcrinEj4qzilJ4hqTY3Dg2U82c8bv+e1xqNqZCrCyc+TL9AUEn5WGKDzm3CfC5RODE/qc96OcbIe33w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.6.tgz", + "integrity": "sha512-9N1LsTwAuE9oj6lHMyyAM+ucxGiVnEqUdp4v7IaMmrwb06ZTEVCIs3oPPplVsnjPfyjmxwHxHMF8b6vzUVAUGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.6.tgz", + "integrity": "sha512-A6bJB41b4lKFWRKNrWoP2LHsjVzNiaurf7wyj/XtFNTsnPuxwEBWHLty+ZE0dWBKuSK1fvKgrKaNjBS7qbFKig==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.6.tgz", + "integrity": "sha512-IjA+DcwoVpjEvyxZddDqBY+uJ2Snc6duLpjmkXm/v4xuS3H+3FkLZlDm9ZsAbF9rsfP3zeA0/ArNDORZgrxR/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.6.tgz", + "integrity": "sha512-dUXuZr5WenIDlMHdMkvDc1FAu4xdWixTCRgP7RQLBOkkGgwuuzaGSYcOpW4jFxzpzL1ejb8yF620UxAqnBrR9g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.6.tgz", + "integrity": "sha512-l8ZCvXP0tbTJ3iaqdNf3pjaOSd5ex/e6/omLIQCVBLmHTlfXW3zAxQ4fnDmPLOB1x9xrcSi/xtCWFwCZRIaEwg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.6.tgz", + "integrity": "sha512-hKrmDa0aOFOr71KQ/19JC7az1P0GWtCN1t2ahYAf4O007DHZt/dW8ym5+CUdJhQ/qkZmI1HAF8KkJbEFtCL7gw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.6.tgz", + "integrity": "sha512-+SqBcAWoB1fYKmpWoQP4pGtx+pUUC//RNYhFdbcSA16617cchuryuhOCRpPsjCblKukAckWsV+aQ3UKT/RMPcA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.6.tgz", + "integrity": "sha512-dyCGxv1/Br7MiSC42qinGL8KkG4kX0pEsdb0+TKhmJZgCUDBGmyo1/ArCjNGiOLiIAgdbWgmWgib4HoCi5t7kA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.6.tgz", + "integrity": "sha512-42QOgcZeZOvXfsCBJF5Afw73t4veOId//XD3i+/9gSkhSV6Gk3VPlWncctI+JcOyERv85FUo7RxuxGy+z8A43Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.6.tgz", + "integrity": "sha512-4AWhgXmDuYN7rJI6ORB+uU9DHLq/erBbuMoAuB4VWJTu5KtCgcKYPynF0YI1VkBNuEfjNlLrFr9KZPJzrtLkrQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.6.tgz", + "integrity": "sha512-NgJPHHbEpLQgDH2MjQu90pzW/5vvXIZ7KOnPyNBm92A6WgZ/7b6fJyUBjoumLqeOQQGqY2QjQxRo97ah4Sj0cA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", + "integrity": "sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", + "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.0.tgz", + "integrity": "sha512-ENIdc4iLu0d93HeYirvKmrzshzofPw6VkZRKQGe9Nv46ZnWUzcF1xV01dcvEg/1wXUR61OmmlSfyeyO7EvjLxQ==", + "dev": true, + "dependencies": { + "@eslint/object-schema": "^2.1.6", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.0.tgz", + "integrity": "sha512-ViuymvFmcJi04qdZeDc2whTHryouGcDlaxPqarTD0ZE10ISpxGUVZGZDx4w01upyIynL3iu6IXH2bS1NhclQMw==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.14.0.tgz", + "integrity": "sha512-qIbV0/JZr7iSDjqAc60IqbLdsj9GDt16xQtWD+B78d/HAlvysGdZZ6rpJHGAc2T0FQx1X6thsSPdnoiGKdNtdg==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", + "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@eslint/js": { + "version": "9.30.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.30.1.tgz", + "integrity": "sha512-zXhuECFlyep42KZUhWjfvsmXGX39W8K8LFb8AWXM9gSV9dQB+MrJGLKvW6Zw0Ggnbpw0VHTtrhFXYe3Gym18jg==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.6.tgz", + "integrity": "sha512-RBMg5FRL0I0gs51M/guSAj5/e14VQ4tpZnQNWwuDT66P14I43ItmPfIZRhO9fUVIPOAQXU47atlywZ/czoqFPA==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.3.3.tgz", + "integrity": "sha512-1+WqvgNMhmlAambTvT3KPtCl/Ibr68VldY2XY40SL1CE0ZXiakFR/cbTspaF5HsnpDMvcYYoJHfl4980NBjGag==", + "dev": true, + "dependencies": { + "@eslint/core": "^0.15.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit/node_modules/@eslint/core": { + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.15.1.tgz", + "integrity": "sha512-bkOp+iumZCCbt1K1CmWf0R9pM5yKpDv+ZXtvSyQpudrI9kuFLp+bM2WOPXImuD/ceQuaa8f5pj93Y7zyECIGNA==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.6", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", + "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", + "dev": true, + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.3.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", + "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", + "dev": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", + "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", + "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.19", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.19.tgz", + "integrity": "sha512-3FL3mnMbPu0muGOCaKAhhFEYmqv9eTfPSJRJmANrCwtgK8VuxpsZDGK+m0LYAGoyO8+0j5uRe4PeyPDK1yA/hA==", + "dev": true + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.44.2.tgz", + "integrity": "sha512-g0dF8P1e2QYPOj1gu7s/3LVP6kze9A7m6x0BZ9iTdXK8N5c2V7cpBKHV3/9A4Zd8xxavdhK0t4PnqjkqVmUc9Q==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.44.2.tgz", + "integrity": "sha512-Yt5MKrOosSbSaAK5Y4J+vSiID57sOvpBNBR6K7xAaQvk3MkcNVV0f9fE20T+41WYN8hDn6SGFlFrKudtx4EoxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.44.2.tgz", + "integrity": "sha512-EsnFot9ZieM35YNA26nhbLTJBHD0jTwWpPwmRVDzjylQT6gkar+zenfb8mHxWpRrbn+WytRRjE0WKsfaxBkVUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.44.2.tgz", + "integrity": "sha512-dv/t1t1RkCvJdWWxQ2lWOO+b7cMsVw5YFaS04oHpZRWehI1h0fV1gF4wgGCTyQHHjJDfbNpwOi6PXEafRBBezw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.44.2.tgz", + "integrity": "sha512-W4tt4BLorKND4qeHElxDoim0+BsprFTwb+vriVQnFFtT/P6v/xO5I99xvYnVzKWrK6j7Hb0yp3x7V5LUbaeOMg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.44.2.tgz", + "integrity": "sha512-tdT1PHopokkuBVyHjvYehnIe20fxibxFCEhQP/96MDSOcyjM/shlTkZZLOufV3qO6/FQOSiJTBebhVc12JyPTA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.44.2.tgz", + "integrity": "sha512-+xmiDGGaSfIIOXMzkhJ++Oa0Gwvl9oXUeIiwarsdRXSe27HUIvjbSIpPxvnNsRebsNdUo7uAiQVgBD1hVriwSQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.44.2.tgz", + "integrity": "sha512-bDHvhzOfORk3wt8yxIra8N4k/N0MnKInCW5OGZaeDYa/hMrdPaJzo7CSkjKZqX4JFUWjUGm88lI6QJLCM7lDrA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.44.2.tgz", + "integrity": "sha512-NMsDEsDiYghTbeZWEGnNi4F0hSbGnsuOG+VnNvxkKg0IGDvFh7UVpM/14mnMwxRxUf9AdAVJgHPvKXf6FpMB7A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.44.2.tgz", + "integrity": "sha512-lb5bxXnxXglVq+7imxykIp5xMq+idehfl+wOgiiix0191av84OqbjUED+PRC5OA8eFJYj5xAGcpAZ0pF2MnW+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.44.2.tgz", + "integrity": "sha512-Yl5Rdpf9pIc4GW1PmkUGHdMtbx0fBLE1//SxDmuf3X0dUC57+zMepow2LK0V21661cjXdTn8hO2tXDdAWAqE5g==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.44.2.tgz", + "integrity": "sha512-03vUDH+w55s680YYryyr78jsO1RWU9ocRMaeV2vMniJJW/6HhoTBwyyiiTPVHNWLnhsnwcQ0oH3S9JSBEKuyqw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.44.2.tgz", + "integrity": "sha512-iYtAqBg5eEMG4dEfVlkqo05xMOk6y/JXIToRca2bAWuqjrJYJlx/I7+Z+4hSrsWU8GdJDFPL4ktV3dy4yBSrzg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.44.2.tgz", + "integrity": "sha512-e6vEbgaaqz2yEHqtkPXa28fFuBGmUJ0N2dOJK8YUfijejInt9gfCSA7YDdJ4nYlv67JfP3+PSWFX4IVw/xRIPg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.44.2.tgz", + "integrity": "sha512-evFOtkmVdY3udE+0QKrV5wBx7bKI0iHz5yEVx5WqDJkxp9YQefy4Mpx3RajIVcM6o7jxTvVd/qpC1IXUhGc1Mw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.44.2.tgz", + "integrity": "sha512-/bXb0bEsWMyEkIsUL2Yt5nFB5naLAwyOWMEviQfQY1x3l5WsLKgvZf66TM7UTfED6erckUVUJQ/jJ1FSpm3pRQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.44.2.tgz", + "integrity": "sha512-3D3OB1vSSBXmkGEZR27uiMRNiwN08/RVAcBKwhUYPaiZ8bcvdeEwWPvbnXvvXHY+A/7xluzcN+kaiOFNiOZwWg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.44.2.tgz", + "integrity": "sha512-VfU0fsMK+rwdK8mwODqYeM2hDrF2WiHaSmCBrS7gColkQft95/8tphyzv2EupVxn3iE0FI78wzffoULH1G+dkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.44.2.tgz", + "integrity": "sha512-+qMUrkbUurpE6DVRjiJCNGZBGo9xM4Y0FXU5cjgudWqIBWbcLkjE3XprJUsOFgC6xjBClwVa9k6O3A7K3vxb5Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.44.2.tgz", + "integrity": "sha512-3+QZROYfJ25PDcxFF66UEk8jGWigHJeecZILvkPkyQN7oc5BvFo4YEXFkOs154j3FTMp9mn9Ky8RCOwastduEA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.7.tgz", + "integrity": "sha512-dkO5fhS7+/oos4ciWxyEyjWe48zmG6wbCheo/G2ZnHx4fs3EU6YC6UM8rk56gAjNJ9P3MTH2jo5jb92/K6wbng==", + "dev": true, + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true + }, + "node_modules/@types/react": { + "version": "19.1.8", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.8.tgz", + "integrity": "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g==", + "dev": true, + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.1.6", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz", + "integrity": "sha512-4hOiT/dwO8Ko0gV1m/TJZYk3y0KBnY9vzDh7W+DH17b2HFSOGgdj33dhihPeuy3l0q23+4e+hoXHV6hCC4dCXw==", + "dev": true, + "peerDependencies": { + "@types/react": "^19.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.36.0.tgz", + "integrity": "sha512-lZNihHUVB6ZZiPBNgOQGSxUASI7UJWhT8nHyUGCnaQ28XFCw98IfrMCG3rUl1uwUWoAvodJQby2KTs79UTcrAg==", + "dev": true, + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "8.36.0", + "@typescript-eslint/type-utils": "8.36.0", + "@typescript-eslint/utils": "8.36.0", + "@typescript-eslint/visitor-keys": "8.36.0", + "graphemer": "^1.4.0", + "ignore": "^7.0.0", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.36.0", + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.36.0.tgz", + "integrity": "sha512-FuYgkHwZLuPbZjQHzJXrtXreJdFMKl16BFYyRrLxDhWr6Qr7Kbcu2s1Yhu8tsiMXw1S0W1pjfFfYEt+R604s+Q==", + "dev": true, + "dependencies": { + "@typescript-eslint/scope-manager": "8.36.0", + "@typescript-eslint/types": "8.36.0", + "@typescript-eslint/typescript-estree": "8.36.0", + "@typescript-eslint/visitor-keys": "8.36.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.36.0.tgz", + "integrity": "sha512-JAhQFIABkWccQYeLMrHadu/fhpzmSQ1F1KXkpzqiVxA/iYI6UnRt2trqXHt1sYEcw1mxLnB9rKMsOxXPxowN/g==", + "dev": true, + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.36.0", + "@typescript-eslint/types": "^8.36.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.36.0.tgz", + "integrity": "sha512-wCnapIKnDkN62fYtTGv2+RY8FlnBYA3tNm0fm91kc2BjPhV2vIjwwozJ7LToaLAyb1ca8BxrS7vT+Pvvf7RvqA==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "8.36.0", + "@typescript-eslint/visitor-keys": "8.36.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.36.0.tgz", + "integrity": "sha512-Nhh3TIEgN18mNbdXpd5Q8mSCBnrZQeY9V7Ca3dqYvNDStNIGRmJA6dmrIPMJ0kow3C7gcQbpsG2rPzy1Ks/AnA==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.36.0.tgz", + "integrity": "sha512-5aaGYG8cVDd6cxfk/ynpYzxBRZJk7w/ymto6uiyUFtdCozQIsQWh7M28/6r57Fwkbweng8qAzoMCPwSJfWlmsg==", + "dev": true, + "dependencies": { + "@typescript-eslint/typescript-estree": "8.36.0", + "@typescript-eslint/utils": "8.36.0", + "debug": "^4.3.4", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.36.0.tgz", + "integrity": "sha512-xGms6l5cTJKQPZOKM75Dl9yBfNdGeLRsIyufewnxT4vZTrjC0ImQT4fj8QmtJK84F58uSh5HVBSANwcfiXxABQ==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.36.0.tgz", + "integrity": "sha512-JaS8bDVrfVJX4av0jLpe4ye0BpAaUW7+tnS4Y4ETa3q7NoZgzYbN9zDQTJ8kPb5fQ4n0hliAt9tA4Pfs2zA2Hg==", + "dev": true, + "dependencies": { + "@typescript-eslint/project-service": "8.36.0", + "@typescript-eslint/tsconfig-utils": "8.36.0", + "@typescript-eslint/types": "8.36.0", + "@typescript-eslint/visitor-keys": "8.36.0", + "debug": "^4.3.4", + "fast-glob": "^3.3.2", + "is-glob": "^4.0.3", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.36.0.tgz", + "integrity": "sha512-VOqmHu42aEMT+P2qYjylw6zP/3E/HvptRwdn/PZxyV27KhZg2IOszXod4NcXisWzPAGSS4trE/g4moNj6XmH2g==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.36.0", + "@typescript-eslint/types": "8.36.0", + "@typescript-eslint/typescript-estree": "8.36.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.36.0.tgz", + "integrity": "sha512-vZrhV2lRPWDuGoxcmrzRZyxAggPL+qp3WzUrlZD+slFueDiYHxeBa34dUXPuC0RmGKzl4lS5kFJYvKCq9cnNDA==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "8.36.0", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.6.0.tgz", + "integrity": "sha512-5Kgff+m8e2PB+9j51eGHEpn5kUzRKH2Ry0qGoe8ItJg7pqnkPrYPkDQZGgGmTa0EGarHrkjLvOdU3b1fzI8otQ==", + "dev": true, + "dependencies": { + "@babel/core": "^7.27.4", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.19", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0-beta.0" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/attr-accept": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.5.tgz", + "integrity": "sha512-0bDNnY/u6pPwHDMoF0FieU354oBi0a8rD9FcsLwzcGWbc8KS8KPIi7y+s13OlVY+gMWc/9xEMUgNE6Qm8ZllYQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.25.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.1.tgz", + "integrity": "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001726", + "electron-to-chromium": "^1.5.173", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001727", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001727.tgz", + "integrity": "sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "dev": true + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "node_modules/electron-to-chromium": { + "version": "1.5.182", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.182.tgz", + "integrity": "sha512-Lv65Btwv9W4J9pyODI6EWpdnhfvrve/us5h1WspW8B2Fb0366REPtY3hX7ounk1CkV/TBjWCEvCBBbYbmV0qCA==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.25.6", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.6.tgz", + "integrity": "sha512-GVuzuUwtdsghE3ocJ9Bs8PNoF13HNQ5TXbEi2AhvVb8xU1Iwt9Fos9FEamfoee+u/TOsn7GUWc04lz46n2bbTg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.6", + "@esbuild/android-arm": "0.25.6", + "@esbuild/android-arm64": "0.25.6", + "@esbuild/android-x64": "0.25.6", + "@esbuild/darwin-arm64": "0.25.6", + "@esbuild/darwin-x64": "0.25.6", + "@esbuild/freebsd-arm64": "0.25.6", + "@esbuild/freebsd-x64": "0.25.6", + "@esbuild/linux-arm": "0.25.6", + "@esbuild/linux-arm64": "0.25.6", + "@esbuild/linux-ia32": "0.25.6", + "@esbuild/linux-loong64": "0.25.6", + "@esbuild/linux-mips64el": "0.25.6", + "@esbuild/linux-ppc64": "0.25.6", + "@esbuild/linux-riscv64": "0.25.6", + "@esbuild/linux-s390x": "0.25.6", + "@esbuild/linux-x64": "0.25.6", + "@esbuild/netbsd-arm64": "0.25.6", + "@esbuild/netbsd-x64": "0.25.6", + "@esbuild/openbsd-arm64": "0.25.6", + "@esbuild/openbsd-x64": "0.25.6", + "@esbuild/openharmony-arm64": "0.25.6", + "@esbuild/sunos-x64": "0.25.6", + "@esbuild/win32-arm64": "0.25.6", + "@esbuild/win32-ia32": "0.25.6", + "@esbuild/win32-x64": "0.25.6" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.30.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.30.1.tgz", + "integrity": "sha512-zmxXPNMOXmwm9E0yQLi5uqXHs7uq2UIiqEKo3Gq+3fwo1XrJ+hijAZImyF7hclW3E6oHz43Yk3RP8at6OTKflQ==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.0", + "@eslint/config-helpers": "^0.3.0", + "@eslint/core": "^0.14.0", + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "9.30.1", + "@eslint/plugin-kit": "^0.3.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "@types/json-schema": "^7.0.15", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-5.2.0.tgz", + "integrity": "sha512-+f15FfK64YQwZdJNELETdn5ibXEUQmW1DZL6KXhNnc2heoy/sg9VJJeT7n8TlMWouzWqSWavFkIhHyIbIAEapg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + } + }, + "node_modules/eslint-plugin-react-refresh": { + "version": "0.4.20", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.20.tgz", + "integrity": "sha512-XpbHQ2q5gUF8BGOX4dHe+71qoirYMhApEPZ7sfhF/dNnOF1UXnCMGZf79SFTBO7Bz5YEIT4TMieSlJBWhP9WBA==", + "dev": true, + "peerDependencies": { + "eslint": ">=8.40" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "node_modules/fastq": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", + "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/file-selector": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-2.1.2.tgz", + "integrity": "sha512-QgXo+mXTe8ljeqUFaX3QVHc5osSItJ/Km+xpocx0aSqWGMSCf6qYs/VnzZgS864Pjn5iceMRFigeAV7AfTlaig==", + "dependencies": { + "tslib": "^2.7.0" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.3.0.tgz", + "integrity": "sha512-bqWEnJ1Nt3neqx2q5SFfGS8r/ahumIakg3HcwtNlrVlwXIeNumWn/c7Pn/wKzGhf6SaW6H6uWXLqC30STCMchQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/react": { + "version": "19.1.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.1.0.tgz", + "integrity": "sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.1.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.0.tgz", + "integrity": "sha512-Xs1hdnE+DyKgeHJeJznQmYMIBG3TKIHJJT95Q58nHLSrElKlGQqDTR2HQ9fx5CN/Gk6Vh/kupBTDLU11/nDk/g==", + "dependencies": { + "scheduler": "^0.26.0" + }, + "peerDependencies": { + "react": "^19.1.0" + } + }, + "node_modules/react-dropzone": { + "version": "14.3.8", + "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.3.8.tgz", + "integrity": "sha512-sBgODnq+lcA4P296DY4wacOZz3JFpD99fp+hb//iBO2HHnyeZU3FwWyXJ6salNpqQdsZrgMrotuko/BdJMV8Ug==", + "dependencies": { + "attr-accept": "^2.2.4", + "file-selector": "^2.1.0", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">= 10.13" + }, + "peerDependencies": { + "react": ">= 16.8 || 18.0.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.44.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.44.2.tgz", + "integrity": "sha512-PVoapzTwSEcelaWGth3uR66u7ZRo6qhPHc0f2uRO9fX6XDVNrIiGYS0Pj9+R8yIIYSD/mCx2b16Ws9itljKSPg==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.44.2", + "@rollup/rollup-android-arm64": "4.44.2", + "@rollup/rollup-darwin-arm64": "4.44.2", + "@rollup/rollup-darwin-x64": "4.44.2", + "@rollup/rollup-freebsd-arm64": "4.44.2", + "@rollup/rollup-freebsd-x64": "4.44.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.44.2", + "@rollup/rollup-linux-arm-musleabihf": "4.44.2", + "@rollup/rollup-linux-arm64-gnu": "4.44.2", + "@rollup/rollup-linux-arm64-musl": "4.44.2", + "@rollup/rollup-linux-loongarch64-gnu": "4.44.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.44.2", + "@rollup/rollup-linux-riscv64-gnu": "4.44.2", + "@rollup/rollup-linux-riscv64-musl": "4.44.2", + "@rollup/rollup-linux-s390x-gnu": "4.44.2", + "@rollup/rollup-linux-x64-gnu": "4.44.2", + "@rollup/rollup-linux-x64-musl": "4.44.2", + "@rollup/rollup-win32-arm64-msvc": "4.44.2", + "@rollup/rollup-win32-ia32-msvc": "4.44.2", + "@rollup/rollup-win32-x64-msvc": "4.44.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/scheduler": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.26.0.tgz", + "integrity": "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz", + "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==", + "dev": true, + "dependencies": { + "fdir": "^6.4.4", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz", + "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==", + "dev": true, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-api-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", + "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", + "dev": true, + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.36.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.36.0.tgz", + "integrity": "sha512-fTCqxthY+h9QbEgSIBfL9iV6CvKDFuoxg6bHPNpJ9HIUzS+jy2lCEyCmGyZRWEBSaykqcDPf1SJ+BfCI8DRopA==", + "dev": true, + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.36.0", + "@typescript-eslint/parser": "8.36.0", + "@typescript-eslint/utils": "8.36.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <5.9.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.0.4.tgz", + "integrity": "sha512-SkaSguuS7nnmV7mfJ8l81JGBFV7Gvzp8IzgE8A8t23+AxuNX61Q5H1Tpz5efduSN7NHC8nQXD3sKQKZAu5mNEA==", + "dev": true, + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.6", + "picomatch": "^4.0.2", + "postcss": "^8.5.6", + "rollup": "^4.40.0", + "tinyglobby": "^0.2.14" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-plugin-singlefile": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/vite-plugin-singlefile/-/vite-plugin-singlefile-2.3.0.tgz", + "integrity": "sha512-DAcHzYypM0CasNLSz/WG0VdKOCxGHErfrjOoyIPiNxTPTGmO6rRD/te93n1YL/s+miXq66ipF1brMBikf99c6A==", + "dev": true, + "dependencies": { + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">18.0.0" + }, + "peerDependencies": { + "rollup": "^4.44.1", + "vite": "^5.4.11 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/vite/node_modules/fdir": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz", + "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==", + "dev": true, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/viewer/package.json b/viewer/package.json new file mode 100644 index 000000000..b98a28f97 --- /dev/null +++ b/viewer/package.json @@ -0,0 +1,34 @@ +{ + "name": "floss-viewer", + "private": true, + "version": "0.0.0", + "type": "module", + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "eslint .", + "preview": "vite preview" + }, + "dependencies": { + "react": "^19.1.0", + "react-dom": "^19.1.0", + "react-dropzone": "^14.3.8" + }, + "devDependencies": { + "@eslint/js": "^9.30.1", + "@types/react": "^19.1.8", + "@types/react-dom": "^19.1.6", + "@vitejs/plugin-react": "^4.6.0", + "eslint": "^9.30.1", + "eslint-plugin-react-hooks": "^5.2.0", + "eslint-plugin-react-refresh": "^0.4.20", + "globals": "^16.3.0", + "typescript": "~5.8.3", + "typescript-eslint": "^8.35.1", + "vite": "^7.0.4", + "vite-plugin-singlefile": "^2.3.0" + } +} diff --git a/viewer/public/favicon.ico b/viewer/public/favicon.ico new file mode 100644 index 000000000..7956c2258 Binary files /dev/null and b/viewer/public/favicon.ico differ diff --git a/viewer/public/favicon.png b/viewer/public/favicon.png new file mode 100644 index 000000000..11ddd4a15 Binary files /dev/null and b/viewer/public/favicon.png differ diff --git a/viewer/public/floss-logo.png b/viewer/public/floss-logo.png new file mode 100644 index 000000000..9e19f4f98 Binary files /dev/null and b/viewer/public/floss-logo.png differ diff --git a/viewer/src/App.css b/viewer/src/App.css new file mode 100644 index 000000000..85be4f544 --- /dev/null +++ b/viewer/src/App.css @@ -0,0 +1,767 @@ +/* ============================================ + FLOSS Graphical Viewer — Monochrome Design System + ============================================ */ + +:root { + --bg: #050505; + --surface: #0e0e0e; + --surface-raised: #151515; + --surface-hover: #1a1a1a; + --border: #222222; + --border-strong: #333333; + --text: #e8e8e8; + --text-secondary: #aaaaaa; + --text-muted: #777777; + --text-dim: #4a4a4a; + --accent: #ffffff; + --highlight: #f0f0f0; + --overlay: rgba(255, 255, 255, 0.02); + --overlay-row: rgba(255, 255, 255, 0.015); + --font-mono: 'Geist Mono', 'JetBrains Mono', 'SF Mono', 'Menlo', monospace; + --radius: 3px; + --transition: 150ms ease; +} + +:root[data-theme="light"] { + --bg: #ffffff; + --surface: #f6f6f6; + --surface-raised: #efefef; + --surface-hover: #e8e8e8; + --border: #e2e2e2; + --border-strong: #d4d4d4; + --text: #161616; + --text-secondary: #4d4d4d; + --text-muted: #777777; + --text-dim: #a3a3a3; + --accent: #000000; + --highlight: #0f0f0f; + --overlay: rgba(0, 0, 0, 0.02); + --overlay-row: rgba(0, 0, 0, 0.02); +} + +@media (prefers-color-scheme: light) { + :root:not([data-theme]) { + --bg: #ffffff; + --surface: #f6f6f6; + --surface-raised: #efefef; + --surface-hover: #e8e8e8; + --border: #e2e2e2; + --border-strong: #d4d4d4; + --text: #161616; + --text-secondary: #4d4d4d; + --text-muted: #777777; + --text-dim: #a3a3a3; + --accent: #000000; + --highlight: #0f0f0f; + --overlay: rgba(0, 0, 0, 0.02); + --overlay-row: rgba(0, 0, 0, 0.02); + } +} + +* { + box-sizing: border-box; +} + +body { + font-family: var(--font-mono); + background-color: var(--bg); + color: var(--text); + margin: 0; + padding: 0; + font-size: 14px; + line-height: 1.5; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +/* ---- App Shell: Sidebar + Main ---- */ + +.App { + display: flex; + height: 100vh; + overflow: hidden; +} + +/* ---- Sidebar ---- */ + +.sidebar { + background: var(--surface); + display: flex; + flex-direction: column; + overflow: hidden; + flex-shrink: 0; + min-width: 260px; + max-width: 600px; +} + +.sidebar-header { + padding: 20px 20px 16px; + border-bottom: 1px solid var(--border); + flex-shrink: 0; +} + +.app-logo { + display: block; + width: 100%; + max-width: 240px; + height: auto; + margin: 0 0 14px 0; +} + +.sidebar-header-buttons { + display: flex; + gap: 8px; +} + +.btn-ghost { + display: inline-flex; + align-items: center; + padding: 7px 14px; + background: transparent; + color: var(--text-secondary); + border: 1px solid var(--border-strong); + border-radius: var(--radius); + cursor: pointer; + font-family: var(--font-mono); + font-size: 12px; + letter-spacing: 0.02em; + transition: all var(--transition); + white-space: nowrap; +} + +.btn-ghost:hover { + color: var(--accent); + border-color: var(--text-muted); + background: var(--surface-hover); +} + +.btn-ghost:active { + background: var(--border); +} + +#file-upload { + display: none; +} + +/* ---- Resize Handle ---- */ + +.resize-handle { + width: 5px; + cursor: col-resize; + background: transparent; + flex-shrink: 0; + position: relative; + z-index: 10; + transition: background var(--transition); +} + +.resize-handle::after { + content: ''; + position: absolute; + top: 0; + left: 2px; + width: 1px; + height: 100%; + background: var(--border); + transition: background var(--transition); +} + +.resize-handle:hover::after, +.resize-handle.dragging::after { + background: var(--text-muted); + width: 2px; + left: 1px; +} + +.resize-handle:hover, +.resize-handle.dragging { + background: var(--overlay); +} + +/* ---- Sidebar Scrollable Body ---- */ + +.sidebar-body { + flex: 1; + overflow-y: auto; + padding: 0; + scrollbar-width: thin; + scrollbar-color: var(--border-strong) transparent; +} + +.sidebar-body::-webkit-scrollbar { + width: 5px; +} + +.sidebar-body::-webkit-scrollbar-track { + background: transparent; +} + +.sidebar-body::-webkit-scrollbar-thumb { + background: var(--border-strong); + border-radius: 3px; +} + +/* ---- Metadata Section ---- */ + +.metadata { + padding: 14px 20px; + border-bottom: 1px solid var(--border); +} + +.meta-row { + display: flex; + gap: 10px; + padding: 2px 0; + font-size: 12px; + line-height: 1.4; +} + +.meta-label { + color: var(--text-muted); + flex-shrink: 0; + width: 56px; +} + +.meta-value { + color: var(--text-secondary); + overflow-wrap: anywhere; + min-width: 0; +} + +.meta-hash { + overflow-wrap: anywhere; +} + +/* ---- Filter Sections ---- */ + +.filter-section { + padding: 14px 20px; + border-bottom: 1px solid var(--border); +} + +.filter-section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 10px; +} + +.filter-section-title { + font-size: 11px; + font-weight: 600; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--text-muted); +} + +.filter-actions { + display: flex; + gap: 8px; +} + +.filter-action-btn { + background: none; + border: none; + color: var(--text-dim); + cursor: pointer; + padding: 0; + font-family: var(--font-mono); + font-size: 11px; + transition: color var(--transition); +} + +.filter-action-btn:hover { + color: var(--text-secondary); +} + +.filter-items { + display: flex; + flex-direction: column; + gap: 1px; +} + +/* ---- Custom Checkboxes ---- */ + +.check-item { + display: flex; + align-items: center; + gap: 8px; + padding: 4px 6px; + border-radius: var(--radius); + cursor: pointer; + font-size: 12.5px; + color: var(--text-secondary); + transition: all var(--transition); + user-select: none; +} + +.check-item:hover { + background: var(--surface-hover); + color: var(--text); +} + +.check-item input[type="checkbox"] { + display: none; +} + +.check-box { + width: 13px; + height: 13px; + border: 1px solid var(--border-strong); + border-radius: 2px; + flex-shrink: 0; + position: relative; + transition: all var(--transition); +} + +.check-item:hover .check-box { + border-color: var(--text-muted); +} + +.check-item input[type="checkbox"]:checked+.check-box { + background: var(--text-muted); + border-color: var(--text-muted); +} + +.check-item input[type="checkbox"]:checked+.check-box::after { + content: ''; + position: absolute; + top: 1px; + left: 3.5px; + width: 3px; + height: 6px; + border: solid var(--surface); + border-width: 0 1.5px 1.5px 0; + transform: rotate(45deg); +} + +.check-count { + color: var(--text-dim); + margin-left: auto; + font-size: 11px; +} + +/* ---- Search Section ---- */ + +.search-section { + padding: 14px 20px; + border-bottom: 1px solid var(--border); +} + +.search-row { + display: flex; + gap: 10px; + align-items: center; +} + +.search-input { + flex: 1; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 8px 10px; + color: var(--text); + font-family: var(--font-mono); + font-size: 13px; + outline: none; + transition: border-color var(--transition); +} + +.search-input:focus { + border-color: var(--text-muted); +} + +.search-input::placeholder { + color: var(--text-dim); +} + +.min-length-group { + display: flex; + align-items: center; + gap: 6px; + flex-shrink: 0; +} + +.min-length-label { + font-size: 11px; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.min-length-input { + width: 48px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 8px 6px; + color: var(--text); + font-family: var(--font-mono); + font-size: 13px; + text-align: center; + outline: none; + transition: border-color var(--transition); +} + +.min-length-input:focus { + border-color: var(--text-muted); +} + +/* hide number spin buttons */ +.min-length-input::-webkit-inner-spin-button, +.min-length-input::-webkit-outer-spin-button { + -webkit-appearance: none; + margin: 0; +} + +.min-length-input { + -moz-appearance: textfield; + appearance: textfield; +} + +/* ---- Sidebar Footer / Actions ---- */ + +.sidebar-footer { + padding: 12px 20px; + border-top: 1px solid var(--border); + flex-shrink: 0; + display: flex; + justify-content: space-between; + align-items: center; +} + +.string-count { + font-size: 12px; + color: var(--text-muted); +} + +.string-count strong { + color: var(--text-secondary); + font-weight: 500; +} + +.string-count-ignored { + color: var(--text-dim); +} + +.btn-copy { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 5px 10px; + background: transparent; + color: var(--text-secondary); + border: 1px solid var(--border-strong); + border-radius: var(--radius); + cursor: pointer; + font-family: var(--font-mono); + font-size: 11px; + letter-spacing: 0.02em; + transition: all var(--transition); +} + +.btn-copy:hover { + color: var(--accent); + border-color: var(--text-muted); +} + +.copy-feedback { + font-size: 11px; + color: var(--text-muted); + margin-left: 8px; + animation: fadeIn 150ms ease; +} + +@keyframes fadeIn { + from { + opacity: 0; + } + + to { + opacity: 1; + } +} + +/* ---- Main Content Area ---- */ + +.main-content { + flex: 1; + overflow-y: auto; + background: var(--bg); + min-width: 0; + scrollbar-width: thin; + scrollbar-color: var(--border-strong) transparent; +} + +.main-content::-webkit-scrollbar { + width: 6px; +} + +.main-content::-webkit-scrollbar-track { + background: transparent; +} + +.main-content::-webkit-scrollbar-thumb { + background: var(--border-strong); + border-radius: 3px; +} + +/* ---- Welcome State ---- */ + +.welcome-state { + display: flex; + align-items: center; + justify-content: center; + height: 100%; + padding: 40px; +} + +.welcome-inner { + text-align: center; +} + +.welcome-title { + font-size: 15px; + font-weight: 500; + color: var(--text-muted); + margin: 0 0 8px 0; + letter-spacing: 0.04em; +} + +.welcome-sub { + font-size: 12px; + color: var(--text-dim); + margin: 0; +} + +/* ---- Layout Sections (String groups) ---- */ + +.layout { + border-bottom: 1px solid var(--border); +} + +.layout .layout { + border-bottom: none; + margin-left: 0; +} + +.layout-header { + height: 35px; + line-height: 35px; + font-size: 11px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-dim); + padding: 0 20px; + background: var(--surface); + border-bottom: 1px solid var(--border); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.layout-content { + padding: 0; +} + +/* ---- Virtual list scroller ---- */ + +.virtual-list { + height: 100%; + overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: var(--border-strong) transparent; +} + +.virtual-list::-webkit-scrollbar { + width: 6px; +} + +.virtual-list::-webkit-scrollbar-track { + background: transparent; +} + +.virtual-list::-webkit-scrollbar-thumb { + background: var(--border-strong); + border-radius: 3px; +} + +/* ---- String Items ---- */ + +.string-view { + display: flex; + align-items: baseline; + height: 26px; + padding: 0 20px; + overflow: hidden; + transition: background var(--transition); + border-bottom: 1px solid transparent; +} + +.string-view--alt { + background: var(--overlay-row); +} + +.string-view:hover { + background: var(--surface-hover); +} + +.string-content { + flex-grow: 1; + padding-right: 16px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + color: var(--text); + font-size: 13px; +} + +.string-content.highlight { + color: var(--accent); + font-weight: 500; +} + +.string-content.mute { + color: var(--text-dim); +} + +.string-tags { + width: 24ch; + flex-shrink: 0; + color: var(--text-muted); + font-size: 12px; + padding-right: 8px; +} + +.string-tags.highlight { + color: var(--text-secondary); +} + +.string-tags.mute { + color: var(--text-dim); +} + +.string-encoding { + width: 3ch; + flex-shrink: 0; + color: var(--text-dim); + font-size: 12px; + padding-right: 8px; +} + +.string-offset-structure { + width: 30ch; + flex-shrink: 0; + font-size: 12px; +} + +.offset-zeros { + color: var(--text-dim); +} + +.offset-digits { + color: var(--text-muted); +} + +.structure-name { + color: var(--text-secondary); + padding-left: 2px; +} + +/* ---- No-select during resize ---- */ + +body.resizing { + user-select: none; + cursor: col-resize; +} + +/* ---- Drag & drop overlay ---- */ + +.App.drag-active > *:not(.drop-overlay) { + filter: blur(6px); +} + +.drop-overlay { + position: fixed; + inset: 0; + z-index: 1000; + display: flex; + align-items: flex-start; + justify-content: center; + padding-top: 24px; + pointer-events: none; +} + +.drop-overlay-inner { + border: 1.5px dashed var(--border-strong); + background: var(--surface-raised); + border-radius: 8px; + padding: 28px 48px; + text-align: center; + box-shadow: 0 12px 48px rgba(0, 0, 0, 0.35); +} + +.drop-overlay-title { + font-size: 14px; + font-weight: 600; + letter-spacing: 0.06em; + color: var(--text); + margin: 0 0 6px 0; +} + +.drop-overlay-sub { + font-size: 12px; + color: var(--text-muted); + margin: 0; +} + +/* ---- Crash fallback ---- */ + +.crash-state { + display: flex; + align-items: center; + justify-content: center; + height: 100vh; + padding: 40px; + background: var(--bg); +} + +.crash-inner { + text-align: center; + max-width: 480px; +} + +.crash-title { + font-size: 15px; + font-weight: 500; + letter-spacing: 0.04em; + color: var(--text); + margin: 0 0 8px 0; +} + +.crash-sub { + font-size: 12px; + color: var(--text-muted); + margin: 0 0 20px 0; + word-break: break-all; +} + +/* ---- Responsive / Edge Cases ---- */ + +@media (max-width: 800px) { + .App { + flex-direction: column; + } + + .sidebar { + border-right: none; + border-bottom: 1px solid var(--border); + max-height: 50vh; + max-width: none; + } + + .resize-handle { + display: none; + } +} \ No newline at end of file diff --git a/viewer/src/App.tsx b/viewer/src/App.tsx new file mode 100644 index 000000000..798ebedd0 --- /dev/null +++ b/viewer/src/App.tsx @@ -0,0 +1,845 @@ +import React, { useState, useCallback, useMemo, useRef, useEffect, useDeferredValue } from 'react'; +import { useDropzone } from 'react-dropzone'; +import './App.css'; +import { type ResultDocument, type ResultLayout, type ResultString, type Analysis, type Strings } from './types'; +import previewData from './pma0303_floss.json'; + +const NOISY_TAGS = ['#common', '#duplicate', '#code', '#reloc', '#code-junk']; + +interface DisplayOptions { + showTags: boolean; + showEncoding: boolean; + showOffsetAndStructure: boolean; +} + +const subsequenceMatch = (query: string, target: string): boolean => { + const qlen = query.length; + if (qlen === 0) return true; + let qi = 0; + for (let ti = 0; ti < target.length && qi < qlen; ti++) { + if (query.charCodeAt(qi) === target.charCodeAt(ti)) qi++; + } + return qi === qlen; +}; + +const ROW_HEIGHT = 26; +const HEADER_HEIGHT = 35; +const OVERSCAN = 10; + +type VirtualRow = + | { kind: 'header'; name: string } + | { kind: 'string'; str: ResultString; alt: boolean }; + +const StringItem: React.FC<{ + str: ResultString; + displayOptions: DisplayOptions; + alt?: boolean; + style?: React.CSSProperties; +}> = React.memo(({ str, displayOptions, alt, style }) => { + const getStyleClass = () => { + const { tags } = str; + if (tags.includes('#capa')) return 'highlight'; + + if (tags.some(t => NOISY_TAGS.includes(t))) return 'mute'; + + return ''; + }; + + const styleClass = getStyleClass(); + + const offsetHex = str.offset.toString(16).padStart(8, '0'); + const firstDigitIndex = offsetHex.search(/[^0]/); + const zeroPart = firstDigitIndex === -1 ? offsetHex : offsetHex.substring(0, firstDigitIndex); + const digitPart = firstDigitIndex === -1 ? '' : offsetHex.substring(firstDigitIndex); + + return ( +
+ {str.string} + {displayOptions.showTags && {str.tags.join(' ')}} + {displayOptions.showEncoding && {str.encoding === 'unicode' ? 'U' : ''}} + {displayOptions.showOffsetAndStructure && ( + + {zeroPart} + {digitPart} + {str.structure && /{str.structure}} + + )} +
+ ); +}); + +const VirtualList: React.FC<{ layout: ResultLayout; displayOptions: DisplayOptions }> = ({ layout, displayOptions }) => { + const { rows, prefixes, totalHeight } = useMemo(() => { + const rows: VirtualRow[] = []; + const prefixes: number[] = [0]; + let alt = true; + let total = 0; + + const push = (row: VirtualRow, height: number) => { + rows.push(row); + total += height; + prefixes.push(total); + }; + + const walk = (l: ResultLayout) => { + push({ kind: 'header', name: l.name }, HEADER_HEIGHT); + for (const s of l.strings) { + push({ kind: 'string', str: s, alt }, ROW_HEIGHT); + alt = !alt; + } + for (const c of l.children) walk(c); + }; + + walk(layout); + return { rows, prefixes, totalHeight: total }; + }, [layout]); + + const containerRef = useRef(null); + const [scrollTop, setScrollTop] = useState(0); + const [viewportH, setViewportH] = useState(0); + + useEffect(() => { + const el = containerRef.current; + if (!el) return; + const update = () => { + setScrollTop(el.scrollTop); + setViewportH(el.clientHeight); + }; + update(); + const ro = new ResizeObserver(update); + ro.observe(el); + el.addEventListener('scroll', update, { passive: true }); + return () => { + ro.disconnect(); + el.removeEventListener('scroll', update); + }; + }, []); + + const n = rows.length; + let startIdx = 0; + let lo = 0; + let hi = n; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (prefixes[mid] <= scrollTop) lo = mid; + else hi = mid - 1; + } + startIdx = lo; + + const endBottom = scrollTop + viewportH; + let endIdx = startIdx; + while (endIdx < n && prefixes[endIdx] < endBottom + OVERSCAN * ROW_HEIGHT) endIdx++; + + const visible: React.ReactNode[] = []; + for (let i = Math.max(0, startIdx - OVERSCAN); i < endIdx; i++) { + const row = rows[i]; + const top = prefixes[i]; + if (row.kind === 'header') { + visible.push( +
+ {row.name} +
+ ); + } else { + visible.push( + + ); + } + } + + return ( +
+
{visible}
+
+ ); +}; + +const CheckItem: React.FC<{ label: string; count?: number; checked: boolean; onChange: () => void }> = ({ label, count, checked, onChange }) => ( + +); + +/** Extract just the filename from a full path */ +const getFilename = (path: string): string => { + const parts = path.replace(/\\/g, '/').split('/'); + return parts[parts.length - 1] || path; +}; + +/** Split a hash into fixed-width 32-char lines for clean rectangular display */ +const chunkHash = (hash: string, charsPerLine = 32): string[] => { + const lines: string[] = []; + for (let i = 0; i < hash.length; i += charsPerLine) { + lines.push(hash.substring(i, i + charsPerLine)); + } + return lines; +}; + +const toNum = (v: unknown, d = 0): number => (typeof v === 'number' && Number.isFinite(v) ? v : d); +const toStr = (v: unknown, d = ''): string => (typeof v === 'string' ? v : d); + +const normalizeString = (raw: unknown): ResultString | null => { + if (typeof raw !== 'object' || raw === null) return null; + const o = raw as Record; + if (typeof o.string !== 'string') return null; + return { + string: o.string, + offset: toNum(o.offset), + size: toNum(o.size), + encoding: toStr(o.encoding), + tags: Array.isArray(o.tags) ? o.tags.filter((t): t is string => typeof t === 'string') : [], + structure: toStr(o.structure), + }; +}; + +const normalizeLayout = (raw: unknown): ResultLayout | null => { + if (typeof raw !== 'object' || raw === null) return null; + const o = raw as Record; + const strings = (Array.isArray(o.strings) ? o.strings.map(normalizeString) : []).filter( + (s): s is ResultString => s !== null + ); + const children = (Array.isArray(o.children) ? o.children.map(normalizeLayout) : []).filter( + (c): c is ResultLayout => c !== null + ); + return { + name: toStr(o.name, 'section'), + offset: toNum(o.offset), + length: toNum(o.length), + strings, + children, + }; +}; + +const normalizeDocument = (raw: unknown): ResultDocument | null => { + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return null; + const o = raw as Record; + if (!o.layout && !o.metadata) return null; + + const metaRaw = (typeof o.metadata === 'object' && o.metadata !== null ? o.metadata : {}) as Record; + const runRaw = (typeof metaRaw.runtime === 'object' && metaRaw.runtime !== null ? metaRaw.runtime : {}) as Record; + + return { + metadata: { + file_path: toStr(metaRaw.file_path, 'unknown'), + md5: toStr(metaRaw.md5), + sha1: toStr(metaRaw.sha1), + sha256: toStr(metaRaw.sha256), + version: toStr(metaRaw.version), + imagebase: toNum(metaRaw.imagebase), + min_length: toNum(metaRaw.min_length), + runtime: { + start_date: toStr(runRaw.start_date), + total: toNum(runRaw.total), + vivisect: toNum(runRaw.vivisect), + find_features: toNum(runRaw.find_features), + static_strings: toNum(runRaw.static_strings), + layout: toNum(runRaw.layout), + tags: toNum(runRaw.tags), + language_strings: toNum(runRaw.language_strings), + stack_strings: toNum(runRaw.stack_strings), + decoded_strings: toNum(runRaw.decoded_strings), + tight_strings: toNum(runRaw.tight_strings), + }, + language: toStr(metaRaw.language), + language_version: toStr(metaRaw.language_version), + language_selected: toStr(metaRaw.language_selected), + }, + analysis: (typeof o.analysis === 'object' && o.analysis !== null ? o.analysis : {}) as Analysis, + strings: (typeof o.strings === 'object' && o.strings !== null ? o.strings : {}) as Strings, + layout: normalizeLayout(o.layout), + }; +}; + +export class ErrorBoundary extends React.Component<{ children: React.ReactNode }, { error: Error | null }> { + state = { error: null as Error | null }; + + static getDerivedStateFromError(error: Error) { + return { error }; + } + + componentDidCatch(error: Error) { + console.error(error); + } + + render() { + if (this.state.error) { + return ( +
+
+

Something went wrong

+

{String(this.state.error)}

+ +
+
+ ); + } + return this.props.children; + } +} + +const App: React.FC = () => { + const [data, setData] = useState(null); + const [searchTerm, setSearchTerm] = useState(''); + const [minStringLength, setMinStringLength] = useState(0); + const [selectedTags, setSelectedTags] = useState([]); + const [showUntagged, setShowUntagged] = useState(true); + const [selectedStructures, setSelectedStructures] = useState([]); + const [showStringsWithoutStructure, setShowStringsWithoutStructure] = useState(true); + const [displayOptions, setDisplayOptions] = useState({ + showTags: true, + showEncoding: true, + showOffsetAndStructure: true, + }); + const [copyFeedback, setCopyFeedback] = useState(''); + + // Theme + const [theme, setTheme] = useState<'light' | 'dark'>(() => { + const saved = localStorage.getItem('floss-viewer-theme'); + if (saved === 'light' || saved === 'dark') return saved; + return window.matchMedia?.('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; + }); + + useEffect(() => { + document.documentElement.setAttribute('data-theme', theme); + localStorage.setItem('floss-viewer-theme', theme); + }, [theme]); + + // Resizable sidebar + const [sidebarWidth, setSidebarWidth] = useState(360); + const isDragging = useRef(false); + const handleRef = useRef(null); + + useEffect(() => { + const handleMouseMove = (e: MouseEvent) => { + if (!isDragging.current) return; + e.preventDefault(); + const newWidth = Math.min(600, Math.max(260, e.clientX)); + setSidebarWidth(newWidth); + }; + + const handleMouseUp = () => { + if (isDragging.current) { + isDragging.current = false; + document.body.classList.remove('resizing'); + handleRef.current?.classList.remove('dragging'); + } + }; + + document.addEventListener('mousemove', handleMouseMove); + document.addEventListener('mouseup', handleMouseUp); + return () => { + document.removeEventListener('mousemove', handleMouseMove); + document.removeEventListener('mouseup', handleMouseUp); + }; + }, []); + + const handleResizeStart = useCallback(() => { + isDragging.current = true; + document.body.classList.add('resizing'); + handleRef.current?.classList.add('dragging'); + }, []); + + const processData = (jsonData: ResultDocument) => { + setData(jsonData); + setSearchTerm(''); + setShowUntagged(true); + setShowStringsWithoutStructure(true); + setMinStringLength(jsonData.metadata.min_length); + + const allTags = new Set(); + const allStructures = new Set(); + const collect = (layout: ResultLayout) => { + layout.strings.forEach(s => { + s.tags.forEach(t => allTags.add(t)); + if (s.structure) { + allStructures.add(s.structure); + } + }); + layout.children.forEach(collect); + }; + if (jsonData.layout) { + collect(jsonData.layout); + } + + const defaultTags = Array.from(allTags).filter( + tag => tag !== '#code' && tag !== '#reloc' + ); + setSelectedTags(defaultTags); + setSelectedStructures(Array.from(allStructures)); + } + + const onDrop = useCallback((acceptedFiles: File[]) => { + const file = acceptedFiles[0]; + if (!file) return; + const reader = new FileReader(); + reader.onload = (e) => { + const content = e.target?.result as string; + let normalized: ResultDocument | null = null; + try { + normalized = normalizeDocument(JSON.parse(content)); + } catch (error) { + console.error("Error parsing JSON:", error); + } + if (!normalized) { + alert("Failed to parse JSON file. Expected a FLOSS result document."); + return; + } + processData(normalized); + }; + reader.onerror = () => { + console.error("Error reading file:", file.name); + alert("Failed to read the file."); + }; + reader.readAsText(file); + }, []); + + const { getRootProps, getInputProps, isDragActive } = useDropzone({ + onDrop, + noClick: true, + noKeyboard: true, + accept: { 'application/json': ['.json'] }, + }); + + const handleSearchChange = (event: React.ChangeEvent) => { + setSearchTerm(event.target.value); + }; + + const handleMinLengthChange = (event: React.ChangeEvent) => { + const value = event.target.value; + setMinStringLength(value === '' ? 0 : parseInt(value, 10)); + }; + + const handleTagChange = (tag: string) => { + setSelectedTags(prev => + prev.includes(tag) ? prev.filter(t => t !== tag) : [...prev, tag] + ); + }; + + const handleStructureChange = (structure: string) => { + setSelectedStructures(prev => + prev.includes(structure) ? prev.filter(s => s !== structure) : [...prev, structure] + ); + }; + + const handleDisplayOptionChange = (option: keyof DisplayOptions) => { + setDisplayOptions(prev => ({ ...prev, [option]: !prev[option] })); + }; + + const tagInfo = useMemo(() => { + if (!data) return { availableTags: [], tagCounts: {}, untaggedCount: 0, totalStringCount: 0 }; + + const counts: { [key: string]: number } = {}; + let untaggedCount = 0; + let totalStringCount = 0; + const collect = (layout: ResultLayout) => { + totalStringCount += layout.strings.length; + for (const s of layout.strings) { + if (s.tags.length === 0) { + untaggedCount++; + } else { + for (const tag of s.tags) { + counts[tag] = (counts[tag] || 0) + 1; + } + } + } + for (const child of layout.children) { + collect(child); + } + }; + if (data.layout) { + collect(data.layout); + } + + return { + availableTags: Object.keys(counts).sort(), + tagCounts: counts, + untaggedCount, + totalStringCount, + }; + }, [data]); + + const structureInfo = useMemo(() => { + if (!data) return { availableStructures: [], structureCounts: {}, withoutStructureCount: 0 }; + + const counts: { [key: string]: number } = {}; + let withoutStructureCount = 0; + const collect = (layout: ResultLayout) => { + for (const s of layout.strings) { + if (!s.structure) { + withoutStructureCount++; + } else { + counts[s.structure] = (counts[s.structure] || 0) + 1; + } + } + for (const child of layout.children) { + collect(child); + } + }; + if (data.layout) { + collect(data.layout); + } + + return { + availableStructures: Object.keys(counts).sort(), + structureCounts: counts, + withoutStructureCount, + }; + }, [data]); + + + const handleSelectAll = () => { + setSelectedTags(tagInfo.availableTags); + setShowUntagged(true); + }; + + const handleSelectNone = () => { + setSelectedTags([]); + setShowUntagged(false); + }; + + const handleFocusView = () => { + const focusedTags = tagInfo.availableTags.filter( + tag => !NOISY_TAGS.includes(tag) + ); + setSelectedTags(focusedTags); + setShowUntagged(true); + }; + + const handlePreview = () => { + // The JSON is now imported directly, so we can just use it. + // The type assertion is safe because we trust the local file. + processData(previewData as ResultDocument); + }; + + const lowercaseMap = useMemo(() => { + const map = new Map(); + const walk = (layout: ResultLayout) => { + layout.strings.forEach(s => map.set(s, s.string.toLowerCase())); + layout.children.forEach(walk); + }; + if (data?.layout) walk(data.layout); + return map; + }, [data]); + + const deferredSearchTerm = useDeferredValue(searchTerm); + + const filteredLayout = useMemo(() => { + if (!data) return null; + if (!data.layout) return null; + + const filter = (layout: ResultLayout): ResultLayout | null => { + const lowerCaseSearchTerm = deferredSearchTerm.toLowerCase(); + + const filteredStrings = layout.strings.filter(s => { + if (s.string.length < minStringLength) return false; + + const searchMatch = deferredSearchTerm === '' + ? true + : subsequenceMatch(lowerCaseSearchTerm, lowercaseMap.get(s) ?? s.string.toLowerCase()); + if (!searchMatch) return false; + + const tagMatch = s.tags.length === 0 + ? showUntagged + : selectedTags.length === 0 ? false : s.tags.every(tag => selectedTags.includes(tag)); + if (!tagMatch) return false; + + const structureMatch = !s.structure + ? showStringsWithoutStructure + : selectedStructures.length === 0 ? false : selectedStructures.includes(s.structure); + if (!structureMatch) return false; + + return true; + }); + + const filteredChildren = layout.children + .map(filter) + .filter((c): c is ResultLayout => c !== null); + + if (filteredStrings.length > 0 || filteredChildren.length > 0) { + return { + ...layout, + strings: filteredStrings, + children: filteredChildren, + }; + } + + return null; + }; + + return filter(data.layout); + }, [data, lowercaseMap, deferredSearchTerm, selectedTags, showUntagged, minStringLength, selectedStructures, showStringsWithoutStructure]); + + const visibleStringCount = useMemo(() => { + if (!filteredLayout) return 0; + let count = 0; + const countStrings = (layout: ResultLayout) => { + count += layout.strings.length; + layout.children.forEach(countStrings); + }; + countStrings(filteredLayout); + return count; + }, [filteredLayout]); + + const ignoredStringCount = tagInfo.totalStringCount - visibleStringCount; + + const handleCopyStrings = () => { + if (!filteredLayout) return; + + const stringsToCopy: string[] = []; + const collectStrings = (layout: ResultLayout) => { + stringsToCopy.push(...layout.strings.map(s => s.string)); + layout.children.forEach(collectStrings); + }; + collectStrings(filteredLayout); + + navigator.clipboard.writeText(stringsToCopy.join('\n')).then(() => { + setCopyFeedback('Copied!'); + setTimeout(() => setCopyFeedback(''), 2000); + }, (err) => { + console.error('Could not copy text: ', err); + setCopyFeedback('Failed to copy.'); + setTimeout(() => setCopyFeedback(''), 2000); + }); + }; + + return ( +
+ {/* ---- Sidebar ---- */} +
+
+ FLOSS +
+ + + + +
+
+ +
+ {data && ( + <> + {/* Metadata */} +
+
+ File + {getFilename(data.metadata.file_path)} +
+
+ MD5 + {chunkHash(data.metadata.md5).map((line, i) =>
{line}
)} +
+
+ SHA256 + {chunkHash(data.metadata.sha256).map((line, i) =>
{line}
)} +
+
+ Time + {new Date(data.metadata.runtime.start_date).toLocaleString()} +
+
+ Ver + {data.metadata.version} +
+
+ + {/* Search */} +
+
+ +
+ Min + { + e.preventDefault(); + setMinStringLength(prev => Math.max(0, prev + (e.deltaY < 0 ? 1 : -1))); + }} + /> +
+
+
+ + {/* Tags Filter */} +
+
+ Tags +
+ + + +
+
+
+ {tagInfo.availableTags.map(tag => ( + handleTagChange(tag)} + /> + ))} + {tagInfo.untaggedCount > 0 && ( + setShowUntagged(p => !p)} + /> + )} +
+
+ + {/* Structures Filter */} +
+
+ Structures +
+
+ {structureInfo.availableStructures.map(structure => ( + handleStructureChange(structure)} + /> + ))} + {structureInfo.withoutStructureCount > 0 && ( + setShowStringsWithoutStructure(p => !p)} + /> + )} +
+
+ + {/* Display Columns */} +
+
+ Columns +
+
+ handleDisplayOptionChange('showTags')} /> + handleDisplayOptionChange('showEncoding')} /> + handleDisplayOptionChange('showOffsetAndStructure')} /> +
+
+ + )} +
+ + {/* Sidebar Footer */} + {data && ( +
+ + {visibleStringCount} / {tagInfo.totalStringCount} + {ignoredStringCount > 0 && ( + <> +  · {ignoredStringCount} ignored + + )} + +
+ + {copyFeedback && {copyFeedback}} +
+
+ )} +
+ + {/* ---- Resize Handle ---- */} +
+ + {/* ---- Main Content ---- */} +
+ {!data ? ( +
+
+

FLOSS Graphical Viewer

+

Drag a JSON file or use the upload button

+
+
+ ) : filteredLayout ? ( + + ) : ( +
+
+

No matches

+

Try adjusting your search or filter settings

+
+
+ )} +
+ + {isDragActive && ( +
+
+

Drop JSON file to load

+

Drop to load a FLOSS JSON result

+
+
+ )} +
+ ); +}; + +export default App; diff --git a/viewer/src/main.tsx b/viewer/src/main.tsx new file mode 100644 index 000000000..14f3c9fbc --- /dev/null +++ b/viewer/src/main.tsx @@ -0,0 +1,12 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import App, { ErrorBoundary } from './App.tsx' + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + + + , +) + diff --git a/viewer/src/pma0303_floss.json b/viewer/src/pma0303_floss.json new file mode 100644 index 000000000..7368dfc70 --- /dev/null +++ b/viewer/src/pma0303_floss.json @@ -0,0 +1,4212 @@ +{ + "metadata": { + "file_path": "/usr/local/google/home/moritzraabe/code/flare-floss/tests/data/pma/Practical Malware Analysis Lab 03-03.exe_", + "md5": "e2bf42217a67e46433da8b6f4507219e", + "sha1": "daf263702f11dc0430d30f9bf443e7885cf91fcb", + "sha256": "ae8a1c7eb64c42ea2a04f97523ebf0844c27029eb040d910048b680f884b9dce", + "version": "0.1.0", + "imagebase": 0, + "min_length": 4, + "runtime": { + "start_date": "2026-04-21T11:19:30.557112", + "total": 0, + "vivisect": 0, + "find_features": 0, + "static_strings": 0, + "layout": 0, + "tags": 0, + "language_strings": 0, + "stack_strings": 0, + "decoded_strings": 0, + "tight_strings": 0 + }, + "language": "", + "language_version": "", + "language_selected": "" + }, + "analysis": { + "enable_static_strings": true, + "enable_stack_strings": true, + "enable_tight_strings": true, + "enable_decoded_strings": true, + "enable_layout": true, + "enable_tags": true, + "functions": { + "discovered": 0, + "library": 0, + "analyzed_stack_strings": 0, + "analyzed_tight_strings": 0, + "analyzed_decoded_strings": 0, + "decoding_function_scores": {} + } + }, + "strings": { + "stack_strings": [], + "tight_strings": [], + "decoded_strings": [], + "static_strings": [], + "language_strings": [], + "language_strings_missed": [] + }, + "layout": { + "name": "pe", + "offset": 0, + "length": 53248, + "strings": [], + "children": [ + { + "name": "header", + "offset": 0, + "length": 4096, + "strings": [ + { + "string": "!This program cannot be run in DOS mode.", + "offset": 77, + "size": 40, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "Rich", + "offset": 192, + "size": 4, + "encoding": "ascii", + "tags": [], + "structure": "rich header" + }, + { + "string": ".text", + "offset": 472, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "section header" + }, + { + "string": "`.rdata", + "offset": 511, + "size": 7, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "section header" + }, + { + "string": "@.data", + "offset": 551, + "size": 6, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "section header" + }, + { + "string": ".rsrc", + "offset": 592, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "section header" + } + ], + "children": [] + }, + { + "name": ".text", + "offset": 4096, + "length": 12288, + "strings": [ + { + "string": "jjjj", + "offset": 4435, + "size": 8, + "encoding": "unicode", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "h@P@", + "offset": 4567, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "hXP@", + "offset": 4572, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "hdP@", + "offset": 4962, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "hlP@", + "offset": 4967, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "Ph0P@", + "offset": 5395, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "GIt#", + "offset": 6585, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "YYh P@", + "offset": 7361, + "size": 6, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "t9UW", + "offset": 7909, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "?=t\"U", + "offset": 7923, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "QQS3", + "offset": 8005, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "PSSW", + "offset": 8069, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "8\"uD", + "offset": 8198, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "8\"uF@", + "offset": 8263, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "8\"u,", + "offset": 8410, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "@@f9", + "offset": 8701, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "@@f9", + "offset": 8708, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#duplicate" + ], + "structure": "" + }, + { + "string": "=|@@", + "offset": 8718, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "SS@SSPVSS", + "offset": 8725, + "size": 9, + "encoding": "ascii", + "tags": [ + "#code", + "#common" + ], + "structure": "" + }, + { + "string": "t#SSUP", + "offset": 8759, + "size": 6, + "encoding": "ascii", + "tags": [ + "#code", + "#common" + ], + "structure": "" + }, + { + "string": "t$$VSS", + "offset": 8766, + "size": 6, + "encoding": "ascii", + "tags": [ + "#code", + "#common" + ], + "structure": "" + }, + { + "string": "_^][YY", + "offset": 8890, + "size": 6, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk", + "#common" + ], + "structure": "" + }, + { + "string": "DSUVWh", + "offset": 8899, + "size": 6, + "encoding": "ascii", + "tags": [ + "#code", + "#common" + ], + "structure": "" + }, + { + "string": "_^][", + "offset": 9316, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "SVWUj", + "offset": 9387, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "]_^[", + "offset": 9408, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "t.;t$$t(", + "offset": 9492, + "size": 8, + "encoding": "ascii", + "tags": [ + "#code", + "#common" + ], + "structure": "" + }, + { + "string": "VC20XC00U", + "offset": 9624, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "SVWU", + "offset": 9638, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code-junk" + ], + "structure": "" + }, + { + "string": "tEVU", + "offset": 9696, + "size": 4, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "t3x<", + "offset": 9714, + "size": 4, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "]_^[", + "offset": 9813, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code-junk", + "#duplicate" + ], + "structure": "" + }, + { + "string": "hxC@", + "offset": 10186, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "58T@", + "offset": 10545, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "90tr", + "offset": 10712, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "%(T@", + "offset": 11074, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "Wj@Y3", + "offset": 11194, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "t7SW", + "offset": 11301, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": " ", + "offset": 11329, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk", + "#msvc" + ], + "structure": "" + }, + { + "string": "@AA;", + "offset": 11539, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "%tT@", + "offset": 11682, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "%xT@", + "offset": 11689, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "j?I_", + "offset": 11831, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "u\t9}", + "offset": 12148, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "ulSj", + "offset": 12439, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "5lT@", + "offset": 12558, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "uY;]", + "offset": 12722, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "pD#U", + "offset": 12859, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "j #M", + "offset": 12942, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "j?^;", + "offset": 12983, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "%tT@", + "offset": 13326, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#duplicate" + ], + "structure": "" + }, + { + "string": "5|T@", + "offset": 13373, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "VWuBh", + "offset": 13782, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "5(@@", + "offset": 13804, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "tzVS", + "offset": 13927, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "GIt%", + "offset": 13961, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "t/Ku", + "offset": 13997, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "h D@", + "offset": 14206, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "9=`T@", + "offset": 14241, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "uFWWj", + "offset": 14247, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "\"WWSh", + "offset": 14284, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "9} u", + "offset": 14388, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#code-junk" + ], + "structure": "" + }, + { + "string": "E WW", + "offset": 14399, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "tMWWS", + "offset": 14520, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "t@9}", + "offset": 14553, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "VSh ", + "offset": 14716, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + }, + { + "string": "h8D@", + "offset": 14797, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code" + ], + "structure": "" + } + ], + "children": [] + }, + { + "name": ".rdata", + "offset": 16384, + "length": 4096, + "strings": [ + { + "string": "runtime error ", + "offset": 16620, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "TLOSS error", + "offset": 16640, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "SING error", + "offset": 16656, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "DOMAIN error", + "offset": 16672, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6028", + "offset": 16688, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- unable to initialize heap", + "offset": 16695, + "size": 27, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6027", + "offset": 16728, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for lowio initialization", + "offset": 16735, + "size": 43, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6026", + "offset": 16784, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for stdio initialization", + "offset": 16791, + "size": 43, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6025", + "offset": 16840, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- pure virtual function call", + "offset": 16847, + "size": 28, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6024", + "offset": 16880, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for _onexit/atexit table", + "offset": 16887, + "size": 43, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6019", + "offset": 16936, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- unable to open console device", + "offset": 16943, + "size": 31, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6018", + "offset": 16980, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- unexpected heap error", + "offset": 16987, + "size": 23, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6017", + "offset": 17016, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- unexpected multithread lock error", + "offset": 17023, + "size": 35, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6016", + "offset": 17064, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for thread data", + "offset": 17071, + "size": 34, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "abnormal program termination", + "offset": 17110, + "size": 28, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6009", + "offset": 17144, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for environment", + "offset": 17151, + "size": 34, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6008", + "offset": 17188, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- not enough space for arguments", + "offset": 17195, + "size": 32, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "R6002", + "offset": 17232, + "size": 5, + "encoding": "ascii", + "tags": [], + "structure": "" + }, + { + "string": "- floating point not loaded", + "offset": 17239, + "size": 27, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "Microsoft Visual C++ Runtime Library", + "offset": 17272, + "size": 36, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "Runtime Error!", + "offset": 17316, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "Program: ", + "offset": 17332, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "", + "offset": 17348, + "size": 22, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "GetLastActivePopup", + "offset": 17372, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "GetActiveWindow", + "offset": 17392, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "MessageBoxA", + "offset": 17408, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "user32.dll", + "offset": 17420, + "size": 10, + "encoding": "ascii", + "tags": [ + "#capa", + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "CloseHandle", + "offset": 17738, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "VirtualFree", + "offset": 17752, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ReadFile", + "offset": 17766, + "size": 8, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "VirtualAlloc", + "offset": 17778, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetFileSize", + "offset": 17794, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "CreateFileA", + "offset": 17808, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ResumeThread", + "offset": 17822, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SetThreadContext", + "offset": 17838, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "WriteProcessMemory", + "offset": 17858, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "VirtualAllocEx", + "offset": 17880, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetProcAddress", + "offset": 17898, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetModuleHandleA", + "offset": 17916, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ReadProcessMemory", + "offset": 17936, + "size": 17, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetThreadContext", + "offset": 17956, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "CreateProcessA", + "offset": 17976, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FreeResource", + "offset": 17994, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SizeofResource", + "offset": 18010, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LockResource", + "offset": 18028, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LoadResource", + "offset": 18044, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FindResourceA", + "offset": 18060, + "size": 13, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetSystemDirectoryA", + "offset": 18076, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "Sleep", + "offset": 18098, + "size": 5, + "encoding": "ascii", + "tags": [ + "#winapi" + ], + "structure": "import table" + }, + { + "string": "KERNEL32.dll", + "offset": 18104, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCommandLineA", + "offset": 18120, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetVersion", + "offset": 18138, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ExitProcess", + "offset": 18152, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "TerminateProcess", + "offset": 18166, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCurrentProcess", + "offset": 18186, + "size": 17, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "UnhandledExceptionFilter", + "offset": 18206, + "size": 24, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetModuleFileNameA", + "offset": 18234, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FreeEnvironmentStringsA", + "offset": 18256, + "size": 23, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FreeEnvironmentStringsW", + "offset": 18282, + "size": 23, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "WideCharToMultiByte", + "offset": 18308, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetEnvironmentStrings", + "offset": 18330, + "size": 21, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetEnvironmentStringsW", + "offset": 18354, + "size": 22, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SetHandleCount", + "offset": 18380, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStdHandle", + "offset": 18398, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetFileType", + "offset": 18414, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStartupInfoA", + "offset": 18428, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapDestroy", + "offset": 18446, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapCreate", + "offset": 18460, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapFree", + "offset": 18474, + "size": 8, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "RtlUnwind", + "offset": 18486, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "WriteFile", + "offset": 18498, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapAlloc", + "offset": 18510, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCPInfo", + "offset": 18522, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetACP", + "offset": 18534, + "size": 6, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetOEMCP", + "offset": 18544, + "size": 8, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapReAlloc", + "offset": 18556, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LoadLibraryA", + "offset": 18570, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "MultiByteToWideChar", + "offset": 18586, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LCMapStringA", + "offset": 18608, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LCMapStringW", + "offset": 18624, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStringTypeA", + "offset": 18640, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStringTypeW", + "offset": 18658, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "import table" + } + ], + "children": [] + }, + { + "name": ".data", + "offset": 20480, + "length": 4096, + "strings": [ + { + "string": "\\svchost.exe", + "offset": 20528, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "NtUnmapViewOfSection", + "offset": 20544, + "size": 20, + "encoding": "ascii", + "tags": [ + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "ntdll.dll", + "offset": 20568, + "size": 9, + "encoding": "ascii", + "tags": [ + "#capa", + "#common", + "#winapi" + ], + "structure": "" + }, + { + "string": "UNICODE", + "offset": 20580, + "size": 7, + "encoding": "ascii", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "LOCALIZATION", + "offset": 20588, + "size": 12, + "encoding": "ascii", + "tags": [], + "structure": "" + } + ], + "children": [] + }, + { + "name": ".rsrc", + "offset": 24576, + "length": 28672, + "strings": [ + { + "string": "UNICODE", + "offset": 24666, + "size": 14, + "encoding": "unicode", + "tags": [ + "#common" + ], + "structure": "" + }, + { + "string": "LOCALIZATION", + "offset": 24682, + "size": 24, + "encoding": "unicode", + "tags": [], + "structure": "" + } + ], + "children": [ + { + "name": "rsrc: UNICODE/LOCALIZATION/0", + "offset": 24708, + "length": 24576, + "strings": [], + "children": [ + { + "name": "pe (XOR decoded with key: 0x41)", + "offset": 24708, + "length": 24576, + "strings": [], + "children": [ + { + "name": "header", + "offset": 24708, + "length": 4096, + "strings": [ + { + "string": "!This program cannot be run in DOS mode.", + "offset": 24785, + "size": 40, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "" + }, + { + "string": "RichS", + "offset": 24900, + "size": 5, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "rich header" + }, + { + "string": ".text", + "offset": 25180, + "size": 5, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "section header" + }, + { + "string": "`.rdata", + "offset": 25219, + "size": 7, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "section header" + }, + { + "string": "@.data", + "offset": 25259, + "size": 6, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "section header" + } + ], + "children": [] + }, + { + "name": ".text", + "offset": 28804, + "length": 12288, + "strings": [ + { + "string": "h@P@", + "offset": 28825, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "hPS@", + "offset": 28864, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "jjjj", + "offset": 28904, + "size": 8, + "encoding": "unicode", + "tags": [ + "#code", + "#code-junk", + "#decoded" + ], + "structure": "" + }, + { + "string": "@hTP@", + "offset": 29033, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "hPW@", + "offset": 29080, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "hPW@", + "offset": 29098, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded", + "#duplicate" + ], + "structure": "" + }, + { + "string": "hPS@", + "offset": 29103, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded", + "#duplicate" + ], + "structure": "" + }, + { + "string": "h0P@", + "offset": 29128, + "size": 4, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "PhPW@", + "offset": 29148, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded" + ], + "structure": "" + }, + { + "string": "PhPW@", + "offset": 29162, + "size": 5, + "encoding": "ascii", + "tags": [ + "#code", + "#decoded", + "#duplicate" + ], + "structure": "" + }, + { + "string": "h", + "offset": 42032, + "size": 22, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "" + }, + { + "string": "GetLastActivePopup", + "offset": 42056, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "" + }, + { + "string": "GetActiveWindow", + "offset": 42076, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "" + }, + { + "string": "MessageBoxA", + "offset": 42092, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "" + }, + { + "string": "user32.dll", + "offset": 42104, + "size": 10, + "encoding": "ascii", + "tags": [ + "#capa", + "#common", + "#decoded", + "#winapi" + ], + "structure": "" + }, + { + "string": "GetModuleHandleA", + "offset": 42422, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "AllocConsole", + "offset": 42442, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "CloseHandle", + "offset": 42458, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "WriteFile", + "offset": 42472, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SetFilePointer", + "offset": 42484, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "CreateFileA", + "offset": 42502, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "KERNEL32.dll", + "offset": 42514, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "UnhookWindowsHookEx", + "offset": 42530, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetMessageA", + "offset": 42552, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SetWindowsHookExA", + "offset": 42566, + "size": 17, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ShowWindow", + "offset": 42586, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FindWindowA", + "offset": 42600, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "CallNextHookEx", + "offset": 42614, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetWindowTextA", + "offset": 42632, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetForegroundWindow", + "offset": 42650, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "USER32.dll", + "offset": 42670, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCommandLineA", + "offset": 42684, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetVersion", + "offset": 42702, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "ExitProcess", + "offset": 42716, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "TerminateProcess", + "offset": 42730, + "size": 16, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCurrentProcess", + "offset": 42750, + "size": 17, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "UnhandledExceptionFilter", + "offset": 42770, + "size": 24, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetModuleFileNameA", + "offset": 42798, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FreeEnvironmentStringsA", + "offset": 42820, + "size": 23, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "FreeEnvironmentStringsW", + "offset": 42846, + "size": 23, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "WideCharToMultiByte", + "offset": 42872, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetEnvironmentStrings", + "offset": 42894, + "size": 21, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetEnvironmentStringsW", + "offset": 42918, + "size": 22, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "SetHandleCount", + "offset": 42944, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStdHandle", + "offset": 42962, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetFileType", + "offset": 42978, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStartupInfoA", + "offset": 42992, + "size": 15, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapDestroy", + "offset": 43010, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapCreate", + "offset": 43024, + "size": 10, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "VirtualFree", + "offset": 43038, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapFree", + "offset": 43052, + "size": 8, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "RtlUnwind", + "offset": 43064, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapAlloc", + "offset": 43076, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetCPInfo", + "offset": 43088, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetACP", + "offset": 43100, + "size": 6, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetOEMCP", + "offset": 43110, + "size": 8, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "VirtualAlloc", + "offset": 43122, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "HeapReAlloc", + "offset": 43138, + "size": 11, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetProcAddress", + "offset": 43152, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LoadLibraryA", + "offset": 43170, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "MultiByteToWideChar", + "offset": 43186, + "size": 19, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LCMapStringA", + "offset": 43208, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "LCMapStringW", + "offset": 43224, + "size": 12, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStringTypeA", + "offset": 43240, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + }, + { + "string": "GetStringTypeW", + "offset": 43258, + "size": 14, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded", + "#winapi" + ], + "structure": "import table" + } + ], + "children": [] + }, + { + "name": ".data", + "offset": 45188, + "length": 4096, + "strings": [ + { + "string": "[Window: ", + "offset": 45238, + "size": 9, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "ConsoleWindowClass", + "offset": 45252, + "size": 18, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "" + }, + { + "string": "practicalmalwareanalysis.log", + "offset": 45272, + "size": 28, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[SHIFT]", + "offset": 45308, + "size": 7, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[ENTER]", + "offset": 45317, + "size": 7, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "" + }, + { + "string": "[BACKSPACE]", + "offset": 45328, + "size": 11, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "BACKSPACE", + "offset": 45340, + "size": 9, + "encoding": "ascii", + "tags": [ + "#common", + "#decoded" + ], + "structure": "" + }, + { + "string": "[TAB]", + "offset": 45352, + "size": 5, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[CTRL]", + "offset": 45360, + "size": 6, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[DEL]", + "offset": 45368, + "size": 5, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[CAPS LOCK]", + "offset": 45416, + "size": 11, + "encoding": "ascii", + "tags": [ + "#decoded" + ], + "structure": "" + }, + { + "string": "[CAPS LOCK]", + "offset": 45428, + "size": 11, + "encoding": "ascii", + "tags": [ + "#decoded", + "#duplicate" + ], + "structure": "" + } + ], + "children": [] + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/viewer/src/types.ts b/viewer/src/types.ts new file mode 100644 index 000000000..b325d5e0a --- /dev/null +++ b/viewer/src/types.ts @@ -0,0 +1,110 @@ +export type ResultString = { + string: string; + offset: number; + size: number; + encoding: string; + tags: string[]; + structure: string; +}; + +export type ResultLayout = { + name: string; + offset: number; + length: number; + strings: ResultString[]; + children: ResultLayout[]; +}; + +export type StackString = { + function: number; + string: string; + encoding: string; + program_counter: number; + stack_pointer: number; + original_stack_pointer: number; + offset: number; + frame_offset: number; +}; + +export type TightString = StackString; + +export type DecodedString = { + address: number; + address_type: string; + string: string; + encoding: string; + decoded_at: number; + decoding_routine: number; +}; + +export type StaticString = { + string: string; + offset: number; + encoding: string; + tags: string[]; + section: string; + structure: string; +}; + +export type Runtime = { + start_date: string; + total: number; + vivisect: number; + find_features: number; + static_strings: number; + layout: number; + tags: number; + language_strings: number; + stack_strings: number; + decoded_strings: number; + tight_strings: number; +}; + +export type Functions = { + discovered: number; + library: number; + analyzed_stack_strings: number; + analyzed_tight_strings: number; + analyzed_decoded_strings: number; + decoding_function_scores: Record; +}; + +export type Analysis = { + enable_static_strings: boolean; + enable_stack_strings: boolean; + enable_tight_strings: boolean; + enable_decoded_strings: boolean; + enable_layout: boolean; + enable_tags: boolean; + functions: Functions; +}; + +export type Metadata = { + file_path: string; + md5: string; + sha1: string; + sha256: string; + version: string; + imagebase: number; + min_length: number; + runtime: Runtime; + language: string; + language_version: string; + language_selected: string; +}; + +export type Strings = { + stack_strings: StackString[]; + tight_strings: TightString[]; + decoded_strings: DecodedString[]; + static_strings: StaticString[]; + language_strings: StaticString[]; + language_strings_missed: StaticString[]; +}; + +export type ResultDocument = { + metadata: Metadata; + analysis: Analysis; + strings: Strings; + layout: ResultLayout | null; +}; diff --git a/viewer/src/vite-env.d.ts b/viewer/src/vite-env.d.ts new file mode 100644 index 000000000..11f02fe2a --- /dev/null +++ b/viewer/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/viewer/tsconfig.app.json b/viewer/tsconfig.app.json new file mode 100644 index 000000000..227a6c672 --- /dev/null +++ b/viewer/tsconfig.app.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["src"] +} diff --git a/viewer/tsconfig.json b/viewer/tsconfig.json new file mode 100644 index 000000000..1ffef600d --- /dev/null +++ b/viewer/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" } + ] +} diff --git a/viewer/tsconfig.node.json b/viewer/tsconfig.node.json new file mode 100644 index 000000000..f85a39906 --- /dev/null +++ b/viewer/tsconfig.node.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + "target": "ES2023", + "lib": ["ES2023"], + "module": "ESNext", + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/viewer/vite.config.ts b/viewer/vite.config.ts new file mode 100644 index 000000000..d7fa0ae78 --- /dev/null +++ b/viewer/vite.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vite' +import react from '@vitejs/plugin-react' +import { viteSingleFile } from 'vite-plugin-singlefile' + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [react(), viteSingleFile()], +}) \ No newline at end of file