ONEKEY baseline

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: ".*\\.md"
+      - id: end-of-file-fixer
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+      - id: check-added-large-files
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.21.0
+    hooks:
+      - id: check-github-actions
+        name: Check Github actions
+      - id: check-github-workflows
+        name: Check Github workflows
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.259"
+    hooks:
+      - id: ruff
+        name: Check python (ruff)
+        args: [--show-source, --fix, --exit-non-zero-on-fix]
+
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        name: Check black
+
+  - repo: https://github.com/jendrikseipp/vulture
+    rev: v2.7
+    hooks:
+      - id: vulture

pyproject.toml

@@ -0,0 +1,49 @@
+[tool.poetry]
+name = "ubi-reader"
+version = "0.8.5"
+description = "Extract files from UBI and UBIFS images."
+authors = ["ONEKEY <[email protected]>", "Jason Pruitt <[email protected]>"]
+license = "GNU GPL"
+readme = "README.md"
+packages = [{include = "ubi_reader"}]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+lzallright = "^0.1.0"
+
+[tool.poetry.group.dev.dependencies]
+pyright = "^1.1.307"
+ruff = "^0.0.265"
+pytest = "^7.3.1"
+pytest-cov = "^4.0.0"
+pre-commit = "^3.3.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+addopts = "--cov=ubi_reader --cov=tests --cov-branch --cov-fail-under=90"
+norecursedirs = """
+  *.egg
+  *_extract
+  .*
+  dist
+  build
+  target
+"""
+
+[tool.vulture]
+paths = ["ubireader/", "scripts/", "vulture_whitelist.py"]
+exclude = []
+
+[tool.pyright]
+exclude = ["build"]
+
+[tool.poetry.scripts]
+ubireader_display_blocks = "scripts.ubireader_display_blocks:main"
+ubireader_display_info = "scripts.ubireader_display_info:main"
+ubireader_extract_files = "scripts.ubireader_extract_files:main"
+ubireader_extract_images = "scripts.ubireader_extract_images:main"
+ubireader_list_files = "scripts.ubireader_list_files:main"
+ubireader_utils_info = "scripts.ubireader_utils_info:main"

scripts/ubireader_display_blocks

@@ -22,69 +22,128 @@
 # matching blocks.
 #############################################################
 
+import argparse
 import os
 import sys
-import argparse
-from ubireader.ubi import ubi_base
-from ubireader.ubi_io import ubi_file
+
 from ubireader import settings
+from ubireader.ubi import ubi_base
 from ubireader.ubi.defines import UBI_EC_HDR_MAGIC
+from ubireader.ubi_io import ubi_file
 from ubireader.ubifs.defines import UBIFS_NODE_MAGIC
-from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size
-
-if __name__=='__main__':
-
-    description = 'Search for specified blocks and display information.'
+from ubireader.utils import (
+    guess_filetype,
+    guess_leb_size,
+    guess_peb_size,
+    guess_start_offset,
+)
+
+if __name__ == "__main__":
+    description = "Search for specified blocks and display information."
     usage = """
     ubireader_display_blocks "{'block.attr': value,...}" path/to/image
         Search for blocks by given parameters and display information about them.
         This is block only, no volume or image information is created, which can
         be used to debug file and image extraction.
     Example:
         "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}"
-        This matches block.peb_num 0, 1, 100, 101, and 102 
+        This matches block.peb_num 0, 1, 100, 101, and 102
         with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks.
         For a full list of parameters check ubireader.ubi.block.description.
     """
     parser = argparse.ArgumentParser(usage=usage, description=description)
 
-    parser.add_argument('-l', '--log', action='store_true', dest='log',
-                      help='Print extraction information to screen.')
-
-    parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose',
-                      help='Prints nearly everything about anything to screen.')
-
-    parser.add_argument('-p', '--peb-size', type=int, dest='block_size',
-                        help='Specify PEB size. (UBI Only)')
-
-    parser.add_argument('-e', '--leb-size', type=int, dest='block_size',
-                        help='Specify LEB size. (UBIFS Only)')
-
-    parser.add_argument('-s', '--start-offset', type=int, dest='start_offset',
-                        help='Specify offset of UBI/UBIFS data in file. (default: 0)')
-
-    parser.add_argument('-n', '--end-offset', type=int, dest='end_offset',
-                        help='Specify end offset of UBI/UBIFS data in file.')
-
-    parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset',
-                        help='Specify offset to start guessing where UBI data is in file. (default: 0)')
-
-    parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors',
-                      help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)')
-
-    parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors',
-                      help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)')
-
-    parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix',
-                      help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)')
-
-    parser.add_argument('block_search_params',
-                      help="""
-                      Double quoted Dict of ubi.block.description attributes, which is run through eval().
-                      Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}"
-                      """)
-
-    parser.add_argument('filepath', help='File with blocks of interest.')
+    parser.add_argument(
+        "-l",
+        "--log",
+        action="store_true",
+        dest="log",
+        help="Print extraction information to screen.",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose-log",
+        action="store_true",
+        dest="verbose",
+        help="Prints nearly everything about anything to screen.",
+    )
+
+    parser.add_argument(
+        "-p",
+        "--peb-size",
+        type=int,
+        dest="block_size",
+        help="Specify PEB size. (UBI Only)",
+    )
+
+    parser.add_argument(
+        "-e",
+        "--leb-size",
+        type=int,
+        dest="block_size",
+        help="Specify LEB size. (UBIFS Only)",
+    )
+
+    parser.add_argument(
+        "-s",
+        "--start-offset",
+        type=int,
+        dest="start_offset",
+        help="Specify offset of UBI/UBIFS data in file. (default: 0)",
+    )
+
+    parser.add_argument(
+        "-n",
+        "--end-offset",
+        type=int,
+        dest="end_offset",
+        help="Specify end offset of UBI/UBIFS data in file.",
+    )
+
+    parser.add_argument(
+        "-g",
+        "--guess-offset",
+        type=int,
+        dest="guess_offset",
+        help="Specify offset to start guessing where UBI data is in file. (default: 0)",
+    )
+
+    parser.add_argument(
+        "-w",
+        "--warn-only-block-read-errors",
+        action="store_true",
+        dest="warn_only_block_read_errors",
+        help="""Attempts to continue extracting files even with bad block reads.
+        Some data will be missing or corrupted! (default: False)""",
+    )
+
+    parser.add_argument(
+        "-i",
+        "--ignore-block-header-errors",
+        action="store_true",
+        dest="ignore_block_header_errors",
+        help="""Forces unused and error containing blocks to be included and
+        also displayed with log/verbose. (default: False)""",
+    )
+
+    parser.add_argument(
+        "-f",
+        "--u-boot-fix",
+        action="store_true",
+        dest="uboot_fix",
+        help="""Assume blocks with image_seq 0 are because of older U-boot
+        implementations and include them. (default: False)""",
+    )
+
+    parser.add_argument(
+        "block_search_params",
+        help="""Double quoted Dict of ubi.block.description attributes,
+        which is run through eval().
+        Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}""",
+    )
+
+    parser.add_argument("filepath", help="File with blocks of interest.")
 
     if len(sys.argv) == 1:
         parser.print_help()
@@ -106,7 +165,7 @@ if __name__=='__main__':
         if not os.path.exists(path):
             parser.error("File path doesn't exist.")
     else:
-        parser.error('File path must be provided.')
+        parser.error("File path must be provided.")
         sys.exit(1)
 
     if args.start_offset:
@@ -123,7 +182,7 @@ if __name__=='__main__':
 
     filetype = guess_filetype(path, start_offset)
     if not filetype:
-        parser.error('Could not determine file type.')
+        parser.error("Could not determine file type.")
 
     if args.block_size:
         block_size = args.block_size
@@ -134,24 +193,28 @@ if __name__=='__main__':
             block_size = guess_leb_size(path)
 
         if not block_size:
-            parser.error('Block size could not be determined.')
+            parser.error("Block size could not be determined.")
 
     if args.block_search_params:
         try:
             search_params = eval(args.block_search_params)
 
             if not isinstance(search_params, dict):
-                parser.error('Search Param Error: Params must be a Dict of block PEB object items:value pairs.')
+                parser.error(
+                    """Search Param Error: Params must be a Dict of block PEB
+                    object items:value pairs."""
+                )
 
-        except NameError as e:
-            parser.error('Search Param Error: Dict key block attrs must be single quoted.')
+        except NameError:
+            parser.error(
+                "Search Param Error: Dict key block attrs must be single quoted."
+            )
 
         except Exception as e:
-            parser.error('Search Param Error: %s' % e)
+            parser.error("Search Param Error: %s" % e)
 
     else:
-        parser.error('No search parameters given, -b arg is required.')
-
+        parser.error("No search parameters given, -b arg is required.")
 
     ufile_obj = ubi_file(path, block_size, start_offset, end_offset)
     ubi_obj = ubi_base(ufile_obj)
@@ -163,7 +226,7 @@ if __name__=='__main__':
         for key in search_params:
             b = ubi_obj.blocks[block]
 
-            for attr in key.split('.'):
+            for attr in key.split("."):
                 if hasattr(b, attr):
                     b = getattr(b, attr)
 
@@ -181,10 +244,10 @@ if __name__=='__main__':
                 match = False
                 break
 
-        if match:                
+        if match:
             blocks.append(ubi_obj.blocks[block])
 
-    print('\nBlock matches: %s' % len(blocks))
+    print("\nBlock matches: %s" % len(blocks))
 
     for block in blocks:
         print(block.display())