From 89579b6f3316d75db1ca719f60b1ec566d7b1a9b Mon Sep 17 00:00:00 2001 From: Quentin Kaiser Date: Fri, 12 May 2023 07:09:53 +0200 Subject: [PATCH 1/4] transition from setuptools to poetry. --- poetry.lock | 24 ++++++++++++++++++++++++ pyproject.toml | 42 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 28 ---------------------------- 3 files changed, 66 insertions(+), 28 deletions(-) create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100755 setup.py diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..4e66d7d --- /dev/null +++ b/poetry.lock @@ -0,0 +1,24 @@ +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. + +[[package]] +name = "lzallright" +version = "0.1.0" +description = "" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "lzallright-0.1.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d4dbdcd418c26ac735a9d4af5d103905cf8ffe86f044cc5cae46f3af9131e01"}, + {file = "lzallright-0.1.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:befa51352f9d3034b8ddaa41431022a87dc2c174ea61e7f224efd83618a48990"}, + {file = "lzallright-0.1.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18f2aba8438dbef22067760fc75903f4af80cc5a48f5a97f512c1bce12042a49"}, + {file = "lzallright-0.1.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7844879cb2dbdf66920915ae2cd890e1429388e116b0fe65a308b59b3d5a765d"}, + {file = "lzallright-0.1.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0ff879402696459222ef9e9837efa93d401f141faf88282d838a83a74d108327"}, + {file = "lzallright-0.1.0-cp38-abi3-win32.whl", hash = "sha256:7c209154c4dcda190ad043f04c18f4b878a7ef3ad33c65799551937b7c1c2a3c"}, + {file = "lzallright-0.1.0-cp38-abi3-win_amd64.whl", hash = "sha256:01b4161c0b87295934f8ed1e3f6b65463942e4a7bc82359dd456ef76faa69f34"}, + {file = "lzallright-0.1.0.tar.gz", hash = "sha256:0de7e0fe110650a79c1e1fca10a188da5b6fc6b2f3730c69dc411782e3b4923a"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "56aada67c1709383460f6cd9aed3aa150be487dddd71f92c43618412ef26148c" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4eb446d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[tool.poetry] +name = "ubi-reader" +version = "0.8.5" +description = "Extract files from UBI and UBIFS images." +authors = ["ONEKEY ", "Jason Pruitt "] +license = "GNU GPL" +readme = "README.md" +packages = [{include = "ubi_reader"}] + +[tool.poetry.dependencies] +python = "^3.8" +lzallright = "^0.1.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +addopts = "--cov=ubi_reader --cov=tests --cov-branch --cov-fail-under=90" +norecursedirs = """ + *.egg + *_extract + .* + dist + build + target +""" + +[tool.vulture] +paths = ["ubireader/", "scripts/", "vulture_whitelist.py"] +exclude = [] + +[tool.pyright] +exclude = ["build"] + +[tool.poetry.scripts] +ubireader_display_blocks = "scripts.ubireader_display_blocks:main" +ubireader_display_info = "scripts.ubireader_display_info:main" +ubireader_extract_files = "scripts.ubireader_extract_files:main" +ubireader_extract_images = "scripts.ubireader_extract_images:main" +ubireader_list_files = "scripts.ubireader_list_files:main" +ubireader_utils_info = "scripts.ubireader_utils_info:main" diff --git a/setup.py b/setup.py deleted file mode 100755 index 8ee5544..0000000 --- a/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup, find_packages - -version = '0.8.5' - -setup( - name='ubi_reader', - version=version, - description='Extract files from UBI and UBIFS images.', - author='Jason Pruitt', - author_email='jrspruitt@gmail.com', - url='https://github.com/jrspruitt/ubi_reader', - long_description='Collection of Python scripts for reading information about and extracting data from UBI and UBIFS images.', - platforms=['Linux'], - license='GNU GPL', - keywords='UBI UBIFS', - - install_requires=['python-lzo'], - packages = find_packages(), - scripts=['scripts/ubireader_display_info', - 'scripts/ubireader_extract_files', - 'scripts/ubireader_list_files', - 'scripts/ubireader_extract_images', - 'scripts/ubireader_utils_info', - 'scripts/ubireader_display_blocks', - ], -) From 83b8962b98647f1ff737a255be9b15adc45f7e21 Mon Sep 17 00:00:00 2001 From: Quentin Kaiser Date: Fri, 12 May 2023 07:10:09 +0200 Subject: [PATCH 2/4] add initial pre-commit config. --- .pre-commit-config.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8854f5a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,39 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + exclude: ".*\\.md" + - id: end-of-file-fixer + - id: check-json + - id: check-toml + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.21.0 + hooks: + - id: check-github-actions + name: Check Github actions + - id: check-github-workflows + name: Check Github workflows + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: "v0.0.259" + hooks: + - id: ruff + name: Check python (ruff) + args: [--show-source, --fix, --exit-non-zero-on-fix] + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + name: Check black + + - repo: https://github.com/jendrikseipp/vulture + rev: v2.7 + hooks: + - id: vulture From bab5d9e7a4d45bf4e1c17d9117f4b502583eecb9 Mon Sep 17 00:00:00 2001 From: Quentin Kaiser Date: Fri, 12 May 2023 07:40:20 +0200 Subject: [PATCH 3/4] pre-commit run --all --- scripts/ubireader_display_blocks | 179 +++++++---- scripts/ubireader_display_info | 184 +++++++---- scripts/ubireader_extract_files | 182 +++++++---- scripts/ubireader_extract_images | 154 ++++++---- scripts/ubireader_list_files | 162 ++++++---- scripts/ubireader_utils_info | 403 +++++++++++++++--------- ubireader/debug.py | 24 +- ubireader/settings.py | 20 +- ubireader/ubi/__init__.py | 51 ++-- ubireader/ubi/block/__init__.py | 101 ++++-- ubireader/ubi/block/layout.py | 9 +- ubireader/ubi/block/sort.py | 32 +- ubireader/ubi/defines.py | 126 ++++---- ubireader/ubi/display.py | 175 ++++++----- ubireader/ubi/headers.py | 67 ++-- ubireader/ubi/image.py | 20 +- ubireader/ubi/volume.py | 45 +-- ubireader/ubi_io.py | 111 +++---- ubireader/ubifs/__init__.py | 74 +++-- ubireader/ubifs/defines.py | 508 ++++++++++++++++--------------- ubireader/ubifs/display.py | 174 +++++------ ubireader/ubifs/list.py | 124 +++++--- ubireader/ubifs/misc.py | 45 ++- ubireader/ubifs/nodes.py | 190 ++++++++---- ubireader/ubifs/output.py | 156 ++++++---- ubireader/ubifs/walk.py | 157 +++++++--- ubireader/utils.py | 77 +++-- vulture_whitelist.py | 174 +++++++++++ 28 files changed, 2348 insertions(+), 1376 deletions(-) create mode 100644 vulture_whitelist.py diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index defd10b..9a394c2 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -22,19 +22,24 @@ # matching blocks. ############################################################# +import argparse import os import sys -import argparse -from ubireader.ubi import ubi_base -from ubireader.ubi_io import ubi_file + from ubireader import settings +from ubireader.ubi import ubi_base from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubi_io import ubi_file from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': - - description = 'Search for specified blocks and display information.' +from ubireader.utils import ( + guess_filetype, + guess_leb_size, + guess_peb_size, + guess_start_offset, +) + +if __name__ == "__main__": + description = "Search for specified blocks and display information." usage = """ ubireader_display_blocks "{'block.attr': value,...}" path/to/image Search for blocks by given parameters and display information about them. @@ -42,49 +47,103 @@ if __name__=='__main__': be used to debug file and image extraction. Example: "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 + This matches block.peb_num 0, 1, 100, 101, and 102 with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. For a full list of parameters check ubireader.ubi.block.description. """ parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) - - parser.add_argument('filepath', help='File with blocks of interest.') + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-p", + "--peb-size", + type=int, + dest="block_size", + help="Specify PEB size. (UBI Only)", + ) + + parser.add_argument( + "-e", + "--leb-size", + type=int, + dest="block_size", + help="Specify LEB size. (UBIFS Only)", + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI/UBIFS data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI/UBIFS data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument( + "block_search_params", + help="""Double quoted Dict of ubi.block.description attributes, + which is run through eval(). + Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}""", + ) + + parser.add_argument("filepath", help="File with blocks of interest.") if len(sys.argv) == 1: parser.print_help() @@ -106,7 +165,7 @@ if __name__=='__main__': if not os.path.exists(path): parser.error("File path doesn't exist.") else: - parser.error('File path must be provided.') + parser.error("File path must be provided.") sys.exit(1) if args.start_offset: @@ -123,7 +182,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if not filetype: - parser.error('Could not determine file type.') + parser.error("Could not determine file type.") if args.block_size: block_size = args.block_size @@ -134,24 +193,28 @@ if __name__=='__main__': block_size = guess_leb_size(path) if not block_size: - parser.error('Block size could not be determined.') + parser.error("Block size could not be determined.") if args.block_search_params: try: search_params = eval(args.block_search_params) if not isinstance(search_params, dict): - parser.error('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') + parser.error( + """Search Param Error: Params must be a Dict of block PEB + object items:value pairs.""" + ) - except NameError as e: - parser.error('Search Param Error: Dict key block attrs must be single quoted.') + except NameError: + parser.error( + "Search Param Error: Dict key block attrs must be single quoted." + ) except Exception as e: - parser.error('Search Param Error: %s' % e) + parser.error("Search Param Error: %s" % e) else: - parser.error('No search parameters given, -b arg is required.') - + parser.error("No search parameters given, -b arg is required.") ufile_obj = ubi_file(path, block_size, start_offset, end_offset) ubi_obj = ubi_base(ufile_obj) @@ -163,7 +226,7 @@ if __name__=='__main__': for key in search_params: b = ubi_obj.blocks[block] - for attr in key.split('.'): + for attr in key.split("."): if hasattr(b, attr): b = getattr(b, attr) @@ -181,10 +244,10 @@ if __name__=='__main__': match = False break - if match: + if match: blocks.append(ubi_obj.blocks[block]) - print('\nBlock matches: %s' % len(blocks)) + print("\nBlock matches: %s" % len(blocks)) for block in blocks: print(block.display()) diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index 5aec8b4..7f594ca 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -18,60 +18,122 @@ # along with this program. If not, see . ############################################################# +import argparse import os import sys import time -import argparse from ubireader import settings from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubi_io import leb_virtual_file, ubi_file from ubireader.ubifs import ubifs from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size -from ubireader.ubi_io import ubi_file, leb_virtual_file - - -if __name__=='__main__': +from ubireader.utils import ( + guess_filetype, + guess_leb_size, + guess_peb_size, + guess_start_offset, +) + +if __name__ == "__main__": start = time.time() - description = 'Show information about UBI or UBIFS image.' - usage = 'ubireader_display_info [options] filepath' + description = "Show information about UBI or UBIFS image." + usage = "ubireader_display_info [options] filepath" parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', - help='Get UBIFS information from inside a UBI image. (default: false)') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('filepath', help='File to extract contents of.') + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-u", + "--ubifs-info", + action="store_true", + dest="ubifs_info", + help="Get UBIFS information from inside a UBI image. (default: false)", + ) + + parser.add_argument( + "-p", + "--peb-size", + type=int, + dest="block_size", + help="Specify PEB size. (UBI Only)", + ) + + parser.add_argument( + "-e", + "--leb-size", + type=int, + dest="block_size", + help="Specify LEB size. (UBIFS Only)", + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI/UBIFS data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI/UBIFS data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument("filepath", help="File to extract contents of.") if len(sys.argv) == 1: parser.print_help() @@ -108,7 +170,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if not filetype: - parser.error('Could not determine file type.') + parser.error("Could not determine file type.") ubifs_info = args.ubifs_info @@ -121,8 +183,7 @@ if __name__=='__main__': block_size = guess_leb_size(path) if not block_size: - parser.error('Block size could not be determined.') - + parser.error("Block size could not be determined.") # Create file object. ufile_obj = ubi_file(path, block_size, start_offset, end_offset) @@ -139,49 +200,48 @@ if __name__=='__main__': for image in ubi_obj.images: # Display image information if not UBIFS request. if not ubifs_info: - print('%s' % image.display('\t')) + print("%s" % image.display("\t")) # Loop through volumes in each image. for volume in image.volumes: # Show UBI or UBIFS info. if not ubifs_info: - # Display volume information. - print(image.volumes[volume].display('\t\t')) + print(image.volumes[volume].display("\t\t")) else: # Get blocks associated with this volume. vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - + # Skip volume if empty. if not len(vol_blocks): continue - + # Create LEB backed virtual file with volume blocks. # Necessary to prevent having to load entire UBI image # into memory. lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - + # Create UBIFS object and print info. ubifs_obj = ubifs(lebv_file) print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) + print(ubifs_obj.superblock_node.display("\t")) + print(ubifs_obj.master_node.display("\t")) try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') + print(ubifs_obj.master_node2.display("\t")) + except Exception: + print("Master Node Error only one valid node.") elif filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) + print(ubifs_obj.superblock_node.display("\t")) + print(ubifs_obj.master_node.display("\t")) try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') + print(ubifs_obj.master_node2.display("\t")) + except Exception: + print("Master Node Error only one valid node.") else: - print('Something went wrong to get here.') + print("Something went wrong to get here.") diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 4ba3c17..11d0c4b 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -18,76 +18,145 @@ # along with this program. If not, see . ############################################################# +import argparse import os import sys import time -import argparse from ubireader import settings +from ubireader.debug import error, log from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubi_io import leb_virtual_file, ubi_file from ubireader.ubifs import ubifs -from ubireader.ubifs.output import extract_files from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size +from ubireader.ubifs.output import extract_files +from ubireader.utils import ( + guess_filetype, + guess_leb_size, + guess_peb_size, + guess_start_offset, +) + def create_output_dir(outpath): if os.path.exists(outpath): if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) + error( + create_output_dir, + "Fatal", + "Output directory is not empty. %s" % outpath, + ) else: try: os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) + log(create_output_dir, "Created output path: %s" % outpath) except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) + error(create_output_dir, "Fatal", "%s" % e) -if __name__=='__main__': +if __name__ == "__main__": start = time.time() - description = 'Extract contents of a UBI or UBIFS image.' - usage = 'ubireader_extract_files [options] filepath' + description = "Extract contents of a UBI or UBIFS image." + usage = "ubireader_extract_files [options] filepath" parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') + parser.add_argument( + "-k", + "--keep-permissions", + action="store_true", + dest="permissions", + help="Maintain file permissions, requires running as root. (default: False)", + ) + + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-p", + "--peb-size", + type=int, + dest="block_size", + help="Specify PEB size. (UBI Only)", + ) + + parser.add_argument( + "-e", + "--leb-size", + type=int, + dest="block_size", + help="Specify LEB size. (UBIFS Only)", + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI/UBIFS data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI/UBIFS data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument( + "-o", "--output-dir", dest="outpath", help="Specify output directory path." + ) + + parser.add_argument("filepath", help="File to extract contents of.") if len(sys.argv) == 1: parser.print_help() @@ -124,7 +193,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if not filetype: - parser.error('Could not determine file type.') + parser.error("Could not determine file type.") if args.outpath: outpath = args.outpath @@ -140,7 +209,7 @@ if __name__=='__main__': block_size = guess_leb_size(path) if not block_size: - parser.error('Block size could not be determined.') + parser.error("Block size could not be determined.") perms = args.permissions @@ -153,20 +222,18 @@ if __name__=='__main__': # Loop through found images in file. for image in ubi_obj.images: - # Create path for specific image # In case multiple images in data - img_outpath = os.path.join(outpath, '%s' % image.image_seq) + img_outpath = os.path.join(outpath, "%s" % image.image_seq) # Loop through volumes in each image. for volume in image.volumes: - # Get blocks associated with this volume. vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) # Create volume data output path. vol_outpath = os.path.join(img_outpath, volume) - + # Create volume output path directory. create_output_dir(vol_outpath) @@ -181,10 +248,9 @@ if __name__=='__main__': # Extract files from UBI image. ubifs_obj = ubifs(lebv_file) - print('Extracting files to: %s' % vol_outpath) + print("Extracting files to: %s" % vol_outpath) extract_files(ubifs_obj, vol_outpath, perms) - elif filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) @@ -193,8 +259,8 @@ if __name__=='__main__': create_output_dir(outpath) # Extract files from UBIFS image. - print('Extracting files to: %s' % outpath) + print("Extracting files to: %s" % outpath) extract_files(ubifs_obj, outpath, perms) else: - print('Something went wrong to get here.') + print("Something went wrong to get here.") diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index 9e7400f..bc3917b 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -18,67 +18,117 @@ # along with this program. If not, see . ############################################################# +import argparse import os import sys import time -import argparse from ubireader import settings +from ubireader.debug import error, log from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC from ubireader.ubi_io import ubi_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size +from ubireader.utils import guess_filetype, guess_peb_size, guess_start_offset + def create_output_dir(outpath): if not os.path.exists(outpath): try: os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) + log(create_output_dir, "Created output path: %s" % outpath) except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) + error(create_output_dir, "Fatal", "%s" % e) -if __name__=='__main__': +if __name__ == "__main__": start = time.time() - description = 'Extract UBI or UBIFS images from file containing UBI data in it.' - usage = 'ubireader_extract_images [options] filepath' + description = "Extract UBI or UBIFS images from file containing UBI data in it." + usage = "ubireader_extract_images [options] filepath" parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-p", "--peb-size", type=int, dest="block_size", help="Specify PEB size." + ) + + parser.add_argument( + "-u", + "--image-type", + dest="image_type", + help="Specify image type to extract UBI or UBIFS. (default: UBIFS)", + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument( + "-o", "--output-dir", dest="outpath", help="Specify output directory path." + ) + + parser.add_argument("filepath", help="File to extract contents of.") if len(sys.argv) == 1: parser.print_help() @@ -115,7 +165,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') + parser.error("File does not look like UBI data.") img_name = os.path.basename(path) if args.outpath: @@ -129,12 +179,12 @@ if __name__=='__main__': block_size = guess_peb_size(path) if not block_size: - parser.error('Block size could not be determined.') + parser.error("Block size could not be determined.") if args.image_type: image_type = args.image_type.upper() else: - image_type = 'UBIFS' + image_type = "UBIFS" # Create file object. ufile_obj = ubi_file(path, block_size, start_offset, end_offset) @@ -144,11 +194,11 @@ if __name__=='__main__': # Loop through found images in file. for image in ubi_obj.images: - if image_type == 'UBI': + if image_type == "UBI": # Create output path and open file. - img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) + img_outpath = os.path.join(outpath, "img-%s.ubi" % image.image_seq) create_output_dir(outpath) - f = open(img_outpath, 'wb') + f = open(img_outpath, "wb") # Loop through UBI image blocks for block in image.get_blocks(ubi_obj.blocks): @@ -156,13 +206,15 @@ if __name__=='__main__': # Write block (PEB) to file f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) - elif image_type == 'UBIFS': + elif image_type == "UBIFS": # Loop through image volumes for volume in image.volumes: # Create output path and open file. - vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) + vol_outpath = os.path.join( + outpath, "img-%s_vol-%s.ubifs" % (image.image_seq, volume) + ) create_output_dir(outpath) - f = open(vol_outpath, 'wb') + f = open(vol_outpath, "wb") # Loop through and write volume block data (LEB) to file. for block in image.volumes[volume].reader(ubi_obj): diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 0571704..3bcea73 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -18,67 +18,123 @@ # along with this program. If not, see . ############################################################# +import argparse import os import sys import time -import argparse from ubireader import settings from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubi_io import leb_virtual_file, ubi_file from ubireader.ubifs import ubifs -from ubireader.ubifs.list import list_files, copy_file from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': +from ubireader.ubifs.list import copy_file, list_files +from ubireader.utils import ( + guess_filetype, + guess_leb_size, + guess_peb_size, + guess_start_offset, +) + +if __name__ == "__main__": start = time.time() - description = 'List and Extract files of a UBI or UBIFS image.' - usage = 'ubireader_list_files [options] filepath' + description = "List and Extract files of a UBI or UBIFS image." + usage = "ubireader_list_files [options] filepath" parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-P', '--path', dest='listpath', - help='Path to list.') - - parser.add_argument('-C', '--copy', dest='copyfile', - help='File to Copy.') - - parser.add_argument('-D', '--copy-dest', dest='copyfiledest', - help='Copy Destination.') - - parser.add_argument('filepath', help='UBI/UBIFS image file.') + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-p", + "--peb-size", + type=int, + dest="block_size", + help="Specify PEB size. (UBI Only)", + ) + + parser.add_argument( + "-e", + "--leb-size", + type=int, + dest="block_size", + help="Specify LEB size. (UBIFS Only)", + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI/UBIFS data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI/UBIFS data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument("-P", "--path", dest="listpath", help="Path to list.") + + parser.add_argument("-C", "--copy", dest="copyfile", help="File to Copy.") + + parser.add_argument( + "-D", "--copy-dest", dest="copyfiledest", help="Copy Destination." + ) + + parser.add_argument("filepath", help="UBI/UBIFS image file.") if len(sys.argv) == 1: parser.print_help() @@ -115,7 +171,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if not filetype: - parser.error('Could not determine file type.') + parser.error("Could not determine file type.") if args.block_size: block_size = args.block_size @@ -126,7 +182,7 @@ if __name__=='__main__': block_size = guess_leb_size(path) if not block_size: - parser.error('Block size could not be determined.') + parser.error("Block size could not be determined.") # Create file object. ufile_obj = ubi_file(path, block_size, start_offset, end_offset) @@ -137,10 +193,8 @@ if __name__=='__main__': # Loop through found images in file. for image in ubi_obj.images: - # Loop through volumes in each image. for volume in image.volumes: - # Get blocks associated with this volume. vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) @@ -171,4 +225,4 @@ if __name__=='__main__': copy_file(ubifs_obj, args.copyfile, args.copyfiledest) else: - print('Something went wrong to get here.') + print("Something went wrong to get here.") diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index 40277b2..90a668c 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -18,33 +18,44 @@ # along with this program. If not, see . ############################################################# +import argparse import os import sys import time -import argparse -if (sys.version_info > (3, 0)): + +if sys.version_info > (3, 0): import configparser else: import ConfigParser as configparser + from ubireader import settings +from ubireader.debug import error, log from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG +from ubireader.ubi.defines import ( + PRINT_VOL_TYPE_LIST, + UBI_EC_HDR_MAGIC, + UBI_VTBL_AUTORESIZE_FLG, +) +from ubireader.ubi_io import leb_virtual_file, ubi_file from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size +from ubireader.ubifs.defines import PRINT_UBIFS_COMPR, PRINT_UBIFS_KEY_HASH +from ubireader.utils import guess_filetype, guess_peb_size, guess_start_offset + def create_output_dir(outpath): if os.path.exists(outpath): if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) + error( + create_output_dir, + "Fatal", + "Output directory is not empty. %s" % outpath, + ) else: try: os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) + log(create_output_dir, "Created output path: %s" % outpath) except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) + error(create_output_dir, "Fatal", "%s" % e) def get_ubi_params(ubi_obj): @@ -56,24 +67,25 @@ def get_ubi_params(ubi_obj): Returns: Dict -- Dict keyed to volume with Dict of args and flags. """ - ubi_flags = {'min_io_size':'-m', - 'max_bud_bytes':'-j', - 'leb_size':'-e', - 'default_compr':'-x', - 'sub_page_size':'-s', - 'fanout':'-f', - 'key_hash':'-k', - 'orph_lebs':'-p', - 'log_lebs':'-l', - 'max_leb_cnt': '-c', - 'peb_size':'-p', - 'sub_page_size':'-s', - 'vid_hdr_offset':'-O', - 'version':'-x', - 'image_seq':'-Q', - 'alignment':'-a', - 'vol_id':'-n', - 'name':'-N'} + ubi_flags = { + "min_io_size": "-m", + "max_bud_bytes": "-j", + "leb_size": "-e", + "default_compr": "-x", + "sub_page_size": "-s", + "fanout": "-f", + "key_hash": "-k", + "orph_lebs": "-p", + "log_lebs": "-l", + "max_leb_cnt": "-c", + "peb_size": "-p", + "vid_hdr_offset": "-O", + "version": "-x", + "image_seq": "-Q", + "alignment": "-a", + "vol_id": "-n", + "name": "-N", + } ubi_params = {} ubi_args = {} @@ -90,49 +102,69 @@ def get_ubi_params(ubi_obj): ini_params[img_seq][volume] = {} # Get ubinize.ini settings - ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] + ini_params[img_seq][volume]["vol_type"] = PRINT_VOL_TYPE_LIST[ + image.volumes[volume].vol_rec.vol_type + ] if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: - ini_params[img_seq][volume]['vol_flags'] = 'autoresize' + ini_params[img_seq][volume]["vol_flags"] = "autoresize" else: - ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags - - ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') - ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment - - ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size + ini_params[img_seq][volume]["vol_flags"] = image.volumes[ + volume + ].vol_rec.flags + + ini_params[img_seq][volume]["vol_id"] = image.volumes[volume].vol_id + ini_params[img_seq][volume]["vol_name"] = ( + image.volumes[volume].name.rstrip(b"\x00").decode("utf-8") + ) + ini_params[img_seq][volume]["vol_alignment"] = image.volumes[ + volume + ].vol_rec.alignment + + ini_params[img_seq][volume]["vol_size"] = ( + image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size + ) # Create file object backed by UBI blocks. - lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) + lebv_file = leb_virtual_file( + ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks) + ) # Create UBIFS object ubifs_obj = ubifs(lebv_file) for key, value in ubifs_obj.superblock_node: - if key == 'key_hash': + if key == "key_hash": value = PRINT_UBIFS_KEY_HASH[value] - elif key == 'default_compr': + elif key == "default_compr": value = PRINT_UBIFS_COMPR[value] if key in ubi_flags: ubi_args[img_seq][volume][key] = value - + for key, value in image.volumes[volume].vol_rec: - if key == 'name': - value = value.rstrip(b'\x00').decode('utf-8') + if key == "name": + value = value.rstrip(b"\x00").decode("utf-8") if key in ubi_flags: ubi_args[img_seq][volume][key] = value - ubi_args[img_seq][volume]['version'] = image.version - ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['image_seq'] = image.image_seq - ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size - ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - - ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} + ubi_args[img_seq][volume]["version"] = image.version + ubi_args[img_seq][volume]["vid_hdr_offset"] = image.vid_hdr_offset + ubi_args[img_seq][volume]["sub_page_size"] = ubi_args[img_seq][volume][ + "vid_hdr_offset" + ] + ubi_args[img_seq][volume]["sub_page_size"] = ubi_args[img_seq][volume][ + "vid_hdr_offset" + ] + ubi_args[img_seq][volume]["image_seq"] = image.image_seq + ubi_args[img_seq][volume]["peb_size"] = ubi_obj.peb_size + ubi_args[img_seq][volume]["vol_id"] = image.volumes[volume].vol_id + + ubi_params[img_seq][volume] = { + "flags": ubi_flags, + "args": ubi_args[img_seq][volume], + "ini": ini_params[img_seq][volume], + } return ubi_params @@ -141,24 +173,24 @@ def print_ubi_params(ubi_obj): ubi_params = get_ubi_params(ubi_obj) for img_params in ubi_params: for volume in ubi_params[img_params]: - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - ini_params = ubi_params[img_params][volume]['ini'] - sorted_keys = sorted(ubi_params[img_params][volume]['args']) - - print('\nVolume %s' % volume) + ubi_flags = ubi_params[img_params][volume]["flags"] + ubi_args = ubi_params[img_params][volume]["args"] + ini_params = ubi_params[img_params][volume]["ini"] + sorted_keys = sorted(ubi_params[img_params][volume]["args"]) + + print("\nVolume %s" % volume) for key in sorted_keys: - if len(key)< 8: - name = '%s\t' % key + if len(key) < 8: + name = "%s\t" % key else: name = key - print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) + print("\t%s\t%s %s" % (name, ubi_flags[key], ubi_args[key])) - print('\n\t#ubinize.ini#') - print('\t[%s]' % ini_params['vol_name']) + print("\n\t#ubinize.ini#") + print("\t[%s]" % ini_params["vol_name"]) for key in ini_params: - if key != 'name': - print('\t%s=%s' % (key, ini_params[key])) + if key != "name": + print("\t%s=%s" % (key, ini_params[key])) def make_files(ubi, outpath): @@ -166,117 +198,185 @@ def make_files(ubi, outpath): for img_params in ubi_params: config = configparser.ConfigParser() - img_outpath = os.path.join(outpath, 'img-%s' % img_params) + img_outpath = os.path.join(outpath, "img-%s" % img_params) if not os.path.exists(img_outpath): os.mkdir(img_outpath) - ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) - ubi_file = os.path.join('img-%s.ubi' % img_params) - script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) - ubifs_files =[] - buf = '#!/bin/sh\n' - print('Writing to: %s' % script_path) + ini_path = os.path.join(img_outpath, "img-%s.ini" % img_params) + ubi_file = os.path.join("img-%s.ubi" % img_params) + script_path = os.path.join(img_outpath, "create_ubi_img-%s.sh" % img_params) + ubifs_files = [] + buf = "#!/bin/sh\n" + print("Writing to: %s" % script_path) - with open(script_path, 'w') as fscr: - with open(ini_path, 'w') as fini: - print('Writing to: %s' % ini_path) + with open(script_path, "w") as fscr: + with open(ini_path, "w") as fini: + print("Writing to: %s" % ini_path) vol_idx = 0 for volume in ubi_params[img_params]: - ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) - ini_params = ubi_params[img_params][volume]['ini'] - ini_file = 'img-%s.ini' % img_params + ubifs_files.append( + os.path.join("img-%s_%s.ubifs" % (img_params, vol_idx)) + ) + ini_params = ubi_params[img_params][volume]["ini"] + ini_file = "img-%s.ini" % img_params config.add_section(volume) - config.set(volume, 'mode', 'ubi') - config.set(volume, 'image', ubifs_files[vol_idx]) - + config.set(volume, "mode", "ubi") + config.set(volume, "image", ubifs_files[vol_idx]) + for i in ini_params: config.set(volume, i, str(ini_params[i])) - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - mkfs_flags = ['min_io_size', - 'leb_size', - 'max_leb_cnt', - 'default_compr', - 'fanout', - 'key_hash', - 'orph_lebs', - 'log_lebs'] - - argstr = '' + ubi_flags = ubi_params[img_params][volume]["flags"] + ubi_args = ubi_params[img_params][volume]["args"] + mkfs_flags = [ + "min_io_size", + "leb_size", + "max_leb_cnt", + "default_compr", + "fanout", + "key_hash", + "orph_lebs", + "log_lebs", + ] + + argstr = "" for flag in mkfs_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) - peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) - min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) - #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) - vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) - sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) - - buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) + argstr += " %s %s" % (ubi_flags[flag], ubi_args[flag]) + + "%s %s" % (ubi_flags["peb_size"], ubi_args["peb_size"]) + "%s %s" % ( + ubi_flags["min_io_size"], + ubi_args["min_io_size"], + ) + "%s %s" % ( + ubi_flags["vid_hdr_offset"], + ubi_args["vid_hdr_offset"], + ) + "%s %s" % ( + ubi_flags["sub_page_size"], + ubi_args["sub_page_size"], + ) + + buf += "/usr/sbin/mkfs.ubifs%s -r $%s %s\n" % ( + argstr, + (vol_idx + 1), + ubifs_files[vol_idx], + ) vol_idx += 1 config.write(fini) - ubinize_flags = ['peb_size', - 'min_io_size', - 'vid_hdr_offset', - 'sub_page_size', - 'version', - 'image_seq'] + ubinize_flags = [ + "peb_size", + "min_io_size", + "vid_hdr_offset", + "sub_page_size", + "version", + "image_seq", + ] - argstr = '' + argstr = "" for flag in ubinize_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) + argstr += " %s %s" % (ubi_flags[flag], ubi_args[flag]) - buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) + buf += "/usr/sbin/ubinize%s -o %s %s\n" % (argstr, ubi_file, ini_file) fscr.write(buf) os.chmod(script_path, 0o755) -if __name__=='__main__': + +if __name__ == "__main__": start = time.time() - description = 'Determine settings for recreating UBI image.' - usage = 'ubireader_utils_info [options] filepath' + description = "Determine settings for recreating UBI image." + usage = "ubireader_utils_info [options] filepath" parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', - help='Print parameters to screen only. (default: false)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') + parser.add_argument( + "-r", + "--show-only", + action="store_true", + dest="show_only", + help="Print parameters to screen only. (default: false)", + ) + + parser.add_argument( + "-l", + "--log", + action="store_true", + dest="log", + help="Print extraction information to screen.", + ) + + parser.add_argument( + "-v", + "--verbose-log", + action="store_true", + dest="verbose", + help="Prints nearly everything about anything to screen.", + ) + + parser.add_argument( + "-p", "--peb-size", type=int, dest="block_size", help="Specify PEB size." + ) + + parser.add_argument( + "-s", + "--start-offset", + type=int, + dest="start_offset", + help="Specify offset of UBI data in file. (default: 0)", + ) + + parser.add_argument( + "-n", + "--end-offset", + type=int, + dest="end_offset", + help="Specify end offset of UBI data in file.", + ) + + parser.add_argument( + "-g", + "--guess-offset", + type=int, + dest="guess_offset", + help="Specify offset to start guessing where UBI data is in file. (default: 0)", + ) + + parser.add_argument( + "-w", + "--warn-only-block-read-errors", + action="store_true", + dest="warn_only_block_read_errors", + help="""Attempts to continue extracting files even with bad block reads. + Some data will be missing or corrupted! (default: False)""", + ) + + parser.add_argument( + "-i", + "--ignore-block-header-errors", + action="store_true", + dest="ignore_block_header_errors", + help="""Forces unused and error containing blocks to be included and + also displayed with log/verbose. (default: False)""", + ) + + parser.add_argument( + "-f", + "--u-boot-fix", + action="store_true", + dest="uboot_fix", + help="""Assume blocks with image_seq 0 are because of older U-boot + implementations and include them. (default: False)""", + ) + + parser.add_argument( + "-o", "--output-dir", dest="outpath", help="Specify output directory path." + ) + + parser.add_argument("filepath", help="File to extract contents of.") if len(sys.argv) == 1: parser.print_help() @@ -313,7 +413,7 @@ if __name__=='__main__': filetype = guess_filetype(path, start_offset) if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') + parser.error("File does not look like UBI data.") img_name = os.path.basename(path) if args.outpath: @@ -327,7 +427,7 @@ if __name__=='__main__': block_size = guess_peb_size(path) if not block_size: - parser.error('Block size could not be determined.') + parser.error("Block size could not be determined.") # Create file object. ufile_obj = ubi_file(path, block_size, start_offset, end_offset) @@ -342,4 +442,3 @@ if __name__=='__main__': create_output_dir(outpath) # Create build scripts. make_files(ubi_obj, outpath) - diff --git a/ubireader/debug.py b/ubireader/debug.py index 7201fe9..4c7800c 100755 --- a/ubireader/debug.py +++ b/ubireader/debug.py @@ -19,35 +19,39 @@ import sys import traceback + from ubireader import settings + def log(obj, message): if settings.logging_on or settings.logging_on_verbose: - print('{} {}'.format(obj.__name__, message)) + print("{} {}".format(obj.__name__, message)) + def verbose_log(obj, message): if settings.logging_on_verbose: log(obj, message) + def verbose_display(displayable_obj): if settings.logging_on_verbose: - print(displayable_obj.display('\t')) + print(displayable_obj.display("\t")) + def error(obj, level, message): - if settings.error_action == 'exit': - print('{} {}: {}'.format(obj.__name__, level, message)) + if settings.error_action == "exit": + print("{} {}: {}".format(obj.__name__, level, message)) if settings.fatal_traceback: traceback.print_exc() sys.exit(1) else: - if level.lower() == 'warn': - print('{} {}: {}'.format(obj.__name__, level, message)) - elif level.lower() == 'fatal': - print('{} {}: {}'.format(obj.__name__, level, message)) + if level.lower() == "warn": + print("{} {}: {}".format(obj.__name__, level, message)) + elif level.lower() == "fatal": + print("{} {}: {}".format(obj.__name__, level, message)) if settings.fatal_traceback: traceback.print_exc() sys.exit(1) else: - print('{} {}: {}'.format(obj.__name__, level, message)) - + print("{} {}: {}".format(obj.__name__, level, message)) diff --git a/ubireader/settings.py b/ubireader/settings.py index 3ad9726..ff2a9a9 100755 --- a/ubireader/settings.py +++ b/ubireader/settings.py @@ -17,18 +17,18 @@ # along with this program. If not, see . ############################################################# -output_dir = 'ubifs-root' +output_dir = "ubifs-root" -error_action = True # if 'exit' on any error exit program. -fatal_traceback = False # Print traceback on fatal errors. +error_action = True # if 'exit' on any error exit program. +fatal_traceback = False # Print traceback on fatal errors. -ignore_block_header_errors = False # Ignore block errors. -warn_only_block_read_errors = False # Warning instead of Fatal error. +ignore_block_header_errors = False # Ignore block errors. +warn_only_block_read_errors = False # Warning instead of Fatal error. -logging_on = False # Print debug info on. -logging_on_verbose = False # Print verbose debug info on. +logging_on = False # Print debug info on. +logging_on_verbose = False # Print verbose debug info on. -use_dummy_socket_file = False # Create regular file place holder for sockets. -use_dummy_devices = False # Create regular file place holder for devices. +use_dummy_socket_file = False # Create regular file place holder for sockets. +use_dummy_devices = False # Create regular file place holder for devices. -uboot_fix = False # Older u-boot sets image_seq to 0 on blocks it's written to. +uboot_fix = False # Older u-boot sets image_seq to 0 on blocks it's written to. diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index cf2cb88..325cc80 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -18,17 +18,17 @@ ############################################################# from ubireader.debug import error -from ubireader.ubi.block import sort, extract_blocks from ubireader.ubi import display +from ubireader.ubi.block import extract_blocks, layout, rm_old_blocks, sort from ubireader.ubi.image import description as image -from ubireader.ubi.block import layout, rm_old_blocks + class ubi_base(object): """UBI Base object Arguments: - Obj:image -- UBI image object - + Obj:image -- UBI image object + Attributes: ubi_file:file -- ubi_file object Int:block_count -- Number of blocks found. @@ -40,20 +40,19 @@ class ubi_base(object): """ def __init__(self, ubi_file): - self.__name__ = 'UBI' + self.__name__ = "UBI" self._file = ubi_file self._first_peb_num = 0 self._blocks = extract_blocks(self) self._block_count = len(self.blocks) if self._block_count <= 0: - error(self, 'Fatal', 'No blocks found.') + error(self, "Fatal", "No blocks found.") arbitrary_block = next(iter(self.blocks.values())) self._min_io_size = arbitrary_block.ec_hdr.vid_hdr_offset self._leb_size = self.file.block_size - arbitrary_block.ec_hdr.data_offset - def _get_file(self): """UBI File object @@ -61,8 +60,8 @@ def _get_file(self): Obj -- UBI File object. """ return self._file - file = property(_get_file) + file = property(_get_file) def _get_block_count(self): """Total amount of UBI blocks in file. @@ -71,20 +70,21 @@ def _get_block_count(self): Int -- Number of blocks """ return self._block_count - block_count = property(_get_block_count) + block_count = property(_get_block_count) def _set_first_peb_num(self, i): self._first_peb_num = i + def _get_first_peb_num(self): """First Physical Erase Block with UBI data - + Returns: Int -- Number of the first PEB. """ return self._first_peb_num - first_peb_num = property(_get_first_peb_num, _set_first_peb_num) + first_peb_num = property(_get_first_peb_num, _set_first_peb_num) def _get_leb_size(self): """LEB size of UBI blocks in file. @@ -93,8 +93,8 @@ def _get_leb_size(self): Int -- LEB Size. """ return self._leb_size - leb_size = property(_get_leb_size) + leb_size = property(_get_leb_size) def _get_peb_size(self): """PEB size of UBI blocks in file. @@ -103,8 +103,8 @@ def _get_peb_size(self): Int -- PEB Size. """ return self.file.block_size - peb_size = property(_get_peb_size) + peb_size = property(_get_peb_size) def _get_min_io_size(self): """Min I/O Size @@ -113,8 +113,8 @@ def _get_min_io_size(self): Int -- Min I/O Size. """ return self._min_io_size - min_io_size = property(_get_min_io_size) + min_io_size = property(_get_min_io_size) def _get_blocks(self): """Main Dict of UBI Blocks @@ -124,6 +124,7 @@ def _get_blocks(self): as there can be thousands. """ return self._blocks + blocks = property(_get_blocks) @@ -151,11 +152,11 @@ def __init__(self, ubi_file): self._data_blocks_list = data_list self._int_vol_blocks_list = int_vol_list self._unknown_blocks_list = unknown_list - + newest_layout_list = rm_old_blocks(self.blocks, self.layout_blocks_list) - + if len(newest_layout_list) < 2: - error(self, 'Fatal', 'Less than 2 layout blocks found.') + error(self, "Fatal", "Less than 2 layout blocks found.") layout_pairs = layout.group_pairs(self.blocks, newest_layout_list) @@ -165,7 +166,6 @@ def __init__(self, ubi_file): for i in range(0, len(layout_infos)): self._images.append(image(self.blocks, layout_infos[i])) - def _get_images(self): """Get UBI images. @@ -173,8 +173,8 @@ def _get_images(self): List -- Of volume objects groupled by image. """ return self._images - images = property(_get_images) + images = property(_get_images) def _get_data_blocks_list(self): """Get all UBI blocks found in file that are data blocks. @@ -183,8 +183,8 @@ def _get_data_blocks_list(self): List -- List of block objects. """ return self._data_blocks_list - data_blocks_list = property(_get_data_blocks_list) + data_blocks_list = property(_get_data_blocks_list) def _get_layout_blocks_list(self): """Get all UBI blocks found in file that are layout volume blocks. @@ -193,8 +193,8 @@ def _get_layout_blocks_list(self): List -- List of block objects. """ return self._layout_blocks_list - layout_blocks_list = property(_get_layout_blocks_list) + layout_blocks_list = property(_get_layout_blocks_list) def _get_int_vol_blocks_list(self): """Get all UBI blocks found in file that are internal volume blocks. @@ -205,8 +205,8 @@ def _get_int_vol_blocks_list(self): This does not include layout blocks. """ return self._int_vol_blocks_list - int_vol_blocks_list = property(_get_int_vol_blocks_list) + int_vol_blocks_list = property(_get_int_vol_blocks_list) def _get_unknown_blocks_list(self): """Get all UBI blocks found in file of unknown type.. @@ -215,11 +215,12 @@ def _get_unknown_blocks_list(self): List -- List of block objects. """ return self._unknown_blocks_list + unknown_blocks_list = property(_get_unknown_blocks_list) - - def display(self, tab=''): + + def display(self, tab=""): """Print information about this object. - + Argument: Str:tab -- '\t' for spacing this object. """ diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index ce80f96..e1732f5 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -18,10 +18,17 @@ ############################################################# from zlib import crc32 + from ubireader import settings from ubireader.debug import error, log, verbose_display, verbose_log from ubireader.ubi import display -from ubireader.ubi.defines import UBI_EC_HDR_SZ, UBI_VID_HDR_SZ, UBI_INTERNAL_VOL_START, UBI_EC_HDR_MAGIC, UBI_CRC32_INIT +from ubireader.ubi.defines import ( + UBI_CRC32_INIT, + UBI_EC_HDR_MAGIC, + UBI_EC_HDR_SZ, + UBI_INTERNAL_VOL_START, + UBI_VID_HDR_SZ, +) from ubireader.ubi.headers import ec_hdr, vid_hdr, vtbl_recs @@ -49,7 +56,6 @@ class description(object): """ def __init__(self, block_buf): - self.file_offset = -1 self.peb_num = -1 self.leb_num = -1 @@ -63,29 +69,35 @@ def __init__(self, block_buf): self.ec_hdr = ec_hdr(block_buf[0:UBI_EC_HDR_SZ]) if not self.ec_hdr.errors or settings.ignore_block_header_errors: - self.vid_hdr = vid_hdr(block_buf[self.ec_hdr.vid_hdr_offset:self.ec_hdr.vid_hdr_offset+UBI_VID_HDR_SZ]) + self.vid_hdr = vid_hdr( + block_buf[ + self.ec_hdr.vid_hdr_offset : self.ec_hdr.vid_hdr_offset + + UBI_VID_HDR_SZ + ] + ) if not self.vid_hdr.errors or settings.ignore_block_header_errors: self.is_internal_vol = self.vid_hdr.vol_id >= UBI_INTERNAL_VOL_START - + if self.vid_hdr.vol_id >= UBI_INTERNAL_VOL_START: - self.vtbl_recs = vtbl_recs(block_buf[self.ec_hdr.data_offset:]) + self.vtbl_recs = vtbl_recs(block_buf[self.ec_hdr.data_offset :]) self.leb_num = self.vid_hdr.lnum self.is_vtbl = bool(self.vtbl_recs) or False - self.is_valid = not self.ec_hdr.errors and not self.vid_hdr.errors or settings.ignore_block_header_errors - + self.is_valid = ( + not self.ec_hdr.errors + and not self.vid_hdr.errors + or settings.ignore_block_header_errors + ) def __repr__(self): - return 'Block: PEB# %s: LEB# %s' % (self.peb_num, self.leb_num) - + return "Block: PEB# %s: LEB# %s" % (self.peb_num, self.leb_num) - def display(self, tab=''): + def display(self, tab=""): return display.block(self, tab) - def get_blocks_in_list(blocks, idx_list): """Retrieve block objects in list of indexes @@ -99,8 +111,7 @@ def get_blocks_in_list(blocks, idx_list): order of idx_list. """ - return {i:blocks[i] for i in idx_list} - + return {i: blocks[i] for i in idx_list} def extract_blocks(ubi): @@ -108,7 +119,7 @@ def extract_blocks(ubi): Arguments:. Obj:ubi -- UBI object. - + Returns: Dict -- Of block objects keyed by PEB number. """ @@ -128,30 +139,42 @@ def extract_blocks(ubi): blk.file_offset = i blk.peb_num = ubi.first_peb_num + peb_count blk.size = ubi.file.block_size - blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & UBI_CRC32_INIT) + blk.data_crc = ( + ~crc32( + buf[ + blk.ec_hdr.data_offset : blk.ec_hdr.data_offset + + blk.vid_hdr.data_size + ] + ) + & UBI_CRC32_INIT + ) blocks[blk.peb_num] = blk peb_count += 1 log(extract_blocks, blk) - verbose_log(extract_blocks, 'file addr: %s' % (ubi.file.last_read_addr())) - ec_hdr_errors = '' - vid_hdr_errors = '' + verbose_log(extract_blocks, "file addr: %s" % (ubi.file.last_read_addr())) + ec_hdr_errors = "" + vid_hdr_errors = "" if blk.ec_hdr.errors: - ec_hdr_errors = ','.join(blk.ec_hdr.errors) + ec_hdr_errors = ",".join(blk.ec_hdr.errors) if blk.vid_hdr and blk.vid_hdr.errors: - vid_hdr_errors = ','.join(blk.vid_hdr.errors) + vid_hdr_errors = ",".join(blk.vid_hdr.errors) if ec_hdr_errors or vid_hdr_errors: if blk.peb_num not in bad_blocks: bad_blocks.append(blk.peb_num) - log(extract_blocks, 'PEB: %s has possible issue EC_HDR [%s], VID_HDR [%s]' % (blk.peb_num, ec_hdr_errors, vid_hdr_errors)) + log( + extract_blocks, + "PEB: %s has possible issue EC_HDR [%s], VID_HDR [%s]" + % (blk.peb_num, ec_hdr_errors, vid_hdr_errors), + ) verbose_display(blk) else: cur_offset += ubi.file.block_size - ubi.first_peb_num = cur_offset//ubi.file.block_size + ubi.first_peb_num = cur_offset // ubi.file.block_size ubi.file.start_offset = cur_offset return blocks @@ -185,10 +208,10 @@ def rm_old_blocks(blocks, block_list): if blocks[i].ec_hdr.image_seq != blocks[k].ec_hdr.image_seq: continue - second_newer = blocks[k].vid_hdr.sqnum > blocks[i].vid_hdr.sqnum + second_newer = blocks[k].vid_hdr.sqnum > blocks[i].vid_hdr.sqnum del_block = None use_block = None - + if second_newer: if blocks[k].vid_hdr.copy_flag == 0: del_block = i @@ -201,7 +224,11 @@ def rm_old_blocks(blocks, block_list): if del_block is not None: del_blocks.append(del_block) - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s, Using PEB%s' % (blocks[del_block].peb_num, blocks[del_block].leb_num, use_block)) + log( + rm_old_blocks, + "Old block removed (copy_flag): PEB %s, LEB %s, Using PEB%s" + % (blocks[del_block].peb_num, blocks[del_block].leb_num, use_block), + ) break if second_newer: @@ -221,16 +248,28 @@ def rm_old_blocks(blocks, block_list): if del_block is not None: del_blocks.append(del_block) - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s, vid_hdr.data_crc %s / %s, Using PEB %s' % (blocks[del_block].peb_num, - blocks[del_block].leb_num, - blocks[del_block].vid_hdr.data_crc, - blocks[del_block].data_crc, - use_block)) + log( + rm_old_blocks, + """Old block removed (data_crc): PEB %s, LEB %s, + vid_hdr.data_crc %s / %s, Using PEB %s""" + % ( + blocks[del_block].peb_num, + blocks[del_block].leb_num, + blocks[del_block].vid_hdr.data_crc, + blocks[del_block].data_crc, + use_block, + ), + ) else: use_block = min(k, i) del_blocks.append(use_block) - error('Warn', rm_old_blocks, 'Multiple PEB [%s] for LEB %s: Using first.' % (', '.join(i, k), blocks[i].leb_num, use_block)) + error( + "Warn", + rm_old_blocks, + "Multiple PEB [%s] for LEB %s: Using first. (use block: %d)" + % (", ".join(i, k), blocks[i].leb_num, use_block), + ) break diff --git a/ubireader/ubi/block/layout.py b/ubireader/ubi/block/layout.py index 67388d9..3ee2c22 100755 --- a/ubireader/ubi/block/layout.py +++ b/ubireader/ubi/block/layout.py @@ -20,6 +20,7 @@ from ubireader.debug import log from ubireader.ubi.block import sort + def group_pairs(blocks, layout_blocks_list): """Sort a list of layout blocks into pairs @@ -31,15 +32,15 @@ def group_pairs(blocks, layout_blocks_list): List -- Layout block pair indexes grouped in a list """ - image_dict={} + image_dict = {} for block_id in layout_blocks_list: - image_seq=blocks[block_id].ec_hdr.image_seq + image_seq = blocks[block_id].ec_hdr.image_seq if image_seq not in image_dict: - image_dict[image_seq]=[block_id] + image_dict[image_seq] = [block_id] else: image_dict[image_seq].append(block_id) - log(group_pairs, 'Layout blocks found at PEBs: %s' % list(image_dict.values())) + log(group_pairs, "Layout blocks found at PEBs: %s" % list(image_dict.values())) return list(image_dict.values()) diff --git a/ubireader/ubi/block/sort.py b/ubireader/ubi/block/sort.py index e9e115b..e5f6785 100644 --- a/ubireader/ubi/block/sort.py +++ b/ubireader/ubi/block/sort.py @@ -19,6 +19,7 @@ from ubireader import settings + def by_image_seq(blocks, image_seq): """Filter blocks to return only those associated with the provided image_seq number. If uboot_fix is set, associate blocks with an image_seq of 0 also. @@ -26,32 +27,42 @@ def by_image_seq(blocks, image_seq): Argument: List:blocks -- List of block objects to sort. Int:image_seq -- image_seq number found in ec_hdr. - + Returns: List -- List of block indexes matching image_seq number. """ if settings.uboot_fix: - return list(filter(lambda block: blocks[block].ec_hdr.image_seq == image_seq or image_seq == 0 or blocks[block].ec_hdr.image_seq == 0, blocks)) + return list( + filter( + lambda block: blocks[block].ec_hdr.image_seq == image_seq + or image_seq == 0 + or blocks[block].ec_hdr.image_seq == 0, + blocks, + ) + ) else: - return list(filter(lambda block: blocks[block].ec_hdr.image_seq == image_seq, blocks)) + return list( + filter(lambda block: blocks[block].ec_hdr.image_seq == image_seq, blocks) + ) + def by_leb(blocks): """Sort blocks by Logical Erase Block number. - + Arguments: List:blocks -- List of block objects to sort. - + Returns: List -- Indexes of blocks sorted by LEB. - """ + """ slist_len = len(blocks) - slist = ['x'] * slist_len + slist = ["x"] * slist_len for block in blocks: if blocks[block].leb_num >= slist_len: add_elements = blocks[block].leb_num - slist_len + 1 - slist += (['x'] * add_elements) + slist += ["x"] * add_elements slist_len = len(slist) slist[blocks[block].leb_num] = block @@ -88,6 +99,7 @@ def by_vol_id(blocks, slist=None): return vol_blocks + def by_type(blocks, slist=None): """Sort blocks into layout, internal volume, data or unknown @@ -99,7 +111,7 @@ def by_type(blocks, slist=None): List:layout -- List of block indexes of blocks containing the volume table records. List:data -- List of block indexes containing filesystem data. - List:int_vol -- List of block indexes containing volume ids + List:int_vol -- List of block indexes containing volume ids greater than UBI_INTERNAL_VOL_START that are not layout volumes. List:unknown -- List of block indexes of blocks that failed validation @@ -110,7 +122,7 @@ def by_type(blocks, slist=None): data = [] int_vol = [] unknown = [] - + for i in blocks: if slist and i not in slist: continue diff --git a/ubireader/ubi/defines.py b/ubireader/ubi/defines.py index 9a26bc0..a2c0c72 100644 --- a/ubireader/ubi/defines.py +++ b/ubireader/ubi/defines.py @@ -35,10 +35,10 @@ import struct # Magic number -UBI_EC_HDR_MAGIC = b'\x55\x42\x49\x23' +UBI_EC_HDR_MAGIC = b"\x55\x42\x49\x23" # Initial CRC32 checksum value. -UBI_CRC32_INIT = 4294967295 #0xFFFFFFFF +UBI_CRC32_INIT = 4294967295 # 0xFFFFFFFF # Max number of volumes allowed. UBI_MAX_VOLUMES = 128 @@ -47,73 +47,79 @@ UBI_INTERNAL_VOL_START = 2147479551 # Error Count header. -EC_HDR_FORMAT = '>4sB3sQIII32sI' -EC_HDR_FIELDS = ['magic', # Magic string UBI# - 'version', # UBI version meant to accept this image. - 'padding', # Reserved for future, zeros. - 'ec', # Erase counter - 'vid_hdr_offset', # Where the VID header starts. - 'data_offset', # Where user data starts. - 'image_seq', # Image sequence number - 'padding2', # Reserved for future, zeros. - 'hdr_crc'] # EC header crc32 checksum. - - -UBI_EC_HDR_SZ = struct.calcsize(EC_HDR_FORMAT) # 64 +EC_HDR_FORMAT = ">4sB3sQIII32sI" +EC_HDR_FIELDS = [ + "magic", # Magic string UBI# + "version", # UBI version meant to accept this image. + "padding", # Reserved for future, zeros. + "ec", # Erase counter + "vid_hdr_offset", # Where the VID header starts. + "data_offset", # Where user data starts. + "image_seq", # Image sequence number + "padding2", # Reserved for future, zeros. + "hdr_crc", +] # EC header crc32 checksum. + + +UBI_EC_HDR_SZ = struct.calcsize(EC_HDR_FORMAT) # 64 # Volume ID header. -UBI_VID_HDR_MAGIC =b'\x55\x42\x49\x21' # UBI! -VID_HDR_FORMAT = '>4sBBBBII4sIIII4sQ12sI' -VID_HDR_FIELDS = ['magic', # Magic string UBI! - 'version', # UBI version meant to accept this image. - 'vol_type', # Volume type, Dynamic/Static - 'copy_flag', # If this is a copied PEB b/c of wear leveling. - 'compat', # Compatibility of this volume UBI_COMPAT_* - 'vol_id', # ID of this volume. - 'lnum', # LEB number. - 'padding', # Reserved for future, zeros. - 'data_size', # How many bytes of data this contains. - # Used for static types only. - 'used_ebs', # Total num of used LEBs in this volume. - 'data_pad', # How many bytes at end of LEB are not used. - 'data_crc', # CRC32 checksum of data, static type only. - 'padding2', # Reserved for future, zeros. - 'sqnum', # Sequence number. - 'padding3', # Reserved for future, zeros. - 'hdr_crc'] # VID header CRC32 checksum. - - -UBI_VID_HDR_SZ = struct.calcsize(VID_HDR_FORMAT) # 64 +UBI_VID_HDR_MAGIC = b"\x55\x42\x49\x21" # UBI! +VID_HDR_FORMAT = ">4sBBBBII4sIIII4sQ12sI" +VID_HDR_FIELDS = [ + "magic", # Magic string UBI! + "version", # UBI version meant to accept this image. + "vol_type", # Volume type, Dynamic/Static + "copy_flag", # If this is a copied PEB b/c of wear leveling. + "compat", # Compatibility of this volume UBI_COMPAT_* + "vol_id", # ID of this volume. + "lnum", # LEB number. + "padding", # Reserved for future, zeros. + "data_size", # How many bytes of data this contains. + # Used for static types only. + "used_ebs", # Total num of used LEBs in this volume. + "data_pad", # How many bytes at end of LEB are not used. + "data_crc", # CRC32 checksum of data, static type only. + "padding2", # Reserved for future, zeros. + "sqnum", # Sequence number. + "padding3", # Reserved for future, zeros. + "hdr_crc", +] # VID header CRC32 checksum. + + +UBI_VID_HDR_SZ = struct.calcsize(VID_HDR_FORMAT) # 64 # Volume table records. -VTBL_REC_FORMAT = '>IIIBBH128sB23sI' -VTBL_REC_FIELDS = ['reserved_pebs', # How many PEBs reserved for this volume. - 'alignment', # Volume alignment. - 'data_pad', # Number of unused bytes at end of PEB. - 'vol_type', # Volume type, static/dynamic. - 'upd_marker', # If vol update started but not finished. - 'name_len', # Length of name. - 'name', # Volume name. - 'flags', # Volume flags - 'padding', # Reserved for future, zeros. - 'crc'] # Vol record CRC32 checksum. - - -UBI_VTBL_REC_SZ = struct.calcsize(VTBL_REC_FORMAT) # 172 +VTBL_REC_FORMAT = ">IIIBBH128sB23sI" +VTBL_REC_FIELDS = [ + "reserved_pebs", # How many PEBs reserved for this volume. + "alignment", # Volume alignment. + "data_pad", # Number of unused bytes at end of PEB. + "vol_type", # Volume type, static/dynamic. + "upd_marker", # If vol update started but not finished. + "name_len", # Length of name. + "name", # Volume name. + "flags", # Volume flags + "padding", # Reserved for future, zeros. + "crc", +] # Vol record CRC32 checksum. + + +UBI_VTBL_REC_SZ = struct.calcsize(VTBL_REC_FORMAT) # 172 # Volume Identifier Header -UBI_VID_DYNAMIC = 1 # Volume can be resized. -UBI_VID_STATIC = 2 # Volume can not be resized. -PRINT_VOL_TYPE_LIST = [0, 'dynamic', 'static'] +UBI_VID_DYNAMIC = 1 # Volume can be resized. +UBI_VID_STATIC = 2 # Volume can not be resized. +PRINT_VOL_TYPE_LIST = [0, "dynamic", "static"] # Volume table record UBI_VTBL_AUTORESIZE_FLG = 1 -UBI_COMPAT_DELETE = 1 # Delete this internal volume before anything written. -UBI_COMPAT_RO = 2 # Attach this device in read-only mode. -UBI_COMPAT_PRESERVE = 4 # Preserve this internal volume - touch nothing. -UBI_COMPAT_REJECT = 5 # Reject this UBI image -PRINT_COMPAT_LIST = [0, 'Delete', 'Read Only', 0, 'Preserve', 'Reject'] +UBI_COMPAT_DELETE = 1 # Delete this internal volume before anything written. +UBI_COMPAT_RO = 2 # Attach this device in read-only mode. +UBI_COMPAT_PRESERVE = 4 # Preserve this internal volume - touch nothing. +UBI_COMPAT_REJECT = 5 # Reject this UBI image +PRINT_COMPAT_LIST = [0, "Delete", "Read Only", 0, "Preserve", "Reject"] # File chunk size for reads. -FILE_CHUNK_SZ = 5 * 1024 * 1024 \ No newline at end of file +FILE_CHUNK_SZ = 5 * 1024 * 1024 diff --git a/ubireader/ubi/display.py b/ubireader/ubi/display.py index 9efbc6a..87faa35 100755 --- a/ubireader/ubi/display.py +++ b/ubireader/ubi/display.py @@ -18,144 +18,151 @@ ############################################################# from ubireader import settings -from ubireader.ubi.defines import PRINT_COMPAT_LIST, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG - -def ubi(ubi, tab=''): - buf = '%sUBI File\n' % (tab) - buf += '%s---------------------\n' % (tab) - buf += '\t%sMin I/O: %s\n' % (tab, ubi.min_io_size) - buf += '\t%sLEB Size: %s\n' % (tab, ubi.leb_size) - buf += '\t%sPEB Size: %s\n' % (tab, ubi.peb_size) - buf += '\t%sTotal Block Count: %s\n' % (tab, ubi.block_count) - buf += '\t%sData Block Count: %s\n' % (tab, len(ubi.data_blocks_list)) - buf += '\t%sLayout Block Count: %s\n' % (tab, len(ubi.layout_blocks_list)) - buf += '\t%sInternal Volume Block Count: %s\n' % (tab, len(ubi.int_vol_blocks_list)) - buf += '\t%sUnknown Block Count: %s\n' % (tab, len(ubi.unknown_blocks_list)) - buf += '\t%sFirst UBI PEB Number: %s\n' % (tab, ubi.first_peb_num) +from ubireader.ubi.defines import ( + PRINT_COMPAT_LIST, + PRINT_VOL_TYPE_LIST, + UBI_VTBL_AUTORESIZE_FLG, +) + + +def ubi(ubi, tab=""): + buf = "%sUBI File\n" % (tab) + buf += "%s---------------------\n" % (tab) + buf += "\t%sMin I/O: %s\n" % (tab, ubi.min_io_size) + buf += "\t%sLEB Size: %s\n" % (tab, ubi.leb_size) + buf += "\t%sPEB Size: %s\n" % (tab, ubi.peb_size) + buf += "\t%sTotal Block Count: %s\n" % (tab, ubi.block_count) + buf += "\t%sData Block Count: %s\n" % (tab, len(ubi.data_blocks_list)) + buf += "\t%sLayout Block Count: %s\n" % (tab, len(ubi.layout_blocks_list)) + buf += "\t%sInternal Volume Block Count: %s\n" % (tab, len(ubi.int_vol_blocks_list)) + buf += "\t%sUnknown Block Count: %s\n" % (tab, len(ubi.unknown_blocks_list)) + buf += "\t%sFirst UBI PEB Number: %s\n" % (tab, ubi.first_peb_num) return buf -def image(image, tab=''): - buf = '%s%s\n' % (tab, image) - buf += '%s---------------------\n' % (tab) - buf += '\t%sImage Sequence Num: %s\n' % (tab, image.image_seq) +def image(image, tab=""): + buf = "%s%s\n" % (tab, image) + buf += "%s---------------------\n" % (tab) + buf += "\t%sImage Sequence Num: %s\n" % (tab, image.image_seq) for volume in image.volumes: - buf += '\t%sVolume Name:%s\n' % (tab, volume) - buf += '\t%sPEB Range: %s - %s\n' % (tab, image.peb_range[0], image.peb_range[1]) + buf += "\t%sVolume Name:%s\n" % (tab, volume) + buf += "\t%sPEB Range: %s - %s\n" % (tab, image.peb_range[0], image.peb_range[1]) return buf -def volume(volume, tab=''): - buf = '%s%s\n' % (tab, volume) - buf += '%s---------------------\n' % (tab) - buf += '\t%sVol ID: %s\n' % (tab, volume.vol_id) - buf += '\t%sName: %s\n' % (tab, volume.name.decode('utf-8')) - buf += '\t%sBlock Count: %s\n' % (tab, volume.block_count) +def volume(volume, tab=""): + buf = "%s%s\n" % (tab, volume) + buf += "%s---------------------\n" % (tab) + buf += "\t%sVol ID: %s\n" % (tab, volume.vol_id) + buf += "\t%sName: %s\n" % (tab, volume.name.decode("utf-8")) + buf += "\t%sBlock Count: %s\n" % (tab, volume.block_count) - buf += '\n' - buf += '\t%sVolume Record\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += vol_rec(volume.vol_rec, '\t\t%s' % tab) + buf += "\n" + buf += "\t%sVolume Record\n" % (tab) + buf += "\t%s---------------------\n" % (tab) + buf += vol_rec(volume.vol_rec, "\t\t%s" % tab) - buf += '\n' + buf += "\n" return buf -def block(block, tab='\t'): - buf = '%s%s\n' % (tab, block) - buf += '%s---------------------\n' % (tab) - buf += '\t%sFile Offset: %s\n' % (tab, block.file_offset) - buf += '\t%sPEB #: %s\n' % (tab, block.peb_num) - buf += '\t%sLEB #: %s\n' % (tab, block.leb_num) - buf += '\t%sBlock Size: %s\n' % (tab, block.size) - buf += '\t%sInternal Volume: %s\n' % (tab, block.is_internal_vol) - buf += '\t%sIs Volume Table: %s\n' % (tab, block.is_vtbl) - buf += '\t%sIs Valid: %s\n' % (tab, block.is_valid) +def block(block, tab="\t"): + buf = "%s%s\n" % (tab, block) + buf += "%s---------------------\n" % (tab) + buf += "\t%sFile Offset: %s\n" % (tab, block.file_offset) + buf += "\t%sPEB #: %s\n" % (tab, block.peb_num) + buf += "\t%sLEB #: %s\n" % (tab, block.leb_num) + buf += "\t%sBlock Size: %s\n" % (tab, block.size) + buf += "\t%sInternal Volume: %s\n" % (tab, block.is_internal_vol) + buf += "\t%sIs Volume Table: %s\n" % (tab, block.is_vtbl) + buf += "\t%sIs Valid: %s\n" % (tab, block.is_valid) if not block.ec_hdr.errors or settings.ignore_block_header_errors: - buf += '\n' - buf += '\t%sErase Count Header\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += ec_hdr(block.ec_hdr, '\t\t%s' % tab) - - if (block.vid_hdr and not block.vid_hdr.errors) or settings.ignore_block_header_errors: - buf += '\n' - buf += '\t%sVID Header\n' % (tab) - buf += '\t%s---------------------\n' % (tab) - buf += vid_hdr(block.vid_hdr, '\t\t%s' % tab) + buf += "\n" + buf += "\t%sErase Count Header\n" % (tab) + buf += "\t%s---------------------\n" % (tab) + buf += ec_hdr(block.ec_hdr, "\t\t%s" % tab) + + if ( + block.vid_hdr and not block.vid_hdr.errors + ) or settings.ignore_block_header_errors: + buf += "\n" + buf += "\t%sVID Header\n" % (tab) + buf += "\t%s---------------------\n" % (tab) + buf += vid_hdr(block.vid_hdr, "\t\t%s" % tab) if block.vtbl_recs: - buf += '\n' - buf += '\t%sVolume Records\n' % (tab) - buf += '\t%s---------------------\n' % (tab) + buf += "\n" + buf += "\t%sVolume Records\n" % (tab) + buf += "\t%s---------------------\n" % (tab) for vol in block.vtbl_recs: - buf += vol_rec(vol, '\t\t%s' % tab) + buf += vol_rec(vol, "\t\t%s" % tab) - buf += '\n' + buf += "\n" return buf -def ec_hdr(ec_hdr, tab=''): - buf = '' +def ec_hdr(ec_hdr, tab=""): + buf = "" for key, value in ec_hdr: - if key == 'errors': - value = ','.join(value) + if key == "errors": + value = ",".join(value) - elif key == 'hdr_crc': + elif key == "hdr_crc": value = hex(value) - buf += '%s%s: %r\n' % (tab, key, value) + buf += "%s%s: %r\n" % (tab, key, value) return buf -def vid_hdr(vid_hdr, tab=''): - buf = '' +def vid_hdr(vid_hdr, tab=""): + buf = "" for key, value in vid_hdr: - if key == 'errors': - value = ','.join(value) + if key == "errors": + value = ",".join(value) - elif key == 'hdr_crc': + elif key == "hdr_crc": value = hex(value) - elif key == 'compat': + elif key == "compat": if value in PRINT_COMPAT_LIST: value = PRINT_COMPAT_LIST[value] else: value = -1 - elif key == 'vol_type': - if value < len(PRINT_VOL_TYPE_LIST): + elif key == "vol_type": + if value < len(PRINT_VOL_TYPE_LIST): value = PRINT_VOL_TYPE_LIST[value] else: value = -1 - buf += '%s%s: %r\n' % (tab, key, value) + buf += "%s%s: %r\n" % (tab, key, value) return buf -def vol_rec(vol_rec, tab=''): - buf = '' +def vol_rec(vol_rec, tab=""): + buf = "" for key, value in vol_rec: - if key == 'errors': - value = ','.join(value) + if key == "errors": + value = ",".join(value) - elif key == 'crc': + elif key == "crc": value = hex(value) - elif key == 'vol_type': + elif key == "vol_type": if value < len(PRINT_VOL_TYPE_LIST): value = PRINT_VOL_TYPE_LIST[value] else: value = -1 - elif key == 'flags' and value == UBI_VTBL_AUTORESIZE_FLG: - value = 'autoresize' + elif key == "flags" and value == UBI_VTBL_AUTORESIZE_FLG: + value = "autoresize" - elif key == 'name': - value = value.strip(b'\x00').decode('utf-8') + elif key == "name": + value = value.strip(b"\x00").decode("utf-8") - elif key == 'padding': - value = value.decode('utf-8') + elif key == "padding": + value = value.decode("utf-8") - buf += '%s%s: %r\n' % (tab, key, value) + buf += "%s%s: %r\n" % (tab, key, value) return buf diff --git a/ubireader/ubi/headers.py b/ubireader/ubi/headers.py index 5fa3905..a628e5b 100755 --- a/ubireader/ubi/headers.py +++ b/ubireader/ubi/headers.py @@ -21,65 +21,76 @@ from zlib import crc32 from ubireader.debug import log -from ubireader.ubi.defines import * +from ubireader.ubi.defines import ( + EC_HDR_FIELDS, + EC_HDR_FORMAT, + UBI_CRC32_INIT, + VID_HDR_FIELDS, + VID_HDR_FORMAT, + UBI_VTBL_REC_SZ, + VTBL_REC_FIELDS, + VTBL_REC_FORMAT, + UBI_MAX_VOLUMES, +) + class ec_hdr(object): def __init__(self, buf): - fields = dict(list(zip(EC_HDR_FIELDS, struct.unpack(EC_HDR_FORMAT,buf)))) + fields = dict(list(zip(EC_HDR_FIELDS, struct.unpack(EC_HDR_FORMAT, buf)))) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) + setattr(self, "errors", []) self._check_errors(buf[:-4]) - + def __repr__(self): - return 'Erase Count Header' + return "Erase Count Header" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) def _check_errors(self, buf_crc): - crc_chk = (~crc32(buf_crc) & UBI_CRC32_INIT) + crc_chk = ~crc32(buf_crc) & UBI_CRC32_INIT if self.hdr_crc != crc_chk: - log(vid_hdr, 'CRC Failed: expected 0x%x got 0x%x' % (crc_chk, self.hdr_crc)) - self.errors.append('crc') + log(vid_hdr, "CRC Failed: expected 0x%x got 0x%x" % (crc_chk, self.hdr_crc)) + self.errors.append("crc") class vid_hdr(object): def __init__(self, buf): - fields = dict(list(zip(VID_HDR_FIELDS, struct.unpack(VID_HDR_FORMAT,buf)))) + fields = dict(list(zip(VID_HDR_FIELDS, struct.unpack(VID_HDR_FORMAT, buf)))) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) + setattr(self, "errors", []) self._check_errors(buf[:-4]) def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) def __repr__(self): - return 'VID Header' + return "VID Header" def _check_errors(self, buf_crc): - crc_chk = (~crc32(buf_crc) & UBI_CRC32_INIT) + crc_chk = ~crc32(buf_crc) & UBI_CRC32_INIT if self.hdr_crc != crc_chk: - log(vid_hdr, 'CRC Failed: expected 0x%x got 0x%x' % (crc_chk, self.hdr_crc)) - self.errors.append('crc') + log(vid_hdr, "CRC Failed: expected 0x%x got 0x%x" % (crc_chk, self.hdr_crc)) + self.errors.append("crc") def vtbl_recs(buf): data_buf = buf vtbl_recs = [] - vtbl_rec_ret = '' + vtbl_rec_ret = "" + + for i in range(0, UBI_MAX_VOLUMES): + offset = i * UBI_VTBL_REC_SZ + vtbl_rec_buf = data_buf[offset : offset + UBI_VTBL_REC_SZ] - for i in range(0, UBI_MAX_VOLUMES): - offset = i*UBI_VTBL_REC_SZ - vtbl_rec_buf = data_buf[offset:offset+UBI_VTBL_REC_SZ] - if len(vtbl_rec_buf) == UBI_VTBL_REC_SZ: vtbl_rec_ret = _vtbl_rec(vtbl_rec_buf) @@ -92,23 +103,23 @@ def vtbl_recs(buf): class _vtbl_rec(object): def __init__(self, buf): - fields = dict(list(zip(VTBL_REC_FIELDS, struct.unpack(VTBL_REC_FORMAT,buf)))) + fields = dict(list(zip(VTBL_REC_FIELDS, struct.unpack(VTBL_REC_FORMAT, buf)))) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) - setattr(self, 'rec_index', -1) + setattr(self, "errors", []) + setattr(self, "rec_index", -1) - self.name = self.name[0: self.name_len] + self.name = self.name[0 : self.name_len] self._check_errors(buf[:-4]) def __repr__(self): - return 'Volume Table Record: %s' % getattr(self, 'name') + return "Volume Table Record: %s" % getattr(self, "name") def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) def _check_errors(self, buf_crc): if self.crc != (~crc32(buf_crc) & 0xFFFFFFFF): - self.errors.append('crc') + self.errors.append("crc") diff --git a/ubireader/ubi/image.py b/ubireader/ubi/image.py index f634b28..bc82023 100755 --- a/ubireader/ubi/image.py +++ b/ubireader/ubi/image.py @@ -19,8 +19,9 @@ from ubireader.debug import log from ubireader.ubi import display -from ubireader.ubi.volume import get_volumes from ubireader.ubi.block import get_blocks_in_list +from ubireader.ubi.volume import get_volumes + class description(object): def __init__(self, blocks, layout_info): @@ -31,30 +32,31 @@ def __init__(self, blocks, layout_info): self._start_peb = min(layout_info[2]) self._end_peb = max(layout_info[2]) self._volumes = get_volumes(blocks, layout_info) - log(description, 'Created Image: %s, Volume Cnt: %s' % (self.image_seq, len(self.volumes))) + log( + description, + "Created Image: %s, Volume Cnt: %s" % (self.image_seq, len(self.volumes)), + ) def __repr__(self): - return 'Image: %s' % (self.image_seq) - + return "Image: %s" % (self.image_seq) def get_blocks(self, blocks): return get_blocks_in_list(blocks, self._block_list) - def _get_peb_range(self): return [self._start_peb, self._end_peb] - peb_range = property(_get_peb_range) + peb_range = property(_get_peb_range) def _get_image_seq(self): return self._image_seq - image_seq = property(_get_image_seq) + image_seq = property(_get_image_seq) def _get_volumes(self): return self._volumes - volumes = property(_get_volumes) + volumes = property(_get_volumes) - def display(self, tab=''): + def display(self, tab=""): return display.image(self, tab) diff --git a/ubireader/ubi/volume.py b/ubireader/ubi/volume.py index f02834b..ee74e66 100755 --- a/ubireader/ubi/volume.py +++ b/ubireader/ubi/volume.py @@ -19,7 +19,8 @@ from ubireader.debug import log from ubireader.ubi import display -from ubireader.ubi.block import sort, get_blocks_in_list, rm_old_blocks +from ubireader.ubi.block import get_blocks_in_list, rm_old_blocks, sort + class description(object): """UBI Volume object @@ -39,57 +40,58 @@ class description(object): Volume object is basically a list of block indexes and some metadata describing a volume found in a UBI image. """ + def __init__(self, vol_id, vol_rec, block_list): self._vol_id = vol_id self._vol_rec = vol_rec self._name = self._vol_rec.name self._block_list = block_list - log(description, 'Create Volume: %s, ID: %s, Block Cnt: %s' % (self.name, self.vol_id, len(self.block_list))) - + log( + description, + "Create Volume: %s, ID: %s, Block Cnt: %s" + % (self.name, self.vol_id, len(self.block_list)), + ) def __repr__(self): - return 'Volume: %s' % (self.name.decode('utf-8')) - + return "Volume: %s" % (self.name.decode("utf-8")) def _get_name(self): return self._name - name = property(_get_name) + name = property(_get_name) def _get_vol_id(self): return self._vol_id - vol_id = property(_get_vol_id) + vol_id = property(_get_vol_id) def _get_block_count(self): return len(self._block_list) - block_count = property(_get_block_count) + block_count = property(_get_block_count) def _get_vol_rec(self): return self._vol_rec + vol_rec = property(_get_vol_rec) - def _get_block_list(self): return self._block_list - block_list = property(_get_block_list) + block_list = property(_get_block_list) def get_blocks(self, blocks): return get_blocks_in_list(blocks, self._block_list) - - def display(self, tab=''): + def display(self, tab=""): return display.volume(self, tab) - def reader(self, ubi): last_leb = 0 for block in sort.by_leb(self.get_blocks(ubi.blocks)): - if block == 'x': + if block == "x": last_leb += 1 - yield b'\xff'*ubi.leb_size + yield b"\xff" * ubi.leb_size else: last_leb += 1 yield ubi.file.read_block_data(ubi.blocks[block]) @@ -111,12 +113,15 @@ def get_volumes(blocks, layout_info): vol_blocks_lists = sort.by_vol_id(blocks, layout_info[2]) for vol_rec in blocks[layout_info[0]].vtbl_recs: - vol_name = vol_rec.name.strip(b'\x00').decode('utf-8') + vol_name = vol_rec.name.strip(b"\x00").decode("utf-8") if vol_rec.rec_index not in vol_blocks_lists: vol_blocks_lists[vol_rec.rec_index] = [] - vol_blocks_lists[vol_rec.rec_index] = rm_old_blocks(blocks, vol_blocks_lists[vol_rec.rec_index]) - volumes[vol_name] = description(vol_rec.rec_index, vol_rec, vol_blocks_lists[vol_rec.rec_index]) - - return volumes + vol_blocks_lists[vol_rec.rec_index] = rm_old_blocks( + blocks, vol_blocks_lists[vol_rec.rec_index] + ) + volumes[vol_name] = description( + vol_rec.rec_index, vol_rec, vol_blocks_lists[vol_rec.rec_index] + ) + return volumes diff --git a/ubireader/ubi_io.py b/ubireader/ubi_io.py index 2a03c68..fcab29f 100755 --- a/ubireader/ubi_io.py +++ b/ubireader/ubi_io.py @@ -20,6 +20,7 @@ from ubireader.debug import error, log, verbose_log from ubireader.ubi.block import sort + class ubi_file(object): """UBI image file object @@ -29,7 +30,7 @@ class ubi_file(object): Int:start_offset -- (optional) Where to start looking in the file for UBI data. Int:end_offset -- (optional) Where to stop looking in the file. - + Methods: seek -- Put file head to specified byte offset. Int:offset @@ -50,90 +51,90 @@ class ubi_file(object): """ def __init__(self, path, block_size, start_offset=0, end_offset=None): - self.__name__ = 'UBI_File' + self.__name__ = "UBI_File" self.is_valid = False try: - log(self, 'Open Path: %s' % path) - self._fhandle = open(path, 'rb') + log(self, "Open Path: %s" % path) + self._fhandle = open(path, "rb") except Exception as e: - error(self, 'Fatal', 'Open file: %s' % e) + error(self, "Fatal", "Open file: %s" % e) - self._fhandle.seek(0,2) + self._fhandle.seek(0, 2) file_size = self.tell() - log(self, 'File Size: %s' % file_size) + log(self, "File Size: %s" % file_size) self._start_offset = start_offset - log(self, 'Start Offset: %s' % (self._start_offset)) + log(self, "Start Offset: %s" % (self._start_offset)) if end_offset: self._end_offset = end_offset else: self._end_offset = file_size - log(self, 'End Offset: %s' % (self._end_offset)) + log(self, "End Offset: %s" % (self._end_offset)) self._block_size = block_size - log(self, 'Block Size: %s' % block_size) + log(self, "Block Size: %s" % block_size) if start_offset > self._end_offset: - error(self, 'Fatal', 'Start offset larger than end offset.') + error(self, "Fatal", "Start offset larger than end offset.") - if ( not end_offset is None ) and ( end_offset > file_size ): - error(self, 'Fatal', 'End offset larger than file size.') + if (end_offset is not None) and (end_offset > file_size): + error(self, "Fatal", "End offset larger than file size.") remainder = (self._end_offset - start_offset) % block_size if remainder != 0: - error(self, 'Warning', 'end_offset - start_offset length is not block aligned, could mean missing data.') + error( + self, + "Warning", + """end_offset - start_offset length is not block aligned, + could mean missing data.""", + ) self._fhandle.seek(self._start_offset) self._last_read_addr = self._fhandle.tell() self.is_valid = True - def _set_start(self, i): self._start_offset = i + def _get_start(self): return self._start_offset - start_offset = property(_get_start, _set_start) + start_offset = property(_get_start, _set_start) def _get_end(self): return self._end_offset - end_offset = property(_get_end) + end_offset = property(_get_end) def _get_block_size(self): return self._block_size - block_size = property(_get_block_size) + block_size = property(_get_block_size) def seek(self, offset): self._fhandle.seek(offset) - def read(self, size): self._last_read_addr = self.tell() - verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size)) + verbose_log(self, "read loc: %s, size: %s" % (self._last_read_addr, size)) return self._fhandle.read(size) - def tell(self): return self._fhandle.tell() - def last_read_addr(self): return self._last_read_addr - def reset(self): self._fhandle.seek(self.start_offset) - def reader(self): self.reset() while True: cur_loc = self._fhandle.tell() if self.end_offset and cur_loc > self.end_offset: - break + break elif self.end_offset and self.end_offset - cur_loc < self.block_size: chunk_size = self.end_offset - cur_loc else: @@ -145,100 +146,104 @@ def reader(self): break yield buf - def read_block(self, block): """Read complete PEB data from file. - + Argument: Obj:block -- Block data is desired for. """ self.seek(block.file_offset) return self._fhandle.read(block.size) - def read_block_data(self, block): """Read LEB data from file - + Argument: Obj:block -- Block data is desired for. """ self.seek(block.file_offset + block.ec_hdr.data_offset) - buf = self._fhandle.read(block.size - block.ec_hdr.data_offset - block.vid_hdr.data_pad) + buf = self._fhandle.read( + block.size - block.ec_hdr.data_offset - block.vid_hdr.data_pad + ) return buf - -class leb_virtual_file(): +class leb_virtual_file: def __init__(self, ubi, block_list): - self.__name__ = 'leb_virtual_file' + self.__name__ = "leb_virtual_file" self.is_valid = False self._ubi = ubi self._last_read_addr = 0 if not len(block_list): - error(self, 'Info', 'Empty block list') + error(self, "Info", "Empty block list") else: self._blocks = sort.by_leb(block_list) self._seek = 0 self._last_leb = -1 - self._last_buf = '' + self._last_buf = "" self.is_valid = True - def read(self, size): - buf = '' + buf = "" leb = int(self.tell() / self._ubi.leb_size) offset = self.tell() % self._ubi.leb_size try: if size < 0: - raise Exception('Bad Read Offset Request') + raise Exception("Bad Read Offset Request") - self._last_read_addr = self._ubi.blocks[self._blocks[leb]].file_offset + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + offset + self._last_read_addr = ( + self._ubi.blocks[self._blocks[leb]].file_offset + + self._ubi.blocks[self._blocks[leb]].ec_hdr.data_offset + + offset + ) - except Exception as e: - error(self.read, 'Error', 'LEB: %s is corrupted or has no data.' % (leb)) - raise Exception('Bad Read Offset Request') + except Exception: + error(self.read, "Error", "LEB: %s is corrupted or has no data." % (leb)) + raise Exception("Bad Read Offset Request") - verbose_log(self, 'read loc: %s, size: %s' % (self._last_read_addr, size)) + verbose_log(self, "read loc: %s, size: %s" % (self._last_read_addr, size)) if leb == self._last_leb: self.seek(self.tell() + size) - return self._last_buf[offset:offset+size] + return self._last_buf[offset : offset + size] else: try: - buf = self._ubi.file.read_block_data(self._ubi.blocks[self._blocks[leb]]) + buf = self._ubi.file.read_block_data( + self._ubi.blocks[self._blocks[leb]] + ) self._last_buf = buf self._last_leb = leb self.seek(self.tell() + size) - return buf[offset:offset+size] + return buf[offset : offset + size] except Exception as e: - error(self, 'Fatal', 'read loc: %s, size: %s, LEB: %s, offset: %s, error: %s' % (self._last_read_addr, size, leb, offset, e)) - + error( + self, + "Fatal", + "read loc: %s, size: %s, LEB: %s, offset: %s, error: %s" + % (self._last_read_addr, size, leb, offset, e), + ) def reset(self): self.seek(0) - def seek(self, offset): self._seek = offset - def tell(self): return self._seek - def last_read_addr(self): """Start address of last physical file read""" return self._last_read_addr - def reader(self): last_leb = 0 for block in self._blocks: while 0 != (self._ubi.blocks[block].leb_num - last_leb): last_leb += 1 - yield b'\xff'*self._ubi.leb_size + yield b"\xff" * self._ubi.leb_size last_leb += 1 yield self._ubi.file.read_block_data(self._ubi.blocks[block]) diff --git a/ubireader/ubifs/__init__.py b/ubireader/ubifs/__init__.py index 3e2bbca..f1a745b 100755 --- a/ubireader/ubifs/__init__.py +++ b/ubireader/ubifs/__init__.py @@ -18,15 +18,23 @@ ############################################################# from ubireader.debug import error, log, verbose_display -from ubireader.ubifs.defines import * -from ubireader.ubifs import nodes, display - -class ubifs(): +from ubireader.ubifs import display, nodes +from ubireader.ubifs.defines import ( + UBIFS_COMMON_HDR_SZ, + UBIFS_SB_NODE, + UBIFS_SB_NODE_SZ, + UBIFS_MST_LNUM, + UBIFS_MST_NODE, + UBIFS_MST_NODE_SZ, +) + + +class ubifs: """UBIFS object Arguments: - Str:path -- File path to UBIFS image. - + Str:path -- File path to UBIFS image. + Attributes: Obj:file -- File object Int:leb_size -- Size of Logical Erase Blocks. @@ -35,13 +43,14 @@ class ubifs(): Obj:mst_node -- Master Node of UBIFS image LEB1 Obj:mst_node2 -- Master Node 2 of UBIFS image LEB2 """ + def __init__(self, ubifs_file): - self.__name__ = 'UBIFS' + self.__name__ = "UBIFS" self._file = ubifs_file try: self.file.reset() sb_chdr = nodes.common_hdr(self.file.read(UBIFS_COMMON_HDR_SZ)) - log(self , '%s file addr: %s' % (sb_chdr, self.file.last_read_addr())) + log(self, "%s file addr: %s" % (sb_chdr, self.file.last_read_addr())) verbose_display(sb_chdr) if sb_chdr.node_type == UBIFS_SB_NODE: @@ -49,57 +58,63 @@ def __init__(self, ubifs_file): buf = self.file.read(UBIFS_SB_NODE_SZ) self._sb_node = nodes.sb_node(buf, self.file.last_read_addr()) self._min_io_size = self._sb_node.min_io_size - self._leb_size = self._sb_node.leb_size - log(self , '%s file addr: %s' % (self._sb_node, self.file.last_read_addr())) + self._leb_size = self._sb_node.leb_size + log( + self, + "%s file addr: %s" % (self._sb_node, self.file.last_read_addr()), + ) verbose_display(self._sb_node) else: - raise Exception('Wrong node type.') + raise Exception("Wrong node type.") except Exception as e: - error(self, 'Fatal', 'Super block error: %s' % e) + error(self, "Fatal", "Super block error: %s" % e) self._mst_nodes = [None, None] for i in range(0, 2): try: - mst_offset = self.leb_size * (UBIFS_MST_LNUM + i) + mst_offset = self.leb_size * (UBIFS_MST_LNUM + i) self.file.seek(mst_offset) mst_chdr = nodes.common_hdr(self.file.read(UBIFS_COMMON_HDR_SZ)) - log(self , '%s file addr: %s' % (mst_chdr, self.file.last_read_addr())) + log(self, "%s file addr: %s" % (mst_chdr, self.file.last_read_addr())) verbose_display(mst_chdr) if mst_chdr.node_type == UBIFS_MST_NODE: self.file.seek(mst_offset + UBIFS_COMMON_HDR_SZ) buf = self.file.read(UBIFS_MST_NODE_SZ) self._mst_nodes[i] = nodes.mst_node(buf, self.file.last_read_addr()) - log(self , '%s%s file addr: %s' % (self._mst_nodes[i], i, self.file.last_read_addr())) + log( + self, + "%s%s file addr: %s" + % (self._mst_nodes[i], i, self.file.last_read_addr()), + ) verbose_display(self._mst_nodes[i]) else: - raise Exception('Wrong node type.') + raise Exception("Wrong node type.") except Exception as e: - error(self, 'Warn', 'Master block %s error: %s' % (i, e)) + error(self, "Warn", "Master block %s error: %s" % (i, e)) if self._mst_nodes[0] is None and self._mst_nodes[1] is None: - error(self, 'Fatal', 'No valid Master Node found.') + error(self, "Fatal", "No valid Master Node found.") elif self._mst_nodes[0] is None and self._mst_nodes[1] is not None: self._mst_nodes[0] = self._mst_nodes[1] self._mst_nodes[1] = None - log(self , 'Swapping Master Nodes due to bad first node.') - + log(self, "Swapping Master Nodes due to bad first node.") def _get_file(self): return self._file - file = property(_get_file) + file = property(_get_file) def _get_superblock(self): - """ Superblock Node Object + """Superblock Node Object Returns: Obj:Superblock Node """ return self._sb_node - superblock_node = property(_get_superblock) + superblock_node = property(_get_superblock) def _get_master_node(self): """Master Node Object @@ -108,8 +123,8 @@ def _get_master_node(self): Obj:Master Node """ return self._mst_nodes[0] - master_node = property(_get_master_node) + master_node = property(_get_master_node) def _get_master_node2(self): """Master Node Object 2 @@ -118,8 +133,8 @@ def _get_master_node2(self): Obj:Master Node """ return self._mst_nodes[1] - master_node2 = property(_get_master_node2) + master_node2 = property(_get_master_node2) def _get_leb_size(self): """LEB size of UBI blocks in file. @@ -128,8 +143,8 @@ def _get_leb_size(self): Int -- LEB Size. """ return self._leb_size - leb_size = property(_get_leb_size) + leb_size = property(_get_leb_size) def _get_min_io_size(self): """Min I/O Size @@ -138,11 +153,12 @@ def _get_min_io_size(self): Int -- Min I/O Size. """ return self._min_io_size + min_io_size = property(_get_min_io_size) - - def display(self, tab=''): + + def display(self, tab=""): """Print information about this object. - + Argument: Str:tab -- '\t' for spacing this object. """ diff --git a/ubireader/ubifs/defines.py b/ubireader/ubifs/defines.py index 1520706..039f64e 100644 --- a/ubireader/ubifs/defines.py +++ b/ubireader/ubifs/defines.py @@ -35,7 +35,7 @@ # Constant defines # Common Header. -UBIFS_NODE_MAGIC = b'\x31\x18\x10\x06' # Set to LSB +UBIFS_NODE_MAGIC = b"\x31\x18\x10\x06" # Set to LSB # Initial CRC32 value. UBIFS_CRC32_INIT = 0xFFFFFFFF @@ -64,7 +64,7 @@ UBIFS_BLOCK_SHIFT = 12 # UBIFS padding byte pattern. -UBIFS_PADDING_BYTE = b'\xCE' +UBIFS_PADDING_BYTE = b"\xCE" # Max key length UBIFS_MAX_KEY_LEN = 16 @@ -86,32 +86,32 @@ UBIFS_LPT_FANOUT_SHIFT = 2 # LEB Properties Tree bit field sizes. -UBIFS_LPT_CRC_BITS = 16 +UBIFS_LPT_CRC_BITS = 16 UBIFS_LPT_CRC_BYTES = 2 UBIFS_LPT_TYPE_BITS = 4 # LEB Properties Tree node types. -UBIFS_LPT_PNODE = 0 # LPT leaf node (contains LEB Properties) -UBIFS_LPT_NNODE = 1 # LPT internal node -UBIFS_LPT_LTAB = 2 # LPT's own lprops table -UBIFS_LPT_LSAVE = 3 # LPT's save table (big model only) -UBIFS_LPT_NODE_CNT = 4 # count of LPT node types -UBIFS_LPT_NOT_A_NODE = (1 << UBIFS_LPT_TYPE_BITS) - 1 # 4 bits of 1 +UBIFS_LPT_PNODE = 0 # LPT leaf node (contains LEB Properties) +UBIFS_LPT_NNODE = 1 # LPT internal node +UBIFS_LPT_LTAB = 2 # LPT's own lprops table +UBIFS_LPT_LSAVE = 3 # LPT's save table (big model only) +UBIFS_LPT_NODE_CNT = 4 # count of LPT node types +UBIFS_LPT_NOT_A_NODE = (1 << UBIFS_LPT_TYPE_BITS) - 1 # 4 bits of 1 # Inode types -UBIFS_ITYPE_REG = 0 # Regular file -UBIFS_ITYPE_DIR = 1 # Directory -UBIFS_ITYPE_LNK = 2 # Soft link -UBIFS_ITYPE_BLK = 3 # Block device node -UBIFS_ITYPE_CHR = 4 # Char device node -UBIFS_ITYPE_FIFO = 5 # FIFO -UBIFS_ITYPE_SOCK = 6 # Socket -UBIFS_ITYPES_CNT = 7 # Support file type count +UBIFS_ITYPE_REG = 0 # Regular file +UBIFS_ITYPE_DIR = 1 # Directory +UBIFS_ITYPE_LNK = 2 # Soft link +UBIFS_ITYPE_BLK = 3 # Block device node +UBIFS_ITYPE_CHR = 4 # Char device node +UBIFS_ITYPE_FIFO = 5 # FIFO +UBIFS_ITYPE_SOCK = 6 # Socket +UBIFS_ITYPES_CNT = 7 # Support file type count # Supported key has functions -UBIFS_KEY_HASH_R5 = 0 # R5 hash -UBIFS_KEY_HASH_TEST = 1 # Test hash, returns first 4 bytes of name -PRINT_UBIFS_KEY_HASH = ['r5', 'test'] +UBIFS_KEY_HASH_R5 = 0 # R5 hash +UBIFS_KEY_HASH_TEST = 1 # Test hash, returns first 4 bytes of name +PRINT_UBIFS_KEY_HASH = ["r5", "test"] # Supported key formats UBIFS_SIMPLE_KEY_FMT = 0 @@ -119,15 +119,15 @@ # Simple key format uses 29 bits for storing UBIFS name and hash. UBIFS_S_KEY_BLOCK_BITS = 29 UBIFS_S_KEY_BLOCK_MASK = 0x1FFFFFFF -UBIFS_S_KEY_HASH_BITS = UBIFS_S_KEY_BLOCK_BITS -UBIFS_S_KEY_HASH_MASK = UBIFS_S_KEY_BLOCK_MASK +UBIFS_S_KEY_HASH_BITS = UBIFS_S_KEY_BLOCK_BITS +UBIFS_S_KEY_HASH_MASK = UBIFS_S_KEY_BLOCK_MASK # Key types -UBIFS_INO_KEY = 0 # Inode node key -UBIFS_DATA_KEY = 1 # Data node key -UBIFS_DENT_KEY = 2 # Directory node key -UBIFS_XENT_KEY = 3 # Extended attribute entry key -UBIFS_KEY_TYPES_CNT = 4 # Supported key count +UBIFS_INO_KEY = 0 # Inode node key +UBIFS_DATA_KEY = 1 # Data node key +UBIFS_DENT_KEY = 2 # Directory node key +UBIFS_XENT_KEY = 3 # Extended attribute entry key +UBIFS_KEY_TYPES_CNT = 4 # Supported key count # Number of reserved LEBs for Superblock area UBIFS_SB_LEBS = 1 @@ -139,167 +139,175 @@ UBIFS_SB_LNUM = 0 # First LEB of the master area -UBIFS_MST_LNUM = (UBIFS_SB_LNUM + UBIFS_SB_LEBS) +UBIFS_MST_LNUM = UBIFS_SB_LNUM + UBIFS_SB_LEBS # First LEB of log area -UBIFS_LOG_LNUM = (UBIFS_MST_LNUM + UBIFS_MST_LEBS) +UBIFS_LOG_LNUM = UBIFS_MST_LNUM + UBIFS_MST_LEBS # On-flash inode flags -UBIFS_COMPR_FL = 1 # Use compression for this inode -UBIFS_SYNC_FL = 2 # Has to be synchronous I/O +UBIFS_COMPR_FL = 1 # Use compression for this inode +UBIFS_SYNC_FL = 2 # Has to be synchronous I/O UBIFS_IMMUTABLE_FL = 4 # Inode is immutable -UBIFS_APPEND_FL = 8 # Writes may only append data -UBIFS_DIRSYNC_FL = 16 # I/O on this directory inode must be synchronous -UBIFS_XATTR_FL = 32 # This inode is inode for extended attributes +UBIFS_APPEND_FL = 8 # Writes may only append data +UBIFS_DIRSYNC_FL = 16 # I/O on this directory inode must be synchronous +UBIFS_XATTR_FL = 32 # This inode is inode for extended attributes # Inode flag bits used by UBIFS UBIFS_FL_MASK = 0x0000001F # Compression alogrithms. -UBIFS_COMPR_NONE = 0 # No compression -UBIFS_COMPR_LZO = 1 # LZO compression -UBIFS_COMPR_ZLIB = 2 # ZLIB compression -UBIFS_COMPR_ZSTD = 3 # ZSTD compression -UBIFS_COMPR_TYPES_CNT = 4 # Count of supported compression types -PRINT_UBIFS_COMPR = ['none','lzo','zlib', 'zstd'] +UBIFS_COMPR_NONE = 0 # No compression +UBIFS_COMPR_LZO = 1 # LZO compression +UBIFS_COMPR_ZLIB = 2 # ZLIB compression +UBIFS_COMPR_ZSTD = 3 # ZSTD compression +UBIFS_COMPR_TYPES_CNT = 4 # Count of supported compression types +PRINT_UBIFS_COMPR = ["none", "lzo", "zlib", "zstd"] # UBIFS node types -UBIFS_INO_NODE = 0 # Inode node -UBIFS_DATA_NODE = 1 # Data node -UBIFS_DENT_NODE = 2 # Directory entry node -UBIFS_XENT_NODE = 3 # Extended attribute node -UBIFS_TRUN_NODE = 4 # Truncation node -UBIFS_PAD_NODE = 5 # Padding node -UBIFS_SB_NODE = 6 # Superblock node -UBIFS_MST_NODE = 7 # Master node -UBIFS_REF_NODE = 8 # LEB reference node -UBIFS_IDX_NODE = 9 # Index node -UBIFS_CS_NODE = 10 # Commit start node -UBIFS_ORPH_NODE = 11 # Orphan node -UBIFS_AUTH_NODE = 12 # Authentication node -UBIFS_SIG_NODE = 13 # Signature node -UBIFS_NODE_TYPES_CNT = 14 # Count of supported node types +UBIFS_INO_NODE = 0 # Inode node +UBIFS_DATA_NODE = 1 # Data node +UBIFS_DENT_NODE = 2 # Directory entry node +UBIFS_XENT_NODE = 3 # Extended attribute node +UBIFS_TRUN_NODE = 4 # Truncation node +UBIFS_PAD_NODE = 5 # Padding node +UBIFS_SB_NODE = 6 # Superblock node +UBIFS_MST_NODE = 7 # Master node +UBIFS_REF_NODE = 8 # LEB reference node +UBIFS_IDX_NODE = 9 # Index node +UBIFS_CS_NODE = 10 # Commit start node +UBIFS_ORPH_NODE = 11 # Orphan node +UBIFS_AUTH_NODE = 12 # Authentication node +UBIFS_SIG_NODE = 13 # Signature node +UBIFS_NODE_TYPES_CNT = 14 # Count of supported node types # Master node flags -UBIFS_MST_DIRTY = 1 # Rebooted uncleanly -UBIFS_MST_NO_ORPHS = 2 # No orphans present -UBIFS_MST_RCVRY = 4 # Written by recovery -PRINT_UBIFS_MST = [[UBIFS_MST_DIRTY, 'Dirty'], - [UBIFS_MST_NO_ORPHS, 'No orphans'], - [UBIFS_MST_RCVRY, 'Recovery write'], - ] +UBIFS_MST_DIRTY = 1 # Rebooted uncleanly +UBIFS_MST_NO_ORPHS = 2 # No orphans present +UBIFS_MST_RCVRY = 4 # Written by recovery +PRINT_UBIFS_MST = [ + [UBIFS_MST_DIRTY, "Dirty"], + [UBIFS_MST_NO_ORPHS, "No orphans"], + [UBIFS_MST_RCVRY, "Recovery write"], +] # Node group type -UBIFS_NO_NODE_GROUP = 0 # This node is not part of a group -UBIFS_IN_NODE_GROUP = 1 # This node is part of a group -UBIFS_LAST_OF_NODE_GROUP = 2 # This node is the last in a group +UBIFS_NO_NODE_GROUP = 0 # This node is not part of a group +UBIFS_IN_NODE_GROUP = 1 # This node is part of a group +UBIFS_LAST_OF_NODE_GROUP = 2 # This node is the last in a group # Superblock flags -UBIFS_FLG_BIGLPT = 2 # If 'big' LPT model is used if set. -UBIFS_FLG_SPACE_FIXUP = 4 # First-mount 'fixup' of free space within. -UBIFS_FLG_DOUBLE_HASH = 8 # Store 32bit cookie for 64bit support. -UBIFS_FLG_ENCRYPTION = 16 # If filesystem contains encrypted files. -UBIFS_FLG_AUTHENTICATION = 32 # If contains hashes for authentication. -PRINT_UBIFS_FLGS = [[UBIFS_FLG_BIGLPT, 'Big LPT'], - [UBIFS_FLG_SPACE_FIXUP, 'Space fixup'], - [UBIFS_FLG_DOUBLE_HASH, 'Double hash'], - [UBIFS_FLG_ENCRYPTION, 'Encryption'], - [UBIFS_FLG_AUTHENTICATION,'Authentication'], - ] +UBIFS_FLG_BIGLPT = 2 # If 'big' LPT model is used if set. +UBIFS_FLG_SPACE_FIXUP = 4 # First-mount 'fixup' of free space within. +UBIFS_FLG_DOUBLE_HASH = 8 # Store 32bit cookie for 64bit support. +UBIFS_FLG_ENCRYPTION = 16 # If filesystem contains encrypted files. +UBIFS_FLG_AUTHENTICATION = 32 # If contains hashes for authentication. +PRINT_UBIFS_FLGS = [ + [UBIFS_FLG_BIGLPT, "Big LPT"], + [UBIFS_FLG_SPACE_FIXUP, "Space fixup"], + [UBIFS_FLG_DOUBLE_HASH, "Double hash"], + [UBIFS_FLG_ENCRYPTION, "Encryption"], + [UBIFS_FLG_AUTHENTICATION, "Authentication"], +] # Struct defines # Common header node -UBIFS_COMMON_HDR_FORMAT = ' 0: pnames.append(i) - filename = pnames[len(pnames)-1] + filename = pnames[len(pnames) - 1] del pnames[-1] inodes = {} bad_blocks = [] - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) + walk.index( + ubifs, + ubifs.master_node.root_lnum, + ubifs.master_node.root_offs, + inodes, + bad_blocks, + ) if len(inodes) < 2: return False inum = find_dir(inodes, 1, pnames, 0) - if inum == None: + if inum is None: return False - if not 'dent' in inodes[inum]: + if "dent" not in inodes[inum]: return False - for dent in inodes[inum]['dent']: + for dent in inodes[inum]["dent"]: if dent.name == filename: filedata = _process_reg_file(ubifs, inodes[dent.inum], filepath) if os.path.isdir(destpath): destpath = os.path.join(destpath, filename) - with open(destpath, 'wb') as f: + with open(destpath, "wb") as f: f.write(filedata) return True return False @@ -98,12 +116,12 @@ def copy_file(ubifs, filepath, destpath): def find_dir(inodes, inum, names, idx): if len(names) == 0: return 1 - for dent in inodes[inum]['dent']: + for dent in inodes[inum]["dent"]: if dent.name == names[idx]: - if len(names) == idx+1: + if len(names) == idx + 1: return dent.inum else: - return find_dir(inodes, dent.inum, names, idx+1) + return find_dir(inodes, dent.inum, names, idx + 1) return None @@ -114,28 +132,39 @@ def print_dent(ubifs, inodes, dent_node, long=True, longts=False): lnk = "" if dent_node.type == UBIFS_ITYPE_LNK: - lnk = " -> " + inode['ino'].data.decode('utf-8') + lnk = " -> " + inode["ino"].data.decode("utf-8") if longts: - mtime = inode['ino'].mtime_sec + mtime = inode["ino"].mtime_sec else: - mtime = time.strftime("%b %d %H:%M", time.gmtime(inode['ino'].mtime_sec)) - - print('%6o %2d %s %s %7d %s %s%s' % (inode['ino'].mode, inode['ino'].nlink, inode['ino'].uid, inode['ino'].gid, fl, mtime, dent_node.name, lnk)) + mtime = time.strftime("%b %d %H:%M", time.gmtime(inode["ino"].mtime_sec)) + + print( + "%6o %2d %s %s %7d %s %s%s" + % ( + inode["ino"].mode, + inode["ino"].nlink, + inode["ino"].uid, + inode["ino"].gid, + fl, + mtime, + dent_node.name, + lnk, + ) + ) else: print(dent_node.name) def file_leng(ubifs, inode): fl = 0 - if 'data' in inode: - compr_type = 0 - sorted_data = sorted(inode['data'], key=lambda x: x.key['khash']) - last_khash = sorted_data[0].key['khash']-1 + if "data" in inode: + sorted_data = sorted(inode["data"], key=lambda x: x.key["khash"]) + last_khash = sorted_data[0].key["khash"] - 1 for data in sorted_data: - if data.key['khash'] - last_khash != 1: - while 1 != (data.key['khash'] - last_khash): + if data.key["khash"] - last_khash != 1: + while 1 != (data.key["khash"] - last_khash): last_khash += 1 fl = fl + UBIFS_BLOCK_SIZE fl = fl + data.size @@ -146,32 +175,39 @@ def file_leng(ubifs, inode): def _process_reg_file(ubifs, inode, path): try: buf = bytearray() - if 'data' in inode: + if "data" in inode: compr_type = 0 - sorted_data = sorted(inode['data'], key=lambda x: x.key['khash']) - last_khash = sorted_data[0].key['khash']-1 + sorted_data = sorted(inode["data"], key=lambda x: x.key["khash"]) + last_khash = sorted_data[0].key["khash"] - 1 for data in sorted_data: - # If data nodes are missing in sequence, fill in blanks # with \x00 * UBIFS_BLOCK_SIZE - if data.key['khash'] - last_khash != 1: - while 1 != (data.key['khash'] - last_khash): - buf += b'\x00'*UBIFS_BLOCK_SIZE + if data.key["khash"] - last_khash != 1: + while 1 != (data.key["khash"] - last_khash): + buf += b"\x00" * UBIFS_BLOCK_SIZE last_khash += 1 compr_type = data.compr_type ubifs.file.seek(data.offset) d = ubifs.file.read(data.compr_len) buf += decompress(compr_type, data.size, d) - last_khash = data.key['khash'] - verbose_log(_process_reg_file, 'ino num: %s, compression: %s, path: %s' % (inode['ino'].key['ino_num'], compr_type, path)) + last_khash = data.key["khash"] + verbose_log( + _process_reg_file, + "ino num: %s, compression: %s, path: %s" + % (inode["ino"].key["ino_num"], compr_type, path), + ) except Exception as e: - error(_process_reg_file, 'Warn', 'inode num:%s :%s' % (inode['ino'].key['ino_num'], e)) - + error( + _process_reg_file, + "Warn", + "inode num:%s :%s" % (inode["ino"].key["ino_num"], e), + ) + # Pad end of file with \x00 if needed. - if inode['ino'].size > len(buf): - buf += b'\x00' * (inode['ino'].size - len(buf)) - + if inode["ino"].size > len(buf): + buf += b"\x00" * (inode["ino"].size - len(buf)) + return bytes(buf) diff --git a/ubireader/ubifs/misc.py b/ubireader/ubifs/misc.py index c6ce0ea..b9779f6 100755 --- a/ubireader/ubifs/misc.py +++ b/ubireader/ubifs/misc.py @@ -17,16 +17,37 @@ # along with this program. If not, see . ############################################################# -import lzo import struct import zlib -from ubireader.ubifs.defines import * + +import lzo + from ubireader.debug import error +from ubireader.ubifs.defines import ( + UBIFS_SK_LEN, + UBIFS_S_KEY_HASH_MASK, + UBIFS_S_KEY_BLOCK_BITS, + UBIFS_COMPR_LZO, + UBIFS_COMPR_ZLIB, +) # For happy printing -ino_types = ['file', 'dir','lnk','blk','chr','fifo','sock'] -node_types = ['ino','data','dent','xent','trun','pad','sb','mst','ref','idx','cs','orph'] -key_types = ['ino','data','dent','xent'] +ino_types = ["file", "dir", "lnk", "blk", "chr", "fifo", "sock"] +node_types = [ + "ino", + "data", + "dent", + "xent", + "trun", + "pad", + "sb", + "mst", + "ref", + "idx", + "cs", + "orph", +] +key_types = ["ino", "data", "dent", "xent"] def parse_key(key): @@ -40,13 +61,13 @@ def parse_key(key): Int:ino_num -- Inode number. Int:khash -- Key hash. """ - hkey, lkey = struct.unpack('> UBIFS_S_KEY_BLOCK_BITS khash = lkey - #if key_type < UBIFS_KEY_TYPES_CNT: - return {'type':key_type, 'ino_num':ino_num, 'khash': khash} + # if key_type < UBIFS_KEY_TYPES_CNT: + return {"type": key_type, "ino_num": ino_num, "khash": khash} def decompress(ctype, unc_len, data): @@ -62,15 +83,13 @@ def decompress(ctype, unc_len, data): """ if ctype == UBIFS_COMPR_LZO: try: - return lzo.decompress(b''.join((b'\xf0', struct.pack('>I', unc_len), data))) + return lzo.decompress(b"".join((b"\xf0", struct.pack(">I", unc_len), data))) except Exception as e: - error(decompress, 'Warn', 'LZO Error: %s' % e) + error(decompress, "Warn", "LZO Error: %s" % e) elif ctype == UBIFS_COMPR_ZLIB: try: return zlib.decompress(data, -11) except Exception as e: - error(decompress, 'Warn', 'ZLib Error: %s' % e) + error(decompress, "Warn", "ZLib Error: %s" % e) else: return data - - diff --git a/ubireader/ubifs/nodes.py b/ubireader/ubifs/nodes.py index fd2bcb7..e99701e 100755 --- a/ubireader/ubifs/nodes.py +++ b/ubireader/ubifs/nodes.py @@ -17,9 +17,33 @@ # along with this program. If not, see . ############################################################# -from ubireader.ubifs.misc import parse_key -from ubireader.ubifs.defines import * +import struct from ubireader.ubifs import display +from ubireader.ubifs.defines import ( + UBIFS_COMMON_HDR_FIELDS, + UBIFS_COMMON_HDR_FORMAT, + UBIFS_INO_NODE_FIELDS, + UBIFS_INO_NODE_FORMAT, + UBIFS_INO_NODE_SZ, + UBIFS_DENT_NODE_FIELDS, + UBIFS_DENT_NODE_FORMAT, + UBIFS_DENT_NODE_SZ, + UBIFS_DATA_NODE_FIELDS, + UBIFS_DATA_NODE_FORMAT, + UBIFS_DATA_NODE_SZ, + UBIFS_IDX_NODE_FIELDS, + UBIFS_IDX_NODE_FORMAT, + UBIFS_IDX_NODE_SZ, + UBIFS_BRANCH_SZ, + UBIFS_BRANCH_FIELDS, + UBIFS_BRANCH_FORMAT, + UBIFS_MST_NODE_FIELDS, + UBIFS_MST_NODE_FORMAT, + UBIFS_SB_NODE_FIELDS, + UBIFS_SB_NODE_FORMAT, +) +from ubireader.ubifs.misc import parse_key + class common_hdr(object): """Get common header at given LEB number + offset. @@ -29,23 +53,29 @@ class common_hdr(object): See ubifs/defines.py for object attributes. """ - def __init__(self, buf): - fields = dict(list(zip(UBIFS_COMMON_HDR_FIELDS, struct.unpack(UBIFS_COMMON_HDR_FORMAT, buf)))) + def __init__(self, buf): + fields = dict( + list( + zip( + UBIFS_COMMON_HDR_FIELDS, struct.unpack(UBIFS_COMMON_HDR_FORMAT, buf) + ) + ) + ) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) - + setattr(self, "errors", []) + def __repr__(self): - return 'UBIFS Common Header' + return "UBIFS Common Header" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.common_hdr(self, tab) @@ -57,27 +87,34 @@ class ino_node(object): See ubifs/defines.py for object attributes. """ - def __init__(self, buf): - fields = dict(list(zip(UBIFS_INO_NODE_FIELDS, struct.unpack(UBIFS_INO_NODE_FORMAT, buf[0:UBIFS_INO_NODE_SZ])))) + def __init__(self, buf): + fields = dict( + list( + zip( + UBIFS_INO_NODE_FIELDS, + struct.unpack(UBIFS_INO_NODE_FORMAT, buf[0:UBIFS_INO_NODE_SZ]), + ) + ) + ) for key in fields: - if key == 'key': + if key == "key": setattr(self, key, parse_key(fields[key])) else: setattr(self, key, fields[key]) - setattr(self, 'data', buf[UBIFS_INO_NODE_SZ:]) - setattr(self, 'errors', []) + setattr(self, "data", buf[UBIFS_INO_NODE_SZ:]) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Ino Node' + return "UBIFS Ino Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.ino_node(self, tab) @@ -89,26 +126,34 @@ class dent_node(object): See ubifs/defines.py for object attributes. """ + def __init__(self, buf): - fields = dict(list(zip(UBIFS_DENT_NODE_FIELDS, struct.unpack(UBIFS_DENT_NODE_FORMAT, buf[0:UBIFS_DENT_NODE_SZ])))) + fields = dict( + list( + zip( + UBIFS_DENT_NODE_FIELDS, + struct.unpack(UBIFS_DENT_NODE_FORMAT, buf[0:UBIFS_DENT_NODE_SZ]), + ) + ) + ) for key in fields: - if key == 'key': + if key == "key": setattr(self, key, parse_key(fields[key])) else: setattr(self, key, fields[key]) - setattr(self, 'name', '%s' % buf[-self.nlen-1:-1].decode('utf-8')) - setattr(self, 'errors', []) + setattr(self, "name", "%s" % buf[-self.nlen - 1 : -1].decode("utf-8")) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Directory Entry Node' + return "UBIFS Directory Entry Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.dent_node(self, tab) @@ -121,28 +166,35 @@ class data_node(object): See ubifs/defines.py for object attributes. """ - def __init__(self, buf, file_offset): - fields = dict(list(zip(UBIFS_DATA_NODE_FIELDS, struct.unpack(UBIFS_DATA_NODE_FORMAT, buf[0:UBIFS_DATA_NODE_SZ])))) + def __init__(self, buf, file_offset): + fields = dict( + list( + zip( + UBIFS_DATA_NODE_FIELDS, + struct.unpack(UBIFS_DATA_NODE_FORMAT, buf[0:UBIFS_DATA_NODE_SZ]), + ) + ) + ) for key in fields: - if key == 'key': + if key == "key": setattr(self, key, parse_key(fields[key])) else: setattr(self, key, fields[key]) - setattr(self, 'offset', file_offset) - setattr(self, 'compr_len', (len(buf) - UBIFS_DATA_NODE_SZ)) - setattr(self, 'errors', []) + setattr(self, "offset", file_offset) + setattr(self, "compr_len", (len(buf) - UBIFS_DATA_NODE_SZ)) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Data Node' + return "UBIFS Data Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.data_node(self, tab) @@ -154,55 +206,73 @@ class idx_node(object): See ubifs/defines.py for object attributes. """ + def __init__(self, buf): - fields = dict(list(zip(UBIFS_IDX_NODE_FIELDS, struct.unpack(UBIFS_IDX_NODE_FORMAT, buf[0:UBIFS_IDX_NODE_SZ])))) + fields = dict( + list( + zip( + UBIFS_IDX_NODE_FIELDS, + struct.unpack(UBIFS_IDX_NODE_FORMAT, buf[0:UBIFS_IDX_NODE_SZ]), + ) + ) + ) for key in fields: setattr(self, key, fields[key]) idxs = UBIFS_IDX_NODE_SZ brs = UBIFS_BRANCH_SZ - setattr(self, 'branches', [branch(buf[idxs+(brs*i):idxs+(brs*i)+brs]) for i in range(0, self.child_cnt)]) - setattr(self, 'errors', []) + setattr( + self, + "branches", + [ + branch(buf[idxs + (brs * i) : idxs + (brs * i) + brs]) + for i in range(0, self.child_cnt) + ], + ) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Index Node' + return "UBIFS Index Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.idx_node(self, tab) class branch(object): - """ Create branch from given idx_node data buf. + """Create branch from given idx_node data buf. Arguments: Bin:buf -- Raw data to extract header information from. """ + def __init__(self, buf): - fields = dict(list(zip(UBIFS_BRANCH_FIELDS, struct.unpack(UBIFS_BRANCH_FORMAT, buf)))) + fields = dict( + list(zip(UBIFS_BRANCH_FIELDS, struct.unpack(UBIFS_BRANCH_FORMAT, buf))) + ) for key in fields: - if key == 'key': + if key == "key": setattr(self, key, parse_key(fields[key])) else: setattr(self, key, fields[key]) - setattr(self, 'errors', []) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Branch' + return "UBIFS Branch" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.branch(self, tab) - + class sb_node(object): """Get superblock node at given LEB number + offset. @@ -213,23 +283,26 @@ class sb_node(object): See ubifs/defines.py for object attributes. """ + def __init__(self, buf, file_offset=-1): self.file_offset = file_offset - fields = dict(list(zip(UBIFS_SB_NODE_FIELDS, struct.unpack(UBIFS_SB_NODE_FORMAT, buf)))) + fields = dict( + list(zip(UBIFS_SB_NODE_FIELDS, struct.unpack(UBIFS_SB_NODE_FORMAT, buf))) + ) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Super Block Node' + return "UBIFS Super Block Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.sb_node(self, tab) @@ -242,21 +315,24 @@ class mst_node(object): See ubifs/defines.py for object attributes. """ + def __init__(self, buf, file_offset=-1): self.file_offset = file_offset - fields = dict(list(zip(UBIFS_MST_NODE_FIELDS, struct.unpack(UBIFS_MST_NODE_FORMAT, buf)))) + fields = dict( + list(zip(UBIFS_MST_NODE_FIELDS, struct.unpack(UBIFS_MST_NODE_FORMAT, buf))) + ) for key in fields: setattr(self, key, fields[key]) - setattr(self, 'errors', []) + setattr(self, "errors", []) def __repr__(self): - return 'UBIFS Master Block Node' + return "UBIFS Master Block Node" def __iter__(self): for key in dir(self): - if not key.startswith('_'): + if not key.startswith("_"): yield key, getattr(self, key) - def display(self, tab=''): + def display(self, tab=""): return display.mst_node(self, tab) diff --git a/ubireader/ubifs/output.py b/ubireader/ubifs/output.py index b4aa653..083404f 100755 --- a/ubireader/ubifs/output.py +++ b/ubireader/ubifs/output.py @@ -21,10 +21,22 @@ import struct from ubireader import settings -from ubireader.ubifs.defines import * +from ubireader.debug import error, log, verbose_log from ubireader.ubifs import walk +from ubireader.ubifs.defines import ( + UBIFS_ITYPE_DIR, + UBIFS_ITYPE_REG, + UBIFS_ITYPE_LNK, + UBIFS_ITYPE_BLK, + UBIFS_ITYPE_CHR, + UBIFS_ITYPE_FIFO, + UBIFS_ITYPE_SOCK, + UBIFS_DATA_KEY, + UBIFS_S_KEY_BLOCK_BITS, + UBIFS_BLOCK_SIZE, +) from ubireader.ubifs.misc import decompress -from ubireader.debug import error, log, verbose_log + def is_safe_path(basedir, path): basedir = os.path.realpath(basedir) @@ -43,30 +55,45 @@ def extract_files(ubifs, out_path, perms=False): inodes = {} bad_blocks = [] - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) + walk.index( + ubifs, + ubifs.master_node.root_lnum, + ubifs.master_node.root_offs, + inodes, + bad_blocks, + ) if len(inodes) < 2: - raise Exception('No inodes found') + raise Exception("No inodes found") - for dent in inodes[1]['dent']: + for dent in inodes[1]["dent"]: extract_dents(ubifs, inodes, dent, out_path, perms) if len(bad_blocks): - error(extract_files, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) + error( + extract_files, + "Warn", + "Data may be missing or corrupted, bad blocks, LEB [%s]" + % ",".join(map(str, bad_blocks)), + ) except Exception as e: - error(extract_files, 'Error', '%s' % e) + error(extract_files, "Error", "%s" % e) -def extract_dents(ubifs, inodes, dent_node, path='', perms=False): +def extract_dents(ubifs, inodes, dent_node, path="", perms=False): if dent_node.inum not in inodes: - error(extract_dents, 'Error', 'inum: %s not found in inodes' % (dent_node.inum)) + error(extract_dents, "Error", "inum: %s not found in inodes" % (dent_node.inum)) return inode = inodes[dent_node.inum] if not is_safe_path(path, dent_node.name): - error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) + error( + extract_dents, + "Warn", + "Path traversal attempt: %s, discarding." % (dent_node.name), + ) return dent_path = os.path.realpath(os.path.join(path, dent_node.name)) @@ -74,29 +101,32 @@ def extract_dents(ubifs, inodes, dent_node, path='', perms=False): try: if not os.path.exists(dent_path): os.mkdir(dent_path) - log(extract_dents, 'Make Dir: %s' % (dent_path)) + log(extract_dents, "Make Dir: %s" % (dent_path)) if perms: _set_file_perms(dent_path, inode) except Exception as e: - error(extract_dents, 'Warn', 'DIR Fail: %s' % e) + error(extract_dents, "Warn", "DIR Fail: %s" % e) - if 'dent' in inode: - for dnode in inode['dent']: + if "dent" in inode: + for dnode in inode["dent"]: extract_dents(ubifs, inodes, dnode, dent_path, perms) _set_file_timestamps(dent_path, inode) elif dent_node.type == UBIFS_ITYPE_REG: try: - if inode['ino'].nlink > 1: - if 'hlink' not in inode: - inode['hlink'] = dent_path + if inode["ino"].nlink > 1: + if "hlink" not in inode: + inode["hlink"] = dent_path buf = _process_reg_file(ubifs, inode, dent_path) _write_reg_file(dent_path, buf) else: - os.link(inode['hlink'], dent_path) - log(extract_dents, 'Make Link: %s > %s' % (dent_path, inode['hlink'])) + os.link(inode["hlink"], dent_path) + log( + extract_dents, + "Make Link: %s > %s" % (dent_path, inode["hlink"]), + ) else: buf = _process_reg_file(ubifs, inode, dent_path) _write_reg_file(dent_path, buf) @@ -107,100 +137,118 @@ def extract_dents(ubifs, inodes, dent_node, path='', perms=False): _set_file_perms(dent_path, inode) except Exception as e: - error(extract_dents, 'Warn', 'FILE Fail: %s' % e) + error(extract_dents, "Warn", "FILE Fail: %s" % e) elif dent_node.type == UBIFS_ITYPE_LNK: try: # probably will need to decompress ino data if > UBIFS_MIN_COMPR_LEN - os.symlink('%s' % inode['ino'].data.decode('utf-8'), dent_path) - log(extract_dents, 'Make Symlink: %s > %s' % (dent_path, inode['ino'].data)) + os.symlink("%s" % inode["ino"].data.decode("utf-8"), dent_path) + log(extract_dents, "Make Symlink: %s > %s" % (dent_path, inode["ino"].data)) except Exception as e: - error(extract_dents, 'Warn', 'SYMLINK Fail: %s' % e) + error(extract_dents, "Warn", "SYMLINK Fail: %s" % e) elif dent_node.type in [UBIFS_ITYPE_BLK, UBIFS_ITYPE_CHR]: try: - dev = struct.unpack(' len(buf): - buf += b'\x00' * (inode['ino'].size - len(buf)) - + if inode["ino"].size > len(buf): + buf += b"\x00" * (inode["ino"].size - len(buf)) + return bytes(buf) diff --git a/ubireader/ubifs/walk.py b/ubireader/ubifs/walk.py index 370caf5..9c76345 100755 --- a/ubireader/ubifs/walk.py +++ b/ubireader/ubifs/walk.py @@ -18,9 +18,19 @@ ############################################################# from ubireader import settings +from ubireader.debug import error, log, verbose_display, verbose_log from ubireader.ubifs import nodes -from ubireader.ubifs.defines import * -from ubireader.debug import error, log, verbose_log, verbose_display +from ubireader.ubifs.defines import ( + UBIFS_COMMON_HDR_SZ, + UBIFS_IDX_NODE, + UBIFS_IDX_NODE_SZ, + UBIFS_BRANCH_SZ, + UBIFS_INO_NODE, + UBIFS_DATA_NODE, + UBIFS_DATA_NODE_SZ, + UBIFS_DENT_NODE, +) + def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): """Walk the index gathering Inode, Dir Entry, and File nodes. @@ -46,14 +56,22 @@ def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): if len(buf) < UBIFS_COMMON_HDR_SZ: if settings.warn_only_block_read_errors: - error(index, 'Error', 'LEB: %s, Common Hdr Size smaller than expected.' % (lnum)) + error( + index, + "Error", + "LEB: %s, Common Hdr Size smaller than expected." % (lnum), + ) return else: - error(index, 'Fatal', 'LEB: %s, Common Hdr Size smaller than expected.' % (lnum)) + error( + index, + "Fatal", + "LEB: %s, Common Hdr Size smaller than expected." % (lnum), + ) chdr = nodes.common_hdr(buf) - log(index , '%s file addr: %s' % (chdr, ubifs.file.last_read_addr())) + log(index, "%s file addr: %s" % (chdr, ubifs.file.last_read_addr())) verbose_display(chdr) read_size = chdr.len - UBIFS_COMMON_HDR_SZ node_buf = ubifs.file.read(read_size) @@ -61,11 +79,19 @@ def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): if len(node_buf) < read_size: if settings.warn_only_block_read_errors: - error(index, 'Error', 'LEB: %s at %s, Node size smaller than expected.' % (lnum, file_offset)) + error( + index, + "Error", + "LEB: %s at %s, Node size smaller than expected." % (lnum, file_offset), + ) return else: - error(index, 'Fatal', 'LEB: %s at %s, Node size smaller than expected.' % (lnum, file_offset)) + error( + index, + "Fatal", + "LEB: %s at %s, Node size smaller than expected." % (lnum, file_offset), + ) if chdr.node_type == UBIFS_IDX_NODE: try: @@ -73,19 +99,36 @@ def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): except Exception as e: if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e)) + error( + index, + "Error", + "Problem at file address: %s extracting idx_node: %s" + % (file_offset, e), + ) return else: - error(index, 'Fatal', 'Problem at file address: %s extracting idx_node: %s' % (file_offset, e)) - - log(index, '%s file addr: %s' % (idxn, file_offset)) + error( + index, + "Fatal", + "Problem at file address: %s extracting idx_node: %s" + % (file_offset, e), + ) + + log(index, "%s file addr: %s" % (idxn, file_offset)) verbose_display(idxn) branch_idx = 0 for branch in idxn.branches: - verbose_log(index, '-------------------') - log(index, '%s file addr: %s' % (branch, file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ))) + verbose_log(index, "-------------------") + log( + index, + "%s file addr: %s" + % ( + branch, + file_offset + UBIFS_IDX_NODE_SZ + (branch_idx * UBIFS_BRANCH_SZ), + ), + ) verbose_display(branch) index(ubifs, branch.lnum, branch.offs, inodes, bad_blocks) branch_idx += 1 @@ -96,44 +139,70 @@ def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): except Exception as e: if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e)) + error( + index, + "Error", + "Problem at file address: %s extracting ino_node: %s" + % (file_offset, e), + ) return else: - error(index, 'Fatal', 'Problem at file address: %s extracting ino_node: %s' % (file_offset, e)) - - ino_num = inon.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (inon, file_offset, ino_num)) + error( + index, + "Fatal", + "Problem at file address: %s extracting ino_node: %s" + % (file_offset, e), + ) + + ino_num = inon.key["ino_num"] + log(index, "%s file addr: %s, ino num: %s" % (inon, file_offset, ino_num)) verbose_display(inon) - if not ino_num in inodes: + if ino_num not in inodes: inodes[ino_num] = {} - inodes[ino_num]['ino'] = inon + inodes[ino_num]["ino"] = inon elif chdr.node_type == UBIFS_DATA_NODE: try: - datn = nodes.data_node(node_buf, (ubifs.leb_size * lnum) + UBIFS_COMMON_HDR_SZ + offset + UBIFS_DATA_NODE_SZ) + datn = nodes.data_node( + node_buf, + (ubifs.leb_size * lnum) + + UBIFS_COMMON_HDR_SZ + + offset + + UBIFS_DATA_NODE_SZ, + ) except Exception as e: if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e)) + error( + index, + "Error", + "Problem at file address: %s extracting data_node: %s" + % (file_offset, e), + ) return else: - error(index, 'Fatal', 'Problem at file address: %s extracting data_node: %s' % (file_offset, e)) - - ino_num = datn.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (datn, file_offset, ino_num)) + error( + index, + "Fatal", + "Problem at file address: %s extracting data_node: %s" + % (file_offset, e), + ) + + ino_num = datn.key["ino_num"] + log(index, "%s file addr: %s, ino num: %s" % (datn, file_offset, ino_num)) verbose_display(datn) - if not ino_num in inodes: + if ino_num not in inodes: inodes[ino_num] = {} - if not 'data' in inodes[ino_num]: - inodes[ino_num]['data']= [] + if "data" not in inodes[ino_num]: + inodes[ino_num]["data"] = [] - inodes[ino_num]['data'].append(datn) + inodes[ino_num]["data"].append(datn) elif chdr.node_type == UBIFS_DENT_NODE: try: @@ -141,20 +210,30 @@ def index(ubifs, lnum, offset, inodes={}, bad_blocks=[]): except Exception as e: if settings.warn_only_block_read_errors: - error(index, 'Error', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e)) + error( + index, + "Error", + "Problem at file address: %s extracting dent_node: %s" + % (file_offset, e), + ) return else: - error(index, 'Fatal', 'Problem at file address: %s extracting dent_node: %s' % (file_offset, e)) - - ino_num = dn.key['ino_num'] - log(index, '%s file addr: %s, ino num: %s' % (dn, file_offset, ino_num)) + error( + index, + "Fatal", + "Problem at file address: %s extracting dent_node: %s" + % (file_offset, e), + ) + + ino_num = dn.key["ino_num"] + log(index, "%s file addr: %s, ino num: %s" % (dn, file_offset, ino_num)) verbose_display(dn) - if not ino_num in inodes: + if ino_num not in inodes: inodes[ino_num] = {} - if not 'dent' in inodes[ino_num]: - inodes[ino_num]['dent']= [] + if "dent" not in inodes[ino_num]: + inodes[ino_num]["dent"] = [] - inodes[ino_num]['dent'].append(dn) + inodes[ino_num]["dent"].append(dn) diff --git a/ubireader/utils.py b/ubireader/utils.py index 1793285..a48d41c 100755 --- a/ubireader/utils.py +++ b/ubireader/utils.py @@ -18,17 +18,24 @@ ############################################################# import re + from ubireader.debug import error, log -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC, UBIFS_SB_NODE_SZ, UBIFS_SB_NODE, UBIFS_COMMON_HDR_SZ +from ubireader.ubi.defines import FILE_CHUNK_SZ, UBI_EC_HDR_MAGIC from ubireader.ubifs import nodes +from ubireader.ubifs.defines import ( + UBIFS_COMMON_HDR_SZ, + UBIFS_NODE_MAGIC, + UBIFS_SB_NODE, + UBIFS_SB_NODE_SZ, +) + def guess_start_offset(path, guess_offset=0): file_offset = guess_offset - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 + f = open(path, "rb") + f.seek(0, 2) + file_size = f.tell() + 1 f.seek(guess_offset) for _ in range(0, file_size, FILE_CHUNK_SZ): @@ -46,38 +53,44 @@ def guess_start_offset(path, guess_offset=0): ubifs_loc = file_size + 1 if ubi_loc < ubifs_loc: - log(guess_start_offset, 'Found UBI magic number at %s' % (file_offset + ubi_loc)) - return file_offset + ubi_loc + log( + guess_start_offset, + "Found UBI magic number at %s" % (file_offset + ubi_loc), + ) + return file_offset + ubi_loc elif ubifs_loc < ubi_loc: - log(guess_start_offset, 'Found UBIFS magic number at %s' % (file_offset + ubifs_loc)) + log( + guess_start_offset, + "Found UBIFS magic number at %s" % (file_offset + ubifs_loc), + ) return file_offset + ubifs_loc else: - error(guess_start_offset, 'Fatal', 'Could not determine start offset.') + error(guess_start_offset, "Fatal", "Could not determine start offset.") else: - error(guess_start_offset, 'Fatal', 'Could not determine start offset.') + error(guess_start_offset, "Fatal", "Could not determine start offset.") f.close() def guess_filetype(path, start_offset=0): - log(guess_filetype, 'Looking for file type at %s' % start_offset) + log(guess_filetype, "Looking for file type at %s" % start_offset) - with open(path, 'rb') as f: + with open(path, "rb") as f: f.seek(start_offset) buf = f.read(4) if buf == UBI_EC_HDR_MAGIC: ftype = UBI_EC_HDR_MAGIC - log(guess_filetype, 'File looks like a UBI image.') + log(guess_filetype, "File looks like a UBI image.") elif buf == UBIFS_NODE_MAGIC: ftype = UBIFS_NODE_MAGIC - log(guess_filetype, 'File looks like a UBIFS image.') + log(guess_filetype, "File looks like a UBIFS image.") else: ftype = None - error(guess_filetype, 'Fatal', 'Could not determine file type.') - + error(guess_filetype, "Fatal", "Could not determine file type.") + return ftype @@ -86,16 +99,16 @@ def guess_leb_size(path): Arguments: Str:path -- Path to file. - + Returns: Int -- LEB size. - + Searches file for superblock and retrieves leb size. """ - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 + f = open(path, "rb") + f.seek(0, 2) + file_size = f.tell() + 1 f.seek(0) block_size = None @@ -104,7 +117,7 @@ def guess_leb_size(path): for m in re.finditer(UBIFS_NODE_MAGIC, buf): start = m.start() - chdr = nodes.common_hdr(buf[start:start+UBIFS_COMMON_HDR_SZ]) + chdr = nodes.common_hdr(buf[start : start + UBIFS_COMMON_HDR_SZ]) if chdr and chdr.node_type == UBIFS_SB_NODE: sb_start = start + UBIFS_COMMON_HDR_SZ @@ -130,18 +143,18 @@ def guess_peb_size(path): Arguments: Str:path -- Path to file. - + Returns: Int -- PEB size. - - Searches file for Magic Number, picks most + + Searches file for Magic Number, picks most common length between them. """ file_offset = 0 offsets = [] - f = open(path, 'rb') - f.seek(0,2) - file_size = f.tell()+1 + f = open(path, "rb") + f.seek(0, 2) + file_size = f.tell() + 1 f.seek(0) for _ in range(0, file_size, FILE_CHUNK_SZ): @@ -153,7 +166,7 @@ def guess_peb_size(path): file_offset = start idx = start else: - idx = start+file_offset + idx = start + file_offset offsets.append(idx) @@ -163,8 +176,8 @@ def guess_peb_size(path): occurrences = {} for i in range(0, len(offsets)): try: - diff = offsets[i] - offsets[i-1] - except: + diff = offsets[i] - offsets[i - 1] + except Exception: diff = offsets[i] if diff not in occurrences: @@ -180,4 +193,4 @@ def guess_peb_size(path): most_frequent = occurrences[offset] block_size = offset - return block_size \ No newline at end of file + return block_size diff --git a/vulture_whitelist.py b/vulture_whitelist.py new file mode 100644 index 0000000..1161211 --- /dev/null +++ b/vulture_whitelist.py @@ -0,0 +1,174 @@ +from ubireader.settings.py import output_dir +from ubireader.ubi import images +from ubireader.ubi.defines.py import ( + UBI_VID_HDR_MAGIC, + UBI_VID_DYNAMIC, + UBI_VID_STATIC, + UBI_COMPAT_DELETE, + UBI_COMPAT_RO, + UBI_COMPAT_PRESERVE, + UBI_COMPAT_REJECT, +) +from ubireader.ubi_io import reader, read_block, leb_virtual_file +from ubireader.ubifs import superblock_node, master_node2 + +from ubireader.ubifs.defines import ( + UBIFS_CRC32_INIT, + UBIFS_MIN_COMPR_LEN, + UBIFS_MIN_COMPRESS_DIFF, + UBIFS_ROOT_INO, + UBIFS_FIRST_INO, + UBIFS_MAX_NLEN, + UBIFS_MAX_JHEADS, + UBIFS_BLOCK_SHIFT, + UBIFS_PADDING_BYTE, + UBIFS_MIN_FANOUT, + UBIFS_MAX_LEVELS, + UBIFS_MAX_INO_DATA, + UBIFS_LPT_FANOUT, + UBIFS_LPT_FANOUT_SHIFT, + UBIFS_LPT_CRC_BITS, + UBIFS_LPT_CRC_BYTES, + UBIFS_LPT_PNODE, + UBIFS_LPT_NNODE, + UBIFS_LPT_LTAB, + UBIFS_LPT_LSAVE, + UBIFS_LPT_NODE_CNT, + UBIFS_LPT_NOT_A_NODE, + UBIFS_ITYPES_CNT, + UBIFS_KEY_HASH_R5, + UBIFS_KEY_HASH_TEST, + PRINT_UBIFS_KEY_HASH, + UBIFS_SIMPLE_KEY_FMT, + UBIFS_S_KEY_HASH_BITS, + UBIFS_INO_KEY, + UBIFS_DENT_KEY, + UBIFS_XENT_KEY, + UBIFS_KEY_TYPES_CNT, + UBIFS_LOG_LNUM, + UBIFS_COMPR_FL, + UBIFS_SYNC_FL, + UBIFS_IMMUTABLE_FL, + UBIFS_APPEND_FL, + UBIFS_DIRSYNC_FL, + UBIFS_XATTR_FL, + UBIFS_FL_MASK, + UBIFS_COMPR_NONE, + UBIFS_COMPR_ZSTD, + UBIFS_COMPR_TYPES_CNT, + PRINT_UBIFS_COMPR, + UBIFS_XENT_NODE, + UBIFS_TRUN_NODE, + UBIFS_PAD_NODE, + UBIFS_REF_NODE, + UBIFS_CS_NODE, + UBIFS_ORPH_NODE, + UBIFS_AUTH_NODE, + UBIFS_SIG_NODE, + UBIFS_NODE_TYPES_CNT, + UBIFS_NO_NODE_GROUP, + UBIFS_IN_NODE_GROUP, + UBIFS_LAST_OF_NODE_GROUP, + UBIFS_KEY_OFFSET, + UBIFS_DEV_DESC_FIELDS, + UBIFS_DEV_DESC_SZ, + UBIFS_TRUN_NODE_FIELDS, + UBIFS_TRUN_NODE_SZ, + UBIFS_PAD_NODE_FIELDS, + UBIFS_PAD_NODE_SZ, + UBIFS_REF_NODE_FIELDS, + UBIFS_REF_NODE_SZ, + UBIFS_SIG_NODE_FIELDS, + UBIFS_SIG_NODE_SZ, +) +from ubireader.ubifs.list import copy_file +from ubireader.ubifs.misc import ino_types, node_types, key_types +from ubireader.utils import guess_leb_size, guess_peb_size + +output_dir +images +UBI_VID_HDR_MAGIC, +UBI_VID_DYNAMIC, +UBI_VID_STATIC, +UBI_COMPAT_DELETE, +UBI_COMPAT_RO, +UBI_COMPAT_PRESERVE, +UBI_COMPAT_REJECT +reader +read_block, +leb_virtual_file +superblock_node +master_node2 +UBIFS_CRC32_INIT +UBIFS_MIN_COMPR_LEN +UBIFS_MIN_COMPRESS_DIFF +UBIFS_ROOT_INO +UBIFS_FIRST_INO +UBIFS_MAX_NLEN +UBIFS_MAX_JHEADS +UBIFS_BLOCK_SHIFT +UBIFS_PADDING_BYTE +UBIFS_MIN_FANOUT +UBIFS_MAX_LEVELS +UBIFS_MAX_INO_DATA +UBIFS_LPT_FANOUT +UBIFS_LPT_FANOUT_SHIFT +UBIFS_LPT_CRC_BITS +UBIFS_LPT_CRC_BYTES +UBIFS_LPT_PNODE +UBIFS_LPT_NNODE +UBIFS_LPT_LTAB +UBIFS_LPT_LSAVE +UBIFS_LPT_NODE_CNT +UBIFS_LPT_NOT_A_NODE +UBIFS_ITYPES_CNT +UBIFS_KEY_HASH_R5 +UBIFS_KEY_HASH_TEST +PRINT_UBIFS_KEY_HASH +UBIFS_SIMPLE_KEY_FMT +UBIFS_S_KEY_HASH_BITS +UBIFS_INO_KEY +UBIFS_DENT_KEY +UBIFS_XENT_KEY +UBIFS_KEY_TYPES_CNT +UBIFS_LOG_LNUM +UBIFS_COMPR_FL +UBIFS_SYNC_FL +UBIFS_IMMUTABLE_FL +UBIFS_APPEND_FL +UBIFS_DIRSYNC_FL +UBIFS_XATTR_FL +UBIFS_FL_MASK +UBIFS_COMPR_NONE +UBIFS_COMPR_ZSTD +UBIFS_COMPR_TYPES_CNT +PRINT_UBIFS_COMPR +UBIFS_XENT_NODE +UBIFS_TRUN_NODE +UBIFS_PAD_NODE +UBIFS_REF_NODE +UBIFS_CS_NODE +UBIFS_ORPH_NODE +UBIFS_AUTH_NODE +UBIFS_SIG_NODE +UBIFS_NODE_TYPES_CNT +UBIFS_NO_NODE_GROUP +UBIFS_IN_NODE_GROUP +UBIFS_LAST_OF_NODE_GROUP +UBIFS_KEY_OFFSET +UBIFS_DEV_DESC_FIELDS +UBIFS_DEV_DESC_SZ +UBIFS_TRUN_NODE_FIELDS +UBIFS_TRUN_NODE_SZ +UBIFS_PAD_NODE_FIELDS +UBIFS_PAD_NODE_SZ +UBIFS_REF_NODE_FIELDS +UBIFS_REF_NODE_SZ +UBIFS_SIG_NODE_FIELDS +UBIFS_SIG_NODE_SZ +copy_file +ino_types +node_types +key_types +guess_leb_size +guess_peb_size From 3c4e6313d6e059dbf0631b8d1dc61be1d93f09e3 Mon Sep 17 00:00:00 2001 From: Quentin Kaiser Date: Fri, 12 May 2023 07:46:56 +0200 Subject: [PATCH 4/4] add dev dependencies --- poetry.lock | 429 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 7 + 2 files changed, 435 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 4e66d7d..0b58902 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,166 @@ # This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +[[package]] +name = "cfgv" +version = "3.3.1" +description = "Validate configuration and produce human readable error messages." +category = "dev" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.2.5" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "coverage-7.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:883123d0bbe1c136f76b56276074b0c79b5817dd4238097ffa64ac67257f4b6c"}, + {file = "coverage-7.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fbc2a127e857d2f8898aaabcc34c37771bf78a4d5e17d3e1f5c30cd0cbc62a"}, + {file = "coverage-7.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f3671662dc4b422b15776cdca89c041a6349b4864a43aa2350b6b0b03bbcc7f"}, + {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780551e47d62095e088f251f5db428473c26db7829884323e56d9c0c3118791a"}, + {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:066b44897c493e0dcbc9e6a6d9f8bbb6607ef82367cf6810d387c09f0cd4fe9a"}, + {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9a4ee55174b04f6af539218f9f8083140f61a46eabcaa4234f3c2a452c4ed11"}, + {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:706ec567267c96717ab9363904d846ec009a48d5f832140b6ad08aad3791b1f5"}, + {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ae453f655640157d76209f42c62c64c4d4f2c7f97256d3567e3b439bd5c9b06c"}, + {file = "coverage-7.2.5-cp310-cp310-win32.whl", hash = "sha256:f81c9b4bd8aa747d417407a7f6f0b1469a43b36a85748145e144ac4e8d303cb5"}, + {file = "coverage-7.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:dc945064a8783b86fcce9a0a705abd7db2117d95e340df8a4333f00be5efb64c"}, + {file = "coverage-7.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cc0f91c6cde033da493227797be2826cbf8f388eaa36a0271a97a332bfd7ce"}, + {file = "coverage-7.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a66e055254a26c82aead7ff420d9fa8dc2da10c82679ea850d8feebf11074d88"}, + {file = "coverage-7.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c10fbc8a64aa0f3ed136b0b086b6b577bc64d67d5581acd7cc129af52654384e"}, + {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a22cbb5ede6fade0482111fa7f01115ff04039795d7092ed0db43522431b4f2"}, + {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292300f76440651529b8ceec283a9370532f4ecba9ad67d120617021bb5ef139"}, + {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7ff8f3fb38233035028dbc93715551d81eadc110199e14bbbfa01c5c4a43f8d8"}, + {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a08c7401d0b24e8c2982f4e307124b671c6736d40d1c39e09d7a8687bddf83ed"}, + {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef9659d1cda9ce9ac9585c045aaa1e59223b143f2407db0eaee0b61a4f266fb6"}, + {file = "coverage-7.2.5-cp311-cp311-win32.whl", hash = "sha256:30dcaf05adfa69c2a7b9f7dfd9f60bc8e36b282d7ed25c308ef9e114de7fc23b"}, + {file = "coverage-7.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:97072cc90f1009386c8a5b7de9d4fc1a9f91ba5ef2146c55c1f005e7b5c5e068"}, + {file = "coverage-7.2.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bebea5f5ed41f618797ce3ffb4606c64a5de92e9c3f26d26c2e0aae292f015c1"}, + {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828189fcdda99aae0d6bf718ea766b2e715eabc1868670a0a07bf8404bf58c33"}, + {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e8a95f243d01ba572341c52f89f3acb98a3b6d1d5d830efba86033dd3687ade"}, + {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8834e5f17d89e05697c3c043d3e58a8b19682bf365048837383abfe39adaed5"}, + {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d1f25ee9de21a39b3a8516f2c5feb8de248f17da7eead089c2e04aa097936b47"}, + {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1637253b11a18f453e34013c665d8bf15904c9e3c44fbda34c643fbdc9d452cd"}, + {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8e575a59315a91ccd00c7757127f6b2488c2f914096077c745c2f1ba5b8c0969"}, + {file = "coverage-7.2.5-cp37-cp37m-win32.whl", hash = "sha256:509ecd8334c380000d259dc66feb191dd0a93b21f2453faa75f7f9cdcefc0718"}, + {file = "coverage-7.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:12580845917b1e59f8a1c2ffa6af6d0908cb39220f3019e36c110c943dc875b0"}, + {file = "coverage-7.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b5016e331b75310610c2cf955d9f58a9749943ed5f7b8cfc0bb89c6134ab0a84"}, + {file = "coverage-7.2.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:373ea34dca98f2fdb3e5cb33d83b6d801007a8074f992b80311fc589d3e6b790"}, + {file = "coverage-7.2.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a063aad9f7b4c9f9da7b2550eae0a582ffc7623dca1c925e50c3fbde7a579771"}, + {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c0a497a000d50491055805313ed83ddba069353d102ece8aef5d11b5faf045"}, + {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b3b05e22a77bb0ae1a3125126a4e08535961c946b62f30985535ed40e26614"}, + {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0342a28617e63ad15d96dca0f7ae9479a37b7d8a295f749c14f3436ea59fdcb3"}, + {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf97ed82ca986e5c637ea286ba2793c85325b30f869bf64d3009ccc1a31ae3fd"}, + {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c2c41c1b1866b670573657d584de413df701f482574bad7e28214a2362cb1fd1"}, + {file = "coverage-7.2.5-cp38-cp38-win32.whl", hash = "sha256:10b15394c13544fce02382360cab54e51a9e0fd1bd61ae9ce012c0d1e103c813"}, + {file = "coverage-7.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:a0b273fe6dc655b110e8dc89b8ec7f1a778d78c9fd9b4bda7c384c8906072212"}, + {file = "coverage-7.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c587f52c81211d4530fa6857884d37f514bcf9453bdeee0ff93eaaf906a5c1b"}, + {file = "coverage-7.2.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4436cc9ba5414c2c998eaedee5343f49c02ca93b21769c5fdfa4f9d799e84200"}, + {file = "coverage-7.2.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6599bf92f33ab041e36e06d25890afbdf12078aacfe1f1d08c713906e49a3fe5"}, + {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:857abe2fa6a4973f8663e039ead8d22215d31db613ace76e4a98f52ec919068e"}, + {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f5cab2d7f0c12f8187a376cc6582c477d2df91d63f75341307fcdcb5d60303"}, + {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aa387bd7489f3e1787ff82068b295bcaafbf6f79c3dad3cbc82ef88ce3f48ad3"}, + {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:156192e5fd3dbbcb11cd777cc469cf010a294f4c736a2b2c891c77618cb1379a"}, + {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd3b4b8175c1db502adf209d06136c000df4d245105c8839e9d0be71c94aefe1"}, + {file = "coverage-7.2.5-cp39-cp39-win32.whl", hash = "sha256:ddc5a54edb653e9e215f75de377354e2455376f416c4378e1d43b08ec50acc31"}, + {file = "coverage-7.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:338aa9d9883aaaad53695cb14ccdeb36d4060485bb9388446330bef9c361c252"}, + {file = "coverage-7.2.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:8877d9b437b35a85c18e3c6499b23674684bf690f5d96c1006a1ef61f9fdf0f3"}, + {file = "coverage-7.2.5.tar.gz", hash = "sha256:f99ef080288f09ffc687423b8d60978cf3a465d3f404a18d1a05474bd8575a47"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.12.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "filelock-3.12.0-py3-none-any.whl", hash = "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9"}, + {file = "filelock-3.12.0.tar.gz", hash = "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"}, +] + +[package.extras] +docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "identify" +version = "2.5.24" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "identify-2.5.24-py2.py3-none-any.whl", hash = "sha256:986dbfb38b1140e763e413e6feb44cd731faf72d1909543178aa79b0e258265d"}, + {file = "identify-2.5.24.tar.gz", hash = "sha256:0aac67d5b4812498056d28a9a512a483f5085cc28640b02b258a59dac34301d4"}, +] + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "lzallright" version = "0.1.0" @@ -18,7 +179,273 @@ files = [ {file = "lzallright-0.1.0.tar.gz", hash = "sha256:0de7e0fe110650a79c1e1fca10a188da5b6fc6b2f3730c69dc411782e3b4923a"}, ] +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "platformdirs" +version = "3.5.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.5.1-py3-none-any.whl", hash = "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"}, + {file = "platformdirs-3.5.1.tar.gz", hash = "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f"}, +] + +[package.extras] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.2.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pre-commit" +version = "3.3.1" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pre_commit-3.3.1-py2.py3-none-any.whl", hash = "sha256:218e9e3f7f7f3271ebc355a15598a4d3893ad9fc7b57fe446db75644543323b9"}, + {file = "pre_commit-3.3.1.tar.gz", hash = "sha256:733f78c9a056cdd169baa6cd4272d51ecfda95346ef8a89bf93712706021b907"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "pyright" +version = "1.1.307" +description = "Command line wrapper for pyright" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyright-1.1.307-py3-none-any.whl", hash = "sha256:6b360d2e018311bdf8acea73ef1f21bf0b5b502345aa94bc6763cf197b2e75b3"}, + {file = "pyright-1.1.307.tar.gz", hash = "sha256:b7a8734fad4a2438b8bb0dfbe462f529c9d4eb31947bdae85b9b4e7a97ff6a49"}, +] + +[package.dependencies] +nodeenv = ">=1.6.0" + +[package.extras] +all = ["twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] + +[[package]] +name = "pytest" +version = "7.3.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "4.0.0" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-cov-4.0.0.tar.gz", hash = "sha256:996b79efde6433cdbd0088872dbc5fb3ed7fe1578b68cdbba634f14bb8dd0470"}, + {file = "pytest_cov-4.0.0-py3-none-any.whl", hash = "sha256:2feb1b751d66a8bd934e5edfa2e961d11309dc37b73b0eabe73b5945fee20f6b"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + +[[package]] +name = "ruff" +version = "0.0.265" +description = "An extremely fast Python linter, written in Rust." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.0.265-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:30ddfe22de6ce4eb1260408f4480bbbce998f954dbf470228a21a9b2c45955e4"}, + {file = "ruff-0.0.265-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:a11bd0889e88d3342e7bc514554bb4461bf6cc30ec115821c2425cfaac0b1b6a"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a9b38bdb40a998cbc677db55b6225a6c4fadcf8819eb30695e1b8470942426b"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8b44a245b60512403a6a03a5b5212da274d33862225c5eed3bcf12037eb19bb"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b279fa55ea175ef953208a6d8bfbcdcffac1c39b38cdb8c2bfafe9222add70bb"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5028950f7af9b119d43d91b215d5044976e43b96a0d1458d193ef0dd3c587bf8"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4057eb539a1d88eb84e9f6a36e0a999e0f261ed850ae5d5817e68968e7b89ed9"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d586e69ab5cbf521a1910b733412a5735936f6a610d805b89d35b6647e2a66aa"}, + {file = "ruff-0.0.265-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa17b13cd3f29fc57d06bf34c31f21d043735cc9a681203d634549b0e41047d1"}, + {file = "ruff-0.0.265-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9ac13b11d9ad3001de9d637974ec5402a67cefdf9fffc3929ab44c2fcbb850a1"}, + {file = "ruff-0.0.265-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:62a9578b48cfd292c64ea3d28681dc16b1aa7445b7a7709a2884510fc0822118"}, + {file = "ruff-0.0.265-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d0f9967f84da42d28e3d9d9354cc1575f96ed69e6e40a7d4b780a7a0418d9409"}, + {file = "ruff-0.0.265-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1d5a8de2fbaf91ea5699451a06f4074e7a312accfa774ad9327cde3e4fda2081"}, + {file = "ruff-0.0.265-py3-none-win32.whl", hash = "sha256:9e9db5ccb810742d621f93272e3cc23b5f277d8d00c4a79668835d26ccbe48dd"}, + {file = "ruff-0.0.265-py3-none-win_amd64.whl", hash = "sha256:f54facf286103006171a00ce20388d88ed1d6732db3b49c11feb9bf3d46f90e9"}, + {file = "ruff-0.0.265-py3-none-win_arm64.whl", hash = "sha256:c78470656e33d32ddc54e8482b1b0fc6de58f1195586731e5ff1405d74421499"}, + {file = "ruff-0.0.265.tar.gz", hash = "sha256:53c17f0dab19ddc22b254b087d1381b601b155acfa8feed514f0d6a413d0ab3a"}, +] + +[[package]] +name = "setuptools" +version = "67.7.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, + {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "virtualenv" +version = "20.23.0" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.23.0-py3-none-any.whl", hash = "sha256:6abec7670e5802a528357fdc75b26b9f57d5d92f29c5462ba0fbe45feacc685e"}, + {file = "virtualenv-20.23.0.tar.gz", hash = "sha256:a85caa554ced0c0afbd0d638e7e2d7b5f92d23478d05d17a76daeac8f279f924"}, +] + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.11,<4" +platformdirs = ">=3.2,<4" + +[package.extras] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=67.7.1)", "time-machine (>=2.9)"] + [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "56aada67c1709383460f6cd9aed3aa150be487dddd71f92c43618412ef26148c" +content-hash = "ebd2429909e1b6d0af75389e33094d04bf46dc0284c5f5aa164165110f5e2717" diff --git a/pyproject.toml b/pyproject.toml index 4eb446d..7b77f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,13 @@ packages = [{include = "ubi_reader"}] python = "^3.8" lzallright = "^0.1.0" +[tool.poetry.group.dev.dependencies] +pyright = "^1.1.307" +ruff = "^0.0.265" +pytest = "^7.3.1" +pytest-cov = "^4.0.0" +pre-commit = "^3.3.1" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api"