diff --git a/data/projects.ini b/data/projects.ini index 3e674ad1..b70c8cb8 100644 --- a/data/projects.ini +++ b/data/projects.ini @@ -207,6 +207,7 @@ description_long: Distributed Task Queue version: >=2017.7.27.1 dpkg_name: python-certifi maintainer: Kenneth Reitz +license: MPL-2.0 homepage_url: http://certifi.io/ download_url: https://pypi.org/project/certifi git_url: https://github.com/certifi/python-certifi.git @@ -233,6 +234,7 @@ description_short: Foreign Function Interface for Python calling C code description_long: Foreign Function Interface for Python calling C code [chardet] +build_system: setuptools version: >=3.0.4 dpkg_name: python-chardet maintainer: Daniel Blanchard @@ -336,6 +338,7 @@ description_long: The dateutil module provides powerful extensions to the [defusedxml] version: >=0.6.0 maintainer: Christian Heimes +license: PSFL homepage_url: https://github.com/tiran/defusedxml download_url: https://pypi.org/project/defusedxml git_url: https://github.com/tiran/defusedxml.git @@ -400,6 +403,8 @@ description_long: Docutils is a modular system for processing documentation [dtfabric] version: >=20170413 +dpkg_build_dependencies: python3-build,python3-wheel +rpm_build_dependencies: python3-build,python3-wheel download_url: https://github.com/libyal/dtfabric/releases git_url: https://github.com/libyal/dtfabric.git @@ -765,6 +770,10 @@ description_long: A comprehensive HTTP client library. [idna] version: >=2.5 +build_system: flit +dpkg_build_dependencies: flit +rpm_build_dependencies: python3-flit-core,python3-pip +license: BSD License maintainer: Kim Davies homepage_url: https://github.com/kjd/idna download_url: https://pypi.org/project/idna @@ -1178,6 +1187,7 @@ build_dependencies: fuse,libcrypto,zlib optional_build_dependencies: bzip2,liblzma dpkg_build_dependencies: libbz2-dev,libfuse-dev,liblzma-dev,libssl-dev,zlib1g-dev pkg_configure_options: --enable-python,--with-pyprefix,--with-libcaes=no +rpm_build_dependencies: bzip2-devel,xz-devel wheel_name: libmodi_python version: >=20210403 download_url: https://github.com/libyal/libmodi/releases @@ -1501,9 +1511,11 @@ description_long: A generic, spec-compliant, thorough implementation of the OAuth request-signing logic. [opensearch-py] +build_system: setup_py dpkg_name: python-opensearch dpkg_dependencies: python3-certifi,python3-urllib3 pypi_name: opensearch-py +pypi_source_name: opensearch_py rpm_name: python-opensearch rpm_template_spec: opensearch-py.spec maintainer: Aleksei Atavin @@ -1744,8 +1756,9 @@ description_long: Google Commandline Flags Module architecture_dependent: true dpkg_build_dependencies: python3-pkgconfig,python3-pytest-runner,python3-setuptools-scm dpkg_template_rules: python-lz4-rules -rpm_build_dependencies: python3-setuptools_scm,python3-pkgconfig,python3-pytest-runner +rpm_build_dependencies: python3-pkgconfig,python3-pytest-runner,python3-setuptools_scm setup_name: lz4 +license: BSD License maintainer: Jonathan Underwood homepage_url: https://github.com/python-lz4/python-lz4 download_url: https://pypi.org/project/lz4 @@ -1833,7 +1846,7 @@ version: >=14.7.0 dpkg_name: python-zmq dpkg_build_dependencies: cython3,python3-packaging dpkg_template_rules: pyzmq-rules -rpm_build_dependencies: gcc-c++,python3-packaging,python3-scikit-build-core +rpm_build_dependencies: gcc-c++,python3-cython,python3-packaging,python3-scikit-build-core rpm_name: zmq rpm_template_spec: pyzmq.spec maintainer: PyZMQ project @@ -1863,6 +1876,7 @@ description_long: The Python interface to the Redis key-value store version: >=2.2.1 dpkg_dependencies: python3-certifi,python3-chardet,python3-idna,python3-urllib3 rpm_dependencies: python3-certifi,python3-chardet,python3-idna,python3-urllib3 +license: Apache-2.0 maintainer: Kenneth Reitz homepage_url: http://docs.python-requests.org/en/latest download_url: https://pypi.org/project/requests @@ -2115,7 +2129,7 @@ disabled: windows version: >=0.9.7 dpkg_build_dependencies: python3-cffi,python3-setuptools,python3-toml,python3-wheel dpkg_name: python-xattr -rpm_build_dependencies: python3-cffi,python3-pip,python3-setuptools,python3-wheel,tox-current-env +rpm_build_dependencies: python3-cffi,python3-pip,python3-setuptools,python3-tox-current-env,python3-wheel rpm_name: python-xattr rpm_template_spec: xattr.spec maintainer: John Machin @@ -2193,11 +2207,13 @@ description_long: The pattern matching swiss knife. [yara-python] architecture_dependent: true +build_system: setup_py version: >=4.0.0 dpkg_name: python-yara dpkg_build_dependencies: libmagic-dev,libssl-dev dpkg_template_control: yara-python-control dpkg_template_rules: yara-python-rules +pypi_source_name: yara_python rpm_name: yara rpm_build_dependencies: file-devel,openssl-devel rpm_template_spec: yara-python.spec diff --git a/data/rpm_templates/PyYAML.spec b/data/rpm_templates/PyYAML.spec index 568a234d..614029d6 100644 --- a/data/rpm_templates/PyYAML.spec +++ b/data/rpm_templates/PyYAML.spec @@ -1,6 +1,6 @@ %define name PyYAML %define version {version} -%define unmangled_name PyYAML +%define unmangled_name pyyaml %define unmangled_version {version} %define release 1 diff --git a/data/rpm_templates/opensearch-py.spec b/data/rpm_templates/opensearch-py.spec index 2897ad7d..4f3c0d4e 100644 --- a/data/rpm_templates/opensearch-py.spec +++ b/data/rpm_templates/opensearch-py.spec @@ -1,6 +1,6 @@ %define name opensearch-py %define version {version} -%define unmangled_name opensearch-py +%define unmangled_name opensearch_py %define unmangled_version {version} %define release 1 @@ -8,10 +8,10 @@ Summary: Python client for OpenSearch Name: %{{name}} Version: %{{version}} Release: %{{release}} -Source0: opensearch-py-%{{unmangled_version}}.tar.gz +Source0: %{{unmangled_name}}-%{{unmangled_version}}.tar.gz License: Apache License, Version 2.0 Group: Development/Libraries -BuildRoot: %{{_tmppath}}/opensearch-py-%{{version}}-%{{release}}-buildroot +BuildRoot: %{{_tmppath}}/%{{unmangled_name}}-%{{version}}-%{{release}}-buildroot Prefix: %{{_prefix}} BuildArch: noarch Vendor: Honza Král @@ -29,7 +29,7 @@ Requires: python3-certifi, python3-urllib3 Python client for OpenSearch. %prep -%autosetup -n opensearch-py-%{{unmangled_version}} +%autosetup -n %{{unmangled_name}}-%{{unmangled_version}} %build %py3_build diff --git a/data/rpm_templates/pefile.spec b/data/rpm_templates/pefile.spec index ef017b0c..e1cefd64 100644 --- a/data/rpm_templates/pefile.spec +++ b/data/rpm_templates/pefile.spec @@ -45,7 +45,6 @@ rm -rf %{{buildroot}} %files -n python3-%{{name}} %license LICENSE -%doc README %{{python3_sitelib}}/ %changelog diff --git a/data/rpm_templates/pyzmq.spec b/data/rpm_templates/pyzmq.spec index 0b018f97..e1f9157a 100644 --- a/data/rpm_templates/pyzmq.spec +++ b/data/rpm_templates/pyzmq.spec @@ -13,6 +13,10 @@ Vendor: Brian E. Granger, Min Ragan-Kelley Source0: %{{name}}-%{{version}}.tar.gz BuildRequires: gcc, gcc-c++, python3-devel, python3-packaging, python3-scikit-build-core, python3-setuptools +%{{?python_disable_dependency_generator}} + +%global debug_package %{nil} + %description PyZMQ is the official Python binding for the ZeroMQ Messaging Library (http://www.zeromq.org). @@ -34,7 +38,7 @@ Messaging Library (http://www.zeromq.org). %pyproject_install %files -n python3-zmq -%license LICENSE.BSD LICENSE.LESSER +%license LICENSE.md %doc AUTHORS.md README.md %{{_libdir}}/python3*/site-packages/zmq %{{_libdir}}/python3*/site-packages/pyzmq*.dist-info diff --git a/data/rpm_templates/sleuthkit.spec b/data/rpm_templates/sleuthkit.spec index f7abca0e..3662c804 100644 --- a/data/rpm_templates/sleuthkit.spec +++ b/data/rpm_templates/sleuthkit.spec @@ -13,8 +13,8 @@ Source0: %{{unmangled_name}}-%{{unmangled_version}}.tar.gz BuildRoot: %{{_tmppath}}/%{{unmangled_name}}-release-%{{version}}-%{{release}}-buildroot Prefix: %{{_prefix}} URL: http://www.sleuthkit.org -BuildRequires: gcc gcc-c++ sqlite-devel zlib-devel -Requires: sleuthkit-libs = %{{version}}-%{{release}} sqlite zlib +BuildRequires: gcc gcc-c++ sqlite-devel zlib-ng-devel +Requires: sleuthkit-libs = %{{version}}-%{{release}} sqlite zlib-ng %description The Sleuth Kit (TSK) is a collection of UNIX-based command line tools that diff --git a/data/rpm_templates/yara-python.spec b/data/rpm_templates/yara-python.spec index ccf152dd..81ced63a 100644 --- a/data/rpm_templates/yara-python.spec +++ b/data/rpm_templates/yara-python.spec @@ -1,6 +1,6 @@ %define name yara-python %define version {version} -%define unmangled_name yara-python +%define unmangled_name yara_python %define unmangled_version {version} %define release 1 diff --git a/l2tdevtools/build_helpers/factory.py b/l2tdevtools/build_helpers/factory.py index 3a747f49..c33bc047 100644 --- a/l2tdevtools/build_helpers/factory.py +++ b/l2tdevtools/build_helpers/factory.py @@ -82,6 +82,8 @@ def NewBuildHelper( build_helper_class = cls._FLIT_BUILD_HELPER_CLASSES.get( build_target, None) + # TODO: add support for hatchling (used by urllib3) + elif project_definition.build_system == 'poetry': build_helper_class = cls._POETRY_BUILD_HELPER_CLASSES.get( build_target, None) diff --git a/l2tdevtools/build_helpers/rpm.py b/l2tdevtools/build_helpers/rpm.py index 8b20dbbd..29ca9a02 100644 --- a/l2tdevtools/build_helpers/rpm.py +++ b/l2tdevtools/build_helpers/rpm.py @@ -38,13 +38,13 @@ class BaseRPMBuildHelper(interface.BuildHelper): _BUILD_DEPENDENCY_PACKAGE_NAMES = { 'bzip2': ['bzip2-devel'], - 'fuse': ['fuse-devel'], + 'fuse': ['fuse3-devel'], 'libcrypto': ['openssl-devel'], 'liblzma': ['xz-devel'], 'pytest-runner': ['python3-pytest-runner'], 'sqlite': ['sqlite-devel'], 'zeromq': ['libzmq3-devel'], - 'zlib': ['zlib-devel'] + 'zlib': ['zlib-ng-devel'] } def __init__( @@ -90,7 +90,7 @@ def _BuildFromSpecFile(self, spec_filename, rpmbuild_flags='-ba'): rpmbuild_flags, spec_filename, self.LOG_FILENAME) exit_code = subprocess.call(command, shell=True) if exit_code != 0: - logging.error('Running: "{0:s}" failed.'.format(command)) + logging.error(f'Running: "{command:s}" failed.') os.chdir(current_path) @@ -113,7 +113,7 @@ def _BuildFromSourcePackage( rpmbuild_flags, source_package_filename, self.LOG_FILENAME) exit_code = subprocess.call(command, shell=True) if exit_code != 0: - logging.error('Running: "{0:s}" failed.'.format(command)) + logging.error(f'Running: "{command:s}" failed.') return False return True @@ -165,7 +165,7 @@ def _CreateSpecFile(self, project_name, spec_file_data): spec_file_data (str): spec file data. """ spec_filename = os.path.join( - self._rpmbuild_specs_path, '{0:s}.spec'.format(project_name)) + self._rpmbuild_specs_path, f'{project_name:s}.spec') with open(spec_filename, 'w', encoding='utf-8') as rpm_spec_file: rpm_spec_file.write(spec_file_data) @@ -207,7 +207,7 @@ def _MoveFilesToCurrentDirectory(self, filenames_glob): """ filenames = glob.glob(filenames_glob) for filename in filenames: - logging.info('Moving: {0:s}'.format(filename)) + logging.info(f'Moving: {filename:s}') local_filename = os.path.basename(filename) if os.path.exists(local_filename): @@ -246,15 +246,15 @@ def _RemoveBuildDirectory(self, project_name, project_version): project_name (str): name of the project. project_version (str): version of the project. """ - filename = '{0:s}-{1!s}'.format(project_name, project_version) - filename = os.path.join(self.rpmbuild_path, 'BUILD', filename) + filename = os.path.join( + self.rpmbuild_path, 'BUILD', f'{project_name:s}-{project_version!s}') if os.path.exists(filename): - logging.info('Removing: {0:s}'.format(filename)) try: shutil.rmtree(filename) + logging.info(f'Removed: {filename:s}') except OSError: - logging.warning('Unable to remove: {0:s}'.format(filename)) + logging.warning(f'Unable to remove: {filename:s}') def _RemoveOlderBuildDirectory(self, project_name, project_version): """Removes previous versions of build directories. @@ -263,9 +263,7 @@ def _RemoveOlderBuildDirectory(self, project_name, project_version): project_name (str): name of the project. project_version (str): version of the project. """ - filenames_to_ignore = '{0:s}-{1!s}'.format( - project_name, project_version) - filenames_to_ignore = re.compile(filenames_to_ignore) + filenames_to_ignore = re.compile(f'{project_name:s}-{project_version!s}') filenames_glob = os.path.join( self.rpmbuild_path, 'BUILD', '{0:s}-*'.format(project_name)) @@ -442,14 +440,16 @@ def _GenerateSpecFile( if project_name.startswith('python-'): project_name = project_name[7:] - spec_filename = '{0:s}.spec'.format(project_name) - output_file_path = os.path.join(self._rpmbuild_specs_path, spec_filename) + output_file_path = os.path.join( + self._rpmbuild_specs_path, f'{project_name:s}.spec') try: result = spec_file_generator.Generate( self._project_definition, source_directory, source_package_filename, project_name, project_version, output_file_path) - except (FileNotFoundError, TypeError): + except (FileNotFoundError, TypeError, ValueError) as exception: + logging.warning( + f'Unable to gerenate rpm spec file with error: {exception!s}') result = False if not result: @@ -717,14 +717,14 @@ def _GenerateSpecFile( if project_name.startswith('python-'): project_name = project_name[7:] - spec_filename = '{0:s}.spec'.format(project_name) - output_file_path = os.path.join(self._rpmbuild_specs_path, spec_filename) + output_file_path = os.path.join( + self._rpmbuild_specs_path, f'{project_name:s}.spec') try: result = spec_file_generator.Generate( self._project_definition, source_directory, source_package_filename, project_name, project_version, output_file_path) - except (FileNotFoundError, TypeError): + except (FileNotFoundError, TypeError, ValueError): result = False if not result: diff --git a/l2tdevtools/projects.py b/l2tdevtools/projects.py index 540c0378..3d709255 100644 --- a/l2tdevtools/projects.py +++ b/l2tdevtools/projects.py @@ -41,6 +41,7 @@ class ProjectDefinition(object): github_release_prefix (str): github release prefix. github_release_tag_prefix (str): github release tag prefix. homepage_url (str): project homepage URL. + license (str): license. maintainer (str): name and email address of the maintainer. name (str): name of the project. optional_build_dependencies (list[str]): optional build dependencies. @@ -89,6 +90,7 @@ def __init__(self, name): self.github_release_prefix = None self.github_release_tag_prefix = None self.homepage_url = None + self.license = None self.maintainer = None self.name = name self.optional_build_dependencies = None @@ -262,6 +264,8 @@ def Read(self, file_object): config_parser, section_name, 'github_release_tag_prefix') project_definition.homepage_url = self._GetConfigValue( config_parser, section_name, 'homepage_url') + project_definition.license = self._GetConfigValue( + config_parser, section_name, 'license') project_definition.maintainer = self._GetConfigValue( config_parser, section_name, 'maintainer') project_definition.rpm_build_dependencies = self._GetConfigValue( diff --git a/l2tdevtools/review_helpers/git.py b/l2tdevtools/review_helpers/git.py index 27acb985..3584788e 100644 --- a/l2tdevtools/review_helpers/git.py +++ b/l2tdevtools/review_helpers/git.py @@ -158,7 +158,7 @@ def GetChangedFiles(self, diffbase=None): list[str]: names of the changed files. """ if diffbase: - command = 'git diff --name-only {0:s}'.format(diffbase) + command = f'git diff --name-only {diffbase:s}' else: command = 'git ls-files' diff --git a/l2tdevtools/review_helpers/review.py b/l2tdevtools/review_helpers/review.py index b5833e18..eb2eb9af 100644 --- a/l2tdevtools/review_helpers/review.py +++ b/l2tdevtools/review_helpers/review.py @@ -181,7 +181,7 @@ def Lint(self): if self._all_files: diffbase = None else: - diffbase = 'origin/main' + diffbase = 'main' changed_python_files = self._git_helper.GetChangedPythonFiles( diffbase=diffbase) diff --git a/l2tdevtools/spec_file.py b/l2tdevtools/spec_file.py index dea13ba2..9550079e 100644 --- a/l2tdevtools/spec_file.py +++ b/l2tdevtools/spec_file.py @@ -2,6 +2,7 @@ """RPM spec file generator.""" import datetime +import glob import os from setuptools.config import setupcfg @@ -32,9 +33,9 @@ class RPMSpecFileGenerator(object): ''] _SPEC_TEMPLATE_PYTHON3_BODY = [ - '%generate_buildrequires', - '%pyproject_buildrequires -R', - '', + '# %generate_buildrequires', + '# %pyproject_buildrequires -R', + '#', '%prep', '%autosetup -p1 -n %{{name}}-%{{version}}', '', @@ -83,6 +84,9 @@ def _GenerateSpecFile( Returns: bool: True if successful, False otherwise. + + Raises: + ValueError: if required configuration values are missing. """ configuration = { 'description': None, @@ -116,8 +120,20 @@ def _GenerateSpecFile( package_name = package_name[7:] if project_definition.description_long: - configuration['description'] = '{0:s}\n\n'.format( - project_definition.description_long) + configuration['description'] = ( + f'{project_definition.description_long:s}\n\n') + + if project_definition.description_short: + configuration['summary'] = project_definition.description_short + + if project_definition.homepage_url: + configuration['url'] = project_definition.homepage_url + + if project_definition.license: + configuration['license'] = project_definition.license + + if project_definition.maintainer: + configuration['vendor'] = project_definition.maintainer # TODO: add support for pyproject.toml # build.util.project_wheel_metadata @@ -146,6 +162,24 @@ def _GenerateSpecFile( if not configuration['vendor']: configuration['vendor'] = setup_cfg_metadata.get('maintainer', None) + if not configuration['version']: + for version_file in glob.glob(os.path.join( + source_directory, '**', 'version.py')): + with open(version_file, 'r', encoding='utf8') as file_object: + for line in file_object: + if '__version__' in line and '=' in line: + version = line.strip().rsplit('=', maxsplit=1)[-1] + configuration['version'] = version.strip().strip('\'').strip('"') + + if not configuration['version']: + for version_file in glob.glob(os.path.join( + source_directory, '**', '__init__.py')): + with open(version_file, 'r', encoding='utf8') as file_object: + for line in file_object: + if '__version__' in line and '=' in line: + version = line.strip().rsplit('=', maxsplit=1)[-1] + configuration['version'] = version.strip().strip('\'').strip('"') + if rpm_build_dependencies: build_requires = rpm_build_dependencies else: @@ -153,12 +187,16 @@ def _GenerateSpecFile( configuration['name'] = project_name + for key, value in configuration.items(): + if value is None: + raise ValueError(f'Missing configuration value: {key:s}') + self._WriteSourcePackageDefinition( output_file_object, source_package_filename, project_definition, build_requires, configuration) if project_name != package_name: - python_package_name = 'python3-{0:s}'.format(package_name) + python_package_name = f'python3-{package_name:s}' else: python_package_name = 'python3-%{name}' @@ -213,9 +251,11 @@ def _GetDocumentationFilesDefinition(self, source_directory): if os.path.exists(doc_file_path): doc_files.append(doc_file) - doc_file_definition = '' - if doc_files: - doc_file_definition = '%doc {0:s}\n'.format(' '.join(doc_files)) + if not doc_files: + doc_file_definition = '' + else: + doc_files = ' '.join(doc_files) + doc_file_definition = f'%doc {doc_files:s}\n' return doc_file_definition diff --git a/tests/download_helpers/pypi.py b/tests/download_helpers/pypi.py index 697bc9bb..029c5a44 100644 --- a/tests/download_helpers/pypi.py +++ b/tests/download_helpers/pypi.py @@ -20,7 +20,7 @@ class PyPIDownloadHelperTest(test_lib.BaseTestCase): _PROJECT_NAME = 'dfvfs' _PROJECT_VERSION = '20240317' - _PYPI_VERSION = '20240115' + _PYPI_VERSION = '20240505' @classmethod def setUpClass(cls): diff --git a/tools/build.py b/tools/build.py index c7163bd8..101a0ebc 100755 --- a/tools/build.py +++ b/tools/build.py @@ -77,11 +77,11 @@ def _BuildProject( return True if not os.path.exists(build_helper_object.LOG_FILENAME): - logging.warning('Build of: {0:s} failed.'.format( - source_helper_object.project_name)) + logging.warning( + f'Build of: {source_helper_object.project_name:s} failed.') else: - log_filename = '{0:s}_{1:s}'.format( - source_helper_object.project_name, build_helper_object.LOG_FILENAME) + log_filename = '_'.join([ + source_helper_object.project_name, build_helper_object.LOG_FILENAME]) # Remove older logfiles if they exists otherwise the rename # fails on Windows. @@ -90,9 +90,8 @@ def _BuildProject( os.rename(build_helper_object.LOG_FILENAME, log_filename) logging.warning(( - 'Build of: {0:s} failed, for more information check ' - '{1:s}').format( - source_helper_object.project_name, log_filename)) + f'Build of: {source_helper_object.project_name:s} failed, for more ' + f'information check {log_filename:s}')) return False @@ -156,8 +155,7 @@ def Build(self, project_definition, distributions=None): return False if os.path.exists(build_helper_object.LOG_FILENAME): - logging.info('Removing: {0:s}'.format( - build_helper_object.LOG_FILENAME)) + logging.info(f'Removing: {build_helper_object.LOG_FILENAME:s}') os.remove(build_helper_object.LOG_FILENAME) return True @@ -523,31 +521,31 @@ def Main(): if undefined_projects: print('') print('Undefined projects:') - for name in undefined_projects: + for name in sorted(undefined_projects): print('\t{0:s}'.format(name)) if configuration_errors: print('') print('Projects with configuration errors:') - for name in configuration_errors: + for name in sorted(configuration_errors): print('\t{0:s}'.format(name)) if failed_downloads: print('') print('Failed downloading:') - for name in failed_downloads: + for name in sorted(failed_downloads): print('\t{0:s}'.format(name)) if missing_build_dependencies: print('') print('Missing build dependencies:') - for dependency in missing_build_dependencies: + for dependency in sorted(missing_build_dependencies): print('\t{0:s}'.format(dependency)) if failed_builds: print('') print('Failed building:') - for name in failed_builds: + for name in sorted(failed_builds): print('\t{0:s}'.format(name)) return (not failed_downloads and not missing_build_dependencies and