From 7f052aa032a27fea334f4ded3629053600857333 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Mon, 1 Jan 2024 12:10:24 -0400 Subject: [PATCH 01/17] fix columns and filters - add formatters to remove milliseconds on dates - add column labels to be used with filters - add new filters for version and build views --- spkrepo/views/admin.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 08d7533..560e553 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -35,6 +35,10 @@ def is_accessible(self): # View column_list = ("username", "email", "roles", "active", "confirmed_at") + column_formatters = { + "confirmed_at": lambda v, c, m, p: m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") + } + # Form form_columns = ("username", "roles", "active") form_overrides = {"password": PasswordField} @@ -143,6 +147,11 @@ def is_accessible(self): return current_user.is_authenticated and current_user.has_role("package_admin") # View + column_labels = { + "package.name": "Package Name", + "path": "Screenshot", + } + def _display(view, context, model, name): return Markup( 'screenshot' @@ -221,6 +230,10 @@ def on_model_delete(self, model): ("insert_date", "insert_date"), ) + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } + # Form form_columns = ("name", "author", "maintainers") form_args = {"name": {"validators": [Regexp(SPK.package_re)]}} @@ -276,21 +289,33 @@ def on_model_delete(self, model): "startable", ) column_labels = { + "package.name": "Package Name", "version_string": "Version", "dependencies": "Dependencies", "service_dependencies": "Services", } - column_filters = ("package.name", "version", "upstream_version") + column_filters = ( + "package.name", + "upstream_version", + "version", + "beta", + "all_builds_active", + ) column_sortable_list = ( ("package", "package.name"), ("upstream_version", "upstream_version"), ("version", "version"), + ("beta", "beta"), ("insert_date", "insert_date"), + ("all_builds_active", "all_builds_active"), ("install_wizard", "install_wizard"), ("upgrade_wizard", "upgrade_wizard"), ("startable", "startable"), ) - # TODO: Add beta and all_builds_active with Flask-Admin>1.0.8 + + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } column_default_sort = (Version.insert_date, True) # Custom queries @@ -516,16 +541,24 @@ def can_unsign(self): ) column_labels = { "version.package": "Package", + "version.package.name": "Package Name", "version.upstream_version": "Upstream Version", "version.version": "Version", + "architectures.code": "Architecture", + "firmware.version": "Firmware Version", + "publisher.username": "Publisher Username", } column_filters = ( "version.package.name", "version.upstream_version", "version.version", + "architectures.code", + "firmware.version", "publisher.username", + "active", ) column_sortable_list = ( + ("version.package", "version.package.name"), ("version.upstream_version", "version.upstream_version"), ("version.version", "version.version"), ("firmware", "firmware.build"), @@ -533,7 +566,10 @@ def can_unsign(self): ("insert_date", "insert_date"), ("active", "active"), ) - # TODO: Add version.package with Flask-Admin>1.0.8 + + column_formatters = { + "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S") + } column_default_sort = (Build.insert_date, True) # Custom queries From c21541a6c8b7693c58f82f05f50bc3d1532fa0e8 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Tue, 2 Jan 2024 18:35:35 -0400 Subject: [PATCH 02/17] fix screenshot management - remove edit function - fix file removal with row deletion --- spkrepo/views/admin.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 560e553..955861d 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -124,18 +124,6 @@ def screenshot_namegen(obj, file_data): return os.path.join(obj.package.name, pattern % (i, ext)) -# TODO: Not necessary with Flask-Admin>1.0.8 -# see https://github.com/mrjoes/flask-admin/pull/705 -class SpkrepoImageUploadField(ImageUploadField): - def _get_path(self, filename): - if not self.base_path: # pragma: no cover - raise ValueError("FileUploadField field requires base_path to be set.") - - if callable(self.base_path): - return os.path.join(self.base_path(), filename) - return os.path.join(self.base_path, filename) # pragma: no cover - - class ScreenshotView(ModelView): """View for :class:`~spkrepo.models.Screenshot`""" @@ -146,6 +134,8 @@ def __init__(self, **kwargs): def is_accessible(self): return current_user.is_authenticated and current_user.has_role("package_admin") + can_edit = False + # View column_labels = { "package.name": "Package Name", @@ -163,8 +153,14 @@ def _display(view, context, model, name): column_default_sort = (Package.name, True) column_filters = ("package.name",) + # Hooks + def on_model_delete(self, model): + build_path = os.path.join(current_app.config["DATA_PATH"], model.path) + if os.path.exists(build_path): + os.remove(build_path) + # Form - form_overrides = {"path": SpkrepoImageUploadField} + form_overrides = {"path": ImageUploadField} form_args = { "path": { "label": "Screenshot", From e48a3c4f7d565d802854a6bbb0fe0964171fb019 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Tue, 2 Jan 2024 19:04:48 -0400 Subject: [PATCH 03/17] fix Babel initialisation --- .flake8 | 2 +- spkrepo/app.py | 9 ++++----- spkrepo/ext.py | 4 ++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.flake8 b/.flake8 index 487c389..0e544b6 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 88 extend-ignore = E203 -per-file-ignores = __init__.py:F401 spkrepo/app.py:F841 +per-file-ignores = __init__.py:F401 exclude = docs/* migrations/* diff --git a/spkrepo/app.py b/spkrepo/app.py index 8477edb..ee63518 100644 --- a/spkrepo/app.py +++ b/spkrepo/app.py @@ -2,12 +2,11 @@ import jinja2 from flask import Flask from flask_admin import Admin -from flask_babel import Babel from wtforms import HiddenField from . import config as default_config from .cli import spkrepo as spkrepo_cli -from .ext import cache, db, debug_toolbar, mail, migrate, security +from .ext import babel, cache, db, debug_toolbar, mail, migrate, security from .models import user_datastore from .views import ( ArchitectureView, @@ -59,12 +58,12 @@ def create_app(config=None, register_blueprints=True, init_admin=True): admin.add_view(BuildView()) admin.init_app(app) - # Initialize Flask-Babel - babel = Babel(app) - # Commands app.cli.add_command(spkrepo_cli) + # Flask-Babel + babel.init_app(app) + # SQLAlchemy db.init_app(app) diff --git a/spkrepo/ext.py b/spkrepo/ext.py index a029bd7..06360a9 100644 --- a/spkrepo/ext.py +++ b/spkrepo/ext.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from flask_babel import Babel from flask_caching import Cache from flask_debugtoolbar import DebugToolbarExtension from flask_mail import Mail @@ -6,6 +7,9 @@ from flask_security import Security from flask_sqlalchemy import SQLAlchemy +# Flask-Babel +babel = Babel() + # Cache cache = Cache() From ec69e1ea3819d543b15bfc786c735c3dab86c3b0 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Wed, 3 Jan 2024 00:34:48 -0400 Subject: [PATCH 04/17] add service view and api check - add service view to admin interface - add service check on package upload --- spkrepo/app.py | 2 ++ spkrepo/views/__init__.py | 1 + spkrepo/views/admin.py | 26 +++++++++++++++++++++++++- spkrepo/views/api.py | 10 ++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/spkrepo/app.py b/spkrepo/app.py index ee63518..55083ea 100644 --- a/spkrepo/app.py +++ b/spkrepo/app.py @@ -15,6 +15,7 @@ IndexView, PackageView, ScreenshotView, + ServiceView, SpkrepoConfirmRegisterForm, UserView, VersionView, @@ -52,6 +53,7 @@ def create_app(config=None, register_blueprints=True, init_admin=True): admin.add_view(UserView()) admin.add_view(ArchitectureView()) admin.add_view(FirmwareView()) + admin.add_view(ServiceView()) admin.add_view(ScreenshotView()) admin.add_view(PackageView()) admin.add_view(VersionView()) diff --git a/spkrepo/views/__init__.py b/spkrepo/views/__init__.py index 2312443..8425c5b 100644 --- a/spkrepo/views/__init__.py +++ b/spkrepo/views/__init__.py @@ -6,6 +6,7 @@ IndexView, PackageView, ScreenshotView, + ServiceView, UserView, VersionView, ) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 955861d..943ecff 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -16,7 +16,16 @@ from wtforms.validators import Regexp from ..ext import db -from ..models import Architecture, Build, Firmware, Package, Screenshot, User, Version +from ..models import ( + Architecture, + Build, + Firmware, + Package, + Screenshot, + Service, + User, + Version, +) from ..utils import SPK @@ -111,6 +120,21 @@ def is_accessible(self): can_delete = False +class ServiceView(ModelView): + """View for :class:`~spkrepo.models.Service`""" + + def __init__(self, **kwargs): + super(ServiceView, self).__init__(Service, db.session, **kwargs) + + # Permissions + def is_accessible(self): + return current_user.is_authenticated and current_user.has_role("package_admin") + + can_edit = False + + can_delete = False + + def screenshot_namegen(obj, file_data): pattern = "screenshot_%0d%s" ext = os.path.splitext(file_data.filename)[1] diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py index 5d4565d..9580bfd 100644 --- a/spkrepo/views/api.py +++ b/spkrepo/views/api.py @@ -130,6 +130,16 @@ def post(self): if firmware is None: abort(422, message="Unknown firmware") + # Services + input_install_dep_services = spk.info.get("install_dep_services", None) + if input_install_dep_services: + for info_dep_service in input_install_dep_services.split(): + service_name = Service.find(info_dep_service) + if service_name is None: + abort( + 422, message="Unknown dependent service: %s" % info_dep_service + ) + # Package create_package = False package = Package.find(spk.info["package"]) From bd613dca0c7ff64bd0dcb79df3c93a62e1d454cc Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:13:05 -0400 Subject: [PATCH 05/17] add and maintain md5 hashes - add a functions to get md5 hash in build model and spk - use function in build model when uploading build - use function in spk when signing/unsigning spk --- spkrepo/models.py | 19 +++++++++++++++++++ spkrepo/tests/common.py | 3 +-- spkrepo/utils.py | 12 ++++++++++++ spkrepo/views/admin.py | 12 ++++++++++++ spkrepo/views/api.py | 2 ++ 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/spkrepo/models.py b/spkrepo/models.py index 6394480..c94dc53 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import hashlib import io import os import shutil @@ -360,6 +361,24 @@ def save(self, stream): ) as f: f.write(stream.read()) + def calculate_md5(self): + if not self.path: + raise ValueError("Path cannot be empty.") + + file_path = os.path.join(current_app.config["DATA_PATH"], self.path) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found at path: {file_path}") + + if self.md5 is None: + with io.open(file_path, "rb") as f: + md5_hash = hashlib.md5() + for chunk in iter(lambda: f.read(4096), b""): + md5_hash.update(chunk) + return md5_hash.hexdigest() + + return self.md5 + def _after_insert(self): assert os.path.exists(os.path.join(current_app.config["DATA_PATH"], self.path)) diff --git a/spkrepo/tests/common.py b/spkrepo/tests/common.py index cb43261..1682b9b 100644 --- a/spkrepo/tests/common.py +++ b/spkrepo/tests/common.py @@ -281,8 +281,7 @@ def create_spk(self, create, extracted, **kwargs): with create_spk(self) as spk_stream: self.save(spk_stream) if self.md5 is None: - spk_stream.seek(0) - self.md5 = hashlib.md5(spk_stream.read()).hexdigest() + self.md5 = self.calculate_md5() spk_stream.close() @classmethod diff --git a/spkrepo/utils.py b/spkrepo/utils.py index c5d4043..aa0b0ab 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -345,6 +345,18 @@ def unsign(self): self.stream.truncate() self.stream.seek(0) + def calculate_md5(self): + md5_hash = hashlib.md5() + + # Ensure the stream position is at the beginning + self.stream.seek(0) + + # Update MD5 hash directly from the stream + for chunk in iter(lambda: self.stream.read(4096), b""): + md5_hash.update(chunk) + + return md5_hash.hexdigest() + def _generate_signature(self, stream, timestamp_url, gnupghome): # pragma: no cover # generate the signature gpg = gnupg.GPG(gnupghome=gnupghome) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 943ecff..0b096de 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -428,8 +428,11 @@ def action_sign(self, ids): current_app.config["GNUPG_TIMESTAMP_URL"], current_app.config["GNUPG_PATH"], ) + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -479,8 +482,11 @@ def action_unsign(self, ids): continue try: spk.unsign() + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -683,8 +689,11 @@ def action_sign(self, ids): current_app.config["GNUPG_TIMESTAMP_URL"], current_app.config["GNUPG_PATH"], ) + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: @@ -733,8 +742,11 @@ def action_unsign(self, ids): continue try: spk.unsign() + build.md5 = spk.calculate_md5() + self.session.commit() success.append(filename) except Exception: + self.session.rollback() failed.append(filename) if failed: if len(failed) == 1: diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py index 9580bfd..593a73f 100644 --- a/spkrepo/views/api.py +++ b/spkrepo/views/api.py @@ -278,6 +278,8 @@ def post(self): for size, icon in build.version.icons.items(): icon.save(spk.icons[size]) build.save(spk.stream) + # generate md5 hash + build.md5 = build.calculate_md5() except Exception as e: # pragma: no cover if create_package: shutil.rmtree(os.path.join(data_path, package.name), ignore_errors=True) From 6cc2ef7f55db0721e3060259629a622707c14b04 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Fri, 5 Jan 2024 07:52:43 -0400 Subject: [PATCH 06/17] remove builds from architecture create form --- spkrepo/views/admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 0b096de..1433246 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -104,6 +104,9 @@ def is_accessible(self): can_delete = False + # Form + form_excluded_columns = "builds" + class FirmwareView(ModelView): """View for :class:`~spkrepo.models.Firmware`""" From 63cebf8889e205e0d0f2a8bd315f6246a4b90a32 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Tue, 9 Jan 2024 13:32:47 -0400 Subject: [PATCH 07/17] fix build environment and model - fix alembic setup environment - update model to match database - fix default sort for screenshot --- migrations/alembic.ini | 2 +- migrations/env.py | 5 +++++ migrations/versions/dc7687894ba7_increase_field_sizes.py | 4 ---- spkrepo/models.py | 2 +- spkrepo/views/admin.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/migrations/alembic.ini b/migrations/alembic.ini index 7eae6d2..ea28df6 100644 --- a/migrations/alembic.ini +++ b/migrations/alembic.ini @@ -1,5 +1,5 @@ [alembic] -script_location = . +script_location = ./migrations [loggers] keys = root,sqlalchemy,alembic diff --git a/migrations/env.py b/migrations/env.py index 2127eba..6e177d5 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -5,9 +5,14 @@ from flask import current_app from sqlalchemy import engine_from_config, pool +from spkrepo import create_app + config = context.config fileConfig(config.config_file_name) +# Set up the Flask application context +app = create_app() # Create the Flask app +app.app_context().push() # Push the app context config.set_main_option( "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") diff --git a/migrations/versions/dc7687894ba7_increase_field_sizes.py b/migrations/versions/dc7687894ba7_increase_field_sizes.py index c4209fe..b1bb233 100644 --- a/migrations/versions/dc7687894ba7_increase_field_sizes.py +++ b/migrations/versions/dc7687894ba7_increase_field_sizes.py @@ -13,7 +13,6 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "version", "conf_dependencies", @@ -42,11 +41,9 @@ def upgrade(): type_=sa.UnicodeText(), existing_nullable=True, ) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "version", "conf_resource", @@ -75,4 +72,3 @@ def downgrade(): type_=sa.VARCHAR(length=255), existing_nullable=True, ) - # ### end Alembic commands ### diff --git a/spkrepo/models.py b/spkrepo/models.py index c94dc53..f61f24c 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -329,7 +329,7 @@ class Build(db.Model): publisher_user_id = db.Column(db.Integer, db.ForeignKey("user.id")) checksum = db.Column(db.Unicode(32)) extract_size = db.Column(db.Integer) - path = db.Column(db.Unicode(100)) + path = db.Column(db.Unicode(2048)) md5 = db.Column(db.Unicode(32)) insert_date = db.Column(db.DateTime, default=db.func.now(), nullable=False) active = db.Column(db.Boolean(), default=False, nullable=False) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 1433246..5da8ead 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -177,7 +177,7 @@ def _display(view, context, model, name): column_formatters = {"path": _display} column_sortable_list = (("package", "package.name"),) - column_default_sort = (Package.name, True) + column_default_sort = "package.name" column_filters = ("package.name",) # Hooks From 3096f24a52f44d9d7e05e551a343898a38406b2b Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Tue, 9 Jan 2024 19:59:09 -0400 Subject: [PATCH 08/17] Update nas test for major_version check --- spkrepo/tests/test_nas.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py index 123140b..eff207b 100644 --- a/spkrepo/tests/test_nas.py +++ b/spkrepo/tests/test_nas.py @@ -227,11 +227,11 @@ def test_stable_build_active_stable(self): catalog[0], build, data, dict(arch="88f628x", build="1594") ) - def test_stable_build_active_stable_5004(self): + def test_stable_noarch_build_active_stable_5004(self): build = BuildFactory( active=True, version__report_url=None, - architectures=[Architecture.find("88f6281", syno=True)], + architectures=[Architecture.find("noarch", syno=True)], firmware=Firmware.find(1594), ) db.session.commit() @@ -247,6 +247,23 @@ def test_stable_build_active_stable_5004(self): catalog["packages"][0], build, data, dict(arch="88f628x", build="5004") ) + def test_stable_arch_build_active_stable_5004(self): + BuildFactory( + active=True, + version__report_url=None, + architectures=[Architecture.find("88f6281", syno=True)], + firmware=Firmware.find(1594), + ) + db.session.commit() + data = dict(arch="88f6281", build="5004", language="enu") + response = self.client.post(url_for("nas.catalog"), data=data) + self.assert200(response) + self.assertHeader(response, "Content-Type", "application/json") + catalog = json.loads(response.data.decode()) + self.assertIn("packages", catalog) + self.assertIn("keyrings", catalog) + self.assertEqual(len(catalog["packages"]), 0) + def test_stable_build_active_stable_download_count(self): package = PackageFactory() build = BuildFactory( From f03de30a9cadcb6bcd236d2e4eaed3a00685917e Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:29:31 -0400 Subject: [PATCH 09/17] add major_version check - identify the major DSM version based on a closest match to the build - filter package versions based on match to major DSM version - include earlier "noarch" package version when major DSM version < 6 --- spkrepo/views/nas.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index 3db8f56..f7d9cf0 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -44,12 +44,28 @@ def is_valid_language(language): @cache.memoize(timeout=600) def get_catalog(arch, build, language, beta): - # latest version per package + # Find the closest matching firmware for the provided build + closest_firmware = ( + Firmware.query.filter(Firmware.build <= build) + .order_by(Firmware.build.desc()) + .first() + ) + + # Extract major version from the closest matching firmware + major_version = ( + int(closest_firmware.version.split(".")[0]) + if closest_firmware and closest_firmware.version + else None + ) + + # latest version per package and major version latest_version = db.session.query( Version.package_id, db.func.max(Version.version).label("latest_version") ).select_from(Version) + if not beta: latest_version = latest_version.filter(Version.report_url.is_(None)) + latest_version = ( latest_version.join(Build) .filter(Build.active) @@ -57,6 +73,19 @@ def get_catalog(arch, build, language, beta): .filter(Architecture.code.in_(["noarch", arch])) .join(Build.firmware) .filter(Firmware.build <= build) + .filter( + db.or_( + # Check if major_version is not None before applying the filter + (major_version is not None) + and Firmware.version.startswith(f"{major_version}."), + # Include earlier "noarch" version when major_version < 6 + db.and_( + Architecture.code == "noarch", + (major_version is not None) and (major_version < 6), + Firmware.version.startswith("3."), + ), + ) + ) .group_by(Version.package_id) ).subquery() From bea771851e8789d152a0bbc22cdd75ab0ce902f6 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Thu, 11 Jan 2024 17:05:45 -0400 Subject: [PATCH 10/17] fix download counter - rewrite catalog download links using md5 hashes for id - allow downloads of noarch builds to pass arch checks - rewrite nas tests for new url structure --- spkrepo/tests/test_nas.py | 36 ++++++++++++++++++------------------ spkrepo/views/nas.py | 37 +++++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py index eff207b..deb79ae 100644 --- a/spkrepo/tests/test_nas.py +++ b/spkrepo/tests/test_nas.py @@ -560,9 +560,9 @@ def test_generic(self): response = self.client.get( url_for( "nas.download", - architecture_id=architecture.id, - firmware_build=4458, - build_id=build.id, + md5=build.md5, + arch=architecture.code, + build=4458, ), environ_base={"REMOTE_ADDR": "127.0.0.1"}, headers={"User-Agent": "My User Agent"}, @@ -592,9 +592,9 @@ def test_wrong_build(self): response = self.client.get( url_for( "nas.download", - architecture_id=architecture.id, - firmware_build=4458, - build_id=build.id + 1, + md5=build.md5 + "1", + arch=architecture.code, + build=4458, ) ) self.assert404(response) @@ -613,9 +613,9 @@ def test_inactive_build(self): response = self.client.get( url_for( "nas.download", - architecture_id=architecture.id, - firmware_build=4458, - build_id=build.id, + md5=build.md5, + arch=architecture.code, + build=4458, ) ) self.assert403(response) @@ -634,9 +634,9 @@ def test_wrong_architecture(self): response = self.client.get( url_for( "nas.download", - architecture_id=10, - firmware_build=4458, - build_id=build.id, + md5=build.md5, + arch=Architecture.find(10).code if Architecture.find(10) else "", + build=4458, ) ) self.assert404(response) @@ -655,9 +655,9 @@ def test_incorrect_architecture(self): response = self.client.get( url_for( "nas.download", - architecture_id=Architecture.find("cedarview").id, - firmware_build=4458, - build_id=build.id, + md5=build.md5, + arch=Architecture.find("cedarview").code, + build=4458, ) ) self.assert400(response) @@ -676,9 +676,9 @@ def test_incorrect_firmware_build(self): response = self.client.get( url_for( "nas.download", - architecture_id=architecture.id, - firmware_build=1593, - build_id=build.id, + md5=build.md5, + arch=architecture.code, + build=1593, ) ) self.assert400(response) diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index f7d9cf0..fc0c880 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -144,6 +144,10 @@ def get_catalog(arch, build, language, beta): # fill the catalog entries = [] for b in latest_build.all(): + # correct any missing md5 hashes + if b.md5 is None: + b.md5 = b.calculate_md5() + db.session.commit() entry = { "package": b.version.package.name, "version": b.version.version_string, @@ -154,7 +158,11 @@ def get_catalog(arch, build, language, beta): language, b.version.descriptions["enu"] ).description, "link": url_for( - ".data", path=b.path, arch=arch, build=build, _external=True + ".download", + md5=b.md5, + arch=arch, + build=build, + _external=True, ), "thumbnail": [ url_for(".data", path=icon.path, _external=True) @@ -251,25 +259,34 @@ def catalog(): return Response(json.dumps(catalog), mimetype="application/json") -@nas.route("/download///") -def download(architecture_id, firmware_build, build_id): +@nas.route("/download///") +def download(md5, arch, build): # check build - build = Build.query.get_or_404(build_id) - if not build.active: + build_obj = Build.query.filter_by(md5=md5).one_or_none() + + if build_obj is None: + abort(404) + elif not build_obj.active: abort(403) # architecture - architecture = Architecture.query.get_or_404(architecture_id) + architecture = Architecture.query.filter_by(code=arch).one_or_none() + if architecture is None: + abort(404) # check consistency - if architecture not in build.architectures or firmware_build < build.firmware.build: + elif build < build_obj.firmware.build: + abort(400) + elif architecture not in build_obj.architectures and not any( + arch.code == "noarch" for arch in build_obj.architectures + ): abort(400) # insert in database download = Download( - build=build, + build=build_obj, architecture=architecture, - firmware_build=firmware_build, + firmware_build=build, ip_address=request.remote_addr, user_agent=request.user_agent.string, ) @@ -277,7 +294,7 @@ def download(architecture_id, firmware_build, build_id): db.session.commit() # redirect - return redirect(url_for(".data", path=build.path)) + return redirect(url_for(".data", path=build_obj.path)) @nas.route("/") From db26f75a96849e32acf3d380b318bb4ea9646e16 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Thu, 11 Jan 2024 18:15:48 -0400 Subject: [PATCH 11/17] Fix depopulate db function --- spkrepo/cli.py | 9 ++++++++- spkrepo/models.py | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spkrepo/cli.py b/spkrepo/cli.py index 7c23e70..ac43884 100644 --- a/spkrepo/cli.py +++ b/spkrepo/cli.py @@ -180,8 +180,15 @@ def depopulate_db(): from spkrepo.models import Package for package in Package.query.all(): - shutil.rmtree(os.path.join(current_app.config["DATA_PATH"], package.name)) + # Delete the package and its associated versions and builds db.session.delete(package) + + # Remove the directory associated with the package (if it exists) + shutil.rmtree( + os.path.join(current_app.config["DATA_PATH"], package.name), + ignore_errors=True, + ) + db.session.commit() diff --git a/spkrepo/models.py b/spkrepo/models.py index f61f24c..0d0c294 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -344,7 +344,11 @@ class Build(db.Model): ) firmware = db.relationship("Firmware", lazy=False) publisher = db.relationship("User", foreign_keys=[publisher_user_id]) - downloads = db.relationship("Download", back_populates="build") + downloads = db.relationship( + "Download", + back_populates="build", + cascade="save-update, merge, delete, delete-orphan", + ) @classmethod def generate_filename(cls, package, version, firmware, architectures): From 73981ae9bf4f249e60f97bc1bb294c5cbcf96347 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Fri, 12 Jan 2024 15:30:22 -0400 Subject: [PATCH 12/17] Revert "fix download counter" This reverts commit bea771851e8789d152a0bbc22cdd75ab0ce902f6. --- spkrepo/tests/test_nas.py | 36 ++++++++++++++++++------------------ spkrepo/views/nas.py | 37 ++++++++++--------------------------- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py index deb79ae..eff207b 100644 --- a/spkrepo/tests/test_nas.py +++ b/spkrepo/tests/test_nas.py @@ -560,9 +560,9 @@ def test_generic(self): response = self.client.get( url_for( "nas.download", - md5=build.md5, - arch=architecture.code, - build=4458, + architecture_id=architecture.id, + firmware_build=4458, + build_id=build.id, ), environ_base={"REMOTE_ADDR": "127.0.0.1"}, headers={"User-Agent": "My User Agent"}, @@ -592,9 +592,9 @@ def test_wrong_build(self): response = self.client.get( url_for( "nas.download", - md5=build.md5 + "1", - arch=architecture.code, - build=4458, + architecture_id=architecture.id, + firmware_build=4458, + build_id=build.id + 1, ) ) self.assert404(response) @@ -613,9 +613,9 @@ def test_inactive_build(self): response = self.client.get( url_for( "nas.download", - md5=build.md5, - arch=architecture.code, - build=4458, + architecture_id=architecture.id, + firmware_build=4458, + build_id=build.id, ) ) self.assert403(response) @@ -634,9 +634,9 @@ def test_wrong_architecture(self): response = self.client.get( url_for( "nas.download", - md5=build.md5, - arch=Architecture.find(10).code if Architecture.find(10) else "", - build=4458, + architecture_id=10, + firmware_build=4458, + build_id=build.id, ) ) self.assert404(response) @@ -655,9 +655,9 @@ def test_incorrect_architecture(self): response = self.client.get( url_for( "nas.download", - md5=build.md5, - arch=Architecture.find("cedarview").code, - build=4458, + architecture_id=Architecture.find("cedarview").id, + firmware_build=4458, + build_id=build.id, ) ) self.assert400(response) @@ -676,9 +676,9 @@ def test_incorrect_firmware_build(self): response = self.client.get( url_for( "nas.download", - md5=build.md5, - arch=architecture.code, - build=1593, + architecture_id=architecture.id, + firmware_build=1593, + build_id=build.id, ) ) self.assert400(response) diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index fc0c880..f7d9cf0 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -144,10 +144,6 @@ def get_catalog(arch, build, language, beta): # fill the catalog entries = [] for b in latest_build.all(): - # correct any missing md5 hashes - if b.md5 is None: - b.md5 = b.calculate_md5() - db.session.commit() entry = { "package": b.version.package.name, "version": b.version.version_string, @@ -158,11 +154,7 @@ def get_catalog(arch, build, language, beta): language, b.version.descriptions["enu"] ).description, "link": url_for( - ".download", - md5=b.md5, - arch=arch, - build=build, - _external=True, + ".data", path=b.path, arch=arch, build=build, _external=True ), "thumbnail": [ url_for(".data", path=icon.path, _external=True) @@ -259,34 +251,25 @@ def catalog(): return Response(json.dumps(catalog), mimetype="application/json") -@nas.route("/download///") -def download(md5, arch, build): +@nas.route("/download///") +def download(architecture_id, firmware_build, build_id): # check build - build_obj = Build.query.filter_by(md5=md5).one_or_none() - - if build_obj is None: - abort(404) - elif not build_obj.active: + build = Build.query.get_or_404(build_id) + if not build.active: abort(403) # architecture - architecture = Architecture.query.filter_by(code=arch).one_or_none() + architecture = Architecture.query.get_or_404(architecture_id) - if architecture is None: - abort(404) # check consistency - elif build < build_obj.firmware.build: - abort(400) - elif architecture not in build_obj.architectures and not any( - arch.code == "noarch" for arch in build_obj.architectures - ): + if architecture not in build.architectures or firmware_build < build.firmware.build: abort(400) # insert in database download = Download( - build=build_obj, + build=build, architecture=architecture, - firmware_build=build, + firmware_build=firmware_build, ip_address=request.remote_addr, user_agent=request.user_agent.string, ) @@ -294,7 +277,7 @@ def download(md5, arch, build): db.session.commit() # redirect - return redirect(url_for(".data", path=build_obj.path)) + return redirect(url_for(".data", path=build.path)) @nas.route("/") From ad1cf1772d925b6ee7e7c1bdb83db7f85360755c Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:31:41 -0400 Subject: [PATCH 13/17] Amend major_version check - Add type column to Firmware table - Increase length of version column - Filter by type for closest firmware when getting catalog - Update populate_db with firmware type --- ...add_firmware_type_and_increase_version_.py | 47 +++++++++++++++++++ spkrepo/models.py | 3 +- spkrepo/utils.py | 5 +- spkrepo/views/nas.py | 2 +- 4 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py diff --git a/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py new file mode 100644 index 0000000..564122c --- /dev/null +++ b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py @@ -0,0 +1,47 @@ +"""Add firmware type and increase version length + +Revision ID: f95855ce9471 +Revises: 76d559b4e873 +Create Date: 2024-01-15 13:58:34.160242 + +""" +revision = "f95855ce9471" +down_revision = "76d559b4e873" + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + op.add_column("firmware", sa.Column("type", sa.Unicode(length=4))) + # Set type based on version + op.execute( + """ + UPDATE firmware + SET type = CASE + WHEN version LIKE '1.%' THEN 'srm' + ELSE 'dsm' + END + """ + ) + # Modify the column to be NOT NULL after setting the values + op.alter_column("firmware", "type", nullable=False) + + op.alter_column( + "firmware", + "version", + existing_type=sa.VARCHAR(length=3), + type_=sa.Unicode(length=4), + existing_nullable=False, + ) + + +def downgrade(): + op.alter_column( + "firmware", + "version", + existing_type=sa.Unicode(length=4), + type_=sa.VARCHAR(length=3), + existing_nullable=False, + ) + op.drop_column("firmware", "type") diff --git a/spkrepo/models.py b/spkrepo/models.py index 0d0c294..2cc0613 100644 --- a/spkrepo/models.py +++ b/spkrepo/models.py @@ -134,8 +134,9 @@ class Firmware(db.Model): # Columns id = db.Column(db.Integer, primary_key=True) - version = db.Column(db.Unicode(3), nullable=False) + version = db.Column(db.Unicode(4), nullable=False) build = db.Column(db.Integer, unique=True, nullable=False) + type = db.Column(db.Unicode(4), nullable=False) @classmethod def find(cls, build): diff --git a/spkrepo/utils.py b/spkrepo/utils.py index aa0b0ab..d55abbd 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -398,7 +398,10 @@ def populate_db(): ) db.session.execute( Firmware.__table__.insert().values( - [{"version": "3.1", "build": 1594}, {"version": "5.0", "build": 4458}] + [ + {"version": "3.1", "build": 1594, "type": "dsm"}, + {"version": "5.0", "build": 4458, "type": "dsm"}, + ] ) ) db.session.execute( diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py index f7d9cf0..9cbeeed 100644 --- a/spkrepo/views/nas.py +++ b/spkrepo/views/nas.py @@ -46,7 +46,7 @@ def is_valid_language(language): def get_catalog(arch, build, language, beta): # Find the closest matching firmware for the provided build closest_firmware = ( - Firmware.query.filter(Firmware.build <= build) + Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm") .order_by(Firmware.build.desc()) .first() ) From 2bc6a2950fa47037e56b2a67e419b5901961a5e9 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:04:48 -0400 Subject: [PATCH 14/17] add validators for firmware input --- spkrepo/utils.py | 4 ++++ spkrepo/views/admin.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/spkrepo/utils.py b/spkrepo/utils.py index d55abbd..b0b0656 100644 --- a/spkrepo/utils.py +++ b/spkrepo/utils.py @@ -58,6 +58,10 @@ class SPK(object): #: Regex for files in conf conf_filename_re = re.compile(r"^conf/.+$") + #: Regex for firmware input + firmware_version_re = re.compile(r"^\d+\.\d$") + firmware_type_re = re.compile(r"^([a-z]){3,}$") + def __init__(self, stream): self.info = {} self.icons = {} diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index 5da8ead..c52800b 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -122,6 +122,13 @@ def is_accessible(self): can_delete = False + # Form + form_columns = ("version", "build", "type") + form_args = { + "version": {"validators": [Regexp(SPK.firmware_version_re)]}, + "type": {"validators": [Regexp(SPK.firmware_type_re)]}, + } + class ServiceView(ModelView): """View for :class:`~spkrepo.models.Service`""" From 0159658cb15f3cff80fb34f5b3c89d90cd55bec0 Mon Sep 17 00:00:00 2001 From: publicarray Date: Sun, 25 Feb 2024 10:52:55 +1100 Subject: [PATCH 15/17] fix Popped wrong app context. --- migrations/env.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/migrations/env.py b/migrations/env.py index 6e177d5..3273e03 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -5,15 +5,9 @@ from flask import current_app from sqlalchemy import engine_from_config, pool -from spkrepo import create_app - config = context.config fileConfig(config.config_file_name) -# Set up the Flask application context -app = create_app() # Create the Flask app -app.app_context().push() # Push the app context - config.set_main_option( "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") ) From 95fa3c6e73db9651e2c935c64e439b44feeadb26 Mon Sep 17 00:00:00 2001 From: publicarray Date: Sun, 25 Feb 2024 11:52:32 +1100 Subject: [PATCH 16/17] fix 500 page on users: 'NoneType' object has no attribute 'strftime' BaseModelView.index_view() got an unexpected keyword argument 'cls' Flask-SQLAlchemy uses a custom formatter to handle date formatting when displaying data in the templates. It dosn't support overriding the value. --- spkrepo/views/admin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index c52800b..f97fea5 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -44,10 +44,6 @@ def is_accessible(self): # View column_list = ("username", "email", "roles", "active", "confirmed_at") - column_formatters = { - "confirmed_at": lambda v, c, m, p: m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") - } - # Form form_columns = ("username", "roles", "active") form_overrides = {"password": PasswordField} From 5345bc728901c06525609ae83cf589f87037f3d3 Mon Sep 17 00:00:00 2001 From: mreid-tt <943378+mreid-tt@users.noreply.github.com> Date: Sun, 25 Feb 2024 05:57:46 -0400 Subject: [PATCH 17/17] fix column formatting --- spkrepo/views/admin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py index f97fea5..d4af9b9 100644 --- a/spkrepo/views/admin.py +++ b/spkrepo/views/admin.py @@ -44,6 +44,12 @@ def is_accessible(self): # View column_list = ("username", "email", "roles", "active", "confirmed_at") + column_formatters = { + "confirmed_at": lambda v, c, m, p: ( + m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") if m.confirmed_at else None + ) + } + # Form form_columns = ("username", "roles", "active") form_overrides = {"password": PasswordField}