From 7f052aa032a27fea334f4ded3629053600857333 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Mon, 1 Jan 2024 12:10:24 -0400
Subject: [PATCH 01/17] fix columns and filters
- add formatters to remove milliseconds on dates
- add column labels to be used with filters
- add new filters for version and build views
---
spkrepo/views/admin.py | 42 +++++++++++++++++++++++++++++++++++++++---
1 file changed, 39 insertions(+), 3 deletions(-)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 08d7533..560e553 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -35,6 +35,10 @@ def is_accessible(self):
# View
column_list = ("username", "email", "roles", "active", "confirmed_at")
+ column_formatters = {
+ "confirmed_at": lambda v, c, m, p: m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S")
+ }
+
# Form
form_columns = ("username", "roles", "active")
form_overrides = {"password": PasswordField}
@@ -143,6 +147,11 @@ def is_accessible(self):
return current_user.is_authenticated and current_user.has_role("package_admin")
# View
+ column_labels = {
+ "package.name": "Package Name",
+ "path": "Screenshot",
+ }
+
def _display(view, context, model, name):
return Markup(
'
'
@@ -221,6 +230,10 @@ def on_model_delete(self, model):
("insert_date", "insert_date"),
)
+ column_formatters = {
+ "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
+ }
+
# Form
form_columns = ("name", "author", "maintainers")
form_args = {"name": {"validators": [Regexp(SPK.package_re)]}}
@@ -276,21 +289,33 @@ def on_model_delete(self, model):
"startable",
)
column_labels = {
+ "package.name": "Package Name",
"version_string": "Version",
"dependencies": "Dependencies",
"service_dependencies": "Services",
}
- column_filters = ("package.name", "version", "upstream_version")
+ column_filters = (
+ "package.name",
+ "upstream_version",
+ "version",
+ "beta",
+ "all_builds_active",
+ )
column_sortable_list = (
("package", "package.name"),
("upstream_version", "upstream_version"),
("version", "version"),
+ ("beta", "beta"),
("insert_date", "insert_date"),
+ ("all_builds_active", "all_builds_active"),
("install_wizard", "install_wizard"),
("upgrade_wizard", "upgrade_wizard"),
("startable", "startable"),
)
- # TODO: Add beta and all_builds_active with Flask-Admin>1.0.8
+
+ column_formatters = {
+ "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
+ }
column_default_sort = (Version.insert_date, True)
# Custom queries
@@ -516,16 +541,24 @@ def can_unsign(self):
)
column_labels = {
"version.package": "Package",
+ "version.package.name": "Package Name",
"version.upstream_version": "Upstream Version",
"version.version": "Version",
+ "architectures.code": "Architecture",
+ "firmware.version": "Firmware Version",
+ "publisher.username": "Publisher Username",
}
column_filters = (
"version.package.name",
"version.upstream_version",
"version.version",
+ "architectures.code",
+ "firmware.version",
"publisher.username",
+ "active",
)
column_sortable_list = (
+ ("version.package", "version.package.name"),
("version.upstream_version", "version.upstream_version"),
("version.version", "version.version"),
("firmware", "firmware.build"),
@@ -533,7 +566,10 @@ def can_unsign(self):
("insert_date", "insert_date"),
("active", "active"),
)
- # TODO: Add version.package with Flask-Admin>1.0.8
+
+ column_formatters = {
+ "insert_date": lambda v, c, m, p: m.insert_date.strftime("%Y-%m-%d %H:%M:%S")
+ }
column_default_sort = (Build.insert_date, True)
# Custom queries
From c21541a6c8b7693c58f82f05f50bc3d1532fa0e8 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Tue, 2 Jan 2024 18:35:35 -0400
Subject: [PATCH 02/17] fix screenshot management
- remove edit function
- fix file removal with row deletion
---
spkrepo/views/admin.py | 22 +++++++++-------------
1 file changed, 9 insertions(+), 13 deletions(-)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 560e553..955861d 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -124,18 +124,6 @@ def screenshot_namegen(obj, file_data):
return os.path.join(obj.package.name, pattern % (i, ext))
-# TODO: Not necessary with Flask-Admin>1.0.8
-# see https://github.com/mrjoes/flask-admin/pull/705
-class SpkrepoImageUploadField(ImageUploadField):
- def _get_path(self, filename):
- if not self.base_path: # pragma: no cover
- raise ValueError("FileUploadField field requires base_path to be set.")
-
- if callable(self.base_path):
- return os.path.join(self.base_path(), filename)
- return os.path.join(self.base_path, filename) # pragma: no cover
-
-
class ScreenshotView(ModelView):
"""View for :class:`~spkrepo.models.Screenshot`"""
@@ -146,6 +134,8 @@ def __init__(self, **kwargs):
def is_accessible(self):
return current_user.is_authenticated and current_user.has_role("package_admin")
+ can_edit = False
+
# View
column_labels = {
"package.name": "Package Name",
@@ -163,8 +153,14 @@ def _display(view, context, model, name):
column_default_sort = (Package.name, True)
column_filters = ("package.name",)
+ # Hooks
+ def on_model_delete(self, model):
+ build_path = os.path.join(current_app.config["DATA_PATH"], model.path)
+ if os.path.exists(build_path):
+ os.remove(build_path)
+
# Form
- form_overrides = {"path": SpkrepoImageUploadField}
+ form_overrides = {"path": ImageUploadField}
form_args = {
"path": {
"label": "Screenshot",
From e48a3c4f7d565d802854a6bbb0fe0964171fb019 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Tue, 2 Jan 2024 19:04:48 -0400
Subject: [PATCH 03/17] fix Babel initialisation
---
.flake8 | 2 +-
spkrepo/app.py | 9 ++++-----
spkrepo/ext.py | 4 ++++
3 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/.flake8 b/.flake8
index 487c389..0e544b6 100644
--- a/.flake8
+++ b/.flake8
@@ -1,7 +1,7 @@
[flake8]
max-line-length = 88
extend-ignore = E203
-per-file-ignores = __init__.py:F401 spkrepo/app.py:F841
+per-file-ignores = __init__.py:F401
exclude =
docs/*
migrations/*
diff --git a/spkrepo/app.py b/spkrepo/app.py
index 8477edb..ee63518 100644
--- a/spkrepo/app.py
+++ b/spkrepo/app.py
@@ -2,12 +2,11 @@
import jinja2
from flask import Flask
from flask_admin import Admin
-from flask_babel import Babel
from wtforms import HiddenField
from . import config as default_config
from .cli import spkrepo as spkrepo_cli
-from .ext import cache, db, debug_toolbar, mail, migrate, security
+from .ext import babel, cache, db, debug_toolbar, mail, migrate, security
from .models import user_datastore
from .views import (
ArchitectureView,
@@ -59,12 +58,12 @@ def create_app(config=None, register_blueprints=True, init_admin=True):
admin.add_view(BuildView())
admin.init_app(app)
- # Initialize Flask-Babel
- babel = Babel(app)
-
# Commands
app.cli.add_command(spkrepo_cli)
+ # Flask-Babel
+ babel.init_app(app)
+
# SQLAlchemy
db.init_app(app)
diff --git a/spkrepo/ext.py b/spkrepo/ext.py
index a029bd7..06360a9 100644
--- a/spkrepo/ext.py
+++ b/spkrepo/ext.py
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+from flask_babel import Babel
from flask_caching import Cache
from flask_debugtoolbar import DebugToolbarExtension
from flask_mail import Mail
@@ -6,6 +7,9 @@
from flask_security import Security
from flask_sqlalchemy import SQLAlchemy
+# Flask-Babel
+babel = Babel()
+
# Cache
cache = Cache()
From ec69e1ea3819d543b15bfc786c735c3dab86c3b0 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Wed, 3 Jan 2024 00:34:48 -0400
Subject: [PATCH 04/17] add service view and api check
- add service view to admin interface
- add service check on package upload
---
spkrepo/app.py | 2 ++
spkrepo/views/__init__.py | 1 +
spkrepo/views/admin.py | 26 +++++++++++++++++++++++++-
spkrepo/views/api.py | 10 ++++++++++
4 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/spkrepo/app.py b/spkrepo/app.py
index ee63518..55083ea 100644
--- a/spkrepo/app.py
+++ b/spkrepo/app.py
@@ -15,6 +15,7 @@
IndexView,
PackageView,
ScreenshotView,
+ ServiceView,
SpkrepoConfirmRegisterForm,
UserView,
VersionView,
@@ -52,6 +53,7 @@ def create_app(config=None, register_blueprints=True, init_admin=True):
admin.add_view(UserView())
admin.add_view(ArchitectureView())
admin.add_view(FirmwareView())
+ admin.add_view(ServiceView())
admin.add_view(ScreenshotView())
admin.add_view(PackageView())
admin.add_view(VersionView())
diff --git a/spkrepo/views/__init__.py b/spkrepo/views/__init__.py
index 2312443..8425c5b 100644
--- a/spkrepo/views/__init__.py
+++ b/spkrepo/views/__init__.py
@@ -6,6 +6,7 @@
IndexView,
PackageView,
ScreenshotView,
+ ServiceView,
UserView,
VersionView,
)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 955861d..943ecff 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -16,7 +16,16 @@
from wtforms.validators import Regexp
from ..ext import db
-from ..models import Architecture, Build, Firmware, Package, Screenshot, User, Version
+from ..models import (
+ Architecture,
+ Build,
+ Firmware,
+ Package,
+ Screenshot,
+ Service,
+ User,
+ Version,
+)
from ..utils import SPK
@@ -111,6 +120,21 @@ def is_accessible(self):
can_delete = False
+class ServiceView(ModelView):
+ """View for :class:`~spkrepo.models.Service`"""
+
+ def __init__(self, **kwargs):
+ super(ServiceView, self).__init__(Service, db.session, **kwargs)
+
+ # Permissions
+ def is_accessible(self):
+ return current_user.is_authenticated and current_user.has_role("package_admin")
+
+ can_edit = False
+
+ can_delete = False
+
+
def screenshot_namegen(obj, file_data):
pattern = "screenshot_%0d%s"
ext = os.path.splitext(file_data.filename)[1]
diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py
index 5d4565d..9580bfd 100644
--- a/spkrepo/views/api.py
+++ b/spkrepo/views/api.py
@@ -130,6 +130,16 @@ def post(self):
if firmware is None:
abort(422, message="Unknown firmware")
+ # Services
+ input_install_dep_services = spk.info.get("install_dep_services", None)
+ if input_install_dep_services:
+ for info_dep_service in input_install_dep_services.split():
+ service_name = Service.find(info_dep_service)
+ if service_name is None:
+ abort(
+ 422, message="Unknown dependent service: %s" % info_dep_service
+ )
+
# Package
create_package = False
package = Package.find(spk.info["package"])
From bd613dca0c7ff64bd0dcb79df3c93a62e1d454cc Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Wed, 3 Jan 2024 11:13:05 -0400
Subject: [PATCH 05/17] add and maintain md5 hashes
- add a functions to get md5 hash in build model and spk
- use function in build model when uploading build
- use function in spk when signing/unsigning spk
---
spkrepo/models.py | 19 +++++++++++++++++++
spkrepo/tests/common.py | 3 +--
spkrepo/utils.py | 12 ++++++++++++
spkrepo/views/admin.py | 12 ++++++++++++
spkrepo/views/api.py | 2 ++
5 files changed, 46 insertions(+), 2 deletions(-)
diff --git a/spkrepo/models.py b/spkrepo/models.py
index 6394480..c94dc53 100644
--- a/spkrepo/models.py
+++ b/spkrepo/models.py
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+import hashlib
import io
import os
import shutil
@@ -360,6 +361,24 @@ def save(self, stream):
) as f:
f.write(stream.read())
+ def calculate_md5(self):
+ if not self.path:
+ raise ValueError("Path cannot be empty.")
+
+ file_path = os.path.join(current_app.config["DATA_PATH"], self.path)
+
+ if not os.path.exists(file_path):
+ raise FileNotFoundError(f"File not found at path: {file_path}")
+
+ if self.md5 is None:
+ with io.open(file_path, "rb") as f:
+ md5_hash = hashlib.md5()
+ for chunk in iter(lambda: f.read(4096), b""):
+ md5_hash.update(chunk)
+ return md5_hash.hexdigest()
+
+ return self.md5
+
def _after_insert(self):
assert os.path.exists(os.path.join(current_app.config["DATA_PATH"], self.path))
diff --git a/spkrepo/tests/common.py b/spkrepo/tests/common.py
index cb43261..1682b9b 100644
--- a/spkrepo/tests/common.py
+++ b/spkrepo/tests/common.py
@@ -281,8 +281,7 @@ def create_spk(self, create, extracted, **kwargs):
with create_spk(self) as spk_stream:
self.save(spk_stream)
if self.md5 is None:
- spk_stream.seek(0)
- self.md5 = hashlib.md5(spk_stream.read()).hexdigest()
+ self.md5 = self.calculate_md5()
spk_stream.close()
@classmethod
diff --git a/spkrepo/utils.py b/spkrepo/utils.py
index c5d4043..aa0b0ab 100644
--- a/spkrepo/utils.py
+++ b/spkrepo/utils.py
@@ -345,6 +345,18 @@ def unsign(self):
self.stream.truncate()
self.stream.seek(0)
+ def calculate_md5(self):
+ md5_hash = hashlib.md5()
+
+ # Ensure the stream position is at the beginning
+ self.stream.seek(0)
+
+ # Update MD5 hash directly from the stream
+ for chunk in iter(lambda: self.stream.read(4096), b""):
+ md5_hash.update(chunk)
+
+ return md5_hash.hexdigest()
+
def _generate_signature(self, stream, timestamp_url, gnupghome): # pragma: no cover
# generate the signature
gpg = gnupg.GPG(gnupghome=gnupghome)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 943ecff..0b096de 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -428,8 +428,11 @@ def action_sign(self, ids):
current_app.config["GNUPG_TIMESTAMP_URL"],
current_app.config["GNUPG_PATH"],
)
+ build.md5 = spk.calculate_md5()
+ self.session.commit()
success.append(filename)
except Exception:
+ self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -479,8 +482,11 @@ def action_unsign(self, ids):
continue
try:
spk.unsign()
+ build.md5 = spk.calculate_md5()
+ self.session.commit()
success.append(filename)
except Exception:
+ self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -683,8 +689,11 @@ def action_sign(self, ids):
current_app.config["GNUPG_TIMESTAMP_URL"],
current_app.config["GNUPG_PATH"],
)
+ build.md5 = spk.calculate_md5()
+ self.session.commit()
success.append(filename)
except Exception:
+ self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
@@ -733,8 +742,11 @@ def action_unsign(self, ids):
continue
try:
spk.unsign()
+ build.md5 = spk.calculate_md5()
+ self.session.commit()
success.append(filename)
except Exception:
+ self.session.rollback()
failed.append(filename)
if failed:
if len(failed) == 1:
diff --git a/spkrepo/views/api.py b/spkrepo/views/api.py
index 9580bfd..593a73f 100644
--- a/spkrepo/views/api.py
+++ b/spkrepo/views/api.py
@@ -278,6 +278,8 @@ def post(self):
for size, icon in build.version.icons.items():
icon.save(spk.icons[size])
build.save(spk.stream)
+ # generate md5 hash
+ build.md5 = build.calculate_md5()
except Exception as e: # pragma: no cover
if create_package:
shutil.rmtree(os.path.join(data_path, package.name), ignore_errors=True)
From 6cc2ef7f55db0721e3060259629a622707c14b04 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Fri, 5 Jan 2024 07:52:43 -0400
Subject: [PATCH 06/17] remove builds from architecture create form
---
spkrepo/views/admin.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 0b096de..1433246 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -104,6 +104,9 @@ def is_accessible(self):
can_delete = False
+ # Form
+ form_excluded_columns = "builds"
+
class FirmwareView(ModelView):
"""View for :class:`~spkrepo.models.Firmware`"""
From 63cebf8889e205e0d0f2a8bd315f6246a4b90a32 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Tue, 9 Jan 2024 13:32:47 -0400
Subject: [PATCH 07/17] fix build environment and model
- fix alembic setup environment
- update model to match database
- fix default sort for screenshot
---
migrations/alembic.ini | 2 +-
migrations/env.py | 5 +++++
migrations/versions/dc7687894ba7_increase_field_sizes.py | 4 ----
spkrepo/models.py | 2 +-
spkrepo/views/admin.py | 2 +-
5 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/migrations/alembic.ini b/migrations/alembic.ini
index 7eae6d2..ea28df6 100644
--- a/migrations/alembic.ini
+++ b/migrations/alembic.ini
@@ -1,5 +1,5 @@
[alembic]
-script_location = .
+script_location = ./migrations
[loggers]
keys = root,sqlalchemy,alembic
diff --git a/migrations/env.py b/migrations/env.py
index 2127eba..6e177d5 100644
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -5,9 +5,14 @@
from flask import current_app
from sqlalchemy import engine_from_config, pool
+from spkrepo import create_app
+
config = context.config
fileConfig(config.config_file_name)
+# Set up the Flask application context
+app = create_app() # Create the Flask app
+app.app_context().push() # Push the app context
config.set_main_option(
"sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
diff --git a/migrations/versions/dc7687894ba7_increase_field_sizes.py b/migrations/versions/dc7687894ba7_increase_field_sizes.py
index c4209fe..b1bb233 100644
--- a/migrations/versions/dc7687894ba7_increase_field_sizes.py
+++ b/migrations/versions/dc7687894ba7_increase_field_sizes.py
@@ -13,7 +13,6 @@
def upgrade():
- # ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"version",
"conf_dependencies",
@@ -42,11 +41,9 @@ def upgrade():
type_=sa.UnicodeText(),
existing_nullable=True,
)
- # ### end Alembic commands ###
def downgrade():
- # ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"version",
"conf_resource",
@@ -75,4 +72,3 @@ def downgrade():
type_=sa.VARCHAR(length=255),
existing_nullable=True,
)
- # ### end Alembic commands ###
diff --git a/spkrepo/models.py b/spkrepo/models.py
index c94dc53..f61f24c 100644
--- a/spkrepo/models.py
+++ b/spkrepo/models.py
@@ -329,7 +329,7 @@ class Build(db.Model):
publisher_user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
checksum = db.Column(db.Unicode(32))
extract_size = db.Column(db.Integer)
- path = db.Column(db.Unicode(100))
+ path = db.Column(db.Unicode(2048))
md5 = db.Column(db.Unicode(32))
insert_date = db.Column(db.DateTime, default=db.func.now(), nullable=False)
active = db.Column(db.Boolean(), default=False, nullable=False)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 1433246..5da8ead 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -177,7 +177,7 @@ def _display(view, context, model, name):
column_formatters = {"path": _display}
column_sortable_list = (("package", "package.name"),)
- column_default_sort = (Package.name, True)
+ column_default_sort = "package.name"
column_filters = ("package.name",)
# Hooks
From 3096f24a52f44d9d7e05e551a343898a38406b2b Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Tue, 9 Jan 2024 19:59:09 -0400
Subject: [PATCH 08/17] Update nas test for major_version check
---
spkrepo/tests/test_nas.py | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py
index 123140b..eff207b 100644
--- a/spkrepo/tests/test_nas.py
+++ b/spkrepo/tests/test_nas.py
@@ -227,11 +227,11 @@ def test_stable_build_active_stable(self):
catalog[0], build, data, dict(arch="88f628x", build="1594")
)
- def test_stable_build_active_stable_5004(self):
+ def test_stable_noarch_build_active_stable_5004(self):
build = BuildFactory(
active=True,
version__report_url=None,
- architectures=[Architecture.find("88f6281", syno=True)],
+ architectures=[Architecture.find("noarch", syno=True)],
firmware=Firmware.find(1594),
)
db.session.commit()
@@ -247,6 +247,23 @@ def test_stable_build_active_stable_5004(self):
catalog["packages"][0], build, data, dict(arch="88f628x", build="5004")
)
+ def test_stable_arch_build_active_stable_5004(self):
+ BuildFactory(
+ active=True,
+ version__report_url=None,
+ architectures=[Architecture.find("88f6281", syno=True)],
+ firmware=Firmware.find(1594),
+ )
+ db.session.commit()
+ data = dict(arch="88f6281", build="5004", language="enu")
+ response = self.client.post(url_for("nas.catalog"), data=data)
+ self.assert200(response)
+ self.assertHeader(response, "Content-Type", "application/json")
+ catalog = json.loads(response.data.decode())
+ self.assertIn("packages", catalog)
+ self.assertIn("keyrings", catalog)
+ self.assertEqual(len(catalog["packages"]), 0)
+
def test_stable_build_active_stable_download_count(self):
package = PackageFactory()
build = BuildFactory(
From f03de30a9cadcb6bcd236d2e4eaed3a00685917e Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Tue, 9 Jan 2024 17:29:31 -0400
Subject: [PATCH 09/17] add major_version check
- identify the major DSM version based on a closest match to the build
- filter package versions based on match to major DSM version
- include earlier "noarch" package version when major DSM version < 6
---
spkrepo/views/nas.py | 31 ++++++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py
index 3db8f56..f7d9cf0 100644
--- a/spkrepo/views/nas.py
+++ b/spkrepo/views/nas.py
@@ -44,12 +44,28 @@ def is_valid_language(language):
@cache.memoize(timeout=600)
def get_catalog(arch, build, language, beta):
- # latest version per package
+ # Find the closest matching firmware for the provided build
+ closest_firmware = (
+ Firmware.query.filter(Firmware.build <= build)
+ .order_by(Firmware.build.desc())
+ .first()
+ )
+
+ # Extract major version from the closest matching firmware
+ major_version = (
+ int(closest_firmware.version.split(".")[0])
+ if closest_firmware and closest_firmware.version
+ else None
+ )
+
+ # latest version per package and major version
latest_version = db.session.query(
Version.package_id, db.func.max(Version.version).label("latest_version")
).select_from(Version)
+
if not beta:
latest_version = latest_version.filter(Version.report_url.is_(None))
+
latest_version = (
latest_version.join(Build)
.filter(Build.active)
@@ -57,6 +73,19 @@ def get_catalog(arch, build, language, beta):
.filter(Architecture.code.in_(["noarch", arch]))
.join(Build.firmware)
.filter(Firmware.build <= build)
+ .filter(
+ db.or_(
+ # Check if major_version is not None before applying the filter
+ (major_version is not None)
+ and Firmware.version.startswith(f"{major_version}."),
+ # Include earlier "noarch" version when major_version < 6
+ db.and_(
+ Architecture.code == "noarch",
+ (major_version is not None) and (major_version < 6),
+ Firmware.version.startswith("3."),
+ ),
+ )
+ )
.group_by(Version.package_id)
).subquery()
From bea771851e8789d152a0bbc22cdd75ab0ce902f6 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Thu, 11 Jan 2024 17:05:45 -0400
Subject: [PATCH 10/17] fix download counter
- rewrite catalog download links using md5 hashes for id
- allow downloads of noarch builds to pass arch checks
- rewrite nas tests for new url structure
---
spkrepo/tests/test_nas.py | 36 ++++++++++++++++++------------------
spkrepo/views/nas.py | 37 +++++++++++++++++++++++++++----------
2 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py
index eff207b..deb79ae 100644
--- a/spkrepo/tests/test_nas.py
+++ b/spkrepo/tests/test_nas.py
@@ -560,9 +560,9 @@ def test_generic(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=architecture.id,
- firmware_build=4458,
- build_id=build.id,
+ md5=build.md5,
+ arch=architecture.code,
+ build=4458,
),
environ_base={"REMOTE_ADDR": "127.0.0.1"},
headers={"User-Agent": "My User Agent"},
@@ -592,9 +592,9 @@ def test_wrong_build(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=architecture.id,
- firmware_build=4458,
- build_id=build.id + 1,
+ md5=build.md5 + "1",
+ arch=architecture.code,
+ build=4458,
)
)
self.assert404(response)
@@ -613,9 +613,9 @@ def test_inactive_build(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=architecture.id,
- firmware_build=4458,
- build_id=build.id,
+ md5=build.md5,
+ arch=architecture.code,
+ build=4458,
)
)
self.assert403(response)
@@ -634,9 +634,9 @@ def test_wrong_architecture(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=10,
- firmware_build=4458,
- build_id=build.id,
+ md5=build.md5,
+ arch=Architecture.find(10).code if Architecture.find(10) else "",
+ build=4458,
)
)
self.assert404(response)
@@ -655,9 +655,9 @@ def test_incorrect_architecture(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=Architecture.find("cedarview").id,
- firmware_build=4458,
- build_id=build.id,
+ md5=build.md5,
+ arch=Architecture.find("cedarview").code,
+ build=4458,
)
)
self.assert400(response)
@@ -676,9 +676,9 @@ def test_incorrect_firmware_build(self):
response = self.client.get(
url_for(
"nas.download",
- architecture_id=architecture.id,
- firmware_build=1593,
- build_id=build.id,
+ md5=build.md5,
+ arch=architecture.code,
+ build=1593,
)
)
self.assert400(response)
diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py
index f7d9cf0..fc0c880 100644
--- a/spkrepo/views/nas.py
+++ b/spkrepo/views/nas.py
@@ -144,6 +144,10 @@ def get_catalog(arch, build, language, beta):
# fill the catalog
entries = []
for b in latest_build.all():
+ # correct any missing md5 hashes
+ if b.md5 is None:
+ b.md5 = b.calculate_md5()
+ db.session.commit()
entry = {
"package": b.version.package.name,
"version": b.version.version_string,
@@ -154,7 +158,11 @@ def get_catalog(arch, build, language, beta):
language, b.version.descriptions["enu"]
).description,
"link": url_for(
- ".data", path=b.path, arch=arch, build=build, _external=True
+ ".download",
+ md5=b.md5,
+ arch=arch,
+ build=build,
+ _external=True,
),
"thumbnail": [
url_for(".data", path=icon.path, _external=True)
@@ -251,25 +259,34 @@ def catalog():
return Response(json.dumps(catalog), mimetype="application/json")
-@nas.route("/download///")
-def download(architecture_id, firmware_build, build_id):
+@nas.route("/download///")
+def download(md5, arch, build):
# check build
- build = Build.query.get_or_404(build_id)
- if not build.active:
+ build_obj = Build.query.filter_by(md5=md5).one_or_none()
+
+ if build_obj is None:
+ abort(404)
+ elif not build_obj.active:
abort(403)
# architecture
- architecture = Architecture.query.get_or_404(architecture_id)
+ architecture = Architecture.query.filter_by(code=arch).one_or_none()
+ if architecture is None:
+ abort(404)
# check consistency
- if architecture not in build.architectures or firmware_build < build.firmware.build:
+ elif build < build_obj.firmware.build:
+ abort(400)
+ elif architecture not in build_obj.architectures and not any(
+ arch.code == "noarch" for arch in build_obj.architectures
+ ):
abort(400)
# insert in database
download = Download(
- build=build,
+ build=build_obj,
architecture=architecture,
- firmware_build=firmware_build,
+ firmware_build=build,
ip_address=request.remote_addr,
user_agent=request.user_agent.string,
)
@@ -277,7 +294,7 @@ def download(architecture_id, firmware_build, build_id):
db.session.commit()
# redirect
- return redirect(url_for(".data", path=build.path))
+ return redirect(url_for(".data", path=build_obj.path))
@nas.route("/")
From db26f75a96849e32acf3d380b318bb4ea9646e16 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Thu, 11 Jan 2024 18:15:48 -0400
Subject: [PATCH 11/17] Fix depopulate db function
---
spkrepo/cli.py | 9 ++++++++-
spkrepo/models.py | 6 +++++-
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/spkrepo/cli.py b/spkrepo/cli.py
index 7c23e70..ac43884 100644
--- a/spkrepo/cli.py
+++ b/spkrepo/cli.py
@@ -180,8 +180,15 @@ def depopulate_db():
from spkrepo.models import Package
for package in Package.query.all():
- shutil.rmtree(os.path.join(current_app.config["DATA_PATH"], package.name))
+ # Delete the package and its associated versions and builds
db.session.delete(package)
+
+ # Remove the directory associated with the package (if it exists)
+ shutil.rmtree(
+ os.path.join(current_app.config["DATA_PATH"], package.name),
+ ignore_errors=True,
+ )
+
db.session.commit()
diff --git a/spkrepo/models.py b/spkrepo/models.py
index f61f24c..0d0c294 100644
--- a/spkrepo/models.py
+++ b/spkrepo/models.py
@@ -344,7 +344,11 @@ class Build(db.Model):
)
firmware = db.relationship("Firmware", lazy=False)
publisher = db.relationship("User", foreign_keys=[publisher_user_id])
- downloads = db.relationship("Download", back_populates="build")
+ downloads = db.relationship(
+ "Download",
+ back_populates="build",
+ cascade="save-update, merge, delete, delete-orphan",
+ )
@classmethod
def generate_filename(cls, package, version, firmware, architectures):
From 73981ae9bf4f249e60f97bc1bb294c5cbcf96347 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Fri, 12 Jan 2024 15:30:22 -0400
Subject: [PATCH 12/17] Revert "fix download counter"
This reverts commit bea771851e8789d152a0bbc22cdd75ab0ce902f6.
---
spkrepo/tests/test_nas.py | 36 ++++++++++++++++++------------------
spkrepo/views/nas.py | 37 ++++++++++---------------------------
2 files changed, 28 insertions(+), 45 deletions(-)
diff --git a/spkrepo/tests/test_nas.py b/spkrepo/tests/test_nas.py
index deb79ae..eff207b 100644
--- a/spkrepo/tests/test_nas.py
+++ b/spkrepo/tests/test_nas.py
@@ -560,9 +560,9 @@ def test_generic(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5,
- arch=architecture.code,
- build=4458,
+ architecture_id=architecture.id,
+ firmware_build=4458,
+ build_id=build.id,
),
environ_base={"REMOTE_ADDR": "127.0.0.1"},
headers={"User-Agent": "My User Agent"},
@@ -592,9 +592,9 @@ def test_wrong_build(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5 + "1",
- arch=architecture.code,
- build=4458,
+ architecture_id=architecture.id,
+ firmware_build=4458,
+ build_id=build.id + 1,
)
)
self.assert404(response)
@@ -613,9 +613,9 @@ def test_inactive_build(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5,
- arch=architecture.code,
- build=4458,
+ architecture_id=architecture.id,
+ firmware_build=4458,
+ build_id=build.id,
)
)
self.assert403(response)
@@ -634,9 +634,9 @@ def test_wrong_architecture(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5,
- arch=Architecture.find(10).code if Architecture.find(10) else "",
- build=4458,
+ architecture_id=10,
+ firmware_build=4458,
+ build_id=build.id,
)
)
self.assert404(response)
@@ -655,9 +655,9 @@ def test_incorrect_architecture(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5,
- arch=Architecture.find("cedarview").code,
- build=4458,
+ architecture_id=Architecture.find("cedarview").id,
+ firmware_build=4458,
+ build_id=build.id,
)
)
self.assert400(response)
@@ -676,9 +676,9 @@ def test_incorrect_firmware_build(self):
response = self.client.get(
url_for(
"nas.download",
- md5=build.md5,
- arch=architecture.code,
- build=1593,
+ architecture_id=architecture.id,
+ firmware_build=1593,
+ build_id=build.id,
)
)
self.assert400(response)
diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py
index fc0c880..f7d9cf0 100644
--- a/spkrepo/views/nas.py
+++ b/spkrepo/views/nas.py
@@ -144,10 +144,6 @@ def get_catalog(arch, build, language, beta):
# fill the catalog
entries = []
for b in latest_build.all():
- # correct any missing md5 hashes
- if b.md5 is None:
- b.md5 = b.calculate_md5()
- db.session.commit()
entry = {
"package": b.version.package.name,
"version": b.version.version_string,
@@ -158,11 +154,7 @@ def get_catalog(arch, build, language, beta):
language, b.version.descriptions["enu"]
).description,
"link": url_for(
- ".download",
- md5=b.md5,
- arch=arch,
- build=build,
- _external=True,
+ ".data", path=b.path, arch=arch, build=build, _external=True
),
"thumbnail": [
url_for(".data", path=icon.path, _external=True)
@@ -259,34 +251,25 @@ def catalog():
return Response(json.dumps(catalog), mimetype="application/json")
-@nas.route("/download///")
-def download(md5, arch, build):
+@nas.route("/download///")
+def download(architecture_id, firmware_build, build_id):
# check build
- build_obj = Build.query.filter_by(md5=md5).one_or_none()
-
- if build_obj is None:
- abort(404)
- elif not build_obj.active:
+ build = Build.query.get_or_404(build_id)
+ if not build.active:
abort(403)
# architecture
- architecture = Architecture.query.filter_by(code=arch).one_or_none()
+ architecture = Architecture.query.get_or_404(architecture_id)
- if architecture is None:
- abort(404)
# check consistency
- elif build < build_obj.firmware.build:
- abort(400)
- elif architecture not in build_obj.architectures and not any(
- arch.code == "noarch" for arch in build_obj.architectures
- ):
+ if architecture not in build.architectures or firmware_build < build.firmware.build:
abort(400)
# insert in database
download = Download(
- build=build_obj,
+ build=build,
architecture=architecture,
- firmware_build=build,
+ firmware_build=firmware_build,
ip_address=request.remote_addr,
user_agent=request.user_agent.string,
)
@@ -294,7 +277,7 @@ def download(md5, arch, build):
db.session.commit()
# redirect
- return redirect(url_for(".data", path=build_obj.path))
+ return redirect(url_for(".data", path=build.path))
@nas.route("/")
From ad1cf1772d925b6ee7e7c1bdb83db7f85360755c Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Mon, 15 Jan 2024 14:31:41 -0400
Subject: [PATCH 13/17] Amend major_version check
- Add type column to Firmware table
- Increase length of version column
- Filter by type for closest firmware when getting catalog
- Update populate_db with firmware type
---
...add_firmware_type_and_increase_version_.py | 47 +++++++++++++++++++
spkrepo/models.py | 3 +-
spkrepo/utils.py | 5 +-
spkrepo/views/nas.py | 2 +-
4 files changed, 54 insertions(+), 3 deletions(-)
create mode 100644 migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py
diff --git a/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py
new file mode 100644
index 0000000..564122c
--- /dev/null
+++ b/migrations/versions/f95855ce9471_add_firmware_type_and_increase_version_.py
@@ -0,0 +1,47 @@
+"""Add firmware type and increase version length
+
+Revision ID: f95855ce9471
+Revises: 76d559b4e873
+Create Date: 2024-01-15 13:58:34.160242
+
+"""
+revision = "f95855ce9471"
+down_revision = "76d559b4e873"
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+ op.add_column("firmware", sa.Column("type", sa.Unicode(length=4)))
+ # Set type based on version
+ op.execute(
+ """
+ UPDATE firmware
+ SET type = CASE
+ WHEN version LIKE '1.%' THEN 'srm'
+ ELSE 'dsm'
+ END
+ """
+ )
+ # Modify the column to be NOT NULL after setting the values
+ op.alter_column("firmware", "type", nullable=False)
+
+ op.alter_column(
+ "firmware",
+ "version",
+ existing_type=sa.VARCHAR(length=3),
+ type_=sa.Unicode(length=4),
+ existing_nullable=False,
+ )
+
+
+def downgrade():
+ op.alter_column(
+ "firmware",
+ "version",
+ existing_type=sa.Unicode(length=4),
+ type_=sa.VARCHAR(length=3),
+ existing_nullable=False,
+ )
+ op.drop_column("firmware", "type")
diff --git a/spkrepo/models.py b/spkrepo/models.py
index 0d0c294..2cc0613 100644
--- a/spkrepo/models.py
+++ b/spkrepo/models.py
@@ -134,8 +134,9 @@ class Firmware(db.Model):
# Columns
id = db.Column(db.Integer, primary_key=True)
- version = db.Column(db.Unicode(3), nullable=False)
+ version = db.Column(db.Unicode(4), nullable=False)
build = db.Column(db.Integer, unique=True, nullable=False)
+ type = db.Column(db.Unicode(4), nullable=False)
@classmethod
def find(cls, build):
diff --git a/spkrepo/utils.py b/spkrepo/utils.py
index aa0b0ab..d55abbd 100644
--- a/spkrepo/utils.py
+++ b/spkrepo/utils.py
@@ -398,7 +398,10 @@ def populate_db():
)
db.session.execute(
Firmware.__table__.insert().values(
- [{"version": "3.1", "build": 1594}, {"version": "5.0", "build": 4458}]
+ [
+ {"version": "3.1", "build": 1594, "type": "dsm"},
+ {"version": "5.0", "build": 4458, "type": "dsm"},
+ ]
)
)
db.session.execute(
diff --git a/spkrepo/views/nas.py b/spkrepo/views/nas.py
index f7d9cf0..9cbeeed 100644
--- a/spkrepo/views/nas.py
+++ b/spkrepo/views/nas.py
@@ -46,7 +46,7 @@ def is_valid_language(language):
def get_catalog(arch, build, language, beta):
# Find the closest matching firmware for the provided build
closest_firmware = (
- Firmware.query.filter(Firmware.build <= build)
+ Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm")
.order_by(Firmware.build.desc())
.first()
)
From 2bc6a2950fa47037e56b2a67e419b5901961a5e9 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Wed, 17 Jan 2024 22:04:48 -0400
Subject: [PATCH 14/17] add validators for firmware input
---
spkrepo/utils.py | 4 ++++
spkrepo/views/admin.py | 7 +++++++
2 files changed, 11 insertions(+)
diff --git a/spkrepo/utils.py b/spkrepo/utils.py
index d55abbd..b0b0656 100644
--- a/spkrepo/utils.py
+++ b/spkrepo/utils.py
@@ -58,6 +58,10 @@ class SPK(object):
#: Regex for files in conf
conf_filename_re = re.compile(r"^conf/.+$")
+ #: Regex for firmware input
+ firmware_version_re = re.compile(r"^\d+\.\d$")
+ firmware_type_re = re.compile(r"^([a-z]){3,}$")
+
def __init__(self, stream):
self.info = {}
self.icons = {}
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index 5da8ead..c52800b 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -122,6 +122,13 @@ def is_accessible(self):
can_delete = False
+ # Form
+ form_columns = ("version", "build", "type")
+ form_args = {
+ "version": {"validators": [Regexp(SPK.firmware_version_re)]},
+ "type": {"validators": [Regexp(SPK.firmware_type_re)]},
+ }
+
class ServiceView(ModelView):
"""View for :class:`~spkrepo.models.Service`"""
From 0159658cb15f3cff80fb34f5b3c89d90cd55bec0 Mon Sep 17 00:00:00 2001
From: publicarray
Date: Sun, 25 Feb 2024 10:52:55 +1100
Subject: [PATCH 15/17] fix Popped wrong app context.
---
migrations/env.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/migrations/env.py b/migrations/env.py
index 6e177d5..3273e03 100644
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -5,15 +5,9 @@
from flask import current_app
from sqlalchemy import engine_from_config, pool
-from spkrepo import create_app
-
config = context.config
fileConfig(config.config_file_name)
-# Set up the Flask application context
-app = create_app() # Create the Flask app
-app.app_context().push() # Push the app context
-
config.set_main_option(
"sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
)
From 95fa3c6e73db9651e2c935c64e439b44feeadb26 Mon Sep 17 00:00:00 2001
From: publicarray
Date: Sun, 25 Feb 2024 11:52:32 +1100
Subject: [PATCH 16/17] fix 500 page on users: 'NoneType' object has no
attribute 'strftime'
BaseModelView.index_view() got an unexpected keyword argument 'cls'
Flask-SQLAlchemy uses a custom formatter to handle date formatting when displaying data in the templates. It dosn't support overriding the value.
---
spkrepo/views/admin.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index c52800b..f97fea5 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -44,10 +44,6 @@ def is_accessible(self):
# View
column_list = ("username", "email", "roles", "active", "confirmed_at")
- column_formatters = {
- "confirmed_at": lambda v, c, m, p: m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S")
- }
-
# Form
form_columns = ("username", "roles", "active")
form_overrides = {"password": PasswordField}
From 5345bc728901c06525609ae83cf589f87037f3d3 Mon Sep 17 00:00:00 2001
From: mreid-tt <943378+mreid-tt@users.noreply.github.com>
Date: Sun, 25 Feb 2024 05:57:46 -0400
Subject: [PATCH 17/17] fix column formatting
---
spkrepo/views/admin.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/spkrepo/views/admin.py b/spkrepo/views/admin.py
index f97fea5..d4af9b9 100644
--- a/spkrepo/views/admin.py
+++ b/spkrepo/views/admin.py
@@ -44,6 +44,12 @@ def is_accessible(self):
# View
column_list = ("username", "email", "roles", "active", "confirmed_at")
+ column_formatters = {
+ "confirmed_at": lambda v, c, m, p: (
+ m.confirmed_at.strftime("%Y-%m-%d %H:%M:%S") if m.confirmed_at else None
+ )
+ }
+
# Form
form_columns = ("username", "roles", "active")
form_overrides = {"password": PasswordField}