Refactor foreign keys and relationships to pure logic (#901)

* Refactor foreign keys and relationships to pure logic

* Revert of some changes

* More revert

* Update the user paginate

* Update user create and update

* Update dept select and delete

* Rename the join query functions

* Update select_join_serialize doc and README

* Fix typo in README

* Update the user delete

* Update the user social

* Update the dict plugin crud

* Update the dict plugin version

* Bump dependencies and pre-commits

* Update the code generator plugin crud

* Update the menu crud

* Update the role crud

* Update the data scope and rule crud

* Restore get_paginated to get_select

* Update the code generator plugin version

* Add the py version in pre-commit

* Remove the plugin include parameter config

* Add more cache cleaning TODO

* Rename get_with_relation to get_join

* Add the user cache clear

* Fix known compatibility issues

* Update the version number to 1.11.0

* Fix lint

* Optimize select_join_serialize logic

* Delete cache cleanup comments

* Update the oauth2 plugin version

* Fix user-role table cleanup when user update

Author: wu-clan

.gitignore

@@ -7,3 +7,4 @@ venv/
 .python-version
 .ruff_cache/
 .pytest_cache/
+.claude/

.pre-commit-config.yaml

@@ -1,8 +1,12 @@
+default_language_version:
+    python: '>= 3.10'
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
+      - id: check-json
       - id: check-yaml
       - id: check-toml
 

backend/README.md

@@ -50,10 +50,10 @@
 
 4. Format and Lint
 
-   Auto-formatting and lint via `pre-commit`
+   Auto-formatting and lint via `prek`
 
    ```shell
-   pre-commit run --all-files
+   prek run --all-files
    ```
 
 5. Commit and push
@@ -78,6 +78,6 @@
 
 - `scripts/format.sh`: Perform ruff format check
 
-- `scripts/lint.sh`: Perform pre-commit formatting
+- `scripts/lint.sh`: Perform prek formatting
 
 - `scripts/export.sh`: Execute uv export dependency package

backend/__init__.py

@@ -1,6 +1,6 @@
 from backend.common.i18n import i18n
 
-__version__ = '1.10.4'
+__version__ = '1.11.0'
 
 
 # 初始化 i18n

backend/app/admin/crud/crud_data_rule.py

@@ -33,7 +33,7 @@ async def get_select(self, name: str | None) -> Select:
         if name is not None:
             filters['name__like'] = f'%{name}%'
 
-        return await self.select_order('id', load_strategies={'scopes': 'noload'}, **filters)
+        return await self.select_order('id', **filters)
 
     async def get_by_name(self, db: AsyncSession, name: str) -> DataRule | None:
         """

backend/app/admin/crud/crud_data_scope.py

@@ -1,11 +1,19 @@
 from collections.abc import Sequence
+from typing import Any
 
-from sqlalchemy import Select, select
+from sqlalchemy import Select, delete, insert
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy_crud_plus import CRUDPlus
+from sqlalchemy_crud_plus import CRUDPlus, JoinConfig
 
 from backend.app.admin.model import DataRule, DataScope
-from backend.app.admin.schema.data_scope import CreateDataScopeParam, UpdateDataScopeParam, UpdateDataScopeRuleParam
+from backend.app.admin.model.m2m import data_scope_rule
+from backend.app.admin.schema.data_scope import (
+    CreateDataScopeParam,
+    CreateDataScopeRuleParam,
+    UpdateDataScopeParam,
+    UpdateDataScopeRuleParam,
+)
+from backend.utils.serializers import select_join_serialize
 
 
 class CRUDDataScope(CRUDPlus[DataScope]):
@@ -31,15 +39,24 @@ async def get_by_name(self, db: AsyncSession, name: str) -> DataScope | None:
         """
         return await self.select_model_by_column(db, name=name)
 
-    async def get_with_relation(self, db: AsyncSession, pk: int) -> DataScope:
+    async def get_join(self, db: AsyncSession, pk: int) -> Any:
         """
         获取数据范围关联数据
 
         :param db: 数据库会话
         :param pk: 范围 ID
         :return:
         """
-        return await self.select_model(db, pk, load_strategies=['rules'])
+        result = await self.select_models(
+            db,
+            id=pk,
+            join_conditions=[
+                JoinConfig(model=data_scope_rule, join_on=data_scope_rule.c.data_scope_id == self.model.id),
+                JoinConfig(model=DataRule, join_on=DataRule.id == data_scope_rule.c.data_rule_id, fill_result=True),
+            ],
+        )
+
+        return select_join_serialize(result, relationships=['DataScope-m2m-DataRule:rules'])
 
     async def get_all(self, db: AsyncSession) -> Sequence[DataScope]:
         """
@@ -65,7 +82,7 @@ async def get_select(self, name: str | None, status: int | None) -> Select:
         if status is not None:
             filters['status'] = status
 
-        return await self.select_order('id', load_strategies={'rules': 'noload', 'roles': 'noload'}, **filters)
+        return await self.select_order('id', **filters)
 
     async def create(self, db: AsyncSession, obj: CreateDataScopeParam) -> None:
         """
@@ -88,7 +105,8 @@ async def update(self, db: AsyncSession, pk: int, obj: UpdateDataScopeParam) ->
         """
         return await self.update_model(db, pk, obj)
 
-    async def update_rules(self, db: AsyncSession, pk: int, rule_ids: UpdateDataScopeRuleParam) -> int:
+    @staticmethod
+    async def update_rules(db: AsyncSession, pk: int, rule_ids: UpdateDataScopeRuleParam) -> int:
         """
         更新数据范围规则
 
@@ -97,11 +115,16 @@ async def update_rules(self, db: AsyncSession, pk: int, rule_ids: UpdateDataScop
         :param rule_ids: 数据规则 ID 列表
         :return:
         """
-        current_data_scope = await self.get_with_relation(db, pk)
-        stmt = select(DataRule).where(DataRule.id.in_(rule_ids.rules))
-        rules = await db.execute(stmt)
-        current_data_scope.rules = rules.scalars().all()
-        return len(current_data_scope.rules)
+        data_scope_rule_stmt = delete(data_scope_rule).where(data_scope_rule.c.data_scope_id == pk)
+        await db.execute(data_scope_rule_stmt)
+
+        data_scope_rule_data = [
+            CreateDataScopeRuleParam(data_scope_id=pk, data_rule_id=rule_id).model_dump() for rule_id in rule_ids.rules
+        ]
+        data_scope_rule_stmt = insert(data_scope_rule)
+        await db.execute(data_scope_rule_stmt, data_scope_rule_data)
+
+        return len(rule_ids.rules)
 
     async def delete(self, db: AsyncSession, pks: list[int]) -> int:
         """

backend/app/admin/crud/crud_dept.py

@@ -1,12 +1,14 @@
 from collections.abc import Sequence
+from typing import Any
 
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy_crud_plus import CRUDPlus
+from sqlalchemy_crud_plus import CRUDPlus, JoinConfig
 
-from backend.app.admin.model import Dept
+from backend.app.admin.model import Dept, User
 from backend.app.admin.schema.dept import CreateDeptParam, UpdateDeptParam
 from backend.app.admin.schema.user import GetUserInfoWithRelationDetail
 from backend.common.security.permission import filter_data_permission
+from backend.utils.serializers import select_join_serialize
 
 
 class CRUDDept(CRUDPlus[Dept]):
@@ -63,8 +65,8 @@ async def get_all(
         if status is not None:
             filters['status'] = status
 
-        data_filtered = filter_data_permission(request_user)
-        return await self.select_models_order(db, 'sort', 'desc', data_filtered, **filters)
+        data_filter = filter_data_permission(request_user)
+        return await self.select_models_order(db, 'sort', 'desc', data_filter, **filters)
 
     async def create(self, db: AsyncSession, obj: CreateDeptParam) -> None:
         """
@@ -97,15 +99,20 @@ async def delete(self, db: AsyncSession, dept_id: int) -> int:
         """
         return await self.delete_model_by_column(db, id=dept_id, logical_deletion=True, deleted_flag_column='del_flag')
 
-    async def get_with_relation(self, db: AsyncSession, dept_id: int) -> Dept | None:
+    async def get_join(self, db: AsyncSession, dept_id: int) -> Any | None:
         """
         获取部门及关联数据
 
         :param db: 数据库会话
         :param dept_id: 部门 ID
         :return:
         """
-        return await self.select_model(db, dept_id, load_strategies=['users'])
+        result = await self.select_model(
+            db,
+            dept_id,
+            join_conditions=[JoinConfig(model=User, join_on=User.dept_id == self.model.id, fill_result=True)],
+        )
+        return select_join_serialize(result, relationships=['Dept-o2m-User'])
 
     async def get_children(self, db: AsyncSession, dept_id: int) -> Sequence[Dept | None]:
         """

backend/app/admin/crud/crud_menu.py

@@ -1,9 +1,11 @@
 from collections.abc import Sequence
 
+from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy_crud_plus import CRUDPlus
 
 from backend.app.admin.model import Menu
+from backend.app.admin.model.m2m import role_menu
 from backend.app.admin.schema.menu import CreateMenuParam, UpdateMenuParam
 
 
@@ -92,6 +94,9 @@ async def delete(self, db: AsyncSession, menu_id: int) -> int:
         :param menu_id: 菜单 ID
         :return:
         """
+        role_menu_stmt = delete(role_menu).where(role_menu.c.menu_id == menu_id)
+        await db.execute(role_menu_stmt)
+
         return await self.delete_model(db, menu_id)
 
     async def get_children(self, db: AsyncSession, menu_id: int) -> Sequence[Menu | None]:

backend/app/admin/crud/crud_role.py

@@ -1,16 +1,21 @@
 from collections.abc import Sequence
+from typing import Any
 
-from sqlalchemy import Select, select
+from sqlalchemy import Select, delete, insert, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy_crud_plus import CRUDPlus
+from sqlalchemy_crud_plus import CRUDPlus, JoinConfig
 
 from backend.app.admin.model import DataScope, Menu, Role
+from backend.app.admin.model.m2m import role_data_scope, role_menu
 from backend.app.admin.schema.role import (
+    CreateRoleMenuParam,
     CreateRoleParam,
+    CreateRoleScopeParam,
     UpdateRoleMenuParam,
     UpdateRoleParam,
     UpdateRoleScopeParam,
 )
+from backend.utils.serializers import select_join_serialize
 
 
 class CRUDRole(CRUDPlus[Role]):
@@ -26,15 +31,39 @@ async def get(self, db: AsyncSession, role_id: int) -> Role | None:
         """
         return await self.select_model(db, role_id)
 
-    async def get_with_relation(self, db: AsyncSession, role_id: int) -> Role | None:
+    @staticmethod
+    async def get_menus(db: AsyncSession, role_id: int) -> Sequence[Menu] | None:
+        """
+        获取角色菜单
+
+        :param db: 数据库会话
+        :param role_id: 角色 ID
+        :return:
+        """
+        menu_stmt = select(Menu).join(role_menu, Menu.id == role_menu.c.menu_id).where(role_menu.c.role_id == role_id)
+        result = await db.execute(menu_stmt)
+        return result.scalars().all()
+
+    async def get_join(self, db: AsyncSession, role_id: int) -> Any:
         """
         获取角色及关联数据
 
         :param db: 数据库会话
         :param role_id: 角色 ID
         :return:
         """
-        return await self.select_model(db, role_id, load_strategies=['menus', 'scopes'])
+        result = await self.select_models(
+            db,
+            id=role_id,
+            join_conditions=[
+                JoinConfig(model=role_menu, join_on=role_menu.c.role_id == self.model.id),
+                JoinConfig(model=Menu, join_on=Menu.id == role_menu.c.menu_id, fill_result=True),
+                JoinConfig(model=role_data_scope, join_on=role_data_scope.c.role_id == self.model.id),
+                JoinConfig(model=DataScope, join_on=DataScope.id == role_data_scope.c.data_scope_id, fill_result=True),
+            ],
+        )
+
+        return select_join_serialize(result, relationships=['Role-m2m-Menu', 'Role-m2m-DataScope:scopes'])
 
     async def get_all(self, db: AsyncSession) -> Sequence[Role]:
         """
@@ -61,15 +90,7 @@ async def get_select(self, name: str | None, status: int | None) -> Select:
         if status is not None:
             filters['status'] = status
 
-        return await self.select_order(
-            'id',
-            load_strategies={
-                'users': 'noload',
-                'menus': 'noload',
-                'scopes': 'noload',
-            },
-            **filters,
-        )
+        return await self.select_order('id', **filters)
 
     async def get_by_name(self, db: AsyncSession, name: str) -> Role | None:
         """
@@ -102,7 +123,8 @@ async def update(self, db: AsyncSession, role_id: int, obj: UpdateRoleParam) ->
         """
         return await self.update_model(db, role_id, obj)
 
-    async def update_menus(self, db: AsyncSession, role_id: int, menu_ids: UpdateRoleMenuParam) -> int:
+    @staticmethod
+    async def update_menus(db: AsyncSession, role_id: int, menu_ids: UpdateRoleMenuParam) -> int:
         """
         更新角色菜单
 
@@ -111,13 +133,19 @@ async def update_menus(self, db: AsyncSession, role_id: int, menu_ids: UpdateRol
         :param menu_ids: 菜单 ID 列表
         :return:
         """
-        current_role = await self.get_with_relation(db, role_id)
-        stmt = select(Menu).where(Menu.id.in_(menu_ids.menus))
-        menus = await db.execute(stmt)
-        current_role.menus = menus.scalars().all()
-        return len(current_role.menus)
+        role_menu_stmt = delete(role_menu).where(role_menu.c.role_id == role_id)
+        await db.execute(role_menu_stmt)
+
+        role_menu_data = [
+            CreateRoleMenuParam(role_id=role_id, menu_id=menu_id).model_dump() for menu_id in menu_ids.menus
+        ]
+        role_menu_stmt = insert(role_menu)
+        await db.execute(role_menu_stmt, role_menu_data)
 
-    async def update_scopes(self, db: AsyncSession, role_id: int, scope_ids: UpdateRoleScopeParam) -> int:
+        return len(menu_ids.menus)
+
+    @staticmethod
+    async def update_scopes(db: AsyncSession, role_id: int, scope_ids: UpdateRoleScopeParam) -> int:
         """
         更新角色数据范围
 
@@ -126,11 +154,16 @@ async def update_scopes(self, db: AsyncSession, role_id: int, scope_ids: UpdateR
         :param scope_ids: 权限范围 ID 列表
         :return:
         """
-        current_role = await self.get_with_relation(db, role_id)
-        stmt = select(DataScope).where(DataScope.id.in_(scope_ids.scopes))
-        scopes = await db.execute(stmt)
-        current_role.scopes = scopes.scalars().all()
-        return len(current_role.scopes)
+        role_scope_stmt = delete(role_data_scope).where(role_data_scope.c.role_id == role_id)
+        await db.execute(role_scope_stmt)
+
+        role_scope_data = [
+            CreateRoleScopeParam(role_id=role_id, data_scope_id=scope_id).model_dump() for scope_id in scope_ids.scopes
+        ]
+        role_scope_stmt = insert(role_data_scope)
+        await db.execute(role_scope_stmt, role_scope_data)
+
+        return len(scope_ids.scopes)
 
     async def delete(self, db: AsyncSession, role_ids: list[int]) -> int:
         """