diff --git a/bk-monitor-base b/bk-monitor-base index 31d64df94fd..dcc13549f76 160000 --- a/bk-monitor-base +++ b/bk-monitor-base @@ -1 +1 @@ -Subproject commit 31d64df94fd123f1011544d0b50b2433a21c3464 +Subproject commit dcc13549f76de325828f597df27c55de1f5c8bee diff --git a/bkmonitor/bkmonitor/management/commands/clean_strategy_history.py b/bkmonitor/bkmonitor/management/commands/clean_strategy_history.py new file mode 100644 index 00000000000..861bc52546b --- /dev/null +++ b/bkmonitor/bkmonitor/management/commands/clean_strategy_history.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError + +from bkmonitor.strategy.history import MIN_CLEAN_STRATEGY_HISTORY_DAYS +from bkmonitor.strategy.history import CleanStrategyHistoryParams +from bkmonitor.strategy.history import clean_strategy_history + + +class Command(BaseCommand): + """清理过期的策略变更历史。""" + + help = ( + "清理指定天数以前的策略历史，并保留最近成功快照和必要的删除记录。" + f"默认 dry-run；真正删除需加 --execute。--days 不得小于 {MIN_CLEAN_STRATEGY_HISTORY_DAYS}。" + ) + command_name = "clean strategy history" + deprecation_warning = "" + + @staticmethod + def cleanup(params: CleanStrategyHistoryParams) -> int: + return clean_strategy_history(params) + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--days", + type=int, + required=True, + help=f"保留最近 N 天的全部策略历史，最小 {MIN_CLEAN_STRATEGY_HISTORY_DAYS}", + ) + parser.add_argument( + "--batch_size", + type=int, + default=1000, + help="单批删除数量，默认 1000", + ) + parser.add_argument( + "--keep_latest_snapshots", + type=int, + default=1, + help="每个策略额外保留的最近成功快照数量，默认 1", + ) + parser.add_argument( + "--execute", + action="store_true", + help="真正执行删除；省略时仅 dry-run 统计预计删除数量", + ) + + def handle(self, *_args, **options) -> None: + try: + params = CleanStrategyHistoryParams( + days=options["days"], + batch_size=options["batch_size"], + keep_latest_snapshots=options["keep_latest_snapshots"], + dry_run=not options["execute"], + ) + except ValueError as exc: + raise CommandError(str(exc)) from exc + + if params.days < MIN_CLEAN_STRATEGY_HISTORY_DAYS: + raise CommandError(f"days must be >= {MIN_CLEAN_STRATEGY_HISTORY_DAYS}, got {params.days}") + + if self.deprecation_warning: + self.stdout.write(self.style.WARNING(self.deprecation_warning)) + self.stdout.write( + f"{self.command_name} start: " + f"days={params.days}, batch_size={params.batch_size}, " + f"keep_latest_snapshots={params.keep_latest_snapshots}, dry_run={params.dry_run}" + ) + matched = self.cleanup(params) + if params.dry_run: + self.stdout.write(self.style.WARNING(f"{self.command_name} dry-run done: would delete {matched} records")) + else: + self.stdout.write(self.style.SUCCESS(f"{self.command_name} done: deleted {matched} records")) diff --git a/bkmonitor/bkmonitor/management/commands/clean_strategy_history_compat.py b/bkmonitor/bkmonitor/management/commands/clean_strategy_history_compat.py new file mode 100644 index 00000000000..64103b92dd4 --- /dev/null +++ b/bkmonitor/bkmonitor/management/commands/clean_strategy_history_compat.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from bkmonitor.management.commands.clean_strategy_history import Command as CleanStrategyHistoryCommand +from bkmonitor.strategy.history import MIN_CLEAN_STRATEGY_HISTORY_DAYS +from bkmonitor.strategy.history import CleanStrategyHistoryParams +from bkmonitor.strategy.history import clean_strategy_history_compat + + +class Command(CleanStrategyHistoryCommand): + """兼容旧批量操作历史状态的临时清理命令。""" + + help = ( + "临时兼容清理命令：为窗口外旧版批量 update 历史额外保留最近快照，" + "并为已删除策略保留最新 delete 记录；其余规则与 clean_strategy_history 一致。" + f"默认 dry-run；真正删除需加 --execute。--days 不得小于 {MIN_CLEAN_STRATEGY_HISTORY_DAYS}。" + ) + command_name = "clean strategy history compatibility" + deprecation_warning = ( + "deprecated compatibility command: only use while legacy bulk update/delete status records need cleanup" + ) + + @staticmethod + def cleanup(params: CleanStrategyHistoryParams) -> int: + return clean_strategy_history_compat(params) diff --git a/bkmonitor/bkmonitor/models/strategy.py b/bkmonitor/bkmonitor/models/strategy.py index eeb23835c2e..7aea79cd536 100644 --- a/bkmonitor/bkmonitor/models/strategy.py +++ b/bkmonitor/bkmonitor/models/strategy.py @@ -439,6 +439,7 @@ class StrategyHistoryModel(Model): ("delete", _lazy("删除")), ("create", _lazy("创建")), ("update", _lazy("更新")), + ("bulk_update", _lazy("批量更新")), ), db_index=True, max_length=12, diff --git a/bkmonitor/bkmonitor/strategy/history.py b/bkmonitor/bkmonitor/strategy/history.py new file mode 100644 index 00000000000..4ba11e40374 --- /dev/null +++ b/bkmonitor/bkmonitor/strategy/history.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from collections.abc import Iterator +from datetime import datetime +from datetime import timedelta + +from django.db.models import OuterRef +from django.db.models import QuerySet +from django.db.models import Subquery +from django.utils import timezone + +from bkmonitor.models import StrategyHistoryModel +from bkmonitor.models import StrategyModel + +SNAPSHOT_OPERATIONS = ("create", "update", "bulk_update") +DELETE_OPERATIONS = ("delete",) +LEGACY_SNAPSHOT_OPERATION = "update" +STRATEGY_ID_CHUNK_SIZE = 500 +# 管理命令允许的最小保留天数；业务层 CleanStrategyHistoryParams 仍只要求正整数，便于单测构造窗口。 +MIN_CLEAN_STRATEGY_HISTORY_DAYS = 30 + + +class CleanStrategyHistoryParams: + """策略历史清理参数。 + + 校验逻辑集中在此类中：days、batch_size、keep_latest_snapshots 均须为正整数。 + """ + + def __init__( + self, + days: int, + batch_size: int = 1000, + keep_latest_snapshots: int = 1, + dry_run: bool = False, + ): + self.days = self._require_positive_int("days", days) + self.batch_size = self._require_positive_int("batch_size", batch_size) + self.keep_latest_snapshots = self._require_positive_int("keep_latest_snapshots", keep_latest_snapshots) + if not isinstance(dry_run, bool): + raise ValueError("dry_run must be a boolean") + self.dry_run = dry_run + + @staticmethod + def _require_positive_int(name: str, value: object) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _collect_latest_history_ids(queryset: QuerySet, limit: int = 1) -> set[int]: + """获取查询集中每个策略按时间排序后的最近若干条历史 ID。 + + Args: + queryset: 已包含业务过滤条件的策略历史查询集。 + limit: 每个策略保留的最近记录条数。 + + Returns: + 每个策略最近 limit 条历史记录的 ID 集合。 + """ + keep_history_ids: set[int] = set() + remaining = queryset.order_by() + + for _ in range(limit): + latest_id = ( + remaining.filter(strategy_id=OuterRef("strategy_id")).order_by("-create_time", "-id").values("id")[:1] + ) + batch_ids = set(remaining.filter(id=Subquery(latest_id)).values_list("id", flat=True)) + if not batch_ids: + break + keep_history_ids.update(batch_ids) + remaining = remaining.exclude(id__in=batch_ids) + + return keep_history_ids + + +def _collect_keep_history_ids( + strategy_ids: list[int], + keep_latest_snapshots: int = 1, + legacy_status_before: datetime | None = None, +) -> set[int]: + """计算一批策略必须保留的历史记录 ID。 + + Args: + strategy_ids: 本批需要处理的策略 ID。 + keep_latest_snapshots: 每个策略保留的最近成功快照条数。 + legacy_status_before: 额外保留此时间之前状态未回写的旧批量更新快照。 + + Returns: + 最近成功快照，以及已删除策略的最新删除记录 ID。 + """ + if not strategy_ids: + return set() + + existing_strategy_ids = set(StrategyModel.objects.filter(id__in=strategy_ids).values_list("id", flat=True)) + deleted_strategy_ids = set(strategy_ids) - existing_strategy_ids + + recoverable_snapshots = ( + StrategyHistoryModel.objects.filter( + strategy_id__in=strategy_ids, + operate__in=SNAPSHOT_OPERATIONS, + status=True, + ) + .exclude(content={}) + .exclude(content__isnull=True) + ) + keep_history_ids = _collect_latest_history_ids(recoverable_snapshots, limit=keep_latest_snapshots) + + if legacy_status_before is not None: + legacy_snapshots = ( + StrategyHistoryModel.objects.filter( + strategy_id__in=strategy_ids, + operate=LEGACY_SNAPSHOT_OPERATION, + status=False, + message="", + create_time__lt=legacy_status_before, + ) + .exclude(content={}) + .exclude(content__isnull=True) + ) + keep_history_ids.update(_collect_latest_history_ids(legacy_snapshots, limit=keep_latest_snapshots)) + + if deleted_strategy_ids: + delete_histories = StrategyHistoryModel.objects.filter( + strategy_id__in=deleted_strategy_ids, + operate__in=DELETE_OPERATIONS, + ) + keep_history_ids.update(_collect_latest_history_ids(delete_histories)) + + return keep_history_ids + + +def _iter_expired_strategy_id_chunks( + before: datetime, + chunk_size: int = STRATEGY_ID_CHUNK_SIZE, +) -> Iterator[list[int]]: + """按 strategy_id 分批产出存在过期历史的策略。 + + Args: + before: 清理截止时间。 + chunk_size: 每批策略数量。 + + Yields: + 按 strategy_id 升序排列的策略 ID 列表。 + """ + last_strategy_id = None + while True: + queryset = StrategyHistoryModel.objects.filter(create_time__lt=before) + if last_strategy_id is not None: + queryset = queryset.filter(strategy_id__gt=last_strategy_id) + + strategy_ids = list( + queryset.order_by("strategy_id").values_list("strategy_id", flat=True).distinct()[:chunk_size] + ) + if not strategy_ids: + return + + yield strategy_ids + last_strategy_id = strategy_ids[-1] + + +def _delete_queryset_in_batches(queryset: QuerySet, batch_size: int) -> int: + """按主键分批删除查询集中的记录。 + + Args: + queryset: 待删除的策略历史查询集。 + batch_size: 单批删除数量。 + + Returns: + 实际删除的记录数量。 + """ + model = queryset.model + last_pk = None + deleted = 0 + + while True: + batch = queryset.order_by("pk") + if last_pk is not None: + batch = batch.filter(pk__gt=last_pk) + + history_ids = list(batch.values_list("pk", flat=True)[:batch_size]) + if not history_ids: + return deleted + + deleted_count, _ = model._default_manager.using(queryset.db).filter(pk__in=history_ids).delete() + deleted += deleted_count + last_pk = history_ids[-1] + + +def _clean_strategy_history( + params: CleanStrategyHistoryParams, + legacy_status_compat: bool = False, +) -> int: + """清理指定天数以前的冗余策略历史。 + + 保留最近 ``params.days`` 天的全部历史；更早的记录中，每个策略额外保留全局最近 + ``params.keep_latest_snapshots`` 条成功快照。已删除策略额外保留全局最新删除记录。 + + Args: + params: 清理参数，校验在 ``CleanStrategyHistoryParams`` 中完成。 + legacy_status_compat: 是否额外保留状态未回写的旧批量更新快照。 + + Returns: + 实际删除的历史记录数量。 + + Raises: + ValueError: 由 ``CleanStrategyHistoryParams`` 在构造时抛出。 + """ + before = timezone.now() - timedelta(days=params.days) + deleted = 0 + + for strategy_ids in _iter_expired_strategy_id_chunks(before): + keep_history_ids = _collect_keep_history_ids( + strategy_ids, + keep_latest_snapshots=params.keep_latest_snapshots, + legacy_status_before=before if legacy_status_compat else None, + ) + queryset = StrategyHistoryModel.objects.filter( + strategy_id__in=strategy_ids, + create_time__lt=before, + ) + if keep_history_ids: + queryset = queryset.exclude(id__in=keep_history_ids) + if params.dry_run: + deleted += queryset.count() + else: + deleted += _delete_queryset_in_batches(queryset, params.batch_size) + + return deleted + + +def clean_strategy_history(params: CleanStrategyHistoryParams) -> int: + """按标准规则清理策略历史。""" + return _clean_strategy_history(params) + + +def clean_strategy_history_compat(params: CleanStrategyHistoryParams) -> int: + """兼容旧批量操作状态的临时清理入口。 + + 旧批量更新和删除与普通操作共用 ``update/delete`` 类型，且成功状态未回写。 + 兼容模式为清理窗口外 ``status=False``、空错误消息的旧 ``update`` 记录 + 提供独立快照保留名额，避免其挤掉已确认成功快照；删除记录仍只为已删除策略 + 保留最新一条。该入口不修改历史状态，待旧数据不再需要兼容后，应弃用并恢复 + 标准清理命令。 + """ + return _clean_strategy_history(params, legacy_status_compat=True) diff --git a/bkmonitor/bkmonitor/strategy/new_strategy.py b/bkmonitor/bkmonitor/strategy/new_strategy.py index b9f0f26be98..f7133eb1777 100644 --- a/bkmonitor/bkmonitor/strategy/new_strategy.py +++ b/bkmonitor/bkmonitor/strategy/new_strategy.py @@ -23,7 +23,7 @@ import arrow import xxhash from django.conf import settings -from django.db import transaction +from django.db import router, transaction from django.db.models import Model, QuerySet from django.utils import timezone from django.utils.translation import gettext as _ @@ -3076,8 +3076,15 @@ def delete(self): @classmethod def delete_by_strategy_ids(cls, strategy_ids: list[int]): + with transaction.atomic(using=router.db_for_write(StrategyModel)): + return cls._delete_by_strategy_ids(strategy_ids) + + @classmethod + def _delete_by_strategy_ids(cls, strategy_ids: list[int]): """ 批量删除策略 + + 删除与成功历史写入放在同一事务中，避免中途失败后主表已删但无删除历史。 """ from bkmonitor.models.issue import StrategyIssueConfig @@ -3088,10 +3095,9 @@ def delete_by_strategy_ids(cls, strategy_ids: list[int]): create_user=cls._get_username(), strategy_id=strategy_id, operate="delete", + status=True, ) ) - StrategyHistoryModel.objects.bulk_create(histories, batch_size=100) - StrategyModel.objects.filter(id__in=strategy_ids).delete() RelationModel.objects.filter(strategy_id__in=strategy_ids).delete() DetectModel.objects.filter(strategy_id__in=strategy_ids).delete() @@ -3101,6 +3107,8 @@ def delete_by_strategy_ids(cls, strategy_ids: list[int]): StrategyLabel.objects.filter(strategy_id__in=strategy_ids).delete() StrategyIssueConfig.objects.filter(strategy_id__in=strategy_ids).delete() + StrategyHistoryModel.objects.bulk_create(histories, batch_size=100) + @classmethod def from_models(cls, strategies: list[StrategyModel] | QuerySet) -> list["Strategy"]: """ diff --git a/bkmonitor/bkmonitor/strategy/tests/test_clean_strategy_history_boundaries.py b/bkmonitor/bkmonitor/strategy/tests/test_clean_strategy_history_boundaries.py new file mode 100644 index 00000000000..4744978a2cc --- /dev/null +++ b/bkmonitor/bkmonitor/strategy/tests/test_clean_strategy_history_boundaries.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from datetime import datetime +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import router +from django.utils import timezone + +from bkmonitor.models import StrategyHistoryModel +from bkmonitor.models import StrategyModel +from bkmonitor.strategy.history import CleanStrategyHistoryParams +from bkmonitor.strategy.history import _collect_keep_history_ids +from bkmonitor.strategy.history import _delete_queryset_in_batches +from bkmonitor.strategy.history import clean_strategy_history + +FIXED_NOW = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) +DAYS = 30 +BEFORE = FIXED_NOW - timedelta(days=DAYS) +OLD = BEFORE - timedelta(hours=1) +OLDER = BEFORE - timedelta(hours=2) +OLDEST = BEFORE - timedelta(hours=3) +RECENT = FIXED_NOW - timedelta(hours=1) +CMD = "clean_strategy_history" + +pytestmark = pytest.mark.django_db(databases=("default", "monitor_api")) + + +def _freeze_now(monkeypatch, now: datetime = FIXED_NOW) -> datetime: + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + return now + + +def _create_strategy(name: str) -> StrategyModel: + return StrategyModel.objects.create( + bk_biz_id=2, + name=name, + scenario="os", + type=StrategyModel.StrategyType.Monitor, + ) + + +def _create_history( + strategy_id: int, + create_time: datetime, + *, + operate: str = "update", + status: bool = True, + content: dict | None = None, + message: str = "", +) -> StrategyHistoryModel: + history = StrategyHistoryModel.objects.create( + strategy_id=strategy_id, + create_user="admin", + operate=operate, + status=status, + content=content if content is not None else {"id": strategy_id}, + message=message, + ) + StrategyHistoryModel.objects.filter(id=history.id).update(create_time=create_time) + history.refresh_from_db() + return history + + +def _remaining_ids() -> set[int]: + return set(StrategyHistoryModel.objects.values_list("id", flat=True)) + + +def _params(**kwargs) -> CleanStrategyHistoryParams: + defaults = {"days": DAYS, "batch_size": 2, "keep_latest_snapshots": 1} + defaults.update(kwargs) + return CleanStrategyHistoryParams(**defaults) + + +class TestCleanStrategyHistoryBoundaryMatrix: + """策略历史删除大边界套件：截止窗口、保留规则、分片删除与命令端到端。""" + + def test_cutoff_boundary_equal_before_is_not_deleted(self, monkeypatch): + """create_time == before 不进入清理窗口（严格 <）。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("cutoff-eq") + at_boundary = _create_history(strategy.id, BEFORE, status=False) + older = _create_history(strategy.id, OLD, status=False) + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {at_boundary.id} + assert not StrategyHistoryModel.objects.filter(id=older.id).exists() + + def test_recent_records_outside_window_never_deleted(self, monkeypatch): + """窗口内历史全部保留，清理应返回 0。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("recent-keep-all") + recent_a = _create_history(strategy.id, RECENT - timedelta(minutes=1)) + recent_b = _create_history(strategy.id, RECENT) + + deleted = clean_strategy_history(_params()) + + assert deleted == 0 + assert _remaining_ids() == {recent_a.id, recent_b.id} + + def test_empty_content_is_not_recoverable_snapshot(self, monkeypatch): + """空 content 不能作为可恢复快照淘汰更早成功记录。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("empty-content") + kept = _create_history(strategy.id, OLDEST) + empty = _create_history(strategy.id, OLD, content={}) + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept.id} + assert not StrategyHistoryModel.objects.filter(id=empty.id).exists() + + def test_failed_update_with_traceback_is_never_kept_as_snapshot(self, monkeypatch): + """失败更新即使时间更新也不能成为保留快照。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("failed-trace") + kept = _create_history(strategy.id, OLDEST) + failed = _create_history( + strategy.id, + OLD, + status=False, + message="Traceback (most recent call last):\nValueError", + ) + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept.id} + assert not StrategyHistoryModel.objects.filter(id=failed.id).exists() + + def test_legacy_empty_message_with_failed_status_is_not_recoverable(self, monkeypatch): + """存量失败记录即使 message 为空，也不能按成功快照保留。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("legacy-empty") + kept = _create_history(strategy.id, OLDEST) + legacy_failed = _create_history(strategy.id, OLD, status=False, message="") + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept.id} + assert not StrategyHistoryModel.objects.filter(id=legacy_failed.id).exists() + + def test_recent_failed_record_does_not_steal_keep_slot(self, monkeypatch): + """窗口内的失败记录不占用 keep_latest_snapshots 名额。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("failed-recent") + older = _create_history(strategy.id, OLDEST) + kept = _create_history(strategy.id, OLD) + recent_failed = _create_history(strategy.id, RECENT, status=False, message="") + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept.id, recent_failed.id} + assert not StrategyHistoryModel.objects.filter(id=older.id).exists() + + def test_bulk_update_status_true_is_preferred_over_older_update(self, monkeypatch): + """同策略多条成功快照时，保留全局最新一条。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("bulk-prefer") + old_update = _create_history(strategy.id, OLDEST, operate="update") + kept_bulk = _create_history(strategy.id, OLD, operate="bulk_update") + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept_bulk.id} + assert not StrategyHistoryModel.objects.filter(id=old_update.id).exists() + + def test_create_snapshot_can_be_the_only_kept_record(self, monkeypatch): + """现存策略只保留成功快照，不保留过期 delete。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("create-only") + create_snap = _create_history(strategy.id, OLD, operate="create") + delete_row = _create_history(strategy.id, OLD + timedelta(minutes=1), operate="delete", status=False) + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {create_snap.id} + assert not StrategyHistoryModel.objects.filter(id=delete_row.id).exists() + + def test_deleted_strategy_keeps_latest_snapshot_and_latest_delete(self, monkeypatch): + """已删除策略保留最新成功快照和最新删除事实。""" + _freeze_now(monkeypatch) + deleted_id = 910001 + old_snap = _create_history(deleted_id, OLDEST, operate="update") + kept_snap = _create_history(deleted_id, OLDER, operate="bulk_update") + old_delete = _create_history(deleted_id, OLD - timedelta(minutes=2), operate="delete", status=False) + kept_delete = _create_history(deleted_id, OLD, operate="delete", status=False) + + deleted = clean_strategy_history(_params()) + + assert deleted == 2 + assert _remaining_ids() == {kept_snap.id, kept_delete.id} + assert not StrategyHistoryModel.objects.filter(id__in=[old_snap.id, old_delete.id]).exists() + + def test_deleted_strategy_with_only_delete_rows_keeps_latest_delete(self, monkeypatch): + """仅有删除记录时，仍保留最新一条删除事实。""" + _freeze_now(monkeypatch) + deleted_id = 910002 + old_delete = _create_history(deleted_id, OLDEST, operate="delete", status=False) + kept_delete = _create_history(deleted_id, OLD, operate="delete", status=False) + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {kept_delete.id} + assert not StrategyHistoryModel.objects.filter(id=old_delete.id).exists() + + def test_same_create_time_prefers_greater_id(self, monkeypatch): + """相同 create_time 时按更大 id 作为更新记录保留。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("tie-break") + first = _create_history(strategy.id, OLD, content={"v": 1}) + second = _create_history(strategy.id, OLD, content={"v": 2}) + assert second.id > first.id + + deleted = clean_strategy_history(_params()) + + assert deleted == 1 + assert _remaining_ids() == {second.id} + + def test_strategy_id_zero_full_cleanup_keeps_latest_recoverable(self, monkeypatch): + """strategy_id=0 也要走完整清理路径，并保留最新成功快照。""" + _freeze_now(monkeypatch) + older = _create_history(0, OLDEST, operate="create") + kept = _create_history(0, OLD, operate="create") + failed = _create_history( + 0, + OLD + timedelta(minutes=1), + operate="create", + status=False, + message="create failed", + ) + + deleted = clean_strategy_history(_params(batch_size=1)) + + assert deleted == 2 + assert _remaining_ids() == {kept.id} + assert not StrategyHistoryModel.objects.filter(id__in=[older.id, failed.id]).exists() + + def test_multi_chunk_cleanup_does_not_skip_strategy_groups(self, monkeypatch): + """跨多个 strategy_id chunk 时，每组都应保留最新成功快照。""" + _freeze_now(monkeypatch) + strategies = [_create_strategy(f"chunk-{i}") for i in range(5)] + kept = [] + removable = [] + for strategy in strategies: + removable.append(_create_history(strategy.id, OLDEST).id) + kept.append(_create_history(strategy.id, OLD).id) + + monkeypatch.setattr("bkmonitor.strategy.history.STRATEGY_ID_CHUNK_SIZE", 2) + deleted = clean_strategy_history(_params(batch_size=1)) + + assert deleted == 5 + assert _remaining_ids() == set(kept) + assert not StrategyHistoryModel.objects.filter(id__in=removable).exists() + + def test_keyset_delete_handles_non_contiguous_pks(self, monkeypatch): + """按主键 keyset 分页删除时，pk 空洞不应导致漏删。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("pk-gaps") + row_ids = [] + for minute in (5, 4, 3, 2, 1): + row_ids.append(_create_history(strategy.id, OLD - timedelta(minutes=minute)).id) + + filler = _create_history(strategy.id + 100000, OLD) + StrategyHistoryModel.objects.filter(id=filler.id).delete() + + queryset = StrategyHistoryModel.objects.filter(id__in=row_ids[:-1]) + deleted = _delete_queryset_in_batches(queryset, batch_size=2) + + assert deleted == 4 + assert list(StrategyHistoryModel.objects.filter(id__in=row_ids).values_list("id", flat=True)) == [row_ids[-1]] + + def test_keyset_delete_preserves_explicit_database_alias(self, monkeypatch): + """删除必须沿用传入 queryset 的数据库，不能重新经过写路由。""" + strategy = _create_strategy("explicit-database-alias") + history = _create_history(strategy.id, OLD) + queryset = StrategyHistoryModel.objects.using("monitor_api").filter(id=history.id) + monkeypatch.setattr(router, "db_for_write", lambda *_args, **_kwargs: "default") + + deleted = _delete_queryset_in_batches(queryset, batch_size=1) + + assert deleted == 1 + assert not StrategyHistoryModel.objects.using("monitor_api").filter(id=history.id).exists() + + def test_window_outside_latest_recoverable_allows_deleting_all_old_snapshots(self, monkeypatch): + """窗口外已有更新成功快照时，过期旧快照可全部删除。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("outside-latest") + old_a = _create_history(strategy.id, OLDEST) + old_b = _create_history(strategy.id, OLD) + recent = _create_history(strategy.id, RECENT, operate="bulk_update") + + deleted = clean_strategy_history(_params()) + + assert deleted == 2 + assert _remaining_ids() == {recent.id} + assert not StrategyHistoryModel.objects.filter(id__in=[old_a.id, old_b.id]).exists() + + def test_keep_latest_snapshots_counts_recent_and_expired_together(self, monkeypatch): + """keep_latest_snapshots 统计全局成功快照，窗口内快照会挤占窗外保留名额。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("keep-n-window") + expired_old = _create_history(strategy.id, OLDEST, operate="create") + expired_kept = _create_history(strategy.id, OLD) + recent = _create_history(strategy.id, RECENT, operate="bulk_update") + + deleted = clean_strategy_history(_params(keep_latest_snapshots=2)) + + assert deleted == 1 + assert _remaining_ids() == {expired_kept.id, recent.id} + assert not StrategyHistoryModel.objects.filter(id=expired_old.id).exists() + + def test_mixed_operate_types_only_recoverable_ops_compete(self, monkeypatch): + """只有 create/update/bulk_update 成功记录参与快照竞选。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("mixed-ops") + create_row = _create_history(strategy.id, OLDEST, operate="create") + update_row = _create_history(strategy.id, OLDER, operate="update") + bulk_row = _create_history(strategy.id, OLD, operate="bulk_update") + delete_row = _create_history(strategy.id, OLD + timedelta(minutes=1), operate="delete", status=False) + + deleted = clean_strategy_history(_params()) + + assert deleted == 3 + assert _remaining_ids() == {bulk_row.id} + assert not StrategyHistoryModel.objects.filter(id__in=[create_row.id, update_row.id, delete_row.id]).exists() + + def test_collect_keep_history_ids_for_existing_strategy_excludes_delete(self, monkeypatch): + """现存策略的 keep 集合不应包含 delete 记录。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("keep-ids-existing") + snap = _create_history(strategy.id, OLD) + delete_row = _create_history(strategy.id, OLD + timedelta(minutes=1), operate="delete", status=False) + + keep_ids = _collect_keep_history_ids([strategy.id], keep_latest_snapshots=1) + + assert keep_ids == {snap.id} + assert delete_row.id not in keep_ids + + def test_cleanup_is_idempotent_on_mixed_dataset(self, monkeypatch): + """混合数据集重复清理时，第二次应删除 0 条。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("idempotent-matrix") + _create_history(strategy.id, OLDEST) + kept = _create_history(strategy.id, OLD, operate="bulk_update") + deleted_id = 910010 + _create_history(deleted_id, OLDEST, operate="update") + kept_snap = _create_history(deleted_id, OLDER, operate="update") + kept_delete = _create_history(deleted_id, OLD, operate="delete", status=False) + + params = _params(batch_size=1) + first = clean_strategy_history(params) + second = clean_strategy_history(params) + + assert first == 2 + assert second == 0 + assert _remaining_ids() == {kept.id, kept_snap.id, kept_delete.id} + + def test_command_real_delete_end_to_end(self, monkeypatch): + """管理命令应真正删除过期冗余历史并输出删除数量。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("cmd-real") + old = _create_history(strategy.id, OLDEST) + kept = _create_history(strategy.id, OLD) + stdout = StringIO() + + call_command(CMD, days=DAYS, batch_size=1, keep_latest_snapshots=1, execute=True, stdout=stdout) + + assert "deleted 1 records" in stdout.getvalue() + assert not StrategyHistoryModel.objects.filter(id=old.id).exists() + assert StrategyHistoryModel.objects.filter(id=kept.id).exists() + + def test_command_rejects_non_positive_keep_latest_snapshots(self): + """命令层应将非法 keep_latest_snapshots 转为 CommandError。""" + with pytest.raises(CommandError, match="must be a positive integer"): + call_command(CMD, days=DAYS, keep_latest_snapshots=0, stdout=StringIO()) + + def test_params_reject_bool_and_non_int_values(self): + """bool / 浮点 / 字符串都不能冒充正整数参数。""" + for kwargs in ( + {"days": True, "batch_size": 1000, "keep_latest_snapshots": 1}, + {"days": 30, "batch_size": 1.5, "keep_latest_snapshots": 1}, + {"days": 30, "batch_size": 1000, "keep_latest_snapshots": "1"}, + ): + with pytest.raises(ValueError, match="must be a positive integer"): + CleanStrategyHistoryParams(**kwargs) + + def test_params_reject_non_boolean_dry_run(self): + """dry_run 只接受布尔值，避免字符串参数误触真实删除。""" + with pytest.raises(ValueError, match="dry_run must be a boolean"): + CleanStrategyHistoryParams(days=30, dry_run="true") + + def test_no_old_histories_returns_zero(self, monkeypatch): + """没有过期历史时返回 0。""" + _freeze_now(monkeypatch) + strategy = _create_strategy("no-old") + recent = _create_history(strategy.id, RECENT) + + assert clean_strategy_history(_params()) == 0 + assert _remaining_ids() == {recent.id} + + def test_full_matrix_across_existing_deleted_zero_and_chunks(self, monkeypatch): + """综合矩阵：现存策略 / 已删策略 / strategy_id=0 / 多 chunk / 窗口内噪声。""" + _freeze_now(monkeypatch) + monkeypatch.setattr("bkmonitor.strategy.history.STRATEGY_ID_CHUNK_SIZE", 2) + + existing = _create_strategy("matrix-existing") + existing_old = _create_history(existing.id, OLDEST, operate="update") + existing_kept = _create_history(existing.id, OLD, operate="bulk_update") + + failed_noise_strategy = _create_strategy("matrix-failed") + failed_old = _create_history( + failed_noise_strategy.id, + OLDEST, + operate="update", + status=False, + message="update failed", + ) + failed_kept = _create_history(failed_noise_strategy.id, OLD, operate="update") + + deleted_id = 910003 + deleted_old_snap = _create_history(deleted_id, OLDEST, operate="update") + deleted_kept_snap = _create_history(deleted_id, OLDER, operate="update") + deleted_old_del = _create_history(deleted_id, OLD - timedelta(minutes=1), operate="delete", status=False) + deleted_kept_del = _create_history(deleted_id, OLD, operate="delete", status=False) + + zero_old = _create_history(0, OLDEST, operate="create") + zero_kept = _create_history(0, OLD, operate="create") + + recent_noise = _create_history(existing.id, RECENT, status=False, message="") + + deleted = clean_strategy_history(_params(batch_size=1)) + + assert deleted == 5 + assert _remaining_ids() == { + existing_kept.id, + failed_kept.id, + deleted_kept_snap.id, + deleted_kept_del.id, + zero_kept.id, + recent_noise.id, + } + assert not StrategyHistoryModel.objects.filter( + id__in=[ + existing_old.id, + failed_old.id, + deleted_old_snap.id, + deleted_old_del.id, + zero_old.id, + ] + ).exists() diff --git a/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup.py b/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup.py new file mode 100644 index 00000000000..d057ff457af --- /dev/null +++ b/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from datetime import datetime +from datetime import timedelta + +import pytest +from django.utils import timezone + +from bkmonitor.models import StrategyHistoryModel +from bkmonitor.models import StrategyModel +from bkmonitor.strategy.history import CleanStrategyHistoryParams +from bkmonitor.strategy.history import clean_strategy_history +from bkmonitor.strategy.history import clean_strategy_history_compat + +pytestmark = pytest.mark.django_db(databases=("default", "monitor_api")) + + +def _create_strategy(name: str) -> StrategyModel: + """创建清理测试使用的最小策略记录。""" + return StrategyModel.objects.create( + bk_biz_id=2, + name=name, + scenario="os", + type=StrategyModel.StrategyType.Monitor, + ) + + +def _create_history( + strategy_id: int, + create_time: datetime, + *, + operate: str = "update", + status: bool = True, + content: dict | None = None, + message: str = "", +) -> StrategyHistoryModel: + """创建历史并将自动生成的时间调整到指定测试时间。""" + history = StrategyHistoryModel.objects.create( + strategy_id=strategy_id, + create_user="admin", + operate=operate, + status=status, + content=content if content is not None else {"id": strategy_id}, + message=message, + ) + StrategyHistoryModel.objects.filter(id=history.id).update(create_time=create_time) + history.create_time = create_time + return history + + +def test_compat_cleanup_keeps_legacy_bulk_update_without_displacing_confirmed_snapshot(monkeypatch): + """兼容快照使用独立保留名额，不能挤掉已确认成功快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("legacy-bulk-status") + + old_confirmed_snapshot = _create_history(strategy.id, now - timedelta(days=45)) + kept_legacy_update = _create_history( + strategy.id, + now - timedelta(days=40), + status=False, + message="", + ) + stale_legacy_delete = _create_history( + strategy.id, + now - timedelta(days=35), + operate="delete", + status=False, + content={}, + message="", + ) + confirmed_failure = _create_history( + strategy.id, + now - timedelta(days=34), + status=False, + message="bulk update failed", + ) + + deleted = clean_strategy_history_compat(CleanStrategyHistoryParams(days=30, batch_size=1)) + + assert deleted == 2 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == { + old_confirmed_snapshot.id, + kept_legacy_update.id, + } + assert not StrategyHistoryModel.objects.filter(id=kept_legacy_update.id, status=True).exists() + assert not StrategyHistoryModel.objects.filter(id=stale_legacy_delete.id).exists() + assert not StrategyHistoryModel.objects.filter(id=confirmed_failure.id).exists() + + +def test_compat_cleanup_keeps_latest_delete_for_deleted_strategy(monkeypatch): + """兼容清理为已删除策略保留最新删除记录，不依赖 status。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + deleted_strategy_id = 900099 + + snapshot = _create_history(deleted_strategy_id, now - timedelta(days=45)) + old_delete = _create_history( + deleted_strategy_id, + now - timedelta(days=40), + operate="delete", + status=False, + content={}, + ) + kept_delete = _create_history( + deleted_strategy_id, + now - timedelta(days=35), + operate="delete", + status=False, + content={}, + ) + + deleted = clean_strategy_history_compat(CleanStrategyHistoryParams(days=30, batch_size=1)) + + assert deleted == 1 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {snapshot.id, kept_delete.id} + assert not StrategyHistoryModel.objects.filter(id=old_delete.id).exists() + + +def test_compat_cleanup_does_not_treat_recent_failed_update_as_legacy(monkeypatch): + """窗口内普通失败更新不能挤掉窗口外已确认成功的快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("recent-failed-update") + + confirmed_snapshot = _create_history(strategy.id, now - timedelta(days=40)) + recent_failed_update = _create_history( + strategy.id, + now - timedelta(days=10), + status=False, + message="", + ) + + deleted = clean_strategy_history_compat(CleanStrategyHistoryParams(days=30)) + + assert deleted == 0 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == { + confirmed_snapshot.id, + recent_failed_update.id, + } + + +def test_compat_cleanup_keeps_latest_legacy_when_no_confirmed_snapshot(monkeypatch): + """无确认成功快照时，兼容清理仍应保留最新有内容的旧批量 update。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("legacy-only") + + older_legacy = _create_history( + strategy.id, + now - timedelta(days=45), + status=False, + message="", + ) + kept_legacy = _create_history( + strategy.id, + now - timedelta(days=35), + status=False, + message="", + ) + + deleted = clean_strategy_history_compat(CleanStrategyHistoryParams(days=30, batch_size=1)) + + assert deleted == 1 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_legacy.id} + assert not StrategyHistoryModel.objects.filter(id=older_legacy.id).exists() + + +def test_compat_cleanup_excludes_empty_content_legacy_from_keep_slots(monkeypatch): + """空 content 的旧批量 update 不能占用兼容保留名额。 + + 把空 content 行放在更新时间，若其误占名额会挤掉更早但仍可恢复的 legacy 快照。 + """ + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("empty-legacy-content") + + kept_legacy = _create_history( + strategy.id, + now - timedelta(days=40), + status=False, + message="", + ) + empty_legacy = _create_history( + strategy.id, + now - timedelta(days=35), + status=False, + content={}, + message="", + ) + + deleted = clean_strategy_history_compat(CleanStrategyHistoryParams(days=30, batch_size=1)) + + assert deleted == 1 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_legacy.id} + assert not StrategyHistoryModel.objects.filter(id=empty_legacy.id).exists() + + +def test_existing_strategy_keeps_latest_successful_snapshot(monkeypatch): + """现存策略应只保留全局最新的成功配置快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("existing-strategy") + + old_create = _create_history(strategy.id, now - timedelta(days=40), operate="create") + kept_snapshot = _create_history(strategy.id, now - timedelta(days=35), operate="bulk_update") + failed_update = _create_history(strategy.id, now - timedelta(days=34), status=False) + stale_delete = _create_history(strategy.id, now - timedelta(days=33), operate="delete") + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30, batch_size=2)) + + assert deleted == 3 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_snapshot.id} + assert not StrategyHistoryModel.objects.filter(id__in=[old_create.id, failed_update.id, stale_delete.id]).exists() + + +def test_deleted_strategy_keeps_latest_snapshot_and_latest_delete(monkeypatch): + """已删除策略应保留最新成功快照和最新删除事实。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy_id = 900001 + + old_snapshot = _create_history(strategy_id, now - timedelta(days=45), operate="update") + kept_snapshot = _create_history(strategy_id, now - timedelta(days=40), operate="bulk_update") + old_delete = _create_history(strategy_id, now - timedelta(days=38), operate="delete") + # 修复前的删除历史没有回写成功状态，删除事实仍需按时间保留最新一条。 + kept_delete = _create_history(strategy_id, now - timedelta(days=35), operate="delete", status=False) + failed_update = _create_history(strategy_id, now - timedelta(days=34), status=False) + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30, batch_size=2)) + + assert deleted == 3 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == { + kept_snapshot.id, + kept_delete.id, + } + assert not StrategyHistoryModel.objects.filter(id__in=[old_snapshot.id, old_delete.id, failed_update.id]).exists() + + +def test_recent_snapshot_allows_all_expired_snapshots_to_be_deleted(monkeypatch): + """窗口内已有更新快照时，不再额外保留过期快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("recent-snapshot") + expired_snapshot = _create_history(strategy.id, now - timedelta(days=40)) + recent_snapshot = _create_history(strategy.id, now - timedelta(days=10), operate="bulk_update") + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30)) + + assert deleted == 1 + assert not StrategyHistoryModel.objects.filter(id=expired_snapshot.id).exists() + assert StrategyHistoryModel.objects.filter(id=recent_snapshot.id).exists() + + +def test_failed_and_empty_histories_are_not_recoverable_snapshots(monkeypatch): + """失败记录和空内容记录不能淘汰更早的有效快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("invalid-snapshots") + kept_snapshot = _create_history(strategy.id, now - timedelta(days=45)) + empty_snapshot = _create_history(strategy.id, now - timedelta(days=40), content={}) + failed_snapshot = _create_history(strategy.id, now - timedelta(days=35), status=False) + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30)) + + assert deleted == 2 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_snapshot.id} + assert not StrategyHistoryModel.objects.filter(id__in=[empty_snapshot.id, failed_snapshot.id]).exists() + + +def test_history_at_cutoff_is_not_deleted(monkeypatch): + """清理范围使用严格小于截止时间，边界记录应保留。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("cutoff-boundary") + expired = _create_history(strategy.id, now - timedelta(days=30, seconds=1), status=False) + at_cutoff = _create_history(strategy.id, now - timedelta(days=30), status=False) + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30)) + + assert deleted == 1 + assert not StrategyHistoryModel.objects.filter(id=expired.id).exists() + assert StrategyHistoryModel.objects.filter(id=at_cutoff.id).exists() + + +def test_cleanup_is_idempotent(monkeypatch): + """相同保留周期重复执行不应继续删除保留记录。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("idempotent-cleanup") + _create_history(strategy.id, now - timedelta(days=40)) + kept_snapshot = _create_history(strategy.id, now - timedelta(days=35), operate="bulk_update") + + params = CleanStrategyHistoryParams(days=30, batch_size=1) + first_deleted = clean_strategy_history(params) + second_deleted = clean_strategy_history(params) + + assert first_deleted == 1 + assert second_deleted == 0 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_snapshot.id} + + +def test_dry_run_returns_matched_count_without_deleting_histories(monkeypatch): + """dry-run 应返回预计删除数量，同时保持数据库内容不变。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("dry-run") + removable = _create_history(strategy.id, now - timedelta(days=40)) + kept_snapshot = _create_history(strategy.id, now - timedelta(days=35), operate="bulk_update") + + matched = clean_strategy_history(CleanStrategyHistoryParams(days=30, dry_run=True)) + + assert matched == 1 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == { + removable.id, + kept_snapshot.id, + } + + +def test_keep_latest_snapshots_retains_multiple_successful_versions(monkeypatch): + """keep_latest_snapshots>1 时应保留全局最近多条成功快照。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("multi-snapshots") + + oldest = _create_history(strategy.id, now - timedelta(days=50), operate="create") + kept_older = _create_history(strategy.id, now - timedelta(days=45)) + kept_newer = _create_history(strategy.id, now - timedelta(days=40), operate="bulk_update") + failed = _create_history(strategy.id, now - timedelta(days=35), status=False) + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30, keep_latest_snapshots=2)) + + assert deleted == 2 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {kept_older.id, kept_newer.id} + assert not StrategyHistoryModel.objects.filter(id__in=[oldest.id, failed.id]).exists() + + +def test_keep_latest_snapshots_counts_recent_window_versions(monkeypatch): + """窗口内成功快照会计入 keep_latest_snapshots，从而减少窗外保留条数。""" + now = timezone.make_aware(datetime(2026, 7, 21, 12, 0, 0)) + monkeypatch.setattr("bkmonitor.strategy.history.timezone.now", lambda: now) + strategy = _create_strategy("window-counts") + + expired_old = _create_history(strategy.id, now - timedelta(days=50), operate="create") + expired_kept = _create_history(strategy.id, now - timedelta(days=40)) + recent = _create_history(strategy.id, now - timedelta(days=10), operate="bulk_update") + + deleted = clean_strategy_history(CleanStrategyHistoryParams(days=30, keep_latest_snapshots=2)) + + assert deleted == 1 + assert set(StrategyHistoryModel.objects.values_list("id", flat=True)) == {expired_kept.id, recent.id} + assert not StrategyHistoryModel.objects.filter(id=expired_old.id).exists() + + +@pytest.mark.parametrize( + ("days", "batch_size", "keep_latest_snapshots"), + [ + (0, 1000, 1), + (-1, 1000, 1), + (30, 0, 1), + (30, -1, 1), + (30, 1000, 0), + (30, 1000, -1), + ], +) +def test_cleanup_rejects_non_positive_arguments(days, batch_size, keep_latest_snapshots): + """保留天数、删除批次和快照保留条数必须为正整数。""" + with pytest.raises(ValueError): + CleanStrategyHistoryParams(days=days, batch_size=batch_size, keep_latest_snapshots=keep_latest_snapshots) diff --git a/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup_command.py b/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup_command.py new file mode 100644 index 00000000000..e25bf50f613 --- /dev/null +++ b/bkmonitor/bkmonitor/strategy/tests/test_history_cleanup_command.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from io import StringIO +from unittest import mock + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError + +from bkmonitor.strategy.history import MIN_CLEAN_STRATEGY_HISTORY_DAYS + + +def test_command_passes_cleanup_options_and_prints_deleted_count(): + """带 --execute 时应真正删除，并将参数传给业务层。""" + stdout = StringIO() + + with mock.patch( + "bkmonitor.management.commands.clean_strategy_history.clean_strategy_history", + return_value=42, + ) as clean: + call_command( + "clean_strategy_history", + days=30, + batch_size=200, + keep_latest_snapshots=3, + execute=True, + stdout=stdout, + ) + + params = clean.call_args.args[0] + assert params.days == 30 + assert params.batch_size == 200 + assert params.keep_latest_snapshots == 3 + assert params.dry_run is False + assert "deleted 42 records" in stdout.getvalue() + + +def test_command_defaults_to_dry_run_without_execute(): + """未传 --execute 时应默认 dry-run，不删除。""" + stdout = StringIO() + + with mock.patch( + "bkmonitor.management.commands.clean_strategy_history.clean_strategy_history", + return_value=42, + ) as clean: + call_command("clean_strategy_history", days=30, stdout=stdout) + + params = clean.call_args.args[0] + assert params.batch_size == 1000 + assert params.keep_latest_snapshots == 1 + assert params.dry_run is True + assert "would delete 42 records" in stdout.getvalue() + assert "deleted 42 records" not in stdout.getvalue() + + +def test_compat_command_uses_safe_legacy_status_cleanup(): + """临时兼容命令应调用兼容清理入口，并保持默认 dry-run。""" + stdout = StringIO() + + with mock.patch( + "bkmonitor.management.commands.clean_strategy_history_compat.clean_strategy_history_compat", + return_value=7, + ) as clean: + call_command("clean_strategy_history_compat", days=30, stdout=stdout) + + params = clean.call_args.args[0] + assert params.dry_run is True + assert "deprecated compatibility command" in stdout.getvalue() + assert "would delete 7 records" in stdout.getvalue() + + +def test_command_rejects_days_below_minimum_retention(): + """命令层拒绝低于最小保留天数的 --days。""" + with pytest.raises(CommandError, match=rf"days must be >= {MIN_CLEAN_STRATEGY_HISTORY_DAYS}"): + call_command( + "clean_strategy_history", + days=MIN_CLEAN_STRATEGY_HISTORY_DAYS - 1, + stdout=StringIO(), + ) + + +@pytest.mark.parametrize( + ("option", "value"), + [ + ("days", 0), + ("batch_size", 0), + ("keep_latest_snapshots", 0), + ], +) +def test_command_reports_invalid_cleanup_options_as_command_error(option, value): + """业务参数校验错误应转换为 Django 命令错误。""" + options = { + "days": 30, + "batch_size": 1000, + "keep_latest_snapshots": 1, + option: value, + } + + with pytest.raises(CommandError, match="must be a positive integer"): + call_command("clean_strategy_history", stdout=StringIO(), **options) diff --git a/bkmonitor/bkmonitor/strategy/tests/test_strategy_history.py b/bkmonitor/bkmonitor/strategy/tests/test_strategy_history.py new file mode 100644 index 00000000000..e5b530c53bf --- /dev/null +++ b/bkmonitor/bkmonitor/strategy/tests/test_strategy_history.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- # noqa: UP009 +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from contextlib import ExitStack +from types import SimpleNamespace +from unittest import mock + +import pytest +from django.db import connections + +from alarm_backends.core.cache.strategy import StrategyCacheManager +from bkmonitor.db_routers import UsingDB +from bkmonitor.models import AlgorithmModel +from bkmonitor.models import DetectModel +from bkmonitor.models import ItemModel +from bkmonitor.models import QueryConfigModel +from bkmonitor.models import StrategyActionConfigRelation +from bkmonitor.models import StrategyHistoryModel +from bkmonitor.models import StrategyLabel +from bkmonitor.models import StrategyModel +from bkmonitor.models.issue import StrategyIssueConfig +from bkmonitor.strategy.new_strategy import Strategy +from monitor_web.strategies.resources.v2 import UpdatePartialStrategyV2Resource + + +def test_strategy_history_operate_choices_include_only_bulk_update(): + choices = dict(StrategyHistoryModel._meta.get_field("operate").choices) + + assert choices["bulk_update"] == "批量更新" + assert "bulk_delete" not in choices + + +@pytest.mark.django_db +def test_bulk_delete_does_not_create_success_history_when_delete_fails(): + """中途删除失败时不应写入成功历史；事务保证与主表删除一并回滚。""" + strategy_id = 1001 + + def raise_delete_error(): + raise RuntimeError("delete relation failed") + + with ( + mock.patch.object(StrategyHistoryModel.objects, "bulk_create") as bulk_create_history, + mock.patch.object(StrategyModel.objects, "filter") as strategy_filter, + mock.patch.object(StrategyActionConfigRelation.objects, "filter") as relation_filter, + ): + relation_filter.return_value.delete.side_effect = raise_delete_error + + with pytest.raises(RuntimeError, match="delete relation failed"): + Strategy.delete_by_strategy_ids([strategy_id]) + + strategy_filter.return_value.delete.assert_called_once_with() + bulk_create_history.assert_not_called() + + +@pytest.mark.django_db(databases=("default", "monitor_api"), transaction=True) +def test_bulk_delete_opens_transaction_on_monitor_api(): + """批量删除必须在实际承载策略模型写入的 monitor_api 库开启事务。""" + transaction_state = {} + + def inspect_transaction(): + transaction_state.update( + default=connections["default"].in_atomic_block, + monitor_api=connections["monitor_api"].in_atomic_block, + ) + raise RuntimeError("stop after inspecting transaction") + + with ( + mock.patch.object(StrategyModel.objects, "filter") as strategy_filter, + mock.patch.object(Strategy, "_get_username", return_value="admin"), + ): + strategy_filter.return_value.delete.side_effect = inspect_transaction + + with pytest.raises(RuntimeError, match="stop after inspecting transaction"): + Strategy.delete_by_strategy_ids([1001]) + + assert transaction_state == {"default": False, "monitor_api": True} + + +@pytest.mark.django_db(databases=("default", "monitor_api"), transaction=True) +def test_bulk_delete_opens_transaction_on_runtime_routed_database(): + """动态路由覆盖时，事务必须与实际写入的数据库保持一致。""" + transaction_state = {} + + def inspect_transaction(): + transaction_state.update( + default=connections["default"].in_atomic_block, + monitor_api=connections["monitor_api"].in_atomic_block, + ) + raise RuntimeError("stop after inspecting transaction") + + with ( + mock.patch.object(StrategyModel.objects, "filter") as strategy_filter, + mock.patch.object(Strategy, "_get_username", return_value="admin"), + ): + strategy_filter.return_value.delete.side_effect = inspect_transaction + + with pytest.raises(RuntimeError, match="stop after inspecting transaction"): + with UsingDB("default"): + Strategy.delete_by_strategy_ids([1001]) + + assert transaction_state == {"default": True, "monitor_api": False} + + +@pytest.mark.django_db +def test_bulk_delete_creates_delete_histories_after_all_data_is_deleted(): + strategy_ids = [1001, 1002] + delete_models = ( + StrategyModel, + StrategyActionConfigRelation, + DetectModel, + ItemModel, + AlgorithmModel, + QueryConfigModel, + StrategyLabel, + StrategyIssueConfig, + ) + operations = mock.Mock() + + with ExitStack() as stack: + for index, model in enumerate(delete_models): + model_filter = stack.enter_context(mock.patch.object(model.objects, "filter")) + operations.attach_mock(model_filter.return_value.delete, f"delete_{index}") + + bulk_create_history = stack.enter_context(mock.patch.object(StrategyHistoryModel.objects, "bulk_create")) + operations.attach_mock(bulk_create_history, "create_history") + stack.enter_context(mock.patch.object(Strategy, "_get_username", return_value="admin")) + + Strategy.delete_by_strategy_ids(strategy_ids) + + histories = bulk_create_history.call_args.args[0] + assert [(history.strategy_id, history.operate, history.status) for history in histories] == [ + (1001, "delete", True), + (1002, "delete", True), + ] + assert operations.mock_calls[-1] == mock.call.create_history(histories, batch_size=100) + + +@pytest.mark.django_db +def test_bulk_update_creates_success_history_with_bulk_update_type(): + strategy_id = 1001 + strategy_queryset = mock.Mock() + strategy = mock.Mock() + strategy.id = strategy_id + strategy.instance = mock.Mock() + strategy.items = [] + strategy.to_dict.return_value = {"bk_biz_id": 2, "id": strategy_id, "is_enabled": False} + + with ( + mock.patch.object(StrategyModel.objects, "filter", return_value=strategy_queryset), + mock.patch.object(StrategyModel.objects, "bulk_update"), + mock.patch.object(StrategyHistoryModel.objects, "bulk_create") as bulk_create_history, + mock.patch.object(Strategy, "from_models", return_value=[strategy]), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_relations", return_value=([], {})), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_action_configs", return_value={}), + mock.patch("monitor_web.strategies.resources.v2.get_global_user", return_value="admin"), + ): + result = UpdatePartialStrategyV2Resource().perform_request( + {"bk_biz_id": 2, "edit_data": {"is_enabled": False}, "ids": [strategy_id]} + ) + + histories = bulk_create_history.call_args.args[0] + assert result == [strategy_id] + assert len(histories) == 1 + assert histories[0].strategy_id == strategy_id + assert histories[0].operate == "bulk_update" + assert histories[0].status is True + assert histories[0].content == {"bk_biz_id": 2, "id": strategy_id, "is_enabled": False} + strategy_queryset.update.assert_called_once_with(hash="", snippet="") + + +@pytest.mark.django_db +def test_bulk_update_does_not_create_success_history_when_update_fails(): + """中途 bulk_update 失败时不应写入成功历史；事务保证与配置更新一并回滚。""" + strategy_id = 1001 + strategy_queryset = mock.Mock() + + def raise_update_error(*_args, **_kwargs): + raise RuntimeError("bulk update failed") + + with ( + mock.patch.object(StrategyModel.objects, "filter", return_value=strategy_queryset), + mock.patch.object(StrategyModel.objects, "bulk_update", side_effect=raise_update_error), + mock.patch.object(StrategyHistoryModel.objects, "bulk_create") as bulk_create_history, + mock.patch.object(Strategy, "from_models", return_value=[]), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_relations", return_value=([], {})), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_action_configs", return_value={}), + mock.patch("monitor_web.strategies.resources.v2.get_global_user", return_value="admin"), + ): + with pytest.raises(RuntimeError, match="bulk update failed"): + UpdatePartialStrategyV2Resource().perform_request( + {"bk_biz_id": 2, "edit_data": {"is_enabled": False}, "ids": [strategy_id]} + ) + + bulk_create_history.assert_not_called() + + +@pytest.mark.django_db(databases=("default", "monitor_api"), transaction=True) +def test_bulk_update_opens_transaction_on_monitor_api_before_processing_updates(): + """直接写库的局部更新方法必须位于 monitor_api 事务中。""" + strategy_queryset = mock.Mock() + strategy = mock.Mock(id=1001, instance=mock.Mock(), items=[]) + strategy.to_dict.return_value = {"bk_biz_id": 2, "id": 1001, "labels": ["updated"]} + + def update_labels(_strategy, _labels): + assert connections["default"].in_atomic_block is False + assert connections["monitor_api"].in_atomic_block is True + return None, [], [] + + with ( + mock.patch.object(StrategyModel.objects, "filter", return_value=strategy_queryset), + mock.patch.object(StrategyModel.objects, "bulk_update"), + mock.patch.object(StrategyHistoryModel.objects, "bulk_create"), + mock.patch.object(Strategy, "from_models", return_value=[strategy]), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_relations", return_value=([], {})), + mock.patch.object(UpdatePartialStrategyV2Resource, "get_action_configs", return_value={}), + mock.patch.object(UpdatePartialStrategyV2Resource, "update_labels", side_effect=update_labels), + mock.patch("monitor_web.strategies.resources.v2.get_global_user", return_value="admin"), + ): + UpdatePartialStrategyV2Resource().perform_request( + {"bk_biz_id": 2, "edit_data": {"labels": {"labels": ["updated"]}}, "ids": [1001]} + ) + + +def test_strategy_cache_handles_bulk_update_and_delete_histories(): + histories = [ + SimpleNamespace( + strategy_id=1001, + operate="bulk_update", + content={"bk_biz_id": 2, "is_enabled": True}, + ), + SimpleNamespace( + strategy_id=1002, + operate="bulk_update", + content={"bk_biz_id": 3, "is_enabled": False}, + ), + SimpleNamespace(strategy_id=1003, operate="delete", content={}), + ] + + with mock.patch.object(StrategyCacheManager, "get_strategy_by_id", return_value=None): + target_biz_ids, deleted_strategy_ids = StrategyCacheManager.handle_history_strategies( + histories, with_group_key=False + ) + + assert target_biz_ids == {2, 3} + assert deleted_strategy_ids == {(1002, ""), (1003, "")} diff --git a/bkmonitor/packages/monitor_web/strategies/resources/v2.py b/bkmonitor/packages/monitor_web/strategies/resources/v2.py index 2fe86f4baa8..aded20b6953 100644 --- a/bkmonitor/packages/monitor_web/strategies/resources/v2.py +++ b/bkmonitor/packages/monitor_web/strategies/resources/v2.py @@ -14,7 +14,7 @@ import arrow import pytz from django.conf import settings -from django.db import transaction +from django.db import router, transaction from django.db.models import Count, ExpressionWrapper, F, Q, QuerySet, fields from django.utils.translation import gettext_lazy as _ from rest_framework import serializers @@ -2498,6 +2498,10 @@ def process_extra_data( create_datas[f"extra_{key}_relation"]["objs"].extend(data["objs"]) def perform_request(self, params): + with transaction.atomic(using=router.db_for_write(StrategyModel)): + return self._perform_request(params) + + def _perform_request(self, params): bk_biz_id = params["bk_biz_id"] config: dict = params["edit_data"] username = get_global_user() @@ -2550,7 +2554,8 @@ def perform_request(self, params): StrategyHistoryModel( create_user=username, strategy_id=strategy.id, - operate="update", + operate="bulk_update", + status=True, content=strategy.to_dict(), ) )