diff --git a/bkmonitor/alarm_backends/core/alarm_engine/__init__.py b/bkmonitor/alarm_backends/core/alarm_engine/__init__.py new file mode 100644 index 00000000000..239267df83d --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/__init__.py @@ -0,0 +1,9 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/bkmonitor/alarm_backends/core/alarm_engine/contract.py b/bkmonitor/alarm_backends/core/alarm_engine/contract.py new file mode 100644 index 00000000000..6953e7b41f8 --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/contract.py @@ -0,0 +1,874 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import base64 +import copy +import hashlib +import json +import math +import re +import struct +from collections.abc import Mapping +from typing import Any + + +SCHEMA_MAJOR = 1 +SCHEMA_MINOR = 0 +DETECTION_OUTCOME_SCHEMA = "detection-outcome" +TRIGGER_STRATEGY_IR_SCHEMA = "trigger-strategy-ir" +TRIGGER_DECISION_BATCH_SCHEMA = "trigger-decision-batch" +TRIGGER_DECISION_ALGORITHM = "trigger-window-v1" +TRIGGER_DECISION_ID_VERSION = "trigger-decision-id-v1" +TRIGGER_PARTITION_HASH_VERSION = "trigger-input-partition-v1" + +FEATURE_FULL_LEVEL_EVALUATIONS = "full-level-evaluations-v1" +FEATURE_RAW_JSON = "raw-json-v1" +FEATURE_RAW_STRATEGY_BYTES = "raw-strategy-bytes-v1" +PURPOSES = {"DETECT", "NODATA"} +EVALUATION_RESULTS = {"NORMAL", "ANOMALOUS"} +OUTCOMES = {"NORMAL", "ANOMALOUS", "ERROR", "UNSUPPORTED"} +ERROR_CODES = { + "ERROR": {"ALGORITHM_ERROR", "INTERNAL_ERROR", "INVALID_INPUT"}, + "UNSUPPORTED": {"UNSUPPORTED_FEATURE", "UNSUPPORTED_STRATEGY"}, +} + +_DECIMAL_RE = re.compile(r"(?:0|[1-9][0-9]*)\Z") +_SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +_RECORD_ID_RE = re.compile(r"(?P[0-9a-f]{32})\.(?P0|[1-9][0-9]*)\Z") +_MAX_INT64 = 2**63 - 1 +_MAX_CONTRACT_INT = 2**31 - 1 + + +class ContractValidationError(ValueError): + """Raised when a contract document cannot be interpreted safely.""" + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ContractValidationError(f"duplicate JSON field: {key}") + result[key] = value + return result + + +def _reject_nonfinite_json(value: str) -> None: + raise ContractValidationError(f"non-finite JSON number: {value}") + + +def _parse_finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ContractValidationError(f"non-finite JSON number: {value}") + return parsed + + +def _decode_strict_json_object(payload: bytes, field: str) -> dict: + if payload.startswith(b"\xef\xbb\xbf"): + raise ContractValidationError(f"{field} must not contain a UTF-8 BOM") + try: + decoded = json.loads( + payload, + object_pairs_hook=_reject_duplicate_json_keys, + parse_constant=_reject_nonfinite_json, + parse_float=_parse_finite_float, + ) + except ContractValidationError: + raise + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ContractValidationError(f"{field} must contain valid UTF-8 JSON") from exc + if not isinstance(decoded, dict): + raise ContractValidationError(f"{field} must contain a JSON object") + _validate_json_strings(decoded, field) + return decoded + + +def _validate_json_strings(value: Any, field: str) -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + _validate_utf8_string(key, f"{field} object key") + _validate_json_strings(child, field) + elif isinstance(value, list): + for child in value: + _validate_json_strings(child, field) + elif isinstance(value, str): + _validate_utf8_string(value, field) + + +def _validate_utf8_string(value: Any, field: str) -> str: + if not isinstance(value, str): + raise ContractValidationError(f"{field} must be a string") + try: + value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ContractValidationError(f"{field} must contain valid UTF-8") from exc + return value + + +def _require_mapping(value: Any, field: str) -> Mapping: + if not isinstance(value, Mapping): + raise ContractValidationError(f"{field} must be an object") + return value + + +def _validate_fixed_fields( + value: Any, + field: str, + *, + required: set[str], + optional: set[str] | None = None, + schema_minor: int = SCHEMA_MINOR, + allow_open: bool = False, +) -> Mapping: + value = _require_mapping(value, field) + optional = optional or set() + known = required | optional + if any(not isinstance(key, str) for key in value): + raise ContractValidationError(f"{field} field names must be strings") + missing = required - set(value) + if missing: + raise ContractValidationError(f"{field} missing required field: {sorted(missing)[0]}") + known_casefold = {key.casefold(): key for key in known} + unknown = set(value) - known + for key in unknown: + canonical = known_casefold.get(key.casefold()) + if canonical is not None: + raise ContractValidationError(f"{field}.{key} case-collides with field {canonical}") + if unknown and not allow_open and schema_minor <= SCHEMA_MINOR: + raise ContractValidationError(f"{field} contains unknown field: {sorted(unknown)[0]}") + return value + + +def _require_nonempty_string(value: Any, field: str) -> str: + if not isinstance(value, str) or not value: + raise ContractValidationError(f"{field} must be a non-empty string") + return _validate_utf8_string(value, field) + + +def _canonical_decimal(value: Any, field: str) -> str: + if isinstance(value, bool): + raise ContractValidationError(f"{field} must use canonical decimal form") + if isinstance(value, int): + value = str(value) + if not isinstance(value, str) or not _DECIMAL_RE.fullmatch(value): + raise ContractValidationError(f"{field} must use canonical decimal form") + return value + + +def _require_positive_int(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0 or value > _MAX_CONTRACT_INT: + raise ContractValidationError(f"{field} must be a positive 32-bit signed integer") + return value + + +def _require_source_time(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > _MAX_INT64: + raise ContractValidationError(f"{field} must be a non-negative int64") + return value + + +def _require_sha256(value: Any, field: str) -> str: + if not isinstance(value, str) or not _SHA256_RE.fullmatch(value): + raise ContractValidationError(f"{field} must be 64 lowercase hexadecimal characters") + return value + + +def _parse_record_id(record_id: Any) -> tuple[str, int]: + if not isinstance(record_id, str): + raise ContractValidationError("record_id must be a string") + matched = _RECORD_ID_RE.fullmatch(record_id) + if not matched: + raise ContractValidationError("record_id must use dimensions_md5.source_time canonical form") + source_time = int(matched.group("source_time")) + if source_time > _MAX_INT64: + raise ContractValidationError("record source_time exceeds int64") + return matched.group("dimensions_md5"), source_time + + +def _validate_header(document: Mapping, *, name: str, required_features: set[str]) -> int: + schema = _require_mapping(document.get("schema"), "schema") + if schema.get("name") != name: + raise ContractValidationError(f"schema.name must be {name}") + major = schema.get("major") + if isinstance(major, bool) or not isinstance(major, int) or major != SCHEMA_MAJOR: + raise ContractValidationError(f"unsupported {name} schema major") + minor = schema.get("minor") + if isinstance(minor, bool) or not isinstance(minor, int) or minor < 0 or minor > _MAX_CONTRACT_INT: + raise ContractValidationError("schema.minor must be a non-negative 32-bit signed integer") + _validate_fixed_fields( + schema, + "schema", + required={"name", "major", "minor"}, + schema_minor=minor, + ) + + features = document.get("required_features") + if not isinstance(features, list) or any(not isinstance(feature, str) for feature in features): + raise ContractValidationError("required_features must be a string array") + if len(features) != len(set(features)): + raise ContractValidationError("required_features contains duplicate values") + unknown_features = set(features) - required_features + if unknown_features: + raise ContractValidationError(f"unsupported required feature: {sorted(unknown_features)[0]}") + if not required_features.issubset(features): + raise ContractValidationError(f"missing required feature: {sorted(required_features - set(features))[0]}") + return minor + + +def _normalize_purpose(value: Any) -> str: + if not isinstance(value, str) or value not in PURPOSES: + raise ContractValidationError(f"unsupported purpose: {value}") + return value + + +def _json_values_equal(left: Any, right: Any) -> bool: + if isinstance(left, Mapping) or isinstance(right, Mapping): + return ( + isinstance(left, Mapping) + and isinstance(right, Mapping) + and left.keys() == right.keys() + and all(_json_values_equal(left[key], right[key]) for key in left) + ) + if isinstance(left, list) or isinstance(right, list): + return ( + isinstance(left, list) + and isinstance(right, list) + and len(left) == len(right) + and all(_json_values_equal(left_value, right_value) for left_value, right_value in zip(left, right)) + ) + return type(left) is type(right) and left == right + + +def json_values_equal(left: Any, right: Any) -> bool: + """Compare JSON values without Python's bool/int or int/float coercion.""" + return _json_values_equal(left, right) + + +def _normalize_strategy_ref(strategy_ref: Any, *, schema_minor: int = SCHEMA_MINOR) -> dict[str, str]: + strategy_ref = _validate_fixed_fields( + strategy_ref, + "strategy_ref", + required={"strategy_id", "item_id", "generation", "content_sha256"}, + schema_minor=schema_minor, + ) + return { + "strategy_id": _canonical_decimal(strategy_ref.get("strategy_id"), "strategy_ref.strategy_id"), + "item_id": _canonical_decimal(strategy_ref.get("item_id"), "strategy_ref.item_id"), + "generation": _require_nonempty_string(strategy_ref.get("generation"), "strategy_ref.generation"), + "content_sha256": _require_sha256(strategy_ref.get("content_sha256"), "strategy_ref.content_sha256"), + } + + +def derive_input_id( + *, + tenant_id: str, + purpose: str, + strategy_id: str | int, + item_id: str | int, + strategy_content_sha256: str, + record_id: str, +) -> str: + """Derive the v1 replay-stable input ID from its frozen canonical tuple.""" + + fields = ( + _require_nonempty_string(tenant_id, "tenant_id"), + _normalize_purpose(purpose), + _canonical_decimal(strategy_id, "strategy_id"), + _canonical_decimal(item_id, "item_id"), + _require_sha256(strategy_content_sha256, "strategy_content_sha256"), + record_id, + ) + _parse_record_id(record_id) + + digest = hashlib.sha256() + for field in fields: + try: + encoded = field.encode("utf-8") + except UnicodeEncodeError as exc: + raise ContractValidationError("input_id canonical fields must contain valid UTF-8") from exc + if len(encoded) > 2**32 - 1: + raise ContractValidationError("input_id canonical field exceeds uint32 length") + digest.update(struct.pack(">I", len(encoded))) + digest.update(encoded) + return digest.hexdigest() + + +def derive_trigger_decision_id(input_id: str) -> str: + """Derive the stable Trigger decision coordinate shared with the Go evaluator.""" + + fields = ( + TRIGGER_DECISION_ID_VERSION, + TRIGGER_DECISION_ALGORITHM, + _require_sha256(input_id, "decision_id.input_id"), + ) + digest = hashlib.sha256() + for field in fields: + encoded = field.encode("utf-8") + digest.update(struct.pack(">I", len(encoded))) + digest.update(encoded) + return digest.hexdigest() + + +def build_trigger_strategy_ir( + *, + tenant_id: str, + purpose: str, + strategy_id: str | int, + item_id: str | int, + generation: str, + legacy_json: bytes, + check_window_unit_seconds: int, + trigger_configs: Mapping[int | str, Mapping[str, int]], +) -> dict: + """Build the minimal immutable StrategyIR needed by Trigger v1.""" + + if not isinstance(legacy_json, bytes) or not legacy_json: + raise ContractValidationError("legacy_json must be non-empty bytes") + _decode_strict_json_object(legacy_json, "legacy_json") + trigger_configs = _require_mapping(trigger_configs, "trigger_configs") + + normalized_configs = [] + seen_levels = set() + for raw_level, raw_config in trigger_configs.items(): + level = int(_canonical_decimal(raw_level, "trigger_configs.level")) + if level <= 0: + raise ContractValidationError("trigger_configs.level must be positive") + if level in seen_levels: + raise ContractValidationError("trigger_configs contains duplicate level") + seen_levels.add(level) + config = _require_mapping(raw_config, f"trigger_configs[{level}]") + normalized_configs.append( + { + "level": level, + "check_window_size": _require_positive_int( + config.get("check_window_size"), f"trigger_configs[{level}].check_window_size" + ), + "trigger_count": _require_positive_int( + config.get("trigger_count"), f"trigger_configs[{level}].trigger_count" + ), + } + ) + if not normalized_configs: + raise ContractValidationError("trigger_configs must not be empty") + normalized_configs.sort(key=lambda config: config["level"]) + + content_sha256 = hashlib.sha256(legacy_json).hexdigest() + strategy_ir = { + "schema": {"name": TRIGGER_STRATEGY_IR_SCHEMA, "major": SCHEMA_MAJOR, "minor": SCHEMA_MINOR}, + "required_features": [FEATURE_RAW_STRATEGY_BYTES], + "tenant_id": _require_nonempty_string(tenant_id, "tenant_id"), + "purpose": _normalize_purpose(purpose), + "strategy_ref": { + "strategy_id": _canonical_decimal(strategy_id, "strategy_id"), + "item_id": _canonical_decimal(item_id, "item_id"), + "generation": _require_nonempty_string(generation, "generation"), + "content_sha256": content_sha256, + }, + "required_levels": [config["level"] for config in normalized_configs], + "check_window_unit_seconds": _require_positive_int(check_window_unit_seconds, "check_window_unit_seconds"), + "trigger_configs": normalized_configs, + "legacy_json_b64": base64.b64encode(legacy_json).decode("ascii"), + } + validate_trigger_strategy_ir(strategy_ir) + return strategy_ir + + +def build_trigger_strategy_ir_from_legacy_config( + *, + tenant_id: str, + purpose: str, + strategy: Mapping, + item_id: str | int, + legacy_json: bytes, +) -> dict: + """Project an eligible legacy Threshold strategy into the minimal Trigger StrategyIR.""" + + strategy = _require_mapping(strategy, "strategy") + if purpose != "DETECT": + raise ContractValidationError("unsupported purpose for the first Threshold contract slice") + if not isinstance(legacy_json, bytes) or not legacy_json: + raise ContractValidationError("legacy_json must be non-empty bytes") + decoded_legacy = _decode_strict_json_object(legacy_json, "legacy_json") + if not _json_values_equal(decoded_legacy, strategy): + raise ContractValidationError("legacy_json must represent the supplied strategy without semantic drift") + + normalized_item_id = _canonical_decimal(item_id, "item_id") + items = strategy.get("items") + if not isinstance(items, list) or not items: + raise ContractValidationError("unsupported strategy without items") + item = None + for strategy_item in items: + strategy_item = _require_mapping(strategy_item, "strategy item") + if _canonical_decimal(strategy_item.get("id"), "strategy item id") == normalized_item_id: + item = strategy_item + break + if item is None: + raise ContractValidationError("unsupported strategy item: item_id not found") + algorithms = item.get("algorithms") + if not isinstance(algorithms, list) or not algorithms: + raise ContractValidationError("unsupported strategy item without algorithms") + if any(not isinstance(algorithm, Mapping) or algorithm.get("type") != "Threshold" for algorithm in algorithms): + raise ContractValidationError("unsupported non-Threshold algorithm") + if _require_mapping(item.get("no_data_config", {}), "no_data_config").get("is_enabled"): + raise ContractValidationError("unsupported no-data configuration") + + algorithm_levels = { + _require_positive_int(algorithm.get("level"), "algorithm.level") for algorithm in item["algorithms"] + } + trigger_configs = {} + detects = strategy.get("detects") + if not isinstance(detects, list): + raise ContractValidationError("unsupported strategy without detects") + for detect in detects: + detect = _require_mapping(detect, "detect") + level = _require_positive_int(detect.get("level"), "detect.level") + if level not in algorithm_levels: + continue + trigger_config = _require_mapping(detect.get("trigger_config"), f"detect[{level}].trigger_config") + if trigger_config.get("uptime"): + raise ContractValidationError("unsupported uptime configuration") + if level in trigger_configs: + raise ContractValidationError("unsupported duplicate detect level") + trigger_configs[level] = { + "check_window_size": _require_positive_int( + trigger_config.get("check_window"), f"detect[{level}].trigger_config.check_window" + ), + "trigger_count": _require_positive_int( + trigger_config.get("count"), f"detect[{level}].trigger_config.count" + ), + } + if set(trigger_configs) != algorithm_levels: + raise ContractValidationError("unsupported strategy with incomplete trigger levels") + + query_configs = item.get("query_configs") or [] + if not isinstance(query_configs, list): + raise ContractValidationError("query_configs must be an array") + intervals = [ + _require_positive_int( + _require_mapping(query_config, "query_config").get("agg_interval", 60), + "query_config.agg_interval", + ) + for query_config in query_configs + ] + return build_trigger_strategy_ir( + tenant_id=tenant_id, + purpose=purpose, + strategy_id=strategy.get("id"), + item_id=normalized_item_id, + generation=_canonical_decimal(strategy.get("update_time"), "strategy.update_time"), + legacy_json=legacy_json, + check_window_unit_seconds=min(intervals) if intervals else 60, + trigger_configs=trigger_configs, + ) + + +def validate_trigger_strategy_ir(strategy_ir: Mapping) -> None: + strategy_ir = _require_mapping(strategy_ir, "strategy_ir") + schema_minor = _validate_header( + strategy_ir, + name=TRIGGER_STRATEGY_IR_SCHEMA, + required_features={FEATURE_RAW_STRATEGY_BYTES}, + ) + _validate_fixed_fields( + strategy_ir, + "strategy_ir", + required={ + "schema", + "required_features", + "tenant_id", + "purpose", + "strategy_ref", + "required_levels", + "check_window_unit_seconds", + "trigger_configs", + "legacy_json_b64", + }, + schema_minor=schema_minor, + ) + _require_nonempty_string(strategy_ir.get("tenant_id"), "tenant_id") + _normalize_purpose(strategy_ir.get("purpose")) + strategy_ref = _normalize_strategy_ref(strategy_ir.get("strategy_ref"), schema_minor=schema_minor) + _require_positive_int(strategy_ir.get("check_window_unit_seconds"), "check_window_unit_seconds") + + required_levels = strategy_ir.get("required_levels") + if not isinstance(required_levels, list) or not required_levels: + raise ContractValidationError("required_levels must be a non-empty array") + if any( + isinstance(level, bool) or not isinstance(level, int) or level <= 0 or level > _MAX_CONTRACT_INT + for level in required_levels + ): + raise ContractValidationError("required_levels must contain positive 32-bit signed integers") + if required_levels != sorted(set(required_levels)): + raise ContractValidationError("required_levels must be sorted and unique") + + trigger_configs = strategy_ir.get("trigger_configs") + if not isinstance(trigger_configs, list): + raise ContractValidationError("trigger_configs must be an array") + config_levels = [] + for config in trigger_configs: + config = _validate_fixed_fields( + config, + "trigger_configs entry", + required={"level", "check_window_size", "trigger_count"}, + schema_minor=schema_minor, + ) + level = _require_positive_int(config.get("level"), "trigger_configs.level") + _require_positive_int(config.get("check_window_size"), "trigger_configs.check_window_size") + _require_positive_int(config.get("trigger_count"), "trigger_configs.trigger_count") + config_levels.append(level) + if len(config_levels) != len(set(config_levels)): + raise ContractValidationError("trigger_configs contains duplicate level") + if config_levels != required_levels: + raise ContractValidationError("trigger_configs levels must equal required_levels") + + legacy_json_b64 = _require_nonempty_string(strategy_ir.get("legacy_json_b64"), "legacy_json_b64") + try: + legacy_json = base64.b64decode(legacy_json_b64, validate=True) + except ValueError as exc: + raise ContractValidationError("legacy_json_b64 must contain valid base64-encoded UTF-8 JSON") from exc + if base64.b64encode(legacy_json).decode("ascii") != legacy_json_b64: + raise ContractValidationError("legacy_json_b64 must use canonical base64 encoding") + _decode_strict_json_object(legacy_json, "legacy_json_b64") + if hashlib.sha256(legacy_json).hexdigest() != strategy_ref["content_sha256"]: + raise ContractValidationError("legacy strategy content hash mismatch") + + +def build_detection_outcome( + *, + strategy_ir: Mapping, + batch_id: str, + data_raw: Mapping, + evaluations: list[Mapping], + outcome: str, + error_code: str | None = None, +) -> dict: + """Build one record-scoped DetectionOutcome and validate it fail-closed.""" + + validate_trigger_strategy_ir(strategy_ir) + data_raw = _require_mapping(data_raw, "data_raw") + record_id = data_raw.get("record_id") + dimensions_md5, source_time = _parse_record_id(record_id) + if _require_source_time(data_raw.get("time"), "data_raw.time") != source_time: + raise ContractValidationError("data_raw.time must equal record source_time") + + strategy_ref = _normalize_strategy_ref(strategy_ir.get("strategy_ref")) + tenant_id = _require_nonempty_string(strategy_ir.get("tenant_id"), "strategy_ir.tenant_id") + purpose = _normalize_purpose(strategy_ir.get("purpose")) + document = { + "schema": {"name": DETECTION_OUTCOME_SCHEMA, "major": SCHEMA_MAJOR, "minor": SCHEMA_MINOR}, + "required_features": [FEATURE_FULL_LEVEL_EVALUATIONS, FEATURE_RAW_JSON], + "input_id": derive_input_id( + tenant_id=tenant_id, + purpose=purpose, + strategy_id=strategy_ref["strategy_id"], + item_id=strategy_ref["item_id"], + strategy_content_sha256=strategy_ref["content_sha256"], + record_id=record_id, + ), + "batch_id": _require_nonempty_string(batch_id, "batch_id"), + "tenant_id": tenant_id, + "purpose": purpose, + "strategy_ref": strategy_ref, + "record": { + "record_id": record_id, + "source_time": source_time, + "dimensions_md5": dimensions_md5, + "data_raw": copy.deepcopy(data_raw), + }, + "evaluations": copy.deepcopy(evaluations), + "outcome": outcome, + } + if error_code is not None: + document["error_code"] = error_code + validate_detection_outcome(document, strategy_ir) + return document + + +def validate_detection_outcome(document: Mapping, strategy_ir: Mapping) -> None: + document = _require_mapping(document, "detection_outcome") + validate_trigger_strategy_ir(strategy_ir) + schema_minor = _validate_header( + document, + name=DETECTION_OUTCOME_SCHEMA, + required_features={FEATURE_FULL_LEVEL_EVALUATIONS, FEATURE_RAW_JSON}, + ) + _validate_fixed_fields( + document, + "detection_outcome", + required={ + "schema", + "required_features", + "input_id", + "batch_id", + "tenant_id", + "purpose", + "strategy_ref", + "record", + "evaluations", + "outcome", + }, + optional={"error_code"}, + schema_minor=schema_minor, + ) + + tenant_id = _require_nonempty_string(document.get("tenant_id"), "tenant_id") + purpose = _normalize_purpose(document.get("purpose")) + batch_id = _require_nonempty_string(document.get("batch_id"), "batch_id") + del batch_id + + expected_tenant_id = _require_nonempty_string(strategy_ir.get("tenant_id"), "strategy_ir.tenant_id") + expected_purpose = _normalize_purpose(strategy_ir.get("purpose")) + if tenant_id != expected_tenant_id or purpose != expected_purpose: + raise ContractValidationError("outcome tenant or purpose does not match StrategyIR") + + strategy_ref = _normalize_strategy_ref(document.get("strategy_ref"), schema_minor=schema_minor) + expected_strategy_ref = _normalize_strategy_ref(strategy_ir.get("strategy_ref")) + if strategy_ref != expected_strategy_ref: + raise ContractValidationError("outcome strategy_ref does not match StrategyIR") + + record = _validate_fixed_fields( + document.get("record"), + "record", + required={"record_id", "source_time", "dimensions_md5", "data_raw"}, + schema_minor=schema_minor, + ) + record_id = record.get("record_id") + dimensions_md5, source_time = _parse_record_id(record_id) + if record.get("dimensions_md5") != dimensions_md5: + raise ContractValidationError("record dimensions_md5 does not match record_id") + if _require_source_time(record.get("source_time"), "record.source_time") != source_time: + raise ContractValidationError("record source_time does not match record_id") + data_raw = _validate_fixed_fields( + record.get("data_raw"), + "record.data_raw", + required={"record_id", "time"}, + optional={"values"}, + schema_minor=schema_minor, + allow_open=True, + ) + data_raw_time = _require_source_time(data_raw.get("time"), "record.data_raw.time") + if data_raw.get("record_id") != record_id or data_raw_time != source_time: + raise ContractValidationError("record.data_raw source coordinate mismatch") + values = data_raw.get("values") + if isinstance(values, Mapping) and "timestamp" in values: + values_timestamp = _require_source_time(values["timestamp"], "record.data_raw.values.timestamp") + if values_timestamp != source_time: + raise ContractValidationError("record.data_raw.values timestamp mismatch") + + expected_input_id = derive_input_id( + tenant_id=tenant_id, + purpose=purpose, + strategy_id=strategy_ref["strategy_id"], + item_id=strategy_ref["item_id"], + strategy_content_sha256=strategy_ref["content_sha256"], + record_id=record_id, + ) + if document.get("input_id") != expected_input_id: + raise ContractValidationError("input_id does not match canonical tuple") + + outcome = document.get("outcome") + if not isinstance(outcome, str) or outcome not in OUTCOMES: + raise ContractValidationError(f"unsupported detection outcome: {outcome}") + error_code = document.get("error_code") + if outcome in {"NORMAL", "ANOMALOUS"}: + if "error_code" in document: + raise ContractValidationError("business outcome must not carry error_code") + elif not isinstance(error_code, str) or error_code not in ERROR_CODES[outcome]: + raise ContractValidationError(f"invalid error_code for {outcome}") + + evaluations = document.get("evaluations") + if not isinstance(evaluations, list): + raise ContractValidationError("evaluations must be an array") + required_levels = set(strategy_ir["required_levels"]) + seen_levels = set() + anomalous_count = 0 + for evaluation in evaluations: + evaluation = _validate_fixed_fields( + evaluation, + "evaluation", + required={"level", "result"}, + optional={"anomaly"}, + schema_minor=schema_minor, + ) + level = _require_positive_int(evaluation.get("level"), "evaluation.level") + if level not in required_levels: + raise ContractValidationError("evaluation level is not required by StrategyIR") + if level in seen_levels: + raise ContractValidationError("evaluations contains duplicate level") + seen_levels.add(level) + + result = evaluation.get("result") + if not isinstance(result, str) or result not in EVALUATION_RESULTS: + raise ContractValidationError(f"unsupported evaluation result: {result}") + anomaly = evaluation.get("anomaly") + if result == "NORMAL": + if "anomaly" in evaluation: + raise ContractValidationError("NORMAL evaluation must not carry anomaly") + continue + + anomaly = _validate_fixed_fields( + anomaly, + "ANOMALOUS evaluation anomaly", + required={"anomaly_id"}, + optional={"anomaly_message", "context"}, + schema_minor=schema_minor, + allow_open=True, + ) + expected_anomaly_id = f"{record_id}.{strategy_ref['strategy_id']}.{strategy_ref['item_id']}.{level}" + if anomaly.get("anomaly_id") != expected_anomaly_id: + raise ContractValidationError("anomaly_id does not match record, strategy, item and level") + anomalous_count += 1 + + if outcome in {"NORMAL", "ANOMALOUS"} and seen_levels != required_levels: + raise ContractValidationError("business outcome evaluations must be complete") + if outcome == "NORMAL" and anomalous_count: + raise ContractValidationError("NORMAL outcome must contain only NORMAL evaluations") + if outcome == "ANOMALOUS" and not anomalous_count: + raise ContractValidationError("ANOMALOUS outcome must contain at least one ANOMALOUS evaluation") + + +def can_drive_trigger(document: Mapping, strategy_ir: Mapping) -> bool: + """Return whether a valid DetectionOutcome may advance Trigger state.""" + + validate_detection_outcome(document, strategy_ir) + return document["outcome"] in {"NORMAL", "ANOMALOUS"} + + +def build_trigger_decision_batch(*, strategy_ir: Mapping, batch_id: str, decisions: list[Mapping]) -> dict: + """Build the Trigger decision wire shared by the Go candidate and Python reference.""" + + validate_trigger_strategy_ir(strategy_ir) + document = { + "schema": {"name": TRIGGER_DECISION_BATCH_SCHEMA, "major": SCHEMA_MAJOR, "minor": SCHEMA_MINOR}, + "required_features": [], + "partition_hash_version": TRIGGER_PARTITION_HASH_VERSION, + "batch_id": _require_nonempty_string(batch_id, "trigger decision batch_id"), + "tenant_id": strategy_ir["tenant_id"], + "purpose": strategy_ir["purpose"], + "strategy_ref": copy.deepcopy(strategy_ir["strategy_ref"]), + "decision_algorithm": TRIGGER_DECISION_ALGORITHM, + "decisions": copy.deepcopy(decisions), + } + validate_trigger_decision_batch(document) + return document + + +def validate_trigger_decision_batch(document: Mapping) -> None: + document = _require_mapping(document, "trigger decision batch") + schema = _require_mapping(document.get("schema"), "schema") + schema_minor = schema.get("minor") + if isinstance(schema_minor, bool) or not isinstance(schema_minor, int): + raise ContractValidationError("schema.minor must be a non-negative 32-bit signed integer") + _validate_fixed_fields( + document, + "trigger decision batch", + required={ + "schema", + "required_features", + "partition_hash_version", + "batch_id", + "tenant_id", + "purpose", + "strategy_ref", + "decision_algorithm", + "decisions", + }, + schema_minor=schema_minor, + ) + _validate_header(document, name=TRIGGER_DECISION_BATCH_SCHEMA, required_features=set()) + if document.get("partition_hash_version") != TRIGGER_PARTITION_HASH_VERSION: + raise ContractValidationError("unsupported trigger decision partition hash version") + _require_nonempty_string(document.get("batch_id"), "trigger decision batch_id") + tenant_id = _require_nonempty_string(document.get("tenant_id"), "trigger decision tenant_id") + purpose = _normalize_purpose(document.get("purpose")) + strategy_ref = _normalize_strategy_ref(document.get("strategy_ref"), schema_minor=schema_minor) + if document.get("decision_algorithm") != TRIGGER_DECISION_ALGORITHM: + raise ContractValidationError("unsupported trigger decision algorithm") + + decisions = document.get("decisions") + if not isinstance(decisions, list) or not 1 <= len(decisions) <= 500: + raise ContractValidationError("trigger decision batch must contain between 1 and 500 decisions") + input_ids = set() + decision_ids = set() + for decision in decisions: + _validate_trigger_decision(decision, schema_minor=schema_minor) + expected_input_id = derive_input_id( + tenant_id=tenant_id, + purpose=purpose, + strategy_id=strategy_ref["strategy_id"], + item_id=strategy_ref["item_id"], + strategy_content_sha256=strategy_ref["content_sha256"], + record_id=decision["record_id"], + ) + if decision["input_id"] != expected_input_id: + raise ContractValidationError("trigger decision input_id does not match batch identity and record_id") + if decision["input_id"] in input_ids or decision["decision_id"] in decision_ids: + raise ContractValidationError("trigger decision batch contains duplicate decision identity") + if purpose != "DETECT" and ( + decision["outcome"] != "UNSUPPORTED" or decision["reason_code"] != "UNSUPPORTED_STRATEGY" + ): + raise ContractValidationError("unsupported purpose requires UNSUPPORTED_STRATEGY decision") + input_ids.add(decision["input_id"]) + decision_ids.add(decision["decision_id"]) + + +def _validate_trigger_decision(decision: Any, *, schema_minor: int) -> None: + decision = _validate_fixed_fields( + decision, + "trigger decision", + required={ + "decision_id", + "input_id", + "record_id", + "outcome", + "reason_code", + "anomaly_timestamps", + }, + optional={"level"}, + schema_minor=schema_minor, + ) + input_id = _require_sha256(decision.get("input_id"), "trigger decision input_id") + if decision.get("decision_id") != derive_trigger_decision_id(input_id): + raise ContractValidationError("trigger decision_id does not match canonical tuple") + _, source_time = _parse_record_id(decision.get("record_id")) + if "level" in decision: + _require_positive_int(decision.get("level"), "trigger decision level") + timestamps = decision.get("anomaly_timestamps") + if not isinstance(timestamps, list): + raise ContractValidationError("trigger decision anomaly_timestamps must be an array") + for timestamp in timestamps: + _require_source_time(timestamp, "trigger decision anomaly timestamp") + if timestamp > source_time: + raise ContractValidationError("trigger decision anomaly timestamp exceeds source time") + if any(right <= left for left, right in zip(timestamps, timestamps[1:])): + raise ContractValidationError("trigger decision anomaly_timestamps must be strictly increasing") + + outcome = decision.get("outcome") + reason_code = decision.get("reason_code") + if outcome == "TRIGGER": + if ( + reason_code != "TRIGGER_CONDITION_MET" + or "level" not in decision + or not timestamps + or timestamps[-1] != source_time + ): + raise ContractValidationError("TRIGGER decision requires condition, level and current timestamp") + return + if outcome == "NO_TRIGGER": + if reason_code not in {"INPUT_NORMAL", "TRIGGER_CONDITION_NOT_MET"}: + raise ContractValidationError("unsupported NO_TRIGGER reason_code") + if "level" in decision or timestamps: + raise ContractValidationError("NO_TRIGGER decision must not carry level or timestamps") + return + if outcome in ERROR_CODES: + if reason_code not in ERROR_CODES[outcome] or "level" in decision or timestamps: + raise ContractValidationError(f"invalid {outcome} trigger decision") + return + raise ContractValidationError(f"unsupported trigger decision outcome: {outcome}") diff --git a/bkmonitor/alarm_backends/core/alarm_engine/encoder.py b/bkmonitor/alarm_backends/core/alarm_engine/encoder.py new file mode 100644 index 00000000000..b6c5d46cbb8 --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/encoder.py @@ -0,0 +1,119 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import json +import math +from collections.abc import Mapping +from typing import Any + +from alarm_backends.core.alarm_engine.contract import ContractValidationError + + +MAX_TRIGGER_DECISION_BATCH_BYTES = 512 * 1024 + + +def _validate_json_value(value: Any, field: str = "contract payload") -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + if not isinstance(key, str): + raise ContractValidationError(f"{field} object keys must be strings") + _validate_json_value(key, field) + _validate_json_value(child, field) + elif isinstance(value, list): + for child in value: + _validate_json_value(child, field) + elif isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ContractValidationError(f"{field} must contain valid UTF-8") from exc + elif value is None or isinstance(value, (bool, int)): + return + elif isinstance(value, float): + if not math.isfinite(value): + raise ContractValidationError(f"{field} must not contain non-finite numbers") + else: + raise ContractValidationError(f"{field} contains unsupported JSON value type") + + +def encode_json_document(document: Mapping) -> bytes: + """Encode a contract document deterministically without accepting NaN or infinity.""" + + _validate_json_value(document) + return json.dumps( + document, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ContractValidationError(f"duplicate JSON field: {key}") + result[key] = value + return result + + +def _reject_nonfinite(value: str) -> None: + raise ContractValidationError(f"non-finite JSON number: {value}") + + +def _parse_finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ContractValidationError(f"non-finite JSON number: {value}") + return parsed + + +def decode_json_document(payload: bytes | str) -> dict: + """Decode an object document while preserving integers and rejecting ambiguous JSON.""" + + if (isinstance(payload, bytes) and payload.startswith(b"\xef\xbb\xbf")) or ( + isinstance(payload, str) and payload.startswith("\ufeff") + ): + raise ContractValidationError("contract payload must not contain a UTF-8 BOM") + try: + document = json.loads( + payload, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_nonfinite, + parse_float=_parse_finite_float, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ContractValidationError("contract payload must be valid UTF-8 JSON") from exc + if not isinstance(document, dict): + raise ContractValidationError("contract payload must contain a JSON object") + _validate_json_value(document) + return document + + +def encode_trigger_decision_batch(document: Mapping) -> bytes: + from alarm_backends.core.alarm_engine.contract import validate_trigger_decision_batch + + validate_trigger_decision_batch(document) + payload = encode_json_document(document) + if len(payload) > MAX_TRIGGER_DECISION_BATCH_BYTES: + raise ContractValidationError("trigger decision batch exceeds encoded byte limit") + return payload + + +def decode_trigger_decision_batch(payload: bytes | str) -> dict: + if len(payload.encode("utf-8") if isinstance(payload, str) else payload) > MAX_TRIGGER_DECISION_BATCH_BYTES: + raise ContractValidationError("trigger decision batch exceeds encoded byte limit") + document = decode_json_document(payload) + + from alarm_backends.core.alarm_engine.contract import validate_trigger_decision_batch + + validate_trigger_decision_batch(document) + return document diff --git a/bkmonitor/alarm_backends/core/alarm_engine/publisher.py b/bkmonitor/alarm_backends/core/alarm_engine/publisher.py new file mode 100644 index 00000000000..b23a1ff0ec4 --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/publisher.py @@ -0,0 +1,220 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import hashlib +import struct +from collections.abc import Mapping +from functools import lru_cache + +from alarm_backends.core.alarm_engine.contract import validate_detection_outcome, validate_trigger_strategy_ir +from alarm_backends.core.alarm_engine.encoder import decode_json_document, encode_json_document + +DEFAULT_DELIVERY_TIMEOUT_MS = 3000 +DEFAULT_MAX_ENVELOPE_BYTES = 512 * 1024 +DEFAULT_MAX_OUTCOMES_PER_MESSAGE = 500 +PARTITION_HASH_VERSION = "trigger-input-partition-v1" + + +class DetectionPublishError(RuntimeError): + """Raised when an outcome batch is not acknowledged within the configured bound.""" + + +class KafkaDetectionPublisher: + def __init__( + self, + *, + producer, + topic: str, + flush_timeout: float, + max_outcomes_per_message: int = DEFAULT_MAX_OUTCOMES_PER_MESSAGE, + max_envelope_bytes: int = DEFAULT_MAX_ENVELOPE_BYTES, + ): + if not isinstance(topic, str) or not topic: + raise ValueError("detection topic must be non-empty") + if isinstance(flush_timeout, bool) or not isinstance(flush_timeout, (int, float)) or flush_timeout <= 0: + raise ValueError("flush_timeout must be positive") + if ( + isinstance(max_outcomes_per_message, bool) + or not isinstance(max_outcomes_per_message, int) + or max_outcomes_per_message <= 0 + or max_outcomes_per_message > DEFAULT_MAX_OUTCOMES_PER_MESSAGE + ): + raise ValueError(f"max_outcomes_per_message must be between 1 and {DEFAULT_MAX_OUTCOMES_PER_MESSAGE}") + if ( + isinstance(max_envelope_bytes, bool) + or not isinstance(max_envelope_bytes, int) + or max_envelope_bytes <= 0 + or max_envelope_bytes > DEFAULT_MAX_ENVELOPE_BYTES + ): + raise ValueError(f"max_envelope_bytes must be between 1 and {DEFAULT_MAX_ENVELOPE_BYTES}") + self.producer = producer + self.topic = topic + self.flush_timeout = flush_timeout + self.max_outcomes_per_message = max_outcomes_per_message + self.max_envelope_bytes = max_envelope_bytes + + def publish_batch(self, batch: Mapping) -> int: + if not isinstance(batch, Mapping): + raise DetectionPublishError("detection batch must be an object") + strategy_ir = batch.get("strategy_ir") + outcomes = batch.get("outcomes") + validate_trigger_strategy_ir(strategy_ir) + if not isinstance(outcomes, list): + raise DetectionPublishError("detection batch outcomes must be an array") + if not outcomes: + return 0 + + partition_key = trigger_partition_key(strategy_ir) + microbatches = self._plan_microbatches(strategy_ir, outcomes) + delivery_errors = [] + + def on_delivery(error, _message): + if error is not None: + delivery_errors.append(error) + + try: + for start, end in microbatches: + self.producer.produce( + topic=self.topic, + key=partition_key, + value=encode_json_document(_trigger_input_envelope(strategy_ir, outcomes[start:end])), + on_delivery=on_delivery, + ) + if hasattr(self.producer, "poll"): + self.producer.poll(0) + remaining = self.producer.flush(timeout=self.flush_timeout) + except Exception as error: + raise DetectionPublishError(f"detection publish failed: {error}") from error + if remaining: + raise DetectionPublishError(f"detection publish flush timeout: {remaining} message(s) unacknowledged") + if delivery_errors: + raise DetectionPublishError(f"detection publish broker rejected message: {delivery_errors[0]}") + return len(outcomes) + + def _plan_microbatches(self, strategy_ir: Mapping, outcomes: list[Mapping]) -> list[tuple[int, int]]: + base_size = len(encode_json_document(_trigger_input_envelope(strategy_ir, []))) + current_start = 0 + current_count = 0 + current_size = base_size + microbatches = [] + batch_id = None + input_ids = set() + for index, outcome in enumerate(outcomes): + validate_detection_outcome(outcome, strategy_ir) + if batch_id is None: + batch_id = outcome["batch_id"] + elif outcome["batch_id"] != batch_id: + raise DetectionPublishError("detection outcomes must share one batch_id") + if outcome["input_id"] in input_ids: + raise DetectionPublishError("detection outcomes must not contain duplicate input_id") + input_ids.add(outcome["input_id"]) + outcome_size = len(encode_json_document(outcome)) + added_size = outcome_size + (1 if current_count else 0) + if current_count and ( + current_count >= self.max_outcomes_per_message or current_size + added_size > self.max_envelope_bytes + ): + microbatches.append((current_start, index)) + current_start = index + current_count = 0 + current_size = base_size + added_size = outcome_size + if current_size + added_size > self.max_envelope_bytes: + raise DetectionPublishError("single detection outcome exceeds the envelope byte limit") + current_count += 1 + current_size += added_size + if current_count: + microbatches.append((current_start, len(outcomes))) + return microbatches + + +def build_kafka_detection_publisher(config: Mapping, *, allowed_topics, producer_factory=None): + if not isinstance(config, Mapping): + raise ValueError("detection Kafka config must be an object") + if ( + not isinstance(allowed_topics, (set, frozenset)) + or not allowed_topics + or any(not isinstance(topic, str) or not topic for topic in allowed_topics) + ): + raise ValueError("detection Shadow topic allowlist must be a non-empty string set") + producer_config = dict(config) + topic = producer_config.pop("topic", None) + if topic not in allowed_topics: + raise ValueError(f"detection Kafka topic is not in the Shadow allowlist: {topic}") + + configured_timeouts = [ + producer_config[name] for name in ("message.timeout.ms", "delivery.timeout.ms") if name in producer_config + ] + if len(configured_timeouts) > 1: + raise ValueError("configure only one delivery timeout alias") + raw_timeout = configured_timeouts[0] if configured_timeouts else DEFAULT_DELIVERY_TIMEOUT_MS + if isinstance(raw_timeout, bool): + raise ValueError("delivery timeout must be a positive number") + try: + timeout_ms = int(float(raw_timeout)) + except (TypeError, ValueError) as error: + raise ValueError("delivery timeout must be a positive number") from error + if timeout_ms <= 0: + raise ValueError("delivery timeout must be a positive number") + if not configured_timeouts: + producer_config["message.timeout.ms"] = timeout_ms + + flush_timeout = producer_config.pop("alarm.engine.flush.timeout.seconds", timeout_ms / 1000 + 1) + max_outcomes_per_message = producer_config.pop( + "alarm.engine.max.outcomes.per.message", DEFAULT_MAX_OUTCOMES_PER_MESSAGE + ) + max_envelope_bytes = producer_config.pop("alarm.engine.max.envelope.bytes", DEFAULT_MAX_ENVELOPE_BYTES) + if producer_config.get("enable.idempotence", True) is not True: + raise ValueError("detection Kafka producer idempotence must be enabled") + producer_config["enable.idempotence"] = True + if producer_factory is None: + from confluent_kafka import Producer + + producer_factory = Producer + producer = producer_factory(producer_config) + return KafkaDetectionPublisher( + producer=producer, + topic=topic, + flush_timeout=flush_timeout, + max_outcomes_per_message=max_outcomes_per_message, + max_envelope_bytes=max_envelope_bytes, + ) + + +@lru_cache(maxsize=1) +def get_cached_kafka_detection_publisher(config_json: str, allowed_topics: tuple[str, ...]): + config = decode_json_document(config_json) + return build_kafka_detection_publisher(config, allowed_topics=set(allowed_topics)) + + +def trigger_partition_key(document: Mapping) -> bytes: + ref = document["strategy_ref"] + fields = ( + PARTITION_HASH_VERSION, + document["tenant_id"], + document["purpose"], + ref["strategy_id"], + ref["item_id"], + ) + payload = bytearray() + for field in fields: + encoded = field.encode("utf-8") + payload.extend(struct.pack(">I", len(encoded))) + payload.extend(encoded) + return hashlib.sha256(payload).digest() + + +def _trigger_input_envelope(strategy_ir: Mapping, outcomes: list[Mapping]) -> dict: + return { + "schema": {"name": "trigger-input", "major": 1, "minor": 0}, + "required_features": [], + "partition_hash_version": PARTITION_HASH_VERSION, + "strategy_ir": strategy_ir, + "detection_outcomes": outcomes, + } diff --git a/bkmonitor/alarm_backends/core/alarm_engine/reference.py b/bkmonitor/alarm_backends/core/alarm_engine/reference.py new file mode 100644 index 00000000000..fe2da3b3f01 --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/reference.py @@ -0,0 +1,336 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +from collections.abc import Mapping +from collections.abc import Callable + +from alarm_backends.core.alarm_engine.contract import ( + ContractValidationError, + build_detection_outcome, + build_trigger_decision_batch, + build_trigger_strategy_ir_from_legacy_config, + derive_trigger_decision_id, + json_values_equal, + validate_detection_outcome, + validate_trigger_strategy_ir, +) + + +def build_reference_trigger_decision_batch( + *, + strategy: Mapping, + legacy_json: bytes, + strategy_snapshot_key: str, + tenant_id_resolver: Callable[[int], str], + expected_input_id: str, + item_id: str | int, + point: Mapping, + event_record: Mapping | None, +) -> dict: + """Project one real Trigger result and verify it against an acknowledged Detection input.""" + + return _build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=legacy_json, + strategy_snapshot_key=strategy_snapshot_key, + tenant_id_resolver=tenant_id_resolver, + expected_input_id=expected_input_id, + item_id=item_id, + point=point, + event_record=event_record, + ) + + +def build_reference_trigger_decision_candidate( + *, + strategy: Mapping, + legacy_json: bytes, + strategy_snapshot_key: str, + tenant_id_resolver: Callable[[int], str], + item_id: str | int, + point: Mapping, + event_record: Mapping | None, +) -> dict: + """Project an unconfirmed Trigger result for later TriggerInput correlation.""" + + return _build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=legacy_json, + strategy_snapshot_key=strategy_snapshot_key, + tenant_id_resolver=tenant_id_resolver, + expected_input_id=None, + item_id=item_id, + point=point, + event_record=event_record, + ) + + +def _build_reference_trigger_decision_batch( + *, + strategy: Mapping, + legacy_json: bytes, + strategy_snapshot_key: str, + tenant_id_resolver: Callable[[int], str], + expected_input_id: str | None, + item_id: str | int, + point: Mapping, + event_record: Mapping | None, +) -> dict: + """Project one real legacy Python Trigger result without mutating its Redis point.""" + + point = _require_mapping(point, "reference point") + point_snapshot_key = _require_nonempty_utf8(point.get("strategy_snapshot_key"), "reference point snapshot key") + strategy_snapshot_key = _require_nonempty_utf8(strategy_snapshot_key, "reference strategy snapshot key") + if point_snapshot_key != strategy_snapshot_key: + raise ContractValidationError("reference point does not match the exact strategy snapshot") + strategy = _require_mapping(strategy, "reference strategy") + bk_biz_id = strategy.get("bk_biz_id") + if isinstance(bk_biz_id, bool) or not isinstance(bk_biz_id, int): + raise ContractValidationError("reference strategy bk_biz_id must be an integer") + if not callable(tenant_id_resolver): + raise ContractValidationError("reference tenant_id_resolver must be callable") + tenant_id = tenant_id_resolver(bk_biz_id) + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id=tenant_id, + purpose="DETECT", + strategy=strategy, + item_id=item_id, + legacy_json=legacy_json, + ) + data_raw = _require_mapping(point.get("data"), "reference point data") + anomalies = _require_mapping(point.get("anomaly"), "reference point anomaly") + if not anomalies: + raise ContractValidationError("reference Trigger point must contain an anomaly") + + evaluations = [] + remaining = dict(anomalies) + for level in strategy_ir["required_levels"]: + level_key = str(level) + if level_key not in remaining: + evaluations.append({"level": level, "result": "NORMAL"}) + else: + anomaly = remaining.pop(level_key) + evaluations.append( + { + "level": level, + "result": "ANOMALOUS", + "anomaly": copy.deepcopy(_require_mapping(anomaly, "reference anomaly")), + } + ) + if remaining: + raise ContractValidationError("reference point contains a level outside StrategyIR") + + record_id = data_raw.get("record_id") + source = build_detection_outcome( + strategy_ir=strategy_ir, + batch_id=f"python-reference-{record_id}", + data_raw=data_raw, + evaluations=evaluations, + outcome="ANOMALOUS", + ) + if expected_input_id is not None and source["input_id"] != expected_input_id: + raise ContractValidationError("reference input_id does not match the acknowledged Detect input") + decision = { + "decision_id": derive_trigger_decision_id(source["input_id"]), + "input_id": source["input_id"], + "record_id": record_id, + "outcome": "NO_TRIGGER", + "reason_code": "TRIGGER_CONDITION_NOT_MET", + "anomaly_timestamps": [], + } + if event_record is not None: + level, timestamps = _parse_trigger_result( + event_record, + strategy_ir=strategy_ir, + source=source, + strategy_snapshot_key=strategy_snapshot_key, + ) + decision.update( + outcome="TRIGGER", + reason_code="TRIGGER_CONDITION_MET", + level=level, + anomaly_timestamps=timestamps, + ) + return build_trigger_decision_batch( + strategy_ir=strategy_ir, + batch_id=source["batch_id"], + decisions=[decision], + ) + + +def parse_alarm_engine_shadow_strategy_ids(configured_strategy_ids) -> set[int] | None: + """Parse the shared canonical selector, returning None for invalid configuration.""" + + if isinstance(configured_strategy_ids, str): + configured_strategy_ids = [] if not configured_strategy_ids else configured_strategy_ids.split(",") + try: + allowed_strategy_ids = set() + for configured_strategy_id in configured_strategy_ids: + if isinstance(configured_strategy_id, bool): + raise ValueError + if isinstance(configured_strategy_id, int): + if configured_strategy_id <= 0: + raise ValueError + allowed_strategy_ids.add(configured_strategy_id) + continue + if ( + not isinstance(configured_strategy_id, str) + or not configured_strategy_id.isascii() + or not configured_strategy_id.isdigit() + or configured_strategy_id.startswith("0") + ): + raise ValueError + allowed_strategy_ids.add(int(configured_strategy_id)) + except (TypeError, ValueError): + return None + return allowed_strategy_ids + + +def is_alarm_engine_shadow_strategy_selected(configured_strategy_ids, strategy_id: int) -> bool: + """Return whether a canonical positive strategy ID is explicitly selected.""" + + allowed_strategy_ids = parse_alarm_engine_shadow_strategy_ids(configured_strategy_ids) + return allowed_strategy_ids is not None and strategy_id in allowed_strategy_ids + + +def build_terminal_reference_decision_batches(*, strategy_ir: Mapping, detection_outcomes: list[Mapping]) -> list[dict]: + """Project ACKed non-anomalous DetectionOutcomes without invoking the legacy Trigger.""" + + validate_trigger_strategy_ir(strategy_ir) + if not isinstance(detection_outcomes, list) or not detection_outcomes: + raise ContractValidationError("reference detection_outcomes must be a non-empty array") + batch_id = None + decisions = [] + for source in detection_outcomes: + validate_detection_outcome(source, strategy_ir) + if batch_id is None: + batch_id = source["batch_id"] + elif source["batch_id"] != batch_id: + raise ContractValidationError("reference detection_outcomes must share one batch_id") + if source["outcome"] == "ANOMALOUS": + continue + if source["outcome"] == "NORMAL": + outcome = "NO_TRIGGER" + reason_code = "INPUT_NORMAL" + else: + outcome = source["outcome"] + reason_code = source["error_code"] + decisions.append( + { + "decision_id": derive_trigger_decision_id(source["input_id"]), + "input_id": source["input_id"], + "record_id": source["record"]["record_id"], + "outcome": outcome, + "reason_code": reason_code, + "anomaly_timestamps": [], + } + ) + return [ + build_trigger_decision_batch( + strategy_ir=strategy_ir, batch_id=batch_id, decisions=decisions[start : start + 500] + ) + for start in range(0, len(decisions), 500) + ] + + +def _parse_trigger_result( + event_record: Mapping, + *, + strategy_ir: Mapping, + source: Mapping, + strategy_snapshot_key: str, +) -> tuple[int, list[int]]: + event_record = _require_mapping(event_record, "reference event_record") + point_data = source["record"]["data_raw"] + if not json_values_equal(event_record.get("data"), point_data): + raise ContractValidationError("reference event_record data does not match the source point") + expected_anomalies = { + str(evaluation["level"]): evaluation["anomaly"] + for evaluation in source["evaluations"] + if evaluation["result"] == "ANOMALOUS" + } + if not json_values_equal(event_record.get("anomaly"), expected_anomalies): + raise ContractValidationError("reference event_record anomaly does not match the source point") + if event_record.get("strategy_snapshot_key") != strategy_snapshot_key: + raise ContractValidationError("reference event_record strategy snapshot does not match the source point") + trigger = _require_mapping(event_record.get("trigger"), "reference event_record trigger") + raw_level = trigger.get("level") + if isinstance(raw_level, bool): + raise ContractValidationError("reference trigger level must be a positive integer") + try: + level = int(raw_level) + except (TypeError, ValueError) as exc: + raise ContractValidationError("reference trigger level must be a positive integer") from exc + if str(level) != str(raw_level) or level <= 0: + raise ContractValidationError("reference trigger level must use canonical decimal form") + anomalous_levels = { + evaluation["level"] for evaluation in source["evaluations"] if evaluation["result"] == "ANOMALOUS" + } + if level not in anomalous_levels: + raise ContractValidationError("reference trigger level is not anomalous in the source point") + + anomaly_ids = trigger.get("anomaly_ids") + if not isinstance(anomaly_ids, list) or not anomaly_ids: + raise ContractValidationError("reference trigger anomaly_ids must be a non-empty array") + ref = strategy_ir["strategy_ref"] + ( + record_dimensions, + _, + ) = source["record"]["record_id"].split(".", 1) + timestamps = [] + for anomaly_id in anomaly_ids: + if not isinstance(anomaly_id, str): + raise ContractValidationError("reference trigger anomaly_id must be a string") + parts = anomaly_id.split(".") + if len(parts) != 5: + raise ContractValidationError("reference trigger anomaly_id is not canonical") + dimensions, timestamp, strategy_id, anomaly_item_id, anomaly_level = parts + if ( + dimensions != record_dimensions + or strategy_id != ref["strategy_id"] + or anomaly_item_id != ref["item_id"] + or anomaly_level != str(level) + or not timestamp.isascii() + or not timestamp.isdigit() + or (timestamp.startswith("0") and timestamp != "0") + ): + raise ContractValidationError("reference trigger anomaly_id identity mismatch") + timestamps.append(int(timestamp)) + if any(right <= left for left, right in zip(timestamps, timestamps[1:])): + raise ContractValidationError("reference trigger timestamps must be strictly increasing") + + config = next((item for item in strategy_ir["trigger_configs"] if item["level"] == level), None) + if config is None or len(timestamps) < config["trigger_count"]: + raise ContractValidationError("reference trigger result does not satisfy trigger_count") + source_time = source["record"]["source_time"] + window_start = source_time - strategy_ir["check_window_unit_seconds"] * config["check_window_size"] + 1 + if ( + any(timestamp < window_start or timestamp > source_time for timestamp in timestamps) + or timestamps[-1] != source_time + ): + raise ContractValidationError("reference trigger timestamps fall outside the selected window") + return level, timestamps + + +def _require_mapping(value, field): + if not isinstance(value, Mapping): + raise ContractValidationError(f"{field} must be an object") + return value + + +def _require_nonempty_utf8(value, field): + if not isinstance(value, str) or not value: + raise ContractValidationError(f"{field} must be a non-empty string") + try: + value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ContractValidationError(f"{field} must contain valid UTF-8") from exc + return value diff --git a/bkmonitor/alarm_backends/core/alarm_engine/reference_publisher.py b/bkmonitor/alarm_backends/core/alarm_engine/reference_publisher.py new file mode 100644 index 00000000000..ad3e24c0eef --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/reference_publisher.py @@ -0,0 +1,156 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from collections.abc import Iterable, Mapping +from functools import lru_cache + +from alarm_backends.core.alarm_engine.encoder import ( + MAX_TRIGGER_DECISION_BATCH_BYTES, + decode_json_document, + encode_trigger_decision_batch, +) +from alarm_backends.core.alarm_engine.publisher import DEFAULT_DELIVERY_TIMEOUT_MS, trigger_partition_key + + +class ReferenceDecisionPublishError(RuntimeError): + """Raised when a reference decision batch is not acknowledged.""" + + +class KafkaReferenceDecisionPublisher: + def __init__(self, *, producer, topic: str, flush_timeout: float): + if not isinstance(topic, str) or not topic: + raise ValueError("reference decision topic must be non-empty") + if isinstance(flush_timeout, bool) or not isinstance(flush_timeout, (int, float)) or flush_timeout <= 0: + raise ValueError("flush_timeout must be positive") + self.producer = producer + self.topic = topic + self.flush_timeout = flush_timeout + + def publish_batch(self, batch: Mapping) -> int: + return self.publish_batches([batch]) + + def publish_batches(self, batches: Iterable[Mapping]) -> int: + def publish_prepared(prepared): + delivery_errors = [] + + def on_delivery(error, _message): + if error is not None: + delivery_errors.append(error) + + for partition_key, payload, _decision_count in prepared: + self.producer.produce( + topic=self.topic, + key=partition_key, + value=payload, + on_delivery=on_delivery, + ) + if hasattr(self.producer, "poll"): + self.producer.poll(0) + remaining = self.producer.flush(timeout=self.flush_timeout) + if remaining: + raise ReferenceDecisionPublishError( + f"reference decision publish flush timeout: {remaining} message(s) unacknowledged" + ) + if delivery_errors: + raise ReferenceDecisionPublishError( + f"reference decision publish broker rejected message: {delivery_errors[0]}" + ) + return sum(decision_count for _partition_key, _payload, decision_count in prepared) + + published = 0 + prepared = [] + prepared_bytes = 0 + try: + for batch in batches: + # One encoded lookahead is needed to decide whether the current ACK group is full; + # official per-message validation bounds both the group and lookahead to 512 KiB each. + payload = encode_trigger_decision_batch(batch) + if prepared and prepared_bytes + len(payload) > MAX_TRIGGER_DECISION_BATCH_BYTES: + published += publish_prepared(prepared) + prepared = [] + prepared_bytes = 0 + prepared.append((trigger_partition_key(batch), payload, len(batch["decisions"]))) + prepared_bytes += len(payload) + if prepared: + published += publish_prepared(prepared) + except ReferenceDecisionPublishError: + raise + except Exception as error: + raise ReferenceDecisionPublishError(f"reference decision publish failed: {error}") from error + return published + + +def build_kafka_reference_decision_publisher( + config: Mapping, + *, + allowed_topics, + forbidden_topics=(), + producer_factory=None, +): + if not isinstance(config, Mapping): + raise ValueError("reference decision Kafka config must be an object") + if ( + not isinstance(allowed_topics, (set, frozenset)) + or not allowed_topics + or any(not isinstance(topic, str) or not topic for topic in allowed_topics) + ): + raise ValueError("reference decision Shadow topic allowlist must be a non-empty string set") + forbidden_topics = set(forbidden_topics) + if any(not isinstance(topic, str) or not topic for topic in forbidden_topics): + raise ValueError("reference decision forbidden topics must contain non-empty strings") + if set(allowed_topics) & forbidden_topics: + raise ValueError("reference decision allowlist must not contain forbidden topics") + + producer_config = dict(config) + topic = producer_config.pop("topic", None) + if topic not in allowed_topics or topic in forbidden_topics: + raise ValueError(f"reference decision Kafka topic is not in the isolated Shadow allowlist: {topic}") + + configured_timeouts = [ + producer_config[name] for name in ("message.timeout.ms", "delivery.timeout.ms") if name in producer_config + ] + if len(configured_timeouts) > 1: + raise ValueError("configure only one delivery timeout alias") + raw_timeout = configured_timeouts[0] if configured_timeouts else DEFAULT_DELIVERY_TIMEOUT_MS + if isinstance(raw_timeout, bool): + raise ValueError("delivery timeout must be a positive number") + try: + timeout_ms = int(float(raw_timeout)) + except (TypeError, ValueError) as error: + raise ValueError("delivery timeout must be a positive number") from error + if timeout_ms <= 0: + raise ValueError("delivery timeout must be a positive number") + if not configured_timeouts: + producer_config["message.timeout.ms"] = timeout_ms + + flush_timeout = producer_config.pop("alarm.engine.flush.timeout.seconds", timeout_ms / 1000 + 1) + if producer_config.get("enable.idempotence", True) is not True: + raise ValueError("reference decision Kafka producer idempotence must be enabled") + producer_config["enable.idempotence"] = True + if producer_factory is None: + from confluent_kafka import Producer + + producer_factory = Producer + producer = producer_factory(producer_config) + return KafkaReferenceDecisionPublisher(producer=producer, topic=topic, flush_timeout=flush_timeout) + + +@lru_cache(maxsize=1) +def get_cached_kafka_reference_decision_publisher( + config_json: str, + allowed_topics: tuple[str, ...], + forbidden_topics: tuple[str, ...], +): + config = decode_json_document(config_json) + return build_kafka_reference_decision_publisher( + config, + allowed_topics=set(allowed_topics), + forbidden_topics=set(forbidden_topics), + ) diff --git a/bkmonitor/alarm_backends/core/alarm_engine/runtime.py b/bkmonitor/alarm_backends/core/alarm_engine/runtime.py new file mode 100644 index 00000000000..c356753c921 --- /dev/null +++ b/bkmonitor/alarm_backends/core/alarm_engine/runtime.py @@ -0,0 +1,140 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +from collections.abc import Mapping, Sequence + +from alarm_backends.core.alarm_engine.contract import ( + ContractValidationError, + build_detection_outcome, + build_trigger_strategy_ir_from_legacy_config, + validate_trigger_strategy_ir, +) +from alarm_backends.core.alarm_engine.encoder import encode_json_document + + +class DetectionNotFinalized(ContractValidationError): + """Raised when a record batch cannot safely be projected as a business outcome.""" + + +def prepare_finalized_threshold_batch( + *, + tenant_id: str, + strategy: Mapping, + item_id: str | int, + legacy_json: bytes, + batch_id: str, + data_points: Sequence[Mapping], + anomaly_outputs: Sequence[Mapping], + finalized: bool, +) -> dict: + """Build the first DETECT-only Threshold input batch after finalization is proven.""" + + if finalized is not True: + raise DetectionNotFinalized("detection batch is not finalized") + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id=tenant_id, + purpose="DETECT", + strategy=strategy, + item_id=item_id, + legacy_json=legacy_json, + ) + outcomes = project_detection_outcomes( + strategy_ir=strategy_ir, + batch_id=batch_id, + data_points=data_points, + anomaly_outputs=anomaly_outputs, + ) + return {"strategy_ir": strategy_ir, "outcomes": outcomes} + + +def project_detection_outcomes( + *, + strategy_ir: Mapping, + batch_id: str, + data_points: Sequence[Mapping], + anomaly_outputs: Sequence[Mapping], +) -> list[dict]: + """Project finalized legacy Detect results into one outcome per accepted record.""" + + validate_trigger_strategy_ir(strategy_ir) + anomalies_by_record = _index_anomalies(anomaly_outputs) + seen_records = set() + outcomes = [] + for data_raw in data_points: + if not isinstance(data_raw, Mapping): + raise ContractValidationError("data point must be an object") + record_id = data_raw.get("record_id") + if record_id in seen_records: + raise ContractValidationError(f"duplicate accepted record: {record_id}") + seen_records.add(record_id) + + indexed_output = anomalies_by_record.pop(record_id, None) + if indexed_output is None: + anomalies = {} + else: + if encode_json_document(indexed_output["data"]) != encode_json_document(data_raw): + raise ContractValidationError(f"anomaly output data does not match accepted data: {record_id}") + anomalies = indexed_output["anomalies"] + evaluations = [] + for level in strategy_ir["required_levels"]: + level_key = str(level) + if level_key not in anomalies: + evaluations.append({"level": level, "result": "NORMAL"}) + continue + anomaly = anomalies.pop(level_key) + evaluations.append( + { + "level": level, + "result": "ANOMALOUS", + "anomaly": copy.deepcopy(anomaly), + } + ) + if anomalies: + raise ContractValidationError(f"anomaly contains levels not required by StrategyIR: {sorted(anomalies)}") + outcomes.append( + build_detection_outcome( + strategy_ir=strategy_ir, + batch_id=batch_id, + data_raw=data_raw, + evaluations=evaluations, + outcome="ANOMALOUS" if any(item["result"] == "ANOMALOUS" for item in evaluations) else "NORMAL", + ) + ) + if anomalies_by_record: + raise ContractValidationError(f"anomaly output references unaccepted record: {sorted(anomalies_by_record)}") + return outcomes + + +def _index_anomalies(anomaly_outputs: Sequence[Mapping]) -> dict[str, dict]: + anomalies_by_record = {} + for output in anomaly_outputs: + if not isinstance(output, Mapping): + raise ContractValidationError("anomaly output must be an object") + data = output.get("data") + anomalies = output.get("anomaly") + if not isinstance(data, Mapping) or not isinstance(anomalies, Mapping): + raise ContractValidationError("anomaly output data and anomaly must be objects") + record_id = data.get("record_id") + if not isinstance(record_id, str): + raise ContractValidationError("anomaly output record_id must be a string") + if record_id in anomalies_by_record: + raise ContractValidationError(f"duplicate anomaly output: {record_id}") + if not anomalies: + raise ContractValidationError("anomaly output must contain at least one anomaly") + if any(not isinstance(level, str) for level in anomalies): + raise ContractValidationError("anomaly level keys must be strings") + if any(not isinstance(anomaly, Mapping) for anomaly in anomalies.values()): + raise ContractValidationError("anomaly entries must be objects") + anomalies_by_record[record_id] = { + "data": copy.deepcopy(dict(data)), + "anomalies": copy.deepcopy(dict(anomalies)), + } + return anomalies_by_record diff --git a/bkmonitor/alarm_backends/core/alert/adapter.py b/bkmonitor/alarm_backends/core/alert/adapter.py index 3f33d40308f..896aa5394c8 100644 --- a/bkmonitor/alarm_backends/core/alert/adapter.py +++ b/bkmonitor/alarm_backends/core/alert/adapter.py @@ -34,6 +34,12 @@ class MonitorEventAdapter: SPECIAL_ALERT_TAG_KEY_WHITELIST = [DoubleCheckStrategy.DOUBLE_CHECK_CONTEXT_KEY] + @classmethod + def get_output_topic(cls) -> str: + if get_cluster().is_default(): + return settings.MONITOR_EVENT_KAFKA_TOPIC + return f"{settings.MONITOR_EVENT_KAFKA_TOPIC}_{get_cluster().name}" + @classmethod def push_to_kafka(cls, events: list[dict]): """ @@ -46,14 +52,9 @@ def push_to_kafka(cls, events: list[dict]): if not events: return messages = [json.dumps(event).encode("utf-8") for event in events] - # 默认集群使用默认topic，其他集群使用集群名作为topic后缀 - if get_cluster().is_default(): - topic = settings.MONITOR_EVENT_KAFKA_TOPIC - else: - topic = f"{settings.MONITOR_EVENT_KAFKA_TOPIC}_{get_cluster().name}" # 使用专用kafka集群: ALERT_KAFKA_HOST ALERT_KAFKA_PORT kafka_queue = KafkaQueue.get_alert_kafka_queue() - kafka_queue.set_topic(topic) + kafka_queue.set_topic(cls.get_output_topic()) kafka_queue.put(value=messages) def __init__(self, record: dict, strategy: dict): diff --git a/bkmonitor/alarm_backends/service/detect/process.py b/bkmonitor/alarm_backends/service/detect/process.py index a9341870620..69797007e92 100644 --- a/bkmonitor/alarm_backends/service/detect/process.py +++ b/bkmonitor/alarm_backends/service/detect/process.py @@ -11,6 +11,7 @@ import json import logging import time +import uuid from django.conf import settings @@ -114,6 +115,139 @@ def bootstrap_new_series_empty_batch(item): NewSeries.bootstrap_empty_batch(item) + def prepare_alarm_engine_detection_batches(self): + if not settings.ALARM_ENGINE_DETECTION_SHADOW_ENABLED: + return [] + + from alarm_backends.core.alarm_engine.reference import parse_alarm_engine_shadow_strategy_ids + + allowed_strategy_ids = parse_alarm_engine_shadow_strategy_ids( + settings.ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS + ) + if allowed_strategy_ids is None: + logger.warning("[alarm engine shadow] configured strategy selector is invalid") + return [] + if int(self.strategy_id) not in allowed_strategy_ids: + return [] + + finalized = int(self.strategy_id) not in settings.DOUBLE_CHECK_SUM_STRATEGY_IDS + if not finalized: + return [] + + legacy_json = key.STRATEGY_SNAPSHOT_KEY.client.get(self.strategy.snapshot_key) + if isinstance(legacy_json, str): + legacy_json = legacy_json.encode() + if not isinstance(legacy_json, bytes) or not legacy_json: + logger.warning(f"[alarm engine shadow] strategy({self.strategy_id}) snapshot is unavailable") + return [] + + from alarm_backends.core.alarm_engine.contract import ContractValidationError, json_values_equal + from alarm_backends.core.alarm_engine.runtime import prepare_finalized_threshold_batch + + batch_id = uuid.uuid4().hex + batches = [] + for item in self.strategy.items: + input_points = self.inputs.get(item.id, []) + if not input_points: + continue + source_strategy = input_points[0].item.strategy + if ( + any(data_point.item.strategy is not source_strategy for data_point in input_points) + or int(source_strategy.id) != int(self.strategy_id) + or not json_values_equal(source_strategy.config, self.strategy.config) + ): + logger.warning( + f"[alarm engine shadow] strategy({self.strategy_id}) item({item.id}) input snapshot is stale" + ) + continue + data_points = [data_point.as_dict() for data_point in input_points] + if not data_points: + continue + try: + batch = prepare_finalized_threshold_batch( + tenant_id=self.strategy.bk_tenant_id, + strategy=self.strategy.config, + item_id=item.id, + legacy_json=legacy_json, + batch_id=batch_id, + data_points=data_points, + anomaly_outputs=self.outputs.get(item.id, []), + finalized=finalized, + ) + except ContractValidationError as error: + logger.debug( + f"[alarm engine shadow] strategy({self.strategy_id}) item({item.id}) is ineligible: {error}" + ) + continue + batches.append(batch) + return batches + + @staticmethod + def publish_alarm_engine_detection_batches(batches): + if not batches: + return 0 + + from alarm_backends.core.alarm_engine.publisher import get_cached_kafka_detection_publisher + from alarm_backends.core.alarm_engine.reference import build_terminal_reference_decision_batches + from alarm_backends.core.alarm_engine.reference_publisher import ( + get_cached_kafka_reference_decision_publisher, + ) + + config_json = json.dumps( + settings.ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG, + sort_keys=True, + separators=(",", ":"), + ) + allowed_topics = tuple(sorted(set(settings.ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS))) + publisher = get_cached_kafka_detection_publisher(config_json, allowed_topics) + reference_publisher = None + reference_initialization_failed = False + reference_enabled = settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED + + published = 0 + for batch in batches: + published += publisher.publish_batch(batch) + if not reference_enabled: + continue + try: + reference_batches = build_terminal_reference_decision_batches( + strategy_ir=batch["strategy_ir"], + detection_outcomes=batch["outcomes"], + ) + except Exception: + logger.exception("[alarm engine shadow] failed to project terminal reference decision") + continue + if not reference_batches or reference_initialization_failed: + continue + if reference_publisher is None: + try: + from alarm_backends.core.alert.adapter import MonitorEventAdapter + + reference_config_json = json.dumps( + settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG, + sort_keys=True, + separators=(",", ":"), + ) + reference_allowed_topics = tuple( + sorted(set(settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS)) + ) + forbidden_topics = tuple(sorted(set(allowed_topics) | {MonitorEventAdapter.get_output_topic()})) + reference_publisher = get_cached_kafka_reference_decision_publisher( + reference_config_json, + reference_allowed_topics, + forbidden_topics, + ) + except Exception: + logger.exception("[alarm engine shadow] failed to initialize terminal reference publisher") + reference_initialization_failed = True + continue + try: + for reference_batch in reference_batches: + reference_publisher.publish_batch(reference_batch) + except Exception: + logger.exception("[alarm engine shadow] failed to publish terminal reference decision") + return published + def push_data(self): current_time = time.time() max_latency = 0 @@ -143,6 +277,15 @@ def push_data(self): strategy_name=self.strategy.name, ).observe(max_latency) anomaly_count = self.push_abnormal_data(self.outputs, self.strategy_id) + try: + alarm_engine_batches = self.prepare_alarm_engine_detection_batches() + except Exception: + logger.exception(f"[alarm engine shadow] strategy({self.strategy_id}) failed to prepare detection batch") + alarm_engine_batches = [] + try: + self.publish_alarm_engine_detection_batches(alarm_engine_batches) + except Exception: + logger.exception(f"[alarm engine shadow] strategy({self.strategy_id}) failed to publish detection batch") if anomaly_count > 1000: # 获取 Redis 节点信息（带异常处理） try: diff --git a/bkmonitor/alarm_backends/service/trigger/processor.py b/bkmonitor/alarm_backends/service/trigger/processor.py index ae02b8624f3..b99d8416ea3 100644 --- a/bkmonitor/alarm_backends/service/trigger/processor.py +++ b/bkmonitor/alarm_backends/service/trigger/processor.py @@ -11,17 +11,23 @@ import json import logging import time +from itertools import chain + +from django.conf import settings from alarm_backends.core.alert.adapter import MonitorEventAdapter +from alarm_backends.core.cache import key as cache_key from alarm_backends.core.cache.key import ANOMALY_LIST_KEY, ANOMALY_SIGNAL_KEY, TRIGGER_EVENT_RATE_LIMIT_KEY from alarm_backends.core.control.strategy import Strategy from alarm_backends.core.storage.redis_cluster import get_node_by_strategy_id, routing_snapshot from alarm_backends.service.trigger.checker import AnomalyChecker +from bkmonitor.utils.tenant import bk_biz_id_to_bk_tenant_id from core.errors.alarm_backends import StrategyNotFound from core.prometheus import metrics # 每个（策略, 数据时间戳）计数器的最大 event 数，超过则丢弃 TRIGGER_EVENT_RATE_LIMIT_THRESHOLD = 5000 +ALARM_ENGINE_REFERENCE_BATCHES_PER_FLUSH = 500 logger = logging.getLogger("trigger") @@ -37,8 +43,10 @@ def __init__(self, strategy_id, item_id): self.anomaly_points = [] self.anomaly_records = [] self.event_records = [] + self.reference_candidates = [] # 策略快照数据 self._strategy_snapshots = {} + self._strategy_snapshot_legacy_json = {} self.strategy = Strategy(self.strategy_id) def get_strategy_snapshot(self, key): @@ -56,22 +64,139 @@ def get_strategy_snapshot(self, key): self._strategy_snapshots[key] = snapshot return snapshot + def get_strategy_snapshot_legacy_json(self, snapshot_key): + """Read and cache the exact legacy strategy document used by this Trigger point.""" + + try: + return self._strategy_snapshot_legacy_json[snapshot_key] + except KeyError: + routed_snapshot_key = cache_key.SimilarStr(snapshot_key) + routed_snapshot_key.strategy_id = self.strategy_id + legacy_json = cache_key.STRATEGY_SNAPSHOT_KEY.client.get(routed_snapshot_key) + if isinstance(legacy_json, str): + legacy_json = legacy_json.encode("utf-8") + if not isinstance(legacy_json, bytes) or not legacy_json: + raise StrategyNotFound({"key": snapshot_key}) + self._strategy_snapshot_legacy_json[snapshot_key] = legacy_json + return legacy_json + + def is_alarm_engine_reference_selected(self): + if not settings.ALARM_ENGINE_DETECTION_SHADOW_ENABLED: + return False + if not settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED: + return False + try: + if self.strategy_id in settings.DOUBLE_CHECK_SUM_STRATEGY_IDS: + return False + except TypeError: + return False + + from alarm_backends.core.alarm_engine.reference import is_alarm_engine_shadow_strategy_selected + + return is_alarm_engine_shadow_strategy_selected( + settings.ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS, + self.strategy_id, + ) + + def capture_alarm_engine_reference_candidate(self, *, point, event_record): + try: + self.reference_candidates.append( + { + "strategy_snapshot_key": point["strategy_snapshot_key"], + "point": point, + "event_record": event_record, + } + ) + except Exception: + logger.exception( + "[alarm engine shadow] failed to capture Trigger reference candidate for strategy(%s) item(%s)", + self.strategy_id, + self.item_id, + ) + + def publish_alarm_engine_reference_candidates(self): + if not self.reference_candidates: + return 0 + + from alarm_backends.core.alarm_engine.reference import build_reference_trigger_decision_candidate + from alarm_backends.core.alarm_engine.reference_publisher import ( + get_cached_kafka_reference_decision_publisher, + ) + + publisher = None + published = 0 + for start in range(0, len(self.reference_candidates), ALARM_ENGINE_REFERENCE_BATCHES_PER_FLUSH): + + def iter_batches(): + for candidate in self.reference_candidates[start : start + ALARM_ENGINE_REFERENCE_BATCHES_PER_FLUSH]: + try: + strategy_snapshot_key = candidate["strategy_snapshot_key"] + yield build_reference_trigger_decision_candidate( + strategy=self.get_strategy_snapshot(strategy_snapshot_key), + legacy_json=self.get_strategy_snapshot_legacy_json(strategy_snapshot_key), + strategy_snapshot_key=strategy_snapshot_key, + tenant_id_resolver=bk_biz_id_to_bk_tenant_id, + item_id=self.item_id, + point=candidate["point"], + event_record=candidate["event_record"], + ) + except Exception: + logger.exception( + "[alarm engine shadow] failed to project Trigger reference for strategy(%s) item(%s)", + self.strategy_id, + self.item_id, + ) + + batches = iter_batches() + try: + first_batch = next(batches) + except StopIteration: + continue + if publisher is None: + try: + config_json = json.dumps( + settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG, + sort_keys=True, + separators=(",", ":"), + ) + allowed_topics = tuple(sorted(set(settings.ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS))) + forbidden_topics = tuple( + sorted( + set(settings.ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS) + | {MonitorEventAdapter.get_output_topic()} + ) + ) + publisher = get_cached_kafka_reference_decision_publisher( + config_json, + allowed_topics, + forbidden_topics, + ) + except Exception: + logger.exception("[alarm engine shadow] failed to initialize Trigger reference publisher") + break + try: + published += publisher.publish_batches(chain((first_batch,), batches)) + except Exception: + logger.exception( + "[alarm engine shadow] failed to publish Trigger reference for strategy(%s) item(%s)", + self.strategy_id, + self.item_id, + ) + break + return published + def pull(self): # lrange + ltrim 必须落在同一路由快照：列表长度依赖首读结果，无法无脑打进一个 pipeline， # 用 routing_snapshot 避免 TTL 边界把读/裁切拆到不同 Redis 节点。 with routing_snapshot(): - self.anomaly_points = ANOMALY_LIST_KEY.client.lrange( - self.anomaly_list_key, -self.MAX_PROCESS_COUNT, -1 - ) + self.anomaly_points = ANOMALY_LIST_KEY.client.lrange(self.anomaly_list_key, -self.MAX_PROCESS_COUNT, -1) # 对列表做翻转，按数据从旧到新的顺序处理 self.anomaly_points.reverse() if self.anomaly_points: metrics.TRIGGER_PROCESS_PULL_DATA_COUNT.labels(strategy_id=metrics.TOTAL_TAG).inc( len(self.anomaly_points) ) - ANOMALY_LIST_KEY.client.ltrim( - self.anomaly_list_key, 0, -len(self.anomaly_points) - 1 - ) + ANOMALY_LIST_KEY.client.ltrim(self.anomaly_list_key, 0, -len(self.anomaly_points) - 1) if self.anomaly_points: if len(self.anomaly_points) == self.MAX_PROCESS_COUNT: # 拉取到的数量若等于最大数量，说明还没拉取完，下次需要再次拉取处理 @@ -273,9 +398,19 @@ def push(self): ) metrics.TRIGGER_PROCESS_PUSH_DATA_COUNT.labels(strategy_id=metrics.TOTAL_TAG).inc(len(self.event_records)) + try: + self.publish_alarm_engine_reference_candidates() + except Exception: + logger.exception( + "[alarm engine shadow] unexpected Trigger reference failure for strategy(%s) item(%s)", + self.strategy_id, + self.item_id, + ) + self.anomaly_points = [] self.anomaly_records = [] self.event_records = [] + self.reference_candidates = [] def process(self): self.pull() @@ -299,6 +434,12 @@ def process_point(self, point): checker = AnomalyChecker(point, strategy, self.item_id) anomaly_records, event_record = checker.check() + if self.is_alarm_engine_reference_selected() and not checker.is_no_data_point(point): + self.capture_alarm_engine_reference_candidate( + point=point, + event_record=event_record, + ) + # 暂存结果，最后批量保存 if event_record: self.event_records.append({"anomaly_records": anomaly_records, "event_record": event_record}) diff --git a/bkmonitor/alarm_backends/tests/alarm_engine_fixtures.py b/bkmonitor/alarm_backends/tests/alarm_engine_fixtures.py new file mode 100644 index 00000000000..37bbc3be92b --- /dev/null +++ b/bkmonitor/alarm_backends/tests/alarm_engine_fixtures.py @@ -0,0 +1,232 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +DETECT_STRATEGY = { + "bk_biz_id": 2, + "items": [ + { + "query_configs": [ + { + "metric_field": "idle", + "agg_dimension": ["ip", "bk_cloud_id"], + "id": 2, + "agg_method": "AVG", + "agg_condition": [], + "agg_interval": 60, + "result_table_id": "system.cpu_detail", + "unit": "%", + "data_type_label": "time_series", + "metric_id": "bk_monitor.system.cpu_detail.idle", + "data_source_label": "bk_monitor", + } + ], + "algorithms": [ + { + "config": [[{"threshold": 51.0, "method": "gte"}]], + "level": 3, + "type": "Threshold", + "id": 2, + }, + { + "config": [[{"threshold": 100, "method": "lte"}]], + "level": 3, + "type": "Threshold", + "id": 3, + }, + ], + "no_data_config": {"is_enabled": False, "continuous": 5}, + "id": 2, + "name": "\u7a7a\u95f2\u7387", + "target": [ + [{"field": "ip", "method": "eq", "value": [{"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}]}] + ], + } + ], + "scenario": "os", + "actions": [ + { + "notice_template": {"action_id": 2, "anomaly_template": "aa", "recovery_template": ""}, + "id": 2, + "notice_group_list": [ + { + "notice_receiver": ["user#test"], + "name": "test", + "notice_way": {"1": ["weixin"], "3": ["weixin"], "2": ["weixin"]}, + "notice_group_id": 1, + "message": "", + "notice_group_name": "test", + "id": 1, + } + ], + "type": "notice", + "config": { + "alarm_end_time": "23:59:59", + "send_recovery_alarm": False, + "alarm_start_time": "00:00:00", + "alarm_interval": 120, + }, + } + ], + "detects": [ + { + "level": 3, + "expression": "", + "connector": "and", + "trigger_config": {"count": 1, "check_window": 5}, + "recovery_config": {"check_window": 5}, + } + ], + "update_time": 1569246480, + "source_type": "BKMONITOR", + "id": 1, + "name": "test", +} + + +DETECT_RECORDS = [ + { + "record_id": "342a08e0f85f169a7e099c18db3708ed.1569246480", + "value": 99, + "values": {"timestamp": 1569246480, "load5": 99}, + "dimensions": {"ip": "127.0.0.1"}, + "time": 1569246480, + }, + { + "record_id": "2a1850513fa6018c435f9b6359b3fa7d.1569246481", + "value": 50.1, + "values": {"timestamp": 1569246481, "load5": 50.1}, + "dimensions": {"ip": "10.0.0.1"}, + # 数据点时间戳错开，避免检测记录断言不准确 + "time": 1569246481, + }, +] + + +TRIGGER_STRATEGY = { + "bk_biz_id": 2, + "items": [ + { + "query_configs": [ + { + "metric_field": "idle", + "agg_dimension": ["ip", "bk_cloud_id"], + "id": 2, + "agg_method": "AVG", + "agg_condition": [], + "agg_interval": 60, + "result_table_id": "system.cpu_detail", + "unit": "%", + "data_type_label": "time_series", + "metric_id": "bk_monitor.system.cpu_detail.idle", + "data_source_label": "bk_monitor", + } + ], + "target": [ + [ + { + "field": "ip", + "method": "eq", + "value": [ + {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}, + ], + } + ] + ], + "algorithms": [ + {"config": [{"threshold": 0.1, "method": "gte"}], "level": 1, "type": "Threshold", "id": 1}, + {"config": [{"threshold": 0.1, "method": "gte"}], "level": 2, "type": "Threshold", "id": 2}, + {"config": [{"threshold": 0.1, "method": "gte"}], "level": 3, "type": "Threshold", "id": 3}, + ], + "no_data_config": {"is_enabled": False, "continuous": 5}, + "id": 1, + "name": "\u7a7a\u95f2\u7387", + } + ], + "detects": [ + { + "expression": "", + "connector": "and", + "level": 1, + "trigger_config": {"count": 3, "check_window": 5}, + "recovery_config": {"check_window": 5}, + }, + { + "expression": "", + "connector": "and", + "level": 2, + "trigger_config": {"count": 2, "check_window": 5}, + "recovery_config": {"check_window": 5}, + }, + { + "expression": "", + "connector": "and", + "level": 3, + "trigger_config": {"count": 1, "check_window": 5}, + "recovery_config": {"check_window": 5}, + }, + ], + "scenario": "os", + "actions": [ + { + "notice_template": {"anomaly_template": "aa", "recovery_template": ""}, + "id": 2, + "notice_group_list": [ + { + "notice_receiver": ["user#test"], + "name": "test", + "notice_way": {"1": ["weixin"], "3": ["weixin"], "2": ["weixin"]}, + "notice_group_id": 1, + "message": "", + "notice_group_name": "test", + "id": 1, + } + ], + "type": "notice", + "config": { + "alarm_end_time": "23:59:59", + "send_recovery_alarm": False, + "alarm_start_time": "00:00:00", + "alarm_interval": 120, + }, + } + ], + "source_type": "BKMONITOR", + "id": 1, + "name": "test", +} + + +TRIGGER_POINT = { + "data": { + "record_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480", + "value": 1.38, + "values": {"timestamp": 1569246480, "load5": 1.38}, + "dimensions": {"ip": "10.0.0.1"}, + "time": 1569246480, + }, + "anomaly": { + "1": { + "anomaly_message": "异常测试", + "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.1", + "anomaly_time": "2019-10-10 10:10:00", + }, + "2": { + "anomaly_message": "异常测试", + "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.2", + "anomaly_time": "2019-10-10 10:10:00", + }, + "3": { + "anomaly_message": "异常测试", + "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.3", + "anomaly_time": "2019-10-10 10:10:00", + }, + }, + "strategy_snapshot_key": "xxx", +} diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/__init__.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/__init__.py new file mode 100644 index 00000000000..239267df83d --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/__init__.py @@ -0,0 +1,9 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_contract.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_contract.py new file mode 100644 index 00000000000..1b63ac63b8e --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_contract.py @@ -0,0 +1,674 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import base64 +import copy + +import pytest + +from alarm_backends.core.alarm_engine.contract import ( + ContractValidationError, + build_detection_outcome, + build_trigger_decision_batch, + build_trigger_strategy_ir, + build_trigger_strategy_ir_from_legacy_config, + can_drive_trigger, + derive_input_id, + derive_trigger_decision_id, + validate_detection_outcome, + validate_trigger_decision_batch, + validate_trigger_strategy_ir, +) +from alarm_backends.core.alarm_engine.encoder import ( + MAX_TRIGGER_DECISION_BATCH_BYTES, + decode_json_document, + decode_trigger_decision_batch, + encode_json_document, + encode_trigger_decision_batch, +) + + +LEGACY_JSON = b'{"id":1,"update_time":1569246480}' +LEGACY_JSON_SHA256 = "8a340c044a560d3410cd4d53098151eac966b8321a5ad01b43547b05f960e2c3" +DIMENSIONS_MD5 = "55a76cf628e46c04a052f4e19bdb9dbf" +SOURCE_TIME = 1569246480 +RECORD_ID = f"{DIMENSIONS_MD5}.{SOURCE_TIME}" +EXPECTED_INPUT_ID = "2c82173befc4a450616df467cfd27d903ac5417e44413c7144780dfe43a8ef44" + + +def make_strategy_ir(): + return build_trigger_strategy_ir( + tenant_id="default", + purpose="DETECT", + strategy_id=1, + item_id=1, + generation="1569246480", + legacy_json=LEGACY_JSON, + check_window_unit_seconds=60, + trigger_configs={ + 1: {"check_window_size": 5, "trigger_count": 3}, + 2: {"check_window_size": 5, "trigger_count": 2}, + 3: {"check_window_size": 5, "trigger_count": 1}, + }, + ) + + +def make_data_raw(): + return { + "record_id": RECORD_ID, + "value": 1.38, + "values": {"timestamp": SOURCE_TIME, "huge_counter": 9007199254740993}, + "dimensions": {"ip": "10.0.0.1", "mixed": 7}, + "time": SOURCE_TIME, + } + + +def normal_evaluations(): + return [{"level": level, "result": "NORMAL"} for level in (1, 2, 3)] + + +def anomalous_evaluations(): + return [ + { + "level": level, + "result": "ANOMALOUS", + "anomaly": { + "anomaly_id": f"{RECORD_ID}.1.1.{level}", + "anomaly_message": "异常测试", + "context": {"level": level, "mixed": [1, "2", {"nested": True}]}, + }, + } + for level in (1, 2, 3) + ] + + +def build_outcome(*, outcome="NORMAL", evaluations=None, error_code=None, batch_id="batch-1"): + return build_detection_outcome( + strategy_ir=make_strategy_ir(), + batch_id=batch_id, + data_raw=make_data_raw(), + evaluations=normal_evaluations() if evaluations is None else evaluations, + outcome=outcome, + error_code=error_code, + ) + + +def test_input_id_uses_frozen_length_prefixed_tuple(): + input_id = derive_input_id( + tenant_id="default", + purpose="DETECT", + strategy_id=1, + item_id=1, + strategy_content_sha256=LEGACY_JSON_SHA256, + record_id=RECORD_ID, + ) + + assert input_id == EXPECTED_INPUT_ID + + +def test_trigger_decision_id_matches_go_golden(): + assert derive_trigger_decision_id("a" * 64) == "ff722dbf77ffa1b06945ab996e22adbefd6f22c32dec2173fd1ed659b874a130" + + +def test_trigger_decision_batch_round_trip_contract(): + strategy_ir = make_strategy_ir() + source = build_outcome() + decision = { + "decision_id": derive_trigger_decision_id(source["input_id"]), + "input_id": source["input_id"], + "record_id": source["record"]["record_id"], + "outcome": "NO_TRIGGER", + "reason_code": "INPUT_NORMAL", + "anomaly_timestamps": [], + } + + batch = build_trigger_decision_batch( + strategy_ir=strategy_ir, + batch_id="python-reference-batch", + decisions=[decision], + ) + + validate_trigger_decision_batch(batch) + assert decode_trigger_decision_batch(encode_trigger_decision_batch(batch)) == batch + assert batch["schema"] == {"name": "trigger-decision-batch", "major": 1, "minor": 0} + assert batch["decision_algorithm"] == "trigger-window-v1" + + +def test_trigger_decision_codec_enforces_exact_encoded_byte_limit(): + strategy_ir = make_strategy_ir() + source = build_outcome() + decision = { + "decision_id": derive_trigger_decision_id(source["input_id"]), + "input_id": source["input_id"], + "record_id": source["record"]["record_id"], + "outcome": "NO_TRIGGER", + "reason_code": "INPUT_NORMAL", + "anomaly_timestamps": [], + } + batch = build_trigger_decision_batch( + strategy_ir=strategy_ir, + batch_id="python-reference-batch", + decisions=[decision], + ) + batch["schema"]["minor"] = 1 + batch["padding"] = "" + base_size = len(encode_json_document(batch)) + batch["padding"] = "x" * (MAX_TRIGGER_DECISION_BATCH_BYTES - base_size) + + exact = encode_trigger_decision_batch(batch) + assert len(exact) == MAX_TRIGGER_DECISION_BATCH_BYTES + assert decode_trigger_decision_batch(exact) == batch + + batch["padding"] += "x" + with pytest.raises(ContractValidationError, match="byte limit"): + encode_trigger_decision_batch(batch) + with pytest.raises(ContractValidationError, match="byte limit"): + decode_trigger_decision_batch(encode_json_document(batch)) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda decision: decision.update(decision_id="0" * 64), + lambda decision: decision.update(record_id=f"{DIMENSIONS_MD5}.1569246481"), + lambda decision: decision.update(level=1), + lambda decision: decision.update(anomaly_timestamps=[SOURCE_TIME]), + ], +) +def test_trigger_decision_batch_rejects_identity_and_terminal_drift(mutate): + strategy_ir = make_strategy_ir() + source = build_outcome() + decision = { + "decision_id": derive_trigger_decision_id(source["input_id"]), + "input_id": source["input_id"], + "record_id": source["record"]["record_id"], + "outcome": "NO_TRIGGER", + "reason_code": "INPUT_NORMAL", + "anomaly_timestamps": [], + } + mutate(decision) + + with pytest.raises(ContractValidationError): + build_trigger_decision_batch( + strategy_ir=strategy_ir, + batch_id="python-reference-batch", + decisions=[decision], + ) + + +def test_input_id_uses_utf8_byte_length_and_rejects_invalid_surrogate(): + fields = { + "tenant_id": "租户", + "purpose": "DETECT", + "strategy_id": 1, + "item_id": 1, + "strategy_content_sha256": LEGACY_JSON_SHA256, + "record_id": RECORD_ID, + } + + assert derive_input_id(**fields) == "4b25f5e001820e6de39c45a27bf65b27b665d542009d3a004a73ee1629f014d7" + fields["tenant_id"] = "\ud800" + with pytest.raises(ContractValidationError, match="UTF-8"): + derive_input_id(**fields) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("purpose", "detect"), + ("strategy_id", "01"), + ("item_id", "+1"), + ("strategy_content_sha256", LEGACY_JSON_SHA256.upper()), + ("record_id", f"{DIMENSIONS_MD5}.01569246480"), + ], +) +def test_input_id_rejects_noncanonical_fields(field, value): + fields = { + "tenant_id": "default", + "purpose": "DETECT", + "strategy_id": 1, + "item_id": 1, + "strategy_content_sha256": LEGACY_JSON_SHA256, + "record_id": RECORD_ID, + } + fields[field] = value + + with pytest.raises(ContractValidationError): + derive_input_id(**fields) + + +def test_strategy_ir_preserves_legacy_bytes_and_normalizes_trigger_levels(): + strategy_ir = make_strategy_ir() + + validate_trigger_strategy_ir(strategy_ir) + assert strategy_ir["required_levels"] == [1, 2, 3] + assert [config["level"] for config in strategy_ir["trigger_configs"]] == [1, 2, 3] + assert strategy_ir["strategy_ref"]["content_sha256"] == LEGACY_JSON_SHA256 + assert base64.b64decode(strategy_ir["legacy_json_b64"], validate=True) == LEGACY_JSON + + +def test_strategy_ir_rejects_features_owned_by_another_contract(): + strategy_ir = make_strategy_ir() + strategy_ir["required_features"].append("raw-json-v1") + + with pytest.raises(ContractValidationError, match="unsupported required feature"): + validate_trigger_strategy_ir(strategy_ir) + + +def test_typed_contract_integers_are_bounded_to_int32(): + strategy_ir = make_strategy_ir() + strategy_ir["schema"]["minor"] = 2**31 + with pytest.raises(ContractValidationError, match="32-bit"): + validate_trigger_strategy_ir(strategy_ir) + + for mutate in ( + lambda value: value.update(check_window_unit_seconds=2**31), + lambda value: value["trigger_configs"][0].update(trigger_count=2**31), + lambda value: ( + value["required_levels"].__setitem__(0, 2**31), + value["trigger_configs"][0].update(level=2**31), + ), + ): + strategy_ir = make_strategy_ir() + mutate(strategy_ir) + with pytest.raises(ContractValidationError, match="32-bit|positive"): + validate_trigger_strategy_ir(strategy_ir) + + strategy_ir = make_strategy_ir() + strategy_ir["schema"]["minor"] = 2**31 - 1 + strategy_ir["check_window_unit_seconds"] = 2**31 - 1 + strategy_ir["required_levels"] = [2**31 - 1] + strategy_ir["trigger_configs"] = [ + { + "level": 2**31 - 1, + "check_window_size": 2**31 - 1, + "trigger_count": 2**31 - 1, + } + ] + validate_trigger_strategy_ir(strategy_ir) + + outcome = build_outcome() + outcome["evaluations"][0]["level"] = 2**31 + with pytest.raises(ContractValidationError, match="32-bit"): + validate_detection_outcome(outcome, make_strategy_ir()) + + strategy_ir = make_strategy_ir() + strategy_ir["trigger_configs"][0]["check_window_size"] = 2**31 + with pytest.raises(ContractValidationError, match="32-bit"): + validate_trigger_strategy_ir(strategy_ir) + + +def test_normal_and_anomalous_outcomes_require_complete_levels(): + normal = build_outcome() + anomalous = build_outcome(outcome="ANOMALOUS", evaluations=anomalous_evaluations()) + + validate_detection_outcome(normal, make_strategy_ir()) + validate_detection_outcome(anomalous, make_strategy_ir()) + assert can_drive_trigger(normal, make_strategy_ir()) is True + assert can_drive_trigger(anomalous, make_strategy_ir()) is True + + incomplete = copy.deepcopy(normal) + incomplete["evaluations"].pop() + with pytest.raises(ContractValidationError, match="complete"): + validate_detection_outcome(incomplete, make_strategy_ir()) + + +def test_error_and_unsupported_allow_only_required_level_subsets(): + partial_error = build_outcome( + outcome="ERROR", + evaluations=[{"level": 1, "result": "NORMAL"}], + error_code="ALGORITHM_ERROR", + ) + unsupported = build_outcome( + outcome="UNSUPPORTED", + evaluations=[], + error_code="UNSUPPORTED_STRATEGY", + ) + + validate_detection_outcome(partial_error, make_strategy_ir()) + validate_detection_outcome(unsupported, make_strategy_ir()) + assert can_drive_trigger(partial_error, make_strategy_ir()) is False + assert can_drive_trigger(unsupported, make_strategy_ir()) is False + + duplicate = copy.deepcopy(partial_error) + duplicate["evaluations"].append(copy.deepcopy(duplicate["evaluations"][0])) + with pytest.raises(ContractValidationError, match="duplicate"): + validate_detection_outcome(duplicate, make_strategy_ir()) + + missing = copy.deepcopy(unsupported) + missing.pop("evaluations") + explicit_null = copy.deepcopy(unsupported) + explicit_null["evaluations"] = None + for invalid in (missing, explicit_null): + with pytest.raises(ContractValidationError, match="evaluations"): + validate_detection_outcome(invalid, make_strategy_ir()) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda value: value["record"].update(record_id=f"{DIMENSIONS_MD5}.1569246481"), + lambda value: value["record"]["data_raw"].update(time=1569246481), + lambda value: value["strategy_ref"].update(item_id="2"), + lambda value: value.update(input_id="0" * 64), + ], +) +def test_outcome_rejects_cross_field_mismatches(mutate): + outcome = build_outcome() + mutate(outcome) + + with pytest.raises(ContractValidationError): + validate_detection_outcome(outcome, make_strategy_ir()) + + +def test_evaluation_anomaly_invariants_are_fail_closed(): + normal_with_anomaly = normal_evaluations() + normal_with_anomaly[0]["anomaly"] = anomalous_evaluations()[0]["anomaly"] + with pytest.raises(ContractValidationError, match="NORMAL"): + build_outcome(evaluations=normal_with_anomaly) + + anomalous_without_payload = anomalous_evaluations() + anomalous_without_payload[0].pop("anomaly") + with pytest.raises(ContractValidationError, match="ANOMALOUS"): + build_outcome(outcome="ANOMALOUS", evaluations=anomalous_without_payload) + + explicit_null = build_outcome() + explicit_null["evaluations"][0]["anomaly"] = None + with pytest.raises(ContractValidationError, match="NORMAL"): + validate_detection_outcome(explicit_null, make_strategy_ir()) + + +def test_unknown_major_feature_and_business_enum_are_rejected(): + cases = [] + unknown_major = build_outcome() + unknown_major["schema"]["major"] = 2 + cases.append(unknown_major) + unknown_feature = build_outcome() + unknown_feature["required_features"].append("future-required-feature") + cases.append(unknown_feature) + unknown_outcome = build_outcome() + unknown_outcome["outcome"] = "DEFERRED" + cases.append(unknown_outcome) + + for case in cases: + with pytest.raises(ContractValidationError): + validate_detection_outcome(case, make_strategy_ir()) + + boolean_major = build_outcome() + boolean_major["schema"]["major"] = True + unhashable_purpose = build_outcome() + unhashable_purpose["purpose"] = [] + unhashable_result = build_outcome() + unhashable_result["evaluations"][0]["result"] = [] + for case in (boolean_major, unhashable_purpose, unhashable_result): + with pytest.raises(ContractValidationError): + validate_detection_outcome(case, make_strategy_ir()) + + +def test_business_outcome_rejects_explicit_null_error_code(): + outcome = build_outcome() + outcome["error_code"] = None + + with pytest.raises(ContractValidationError, match="error_code"): + validate_detection_outcome(outcome, make_strategy_ir()) + + +def test_higher_minor_ignores_unknown_optional_fields_when_features_are_supported(): + outcome = build_outcome() + outcome["schema"]["minor"] = 1 + outcome["future_optional_diagnostic"] = {"ignored": True} + + validate_detection_outcome(outcome, make_strategy_ir()) + + +@pytest.mark.parametrize("field", ["future_optional_diagnostic", "Schema"]) +def test_v1_rejects_unknown_or_case_colliding_fixed_fields(field): + outcome = build_outcome() + outcome[field] = {"ignored": True} + + with pytest.raises(ContractValidationError, match="field"): + validate_detection_outcome(outcome, make_strategy_ir()) + + +def test_json_encoder_preserves_large_integers_and_rejects_nonfinite_numbers(): + outcome = build_outcome(outcome="ANOMALOUS", evaluations=anomalous_evaluations()) + + decoded = decode_json_document(encode_json_document(outcome)) + assert decoded == outcome + assert decoded["record"]["data_raw"]["values"]["huge_counter"] == 9007199254740993 + + outcome["record"]["data_raw"]["value"] = float("nan") + with pytest.raises(ValueError): + encode_json_document(outcome) + + +def test_json_decoder_rejects_duplicate_fields_and_nonfinite_numbers(): + with pytest.raises(ContractValidationError, match="duplicate"): + decode_json_document(b'{"input_id":"first","input_id":"second"}') + with pytest.raises(ContractValidationError, match="non-finite"): + decode_json_document(b'{"value":NaN}') + with pytest.raises(ContractValidationError, match="non-finite"): + decode_json_document(b'{"value":1e400}') + + +def test_json_codec_rejects_nonstring_keys_surrogates_and_bom(): + with pytest.raises(ContractValidationError, match="keys must be strings"): + encode_json_document({"raw": {1: "numeric-key"}}) + with pytest.raises(ContractValidationError, match="UTF-8"): + encode_json_document({"raw": "\ud800"}) + with pytest.raises(ContractValidationError, match="UTF-8"): + decode_json_document(b'{"raw":"\\ud800"}') + with pytest.raises(ContractValidationError, match="BOM"): + decode_json_document(b'\xef\xbb\xbf{"raw":true}') + + valid = {"raw": "\ufffd😀"} + assert decode_json_document(encode_json_document(valid)) == valid + + +def test_strategy_ir_rejects_ambiguous_or_nonfinite_legacy_json(): + for legacy_json in (b'{"id":1,"id":2}', b'{"value":NaN}', b"null", b'\xef\xbb\xbf{"id":1}'): + with pytest.raises(ContractValidationError): + build_trigger_strategy_ir( + tenant_id="default", + purpose="DETECT", + strategy_id=1, + item_id=1, + generation="1", + legacy_json=legacy_json, + check_window_unit_seconds=60, + trigger_configs={1: {"check_window_size": 1, "trigger_count": 1}}, + ) + + +def test_record_source_coordinates_reject_boolean_timestamps(): + data_raw = make_data_raw() + data_raw["record_id"] = f"{DIMENSIONS_MD5}.1" + data_raw["time"] = True + data_raw["values"]["timestamp"] = True + + with pytest.raises(ContractValidationError): + build_detection_outcome( + strategy_ir=make_strategy_ir(), + batch_id="batch-1", + data_raw=data_raw, + evaluations=normal_evaluations(), + outcome="NORMAL", + ) + + +@pytest.mark.parametrize("values", [[1, "2"], "opaque", 7]) +def test_record_values_remain_open_raw_json(values): + data_raw = make_data_raw() + data_raw["values"] = values + + outcome = build_detection_outcome( + strategy_ir=make_strategy_ir(), + batch_id="batch-1", + data_raw=data_raw, + evaluations=normal_evaluations(), + outcome="NORMAL", + ) + + assert outcome["record"]["data_raw"]["values"] == values + + +def test_contract_builders_reject_wrong_container_types_with_validation_error(): + with pytest.raises(ContractValidationError): + build_trigger_strategy_ir( + tenant_id="default", + purpose="DETECT", + strategy_id=1, + item_id=1, + generation="1", + legacy_json=b'{"id":1}', + check_window_unit_seconds=60, + trigger_configs=[], + ) + + +def test_retry_changes_batch_id_but_not_input_id(): + first = build_outcome(batch_id="batch-1") + retry = build_outcome(batch_id="batch-retry-2") + + assert first["batch_id"] != retry["batch_id"] + assert first["input_id"] == retry["input_id"] == EXPECTED_INPUT_ID + + +def test_strategy_ir_adapter_uses_threshold_strategy_semantics_without_reencoding_legacy_json(): + # Reduced from the current three-level Trigger fixture in service/trigger/test_checker.py. + strategy = { + "id": 1, + "update_time": SOURCE_TIME, + "items": [ + { + "id": 1, + "query_configs": [{"agg_interval": 60}], + "algorithms": [ + {"level": 1, "type": "Threshold"}, + {"level": 2, "type": "Threshold"}, + {"level": 3, "type": "Threshold"}, + ], + "no_data_config": {"is_enabled": False}, + } + ], + "detects": [ + {"level": 1, "trigger_config": {"count": 3, "check_window": 5}}, + {"level": 2, "trigger_config": {"count": 2, "check_window": 5}}, + {"level": 3, "trigger_config": {"count": 1, "check_window": 5}}, + ], + } + legacy_json = b'{ "update_time": 1569246480, "id": 1, "items": [{"id": 1, "query_configs": [{"agg_interval": 60}], "algorithms": [{"level": 1, "type": "Threshold"}, {"level": 2, "type": "Threshold"}, {"level": 3, "type": "Threshold"}], "no_data_config": {"is_enabled": false}}], "detects": [{"level": 1, "trigger_config": {"count": 3, "check_window": 5}}, {"level": 2, "trigger_config": {"count": 2, "check_window": 5}}, {"level": 3, "trigger_config": {"count": 1, "check_window": 5}}] }' + + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=legacy_json, + ) + + assert strategy_ir["required_levels"] == [1, 2, 3] + assert strategy_ir["check_window_unit_seconds"] == 60 + assert [config["trigger_count"] for config in strategy_ir["trigger_configs"]] == [3, 2, 1] + assert base64.b64decode(strategy_ir["legacy_json_b64"], validate=True) == legacy_json + + +@pytest.mark.parametrize("target_first", [True, False]) +def test_strategy_ir_adapter_only_checks_algorithms_for_target_item(target_first): + target_item = { + "id": 1, + "query_configs": [{"agg_interval": 60}], + "algorithms": [{"level": 1, "type": "Threshold"}], + "no_data_config": {"is_enabled": False}, + } + other_item = { + "id": 2, + "query_configs": [{"agg_interval": 60}], + "algorithms": [{"level": 1, "type": "IntelligentDetect"}], + "no_data_config": {"is_enabled": False}, + } + strategy = { + "id": 1, + "update_time": SOURCE_TIME, + "items": [target_item, other_item] if target_first else [other_item, target_item], + "detects": [{"level": 1, "trigger_config": {"count": 1, "check_window": 5}}], + } + + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=encode_json_document(strategy), + ) + + assert strategy_ir["strategy_ref"]["item_id"] == "1" + assert strategy_ir["required_levels"] == [1] + + +def test_strategy_ir_adapter_rejects_boolean_integer_type_drift(): + strategy = { + "id": 1, + "update_time": SOURCE_TIME, + "items": [ + { + "id": 1, + "query_configs": [{"agg_interval": 60}], + "algorithms": [{"level": 1, "type": "Threshold"}], + "no_data_config": {"is_enabled": False}, + } + ], + "detects": [{"level": 1, "trigger_config": {"count": 1, "check_window": 5}}], + } + legacy_json = encode_json_document({**strategy, "id": True}) + + with pytest.raises(ContractValidationError, match="semantic drift"): + build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=legacy_json, + ) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda strategy: strategy["items"][0]["algorithms"][0].update(type="IntelligentDetect"), + lambda strategy: strategy["items"][0]["no_data_config"].update(is_enabled=True), + lambda strategy: strategy["detects"][0]["trigger_config"].update(uptime={"time_ranges": []}), + ], +) +def test_strategy_ir_adapter_rejects_features_outside_first_threshold_slice(mutate): + strategy = { + "id": 1, + "update_time": SOURCE_TIME, + "items": [ + { + "id": 1, + "query_configs": [{"agg_interval": 60}], + "algorithms": [{"level": 1, "type": "Threshold"}], + "no_data_config": {"is_enabled": False}, + } + ], + "detects": [{"level": 1, "trigger_config": {"count": 1, "check_window": 5}}], + } + mutate(strategy) + legacy_json = encode_json_document(strategy) + + with pytest.raises(ContractValidationError, match="unsupported"): + build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=legacy_json, + ) diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_golden.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_golden.py new file mode 100644 index 00000000000..baa380d9c6d --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_golden.py @@ -0,0 +1,211 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import hashlib +import json +from pathlib import Path + +from alarm_backends.core.alarm_engine.contract import ( + build_detection_outcome, + build_trigger_strategy_ir_from_legacy_config, +) +from alarm_backends.core.alarm_engine.encoder import decode_json_document, encode_json_document +from alarm_backends.tests.alarm_engine_fixtures import ( + DETECT_RECORDS, + DETECT_STRATEGY, + TRIGGER_POINT, + TRIGGER_STRATEGY, +) + + +FIXTURE_DIR = Path(__file__).parent / "testdata" / "python-v1" +FIXTURE_FILE = FIXTURE_DIR / "detection_outcome_v1.json" +CHECKSUM_FILE = FIXTURE_DIR / "SHA256SUMS" +GO_SEMANTIC_FILE = FIXTURE_DIR / "go_semantic_v1.json" + + +def _legacy_bytes(strategy: dict) -> bytes: + # Deliberately retain whitespace and insertion order: StrategyIR must preserve these exact bytes. + return json.dumps(strategy, ensure_ascii=False, indent=1).encode("utf-8") + + +def _build_detect_strategy_ir(): + strategy = copy.deepcopy(DETECT_STRATEGY) + return build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=2, + legacy_json=_legacy_bytes(strategy), + ) + + +def _build_trigger_strategy_ir(): + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + return build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=_legacy_bytes(strategy), + ) + + +def _anomaly(record_id: str, strategy_id: int, item_id: int, level: int) -> dict: + return { + "anomaly_id": f"{record_id}.{strategy_id}.{item_id}.{level}", + "anomaly_message": "异常测试", + "anomaly_time": "2019-10-10 10:10:00", + "context": {"level": level, "mixed": [1, "2", {"nested": True}]}, + } + + +def build_fixture_set() -> list[dict]: + detect_strategy_ir = _build_detect_strategy_ir() + anomalous_data, normal_data = copy.deepcopy(DETECT_RECORDS) + anomalous_data["values"]["huge_counter"] = 9007199254740993 + normal = build_detection_outcome( + strategy_ir=detect_strategy_ir, + batch_id="detect-batch-1", + data_raw=normal_data, + evaluations=[{"level": 3, "result": "NORMAL"}], + outcome="NORMAL", + ) + anomalous = build_detection_outcome( + strategy_ir=detect_strategy_ir, + batch_id="detect-batch-1", + data_raw=anomalous_data, + evaluations=[ + { + "level": 3, + "result": "ANOMALOUS", + "anomaly": _anomaly(anomalous_data["record_id"], 1, 2, 3), + } + ], + outcome="ANOMALOUS", + ) + retry = build_detection_outcome( + strategy_ir=detect_strategy_ir, + batch_id="detect-batch-retry-2", + data_raw=anomalous_data, + evaluations=[ + { + "level": 3, + "result": "ANOMALOUS", + "anomaly": _anomaly(anomalous_data["record_id"], 1, 2, 3), + } + ], + outcome="ANOMALOUS", + ) + + trigger_strategy_ir = _build_trigger_strategy_ir() + trigger_data = copy.deepcopy(TRIGGER_POINT["data"]) + trigger_data["values"]["huge_counter"] = 9007199254740993 + partial_error = build_detection_outcome( + strategy_ir=trigger_strategy_ir, + batch_id="trigger-batch-1", + data_raw=trigger_data, + evaluations=[{"level": 1, "result": "NORMAL"}], + outcome="ERROR", + error_code="ALGORITHM_ERROR", + ) + unsupported = build_detection_outcome( + strategy_ir=trigger_strategy_ir, + batch_id="trigger-batch-1", + data_raw=trigger_data, + evaluations=[], + outcome="UNSUPPORTED", + error_code="UNSUPPORTED_STRATEGY", + ) + + source_tests = [ + "alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle", + "alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init", + ] + return [ + {"name": "normal", "source_tests": source_tests, "strategy_ir": detect_strategy_ir, "outcome": normal}, + { + "name": "anomalous", + "source_tests": source_tests, + "strategy_ir": detect_strategy_ir, + "outcome": anomalous, + }, + { + "name": "error-partial", + "source_tests": source_tests, + "strategy_ir": trigger_strategy_ir, + "outcome": partial_error, + }, + { + "name": "unsupported-empty", + "source_tests": source_tests, + "strategy_ir": trigger_strategy_ir, + "outcome": unsupported, + }, + { + "name": "retry-same-input", + "source_tests": source_tests, + "strategy_ir": detect_strategy_ir, + "outcome": retry, + }, + ] + + +def test_python_v1_golden_matches_current_legacy_objects_without_overwriting_fixture(): + expected = decode_json_document(FIXTURE_FILE.read_bytes()) + + assert expected["schema_version"] == "detection-outcome/1.0" + assert expected["fixtures"] == build_fixture_set() + + +def test_python_v1_golden_checksum_is_current(): + checksums = { + name: digest + for digest, name in ( + line.split(" ", 1) for line in CHECKSUM_FILE.read_text(encoding="ascii").splitlines() if line + ) + } + + assert hashlib.sha256(FIXTURE_FILE.read_bytes()).hexdigest() == checksums[FIXTURE_FILE.name] + assert hashlib.sha256(GO_SEMANTIC_FILE.read_bytes()).hexdigest() == checksums[GO_SEMANTIC_FILE.name] + + +def test_retry_fixture_reuses_input_id_but_changes_transport_correlation(): + fixtures = {fixture["name"]: fixture for fixture in build_fixture_set()} + anomalous = fixtures["anomalous"]["outcome"] + retry = fixtures["retry-same-input"]["outcome"] + + assert retry["input_id"] == anomalous["input_id"] + assert retry["batch_id"] != anomalous["batch_id"] + + +def test_fixture_set_itself_is_json_encodable_without_precision_loss(): + fixture_set = {"schema_version": "detection-outcome/1.0", "fixtures": build_fixture_set()} + + assert decode_json_document(encode_json_document(fixture_set)) == fixture_set + + +def test_go_semantic_projection_matches_python_contract_documents(): + go_projection = decode_json_document(GO_SEMANTIC_FILE.read_bytes()) + python_projection = { + "schema_version": "detection-outcome/1.0", + "fixtures": [ + { + "name": fixture["name"], + "strategy_ir": fixture["strategy_ir"], + "outcome": fixture["outcome"], + } + for fixture in build_fixture_set() + ], + } + + assert go_projection == python_projection diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_publisher.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_publisher.py new file mode 100644 index 00000000000..c1244a783e9 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_publisher.py @@ -0,0 +1,280 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import json + +import pytest + +from alarm_backends.core.alarm_engine import publisher as publisher_module +from alarm_backends.core.alarm_engine.encoder import decode_json_document, encode_json_document +from alarm_backends.core.alarm_engine.publisher import ( + DetectionPublishError, + KafkaDetectionPublisher, + build_kafka_detection_publisher, +) +from alarm_backends.core.alarm_engine.runtime import prepare_finalized_threshold_batch +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY + + +class FakeProducer: + def __init__(self, *, delivery_error=None, remaining=0): + self.delivery_error = delivery_error + self.remaining = remaining + self.messages = [] + + def produce(self, **message): + self.messages.append(message) + + def flush(self, timeout): + self.flush_timeout = timeout + for message in self.messages: + message["on_delivery"](self.delivery_error, None) + return self.remaining + + +def test_kafka_detection_publisher_waits_for_delivery_and_emits_self_contained_microbatch(): + producer = FakeProducer() + publisher = KafkaDetectionPublisher(producer=producer, topic="alarm-engine-detection-shadow", flush_timeout=4) + batch = _batch() + + published = publisher.publish_batch(batch) + + assert published == 1 + assert producer.flush_timeout == 4 + assert len(producer.messages) == 1 + message = producer.messages[0] + assert message["topic"] == "alarm-engine-detection-shadow" + assert isinstance(message["key"], bytes) + assert message["key"].hex() == "76822eff60b83ab18de1ec5ecf6c194f6e933f12af8b28e199f2a43f8a730c27" + envelope = decode_json_document(message["value"]) + assert envelope == { + "schema": {"name": "trigger-input", "major": 1, "minor": 0}, + "required_features": [], + "partition_hash_version": "trigger-input-partition-v1", + "strategy_ir": batch["strategy_ir"], + "detection_outcomes": batch["outcomes"], + } + + +def test_kafka_detection_publisher_splits_microbatches_by_count(): + producer = FakeProducer() + publisher = KafkaDetectionPublisher( + producer=producer, + topic="alarm-engine-detection-shadow", + flush_timeout=4, + max_outcomes_per_message=1, + max_envelope_bytes=512 * 1024, + ) + batch = _batch(include_normal=True) + + assert publisher.publish_batch(batch) == 2 + assert len(producer.messages) == 2 + assert [len(decode_json_document(message["value"])["detection_outcomes"]) for message in producer.messages] == [ + 1, + 1, + ] + + +def test_kafka_detection_publisher_splits_microbatches_by_encoded_bytes(): + producer = FakeProducer() + batch = _batch(include_normal=True) + single_envelope = { + "schema": {"name": "trigger-input", "major": 1, "minor": 0}, + "required_features": [], + "partition_hash_version": "trigger-input-partition-v1", + "strategy_ir": batch["strategy_ir"], + "detection_outcomes": batch["outcomes"][:1], + } + publisher = KafkaDetectionPublisher( + producer=producer, + topic="alarm-engine-detection-shadow", + flush_timeout=4, + max_outcomes_per_message=500, + max_envelope_bytes=len(encode_json_document(single_envelope)), + ) + + assert publisher.publish_batch(batch) == 2 + assert len(producer.messages) == 2 + + +def test_kafka_detection_publisher_rejects_a_single_oversized_outcome_before_produce(): + producer = FakeProducer() + publisher = KafkaDetectionPublisher( + producer=producer, + topic="alarm-engine-detection-shadow", + flush_timeout=4, + max_outcomes_per_message=500, + max_envelope_bytes=1, + ) + + with pytest.raises(DetectionPublishError, match="exceeds"): + publisher.publish_batch(_batch()) + + assert producer.messages == [] + + +@pytest.mark.parametrize( + ("mutate", "error"), + [ + (lambda batch: batch["outcomes"][1].__setitem__("batch_id", "another-batch"), "share one batch_id"), + ( + lambda batch: batch["outcomes"].__setitem__(1, copy.deepcopy(batch["outcomes"][0])), + "duplicate input_id", + ), + ], +) +def test_kafka_detection_publisher_rejects_microbatch_identity_contradictions_before_produce(mutate, error): + producer = FakeProducer() + publisher = KafkaDetectionPublisher(producer=producer, topic="alarm-engine-detection-shadow", flush_timeout=4) + batch = _batch(include_normal=True) + mutate(batch) + + with pytest.raises(DetectionPublishError, match=error): + publisher.publish_batch(batch) + + assert producer.messages == [] + + +@pytest.mark.parametrize( + "kwargs", + [ + {"max_outcomes_per_message": 501}, + {"max_envelope_bytes": 512 * 1024 + 1}, + ], +) +def test_kafka_detection_publisher_rejects_limits_above_wire_v1(kwargs): + with pytest.raises(ValueError, match="must be between"): + KafkaDetectionPublisher( + producer=FakeProducer(), + topic="alarm-engine-detection-shadow", + flush_timeout=4, + **kwargs, + ) + + +@pytest.mark.parametrize( + ("producer", "error"), + [ + (FakeProducer(delivery_error=RuntimeError("broker rejected")), "broker rejected"), + (FakeProducer(remaining=1), "flush timeout"), + ], +) +def test_kafka_detection_publisher_fails_when_broker_does_not_ack(producer, error): + publisher = KafkaDetectionPublisher(producer=producer, topic="alarm-engine-detection-shadow", flush_timeout=4) + + with pytest.raises(DetectionPublishError, match=error): + publisher.publish_batch(_batch()) + + +def test_build_kafka_detection_publisher_enables_idempotence_and_bounded_delivery(): + producer = FakeProducer() + captured = {} + + def producer_factory(config): + captured.update(config) + return producer + + publisher = build_kafka_detection_publisher( + { + "topic": "alarm-engine-detection-shadow", + "bootstrap.servers": "kafka:9092", + "message.timeout.ms": 2500, + }, + producer_factory=producer_factory, + allowed_topics={"alarm-engine-detection-shadow"}, + ) + + assert publisher.producer is producer + assert publisher.topic == "alarm-engine-detection-shadow" + assert publisher.flush_timeout == 3.5 + assert captured == { + "bootstrap.servers": "kafka:9092", + "message.timeout.ms": 2500, + "enable.idempotence": True, + } + + +def test_build_kafka_detection_publisher_rejects_production_topic(): + with pytest.raises(ValueError, match="not in the Shadow allowlist"): + build_kafka_detection_publisher( + {"topic": "monitor-event", "bootstrap.servers": "kafka:9092"}, + producer_factory=lambda _config: FakeProducer(), + allowed_topics={"alarm-engine-detection-shadow"}, + ) + + +def test_build_kafka_detection_publisher_rejects_disabled_idempotence(): + with pytest.raises(ValueError, match="idempotence"): + build_kafka_detection_publisher( + { + "topic": "alarm-engine-detection-shadow", + "bootstrap.servers": "kafka:9092", + "enable.idempotence": False, + }, + producer_factory=lambda _config: FakeProducer(), + allowed_topics={"alarm-engine-detection-shadow"}, + ) + + +def test_cached_kafka_detection_publisher_reuses_process_producer(monkeypatch): + publisher_module.get_cached_kafka_detection_publisher.cache_clear() + expected = object() + calls = [] + + def build(config, *, allowed_topics): + calls.append((config, allowed_topics)) + return expected + + monkeypatch.setattr(publisher_module, "build_kafka_detection_publisher", build) + config_json = json.dumps( + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + sort_keys=True, + separators=(",", ":"), + ) + + first = publisher_module.get_cached_kafka_detection_publisher(config_json, ("alarm-engine-detection-shadow",)) + second = publisher_module.get_cached_kafka_detection_publisher(config_json, ("alarm-engine-detection-shadow",)) + + assert first is expected + assert second is expected + assert calls == [ + ( + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + {"alarm-engine-detection-shadow"}, + ) + ] + publisher_module.get_cached_kafka_detection_publisher.cache_clear() + + +def _batch(*, include_normal=False): + strategy = copy.deepcopy(DETECT_STRATEGY) + records = copy.deepcopy(DETECT_RECORDS if include_normal else DETECT_RECORDS[:1]) + record = records[0] + return prepare_finalized_threshold_batch( + tenant_id="default", + strategy=strategy, + item_id=2, + legacy_json=json.dumps(strategy).encode(), + batch_id="batch-1", + data_points=records, + anomaly_outputs=[ + { + "data": record, + "anomaly": { + "3": { + "anomaly_id": f"{record['record_id']}.1.2.3", + "anomaly_message": "threshold matched", + } + }, + } + ], + finalized=True, + ) diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference.py new file mode 100644 index 00000000000..7684ae468e0 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference.py @@ -0,0 +1,405 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from alarm_backends.core.alarm_engine.contract import ( + ContractValidationError, + build_detection_outcome, + build_trigger_strategy_ir_from_legacy_config, +) +from alarm_backends.core.alarm_engine.encoder import decode_json_document +from alarm_backends.core.alarm_engine.reference import ( + build_reference_trigger_decision_batch, + build_reference_trigger_decision_candidate, + build_terminal_reference_decision_batches, + is_alarm_engine_shadow_strategy_selected, +) +from alarm_backends.core.alarm_engine.runtime import prepare_finalized_threshold_batch +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY, TRIGGER_POINT, TRIGGER_STRATEGY + + +def legacy_bytes(strategy): + return json.dumps(strategy, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + +GOLDEN_FILE = Path(__file__).parent / "testdata" / "python-v1" / "trigger_decision_v1.json" +GOLDEN_SHA256 = "982bfd06f5cf3d98fc2f7c965fafd854346601b1a47d74e86e4c7195a1f93f21" + + +def build_reference(*, point, strategy, event_record, raw=None, tenant_id="default"): + raw = raw or legacy_bytes(strategy) + return build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=raw, + strategy_snapshot_key=point["strategy_snapshot_key"], + tenant_id_resolver=lambda _bk_biz_id: tenant_id, + expected_input_id=detect_input_id(point=point, strategy=strategy, raw=raw), + item_id=1, + point=point, + event_record=event_record, + ) + + +def triggered_event(point): + source_time = point["data"]["time"] + return { + "data": copy.deepcopy(point["data"]), + "anomaly": copy.deepcopy(point["anomaly"]), + "strategy_snapshot_key": point["strategy_snapshot_key"], + "trigger": { + "level": "3", + "anomaly_ids": [f"55a76cf628e46c04a052f4e19bdb9dbf.{source_time}.1.1.3"], + }, + } + + +def detect_input_id(*, point, strategy, raw): + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=raw, + ) + evaluations = [ + {"level": level, "result": "ANOMALOUS", "anomaly": copy.deepcopy(point["anomaly"][str(level)])} + for level in strategy_ir["required_levels"] + ] + return build_detection_outcome( + strategy_ir=strategy_ir, + batch_id="authoritative-detect-batch", + data_raw=point["data"], + evaluations=evaluations, + outcome="ANOMALOUS", + )["input_id"] + + +def test_reference_rebuilds_same_input_id_from_exact_snapshot_for_no_trigger(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + raw = legacy_bytes(strategy) + + first = build_reference(point=point, strategy=strategy, raw=raw, event_record=None) + retry = build_reference(point=point, strategy=strategy, raw=raw, event_record=None) + + assert first == retry + assert first["decisions"][0]["outcome"] == "NO_TRIGGER" + assert first["decisions"][0]["reason_code"] == "TRIGGER_CONDITION_NOT_MET" + + +def test_reference_projects_real_checker_trigger_shape(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + source_time = point["data"]["time"] + batch = build_reference(point=point, strategy=strategy, event_record=triggered_event(point)) + + assert batch["decisions"][0]["outcome"] == "TRIGGER" + assert batch["decisions"][0]["reason_code"] == "TRIGGER_CONDITION_MET" + assert batch["decisions"][0]["level"] == 3 + assert batch["decisions"][0]["anomaly_timestamps"] == [source_time] + + +def test_python_reference_decision_golden_is_current(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + expected = { + "schema_version": "trigger-decision-batch/1.0", + "fixtures": [ + { + "name": "python-trigger-reference", + "batch": build_reference(point=point, strategy=strategy, event_record=triggered_event(point)), + } + ], + } + + payload = GOLDEN_FILE.read_bytes() + assert hashlib.sha256(payload).hexdigest() == GOLDEN_SHA256 + assert decode_json_document(payload) == expected + + +def test_reference_uses_exact_snapshot_bytes_and_fails_closed_on_identity_drift(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + compact = legacy_bytes(strategy) + spaced = json.dumps(strategy, ensure_ascii=False, indent=1).encode("utf-8") + + compact_batch = build_reference(point=point, strategy=strategy, raw=compact, event_record=None) + spaced_batch = build_reference(point=point, strategy=strategy, raw=spaced, event_record=None) + assert compact_batch["decisions"][0]["input_id"] != spaced_batch["decisions"][0]["input_id"] + + point["data"]["time"] += 1 + with pytest.raises(ContractValidationError): + build_reference(point=point, strategy=strategy, raw=compact, event_record=None) + + +def test_reference_rejects_null_anomaly_instead_of_treating_it_as_normal(): + point = copy.deepcopy(TRIGGER_POINT) + point["anomaly"]["1"] = None + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + + with pytest.raises(ContractValidationError, match="object"): + build_reference(point=point, strategy=strategy, event_record=None) + + +def test_reference_input_id_matches_authoritative_detect_projection(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + raw = legacy_bytes(strategy) + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=raw, + ) + evaluations = [ + {"level": level, "result": "ANOMALOUS", "anomaly": copy.deepcopy(point["anomaly"][str(level)])} + for level in strategy_ir["required_levels"] + ] + detect = build_detection_outcome( + strategy_ir=strategy_ir, + batch_id="authoritative-detect-batch", + data_raw=point["data"], + evaluations=evaluations, + outcome="ANOMALOUS", + ) + + reference = build_reference(point=point, strategy=strategy, raw=raw, event_record=None) + + assert reference["decisions"][0]["input_id"] == detect["input_id"] + + +def test_reference_rejects_stale_snapshot_and_event_identity(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + with pytest.raises(ContractValidationError, match="exact strategy snapshot"): + build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=legacy_bytes(strategy), + strategy_snapshot_key="another-snapshot", + tenant_id_resolver=lambda _bk_biz_id: "default", + expected_input_id=detect_input_id(point=point, strategy=strategy, raw=legacy_bytes(strategy)), + item_id=1, + point=point, + event_record=None, + ) + + stale_event = triggered_event(point) + stale_event["data"]["value"] = 999 + with pytest.raises(ContractValidationError, match="event_record data"): + build_reference(point=point, strategy=strategy, event_record=stale_event) + + +@pytest.mark.parametrize("snapshot_key", [None, "", b"snapshot", "\ud800"]) +def test_reference_rejects_missing_or_invalid_snapshot_identity(snapshot_key): + point = copy.deepcopy(TRIGGER_POINT) + point["strategy_snapshot_key"] = snapshot_key + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + + with pytest.raises(ContractValidationError, match="snapshot key"): + build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=legacy_bytes(strategy), + strategy_snapshot_key=snapshot_key, + tenant_id_resolver=lambda _bk_biz_id: "default", + expected_input_id="0" * 64, + item_id=1, + point=point, + event_record=None, + ) + + +def test_reference_rejects_tenant_mapping_drift_from_acknowledged_detect_input(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + + with pytest.raises(ContractValidationError, match="acknowledged Detect input"): + build_reference(point=point, strategy=strategy, event_record=None, tenant_id="tenant-b") + + +def test_unconfirmed_reference_candidate_preserves_the_same_trigger_identity(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + strategy["update_time"] = 1569246480 + raw = legacy_bytes(strategy) + + candidate = build_reference_trigger_decision_candidate( + strategy=strategy, + legacy_json=raw, + strategy_snapshot_key=point["strategy_snapshot_key"], + tenant_id_resolver=lambda _bk_biz_id: "default", + item_id=1, + point=point, + event_record=triggered_event(point), + ) + acknowledged = build_reference( + point=point, + strategy=strategy, + event_record=triggered_event(point), + raw=raw, + ) + + assert candidate == acknowledged + + +@pytest.mark.parametrize("selector", [(True,), (1.9,), ("01",), (" 1 ",), "1,", None]) +def test_alarm_engine_shadow_strategy_selector_rejects_noncanonical_values(selector): + assert not is_alarm_engine_shadow_strategy_selected(selector, 1) + + +@pytest.mark.parametrize("selector", [(1,), ("1",), "1", ("2", 1)]) +def test_alarm_engine_shadow_strategy_selector_accepts_exact_positive_ids(selector): + assert is_alarm_engine_shadow_strategy_selected(selector, 1) + + +def test_terminal_reference_projects_only_detection_terminal_outcomes_after_ack(): + strategy = copy.deepcopy(DETECT_STRATEGY) + detection = prepare_finalized_threshold_batch( + tenant_id="default", + strategy=strategy, + item_id=2, + legacy_json=legacy_bytes(strategy), + batch_id="detect-batch", + data_points=copy.deepcopy(DETECT_RECORDS), + anomaly_outputs=[ + { + "data": copy.deepcopy(DETECT_RECORDS[0]), + "anomaly": { + "3": { + "anomaly_id": f"{DETECT_RECORDS[0]['record_id']}.1.2.3", + "anomaly_message": "threshold matched", + } + }, + } + ], + finalized=True, + ) + + reference_batches = build_terminal_reference_decision_batches( + strategy_ir=detection["strategy_ir"], + detection_outcomes=detection["outcomes"], + ) + reference = reference_batches[0] + + assert [decision["input_id"] for decision in reference["decisions"]] == [detection["outcomes"][1]["input_id"]] + assert reference["decisions"][0]["outcome"] == "NO_TRIGGER" + assert reference["decisions"][0]["reason_code"] == "INPUT_NORMAL" + + +@pytest.mark.parametrize( + ("outcome", "error_code"), + [ + ("ERROR", "ALGORITHM_ERROR"), + ("UNSUPPORTED", "UNSUPPORTED_STRATEGY"), + ], +) +def test_terminal_reference_preserves_detection_error_terminal(outcome, error_code): + strategy = copy.deepcopy(DETECT_STRATEGY) + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=2, + legacy_json=legacy_bytes(strategy), + ) + source = build_detection_outcome( + strategy_ir=strategy_ir, + batch_id="detect-batch", + data_raw=copy.deepcopy(DETECT_RECORDS[0]), + evaluations=[], + outcome=outcome, + error_code=error_code, + ) + + reference_batches = build_terminal_reference_decision_batches( + strategy_ir=strategy_ir, + detection_outcomes=[source], + ) + reference = reference_batches[0] + + assert reference["decisions"][0]["outcome"] == outcome + assert reference["decisions"][0]["reason_code"] == error_code + + +def test_terminal_reference_returns_none_when_all_inputs_require_real_trigger(): + strategy = copy.deepcopy(DETECT_STRATEGY) + detection = prepare_finalized_threshold_batch( + tenant_id="default", + strategy=strategy, + item_id=2, + legacy_json=legacy_bytes(strategy), + batch_id="detect-batch", + data_points=[copy.deepcopy(DETECT_RECORDS[0])], + anomaly_outputs=[ + { + "data": copy.deepcopy(DETECT_RECORDS[0]), + "anomaly": {"3": {"anomaly_id": f"{DETECT_RECORDS[0]['record_id']}.1.2.3"}}, + } + ], + finalized=True, + ) + + assert ( + build_terminal_reference_decision_batches( + strategy_ir=detection["strategy_ir"], + detection_outcomes=detection["outcomes"], + ) + == [] + ) + + +@pytest.mark.parametrize(("outcome_count", "expected_chunk_sizes"), [(500, [500]), (501, [500, 1])]) +def test_terminal_reference_chunks_at_the_wire_outcome_limit(outcome_count, expected_chunk_sizes): + strategy = copy.deepcopy(DETECT_STRATEGY) + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=2, + legacy_json=legacy_bytes(strategy), + ) + template = copy.deepcopy(DETECT_RECORDS[1]) + outcomes = [] + for index in range(outcome_count): + record = copy.deepcopy(template) + record["record_id"] = f"{index:032x}.{index + 1}" + record["time"] = index + 1 + record["values"]["timestamp"] = index + 1 + outcomes.append( + build_detection_outcome( + strategy_ir=strategy_ir, + batch_id="detect-batch", + data_raw=record, + evaluations=[{"level": level, "result": "NORMAL"} for level in strategy_ir["required_levels"]], + outcome="NORMAL", + ) + ) + + batches = build_terminal_reference_decision_batches( + strategy_ir=strategy_ir, + detection_outcomes=outcomes, + ) + + assert [len(batch["decisions"]) for batch in batches] == expected_chunk_sizes diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference_publisher.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference_publisher.py new file mode 100644 index 00000000000..8776b479dbb --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_reference_publisher.py @@ -0,0 +1,222 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import json + +import pytest + +from alarm_backends.core.alarm_engine.encoder import decode_trigger_decision_batch +from alarm_backends.core.alarm_engine.reference import build_terminal_reference_decision_batches +from alarm_backends.core.alarm_engine.reference_publisher import ( + KafkaReferenceDecisionPublisher, + ReferenceDecisionPublishError, + build_kafka_reference_decision_publisher, +) +from alarm_backends.core.alarm_engine.runtime import prepare_finalized_threshold_batch +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY + + +class FakeProducer: + def __init__(self, *, delivery_error=None, remaining=0): + self.delivery_error = delivery_error + self.remaining = remaining + self.messages = [] + self.flush_calls = 0 + self.pending_payload_bytes = 0 + self.flushed_payload_bytes = [] + + def produce(self, **message): + self.messages.append(message) + self.pending_payload_bytes += len(message["value"]) + + def flush(self, timeout): + self.flush_calls += 1 + self.flush_timeout = timeout + self.flushed_payload_bytes.append(self.pending_payload_bytes) + self.pending_payload_bytes = 0 + for message in self.messages: + message["on_delivery"](self.delivery_error, None) + return self.remaining + + +def test_reference_publisher_uses_official_codec_partition_key_and_broker_ack(): + producer = FakeProducer() + publisher = KafkaReferenceDecisionPublisher( + producer=producer, + topic="alarm-engine-reference-shadow", + flush_timeout=4, + ) + batch = _normal_reference_batch() + + assert publisher.publish_batch(batch) == 1 + assert producer.flush_timeout == 4 + assert len(producer.messages) == 1 + message = producer.messages[0] + assert message["topic"] == "alarm-engine-reference-shadow" + assert message["key"].hex() == "76822eff60b83ab18de1ec5ecf6c194f6e933f12af8b28e199f2a43f8a730c27" + assert decode_trigger_decision_batch(message["value"]) == batch + + +def test_reference_publisher_batches_share_one_broker_ack_barrier(): + producer = FakeProducer() + publisher = KafkaReferenceDecisionPublisher( + producer=producer, + topic="alarm-engine-reference-shadow", + flush_timeout=4, + ) + first = _normal_reference_batch() + second = copy.deepcopy(first) + second["batch_id"] = "batch-2" + + assert publisher.publish_batches([first, second]) == 2 + assert len(producer.messages) == 2 + assert producer.flush_calls == 1 + + +def test_reference_publisher_bounds_each_ack_group_by_encoded_bytes(): + producer = FakeProducer() + publisher = KafkaReferenceDecisionPublisher( + producer=producer, + topic="alarm-engine-reference-shadow", + flush_timeout=4, + ) + first = _normal_reference_batch() + second = copy.deepcopy(first) + second["batch_id"] = "batch-2" + for batch in (first, second): + batch["schema"]["minor"] = 1 + batch["padding"] = "x" * (300 * 1024) + + assert publisher.publish_batches([first, second]) == 2 + assert producer.flush_calls == 2 + assert all(payload_bytes <= 512 * 1024 for payload_bytes in producer.flushed_payload_bytes) + + +def test_reference_publisher_does_not_consume_past_one_size_lookahead_after_ack_failure(): + producer = FakeProducer(remaining=1) + publisher = KafkaReferenceDecisionPublisher( + producer=producer, + topic="alarm-engine-reference-shadow", + flush_timeout=4, + ) + batches = [] + for index in range(3): + batch = _normal_reference_batch() + batch["batch_id"] = f"batch-{index}" + batch["schema"]["minor"] = 1 + batch["padding"] = "x" * (300 * 1024) + batches.append(batch) + consumed = [] + + def iter_batches(): + for batch in batches: + consumed.append(batch["batch_id"]) + yield batch + + with pytest.raises(ReferenceDecisionPublishError, match="flush timeout"): + publisher.publish_batches(iter_batches()) + + assert consumed == ["batch-0", "batch-1"] + assert producer.flush_calls == 1 + + +@pytest.mark.parametrize( + ("producer", "error"), + [ + (FakeProducer(delivery_error=RuntimeError("broker rejected")), "broker rejected"), + (FakeProducer(remaining=1), "flush timeout"), + ], +) +def test_reference_publisher_requires_broker_ack(producer, error): + publisher = KafkaReferenceDecisionPublisher( + producer=producer, + topic="alarm-engine-reference-shadow", + flush_timeout=4, + ) + + with pytest.raises(ReferenceDecisionPublishError, match=error): + publisher.publish_batch(_normal_reference_batch()) + + +def test_reference_publisher_config_is_fail_closed_and_idempotent(): + producer = FakeProducer() + captured = {} + + def producer_factory(config): + captured.update(config) + return producer + + publisher = build_kafka_reference_decision_publisher( + { + "topic": "alarm-engine-reference-shadow", + "bootstrap.servers": "kafka:9092", + "message.timeout.ms": 2500, + }, + producer_factory=producer_factory, + allowed_topics={"alarm-engine-reference-shadow"}, + ) + + assert publisher.producer is producer + assert publisher.flush_timeout == 3.5 + assert captured == { + "bootstrap.servers": "kafka:9092", + "message.timeout.ms": 2500, + "enable.idempotence": True, + } + + +@pytest.mark.parametrize( + "config", + [ + {"topic": "monitor-event", "bootstrap.servers": "kafka:9092"}, + { + "topic": "alarm-engine-reference-shadow", + "bootstrap.servers": "kafka:9092", + "enable.idempotence": False, + }, + ], +) +def test_reference_publisher_rejects_unsafe_config(config): + with pytest.raises(ValueError): + build_kafka_reference_decision_publisher( + config, + producer_factory=lambda _config: FakeProducer(), + allowed_topics={"alarm-engine-reference-shadow"}, + ) + + +@pytest.mark.parametrize("forbidden_topic", ["alarm-engine-detection-shadow", "monitor-event-nondefault"]) +def test_reference_publisher_rejects_topics_that_are_explicitly_forbidden(forbidden_topic): + with pytest.raises(ValueError, match="allowlist must not contain forbidden"): + build_kafka_reference_decision_publisher( + {"topic": forbidden_topic, "bootstrap.servers": "kafka:9092"}, + producer_factory=lambda _config: FakeProducer(), + allowed_topics={forbidden_topic}, + forbidden_topics={forbidden_topic}, + ) + + +def _normal_reference_batch(): + strategy = copy.deepcopy(DETECT_STRATEGY) + detection = prepare_finalized_threshold_batch( + tenant_id="default", + strategy=strategy, + item_id=2, + legacy_json=json.dumps(strategy).encode(), + batch_id="batch-1", + data_points=[copy.deepcopy(DETECT_RECORDS[1])], + anomaly_outputs=[], + finalized=True, + ) + return build_terminal_reference_decision_batches( + strategy_ir=detection["strategy_ir"], + detection_outcomes=detection["outcomes"], + )[0] diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/test_runtime.py b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_runtime.py new file mode 100644 index 00000000000..0881255d225 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/test_runtime.py @@ -0,0 +1,231 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import json + +import pytest + +from alarm_backends.core.alarm_engine.contract import ( + ContractValidationError, + build_trigger_strategy_ir, + derive_input_id, +) +from alarm_backends.core.alarm_engine.runtime import ( + DetectionNotFinalized, + prepare_finalized_threshold_batch, + project_detection_outcomes, +) +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY + + +def test_project_detection_outcomes_covers_every_record_and_required_level(): + strategy_ir = _strategy_ir() + records = [ + { + "record_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.100", + "time": 100, + "value": 99, + "values": {"timestamp": 100, "metric": 99}, + "dimensions": {"host": "host-1"}, + }, + { + "record_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.110", + "time": 110, + "value": 10, + "values": {"timestamp": 110, "metric": 10}, + "dimensions": {"host": "host-2"}, + }, + ] + anomaly_outputs = [ + { + "data": records[0], + "anomaly": { + "2": { + "anomaly_id": f"{records[0]['record_id']}.1.2.2", + "anomaly_message": "level 2 anomaly", + }, + "3": { + "anomaly_id": f"{records[0]['record_id']}.1.2.3", + "anomaly_message": "level 3 anomaly", + "context": {"counter": 9007199254740993}, + }, + }, + } + ] + original_records = copy.deepcopy(records) + original_outputs = copy.deepcopy(anomaly_outputs) + + outcomes = project_detection_outcomes( + strategy_ir=strategy_ir, + batch_id="batch-1", + data_points=records, + anomaly_outputs=anomaly_outputs, + ) + + assert [outcome["record"]["record_id"] for outcome in outcomes] == [ + records[0]["record_id"], + records[1]["record_id"], + ] + assert [outcome["outcome"] for outcome in outcomes] == ["ANOMALOUS", "NORMAL"] + assert outcomes[0]["evaluations"] == [ + {"level": 1, "result": "NORMAL"}, + {"level": 2, "result": "ANOMALOUS", "anomaly": anomaly_outputs[0]["anomaly"]["2"]}, + {"level": 3, "result": "ANOMALOUS", "anomaly": anomaly_outputs[0]["anomaly"]["3"]}, + ] + assert outcomes[1]["evaluations"] == [ + {"level": 1, "result": "NORMAL"}, + {"level": 2, "result": "NORMAL"}, + {"level": 3, "result": "NORMAL"}, + ] + assert outcomes[0]["record"]["data_raw"] == records[0] + assert outcomes[0]["record"]["data_raw"] is not records[0] + assert outcomes[0]["input_id"] == derive_input_id( + tenant_id="default", + purpose="DETECT", + strategy_id="1", + item_id="2", + strategy_content_sha256=strategy_ir["strategy_ref"]["content_sha256"], + record_id=records[0]["record_id"], + ) + assert records == original_records + assert anomaly_outputs == original_outputs + + +def test_project_detection_outcomes_rejects_anomaly_for_unaccepted_record(): + strategy_ir = _strategy_ir() + records = [{"record_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.100", "time": 100}] + unexpected_record_id = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.110" + + with pytest.raises(ValueError, match="unaccepted record"): + project_detection_outcomes( + strategy_ir=strategy_ir, + batch_id="batch-1", + data_points=records, + anomaly_outputs=[ + { + "data": {"record_id": unexpected_record_id, "time": 110}, + "anomaly": {"1": {"anomaly_id": f"{unexpected_record_id}.1.2.1"}}, + } + ], + ) + + +@pytest.mark.parametrize("anomalies", [{}, {"1": None}, {"1": "not-an-object"}]) +def test_project_detection_outcomes_rejects_invalid_anomaly_payload(anomalies): + record = {"record_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.100", "time": 100} + + with pytest.raises(ContractValidationError, match="anomaly"): + project_detection_outcomes( + strategy_ir=_strategy_ir(), + batch_id="batch-1", + data_points=[record], + anomaly_outputs=[{"data": record, "anomaly": anomalies}], + ) + + +def test_project_detection_outcomes_rejects_output_data_drift(): + record = {"record_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.100", "time": 100, "value": 1} + drifted_record = {**record, "value": 999} + + with pytest.raises(ContractValidationError, match="does not match accepted data"): + project_detection_outcomes( + strategy_ir=_strategy_ir(), + batch_id="batch-1", + data_points=[record], + anomaly_outputs=[ + { + "data": drifted_record, + "anomaly": {"1": {"anomaly_id": f"{record['record_id']}.1.2.1"}}, + } + ], + ) + + +def test_project_detection_outcomes_rejects_non_string_level_key(): + record = {"record_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.100", "time": 100} + + with pytest.raises(ContractValidationError, match="level keys must be strings"): + project_detection_outcomes( + strategy_ir=_strategy_ir(), + batch_id="batch-1", + data_points=[record], + anomaly_outputs=[ + { + "data": record, + "anomaly": { + "1": {"anomaly_id": f"{record['record_id']}.1.2.1"}, + 4: {"anomaly_id": f"{record['record_id']}.1.2.4"}, + }, + } + ], + ) + + +def test_prepare_finalized_threshold_batch_uses_exact_strategy_snapshot(): + legacy_json = json.dumps(DETECT_STRATEGY).encode() + record = copy.deepcopy(DETECT_RECORDS[0]) + anomaly_output = { + "data": record, + "anomaly": { + "3": { + "anomaly_id": f"{record['record_id']}.1.2.3", + "anomaly_message": "threshold matched", + } + }, + } + + batch = prepare_finalized_threshold_batch( + tenant_id="default", + strategy=DETECT_STRATEGY, + item_id=2, + legacy_json=legacy_json, + batch_id="batch-1", + data_points=[record], + anomaly_outputs=[anomaly_output], + finalized=True, + ) + + assert batch["strategy_ir"]["legacy_json_b64"] + assert batch["strategy_ir"]["strategy_ref"]["generation"] == str(DETECT_STRATEGY["update_time"]) + assert batch["outcomes"][0]["outcome"] == "ANOMALOUS" + + +@pytest.mark.parametrize("finalized", [False, None, 0, 1, "false", object()]) +def test_prepare_finalized_threshold_batch_does_not_infer_normal_before_finalization(finalized): + with pytest.raises(DetectionNotFinalized): + prepare_finalized_threshold_batch( + tenant_id="default", + strategy=DETECT_STRATEGY, + item_id=2, + legacy_json=json.dumps(DETECT_STRATEGY).encode(), + batch_id="batch-1", + data_points=[copy.deepcopy(DETECT_RECORDS[1])], + anomaly_outputs=[], + finalized=finalized, + ) + + +def _strategy_ir(): + legacy_json = json.dumps({"id": 1, "items": [{"id": 2}]}, separators=(",", ":")).encode() + return build_trigger_strategy_ir( + tenant_id="default", + purpose="DETECT", + strategy_id=1, + item_id=2, + generation="1", + legacy_json=legacy_json, + check_window_unit_seconds=10, + trigger_configs={ + 1: {"check_window_size": 3, "trigger_count": 2}, + 2: {"check_window_size": 3, "trigger_count": 2}, + 3: {"check_window_size": 3, "trigger_count": 2}, + }, + ) diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/SHA256SUMS b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/SHA256SUMS new file mode 100644 index 00000000000..d20c9593a43 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/SHA256SUMS @@ -0,0 +1,2 @@ +02e912c8560de3c4db5b59507e845cbb4825dc39ce73977b08038236488c3baf detection_outcome_v1.json +e1482a587af185a74760114c21366a62cc7dca6615abcbf3a6ab2c6431c1af4d go_semantic_v1.json diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/detection_outcome_v1.json b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/detection_outcome_v1.json new file mode 100644 index 00000000000..8c51ea31533 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/detection_outcome_v1.json @@ -0,0 +1 @@ +{"fixtures":[{"name":"normal","outcome":{"batch_id":"detect-batch-1","evaluations":[{"level":3,"result":"NORMAL"}],"input_id":"b30a342620b3a762540e78596b8e79c2d8797d7580e9811ab8475eaf24b0610e","outcome":"NORMAL","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"2a1850513fa6018c435f9b6359b3fa7d.1569246481","time":1569246481,"value":50.1,"values":{"load5":50.1,"timestamp":1569246481}},"dimensions_md5":"2a1850513fa6018c435f9b6359b3fa7d","record_id":"2a1850513fa6018c435f9b6359b3fa7d.1569246481","source_time":1569246481},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"source_tests":["alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle","alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init"],"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"anomalous","outcome":{"batch_id":"detect-batch-1","evaluations":[{"anomaly":{"anomaly_id":"342a08e0f85f169a7e099c18db3708ed.1569246480.1.2.3","anomaly_message":"异常测试","anomaly_time":"2019-10-10 10:10:00","context":{"level":3,"mixed":[1,"2",{"nested":true}]}},"level":3,"result":"ANOMALOUS"}],"input_id":"c4e2b55205fa63110d8e5010440faeee1db9a9cac724044d51fae2b23d688b16","outcome":"ANOMALOUS","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"127.0.0.1"},"record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","time":1569246480,"value":99,"values":{"huge_counter":9007199254740993,"load5":99,"timestamp":1569246480}},"dimensions_md5":"342a08e0f85f169a7e099c18db3708ed","record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"source_tests":["alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle","alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init"],"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"error-partial","outcome":{"batch_id":"trigger-batch-1","error_code":"ALGORITHM_ERROR","evaluations":[{"level":1,"result":"NORMAL"}],"input_id":"6f5bf6ba2cbb0f086d0f3b63034fcabcb654c579161570122557dc6f385ac8ac","outcome":"ERROR","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","time":1569246480,"value":1.38,"values":{"huge_counter":9007199254740993,"load5":1.38,"timestamp":1569246480}},"dimensions_md5":"55a76cf628e46c04a052f4e19bdb9dbf","record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default"},"source_tests":["alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle","alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init"],"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAidGFyZ2V0IjogWwogICAgWwogICAgIHsKICAgICAgImZpZWxkIjogImlwIiwKICAgICAgIm1ldGhvZCI6ICJlcSIsCiAgICAgICJ2YWx1ZSI6IFsKICAgICAgIHsKICAgICAgICAiaXAiOiAiMTI3LjAuMC4xIiwKICAgICAgICAiYmtfY2xvdWRfaWQiOiAwLAogICAgICAgICJia19zdXBwbGllcl9pZCI6IDAKICAgICAgIH0KICAgICAgXQogICAgIH0KICAgIF0KICAgXSwKICAgImFsZ29yaXRobXMiOiBbCiAgICB7CiAgICAgImNvbmZpZyI6IFsKICAgICAgewogICAgICAgInRocmVzaG9sZCI6IDAuMSwKICAgICAgICJtZXRob2QiOiAiZ3RlIgogICAgICB9CiAgICAgXSwKICAgICAibGV2ZWwiOiAxLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMQogICAgfSwKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICB7CiAgICAgICAidGhyZXNob2xkIjogMC4xLAogICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgIH0KICAgICBdLAogICAgICJsZXZlbCI6IDIsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIHsKICAgICAgICJ0aHJlc2hvbGQiOiAwLjEsCiAgICAgICAibWV0aG9kIjogImd0ZSIKICAgICAgfQogICAgIF0sCiAgICAgImxldmVsIjogMywKICAgICAidHlwZSI6ICJUaHJlc2hvbGQiLAogICAgICJpZCI6IDMKICAgIH0KICAgXSwKICAgIm5vX2RhdGFfY29uZmlnIjogewogICAgImlzX2VuYWJsZWQiOiBmYWxzZSwKICAgICJjb250aW51b3VzIjogNQogICB9LAogICAiaWQiOiAxLAogICAibmFtZSI6ICLnqbrpl7LnjociCiAgfQogXSwKICJkZXRlY3RzIjogWwogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAxLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAzLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAyLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAyLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAzLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAxLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9CiBdLAogInNjZW5hcmlvIjogIm9zIiwKICJhY3Rpb25zIjogWwogIHsKICAgIm5vdGljZV90ZW1wbGF0ZSI6IHsKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIsCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwCn0=","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[1,2,3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":1,"trigger_count":3},{"check_window_size":5,"level":2,"trigger_count":2},{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"unsupported-empty","outcome":{"batch_id":"trigger-batch-1","error_code":"UNSUPPORTED_STRATEGY","evaluations":[],"input_id":"6f5bf6ba2cbb0f086d0f3b63034fcabcb654c579161570122557dc6f385ac8ac","outcome":"UNSUPPORTED","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","time":1569246480,"value":1.38,"values":{"huge_counter":9007199254740993,"load5":1.38,"timestamp":1569246480}},"dimensions_md5":"55a76cf628e46c04a052f4e19bdb9dbf","record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default"},"source_tests":["alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle","alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init"],"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAidGFyZ2V0IjogWwogICAgWwogICAgIHsKICAgICAgImZpZWxkIjogImlwIiwKICAgICAgIm1ldGhvZCI6ICJlcSIsCiAgICAgICJ2YWx1ZSI6IFsKICAgICAgIHsKICAgICAgICAiaXAiOiAiMTI3LjAuMC4xIiwKICAgICAgICAiYmtfY2xvdWRfaWQiOiAwLAogICAgICAgICJia19zdXBwbGllcl9pZCI6IDAKICAgICAgIH0KICAgICAgXQogICAgIH0KICAgIF0KICAgXSwKICAgImFsZ29yaXRobXMiOiBbCiAgICB7CiAgICAgImNvbmZpZyI6IFsKICAgICAgewogICAgICAgInRocmVzaG9sZCI6IDAuMSwKICAgICAgICJtZXRob2QiOiAiZ3RlIgogICAgICB9CiAgICAgXSwKICAgICAibGV2ZWwiOiAxLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMQogICAgfSwKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICB7CiAgICAgICAidGhyZXNob2xkIjogMC4xLAogICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgIH0KICAgICBdLAogICAgICJsZXZlbCI6IDIsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIHsKICAgICAgICJ0aHJlc2hvbGQiOiAwLjEsCiAgICAgICAibWV0aG9kIjogImd0ZSIKICAgICAgfQogICAgIF0sCiAgICAgImxldmVsIjogMywKICAgICAidHlwZSI6ICJUaHJlc2hvbGQiLAogICAgICJpZCI6IDMKICAgIH0KICAgXSwKICAgIm5vX2RhdGFfY29uZmlnIjogewogICAgImlzX2VuYWJsZWQiOiBmYWxzZSwKICAgICJjb250aW51b3VzIjogNQogICB9LAogICAiaWQiOiAxLAogICAibmFtZSI6ICLnqbrpl7LnjociCiAgfQogXSwKICJkZXRlY3RzIjogWwogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAxLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAzLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAyLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAyLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAzLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAxLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9CiBdLAogInNjZW5hcmlvIjogIm9zIiwKICJhY3Rpb25zIjogWwogIHsKICAgIm5vdGljZV90ZW1wbGF0ZSI6IHsKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIsCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwCn0=","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[1,2,3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":1,"trigger_count":3},{"check_window_size":5,"level":2,"trigger_count":2},{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"retry-same-input","outcome":{"batch_id":"detect-batch-retry-2","evaluations":[{"anomaly":{"anomaly_id":"342a08e0f85f169a7e099c18db3708ed.1569246480.1.2.3","anomaly_message":"异常测试","anomaly_time":"2019-10-10 10:10:00","context":{"level":3,"mixed":[1,"2",{"nested":true}]}},"level":3,"result":"ANOMALOUS"}],"input_id":"c4e2b55205fa63110d8e5010440faeee1db9a9cac724044d51fae2b23d688b16","outcome":"ANOMALOUS","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"127.0.0.1"},"record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","time":1569246480,"value":99,"values":{"huge_counter":9007199254740993,"load5":99,"timestamp":1569246480}},"dimensions_md5":"342a08e0f85f169a7e099c18db3708ed","record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"source_tests":["alarm_backends/tests/service/detect/test_processor.py::TestProcessorViews::test_processor_handle","alarm_backends/tests/service/trigger/test_checker.py::TestChecker::test_init"],"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}}],"schema_version":"detection-outcome/1.0"} diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/go_semantic_v1.json b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/go_semantic_v1.json new file mode 100644 index 00000000000..0a474a1da7f --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/go_semantic_v1.json @@ -0,0 +1 @@ +{"fixtures":[{"name":"normal","outcome":{"batch_id":"detect-batch-1","evaluations":[{"level":3,"result":"NORMAL"}],"input_id":"b30a342620b3a762540e78596b8e79c2d8797d7580e9811ab8475eaf24b0610e","outcome":"NORMAL","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"2a1850513fa6018c435f9b6359b3fa7d.1569246481","time":1569246481,"value":50.1,"values":{"load5":50.1,"timestamp":1569246481}},"dimensions_md5":"2a1850513fa6018c435f9b6359b3fa7d","record_id":"2a1850513fa6018c435f9b6359b3fa7d.1569246481","source_time":1569246481},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"anomalous","outcome":{"batch_id":"detect-batch-1","evaluations":[{"anomaly":{"anomaly_id":"342a08e0f85f169a7e099c18db3708ed.1569246480.1.2.3","anomaly_message":"异常测试","anomaly_time":"2019-10-10 10:10:00","context":{"level":3,"mixed":[1,"2",{"nested":true}]}},"level":3,"result":"ANOMALOUS"}],"input_id":"c4e2b55205fa63110d8e5010440faeee1db9a9cac724044d51fae2b23d688b16","outcome":"ANOMALOUS","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"127.0.0.1"},"record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","time":1569246480,"value":99,"values":{"huge_counter":9007199254740993,"load5":99,"timestamp":1569246480}},"dimensions_md5":"342a08e0f85f169a7e099c18db3708ed","record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"error-partial","outcome":{"batch_id":"trigger-batch-1","error_code":"ALGORITHM_ERROR","evaluations":[{"level":1,"result":"NORMAL"}],"input_id":"6f5bf6ba2cbb0f086d0f3b63034fcabcb654c579161570122557dc6f385ac8ac","outcome":"ERROR","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","time":1569246480,"value":1.38,"values":{"huge_counter":9007199254740993,"load5":1.38,"timestamp":1569246480}},"dimensions_md5":"55a76cf628e46c04a052f4e19bdb9dbf","record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default"},"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAidGFyZ2V0IjogWwogICAgWwogICAgIHsKICAgICAgImZpZWxkIjogImlwIiwKICAgICAgIm1ldGhvZCI6ICJlcSIsCiAgICAgICJ2YWx1ZSI6IFsKICAgICAgIHsKICAgICAgICAiaXAiOiAiMTI3LjAuMC4xIiwKICAgICAgICAiYmtfY2xvdWRfaWQiOiAwLAogICAgICAgICJia19zdXBwbGllcl9pZCI6IDAKICAgICAgIH0KICAgICAgXQogICAgIH0KICAgIF0KICAgXSwKICAgImFsZ29yaXRobXMiOiBbCiAgICB7CiAgICAgImNvbmZpZyI6IFsKICAgICAgewogICAgICAgInRocmVzaG9sZCI6IDAuMSwKICAgICAgICJtZXRob2QiOiAiZ3RlIgogICAgICB9CiAgICAgXSwKICAgICAibGV2ZWwiOiAxLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMQogICAgfSwKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICB7CiAgICAgICAidGhyZXNob2xkIjogMC4xLAogICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgIH0KICAgICBdLAogICAgICJsZXZlbCI6IDIsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIHsKICAgICAgICJ0aHJlc2hvbGQiOiAwLjEsCiAgICAgICAibWV0aG9kIjogImd0ZSIKICAgICAgfQogICAgIF0sCiAgICAgImxldmVsIjogMywKICAgICAidHlwZSI6ICJUaHJlc2hvbGQiLAogICAgICJpZCI6IDMKICAgIH0KICAgXSwKICAgIm5vX2RhdGFfY29uZmlnIjogewogICAgImlzX2VuYWJsZWQiOiBmYWxzZSwKICAgICJjb250aW51b3VzIjogNQogICB9LAogICAiaWQiOiAxLAogICAibmFtZSI6ICLnqbrpl7LnjociCiAgfQogXSwKICJkZXRlY3RzIjogWwogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAxLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAzLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAyLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAyLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAzLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAxLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9CiBdLAogInNjZW5hcmlvIjogIm9zIiwKICJhY3Rpb25zIjogWwogIHsKICAgIm5vdGljZV90ZW1wbGF0ZSI6IHsKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIsCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwCn0=","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[1,2,3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":1,"trigger_count":3},{"check_window_size":5,"level":2,"trigger_count":2},{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"unsupported-empty","outcome":{"batch_id":"trigger-batch-1","error_code":"UNSUPPORTED_STRATEGY","evaluations":[],"input_id":"6f5bf6ba2cbb0f086d0f3b63034fcabcb654c579161570122557dc6f385ac8ac","outcome":"UNSUPPORTED","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"10.0.0.1"},"record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","time":1569246480,"value":1.38,"values":{"huge_counter":9007199254740993,"load5":1.38,"timestamp":1569246480}},"dimensions_md5":"55a76cf628e46c04a052f4e19bdb9dbf","record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default"},"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAidGFyZ2V0IjogWwogICAgWwogICAgIHsKICAgICAgImZpZWxkIjogImlwIiwKICAgICAgIm1ldGhvZCI6ICJlcSIsCiAgICAgICJ2YWx1ZSI6IFsKICAgICAgIHsKICAgICAgICAiaXAiOiAiMTI3LjAuMC4xIiwKICAgICAgICAiYmtfY2xvdWRfaWQiOiAwLAogICAgICAgICJia19zdXBwbGllcl9pZCI6IDAKICAgICAgIH0KICAgICAgXQogICAgIH0KICAgIF0KICAgXSwKICAgImFsZ29yaXRobXMiOiBbCiAgICB7CiAgICAgImNvbmZpZyI6IFsKICAgICAgewogICAgICAgInRocmVzaG9sZCI6IDAuMSwKICAgICAgICJtZXRob2QiOiAiZ3RlIgogICAgICB9CiAgICAgXSwKICAgICAibGV2ZWwiOiAxLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMQogICAgfSwKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICB7CiAgICAgICAidGhyZXNob2xkIjogMC4xLAogICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgIH0KICAgICBdLAogICAgICJsZXZlbCI6IDIsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIHsKICAgICAgICJ0aHJlc2hvbGQiOiAwLjEsCiAgICAgICAibWV0aG9kIjogImd0ZSIKICAgICAgfQogICAgIF0sCiAgICAgImxldmVsIjogMywKICAgICAidHlwZSI6ICJUaHJlc2hvbGQiLAogICAgICJpZCI6IDMKICAgIH0KICAgXSwKICAgIm5vX2RhdGFfY29uZmlnIjogewogICAgImlzX2VuYWJsZWQiOiBmYWxzZSwKICAgICJjb250aW51b3VzIjogNQogICB9LAogICAiaWQiOiAxLAogICAibmFtZSI6ICLnqbrpl7LnjociCiAgfQogXSwKICJkZXRlY3RzIjogWwogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAxLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAzLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAyLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAyLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9LAogIHsKICAgImV4cHJlc3Npb24iOiAiIiwKICAgImNvbm5lY3RvciI6ICJhbmQiLAogICAibGV2ZWwiOiAzLAogICAidHJpZ2dlcl9jb25maWciOiB7CiAgICAiY291bnQiOiAxLAogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfSwKICAgInJlY292ZXJ5X2NvbmZpZyI6IHsKICAgICJjaGVja193aW5kb3ciOiA1CiAgIH0KICB9CiBdLAogInNjZW5hcmlvIjogIm9zIiwKICJhY3Rpb25zIjogWwogIHsKICAgIm5vdGljZV90ZW1wbGF0ZSI6IHsKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIsCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwCn0=","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[1,2,3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"197f40ff15e95f2778a9d7b2fb32c83aa15ac6b63d8b8217a9b5307ec3efd3df","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":1,"trigger_count":3},{"check_window_size":5,"level":2,"trigger_count":2},{"check_window_size":5,"level":3,"trigger_count":1}]}},{"name":"retry-same-input","outcome":{"batch_id":"detect-batch-retry-2","evaluations":[{"anomaly":{"anomaly_id":"342a08e0f85f169a7e099c18db3708ed.1569246480.1.2.3","anomaly_message":"异常测试","anomaly_time":"2019-10-10 10:10:00","context":{"level":3,"mixed":[1,"2",{"nested":true}]}},"level":3,"result":"ANOMALOUS"}],"input_id":"c4e2b55205fa63110d8e5010440faeee1db9a9cac724044d51fae2b23d688b16","outcome":"ANOMALOUS","purpose":"DETECT","record":{"data_raw":{"dimensions":{"ip":"127.0.0.1"},"record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","time":1569246480,"value":99,"values":{"huge_counter":9007199254740993,"load5":99,"timestamp":1569246480}},"dimensions_md5":"342a08e0f85f169a7e099c18db3708ed","record_id":"342a08e0f85f169a7e099c18db3708ed.1569246480","source_time":1569246480},"required_features":["full-level-evaluations-v1","raw-json-v1"],"schema":{"major":1,"minor":0,"name":"detection-outcome"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default"},"strategy_ir":{"check_window_unit_seconds":60,"legacy_json_b64":"ewogImJrX2Jpel9pZCI6IDIsCiAiaXRlbXMiOiBbCiAgewogICAicXVlcnlfY29uZmlncyI6IFsKICAgIHsKICAgICAibWV0cmljX2ZpZWxkIjogImlkbGUiLAogICAgICJhZ2dfZGltZW5zaW9uIjogWwogICAgICAiaXAiLAogICAgICAiYmtfY2xvdWRfaWQiCiAgICAgXSwKICAgICAiaWQiOiAyLAogICAgICJhZ2dfbWV0aG9kIjogIkFWRyIsCiAgICAgImFnZ19jb25kaXRpb24iOiBbXSwKICAgICAiYWdnX2ludGVydmFsIjogNjAsCiAgICAgInJlc3VsdF90YWJsZV9pZCI6ICJzeXN0ZW0uY3B1X2RldGFpbCIsCiAgICAgInVuaXQiOiAiJSIsCiAgICAgImRhdGFfdHlwZV9sYWJlbCI6ICJ0aW1lX3NlcmllcyIsCiAgICAgIm1ldHJpY19pZCI6ICJia19tb25pdG9yLnN5c3RlbS5jcHVfZGV0YWlsLmlkbGUiLAogICAgICJkYXRhX3NvdXJjZV9sYWJlbCI6ICJia19tb25pdG9yIgogICAgfQogICBdLAogICAiYWxnb3JpdGhtcyI6IFsKICAgIHsKICAgICAiY29uZmlnIjogWwogICAgICBbCiAgICAgICB7CiAgICAgICAgInRocmVzaG9sZCI6IDUxLjAsCiAgICAgICAgIm1ldGhvZCI6ICJndGUiCiAgICAgICB9CiAgICAgIF0KICAgICBdLAogICAgICJsZXZlbCI6IDMsCiAgICAgInR5cGUiOiAiVGhyZXNob2xkIiwKICAgICAiaWQiOiAyCiAgICB9LAogICAgewogICAgICJjb25maWciOiBbCiAgICAgIFsKICAgICAgIHsKICAgICAgICAidGhyZXNob2xkIjogMTAwLAogICAgICAgICJtZXRob2QiOiAibHRlIgogICAgICAgfQogICAgICBdCiAgICAgXSwKICAgICAibGV2ZWwiOiAzLAogICAgICJ0eXBlIjogIlRocmVzaG9sZCIsCiAgICAgImlkIjogMwogICAgfQogICBdLAogICAibm9fZGF0YV9jb25maWciOiB7CiAgICAiaXNfZW5hYmxlZCI6IGZhbHNlLAogICAgImNvbnRpbnVvdXMiOiA1CiAgIH0sCiAgICJpZCI6IDIsCiAgICJuYW1lIjogIuepuumXsueOhyIsCiAgICJ0YXJnZXQiOiBbCiAgICBbCiAgICAgewogICAgICAiZmllbGQiOiAiaXAiLAogICAgICAibWV0aG9kIjogImVxIiwKICAgICAgInZhbHVlIjogWwogICAgICAgewogICAgICAgICJpcCI6ICIxMjcuMC4wLjEiLAogICAgICAgICJia19jbG91ZF9pZCI6IDAsCiAgICAgICAgImJrX3N1cHBsaWVyX2lkIjogMAogICAgICAgfQogICAgICBdCiAgICAgfQogICAgXQogICBdCiAgfQogXSwKICJzY2VuYXJpbyI6ICJvcyIsCiAiYWN0aW9ucyI6IFsKICB7CiAgICJub3RpY2VfdGVtcGxhdGUiOiB7CiAgICAiYWN0aW9uX2lkIjogMiwKICAgICJhbm9tYWx5X3RlbXBsYXRlIjogImFhIiwKICAgICJyZWNvdmVyeV90ZW1wbGF0ZSI6ICIiCiAgIH0sCiAgICJpZCI6IDIsCiAgICJub3RpY2VfZ3JvdXBfbGlzdCI6IFsKICAgIHsKICAgICAibm90aWNlX3JlY2VpdmVyIjogWwogICAgICAidXNlciN0ZXN0IgogICAgIF0sCiAgICAgIm5hbWUiOiAidGVzdCIsCiAgICAgIm5vdGljZV93YXkiOiB7CiAgICAgICIxIjogWwogICAgICAgIndlaXhpbiIKICAgICAgXSwKICAgICAgIjMiOiBbCiAgICAgICAid2VpeGluIgogICAgICBdLAogICAgICAiMiI6IFsKICAgICAgICJ3ZWl4aW4iCiAgICAgIF0KICAgICB9LAogICAgICJub3RpY2VfZ3JvdXBfaWQiOiAxLAogICAgICJtZXNzYWdlIjogIiIsCiAgICAgIm5vdGljZV9ncm91cF9uYW1lIjogInRlc3QiLAogICAgICJpZCI6IDEKICAgIH0KICAgXSwKICAgInR5cGUiOiAibm90aWNlIiwKICAgImNvbmZpZyI6IHsKICAgICJhbGFybV9lbmRfdGltZSI6ICIyMzo1OTo1OSIsCiAgICAic2VuZF9yZWNvdmVyeV9hbGFybSI6IGZhbHNlLAogICAgImFsYXJtX3N0YXJ0X3RpbWUiOiAiMDA6MDA6MDAiLAogICAgImFsYXJtX2ludGVydmFsIjogMTIwCiAgIH0KICB9CiBdLAogImRldGVjdHMiOiBbCiAgewogICAibGV2ZWwiOiAzLAogICAiZXhwcmVzc2lvbiI6ICIiLAogICAiY29ubmVjdG9yIjogImFuZCIsCiAgICJ0cmlnZ2VyX2NvbmZpZyI6IHsKICAgICJjb3VudCI6IDEsCiAgICAiY2hlY2tfd2luZG93IjogNQogICB9LAogICAicmVjb3ZlcnlfY29uZmlnIjogewogICAgImNoZWNrX3dpbmRvdyI6IDUKICAgfQogIH0KIF0sCiAidXBkYXRlX3RpbWUiOiAxNTY5MjQ2NDgwLAogInNvdXJjZV90eXBlIjogIkJLTU9OSVRPUiIsCiAiaWQiOiAxLAogIm5hbWUiOiAidGVzdCIKfQ==","purpose":"DETECT","required_features":["raw-strategy-bytes-v1"],"required_levels":[3],"schema":{"major":1,"minor":0,"name":"trigger-strategy-ir"},"strategy_ref":{"content_sha256":"9fb22814fc4c0159ebd032634ee9fdaf3cfeaaba81f78a2e8fb7c89bab0ab57e","generation":"1569246480","item_id":"2","strategy_id":"1"},"tenant_id":"default","trigger_configs":[{"check_window_size":5,"level":3,"trigger_count":1}]}}],"schema_version":"detection-outcome/1.0"} diff --git a/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/trigger_decision_v1.json b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/trigger_decision_v1.json new file mode 100644 index 00000000000..f6b750c0d68 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/core/alarm_engine/testdata/python-v1/trigger_decision_v1.json @@ -0,0 +1 @@ +{"fixtures":[{"batch":{"batch_id":"python-reference-55a76cf628e46c04a052f4e19bdb9dbf.1569246480","decision_algorithm":"trigger-window-v1","decisions":[{"anomaly_timestamps":[1569246480],"decision_id":"f611274bf630b8921b4b2bf0caa6feb86363f4195012ed57a92b60fff9af7dfa","input_id":"79917dbccf8fd61d15f712653d445b9ad6766228f74fff6e9c5a6afcd15067bc","level":3,"outcome":"TRIGGER","reason_code":"TRIGGER_CONDITION_MET","record_id":"55a76cf628e46c04a052f4e19bdb9dbf.1569246480"}],"partition_hash_version":"trigger-input-partition-v1","purpose":"DETECT","required_features":[],"schema":{"major":1,"minor":0,"name":"trigger-decision-batch"},"strategy_ref":{"content_sha256":"01d64f44affe7983eff637fb9b3e93b49306fca2f88ebb6db2f55aa61c6000b5","generation":"1569246480","item_id":"1","strategy_id":"1"},"tenant_id":"default"},"name":"python-trigger-reference"}],"schema_version":"trigger-decision-batch/1.0"} diff --git a/bkmonitor/alarm_backends/tests/service/alert/test_adapter.py b/bkmonitor/alarm_backends/tests/service/alert/test_adapter.py index 92ebe9ac066..f9553f03dbc 100644 --- a/bkmonitor/alarm_backends/tests/service/alert/test_adapter.py +++ b/bkmonitor/alarm_backends/tests/service/alert/test_adapter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. Copyright (C) 2017-2025 Tencent. All rights reserved. @@ -8,7 +7,11 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from unittest import TestCase + +from types import SimpleNamespace +from unittest import TestCase, mock + +from django.conf import settings from alarm_backends.core.alert.adapter import MonitorEventAdapter @@ -144,6 +147,16 @@ class TestAdapter(TestCase): + def test_output_topic_uses_current_cluster_suffix(self): + with ( + mock.patch.object(settings, "MONITOR_EVENT_KAFKA_TOPIC", "monitor-event"), + mock.patch( + "alarm_backends.core.alert.adapter.get_cluster", + return_value=SimpleNamespace(is_default=lambda: False, name="nondefault"), + ), + ): + self.assertEqual("monitor-event_nondefault", MonitorEventAdapter.get_output_topic()) + def test_adapt(self): adapter = MonitorEventAdapter(record=ANOMALY_EVENT, strategy=STRATEGY) event = adapter.adapt() diff --git a/bkmonitor/alarm_backends/tests/service/detect/test_alarm_engine_shadow.py b/bkmonitor/alarm_backends/tests/service/detect/test_alarm_engine_shadow.py new file mode 100644 index 00000000000..b6dac94a198 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/service/detect/test_alarm_engine_shadow.py @@ -0,0 +1,448 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import json +from types import SimpleNamespace +from unittest import mock + +import pytest +from django.conf import settings + +from alarm_backends.core.cache import key +from alarm_backends.core.alarm_engine import publisher as publisher_module +from alarm_backends.core.alarm_engine import reference_publisher as reference_publisher_module +from alarm_backends.core.alert.adapter import MonitorEventAdapter +from alarm_backends.service.detect.process import DetectProcess +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY + + +def test_alarm_engine_shadow_is_inert_when_disabled(): + processor = object.__new__(DetectProcess) + + with mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", False, create=True): + assert processor.prepare_alarm_engine_detection_batches() == [] + + +def test_alarm_engine_shadow_requires_an_explicit_strategy_selector(): + processor = object.__new__(DetectProcess) + processor.strategy_id = "1" + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", (), create=True), + ): + assert processor.prepare_alarm_engine_detection_batches() == [] + + +@pytest.mark.parametrize("selector", [(True,), (1.9,), ("01",), (" 1 ",), "1,"]) +def test_alarm_engine_shadow_rejects_noncanonical_strategy_selectors(selector): + processor = object.__new__(DetectProcess) + processor.strategy_id = "1" + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", selector, create=True), + ): + assert processor.prepare_alarm_engine_detection_batches() == [] + + +def test_alarm_engine_shadow_projects_finalized_threshold_records(): + strategy = copy.deepcopy(DETECT_STRATEGY) + anomalous_record, normal_record = copy.deepcopy(DETECT_RECORDS) + processor = object.__new__(DetectProcess) + processor.strategy_id = "1" + source_strategy = SimpleNamespace(id=1, config=strategy) + processor.strategy = SimpleNamespace( + id=1, + bk_tenant_id="default", + config=strategy, + items=[SimpleNamespace(id=2)], + snapshot_key="snapshot-key", + ) + processor.inputs = { + 2: [ + SimpleNamespace(item=SimpleNamespace(strategy=source_strategy), as_dict=lambda: anomalous_record), + SimpleNamespace(item=SimpleNamespace(strategy=source_strategy), as_dict=lambda: normal_record), + ] + } + processor.outputs = { + 2: [ + { + "data": anomalous_record, + "anomaly": {"3": {"anomaly_id": f"{anomalous_record['record_id']}.1.2.3"}}, + } + ] + } + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", (1,), create=True), + mock.patch.object(settings, "DOUBLE_CHECK_SUM_STRATEGY_IDS", []), + mock.patch.object(key.STRATEGY_SNAPSHOT_KEY.client, "get", return_value=json.dumps(strategy).encode()), + ): + batches = processor.prepare_alarm_engine_detection_batches() + + assert len(batches) == 1 + assert [outcome["outcome"] for outcome in batches[0]["outcomes"]] == ["ANOMALOUS", "NORMAL"] + assert "_alarm_engine" not in processor.outputs[2][0] + + +@pytest.mark.parametrize("stale_update_time", ["stale", True, 1.0]) +def test_alarm_engine_shadow_rejects_inputs_from_a_stale_strategy_snapshot(stale_update_time): + strategy = copy.deepcopy(DETECT_STRATEGY) + stale_strategy = copy.deepcopy(strategy) + strategy["update_time"] = 1 + stale_strategy["update_time"] = stale_update_time + record = copy.deepcopy(DETECT_RECORDS[0]) + processor = object.__new__(DetectProcess) + processor.strategy_id = "1" + processor.strategy = SimpleNamespace( + id=1, + bk_tenant_id="default", + config=strategy, + items=[SimpleNamespace(id=2)], + snapshot_key="snapshot-key", + ) + processor.inputs = { + 2: [ + SimpleNamespace( + item=SimpleNamespace(strategy=SimpleNamespace(id=1, config=stale_strategy)), + as_dict=lambda: record, + ) + ] + } + processor.outputs = {2: []} + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", (1,), create=True), + mock.patch.object(settings, "DOUBLE_CHECK_SUM_STRATEGY_IDS", []), + mock.patch.object(key.STRATEGY_SNAPSHOT_KEY.client, "get", return_value=json.dumps(strategy).encode()), + ): + assert processor.prepare_alarm_engine_detection_batches() == [] + + +def test_detect_push_keeps_legacy_delivery_before_shadow_publish(): + processor = object.__new__(DetectProcess) + processor.strategy_id = "1" + processor.strategy = SimpleNamespace(bk_biz_id=2, name="strategy") + processor.inputs = {} + processor.outputs = {} + calls = [] + processor.prepare_alarm_engine_detection_batches = lambda: calls.append("prepare") or ["batch"] + processor.push_abnormal_data = lambda *_args: calls.append("legacy") or 0 + processor.publish_alarm_engine_detection_batches = lambda batches: calls.append(("shadow", batches)) + + processor.push_data() + + assert calls == ["legacy", "prepare", ("shadow", ["batch"])] + + +def test_alarm_engine_shadow_publishes_with_process_cached_producer(): + published = [] + fake_publisher = SimpleNamespace(publish_batch=lambda batch: published.append(batch) or len(batch["outcomes"])) + batches = [{"strategy_ir": {}, "outcomes": [{"input_id": "one"}]}] + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=fake_publisher), + ): + assert DetectProcess.publish_alarm_engine_detection_batches(batches) == 1 + + assert published == batches + + +def test_alarm_engine_shadow_publishes_terminal_reference_only_after_detection_ack(): + calls = [] + batch = _prepared_detection_batch() + detection_publisher = SimpleNamespace( + publish_batch=lambda value: calls.append(("detection", value)) or len(value["outcomes"]) + ) + reference_publisher = SimpleNamespace(publish_batch=lambda value: calls.append(("reference", value)) or 1) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + side_effect=lambda *_args: calls.append(("reference-factory", None)) or reference_publisher, + ) as reference_factory, + ): + assert DetectProcess.publish_alarm_engine_detection_batches([batch]) == 2 + + assert [name for name, _value in calls] == ["detection", "reference-factory", "reference"] + assert calls[2][1]["decisions"][0]["reason_code"] == "INPUT_NORMAL" + factory_args = reference_factory.call_args.args + assert factory_args[2] == ("alarm-engine-detection-shadow", "monitor-event-nondefault") + + +def test_alarm_engine_reference_failure_does_not_change_acknowledged_detection_result(): + batch = _prepared_detection_batch() + detection_publisher = SimpleNamespace(publish_batch=lambda value: len(value["outcomes"])) + reference_publisher = SimpleNamespace(publish_batch=mock.Mock(side_effect=RuntimeError("reference failed"))) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + return_value=reference_publisher, + ), + ): + assert DetectProcess.publish_alarm_engine_detection_batches([batch]) == 2 + + reference_publisher.publish_batch.assert_called_once() + + +@pytest.mark.parametrize( + ("reference_config", "reference_allowed_topics"), + [ + ({"topic": object()}, ("alarm-engine-reference-shadow",)), + ({"topic": "alarm-engine-reference-shadow"}, (["not-hashable"],)), + ], +) +def test_invalid_reference_config_does_not_block_detection_ack(reference_config, reference_allowed_topics): + batch = _prepared_detection_batch() + detection_publisher = SimpleNamespace(publish_batch=mock.Mock(return_value=len(batch["outcomes"]))) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + reference_config, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + reference_allowed_topics, + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + ) as reference_factory, + ): + assert DetectProcess.publish_alarm_engine_detection_batches([batch]) == len(batch["outcomes"]) + + detection_publisher.publish_batch.assert_called_once_with(batch) + reference_factory.assert_not_called() + + +def test_detection_publish_failure_never_initializes_or_sends_reference(): + batch = _prepared_detection_batch() + detection_publisher = SimpleNamespace(publish_batch=mock.Mock(side_effect=RuntimeError("detection failed"))) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + ) as reference_factory, + ): + with pytest.raises(RuntimeError, match="detection failed"): + DetectProcess.publish_alarm_engine_detection_batches([batch]) + + reference_factory.assert_not_called() + + +def test_all_anomalous_detection_batch_does_not_initialize_terminal_reference(): + batch = _prepared_detection_batch() + batch["outcomes"] = [batch["outcomes"][0]] + detection_publisher = SimpleNamespace(publish_batch=mock.Mock(return_value=1)) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + ) as reference_factory, + ): + assert DetectProcess.publish_alarm_engine_detection_batches([batch]) == 1 + + reference_factory.assert_not_called() + + +def test_reference_projection_failure_does_not_disable_later_batches(): + first_batch = _prepared_detection_batch() + second_batch = _prepared_detection_batch() + second_batch["outcomes"] = [second_batch["outcomes"][1]] + calls = [] + detection_publisher = SimpleNamespace(publish_batch=lambda batch: len(batch["outcomes"])) + reference_publisher = SimpleNamespace(publish_batch=lambda batch: calls.append(batch) or len(batch["decisions"])) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-detection-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch.object(publisher_module, "get_cached_kafka_detection_publisher", return_value=detection_publisher), + mock.patch( + "alarm_backends.core.alarm_engine.reference.build_terminal_reference_decision_batches", + side_effect=[RuntimeError("projection failed"), [{"decisions": [{"input_id": "second"}]}]], + ), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + return_value=reference_publisher, + ), + ): + assert DetectProcess.publish_alarm_engine_detection_batches([first_batch, second_batch]) == 3 + + assert calls == [{"decisions": [{"input_id": "second"}]}] + + +def _prepared_detection_batch(): + strategy = copy.deepcopy(DETECT_STRATEGY) + anomalous_record, normal_record = copy.deepcopy(DETECT_RECORDS) + from alarm_backends.core.alarm_engine.runtime import prepare_finalized_threshold_batch + + return prepare_finalized_threshold_batch( + tenant_id="default", + strategy=strategy, + item_id=2, + legacy_json=json.dumps(strategy).encode(), + batch_id="batch-1", + data_points=[anomalous_record, normal_record], + anomaly_outputs=[ + { + "data": anomalous_record, + "anomaly": {"3": {"anomaly_id": f"{anomalous_record['record_id']}.1.2.3"}}, + } + ], + finalized=True, + ) diff --git a/bkmonitor/alarm_backends/tests/service/detect/test_processor.py b/bkmonitor/alarm_backends/tests/service/detect/test_processor.py index 6faa5c7c8b2..2ece5dabc6c 100644 --- a/bkmonitor/alarm_backends/tests/service/detect/test_processor.py +++ b/bkmonitor/alarm_backends/tests/service/detect/test_processor.py @@ -21,91 +21,13 @@ from alarm_backends.core.detect_result import ANOMALY_LABEL, CheckResult from alarm_backends.core.storage.redis_cluster import get_node_by_strategy_id from alarm_backends.service.detect.process import DetectProcess +from alarm_backends.tests.alarm_engine_fixtures import DETECT_RECORDS, DETECT_STRATEGY from bkmonitor.models import CacheNode pytestmark = pytest.mark.django_db -strategy_config = { - "bk_biz_id": 2, - "items": [ - { - "query_configs": [ - { - "metric_field": "idle", - "agg_dimension": ["ip", "bk_cloud_id"], - "id": 2, - "agg_method": "AVG", - "agg_condition": [], - "agg_interval": 60, - "result_table_id": "system.cpu_detail", - "unit": "%", - "data_type_label": "time_series", - "metric_id": "bk_monitor.system.cpu_detail.idle", - "data_source_label": "bk_monitor", - } - ], - "algorithms": [ - { - "config": [[{"threshold": 51.0, "method": "gte"}]], - "level": 3, - "type": "Threshold", - "id": 2, - }, - { - "config": [[{"threshold": 100, "method": "lte"}]], - "level": 3, - "type": "Threshold", - "id": 3, - }, - ], - "no_data_config": {"is_enabled": False, "continuous": 5}, - "id": 2, - "name": "\u7a7a\u95f2\u7387", - "target": [ - [{"field": "ip", "method": "eq", "value": [{"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}]}] - ], - } - ], - "scenario": "os", - "actions": [ - { - "notice_template": {"action_id": 2, "anomaly_template": "aa", "recovery_template": ""}, - "id": 2, - "notice_group_list": [ - { - "notice_receiver": ["user#test"], - "name": "test", - "notice_way": {"1": ["weixin"], "3": ["weixin"], "2": ["weixin"]}, - "notice_group_id": 1, - "message": "", - "notice_group_name": "test", - "id": 1, - } - ], - "type": "notice", - "config": { - "alarm_end_time": "23:59:59", - "send_recovery_alarm": False, - "alarm_start_time": "00:00:00", - "alarm_interval": 120, - }, - } - ], - "detects": [ - { - "level": 3, - "expression": "", - "connector": "and", - "trigger_config": {"count": 1, "check_window": 5}, - "recovery_config": {"check_window": 5}, - } - ], - "update_time": 1569246480, - "source_type": "BKMONITOR", - "id": 1, - "name": "test", -} +strategy_config = DETECT_STRATEGY class TestProcessorViews(TestCase): @@ -141,23 +63,7 @@ def test_processor_handle(self): "alarm_backends.core.cache.strategy.StrategyCacheManager.get_strategy_by_id", return_value=copy.deepcopy(strategy_config), ): - records = [ - { - "record_id": "342a08e0f85f169a7e099c18db3708ed.1569246480", - "value": 99, - "values": {"timestamp": 1569246480, "load5": 99}, - "dimensions": {"ip": "127.0.0.1"}, - "time": 1569246480, - }, - { - "record_id": "2a1850513fa6018c435f9b6359b3fa7d.1569246481", - "value": 50.1, - "values": {"timestamp": 1569246481, "load5": 50.1}, - "dimensions": {"ip": "10.0.0.1"}, - # 数据点时间戳错开，避免检测记录断言不准确 - "time": 1569246481, - }, - ] + records = copy.deepcopy(DETECT_RECORDS) dumped_records = list(map(json.dumps, records)) # 环境定义，和mock的strategy_config保持一致 diff --git a/bkmonitor/alarm_backends/tests/service/trigger/test_alarm_engine_reference.py b/bkmonitor/alarm_backends/tests/service/trigger/test_alarm_engine_reference.py new file mode 100644 index 00000000000..f2345d3f2c1 --- /dev/null +++ b/bkmonitor/alarm_backends/tests/service/trigger/test_alarm_engine_reference.py @@ -0,0 +1,361 @@ +""" +Tencent is pleased to support the open source community by making 蓝鲸智云 - 监控平台 (BlueKing - Monitor) available. +Copyright (C) 2017-2025 Tencent. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import copy +import json +from types import SimpleNamespace +from unittest import mock + +import pytest +from django.conf import settings + +from alarm_backends.core.alarm_engine import reference_publisher as reference_publisher_module +from alarm_backends.core.alert.adapter import MonitorEventAdapter +from alarm_backends.service.trigger.processor import TriggerProcessor +from alarm_backends.tests.alarm_engine_fixtures import TRIGGER_POINT, TRIGGER_STRATEGY + + +def test_trigger_reference_capture_is_lightweight_and_follows_real_checker_result(): + point = copy.deepcopy(TRIGGER_POINT) + strategy = copy.deepcopy(TRIGGER_STRATEGY) + event_record = _triggered_event(point) + processor = _processor() + processor.get_strategy_snapshot = mock.Mock(return_value=strategy) + processor.get_strategy_snapshot_legacy_json = mock.Mock() + + checker = mock.Mock() + checker.check.return_value = ([], event_record) + checker.is_no_data_point.return_value = False + with ( + mock.patch("alarm_backends.service.trigger.processor.AnomalyChecker", return_value=checker), + mock.patch.object(processor, "is_alarm_engine_reference_selected", return_value=True), + ): + processor.process_point(json.dumps(point)) + + assert processor.reference_candidates == [ + { + "strategy_snapshot_key": point["strategy_snapshot_key"], + "point": point, + "event_record": event_record, + } + ] + processor.get_strategy_snapshot_legacy_json.assert_not_called() + + +def test_trigger_reference_is_not_captured_outside_the_detection_selector(): + processor = _processor() + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.get_strategy_snapshot_legacy_json = mock.Mock() + checker = mock.Mock() + checker.check.return_value = ([], None) + + with ( + mock.patch("alarm_backends.service.trigger.processor.AnomalyChecker", return_value=checker), + mock.patch.object(processor, "is_alarm_engine_reference_selected", return_value=False), + ): + processor.process_point(json.dumps(copy.deepcopy(TRIGGER_POINT))) + + assert processor.reference_candidates == [] + processor.get_strategy_snapshot_legacy_json.assert_not_called() + + +def test_trigger_reference_does_not_capture_nodata_points(): + processor = _processor() + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.capture_alarm_engine_reference_candidate = mock.Mock() + checker = mock.Mock() + checker.check.return_value = ([], None) + checker.is_no_data_point.return_value = True + + with ( + mock.patch("alarm_backends.service.trigger.processor.AnomalyChecker", return_value=checker), + mock.patch.object(processor, "is_alarm_engine_reference_selected", return_value=True), + ): + processor.process_point(json.dumps(copy.deepcopy(TRIGGER_POINT))) + + processor.capture_alarm_engine_reference_candidate.assert_not_called() + + +def test_trigger_reference_reuses_detection_selector_and_excludes_double_check(): + processor = _processor() + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", (1,), create=True), + mock.patch.object(settings, "DOUBLE_CHECK_SUM_STRATEGY_IDS", [], create=True), + ): + assert processor.is_alarm_engine_reference_selected() + + with ( + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED", True, create=True), + mock.patch.object(settings, "ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS", (1,), create=True), + mock.patch.object(settings, "DOUBLE_CHECK_SUM_STRATEGY_IDS", [1], create=True), + ): + assert not processor.is_alarm_engine_reference_selected() + + +def test_trigger_push_completes_monitor_event_before_reference_publish(): + processor = _processor() + processor.event_records = [{"event_record": {"data": {}}, "anomaly_records": []}] + processor.reference_candidates = [{"input_id": "candidate"}] + calls = [] + processor.push_event_to_kafka = lambda records: calls.append(("monitor-event", records)) + processor.publish_alarm_engine_reference_candidates = lambda: calls.append(("reference", None)) + + processor.push() + + assert [name for name, _value in calls] == ["monitor-event", "reference"] + + +def test_trigger_monitor_event_failure_prevents_reference_publish(): + processor = _processor() + processor.event_records = [{"event_record": {"data": {}}, "anomaly_records": []}] + processor.reference_candidates = [{"input_id": "candidate"}] + processor.push_event_to_kafka = mock.Mock(side_effect=RuntimeError("monitor event failed")) + processor.publish_alarm_engine_reference_candidates = mock.Mock() + + with pytest.raises(RuntimeError, match="monitor event failed"): + processor.push() + + processor.publish_alarm_engine_reference_candidates.assert_not_called() + + +def test_trigger_reference_unexpected_failure_does_not_change_legacy_push_result(): + processor = _processor() + processor.reference_candidates = [{"input_id": "candidate"}] + processor.publish_alarm_engine_reference_candidates = mock.Mock(side_effect=RuntimeError("reference failed")) + + processor.push() + + assert processor.reference_candidates == [] + + +def test_trigger_reference_publisher_uses_candidate_identity_and_is_fail_open(): + processor = _processor() + processor.reference_candidates = [ + {"strategy_snapshot_key": "first", "point": "first", "event_record": None}, + {"strategy_snapshot_key": "second", "point": "second", "event_record": None}, + ] + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.get_strategy_snapshot_legacy_json = mock.Mock(return_value=b"strategy") + batches = [ + {"tenant_id": "default", "purpose": "DETECT", "strategy_ref": {}, "decisions": [{"input_id": "one"}]}, + {"tenant_id": "default", "purpose": "DETECT", "strategy_ref": {}, "decisions": [{"input_id": "two"}]}, + ] + published_groups = [] + + def publish_batches(batch_iter): + published_groups.append(list(batch_iter)) + return len(published_groups[-1]) + + publisher = SimpleNamespace(publish_batches=mock.Mock(side_effect=publish_batches)) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch( + "alarm_backends.core.alarm_engine.reference.build_reference_trigger_decision_candidate", + side_effect=batches, + ) as candidate_builder, + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + return_value=publisher, + ) as factory, + ): + assert processor.publish_alarm_engine_reference_candidates() == 1 + + assert published_groups == [batches] + assert processor.get_strategy_snapshot_legacy_json.call_args_list == [mock.call("first"), mock.call("second")] + assert [call.kwargs["legacy_json"] for call in candidate_builder.call_args_list] == [b"strategy", b"strategy"] + assert factory.call_args.args[2] == ("alarm-engine-detection-shadow", "monitor-event-nondefault") + + +def test_trigger_reference_publisher_flushes_candidates_in_bounded_groups(): + processor = _processor() + processor.reference_candidates = [ + {"strategy_snapshot_key": "snapshot", "point": index, "event_record": None} for index in range(501) + ] + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.get_strategy_snapshot_legacy_json = mock.Mock(return_value=b"strategy") + published_groups = [] + + def publish_batches(batch_iter): + published_groups.append(list(batch_iter)) + return len(published_groups[-1]) + + publisher = SimpleNamespace(publish_batches=mock.Mock(side_effect=publish_batches)) + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch( + "alarm_backends.core.alarm_engine.reference.build_reference_trigger_decision_candidate", + side_effect=lambda **kwargs: {"decisions": [{"input_id": str(kwargs["point"])}]}, + ), + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + return_value=publisher, + ), + ): + assert processor.publish_alarm_engine_reference_candidates() == 501 + + assert [len(group) for group in published_groups] == [500, 1] + + +def test_trigger_reference_stops_projecting_when_the_publisher_stops_consuming(): + processor = _processor() + processor.reference_candidates = [ + {"strategy_snapshot_key": "snapshot", "point": index, "event_record": None} for index in range(501) + ] + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.get_strategy_snapshot_legacy_json = mock.Mock(return_value=b"strategy") + + def fail_after_first_batch(batches): + next(iter(batches)) + raise RuntimeError("broker stopped consuming") + + publisher = SimpleNamespace(publish_batches=mock.Mock(side_effect=fail_after_first_batch)) + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch( + "alarm_backends.core.alarm_engine.reference.build_reference_trigger_decision_candidate", + return_value={"decisions": [{"input_id": "one"}]}, + ) as candidate_builder, + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + return_value=publisher, + ), + ): + assert processor.publish_alarm_engine_reference_candidates() == 0 + + candidate_builder.assert_called_once() + + +def test_trigger_reference_stops_after_publisher_initialization_failure(): + processor = _processor() + processor.reference_candidates = [ + {"strategy_snapshot_key": "snapshot", "point": index, "event_record": None} for index in range(501) + ] + processor.get_strategy_snapshot = mock.Mock(return_value=copy.deepcopy(TRIGGER_STRATEGY)) + processor.get_strategy_snapshot_legacy_json = mock.Mock(return_value=b"strategy") + + with ( + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG", + {"topic": "alarm-engine-reference-shadow", "bootstrap.servers": "kafka:9092"}, + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-reference-shadow",), + create=True, + ), + mock.patch.object( + settings, + "ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS", + ("alarm-engine-detection-shadow",), + create=True, + ), + mock.patch.object(MonitorEventAdapter, "get_output_topic", return_value="monitor-event-nondefault"), + mock.patch( + "alarm_backends.core.alarm_engine.reference.build_reference_trigger_decision_candidate", + return_value={"decisions": [{"input_id": "one"}]}, + ) as candidate_builder, + mock.patch.object( + reference_publisher_module, + "get_cached_kafka_reference_decision_publisher", + side_effect=RuntimeError("publisher initialization failed"), + ), + ): + assert processor.publish_alarm_engine_reference_candidates() == 0 + + candidate_builder.assert_called_once() + + +def _processor(): + processor = object.__new__(TriggerProcessor) + processor.strategy_id = 1 + processor.item_id = 1 + processor.strategy = SimpleNamespace(bk_biz_id=2, name="strategy") + processor.anomaly_points = [] + processor.anomaly_records = [] + processor.event_records = [] + processor.reference_candidates = [] + processor._strategy_snapshot_legacy_json = {} + return processor + + +def _triggered_event(point): + source_time = point["data"]["time"] + event = copy.deepcopy(point) + event["trigger"] = { + "level": "1", + "anomaly_ids": [f"55a76cf628e46c04a052f4e19bdb9dbf.{source_time}.1.1.1"], + } + return event diff --git a/bkmonitor/alarm_backends/tests/service/trigger/test_checker.py b/bkmonitor/alarm_backends/tests/service/trigger/test_checker.py index 4588f4ab25c..54604160b04 100644 --- a/bkmonitor/alarm_backends/tests/service/trigger/test_checker.py +++ b/bkmonitor/alarm_backends/tests/service/trigger/test_checker.py @@ -8,144 +8,31 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ + import copy +import json import arrow import pytest from django.test import TestCase from alarm_backends.constants import NO_DATA_TAG_DIMENSION +from alarm_backends.core.alarm_engine.contract import ( + build_detection_outcome, + build_trigger_strategy_ir_from_legacy_config, +) +from alarm_backends.core.alarm_engine.reference import build_reference_trigger_decision_batch from alarm_backends.core.cache.key import CHECK_RESULT_CACHE_KEY from alarm_backends.core.storage.redis_cluster import get_node_by_strategy_id from alarm_backends.service.trigger.checker import AnomalyChecker +from alarm_backends.tests.alarm_engine_fixtures import TRIGGER_POINT as POINT +from alarm_backends.tests.alarm_engine_fixtures import TRIGGER_STRATEGY as STRATEGY from bkmonitor.models import CacheNode from bkmonitor.utils import time_tools from core.errors.alarm_backends import StrategyItemNotFound pytestmark = pytest.mark.django_db -STRATEGY = { - "bk_biz_id": 2, - "items": [ - { - "query_configs": [ - { - "metric_field": "idle", - "agg_dimension": ["ip", "bk_cloud_id"], - "id": 2, - "agg_method": "AVG", - "agg_condition": [], - "agg_interval": 60, - "result_table_id": "system.cpu_detail", - "unit": "%", - "data_type_label": "time_series", - "metric_id": "bk_monitor.system.cpu_detail.idle", - "data_source_label": "bk_monitor", - } - ], - "target": [ - [ - { - "field": "ip", - "method": "eq", - "value": [ - {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}, - ], - } - ] - ], - "algorithms": [ - {"config": [{"threshold": 0.1, "method": "gte"}], "level": 1, "type": "Threshold", "id": 1}, - {"config": [{"threshold": 0.1, "method": "gte"}], "level": 2, "type": "Threshold", "id": 2}, - {"config": [{"threshold": 0.1, "method": "gte"}], "level": 3, "type": "Threshold", "id": 3}, - ], - "no_data_config": {"is_enabled": False, "continuous": 5}, - "id": 1, - "name": "\u7a7a\u95f2\u7387", - } - ], - "detects": [ - { - "expression": "", - "connector": "and", - "level": 1, - "trigger_config": {"count": 3, "check_window": 5}, - "recovery_config": {"check_window": 5}, - }, - { - "expression": "", - "connector": "and", - "level": 2, - "trigger_config": {"count": 2, "check_window": 5}, - "recovery_config": {"check_window": 5}, - }, - { - "expression": "", - "connector": "and", - "level": 3, - "trigger_config": {"count": 1, "check_window": 5}, - "recovery_config": {"check_window": 5}, - }, - ], - "scenario": "os", - "actions": [ - { - "notice_template": {"anomaly_template": "aa", "recovery_template": ""}, - "id": 2, - "notice_group_list": [ - { - "notice_receiver": ["user#test"], - "name": "test", - "notice_way": {"1": ["weixin"], "3": ["weixin"], "2": ["weixin"]}, - "notice_group_id": 1, - "message": "", - "notice_group_name": "test", - "id": 1, - } - ], - "type": "notice", - "config": { - "alarm_end_time": "23:59:59", - "send_recovery_alarm": False, - "alarm_start_time": "00:00:00", - "alarm_interval": 120, - }, - } - ], - "source_type": "BKMONITOR", - "id": 1, - "name": "test", -} - - -POINT = { - "data": { - "record_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480", - "value": 1.38, - "values": {"timestamp": 1569246480, "load5": 1.38}, - "dimensions": {"ip": "10.0.0.1"}, - "time": 1569246480, - }, - "anomaly": { - "1": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.1", - "anomaly_time": "2019-10-10 10:10:00", - }, - "2": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.2", - "anomaly_time": "2019-10-10 10:10:00", - }, - "3": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.3", - "anomaly_time": "2019-10-10 10:10:00", - }, - }, - "strategy_snapshot_key": "xxx", -} - # 不同的异常数量测试集 CHECK_RESULT_SETS = { 0: [ @@ -195,7 +82,6 @@ class TestChecker(TestCase): - databases = {"monitor_api", "default"} @classmethod @@ -221,6 +107,45 @@ def test_init(self): with self.assertRaises(StrategyItemNotFound): AnomalyChecker(POINT, STRATEGY, 23) + def test_alarm_engine_reference_uses_real_checker_event_shape(self): + self.insert_check_result(3) + point = copy.deepcopy(POINT) + strategy = copy.deepcopy(STRATEGY) + strategy["update_time"] = 1569246480 + raw = json.dumps(strategy, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _, event_record = AnomalyChecker(point, strategy, 1).check() + strategy_ir = build_trigger_strategy_ir_from_legacy_config( + tenant_id="default", + purpose="DETECT", + strategy=strategy, + item_id=1, + legacy_json=raw, + ) + evaluations = [ + {"level": level, "result": "ANOMALOUS", "anomaly": copy.deepcopy(point["anomaly"][str(level)])} + for level in strategy_ir["required_levels"] + ] + detected = build_detection_outcome( + strategy_ir=strategy_ir, + batch_id="authoritative-detect-batch", + data_raw=point["data"], + evaluations=evaluations, + outcome="ANOMALOUS", + ) + + batch = build_reference_trigger_decision_batch( + strategy=strategy, + legacy_json=raw, + strategy_snapshot_key=point["strategy_snapshot_key"], + tenant_id_resolver=lambda _bk_biz_id: "default", + expected_input_id=detected["input_id"], + item_id=1, + point=point, + event_record=event_record, + ) + + assert batch["decisions"][0]["outcome"] == "TRIGGER" + def setUp(self): get_node_by_strategy_id(0) CacheNode.refresh_from_settings() diff --git a/bkmonitor/alarm_backends/tests/service/trigger/test_processor.py b/bkmonitor/alarm_backends/tests/service/trigger/test_processor.py index eb5af4c3fbd..583978d5fa1 100644 --- a/bkmonitor/alarm_backends/tests/service/trigger/test_processor.py +++ b/bkmonitor/alarm_backends/tests/service/trigger/test_processor.py @@ -25,39 +25,11 @@ ) from alarm_backends.core.storage.redis_cluster import get_node_by_strategy_id from alarm_backends.service.trigger.processor import TriggerProcessor +from alarm_backends.tests.alarm_engine_fixtures import TRIGGER_POINT as POINT +from alarm_backends.tests.alarm_engine_fixtures import TRIGGER_STRATEGY as STRATEGY from bkmonitor.models import AnomalyRecord, CacheNode, time_tools from core.errors.alarm_backends import StrategyNotFound -from .test_checker import STRATEGY - -POINT = { - "data": { - "record_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480", - "value": 1.38, - "values": {"timestamp": 1569246480, "load5": 1.38}, - "dimensions": {"ip": "10.0.0.1"}, - "time": 1569246480, - }, - "anomaly": { - "1": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.1", - "anomaly_time": "2019-10-10 10:10:00", - }, - "2": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.2", - "anomaly_time": "2019-10-10 10:10:00", - }, - "3": { - "anomaly_message": "异常测试", - "anomaly_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480.1.1.3", - "anomaly_time": "2019-10-10 10:10:00", - }, - }, - "strategy_snapshot_key": "xxx", -} - EVENT = { "data": { "record_id": "55a76cf628e46c04a052f4e19bdb9dbf.1569246480", diff --git a/bkmonitor/config/default.py b/bkmonitor/config/default.py index 0746d0cc2be..2f4b0eefaee 100644 --- a/bkmonitor/config/default.py +++ b/bkmonitor/config/default.py @@ -1361,6 +1361,14 @@ PUSH_MONITOR_EVENT_TO_FTA = True # 监控推送事件数据给自愈的 kafka topic MONITOR_EVENT_KAFKA_TOPIC = os.getenv("BK_MONITOR_EVENT_KAFKA_TOPIC", "0bkmonitor_backend_event") +# Alarm Engine Trigger-only Shadow 默认关闭，环境期望态显式提供独立 Shadow topic 后才可开启。 +ALARM_ENGINE_DETECTION_SHADOW_ENABLED = False +ALARM_ENGINE_DETECTION_SHADOW_STRATEGY_IDS = () +ALARM_ENGINE_DETECTION_SHADOW_KAFKA_CONFIG = {} +ALARM_ENGINE_DETECTION_SHADOW_ALLOWED_TOPICS = () +ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ENABLED = False +ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_KAFKA_CONFIG = {} +ALARM_ENGINE_TRIGGER_REFERENCE_SHADOW_ALLOWED_TOPICS = () # 监控推送事件数据给自愈的 插件ID MONITOR_EVENT_PLUGIN_ID = "bkmonitor" # 主机监控获取单个进程支持最多port数