diff --git a/src/poly/project.py b/src/poly/project.py index 47e7a508..4b44a8aa 100644 --- a/src/poly/project.py +++ b/src/poly/project.py @@ -922,6 +922,12 @@ def _update_multi_resource_yaml_resources( for file_path in self._sort_paths_for_reverse_deletion(deleted_paths, resource_type): resource_type.delete_resource(file_path, save_to_cache=True) + # The deletions above only reached the cache. Flush them, then clear: a cached + # entry carries the pre-write mtime, so anything left behind makes every later + # read in this process see a file state that is not on disk. + MultiResourceYamlResource.write_cache_to_file() + MultiResourceYamlResource._file_cache.clear() + return files_with_conflicts, progress_offset def _update_pulled_resources( diff --git a/src/poly/resources/agent_settings.py b/src/poly/resources/agent_settings.py index e5253085..fae3b7be 100644 --- a/src/poly/resources/agent_settings.py +++ b/src/poly/resources/agent_settings.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from functools import cached_property @@ -21,6 +22,8 @@ register_resource, ) +logger = logging.getLogger(__name__) + ALLOWED_BEHAVIOUR_REFERENCES = [ "global_functions", "sms", @@ -199,6 +202,10 @@ def validate(self, resource_mappings: list[ResourceMapping] = None, **kwargs) -> @classmethod def from_projection(cls, projection: dict) -> dict[str, "SettingsRules"]: """Parse rules settings from a projection dict.""" + if "agentSettings" not in projection: + logger.debug("No read access to the agent rules - it will not be pulled.") + return {} + agent_settings = projection.get("agentSettings", {}) rules = agent_settings.get("rules", None) if not rules: diff --git a/src/poly/resources/api_integration.py b/src/poly/resources/api_integration.py index 1225a40f..7089891f 100644 --- a/src/poly/resources/api_integration.py +++ b/src/poly/resources/api_integration.py @@ -283,6 +283,10 @@ class ApiIntegration(MultiResourceYamlResource): @classmethod def from_projection(cls, projection: dict) -> dict[str, "ApiIntegration"]: """Parse API integrations from a projection dict.""" + if "apiIntegrations" not in projection: + logger.debug("No read access to API integrations - it will not be pulled.") + return {} + api_integrations = {} for integration_id, integration_data in ( projection.get("apiIntegrations", {}) diff --git a/src/poly/resources/asr_settings.py b/src/poly/resources/asr_settings.py index 05398c61..0b31ea54 100644 --- a/src/poly/resources/asr_settings.py +++ b/src/poly/resources/asr_settings.py @@ -46,6 +46,10 @@ def __init__( @classmethod def from_projection(cls, projection: dict) -> dict[str, "AsrSettings"]: """Parse ASR settings from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to ASR settings - it will not be pulled.") + return {} + asr_settings_data = projection.get("channels", {}).get("voice", {}).get("asrSettings", {}) if not asr_settings_data: return {} diff --git a/src/poly/resources/channel_settings.py b/src/poly/resources/channel_settings.py index 856226b0..45db121b 100644 --- a/src/poly/resources/channel_settings.py +++ b/src/poly/resources/channel_settings.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass, field from functools import cached_property @@ -26,6 +27,8 @@ register_resource, ) +logger = logging.getLogger(__name__) + def _config_path(channel: str) -> str: """Return config path for a channel (e.g. voice/configuration.yaml).""" @@ -122,6 +125,10 @@ def update_command_type(self) -> str: @classmethod def from_projection(cls, projection: dict) -> dict[str, "VoiceDisclaimerMessage"]: """Parse voice disclaimer from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to the voice disclaimer - it will not be pulled.") + return {} + voice_settings = projection.get("channels", {}).get("voice", {}) voice_disclaimer = voice_settings.get("disclaimer", None) if not voice_disclaimer: @@ -268,6 +275,10 @@ class VoiceGreeting(ChannelGreeting): @classmethod def from_projection(cls, projection: dict) -> dict[str, "VoiceGreeting"]: """Parse voice greeting from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to the voice greeting - it will not be pulled.") + return {} + voice_config = projection.get("channels", {}).get("voice", {}).get("config", {}) or {} voice_greeting = voice_config.get("greeting", None) if not voice_greeting: @@ -293,6 +304,10 @@ class ChatGreeting(ChannelGreeting): @classmethod def from_projection(cls, projection: dict) -> dict[str, "ChatGreeting"]: """Parse chat greeting from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to the chat greeting - it will not be pulled.") + return {} + web_chat_settings = projection.get("channels", {}).get("webChat", {}) if not web_chat_settings.get("status", False): return {} @@ -401,6 +416,10 @@ class VoiceStylePrompt(ChannelStylePrompt): @classmethod def from_projection(cls, projection: dict) -> dict[str, "VoiceStylePrompt"]: """Parse voice style prompt from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to the voice style prompt - it will not be pulled.") + return {} + voice_config = projection.get("channels", {}).get("voice", {}).get("config", {}) or {} voice_style_prompt = voice_config.get("stylePrompt", None) if not voice_style_prompt: @@ -425,6 +444,10 @@ class ChatStylePrompt(ChannelStylePrompt): @classmethod def from_projection(cls, projection: dict) -> dict[str, "ChatStylePrompt"]: """Parse chat style prompt from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to the chat style prompt - it will not be pulled.") + return {} + web_chat_settings = projection.get("channels", {}).get("webChat", {}) if not web_chat_settings.get("status", False): return {} diff --git a/src/poly/resources/documents.py b/src/poly/resources/documents.py index a522d5ae..32880b6c 100644 --- a/src/poly/resources/documents.py +++ b/src/poly/resources/documents.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from functools import cached_property @@ -14,6 +15,8 @@ ) from poly.resources.resource import Resource, register_resource +logger = logging.getLogger(__name__) + PLATFORM_CONTEXT_FILE = "CONTEXT.MD" @@ -127,13 +130,17 @@ def discover_resources(base_path: str) -> list[str]: @classmethod def from_projection(cls, projection: dict) -> dict[str, "Document"]: documents = {} - for document_id, document_data in ( - projection.get("documents", {}).get("documents", {}).get("entities", {}).items() + documents_projection = ( + projection.get("documents", {}).get("documents", {}).get("entities", {}) + ) + # The projection carries the proto field name, "content". + if "documents" not in projection or any( + "content" not in doc for doc in documents_projection.values() ): - # The projection carries the proto field name, "content". An entity without it is - # one the current user lacks read permission for, so skip it. - if "content" not in document_data: - continue + logger.debug("No read access to context documents - they will not be pulled.") + return {} + + for document_id, document_data in documents_projection.items(): path = document_data.get("path", "") or "" name = path.removesuffix(".md").removesuffix(".MD") documents[document_id] = Document( diff --git a/src/poly/resources/entities.py b/src/poly/resources/entities.py index 144140a6..7f5dc0de 100644 --- a/src/poly/resources/entities.py +++ b/src/poly/resources/entities.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass, field from enum import Enum @@ -27,6 +28,8 @@ ) from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + class EntityType(str, Enum): """Enum representing the type of an Entity""" @@ -108,9 +111,14 @@ def __init__( def from_projection(cls, projection: dict) -> dict[str, "Entity"]: """Parse entities from a projection dict.""" entities = {} - for entity_id, entity_data in ( - projection.get("entities", {}).get("entities", {}).get("entities", {}).items() + entities_projection = projection.get("entities", {}).get("entities", {}).get("entities", {}) + if "entities" not in projection or any( + "type" not in entity for entity in entities_projection.values() ): + logger.debug("No read access to entities - they will not be pulled.") + return {} + + for entity_id, entity_data in entities_projection.items(): entities[entity_id] = cls( resource_id=entity_id, name=entity_data["name"], diff --git a/src/poly/resources/experimental_config.py b/src/poly/resources/experimental_config.py index d72c4da2..644a3c63 100644 --- a/src/poly/resources/experimental_config.py +++ b/src/poly/resources/experimental_config.py @@ -4,6 +4,7 @@ """ import json +import logging import os from dataclasses import dataclass, field @@ -11,6 +12,8 @@ from poly.handlers.protobuf.experimental_config_pb2 import ExperimentalConfig_UpdateConfig from poly.resources.resource import Resource, register_resource +logger = logging.getLogger(__name__) + @register_resource("experimental_config") @dataclass @@ -27,6 +30,14 @@ def from_projection(cls, projection: dict) -> dict[str, "ExperimentalConfig"]: .get("experimentalConfigs", {}) .get("entities", {}) ) + # "features" is optional in the API schema, so it can't distinguish an + # auth-filtered config from one with no features set. "active" always is. + if "experimentalConfig" not in projection or any( + "active" not in config for config in experimental_configs.values() + ): + logger.debug("No read access to experimental config - it will not be pulled.") + return {} + config_id, config_data = ( next(iter(experimental_configs.items()), ("default", {})) if experimental_configs diff --git a/src/poly/resources/flows.py b/src/poly/resources/flows.py index 6b2519e9..d3c9ba50 100644 --- a/src/poly/resources/flows.py +++ b/src/poly/resources/flows.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import math import os import re @@ -60,6 +61,9 @@ register_resource, ) +logger = logging.getLogger(__name__) + + FUNCTION_REGEX = re.compile(r"{{f[nt]:([\w-]+)}}") # Flow step names: alphanumeric, extended Latin (C0–024F, 1E00–1EFF), and _ &,/.- FLOW_STEP_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9\u00C0-\u024F\u1E00-\u1EFF_ &,/.\-]+$") @@ -109,6 +113,10 @@ def from_projection(cls, projection: dict) -> dict[str, "FlowConfig"]: """Parse flow configs from a projection dict.""" configs = {} flows = projection.get("flows", {}).get("flows", {}).get("entities", {}) + if "flows" not in projection or any("startStepId" not in flow for flow in flows.values()): + logger.debug("No read access to flows - they will not be pulled.") + return {} + for flow_id, flow_data in flows.items(): configs[flow_id] = cls( resource_id=flow_id, @@ -368,6 +376,14 @@ def from_projection(cls, projection: dict) -> dict[str, "FlowStep"]: """Parse flow steps (non-function) from a projection dict.""" steps = {} flows = projection.get("flows", {}).get("flows", {}).get("entities", {}) + if "flows" not in projection or any( + "type" not in step + for flow_data in flows.values() + for step in flow_data.get("steps", {}).get("entities", {}).values() + ): + logger.debug("No read access to flow steps - they will not be pulled.") + return {} + for flow_id, flow_data in flows.items(): for step_id, step in flow_data.get("steps", {}).get("entities", {}).items(): if step.get("type") == "function_step": @@ -1732,6 +1748,14 @@ def from_projection(cls, projection: dict) -> dict[str, "FunctionStep"]: """Parse function steps from a projection dict.""" func_steps = {} flows = projection.get("flows", {}).get("flows", {}).get("entities", {}) + if "flows" not in projection or any( + "type" not in step + for flow_data in flows.values() + for step in flow_data.get("steps", {}).get("entities", {}).values() + ): + logger.debug("No read access to flow steps - they will not be pulled.") + return {} + for flow_id, flow_data in flows.items(): for step_id, step in flow_data.get("steps", {}).get("entities", {}).items(): if step.get("type") != "function_step": diff --git a/src/poly/resources/function.py b/src/poly/resources/function.py index a0d890b6..16e87d92 100644 --- a/src/poly/resources/function.py +++ b/src/poly/resources/function.py @@ -282,7 +282,17 @@ def from_projection(cls, projection: dict) -> dict[str, "Function"]: """Parse functions from a projection dict.""" functions = {} + # Functions are drawn from three projection slices gated on two different + # permissions ("functions" for special and global, "jupiter_flows" for + # transition), so each slice is checked for read access independently - + # losing one must not hide the others. special_functions = projection.get("specialFunctions", {}) + if "specialFunctions" not in projection or any( + "code" not in func for func in special_functions.values() + ): + logger.debug("No read access to start/end functions - they will not be pulled.") + special_functions = {} + for func_type_key, func in special_functions.items(): if func.get("archived", False): continue @@ -316,6 +326,14 @@ def from_projection(cls, projection: dict) -> dict[str, "Function"]: ) flows = projection.get("flows", {}).get("flows", {}).get("entities", {}) + if "flows" not in projection or any( + "code" not in func + for flow_data in flows.values() + for func in flow_data.get("transitionFunctions", {}).get("entities", {}).values() + ): + logger.debug("No read access to transition functions - they will not be pulled.") + flows = {} + for flow_id, flow_data in flows.items(): for func_id, func in ( flow_data.get("transitionFunctions", {}).get("entities", {}).items() @@ -345,9 +363,14 @@ def from_projection(cls, projection: dict) -> dict[str, "Function"]: function_type=FunctionType.TRANSITION, ) - for func_id, func in ( - projection.get("functions", {}).get("functions", {}).get("entities", {}).items() + global_functions = projection.get("functions", {}).get("functions", {}).get("entities", {}) + if "functions" not in projection or any( + "code" not in func for func in global_functions.values() ): + logger.debug("No read access to functions - they will not be pulled.") + global_functions = {} + + for func_id, func in global_functions.items(): if func.get("archived", False): continue diff --git a/src/poly/resources/handoff.py b/src/poly/resources/handoff.py index 81289a35..02dc6018 100644 --- a/src/poly/resources/handoff.py +++ b/src/poly/resources/handoff.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar @@ -24,6 +25,8 @@ register_resource, ) +logger = logging.getLogger(__name__) + VALID_SIP_METHODS = ("invite", "refer", "bye") VALID_ENCRYPTION = ("TLS/SRTP", "UDP/RTP") @@ -107,6 +110,14 @@ def from_projection(cls, projection: dict) -> dict[str, "Handoff"]: """Parse handoffs from a projection dict.""" handoffs_projection = projection.get("handoff", {}).get("handoffs", {}).get("entities", {}) handoffs = {} + # Read access is checked before "active": an auth-filtered handoff has no + # "active" field, and must not be mistaken for a deactivated one. + if "handoff" not in projection or any( + "active" not in handoff for handoff in handoffs_projection.values() + ): + logger.debug("No read access to handoffs - they will not be pulled.") + return {} + for handoff_id, handoff_data in handoffs_projection.items(): if not handoff_data.get("active", False): continue diff --git a/src/poly/resources/keyphrase_boosting.py b/src/poly/resources/keyphrase_boosting.py index 52ef291e..6891d65d 100644 --- a/src/poly/resources/keyphrase_boosting.py +++ b/src/poly/resources/keyphrase_boosting.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -15,6 +16,8 @@ ) from poly.resources.resource import MultiResourceYamlResource, register_resource +logger = logging.getLogger(__name__) + VALID_LEVELS = ("default", "boosted", "maximum") @@ -45,12 +48,16 @@ def __init__( def from_projection(cls, projection: dict) -> dict[str, "KeyphraseBoosting"]: """Parse keyphrase boosting entries from a projection dict.""" keyphrases = {} - for kp_id, kp_data in ( - projection.get("keyphraseBoosting", {}) - .get("keyphraseBoosting", {}) - .get("entities", {}) - .items() + keyphrases_projection = ( + projection.get("keyphraseBoosting", {}).get("keyphraseBoosting", {}).get("entities", {}) + ) + if "keyphraseBoosting" not in projection or any( + "keyphrase" not in kp for kp in keyphrases_projection.values() ): + logger.debug("No read access to keyphrase boosting - it will not be pulled.") + return {} + + for kp_id, kp_data in keyphrases_projection.items(): keyphrases[kp_id] = cls( resource_id=kp_id, name=kp_data.get("keyphrase", ""), diff --git a/src/poly/resources/languages.py b/src/poly/resources/languages.py index b3dcf770..235b3360 100644 --- a/src/poly/resources/languages.py +++ b/src/poly/resources/languages.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar @@ -22,6 +23,8 @@ register_resource, ) +logger = logging.getLogger(__name__) + LANGUAGES_FILE = os.path.join("agent_settings", "languages.yaml") EMPTY_LANGUAGES = {"default_language": None, "additional_languages": []} @@ -77,6 +80,10 @@ def from_yaml_dict( @classmethod def from_projection(cls, projection: dict) -> dict[str, "DefaultLanguage"]: """Parse default language from a projection dict.""" + if "languages" not in projection: + logger.debug("No read access to the default language - it will not be pulled.") + return {} + language_data = projection.get("languages", {}) if not language_data: return {} @@ -195,9 +202,14 @@ def from_projection(cls, projection: dict) -> dict[str, "AdditionalLanguage"]: if not language_data: return {} additional_languages = {} - for lang_id, lang in ( - language_data.get("additionalLanguages", {}).get("entities", {}).items() + languages_projection = language_data.get("additionalLanguages", {}).get("entities", {}) + if "languages" not in projection or any( + "code" not in lang for lang in languages_projection.values() ): + logger.debug("No read access to additional languages - they will not be pulled.") + return {} + + for lang_id, lang in languages_projection.items(): code = lang.get("code") additional_languages[lang_id] = cls( resource_id=lang_id, diff --git a/src/poly/resources/phrase_filter.py b/src/poly/resources/phrase_filter.py index 69e65b1e..dea543d4 100644 --- a/src/poly/resources/phrase_filter.py +++ b/src/poly/resources/phrase_filter.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -17,6 +18,8 @@ from poly.resources.function import Function from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + @register_resource("phrase_filtering") @dataclass @@ -53,9 +56,16 @@ def __init__( def from_projection(cls, projection: dict) -> dict[str, "PhraseFilter"]: """Parse phrase filters from a projection dict.""" phrase_filters = {} - for filter_id, filter_data in ( - projection.get("stopKeywords", {}).get("filters", {}).get("entities", {}).items() + filters_projection = ( + projection.get("stopKeywords", {}).get("filters", {}).get("entities", {}) + ) + if "stopKeywords" not in projection or any( + "title" not in f for f in filters_projection.values() ): + logger.debug("No read access to phrase filtering - it will not be pulled.") + return {} + + for filter_id, filter_data in filters_projection.items(): references = filter_data.get("references", {}) global_functions = references.get("globalFunctions", {}) function_id = next(iter(global_functions), None) if global_functions else None diff --git a/src/poly/resources/pronunciation.py b/src/poly/resources/pronunciation.py index 0dc8d5c2..83f00ffe 100644 --- a/src/poly/resources/pronunciation.py +++ b/src/poly/resources/pronunciation.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -20,6 +21,8 @@ register_resource, ) +logger = logging.getLogger(__name__) + @register_resource("pronunciations") @dataclass @@ -60,12 +63,16 @@ def from_projection(cls, projection: dict) -> dict[str, "Pronunciation"]: """Parse pronunciations from a projection dict.""" pronunciations = {} index = 0 - for pronunciation_id, pronunciation_data in ( - projection.get("pronunciations", {}) - .get("pronunciations", {}) - .get("entities", {}) - .items() + pronunciations_projection = ( + projection.get("pronunciations", {}).get("pronunciations", {}).get("entities", {}) + ) + if "pronunciations" not in projection or any( + "regex" not in p for p in pronunciations_projection.values() ): + logger.debug("No read access to pronunciations - they will not be pulled.") + return {} + + for pronunciation_id, pronunciation_data in pronunciations_projection.items(): pronunciations[pronunciation_id] = cls( resource_id=pronunciation_id, name=pronunciation_data.get("name", ""), diff --git a/src/poly/resources/safety_filters.py b/src/poly/resources/safety_filters.py index 4597be59..abb10066 100644 --- a/src/poly/resources/safety_filters.py +++ b/src/poly/resources/safety_filters.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -18,6 +19,8 @@ ) from poly.resources.resource import ResourceMapping, YamlResource, register_resource +logger = logging.getLogger(__name__) + PRECISION_MAPPING = {"LOOSE": "lenient", "MEDIUM": "medium", "STRICT": "strict"} PRECISION_MAPPING_INVERSE = {v: k for k, v in PRECISION_MAPPING.items()} _AZURE_CATEGORY_KEYS = { @@ -265,6 +268,10 @@ class GeneralSafetyFilters(_BaseSafetyFilters): @classmethod def from_projection(cls, projection: dict) -> dict[str, "GeneralSafetyFilters"]: """Parse general safety filters from a projection dict.""" + if "contentFilterSettings" not in projection: + logger.debug("No read access to safety filters - it will not be pulled.") + return {} + data = projection.get("contentFilterSettings", {}) if not data: return {} @@ -361,6 +368,10 @@ class VoiceSafetyFilters(ChannelSafetyFilters): @classmethod def from_projection(cls, projection: dict) -> dict[str, "VoiceSafetyFilters"]: """Parse voice safety filters from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to voice safety filters - it will not be pulled.") + return {} + voice_config = projection.get("channels", {}).get("voice", {}).get("config", {}) or {} voice_safety_filters = voice_config.get("safetyFilters", None) if not voice_safety_filters: @@ -388,6 +399,10 @@ class ChatSafetyFilters(ChannelSafetyFilters): @classmethod def from_projection(cls, projection: dict) -> dict[str, "ChatSafetyFilters"]: """Parse chat safety filters from a projection dict.""" + if "channels" not in projection: + logger.debug("No read access to chat safety filters - it will not be pulled.") + return {} + web_chat_settings = projection.get("channels", {}).get("webChat", {}) if not web_chat_settings.get("status", False): return {} diff --git a/src/poly/resources/sms.py b/src/poly/resources/sms.py index 891f39ff..896b36f2 100644 --- a/src/poly/resources/sms.py +++ b/src/poly/resources/sms.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -18,6 +19,8 @@ ) from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + @dataclass class EnvPhoneNumbers: @@ -72,6 +75,14 @@ def from_projection(cls, projection: dict) -> dict[str, "SMSTemplate"]: projection.get("sms", {}).get("templates", {}).get("entities", {}) ) sms_templates = {} + # Read access is checked before "active": an auth-filtered template has no + # "active" field, and must not be mistaken for a deactivated one. + if "sms" not in projection or any( + "active" not in template for template in sms_templates_projection.values() + ): + logger.debug("No read access to SMS templates - they will not be pulled.") + return {} + for sms_template_id, sms_template_data in sms_templates_projection.items(): if not sms_template_data.get("active", False): continue diff --git a/src/poly/resources/test_suite.py b/src/poly/resources/test_suite.py index 8fa57eb1..369462d2 100644 --- a/src/poly/resources/test_suite.py +++ b/src/poly/resources/test_suite.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass, field from datetime import date, datetime, timezone @@ -45,6 +46,8 @@ from poly.resources.resource import ResourceMapping, SubResource, YamlResource, register_resource from poly.resources.variant_attributes import Variant +logger = logging.getLogger(__name__) + INTERNAL_TO_CHANNEL = { "chat.polyai": "voice", "webchat.polyai": "webchat", @@ -660,9 +663,16 @@ class TestCase(YamlResource): def from_projection(cls, projection: dict) -> dict[str, "TestCase"]: """Parse test cases from a projection dict.""" test_cases = {} - for test_case_id, test_case_data in ( - projection.get("testing", {}).get("testCases", {}).get("entities", {}).items() + test_cases_projection = ( + projection.get("testing", {}).get("testCases", {}).get("entities", {}) + ) + if "testing" not in projection or any( + "scenario" not in tc for tc in test_cases_projection.values() ): + logger.debug("No read access to test cases - they will not be pulled.") + return {} + + for test_case_id, test_case_data in test_cases_projection.items(): prompt_assertions = [] function_assertions = [] for assertion in test_case_data.get("assertions", []): diff --git a/src/poly/resources/topic.py b/src/poly/resources/topic.py index 28791537..a6ed29d0 100644 --- a/src/poly/resources/topic.py +++ b/src/poly/resources/topic.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os import re from dataclasses import dataclass @@ -17,6 +18,8 @@ ) from poly.resources.resource import ResourceMapping, YamlResource, register_resource +logger = logging.getLogger(__name__) + FUNCTION_REGEX = re.compile(r"{{fn:([\w-]+)}}") FLOW_FUNCTION_REGEX = re.compile(r"{{ft:([\w-]+)}}") @@ -55,9 +58,16 @@ def __init__( def from_projection(cls, projection: dict) -> dict[str, "Topic"]: """Parse topics from a projection dict.""" topics = {} - for topic_id, topic in ( - projection.get("knowledgeBase", {}).get("topics", {}).get("entities", {}).items() + topics_projection = ( + projection.get("knowledgeBase", {}).get("topics", {}).get("entities", {}) + ) + if "knowledgeBase" not in projection or any( + "content" not in topic for topic in topics_projection.values() ): + logger.debug("No read access to the knowledge base - it will not be pulled.") + return {} + + for topic_id, topic in topics_projection.items(): example_queries = topic.get("exampleQueries", []) queries = [ example_queries["query"] diff --git a/src/poly/resources/transcript_correction.py b/src/poly/resources/transcript_correction.py index cadbdc8a..cd4d7407 100644 --- a/src/poly/resources/transcript_correction.py +++ b/src/poly/resources/transcript_correction.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass, field from typing import ClassVar, Optional @@ -22,6 +23,8 @@ ) from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + VALID_REPLACEMENT_TYPES = ("full", "partial", "substring") @@ -92,6 +95,10 @@ def __init__( @classmethod def from_projection(cls, projection: dict) -> dict[str, "TranscriptCorrection"]: """Parse transcript corrections from a projection dict.""" + if "transcriptCorrections" not in projection: + logger.debug("No read access to transcript corrections - it will not be pulled.") + return {} + corrections = {} for correction_id, correction_data in ( projection.get("transcriptCorrections", {}) diff --git a/src/poly/resources/translations.py b/src/poly/resources/translations.py index a4637da3..66c2aad6 100644 --- a/src/poly/resources/translations.py +++ b/src/poly/resources/translations.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass from typing import ClassVar, Optional @@ -18,6 +19,8 @@ from poly.resources.languages import AdditionalLanguage, DefaultLanguage from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + @register_resource("translations") @dataclass @@ -30,12 +33,23 @@ class Translation(MultiResourceYamlResource): @classmethod def from_projection(cls, projection: dict) -> dict[str, "Translation"]: """Parse translations from a projection dict.""" + if "translations" not in projection: + logger.debug("No read access to translations - it will not be pulled.") + return {} + translations_data = ( projection.get("translations", {}).get("translations", {}).get("entities", {}) ) if not translations_data: return {} + if any( + "translations" not in translation_data + for translation_data in translations_data.values() + ): + logger.debug("No read access to translations - they will not be pulled.") + return {} + translations = {} for translation_id, translation_data in translations_data.items(): translations[translation_id] = cls( diff --git a/src/poly/resources/variable.py b/src/poly/resources/variable.py index a6967613..81b22c97 100644 --- a/src/poly/resources/variable.py +++ b/src/poly/resources/variable.py @@ -51,6 +51,10 @@ def __init__( @classmethod def from_projection(cls, projection: dict) -> dict[str, "Variable"]: """Parse variables from a projection dict.""" + if "variables" not in projection: + logger.debug("No read access to variables - it will not be pulled.") + return {} + variables = {} variables_data = projection.get("variables", {}).get("variables", {}).get("entities", {}) for var_id, var_data in variables_data.items(): diff --git a/src/poly/resources/variant_attributes.py b/src/poly/resources/variant_attributes.py index a67cfdb3..63b040bb 100644 --- a/src/poly/resources/variant_attributes.py +++ b/src/poly/resources/variant_attributes.py @@ -3,6 +3,7 @@ Copyright PolyAI Limited """ +import logging import os from dataclasses import dataclass, field from typing import ClassVar @@ -20,6 +21,8 @@ ) from poly.resources.resource import MultiResourceYamlResource, ResourceMapping, register_resource +logger = logging.getLogger(__name__) + @register_resource("variants") @dataclass @@ -62,9 +65,19 @@ def from_yaml_dict(cls, yaml_dict: dict, resource_id: str, name: str, **kwargs) def from_projection(cls, projection: dict) -> dict[str, "Variant"]: """Parse variants from a projection dict.""" variants = {} - for variant_id, variant_data in ( - projection.get("variantManagement", {}).get("variants", {}).get("entities", {}).items() + variants_projection = ( + projection.get("variantManagement", {}).get("variants", {}).get("entities", {}) + ) + # Guard on "isDefault", not "name": names are deliberately exposed to + # filtered readers so test cases can resolve their variant, so a name + # no longer distinguishes a readable variant from a withheld one. + if "variantManagement" not in projection or any( + "isDefault" not in variant for variant in variants_projection.values() ): + logger.debug("No read access to variants - they will not be pulled.") + return {} + + for variant_id, variant_data in variants_projection.items(): variants[variant_id] = cls( resource_id=variant_id, name=variant_data["name"], @@ -215,13 +228,19 @@ def from_yaml_dict( def from_projection(cls, projection: dict) -> dict[str, "VariantAttribute"]: """Parse variant attributes from a projection dict.""" variant_attributes = {} - for attribute_id, attribute_data in ( - projection.get("variantManagement", {}) - .get("attributes", {}) - .get("entities", {}) - .items() + attributes_projection = ( + projection.get("variantManagement", {}).get("attributes", {}).get("entities", {}) + ) + # "archived" is optional in the API schema, so it can't distinguish an + # auth-filtered attribute from an unarchived one. "type" always is. + if "variantManagement" not in projection or any( + "type" not in attribute for attribute in attributes_projection.values() ): - if attribute_data["archived"]: + logger.debug("No read access to variant attributes - they will not be pulled.") + return {} + + for attribute_id, attribute_data in attributes_projection.items(): + if attribute_data.get("archived"): continue variant_attributes[attribute_id] = cls( resource_id=attribute_id, name=attribute_data["name"], mappings={} diff --git a/src/poly/tests/project_test.py b/src/poly/tests/project_test.py index b2f79067..1c036dcc 100644 --- a/src/poly/tests/project_test.py +++ b/src/poly/tests/project_test.py @@ -3878,6 +3878,40 @@ def test_absent_multi_resource_type_deleted_on_pull(self): "delete_resource should be called for every entity when Entity type is absent", ) + def test_absent_multi_resource_type_deletion_reaches_disk(self): + """The deletions are batched into the file cache, so they have to be flushed. + + Asserting only that delete_resource was called says nothing about whether the + pruned file was ever written - the cache is discarded when the pull returns and + the entities stay on disk, needing a second pull to clear. + """ + project = AgentStudioProject.from_dict(PROJECT_DATA, TEST_DIR) + incoming_resources = deepcopy(project.resources) + entity_names = {res.name for res in incoming_resources[Entity].values()} + self.assertGreater(len(entity_names), 0) + + del incoming_resources[Entity] + self.mock_api_handler.pull_resources.return_value = (incoming_resources, {}) + + MultiResourceYamlResource._file_cache.clear() + project.pull_project(force=False) + cache_after_pull = dict(MultiResourceYamlResource._file_cache) + MultiResourceYamlResource._file_cache.clear() + + entities_file = os.path.join(TEST_DIR, "config", "entities.yaml") + written = [ + call[0][0] + for call in self.mock_save_to_file.call_args_list + if call[0][1] == entities_file + ] + self.assertTrue(written, "the pruned entities file should have been written to disk") + for name in entity_names: + self.assertNotIn(name, written[-1]) + + # A cached entry carries the pre-write mtime, so anything left behind makes later + # reads in this process see a file state that is not on disk. + self.assertEqual(cache_after_pull, {}) + def test_not_loaded_resource_type_not_deleted_on_pull(self): """When a resource type is in _not_loaded_resources, it should NOT be deleted even if absent from incoming_resources. This prevents spurious deletions of diff --git a/src/poly/tests/resources_test.py b/src/poly/tests/resources_test.py index 646f3885..ed01415c 100644 --- a/src/poly/tests/resources_test.py +++ b/src/poly/tests/resources_test.py @@ -10518,8 +10518,13 @@ def test_parses_document_fields(self): self.assertEqual(document.name, "faq") self.assertEqual(document.contents, "Frequently asked questions") - def test_skips_document_without_content(self): - """A document missing 'content' means the user lacks read permission, so it's skipped.""" + def test_skips_all_documents_when_any_is_unreadable(self): + """A document missing 'content' means the user lacks read permission. + + Auth filtering is per-slice, so one unreadable document means the whole + slice was filtered and none of it is represented locally - a partially + populated context/ directory would be worse than an absent one. + """ projection = { "documents": { "documents": { @@ -10530,8 +10535,7 @@ def test_skips_document_without_content(self): } } } - documents = Document.from_projection(projection) - self.assertEqual(list(documents), ["DOC-2"]) + self.assertEqual(Document.from_projection(projection), {}) def test_keeps_document_with_empty_content(self): """An empty 'content' is a readable but empty document, not a permission failure.""" diff --git a/src/poly/tests/slim_projection_test.py b/src/poly/tests/slim_projection_test.py new file mode 100644 index 00000000..1a09feeb --- /dev/null +++ b/src/poly/tests/slim_projection_test.py @@ -0,0 +1,271 @@ +"""Tests for handling auth-filtered ("slim") projections + +The API filters projections per slice. A user without read access to a slice +gets a skeleton carrying only identity fields - ids, names, references - with +the substantive fields stripped. A resource the user cannot read is not +represented locally at all: no file, no baseline entry. It looks like a remote +delete, which is honest, since from that user's vantage point it does not exist. + +Copyright PolyAI Limited +""" + +import unittest + +import poly.resources # noqa: F401 - triggers resource registration +from poly.resources.function import Function +from poly.resources.resource import PROJECTION_REGISTRY, RESOURCE_CLASS_TO_NAME +from poly.resources.sms import SMSTemplate +from poly.resources.topic import Topic + +# A projection where every slice came back auth-filtered, built from the fields +# each slice declares in its alwaysPresentJsonPaths allow-list. Slices whose +# allow-list is empty are omitted entirely, as the API omits them. +SKELETON_PROJECTION = { + "documents": { + "documents": {"ids": ["CONTEXT.MD"], "entities": {"CONTEXT.MD": {"path": "CONTEXT.MD"}}} + }, + "knowledgeBase": { + "topics": { + "ids": ["TOPIC-1"], + "entities": {"TOPIC-1": {"id": "TOPIC-1", "name": "billing", "references": {}}}, + }, + "uninstantiatedTopics": {"ids": [], "entities": {}}, + }, + "entities": { + "entities": { + "ids": ["ENTITY-1"], + "entities": {"ENTITY-1": {"id": "ENTITY-1", "name": "customer_name"}}, + } + }, + "functions": { + "functions": { + "ids": ["FUNCTION-1"], + "entities": { + "FUNCTION-1": { + "id": "FUNCTION-1", + "name": "lookup_order", + "references": {}, + "parameters": {"ids": ["P1"], "entities": {"P1": {"id": "P1", "name": "p"}}}, + "latencyControl": {"delayResponses": {"ids": []}}, + } + }, + } + }, + "specialFunctions": { + "startFunction": {"id": "SF-1", "parameters": {"ids": [], "entities": {}}}, + "endFunction": {"id": "EF-1", "parameters": {"ids": [], "entities": {}}}, + }, + "flows": { + "flows": { + "ids": ["FLOW-1"], + "entities": { + "FLOW-1": { + "id": "FLOW-1", + "name": "MyFlow", + "steps": { + "entities": { + "STEP-1": {"id": "STEP-1", "name": "greet", "references": {}}, + "STEP-2": {"id": "STEP-2", "name": "lookup", "references": {}}, + } + }, + "transitionFunctions": { + "ids": ["TF-1"], + "entities": {"TF-1": {"id": "TF-1", "name": "go", "references": {}}}, + }, + } + }, + } + }, + "handoff": { + "handoffs": { + "ids": ["HO-1"], + "entities": {"HO-1": {"id": "HO-1", "name": "agent", "references": {}}}, + } + }, + "sms": { + "templates": { + "ids": ["SMS-1"], + "entities": {"SMS-1": {"id": "SMS-1", "name": "confirm", "references": {}}}, + } + }, + "variables": { + "variables": { + "ids": ["VAR-1"], + "entities": {"VAR-1": {"id": "VAR-1", "name": "order_id", "references": {}}}, + } + }, + "variantManagement": { + "attributes": { + "ids": ["ATTR-1"], + "entities": {"ATTR-1": {"id": "ATTR-1", "name": "brand", "references": {}}}, + }, + # Variant names are exposed to filtered readers so that test cases, gated + # on a different permission, can resolve the variant they run against. + "variants": {"ids": ["V-1"], "entities": {"V-1": {"id": "V-1", "name": "Default"}}}, + "variantAttributeValues": {"ids": ["V-1"], "entities": {"V-1": {"id": "V-1"}}}, + }, + # Likewise translation keys, which topics and behaviour rules embed as {{tn:}}. + "translations": { + "translations": { + "ids": ["TN-1"], + "entities": {"TN-1": {"id": "TN-1", "translationKey": "greeting"}}, + } + }, + "testing": {"testCases": {"ids": ["TC-1"], "entities": {"TC-1": {"id": "TC-1"}}}}, + "pronunciations": {"pronunciations": {"ids": ["PR-1"], "entities": {"PR-1": {"id": "PR-1"}}}}, + "keyphraseBoosting": { + "keyphraseBoosting": {"ids": ["KB-1"], "entities": {"KB-1": {"id": "KB-1"}}} + }, + "stopKeywords": {"filters": {"ids": ["SK-1"], "entities": {"SK-1": {"id": "SK-1"}}}}, + "experimentalConfig": { + "experimentalConfigs": {"ids": ["default"], "entities": {"default": {"id": "default"}}} + }, + "channels": {"webChat": {"status": 1}}, + "languages": {"additionalLanguages": {"ids": ["fr"], "entities": {"fr": {"id": "fr"}}}}, + "csat": {"enabled": True}, + "webchatCsat": { + "enabled": True, + "questions": {"ids": ["Q1"], "entities": {"Q1": {"id": "Q1"}}}, + }, + "childOverwrites": {"knowledgeBase": {"topics": {"ids": [], "entities": {}}}}, +} + +# Variables and their names are the whole of the Variable resource, and the +# variables slice keeps both, so a filtered slice is indistinguishable from - and +# just as usable as - a full one. Nothing else should survive. +TYPES_READABLE_FROM_SKELETON = {"variables"} + + +class SkeletonProjectionParsing(unittest.TestCase): + """Every registered resource must tolerate an auth-filtered projection.""" + + def test_no_resource_type_raises(self): + """A slim projection must never abort the pull.""" + for resource_cls in PROJECTION_REGISTRY: + name = RESOURCE_CLASS_TO_NAME[resource_cls] + with self.subTest(resource=name): + resource_cls.from_projection(SKELETON_PROJECTION) + + def test_only_fully_readable_types_are_represented(self): + """Anything the user cannot read is hidden rather than partly built.""" + represented = { + RESOURCE_CLASS_TO_NAME[cls] + for cls in PROJECTION_REGISTRY + if cls.from_projection(SKELETON_PROJECTION) + } + self.assertEqual(represented, TYPES_READABLE_FROM_SKELETON) + + def test_represented_resources_survive_downstream_operations(self): + """Whatever is kept must be usable, not a half-built object. + + compute_hash and file_path run on every pull and push, so a resource + that parses but blows up here fails far from the cause - which is how + filtered entities and additional_languages used to surface. + """ + for resource_cls in PROJECTION_REGISTRY: + name = RESOURCE_CLASS_TO_NAME[resource_cls] + for resource in resource_cls.from_projection(SKELETON_PROJECTION).values(): + with self.subTest(resource=name): + resource.compute_hash() + resource.validate() + + +class FalsyGuardValuesAreReadable(unittest.TestCase): + """The guards test for a field's presence, never its truthiness. + + An empty string or False is readable data. Getting this wrong would hide + resources the user can perfectly well read - the failure mode this whole + change exists to prevent, just in the opposite direction. + """ + + def test_empty_topic_content_is_kept(self): + projection = { + "knowledgeBase": { + "topics": {"entities": {"TOPIC-1": {"name": "empty", "content": "", "actions": ""}}} + } + } + self.assertEqual(list(Topic.from_projection(projection)), ["TOPIC-1"]) + + def test_inactive_sms_template_is_a_read_not_a_hide(self): + """active=False is a deactivated template, not a filtered one. + + Both end up absent, but only the filtered case should log - so the + guard must see the field and fall through to the "active" check. + """ + projection = { + "sms": {"templates": {"entities": {"SMS-1": {"name": "old", "active": False}}}} + } + with self.assertNoLogs("poly.resources.sms", level="DEBUG"): + self.assertEqual(SMSTemplate.from_projection(projection), {}) + + +class FunctionSliceIndependence(unittest.TestCase): + """Functions are drawn from three slices gated on two different permissions. + + "functions" gates the special and global function slices; "jupiter_flows" + gates transition functions. Losing one must not hide the others. + """ + + @staticmethod + def _full_function(func_id, name): + return {"id": func_id, "name": name, "description": "", "code": "pass"} + + def test_unreadable_transition_functions_keep_global_functions(self): + projection = { + "functions": { + "functions": {"entities": {"FN-1": self._full_function("FN-1", "readable")}} + }, + "flows": { + "flows": { + "entities": { + "FLOW-1": { + "id": "FLOW-1", + "name": "MyFlow", + "transitionFunctions": { + "entities": {"TF-1": {"id": "TF-1", "name": "hidden"}} + }, + } + } + } + }, + } + functions = Function.from_projection(projection) + self.assertEqual(list(functions), ["FN-1"]) + + def test_unreadable_global_functions_keep_transition_functions(self): + projection = { + "functions": {"functions": {"entities": {"FN-1": {"id": "FN-1", "name": "hidden"}}}}, + "flows": { + "flows": { + "entities": { + "FLOW-1": { + "id": "FLOW-1", + "name": "MyFlow", + "transitionFunctions": { + "entities": {"TF-1": self._full_function("TF-1", "readable")} + }, + } + } + } + }, + } + functions = Function.from_projection(projection) + self.assertEqual(list(functions), ["TF-1"]) + + +class WithheldSlicesAreReported(unittest.TestCase): + """A withheld slice arrives as nothing at all, so the absence is the signal.""" + + def test_absent_slice_logs_and_yields_nothing(self): + with self.assertLogs("poly.resources.topic", level="DEBUG"): + self.assertEqual(Topic.from_projection({}), {}) + + def test_present_but_empty_slice_is_silent(self): + """An authorised slice with no entities is emitted, just empty - not withheld.""" + projection = {"knowledgeBase": {"topics": {"entities": {}}}} + with self.assertNoLogs("poly.resources.topic", level="DEBUG"): + self.assertEqual(Topic.from_projection(projection), {}) + + +if __name__ == "__main__": + unittest.main()