From eedf77ffd1999f146e06098bdf36fe8873bc812d Mon Sep 17 00:00:00 2001 From: David Tauriello Date: Tue, 9 Jul 2024 16:30:57 -0400 Subject: [PATCH] Update XULE 30013 - use a new ModelManager when loading instances and taxonomies to prevent Arelle from using the loaded models when validating. - support for trait properties --- plugin/xule/XuleContext.py | 6 ++-- plugin/xule/XuleInstanceFunctions.py | 11 +++--- plugin/xule/XuleProperties.py | 2 ++ plugin/xule/XulePropertiesTrait.py | 52 ++++++++++++++++++++++++++++ plugin/xule/XuleUtility.py | 34 +++++++++++++++++- plugin/xule/XuleValidate.py | 3 +- plugin/xule/__init__.py | 41 +++++++--------------- plugin/xule/version.json | 2 +- 8 files changed, 112 insertions(+), 39 deletions(-) create mode 100644 plugin/xule/XulePropertiesTrait.py diff --git a/plugin/xule/XuleContext.py b/plugin/xule/XuleContext.py index 2b368b65f..88ca2e99c 100644 --- a/plugin/xule/XuleContext.py +++ b/plugin/xule/XuleContext.py @@ -30,7 +30,6 @@ from .XuleValue import XuleValue, XuleValueSet from . import XuleUtility as xu from arelle import FileSource -from arelle import ModelManager from queue import Queue from multiprocessing import Queue as M_Queue, Manager, cpu_count from collections import defaultdict, OrderedDict @@ -288,8 +287,9 @@ def get_other_taxonomies(self, taxonomy_url): start = datetime.datetime.today() rules_taxonomy_filesource = FileSource.openFileSource(taxonomy_url, self.cntlr) #modelManager = ModelManager.initialize(self.cntlr) - modelManager = self.cntlr.modelManager - modelXbrl = modelManager.load(rules_taxonomy_filesource) + #modelManager = self.cntlr.modelManager + import_model_manager = xu.get_model_manager_for_import(self.cntlr) + modelXbrl = import_model_manager.load(rules_taxonomy_filesource) if 'IOerror' in modelXbrl.errors: raise XuleProcessingError(_("Taxonomy {} not found.".format(taxonomy_url))) end = datetime.datetime.today() diff --git a/plugin/xule/XuleInstanceFunctions.py b/plugin/xule/XuleInstanceFunctions.py index 2746a8962..90d8260b5 100644 --- a/plugin/xule/XuleInstanceFunctions.py +++ b/plugin/xule/XuleInstanceFunctions.py @@ -26,7 +26,8 @@ from . import XuleValue as xv from .XuleRunTime import XuleProcessingError from .XuleModelIndexer import index_model -from arelle import ModelXbrl, ModelDocument, FileSource, ModelManager, ModelValue +from .XuleUtility import get_model_manager_for_import +from arelle import ModelXbrl, ModelDocument, FileSource, ModelValue from arelle.PrototypeInstanceObject import DimValuePrototype import datetime import os.path @@ -50,8 +51,9 @@ def func_instance(xule_context, *args): start = datetime.datetime.today() instance_filesource = FileSource.openFileSource(instance_url.value, xule_context.global_context.cntlr) #modelManager = ModelManager.initialize(xule_context.global_context.cntlr) - modelManager = xule_context.global_context.cntlr.modelManager - instance_model = modelManager.load(instance_filesource) + #modelManager = xule_context.global_context.cntlr.modelManager + import_model_manager = get_model_manager_for_import(xule_context.global_context.cntlr) + instance_model = import_model_manager.load(instance_filesource) if 'IOerror' in instance_model.errors: raise XuleProcessingError(_("Instance {} not found.".format(instance_url))) end = datetime.datetime.today() @@ -100,7 +102,8 @@ def func_new_instance(xule_context, *args): raise XuleProcessingError(_("List of arcrole/role refs for new_instance() fucntion must be a set, list, string or uri, found{}".format(args[2].type)), xule_context) # Create the model - model_manager = xule_context.model.modelManager + #model_manager = xule_context.model.modelManager + model_manager = get_model_manager_for_import(xule_context.global_context.cntlr) # Need a filesource for the new instance document. Creating a temporary directory to put the file in temp_directory = tempfile.TemporaryDirectory() inst_file_name = os.path.join(temp_directory.name, f'{inst_name}.json') diff --git a/plugin/xule/XuleProperties.py b/plugin/xule/XuleProperties.py index d15aeb11c..a77eb5741 100644 --- a/plugin/xule/XuleProperties.py +++ b/plugin/xule/XuleProperties.py @@ -3073,6 +3073,8 @@ def property_arcroles(xule_context, object_value, *args): '_list-properties': (property_list_properties, 0, ('unbound',), True), } +from . import XulePropertiesTrait +PROPERTIES.update(XulePropertiesTrait.trait_properties()) #Network tuple NETWORK_INFO = 0 diff --git a/plugin/xule/XulePropertiesTrait.py b/plugin/xule/XulePropertiesTrait.py new file mode 100644 index 000000000..a28b651ab --- /dev/null +++ b/plugin/xule/XulePropertiesTrait.py @@ -0,0 +1,52 @@ + +from . import XuleProperties as xp +from . import XuleValue as xv + +_TRAIT_CONCEPT_2021 = 'http://www.xbrl.org/2021/arcrole/trait-concept' +_TRAIT_CONCEPT_2023 = 'http://www.xbrl.org/2023/arcrole/trait-concept' +_CLASS_SUBCLASS_2021 = 'http://www.xbrl.org/2021/arcrole/class-subclass' +_CLASS_SUBCLASS_2023 = 'http://www.xbrl.org/2023/arcrole/class-subclass' + +def property_traits(xule_context, object_value, *args): + + traits = set() + concepts = {object_value.value,} # add the concept + concepts |= get_ancestors(object_value.value, (_CLASS_SUBCLASS_2021, _CLASS_SUBCLASS_2023), False) + for concept in concepts: + traits |= get_ancestors(concept, (_TRAIT_CONCEPT_2021, _TRAIT_CONCEPT_2023)) + + return_values = frozenset(xv.XuleValue(xule_context, trait, 'concept') for trait in traits) + return xv.XuleValue(xule_context, return_values, 'set', shadow_collection=frozenset(traits)) + +def get_ancestors(concept, arc_roles, parent_only=True): + dts = concept.modelXbrl + network_infos = [] + for arc_role in arc_roles: + network_infos.extend(xp.get_base_set_info(dts, arc_role)) + + ancestors = set() + + for network_info in network_infos: + network = dts.relationshipSet( + network_info[xp.NETWORK_ARCROLE], + network_info[xp.NETWORK_ROLE], + network_info[xp.NETWORK_LINK], + network_info[xp.NETWORK_ARC]) + for parent in network.toModelObject(concept): + ancestors.add(parent.fromModelObject) + + if not parent_only: + next_ancestors = set() + for ancestor in ancestors: + next_ancestors |= get_ancestors(ancestor, arc_roles, parent_only) + ancestors |= next_ancestors + + return ancestors + +def trait_properties(): + props = { + #NEW PROPERTIES + 'traits': (property_traits, 0, ('concept',), False), + } + + return props \ No newline at end of file diff --git a/plugin/xule/XuleUtility.py b/plugin/xule/XuleUtility.py index 0dab5c00b..2338317f5 100644 --- a/plugin/xule/XuleUtility.py +++ b/plugin/xule/XuleUtility.py @@ -23,6 +23,7 @@ DOCSKIP """ from arelle.ModelRelationshipSet import ModelRelationshipSet +from arelle import ModelManager import collections import json import os @@ -34,10 +35,29 @@ from . import XuleConstants as xc from . import XuleRunTime as xrt from .XuleRunTime import XuleProcessingError + # XuleValue is a module. It is imported in the _imports() function to avoid a circular relative import error. XuleValue = None XuleProperties = None +class XuleVars: + + class XuleVarContainer: + pass + + @classmethod + def set(cls, cntlr, name, value): + if not hasattr(cntlr, 'xule_vars'): + cntlr.xule_vars = dict() + + cntlr.xule_vars[name] = value + + @classmethod + def get(cls, cntlr, name): + if hasattr(cntlr, 'xule_vars'): + return cntlr.xule_vars.get(name) + else: + return None def version(ruleset_version=False): # version_type determines if looking at the processor version or the ruleset builder version. @@ -185,6 +205,8 @@ def resolve_role(role_value, role_type, dts, xule_context): and error is raise. This allows short form of an arcrole i.e parent-child. """ _imports() + if dts is None: + raise XuleProcessingError(("Not able to resolve role/arcrole '{}' when there is no taxonomy".format(role_value.value.localName))) if role_value.value.prefix is not None: raise XuleProcessingError(_("Invalid {}. {} should be a string, uri or short role name. Found qname with value of {}".format(role_type, role_type.capitalize(), role_value.format_value()))) else: @@ -441,4 +463,14 @@ def get_rule_set_compatibility_version(): compatibility_json = json.load(compatibility_file) return compatibility_json.get('versionControl') except ValueError: - raise XuleProcessingError(_("Rule set compatibility file does not appear to be a valid JSON file. File: {}".format(xc.RULE_SET_COMPATIBILITY_FILE))) \ No newline at end of file + raise XuleProcessingError(_("Rule set compatibility file does not appear to be a valid JSON file. File: {}".format(xc.RULE_SET_COMPATIBILITY_FILE))) + +def get_model_manager_for_import(cntlr): + import_model_manager = XuleVars.get(cntlr, 'importModelManager') + if import_model_manager is None: + import_model_manager = ModelManager.initialize(cntlr) + import_model_manager.loadCustomTransforms() + #import_model_manager.customTransforms = cntlr.modelManager.customTransforms + XuleVars.set(cntlr, 'importModelManager', import_model_manager) + + return import_model_manager diff --git a/plugin/xule/XuleValidate.py b/plugin/xule/XuleValidate.py index e3c579cc2..8ef4b6391 100644 --- a/plugin/xule/XuleValidate.py +++ b/plugin/xule/XuleValidate.py @@ -159,7 +159,8 @@ def _get_taxonomy_model(self, taxonomy_url, namespace): start = datetime.datetime.today() rules_taxonomy_filesource = FileSource.openFileSource(taxonomy_url, self.cntlr) #modelManager = ModelManager.initialize(self.cntlr) - modelManager = self.cntlr.modelManager + #modelManager = self.cntlr.modelManager + modelManager = xu.get_model_manager_for_import(self.cntlr) modelXbrl = modelManager.load(rules_taxonomy_filesource) if len({'IOerror','FileNotLoadable'} & set(modelXbrl.errors)) > 0: modelXbrl.error("TaxonomyLoadError","Cannot open file {} with namespace {}.".format(taxonomy_url, namespace)) diff --git a/plugin/xule/__init__.py b/plugin/xule/__init__.py index c0e745b1a..06ae2d63e 100644 --- a/plugin/xule/__init__.py +++ b/plugin/xule/__init__.py @@ -84,25 +84,6 @@ class EmptyOptions: pass -class XuleVars: - - class XuleVarContainer: - pass - - @classmethod - def set(cls, cntlr, name, value): - if not hasattr(cntlr, 'xule_vars'): - cntlr.xule_vars = dict() - - cntlr.xule_vars[name] = value - - @classmethod - def get(cls, cntlr, name): - if hasattr(cntlr, 'xule_vars'): - return cntlr.xule_vars.get(name) - else: - return None - def xuleMenuTools(cntlr, menu): import tkinter @@ -118,7 +99,7 @@ def xuleMenuTools(cntlr, menu): activate_xule_label = "Activate" def turnOnXule(): # The validation menu hook is hit before the tools menu hook. the XuleVars for 'validate_menu' is set in the validation menu hook. - validate_menu = XuleVars.get(cntlr, 'validate_menu') + validate_menu = xu.XuleVars.get(cntlr, 'validate_menu') addValidateMenuTools(cntlr, validate_menu, 'Xule', _xule_rule_set_map_name) menu.delete('Xule') xule_menu = addMenuTools(cntlr, menu, 'Xule', '', __file__, _xule_rule_set_map_name, _latest_map_name) @@ -127,7 +108,7 @@ def turnOffXule(): validate_menu.delete('Xule Rules') menu.delete('Xule') new_xule_menu = tkinter.Menu(menu, tearoff=0) - new_xule_menu.add_command(label=activate_xule_label, underline=0, command=XuleVars.get(cntlr, 'activate_xule_function')) + new_xule_menu.add_command(label=activate_xule_label, underline=0, command=xu.XuleVars.get(cntlr, 'activate_xule_function')) menu.add_cascade(label=_("Xule"), menu=new_xule_menu, underline=0) cntlr.config['xule_activated'] = False @@ -136,7 +117,7 @@ def turnOffXule(): xule_menu = tkinter.Menu(menu, tearoff=0) xule_menu.add_command(label=activate_xule_label, underline=0, command=turnOnXule) - XuleVars.set(cntlr, 'activate_xule_function', turnOnXule) + xu.XuleVars.set(cntlr, 'activate_xule_function', turnOnXule) menu.add_cascade(label=_("Xule"), menu=xule_menu, underline=0) @@ -339,7 +320,7 @@ def useEnteredValue(main_window, entry_value, replace): def xuleValidateMenuTools(cntlr, validateMenu, *args, **kwargs): # Save the validationMenu object. - XuleVars.set(cntlr, 'validate_menu', validateMenu) + xu.XuleVars.set(cntlr, 'validate_menu', validateMenu) def addValidateMenuTools(cntlr, validateMenu, name, map_name): # Extend menu with an item for the save infoset plugin @@ -368,7 +349,7 @@ def isXuleDirect(cntlr): global _is_xule_direct if _is_xule_direct is None: _is_xule_direct = False - for plugin_command in getattr(XuleVars.get(cntlr, 'options'), 'plugins', '').split('|'): + for plugin_command in getattr(xu.XuleVars.get(cntlr, 'options'), 'plugins', '').split('|'): if plugin_command.lower().strip().endswith('xule'): _is_xule_direct = True @@ -672,7 +653,7 @@ def xuleCmdOptions(parser): help=_("Validate ruleset")) def saveOptions(cntlr, options, **kwargs): - XuleVars.set(cntlr, 'options', options) + xu.XuleVars.set(cntlr, 'options', options) # Save the options in the xuleparser if xp is not None: xp.setOptions(options) @@ -879,8 +860,10 @@ def xuleCmdUtilityRun(cntlr, options, **kwargs): input_file_name = input_file_name.strip() print("Processing filing", input_file_name) filing_filesource = FileSource.openFileSource(input_file_name, cntlr) + # TODO - #29 #modelManager = ModelManager.initialize(cntlr) - modelManager = cntlr.modelManager + #modelManager = cntlr.modelManager + modelManager = xu.get_model_manager_for_import(cntlr) modelXbrl = modelManager.load(filing_filesource) # Update options new_options = copy.copy(options) @@ -1002,7 +985,7 @@ def xuleValidate(val): # If the controller is not there, pull it from the model. this happens when running as a web service in multi processing mode. _cntlr = _cntlr or val.modelXbrl.modelManager.cntlr - options = XuleVars.get(_cntlr, 'options') + options = xu.XuleVars.get(_cntlr, 'options') if options is None: options = EmptyOptions() @@ -1036,7 +1019,7 @@ def xuleTestXbrlLoaded(modelTestcase, modelXbrl, testVariation): global _test_start global _test_variation_name - if getattr(XuleVars.get(_cntlr,'options'), 'xule_test_debug', False): + if getattr(xu.XuleVars.get(_cntlr,'options'), 'xule_test_debug', False): _test_start = datetime.datetime.today() _test_variation_name = testVariation.id print('{}: Testcase variation {} started'.format(_test_start.isoformat(sep=' '), testVariation.id)) @@ -1046,7 +1029,7 @@ def xuleTestValidated(modelTestcase, modelXbrl): global _test_start global _test_variation_name - if getattr(XuleVars.get(_cntlr, 'options'), 'xule_test_debug', False): + if getattr(xu.XuleVars.get(_cntlr, 'options'), 'xule_test_debug', False): if _test_start is not None: test_end = datetime.datetime.today() print("{}: Test variation {} finished. in {} ".format(test_end.isoformat(sep=' '), _test_variation_name, (test_end - _test_start))) diff --git a/plugin/xule/version.json b/plugin/xule/version.json index f8c9062b6..b5074a5d1 100644 --- a/plugin/xule/version.json +++ b/plugin/xule/version.json @@ -1,4 +1,4 @@ { - "version": "30003", + "version": "30013", "ruleset_version": "23752" } \ No newline at end of file