diff --git a/plugins/fabric-cli/skills/fabric-cli/scripts/README.md b/plugins/fabric-cli/skills/fabric-cli/scripts/README.md index cdfea233..6dc47219 100644 --- a/plugins/fabric-cli/skills/fabric-cli/scripts/README.md +++ b/plugins/fabric-cli/skills/fabric-cli/scripts/README.md @@ -119,6 +119,31 @@ python3 export_semantic_model_as_pbip.py "Sales.Workspace/Sales Model.SemanticMo Creates complete PBIP structure with TMDL definition and blank report. +### get_semantic_model_ai_metadata.py + +Read semantic model AI instructions and AI schema from a Fabric semantic model +definition. This is the service-definition roundtrip path for metadata managed +through Tabular Editor or the VS Code extension. + +```bash +python3 get_semantic_model_ai_metadata.py \ + "Sales.Workspace/Sales Model.SemanticModel" --format json + +python3 get_semantic_model_ai_metadata.py \ + "Sales.Workspace/Sales Model.SemanticModel" \ + --instructions-out instructions.md --schema-out schema.json +``` + +For offline parsing of a saved Fabric definition payload: + +```bash +fab get "Sales.Workspace/Sales Model.SemanticModel" -q definition -f > definition.json +python3 get_semantic_model_ai_metadata.py --definition-file definition.json --format json +``` + +The script reads friendly Copilot files and culture `linguisticMetadata`, +preferring friendly files when both are present. + ### download_workspace.py Download complete workspace with all items and lakehouse files. diff --git a/plugins/fabric-cli/skills/fabric-cli/scripts/get_semantic_model_ai_metadata.py b/plugins/fabric-cli/skills/fabric-cli/scripts/get_semantic_model_ai_metadata.py new file mode 100755 index 00000000..774bf486 --- /dev/null +++ b/plugins/fabric-cli/skills/fabric-cli/scripts/get_semantic_model_ai_metadata.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +""" +Retrieve semantic model AI instructions and AI schema with the Fabric CLI. + +Primary usage: + python3 get_semantic_model_ai_metadata.py "Workspace.Workspace/Model.SemanticModel" + python3 get_semantic_model_ai_metadata.py "Workspace.Workspace/Model.SemanticModel" --instructions-out instructions.md --schema-out schema.json + +For offline parsing of a saved Fabric definition payload: + fab get "Workspace.Workspace/Model.SemanticModel" -q "definition" -f > definition.json + python3 get_semantic_model_ai_metadata.py --definition-file definition.json + +Source precedence: when multiple parts provide AI instructions or AI schemas, +friendly Copilot definition files (e.g. Copilot/Instructions/instructions.md) +rank before culture linguisticMetadata, regardless of part order. The first +entry of aiInstructions/aiSchemas is the winner used by --instructions-out, +--schema-out, and --format text; a warning is emitted when sources disagree. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + + +NO_METADATA_WARNING = "No AI instructions or AI schema metadata found in semantic model definition." + + +def configure_output_streams() -> None: + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors="replace") + except (OSError, ValueError): + pass + + +def run_fab_command(args: list[str]) -> str: + try: + result = subprocess.run( + ["fab", *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as exc: + message = exc.stderr.strip() or exc.stdout.strip() or str(exc) + print(f"Error running fab command: {message}", file=sys.stderr) + sys.exit(exc.returncode or 1) + except FileNotFoundError: + print("Error: fab CLI not found. Install ms-fabric-cli and run 'fab auth login'.", file=sys.stderr) + sys.exit(1) + + +def get_definition(model_path: str) -> dict[str, Any]: + output = run_fab_command(["get", model_path, "-q", "definition", "-f"]) + try: + payload = json.loads(output) + except json.JSONDecodeError as exc: + print(f"Error: fab did not return valid JSON: {exc}", file=sys.stderr) + sys.exit(1) + + return normalize_definition_payload(payload) + + +def load_definition_file(path: Path) -> dict[str, Any]: + try: + raw = path.read_bytes() + except OSError as exc: + print(f"Error reading {path}: {exc}", file=sys.stderr) + sys.exit(1) + text = decode_text_bytes(raw) + if text is None: + print(f"Error reading {path}: file is not valid UTF-8 or UTF-16 text.", file=sys.stderr) + sys.exit(1) + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + print(f"Error parsing {path}: {exc}", file=sys.stderr) + sys.exit(1) + + return normalize_definition_payload(payload) + + +def normalize_definition_payload(payload: Any) -> dict[str, Any]: + if isinstance(payload, dict) and isinstance(payload.get("definition"), dict): + payload = payload["definition"] + if not isinstance(payload, dict) or not isinstance(payload.get("parts"), list): + print("Error: expected a Fabric semantic model definition object with a 'parts' array.", file=sys.stderr) + sys.exit(1) + return payload + + +def definition_files(definition: dict[str, Any], warnings: list[str] | None = None) -> dict[str, str]: + files: dict[str, str] = {} + for part in definition.get("parts", []): + if not isinstance(part, dict): + continue + path = part.get("path") + if not isinstance(path, str) or not path: + continue + content = decode_part_payload(part) + if content is None: + if warnings is not None: + warnings.append(f"Could not decode payload as text for part: {path}") + continue + files[path.replace("\\", "/")] = content + return files + + +def decode_part_payload(part: dict[str, Any]) -> str | None: + payload = part.get("payload") + payload_type = str(part.get("payloadType") or "") + + if isinstance(payload, str): + if payload_type in {"InlineBase64", "DecodeBase64"}: + return try_decode_base64(payload) + return payload + + return json.dumps(payload) + + +def try_decode_base64(value: str) -> str | None: + try: + raw = base64.b64decode(value, validate=True) + except Exception: + return None + return decode_text_bytes(raw) + + +def decode_text_bytes(raw: bytes) -> str | None: + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + try: + return raw.decode("utf-16") + except UnicodeDecodeError: + return None + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError: + text = None + if text is not None and "\x00" not in text: + return text + for encoding in ("utf-16-le", "utf-16-be"): + try: + decoded = raw.decode(encoding) + except UnicodeDecodeError: + continue + if "\x00" not in decoded: + return decoded + return text + + +def parse_metadata(files: dict[str, str], culture_filter: str | None = None) -> dict[str, Any]: + result: dict[str, Any] = { + "aiInstructions": [], + "aiSchemas": [], + "aiSchemaObjects": [], + "sources": [], + "warnings": [], + } + + for source_path, text in files.items(): + lower = source_path.lower() + structured = try_parse_json(text) + is_culture_part = is_linguistic_metadata_path(lower) + if is_culture_part and lower.endswith(".tmdl"): + # TMDL culture files embed the linguistic metadata JSON after a + # "linguisticMetadata" token; .lsdl/.lsdl.json parts are the raw JSON itself. + tmdl_metadata = extract_tmdl_linguistic_metadata(text) + structured = try_parse_json(tmdl_metadata) if tmdl_metadata is not None else None + + if is_ai_instruction_path(lower): + instruction_text = extract_instruction_text(structured if structured is not None else text) + if instruction_text is None: + result["warnings"].append( + f"AI instructions part has no recognized instruction content: {source_path}" + ) + continue + result["aiInstructions"].append( + { + "sourcePath": source_path, + "format": "markdown" if lower.endswith((".md", ".markdown")) else "text", + "length": len(instruction_text), + "text": instruction_text, + } + ) + result["sources"].append({"kind": "aiInstructions", "path": source_path}) + continue + + if is_ai_schema_path(lower): + if not isinstance(structured, dict): + result["warnings"].append(f"AI schema part could not be parsed as JSON: {source_path}") + continue + schema = structured + result["aiSchemas"].append({"sourcePath": source_path, "schema": schema}) + add_ai_schema_objects(result, source_path, schema) + result["sources"].append({"kind": "aiSchema", "path": source_path}) + continue + + if lower.endswith(".bim"): + collect_tmsl_metadata(result, source_path, structured, culture_filter) + continue + + if is_culture_part and isinstance(structured, dict): + culture = culture_from_tmdl(source_path, text) + if culture_filter and culture and culture.lower() != culture_filter.lower(): + continue + collect_linguistic_metadata(result, source_path, structured, culture) + + # Deterministic precedence: friendly Copilot files rank before culture + # linguisticMetadata, independent of part order (see module docstring). + result["aiInstructions"].sort(key=source_precedence) + result["aiSchemas"].sort(key=source_precedence) + + if len(result["aiInstructions"]) > 1: + primary = result["aiInstructions"][0] + for other in result["aiInstructions"][1:]: + if other["text"] != primary["text"]: + result["warnings"].append( + "AI instructions differ between sources " + f"{primary['sourcePath']} and {other['sourcePath']}; using {primary['sourcePath']}." + ) + + if not result["aiInstructions"] and not result["aiSchemas"] and not result["aiSchemaObjects"]: + result["warnings"].append(NO_METADATA_WARNING) + + return result + + +def source_precedence(entry: dict[str, Any]) -> int: + return 1 if entry.get("storage") == "linguisticMetadata" else 0 + + +def collect_linguistic_metadata(result: dict[str, Any], source_path: str, payload: dict[str, Any], culture: str | None) -> None: + instructions = payload.get("CustomInstructions") or payload.get("customInstructions") + if isinstance(instructions, str): + result["aiInstructions"].append( + { + "sourcePath": source_path, + "storage": "linguisticMetadata", + "culture": culture, + "format": "markdown", + "length": len(instructions), + "text": instructions, + } + ) + result["sources"].append( + {"kind": "aiInstructions", "path": source_path, "storage": "linguisticMetadata", "culture": culture} + ) + + schema = schema_from_entities(payload.get("Entities") or payload.get("entities") or {}) + if schema["tables"]: + result["aiSchemas"].append( + { + "sourcePath": source_path, + "storage": "linguisticMetadata", + "culture": culture, + "schema": schema, + } + ) + add_ai_schema_objects(result, source_path, schema) + result["sources"].append( + {"kind": "aiSchema", "path": source_path, "storage": "linguisticMetadata", "culture": culture} + ) + + +def collect_tmsl_metadata(result: dict[str, Any], source_path: str, payload: Any, culture_filter: str | None) -> None: + """Walk a TMSL definition (model.bim): model.cultures[*].linguisticMetadata.content.""" + if not isinstance(payload, dict): + return + model = payload.get("model") + if not isinstance(model, dict): + return + cultures = model.get("cultures") + if not isinstance(cultures, list): + return + for entry in cultures: + if not isinstance(entry, dict): + continue + culture = entry.get("name") if isinstance(entry.get("name"), str) else None + if culture_filter and culture and culture.lower() != culture_filter.lower(): + continue + metadata = entry.get("linguisticMetadata") + if not isinstance(metadata, dict): + continue + content = metadata.get("content") + if isinstance(content, str): + content = try_parse_json(content) + if isinstance(content, dict): + collect_linguistic_metadata(result, source_path, content, culture) + + +def extract_tmdl_linguistic_metadata(text: str) -> str | None: + marker = text.find("linguisticMetadata") + if marker == -1: + return None + + start = text.find("{", marker) + if start == -1: + return None + + depth = 0 + in_string = False + escaped = False + for index in range(start, len(text)): + char = text[index] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return text[start : index + 1] + return None + + +def culture_from_tmdl(source_path: str, text: str) -> str | None: + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("cultureInfo "): + return stripped.removeprefix("cultureInfo ").strip().strip("'\"") + name = Path(source_path).stem + return name or None + + +def schema_from_entities(entities: Any) -> dict[str, Any]: + if not isinstance(entities, dict): + return {"tables": []} + + tables: dict[str, dict[str, Any]] = {} + + def table_for(name: str) -> dict[str, Any]: + if name not in tables: + tables[name] = {"name": name, "include": True, "columns": [], "hierarchies": []} + return tables[name] + + for entity in entities.values(): + if not isinstance(entity, dict): + continue + binding = entity.get("Binding") + if not isinstance(binding, dict): + definition = entity.get("Definition") + binding = definition.get("Binding") if isinstance(definition, dict) else None + if not isinstance(binding, dict): + continue + + table_name = binding.get("ConceptualEntity") or binding.get("conceptualEntity") + if not isinstance(table_name, str) or not table_name: + continue + + include = entity_included(entity) + table = table_for(table_name) + property_name = binding.get("ConceptualProperty") or binding.get("conceptualProperty") + hierarchy_name = binding.get("Hierarchy") or binding.get("hierarchy") + level_name = binding.get("HierarchyLevel") or binding.get("hierarchyLevel") + + if isinstance(level_name, str) and level_name: + hierarchy = hierarchy_for(table, hierarchy_name or "") + hierarchy.setdefault("levels", []).append({"name": level_name, "include": include}) + elif isinstance(hierarchy_name, str) and hierarchy_name: + hierarchy_for(table, hierarchy_name)["include"] = include + elif isinstance(property_name, str) and property_name: + table.setdefault("columns", []).append({"name": property_name, "include": include}) + else: + table["include"] = include + + table_list = list(tables.values()) + for table in table_list: + if not table.get("columns"): + table.pop("columns", None) + if not table.get("hierarchies"): + table.pop("hierarchies", None) + else: + for hierarchy in table["hierarchies"]: + if not hierarchy.get("levels"): + hierarchy.pop("levels", None) + return {"tables": table_list} + + +def hierarchy_for(table: dict[str, Any], name: str) -> dict[str, Any]: + hierarchies = table.setdefault("hierarchies", []) + for hierarchy in hierarchies: + if hierarchy.get("name") == name: + return hierarchy + hierarchy = {"name": name, "include": True, "levels": []} + hierarchies.append(hierarchy) + return hierarchy + + +def entity_included(entity: dict[str, Any]) -> bool: + state = str(entity.get("State") or entity.get("state") or "Generated").lower() + return state not in {"deleted", "hidden", "disabled"} + + +def add_ai_schema_objects(result: dict[str, Any], source_path: str, schema: dict[str, Any]) -> None: + for table_key, table in collection_entries(schema.get("tables") or schema.get("Tables")): + table_name = schema_object_name(table_key, table) + if not table_name: + continue + push_schema_object(result, source_path, table, {"type": "table", "table": table_name}) + + for column_key, column in collection_entries(get_child(table, "columns")): + column_name = schema_object_name(column_key, column) + if column_name: + push_schema_object(result, source_path, column, {"type": "column", "table": table_name, "property": column_name}) + + for measure_key, measure in collection_entries(get_child(table, "measures")): + measure_name = schema_object_name(measure_key, measure) + if measure_name: + push_schema_object(result, source_path, measure, {"type": "measure", "table": table_name, "property": measure_name}) + + for hierarchy_key, hierarchy in collection_entries(get_child(table, "hierarchies")): + hierarchy_name = schema_object_name(hierarchy_key, hierarchy) + if not hierarchy_name: + continue + push_schema_object(result, source_path, hierarchy, {"type": "hierarchy", "table": table_name, "hierarchy": hierarchy_name}) + for level_key, level in collection_entries(get_child(hierarchy, "levels")): + level_name = schema_object_name(level_key, level) + if level_name: + push_schema_object( + result, + source_path, + level, + {"type": "level", "table": table_name, "hierarchy": hierarchy_name, "level": level_name}, + ) + + +def push_schema_object(result: dict[str, Any], source_path: str, value: Any, obj: dict[str, Any]) -> None: + result["aiSchemaObjects"].append( + { + "sourcePath": source_path, + "object": obj, + "include": schema_include(value), + "visibility": schema_property(value, "visibility"), + "index": schema_property(value, "index"), + } + ) + + +def collection_entries(value: Any) -> list[tuple[str | None, Any]]: + if isinstance(value, list): + return [(schema_object_name(None, item), item) for item in value] + if isinstance(value, dict): + return list(value.items()) + return [] + + +def get_child(value: Any, name: str) -> Any: + if not isinstance(value, dict): + return None + return value.get(name) or value.get(name[:1].upper() + name[1:]) + + +def schema_object_name(key: str | None, value: Any) -> str | None: + if isinstance(value, dict): + candidate = value.get("name") or value.get("Name") or value.get("id") or value.get("Id") + if isinstance(candidate, str): + return candidate + return key + + +def schema_include(value: Any) -> bool | None: + if isinstance(value, bool): + return value + include = schema_property(value, "include") + if isinstance(include, bool): + return include + enabled = schema_property(value, "enabled") + if isinstance(enabled, bool): + return enabled + selected = schema_property(value, "selected") + if isinstance(selected, bool): + return selected + visibility = schema_property(value, "visibility") + if isinstance(visibility, str): + if visibility.lower() == "hidden": + return False + if visibility.lower() == "visible": + return True + return None + + +def schema_property(value: Any, name: str) -> Any: + if not isinstance(value, dict): + return None + return value.get(name) if name in value else value.get(name[:1].upper() + name[1:]) + + +def try_parse_json(text: str) -> Any: + stripped = text.lstrip("\ufeff").strip() + if not stripped.startswith(("{", "[")): + return None + try: + return json.loads(stripped) + except json.JSONDecodeError: + return None + + +def extract_instruction_text(payload: Any) -> str | None: + if isinstance(payload, str): + return payload.strip() + if not isinstance(payload, dict): + return None + for key in ["instructions", "aiInstructions", "systemInstructions", "copilotInstructions", "prompt"]: + value = payload.get(key) + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return None + + +def path_tokens(lower_path: str) -> set[str]: + return {token for token in re.split(r"[^a-z0-9]+", lower_path) if token} + + +def is_ai_instruction_path(lower_path: str) -> bool: + if "copilot/instructions/version.json" in lower_path: + return False + if lower_path.endswith("copilot/instructions/instructions.md"): + return True + tokens = path_tokens(lower_path) + return bool(tokens & {"instruction", "instructions", "prompt", "prompts"}) and bool(tokens & {"ai", "copilot"}) + + +def is_ai_schema_path(lower_path: str) -> bool: + return ( + "ai-schema" in lower_path + or "/ai/schema" in lower_path + or "copilot/schema" in lower_path + ) + + +def is_linguistic_metadata_path(lower_path: str) -> bool: + return lower_path.endswith(".lsdl") or lower_path.endswith(".lsdl.json") or ( + lower_path.endswith(".tmdl") and ("/cultures/" in lower_path or lower_path.startswith("cultures/")) + ) + + +def write_outputs(result: dict[str, Any], instructions_out: Path | None, schema_out: Path | None) -> None: + if instructions_out: + if result["aiInstructions"]: + instructions_out.parent.mkdir(parents=True, exist_ok=True) + instructions_out.write_text(result["aiInstructions"][0]["text"], encoding="utf-8") + else: + result["warnings"].append(f"No AI instructions found; not writing {instructions_out}.") + if schema_out: + if result["aiSchemas"]: + schema_out.parent.mkdir(parents=True, exist_ok=True) + schema_out.write_text(json.dumps(result["aiSchemas"][0]["schema"], indent=2) + "\n", encoding="utf-8") + else: + result["warnings"].append(f"No AI schema found; not writing {schema_out}.") + + +def main() -> None: + configure_output_streams() + parser = argparse.ArgumentParser(description="Retrieve semantic model AI instructions and AI schema with fab.") + parser.add_argument("model", nargs="?", help='Fabric path: "Workspace.Workspace/Model.SemanticModel"') + parser.add_argument("--definition-file", type=Path, help="Parse a saved fab definition JSON instead of calling fab.") + parser.add_argument("--culture", help="Culture to use when multiple TMDL culture metadata files exist.") + parser.add_argument("--instructions-out", type=Path, help="Write the first AI instructions payload to this file.") + parser.add_argument("--schema-out", type=Path, help="Write the first AI schema payload to this JSON file.") + parser.add_argument("--format", choices=["json", "text"], default="json", help="Output format. Default: json.") + args = parser.parse_args() + + if not args.definition_file and not args.model: + parser.error("provide a semantic model path or --definition-file") + + definition = load_definition_file(args.definition_file) if args.definition_file else get_definition(args.model) + decode_warnings: list[str] = [] + files = definition_files(definition, decode_warnings) + result = parse_metadata(files, args.culture) + result["warnings"].extend(decode_warnings) + result["model"] = args.model or str(args.definition_file) + result["partCount"] = len(files) + + write_outputs(result, args.instructions_out, args.schema_out) + + if args.format == "text": + for warning in result["warnings"]: + print(warning, file=sys.stderr) + if result["aiInstructions"]: + print(result["aiInstructions"][0]["text"]) + return + sys.exit(1) + + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/plugins/semantic-models/skills/semantic-model/SKILL.md b/plugins/semantic-models/skills/semantic-model/SKILL.md index 33b2a83c..31db0a59 100644 --- a/plugins/semantic-models/skills/semantic-model/SKILL.md +++ b/plugins/semantic-models/skills/semantic-model/SKILL.md @@ -94,6 +94,8 @@ references/refactoring-renaming.md: safe rename workflow (lineage check first references/review-checklist.md: full audit checklist with remediation references/performance.md: performance testing, unused-column detection, memory analysis scripts/get_model_info.py: model metadata overview (mode, size, reports, endorsement, sources, refresh) +scripts/manage-ai-metadata.csx: read/write AI instructions and AI schema through TOM culture linguistic metadata +scripts/get_semantic_model_ai_metadata.py: Fabric CLI service-definition readback for AI instructions and AI schema ``` ## What this skill deliberately leaves out diff --git a/plugins/semantic-models/skills/semantic-model/scripts/README.md b/plugins/semantic-models/skills/semantic-model/scripts/README.md new file mode 100644 index 00000000..7b7ba9a4 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/README.md @@ -0,0 +1,34 @@ +# Semantic Model Scripts + +Utility scripts used by the semantic-model skill when a direct `te` command is +not enough. + +## Semantic model AI metadata + +These scripts manage or inspect semantic model AI metadata: + +- `manage-ai-metadata.csx`: non-interactive `te script` CRUD for AI + instructions and AI schema. +- `edit-ai-instructions-interactive.csx`: TE3 Desktop GUI editor for AI + instructions. +- `edit-ai-schema-interactive.csx`: TE3 Desktop GUI editor for AI schema. +- `manage-ai-metadata-interactive.csx`: original combined TE3 Desktop editor + prototype. +- `get_semantic_model_ai_metadata.py`: Fabric CLI readback and offline + definition parser. + +Use the C# script for fast TOM-backed reads and writes: + +```bash +TE_AI_ACTION=get TE_AI_TARGET=both \ + te script -s "workspace" -d "model" \ + -S scripts/manage-ai-metadata.csx \ + --output-format json --non-interactive +``` + +Use the Python script to confirm the deployed service definition: + +```bash +python3 scripts/get_semantic_model_ai_metadata.py \ + "workspace.Workspace/model.SemanticModel" --format json +``` diff --git a/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-instructions-interactive.csx b/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-instructions-interactive.csx new file mode 100644 index 00000000..f7437ad5 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-instructions-interactive.csx @@ -0,0 +1,517 @@ +#r "System.Drawing" + +// TE3 Desktop GUI editor for semantic model AI instructions. +// Uses Model.Cultures["en-US"].Content -> CustomInstructions. + +using System; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiInstructionsEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiInstructionsEditor.RootMessage(ex), "Semantic Model AI Instructions"); +} + +public static class AiInstructionsEditor +{ + private const string DefaultCultureName = "en-US"; + private const int InstructionsLimit = 10000; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Instructions"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Instructions"); + return; + } + + using (var form = new Form()) + using (var editor = ScriptTextEditor.Create("markdown", true)) + { + form.Text = "Semantic Model AI Instructions"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 980; + form.Height = 760; + form.MinimumSize = new Size(760, 520); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + + (editor.IsScintilla ? " Editor: Scintilla" : " Editor: TextBox") + }; + + var editorPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + editorPanel.Controls.Add(editor.Control); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Margin = new Padding(0, 0, 8, 0) + }; + + var wrapButton = NewFooterButton("Wrap", font); + var reloadButton = NewFooterButton("Reload", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(wrapButton); + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(editorPanel, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + Action refreshStatus = () => + { + var count = NormalizeForStorage(editor.Text).Length; + status.Text = count + " / " + InstructionsLimit + " characters" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + status.ForeColor = count > InstructionsLimit ? Color.Firebrick : + string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + saveButton.Enabled = count <= InstructionsLimit; + }; + + Action load = () => + { + try + { + editor.Text = NormalizeForEditor(GetInstructions(culture)); + editor.SelectStart(); + refreshStatus(); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + editor.TextChanged += (sender, args) => refreshStatus(); + wrapButton.Click += (sender, args) => editor.WordWrap = !editor.WordWrap; + reloadButton.Click += (sender, args) => load(); + saveButton.Click += (sender, args) => + { + try + { + var text = NormalizeForStorage(editor.Text); + if (text.Length > InstructionsLimit) + { + status.Text = "AI instructions must be 10000 characters or fewer."; + status.ForeColor = Color.Firebrick; + return; + } + + SetInstructions(culture, text); + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + load(); + form.Shown += (sender, args) => editor.Focus(); + form.ShowDialog(); + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static string GetInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + return (string)payload["CustomInstructions"] ?? ""; + } + + private static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + private static string NormalizeForStorage(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n"); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly object scintilla; + private readonly TextBox textBox; + private bool wordWrap; + + public Control Control { get; private set; } + public bool IsScintilla { get { return scintilla != null; } } + public event EventHandler TextChanged; + + private ScriptTextEditor(object scintillaControl, TextBox fallbackTextBox, bool initialWordWrap) + { + scintilla = scintillaControl; + textBox = fallbackTextBox; + Control = (Control)(scintillaControl ?? (object)fallbackTextBox); + wordWrap = initialWordWrap; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(string lexerName, bool wordWrap) + { + try + { + var assembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, "ScintillaNET", StringComparison.OrdinalIgnoreCase)) + ?? Assembly.Load("ScintillaNET"); + var type = assembly.GetType("ScintillaNET.Scintilla", true); + var control = (Control)Activator.CreateInstance(type); + ConfigureScintilla(control, lexerName, wordWrap); + return new ScriptTextEditor(control, null, wordWrap); + } + catch + { + var fallback = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(null, fallback, wordWrap); + } + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return wordWrap; } + set + { + wordWrap = value; + if (textBox != null) + { + textBox.WordWrap = value; + return; + } + + SetEnumProperty(scintilla, "WrapMode", value ? "Word" : "None"); + SetProperty(scintilla, "HScrollBar", !value); + } + } + + public void Focus() + { + Control.Focus(); + } + + public void SelectStart() + { + if (textBox != null) + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + return; + } + + SetProperty(scintilla, "CurrentPosition", 0); + SetProperty(scintilla, "AnchorPosition", 0); + } + + public void Dispose() + { + Control?.Dispose(); + } + + private static void ConfigureScintilla(Control control, string lexerName, bool wordWrap) + { + control.Dock = DockStyle.Fill; + control.Font = new Font("Consolas", 10F); + control.BackColor = Color.White; + + var target = (object)control; + SetEnumProperty(target, "BorderStyle", "None"); + SetProperty(target, "LexerName", lexerName); + SetEnumProperty(target, "WrapMode", wordWrap ? "Word" : "None"); + SetEnumProperty(target, "WrapIndentMode", "Indent"); + SetProperty(target, "ScrollWidthTracking", true); + SetProperty(target, "MultipleSelection", true); + SetProperty(target, "AdditionalSelectionTyping", true); + SetProperty(target, "MouseSelectionRectangularSwitch", true); + SetProperty(target, "HScrollBar", !wordWrap); + SetProperty(target, "VScrollBar", true); + ConfigureMargins(target); + ConfigureContextMenu(control, target); + } + + private static void ConfigureMargins(object target) + { + var margins = GetProperty(target, "Margins"); + if (margins == null) return; + + var lineMargin = GetIndexerValue(margins, 0); + if (lineMargin != null) + { + SetProperty(lineMargin, "Width", 42); + SetEnumProperty(lineMargin, "Type", "Number"); + SetEnumProperty(lineMargin, "Cursor", "ReverseArrow"); + } + + var foldMargin = GetIndexerValue(margins, 2); + if (foldMargin != null) + { + SetProperty(foldMargin, "Width", 16); + SetProperty(foldMargin, "Sensitive", true); + SetEnumProperty(foldMargin, "Type", "Symbol"); + SetEnumProperty(foldMargin, "Cursor", "Arrow"); + } + } + + private static void ConfigureContextMenu(Control control, object target) + { + var menu = new ContextMenuStrip(); + AddMenuItem(menu, "Undo", () => InvokeNoArgs(target, "Undo")); + AddMenuItem(menu, "Redo", () => InvokeNoArgs(target, "Redo")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Cut", () => InvokeNoArgs(target, "Cut")); + AddMenuItem(menu, "Copy", () => InvokeNoArgs(target, "Copy")); + AddMenuItem(menu, "Paste", () => InvokeNoArgs(target, "Paste")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Select All", () => InvokeNoArgs(target, "SelectAll")); + control.ContextMenuStrip = menu; + } + + private static void AddMenuItem(ContextMenuStrip menu, string text, Action action) + { + var item = new ToolStripMenuItem(text); + item.Click += (sender, args) => + { + try { action(); } + catch { } + }; + menu.Items.Add(item); + } + + private static void InvokeNoArgs(object target, string methodName) + { + var method = target.GetType().GetMethod(methodName, BindingFlags.Instance | BindingFlags.Public); + if (method != null) method.Invoke(target, null); + } + + private static object GetProperty(object target, string propertyName) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + return prop == null ? null : prop.GetValue(target, null); + } + + private static object GetIndexerValue(object target, int index) + { + var prop = target.GetType().GetProperties() + .FirstOrDefault(p => p.GetIndexParameters().Length == 1 && p.GetIndexParameters()[0].ParameterType == typeof(int)); + return prop == null ? null : prop.GetValue(target, new object[] { index }); + } + + private static void SetProperty(object target, string propertyName, object value) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite) return; + prop.SetValue(target, value, null); + } + + private static void SetEnumProperty(object target, string propertyName, string enumValue) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite || !prop.PropertyType.IsEnum) return; + prop.SetValue(target, Enum.Parse(prop.PropertyType, enumValue), null); + } +} diff --git a/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-schema-interactive.csx b/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-schema-interactive.csx new file mode 100644 index 00000000..80b77368 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/edit-ai-schema-interactive.csx @@ -0,0 +1,1251 @@ +#r "System.Drawing" + +// TE3 Desktop GUI editor for semantic model AI schema. +// Uses Model.Cultures["en-US"].Content -> Entities. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiSchemaEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiSchemaEditor.RootMessage(ex), "Semantic Model AI Schema"); +} + +public static class AiSchemaEditor +{ + private const string DefaultCultureName = "en-US"; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Schema"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Schema"); + return; + } + + using (var form = new Form()) + using (var jsonEditor = ScriptTextEditor.Create("json", false)) + using (var stateImages = BuildStateImages()) + { + form.Text = "Semantic Model AI Schema"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 1040; + form.Height = 780; + form.MinimumSize = new Size(820, 560); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + }; + + var tabs = new TabControl + { + Dock = DockStyle.Fill, + Font = font + }; + var treePage = new TabPage("Objects"); + var jsonPage = new TabPage("JSON"); + tabs.TabPages.Add(treePage); + tabs.TabPages.Add(jsonPage); + + var tree = new TreeView + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.None, + Font = font, + HideSelection = false, + ShowLines = true, + ShowPlusMinus = true, + ShowRootLines = true, + StateImageList = stateImages + }; + + var treePanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle + }; + treePanel.Controls.Add(tree); + + var toolbar = new FlowLayoutPanel + { + Dock = DockStyle.Top, + Height = 44, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Padding = new Padding(0, 6, 0, 4) + }; + var showHidden = new CheckBox { Text = "Show hidden", Checked = true, AutoSize = true, Font = font, Padding = new Padding(0, 7, 12, 0) }; + var checkAllButton = NewToolbarButton("Check all", font); + var clearButton = NewToolbarButton("Clear", font); + var expandButton = NewToolbarButton("Expand", font); + var collapseButton = NewToolbarButton("Collapse", font); + toolbar.Controls.Add(showHidden); + toolbar.Controls.Add(checkAllButton); + toolbar.Controls.Add(clearButton); + toolbar.Controls.Add(expandButton); + toolbar.Controls.Add(collapseButton); + + var treeLayout = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0) }; + treeLayout.Controls.Add(treePanel); + treeLayout.Controls.Add(toolbar); + treePage.Controls.Add(treeLayout); + + var jsonPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + jsonPanel.Controls.Add(jsonEditor.Control); + jsonPage.Controls.Add(jsonPanel); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod, + Margin = new Padding(0, 0, 8, 0) + }; + + var reloadButton = NewFooterButton("Reload", font); + var formatButton = NewFooterButton("Format JSON", font); + var updateJsonButton = NewFooterButton("Update JSON", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(formatButton); + buttonStrip.Controls.Add(updateJsonButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(tabs, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + bool updatingTree = false; + + Action loadSchemaIntoUi = schema => + { + updatingTree = true; + try + { + BuildTree(tree, model, schema, showHidden.Checked); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + } + finally + { + updatingTree = false; + } + }; + + Action reload = () => + { + try + { + loadSchemaIntoUi(GetSchema(culture)); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + tree.NodeMouseClick += (sender, args) => + { + if (updatingTree) return; + try + { + ToggleNode(args.Node); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + showHidden.CheckedChanged += (sender, args) => + { + if (updatingTree) return; + try + { + var current = SchemaFromTree(tree); + loadSchemaIntoUi(current); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + checkAllButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, CheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + clearButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, UncheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + expandButton.Click += (sender, args) => tree.ExpandAll(); + collapseButton.Click += (sender, args) => tree.CollapseAll(); + + reloadButton.Click += (sender, args) => reload(); + formatButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(JObject.Parse(jsonEditor.Text).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + status.Text = "JSON formatted."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + updateJsonButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(SchemaFromTree(tree).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + jsonEditor.SelectStart(); + status.Text = "JSON updated from object tree."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + saveButton.Click += (sender, args) => + { + try + { + JObject schema; + if (tabs.SelectedTab == jsonPage) + { + schema = JObject.Parse(jsonEditor.Text); + SetSchema(culture, schema); + loadSchemaIntoUi(schema); + tabs.SelectedTab = jsonPage; + } + else + { + schema = SchemaFromTree(tree); + SetSchema(culture, schema); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + } + + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + reload(); + form.ShowDialog(); + } + } + + private const int UncheckedState = 0; + private const int CheckedState = 1; + private const int MixedState = 2; + + private static void BuildTree(TreeView tree, TabularEditor.TOMWrapper.Model model, JObject schema, bool showHidden) + { + tree.BeginUpdate(); + try + { + tree.Nodes.Clear(); + var index = BuildSchemaIndex(schema); + var hasSchema = index.Count > 0; + + foreach (var table in model.Tables.OrderBy(t => t.Name)) + { + if (!showHidden && !IsVisibleObject(table)) continue; + + var tableNode = NewNode(table.Name, new SchemaNode("table", table.Name, null, null, null)); + tree.Nodes.Add(tableNode); + + foreach (var column in table.Columns.OrderBy(c => c.Name)) + { + if (!showHidden && !IsVisibleObject(column)) continue; + var node = NewNode(column.Name + " column", new SchemaNode("column", table.Name, column.Name, null, null)); + node.ForeColor = IsVisibleObject(column) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(column)); + tableNode.Nodes.Add(node); + } + + foreach (var measure in table.Measures.OrderBy(m => m.Name)) + { + if (!showHidden && !IsVisibleObject(measure)) continue; + var node = NewNode(measure.Name + " measure", new SchemaNode("measure", table.Name, measure.Name, null, null)); + node.ForeColor = IsVisibleObject(measure) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(measure)); + tableNode.Nodes.Add(node); + } + + foreach (var hierarchy in table.Hierarchies.OrderBy(h => h.Name)) + { + if (!showHidden && !IsVisibleObject(hierarchy)) continue; + var hierarchyNode = NewNode(hierarchy.Name + " hierarchy", new SchemaNode("hierarchy", table.Name, null, hierarchy.Name, null)); + hierarchyNode.ForeColor = IsVisibleObject(hierarchy) ? SystemColors.WindowText : SystemColors.GrayText; + hierarchyNode.StateImageIndex = IncludeState(index, hierarchyNode.Tag as SchemaNode, hasSchema, IsVisibleObject(hierarchy)); + tableNode.Nodes.Add(hierarchyNode); + + foreach (var level in hierarchy.Levels.OrderBy(l => l.Name)) + { + var levelNode = NewNode(level.Name + " level", new SchemaNode("level", table.Name, null, hierarchy.Name, level.Name)); + levelNode.StateImageIndex = IncludeState(index, levelNode.Tag as SchemaNode, hasSchema, true); + hierarchyNode.Nodes.Add(levelNode); + } + + if (hierarchyNode.Nodes.Count > 0 && !index.ContainsKey(Key(hierarchyNode.Tag as SchemaNode))) + { + hierarchyNode.StateImageIndex = AggregateState(hierarchyNode); + } + } + + tableNode.ForeColor = IsVisibleObject(table) ? SystemColors.WindowText : SystemColors.GrayText; + tableNode.StateImageIndex = index.ContainsKey(Key(tableNode.Tag as SchemaNode)) + ? index[Key(tableNode.Tag as SchemaNode)] ? CheckedState : UncheckedState + : tableNode.Nodes.Count == 0 ? (IsVisibleObject(table) || !hasSchema ? CheckedState : UncheckedState) : AggregateState(tableNode); + tableNode.Expand(); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static Button NewToolbarButton(string text, Font font) + { + return new Button + { + Text = text, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Font = font, + Margin = new Padding(6, 0, 0, 0), + MinimumSize = new Size(96, 30), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static TreeNode NewNode(string text, SchemaNode tag) + { + return new TreeNode(text) + { + Tag = tag, + StateImageIndex = CheckedState + }; + } + + private static bool IsVisibleObject(object obj) + { + if (obj is IHideableObject hideable) return hideable.IsVisible; + return true; + } + + private static int IncludeState(Dictionary index, SchemaNode node, bool hasSchema, bool visible) + { + var key = Key(node); + if (index.ContainsKey(key)) return index[key] ? CheckedState : UncheckedState; + return !hasSchema || visible ? CheckedState : UncheckedState; + } + + private static Dictionary BuildSchemaIndex(JObject schema) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tableEntry in CollectionEntries(schema?["tables"] ?? schema?["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + result[Key(new SchemaNode("table", tableName, null, null, null))] = IncludeValue(tableEntry.Value); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var name = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("column", tableName, name, null, null))] = IncludeValue(columnEntry.Value); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var name = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("measure", tableName, name, null, null))] = IncludeValue(measureEntry.Value); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var name = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(name)) continue; + result[Key(new SchemaNode("hierarchy", tableName, null, name, null))] = IncludeValue(hierarchyEntry.Value); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var level = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(level)) result[Key(new SchemaNode("level", tableName, null, name, level))] = IncludeValue(levelEntry.Value); + } + } + } + return result; + } + + private static string Key(SchemaNode node) + { + if (node == null) return ""; + if (node.Kind == "table") return "T|" + node.Table; + if (node.Kind == "column" || node.Kind == "measure") return "P|" + node.Table + "|" + node.Property; + if (node.Kind == "hierarchy") return "H|" + node.Table + "|" + node.Hierarchy; + if (node.Kind == "level") return "L|" + node.Table + "|" + node.Hierarchy + "|" + node.Level; + return ""; + } + + private static void ToggleNode(TreeNode node) + { + var next = node.StateImageIndex == CheckedState ? UncheckedState : CheckedState; + SetNodeAndChildren(node, next); + RefreshAncestors(node.Parent); + } + + private static void SetAllTreeNodes(TreeView tree, int state) + { + tree.BeginUpdate(); + try + { + foreach (TreeNode node in tree.Nodes) + { + SetNodeAndChildren(node, state); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static void SetNodeAndChildren(TreeNode node, int state) + { + node.StateImageIndex = state; + foreach (TreeNode child in node.Nodes) + { + SetNodeAndChildren(child, state); + } + } + + private static void RefreshAncestors(TreeNode node) + { + while (node != null) + { + node.StateImageIndex = AggregateState(node); + node = node.Parent; + } + } + + private static int AggregateState(TreeNode node) + { + if (node.Nodes.Count == 0) return node.StateImageIndex == CheckedState ? CheckedState : UncheckedState; + + var checkedCount = 0; + var uncheckedCount = 0; + foreach (TreeNode child in node.Nodes) + { + if (child.StateImageIndex == CheckedState) checkedCount++; + else if (child.StateImageIndex == UncheckedState) uncheckedCount++; + else return MixedState; + } + + if (checkedCount == node.Nodes.Count) return CheckedState; + if (uncheckedCount == node.Nodes.Count) return UncheckedState; + return MixedState; + } + + private static JObject SchemaFromTree(TreeView tree) + { + var tables = new JArray(); + foreach (TreeNode tableNode in tree.Nodes) + { + var tableTag = tableNode.Tag as SchemaNode; + if (tableTag == null || tableTag.Kind != "table") continue; + + var table = new JObject + { + ["name"] = tableTag.Table, + ["include"] = tableNode.StateImageIndex != UncheckedState + }; + var columns = new JArray(); + var measures = new JArray(); + var hierarchies = new JArray(); + + foreach (TreeNode child in tableNode.Nodes) + { + var tag = child.Tag as SchemaNode; + if (tag == null) continue; + + if (tag.Kind == "column") + { + columns.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "measure") + { + measures.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "hierarchy") + { + var hierarchy = new JObject + { + ["name"] = tag.Hierarchy, + ["include"] = child.StateImageIndex != UncheckedState + }; + var levels = new JArray(); + foreach (TreeNode levelNode in child.Nodes) + { + var levelTag = levelNode.Tag as SchemaNode; + if (levelTag != null && levelTag.Kind == "level") + { + levels.Add(new JObject { ["name"] = levelTag.Level, ["include"] = levelNode.StateImageIndex == CheckedState }); + } + } + if (levels.Count > 0) hierarchy["levels"] = levels; + hierarchies.Add(hierarchy); + } + } + + if (columns.Count > 0) table["columns"] = columns; + if (measures.Count > 0) table["measures"] = measures; + if (hierarchies.Count > 0) table["hierarchies"] = hierarchies; + tables.Add(table); + } + + return new JObject { ["tables"] = tables }; + } + + private static string StatusText(TreeView tree, string startupWarning) + { + var total = 0; + var included = 0; + CountNodes(tree.Nodes, ref total, ref included); + return included + " included / " + total + " objects" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + } + + private static void CountNodes(TreeNodeCollection nodes, ref int total, ref int included) + { + foreach (TreeNode node in nodes) + { + if (node.Tag is SchemaNode) + { + total++; + if (node.StateImageIndex != UncheckedState) included++; + } + CountNodes(node.Nodes, ref total, ref included); + } + } + + private static ImageList BuildStateImages() + { + var list = new ImageList { ImageSize = new Size(16, 16), ColorDepth = ColorDepth.Depth32Bit }; + list.Images.Add(DrawStateImage(UncheckedState)); + list.Images.Add(DrawStateImage(CheckedState)); + list.Images.Add(DrawStateImage(MixedState)); + return list; + } + + private static Bitmap DrawStateImage(int state) + { + var bmp = new Bitmap(16, 16); + using (var g = Graphics.FromImage(bmp)) + using (var border = new Pen(Color.FromArgb(120, 120, 120))) + using (var fill = new SolidBrush(Color.White)) + using (var mark = new Pen(Color.FromArgb(0, 120, 215), 2F)) + using (var mixed = new SolidBrush(Color.FromArgb(0, 120, 215))) + { + g.Clear(Color.Transparent); + g.FillRectangle(fill, 2, 2, 12, 12); + g.DrawRectangle(border, 2, 2, 12, 12); + if (state == CheckedState) + { + g.DrawLines(mark, new[] { new Point(4, 8), new Point(7, 11), new Point(12, 5) }); + } + else if (state == MixedState) + { + g.FillRectangle(mixed, 5, 7, 6, 2); + } + } + return bmp; + } + + private sealed class SchemaNode + { + public readonly string Kind; + public readonly string Table; + public readonly string Property; + public readonly string Hierarchy; + public readonly string Level; + + public SchemaNode(string kind, string table, string property, string hierarchy, string level) + { + Kind = kind; + Table = table; + Property = property; + Hierarchy = hierarchy; + Level = level; + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static JObject GetSchema(Culture culture) + { + var payload = GetPayload(culture, false); + return SchemaFromEntities(payload["Entities"] as JObject); + } + + private static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) return new JObject { ["tables"] = orderedTables }; + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly object scintilla; + private readonly TextBox textBox; + private bool wordWrap; + + public Control Control { get; private set; } + public bool IsScintilla { get { return scintilla != null; } } + public event EventHandler TextChanged; + + private ScriptTextEditor(object scintillaControl, TextBox fallbackTextBox, bool initialWordWrap) + { + scintilla = scintillaControl; + textBox = fallbackTextBox; + Control = (Control)(scintillaControl ?? (object)fallbackTextBox); + wordWrap = initialWordWrap; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(string lexerName, bool wordWrap) + { + try + { + var assembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, "ScintillaNET", StringComparison.OrdinalIgnoreCase)) + ?? Assembly.Load("ScintillaNET"); + var type = assembly.GetType("ScintillaNET.Scintilla", true); + var control = (Control)Activator.CreateInstance(type); + ConfigureScintilla(control, lexerName, wordWrap); + return new ScriptTextEditor(control, null, wordWrap); + } + catch + { + var fallback = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(null, fallback, wordWrap); + } + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return wordWrap; } + set + { + wordWrap = value; + if (textBox != null) + { + textBox.WordWrap = value; + return; + } + + SetEnumProperty(scintilla, "WrapMode", value ? "Word" : "None"); + SetProperty(scintilla, "HScrollBar", !value); + } + } + + public void SelectStart() + { + if (textBox != null) + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + return; + } + + SetProperty(scintilla, "CurrentPosition", 0); + SetProperty(scintilla, "AnchorPosition", 0); + } + + public void Dispose() + { + Control?.Dispose(); + } + + private static void ConfigureScintilla(Control control, string lexerName, bool wordWrap) + { + control.Dock = DockStyle.Fill; + control.Font = new Font("Consolas", 10F); + control.BackColor = Color.White; + + var target = (object)control; + SetEnumProperty(target, "BorderStyle", "None"); + SetProperty(target, "LexerName", lexerName); + SetEnumProperty(target, "WrapMode", wordWrap ? "Word" : "None"); + SetEnumProperty(target, "WrapIndentMode", "Indent"); + SetProperty(target, "ScrollWidthTracking", true); + SetProperty(target, "MultipleSelection", true); + SetProperty(target, "AdditionalSelectionTyping", true); + SetProperty(target, "MouseSelectionRectangularSwitch", true); + SetProperty(target, "HScrollBar", !wordWrap); + SetProperty(target, "VScrollBar", true); + ConfigureMargins(target); + ConfigureContextMenu(control, target); + } + + private static void ConfigureMargins(object target) + { + var margins = GetProperty(target, "Margins"); + if (margins == null) return; + + var lineMargin = GetIndexerValue(margins, 0); + if (lineMargin != null) + { + SetProperty(lineMargin, "Width", 42); + SetEnumProperty(lineMargin, "Type", "Number"); + SetEnumProperty(lineMargin, "Cursor", "ReverseArrow"); + } + + var foldMargin = GetIndexerValue(margins, 2); + if (foldMargin != null) + { + SetProperty(foldMargin, "Width", 16); + SetProperty(foldMargin, "Sensitive", true); + SetEnumProperty(foldMargin, "Type", "Symbol"); + SetEnumProperty(foldMargin, "Cursor", "Arrow"); + } + } + + private static void ConfigureContextMenu(Control control, object target) + { + var menu = new ContextMenuStrip(); + AddMenuItem(menu, "Undo", () => InvokeNoArgs(target, "Undo")); + AddMenuItem(menu, "Redo", () => InvokeNoArgs(target, "Redo")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Cut", () => InvokeNoArgs(target, "Cut")); + AddMenuItem(menu, "Copy", () => InvokeNoArgs(target, "Copy")); + AddMenuItem(menu, "Paste", () => InvokeNoArgs(target, "Paste")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Select All", () => InvokeNoArgs(target, "SelectAll")); + control.ContextMenuStrip = menu; + } + + private static void AddMenuItem(ContextMenuStrip menu, string text, Action action) + { + var item = new ToolStripMenuItem(text); + item.Click += (sender, args) => + { + try { action(); } + catch { } + }; + menu.Items.Add(item); + } + + private static void InvokeNoArgs(object target, string methodName) + { + var method = target.GetType().GetMethod(methodName, BindingFlags.Instance | BindingFlags.Public); + if (method != null) method.Invoke(target, null); + } + + private static object GetProperty(object target, string propertyName) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + return prop == null ? null : prop.GetValue(target, null); + } + + private static object GetIndexerValue(object target, int index) + { + var prop = target.GetType().GetProperties() + .FirstOrDefault(p => p.GetIndexParameters().Length == 1 && p.GetIndexParameters()[0].ParameterType == typeof(int)); + return prop == null ? null : prop.GetValue(target, new object[] { index }); + } + + private static void SetProperty(object target, string propertyName, object value) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite) return; + prop.SetValue(target, value, null); + } + + private static void SetEnumProperty(object target, string propertyName, string enumValue) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite || !prop.PropertyType.IsEnum) return; + prop.SetValue(target, Enum.Parse(prop.PropertyType, enumValue), null); + } +} diff --git a/plugins/semantic-models/skills/semantic-model/scripts/get_semantic_model_ai_metadata.py b/plugins/semantic-models/skills/semantic-model/scripts/get_semantic_model_ai_metadata.py new file mode 100755 index 00000000..774bf486 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/get_semantic_model_ai_metadata.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +""" +Retrieve semantic model AI instructions and AI schema with the Fabric CLI. + +Primary usage: + python3 get_semantic_model_ai_metadata.py "Workspace.Workspace/Model.SemanticModel" + python3 get_semantic_model_ai_metadata.py "Workspace.Workspace/Model.SemanticModel" --instructions-out instructions.md --schema-out schema.json + +For offline parsing of a saved Fabric definition payload: + fab get "Workspace.Workspace/Model.SemanticModel" -q "definition" -f > definition.json + python3 get_semantic_model_ai_metadata.py --definition-file definition.json + +Source precedence: when multiple parts provide AI instructions or AI schemas, +friendly Copilot definition files (e.g. Copilot/Instructions/instructions.md) +rank before culture linguisticMetadata, regardless of part order. The first +entry of aiInstructions/aiSchemas is the winner used by --instructions-out, +--schema-out, and --format text; a warning is emitted when sources disagree. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + + +NO_METADATA_WARNING = "No AI instructions or AI schema metadata found in semantic model definition." + + +def configure_output_streams() -> None: + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors="replace") + except (OSError, ValueError): + pass + + +def run_fab_command(args: list[str]) -> str: + try: + result = subprocess.run( + ["fab", *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as exc: + message = exc.stderr.strip() or exc.stdout.strip() or str(exc) + print(f"Error running fab command: {message}", file=sys.stderr) + sys.exit(exc.returncode or 1) + except FileNotFoundError: + print("Error: fab CLI not found. Install ms-fabric-cli and run 'fab auth login'.", file=sys.stderr) + sys.exit(1) + + +def get_definition(model_path: str) -> dict[str, Any]: + output = run_fab_command(["get", model_path, "-q", "definition", "-f"]) + try: + payload = json.loads(output) + except json.JSONDecodeError as exc: + print(f"Error: fab did not return valid JSON: {exc}", file=sys.stderr) + sys.exit(1) + + return normalize_definition_payload(payload) + + +def load_definition_file(path: Path) -> dict[str, Any]: + try: + raw = path.read_bytes() + except OSError as exc: + print(f"Error reading {path}: {exc}", file=sys.stderr) + sys.exit(1) + text = decode_text_bytes(raw) + if text is None: + print(f"Error reading {path}: file is not valid UTF-8 or UTF-16 text.", file=sys.stderr) + sys.exit(1) + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + print(f"Error parsing {path}: {exc}", file=sys.stderr) + sys.exit(1) + + return normalize_definition_payload(payload) + + +def normalize_definition_payload(payload: Any) -> dict[str, Any]: + if isinstance(payload, dict) and isinstance(payload.get("definition"), dict): + payload = payload["definition"] + if not isinstance(payload, dict) or not isinstance(payload.get("parts"), list): + print("Error: expected a Fabric semantic model definition object with a 'parts' array.", file=sys.stderr) + sys.exit(1) + return payload + + +def definition_files(definition: dict[str, Any], warnings: list[str] | None = None) -> dict[str, str]: + files: dict[str, str] = {} + for part in definition.get("parts", []): + if not isinstance(part, dict): + continue + path = part.get("path") + if not isinstance(path, str) or not path: + continue + content = decode_part_payload(part) + if content is None: + if warnings is not None: + warnings.append(f"Could not decode payload as text for part: {path}") + continue + files[path.replace("\\", "/")] = content + return files + + +def decode_part_payload(part: dict[str, Any]) -> str | None: + payload = part.get("payload") + payload_type = str(part.get("payloadType") or "") + + if isinstance(payload, str): + if payload_type in {"InlineBase64", "DecodeBase64"}: + return try_decode_base64(payload) + return payload + + return json.dumps(payload) + + +def try_decode_base64(value: str) -> str | None: + try: + raw = base64.b64decode(value, validate=True) + except Exception: + return None + return decode_text_bytes(raw) + + +def decode_text_bytes(raw: bytes) -> str | None: + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + try: + return raw.decode("utf-16") + except UnicodeDecodeError: + return None + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError: + text = None + if text is not None and "\x00" not in text: + return text + for encoding in ("utf-16-le", "utf-16-be"): + try: + decoded = raw.decode(encoding) + except UnicodeDecodeError: + continue + if "\x00" not in decoded: + return decoded + return text + + +def parse_metadata(files: dict[str, str], culture_filter: str | None = None) -> dict[str, Any]: + result: dict[str, Any] = { + "aiInstructions": [], + "aiSchemas": [], + "aiSchemaObjects": [], + "sources": [], + "warnings": [], + } + + for source_path, text in files.items(): + lower = source_path.lower() + structured = try_parse_json(text) + is_culture_part = is_linguistic_metadata_path(lower) + if is_culture_part and lower.endswith(".tmdl"): + # TMDL culture files embed the linguistic metadata JSON after a + # "linguisticMetadata" token; .lsdl/.lsdl.json parts are the raw JSON itself. + tmdl_metadata = extract_tmdl_linguistic_metadata(text) + structured = try_parse_json(tmdl_metadata) if tmdl_metadata is not None else None + + if is_ai_instruction_path(lower): + instruction_text = extract_instruction_text(structured if structured is not None else text) + if instruction_text is None: + result["warnings"].append( + f"AI instructions part has no recognized instruction content: {source_path}" + ) + continue + result["aiInstructions"].append( + { + "sourcePath": source_path, + "format": "markdown" if lower.endswith((".md", ".markdown")) else "text", + "length": len(instruction_text), + "text": instruction_text, + } + ) + result["sources"].append({"kind": "aiInstructions", "path": source_path}) + continue + + if is_ai_schema_path(lower): + if not isinstance(structured, dict): + result["warnings"].append(f"AI schema part could not be parsed as JSON: {source_path}") + continue + schema = structured + result["aiSchemas"].append({"sourcePath": source_path, "schema": schema}) + add_ai_schema_objects(result, source_path, schema) + result["sources"].append({"kind": "aiSchema", "path": source_path}) + continue + + if lower.endswith(".bim"): + collect_tmsl_metadata(result, source_path, structured, culture_filter) + continue + + if is_culture_part and isinstance(structured, dict): + culture = culture_from_tmdl(source_path, text) + if culture_filter and culture and culture.lower() != culture_filter.lower(): + continue + collect_linguistic_metadata(result, source_path, structured, culture) + + # Deterministic precedence: friendly Copilot files rank before culture + # linguisticMetadata, independent of part order (see module docstring). + result["aiInstructions"].sort(key=source_precedence) + result["aiSchemas"].sort(key=source_precedence) + + if len(result["aiInstructions"]) > 1: + primary = result["aiInstructions"][0] + for other in result["aiInstructions"][1:]: + if other["text"] != primary["text"]: + result["warnings"].append( + "AI instructions differ between sources " + f"{primary['sourcePath']} and {other['sourcePath']}; using {primary['sourcePath']}." + ) + + if not result["aiInstructions"] and not result["aiSchemas"] and not result["aiSchemaObjects"]: + result["warnings"].append(NO_METADATA_WARNING) + + return result + + +def source_precedence(entry: dict[str, Any]) -> int: + return 1 if entry.get("storage") == "linguisticMetadata" else 0 + + +def collect_linguistic_metadata(result: dict[str, Any], source_path: str, payload: dict[str, Any], culture: str | None) -> None: + instructions = payload.get("CustomInstructions") or payload.get("customInstructions") + if isinstance(instructions, str): + result["aiInstructions"].append( + { + "sourcePath": source_path, + "storage": "linguisticMetadata", + "culture": culture, + "format": "markdown", + "length": len(instructions), + "text": instructions, + } + ) + result["sources"].append( + {"kind": "aiInstructions", "path": source_path, "storage": "linguisticMetadata", "culture": culture} + ) + + schema = schema_from_entities(payload.get("Entities") or payload.get("entities") or {}) + if schema["tables"]: + result["aiSchemas"].append( + { + "sourcePath": source_path, + "storage": "linguisticMetadata", + "culture": culture, + "schema": schema, + } + ) + add_ai_schema_objects(result, source_path, schema) + result["sources"].append( + {"kind": "aiSchema", "path": source_path, "storage": "linguisticMetadata", "culture": culture} + ) + + +def collect_tmsl_metadata(result: dict[str, Any], source_path: str, payload: Any, culture_filter: str | None) -> None: + """Walk a TMSL definition (model.bim): model.cultures[*].linguisticMetadata.content.""" + if not isinstance(payload, dict): + return + model = payload.get("model") + if not isinstance(model, dict): + return + cultures = model.get("cultures") + if not isinstance(cultures, list): + return + for entry in cultures: + if not isinstance(entry, dict): + continue + culture = entry.get("name") if isinstance(entry.get("name"), str) else None + if culture_filter and culture and culture.lower() != culture_filter.lower(): + continue + metadata = entry.get("linguisticMetadata") + if not isinstance(metadata, dict): + continue + content = metadata.get("content") + if isinstance(content, str): + content = try_parse_json(content) + if isinstance(content, dict): + collect_linguistic_metadata(result, source_path, content, culture) + + +def extract_tmdl_linguistic_metadata(text: str) -> str | None: + marker = text.find("linguisticMetadata") + if marker == -1: + return None + + start = text.find("{", marker) + if start == -1: + return None + + depth = 0 + in_string = False + escaped = False + for index in range(start, len(text)): + char = text[index] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return text[start : index + 1] + return None + + +def culture_from_tmdl(source_path: str, text: str) -> str | None: + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("cultureInfo "): + return stripped.removeprefix("cultureInfo ").strip().strip("'\"") + name = Path(source_path).stem + return name or None + + +def schema_from_entities(entities: Any) -> dict[str, Any]: + if not isinstance(entities, dict): + return {"tables": []} + + tables: dict[str, dict[str, Any]] = {} + + def table_for(name: str) -> dict[str, Any]: + if name not in tables: + tables[name] = {"name": name, "include": True, "columns": [], "hierarchies": []} + return tables[name] + + for entity in entities.values(): + if not isinstance(entity, dict): + continue + binding = entity.get("Binding") + if not isinstance(binding, dict): + definition = entity.get("Definition") + binding = definition.get("Binding") if isinstance(definition, dict) else None + if not isinstance(binding, dict): + continue + + table_name = binding.get("ConceptualEntity") or binding.get("conceptualEntity") + if not isinstance(table_name, str) or not table_name: + continue + + include = entity_included(entity) + table = table_for(table_name) + property_name = binding.get("ConceptualProperty") or binding.get("conceptualProperty") + hierarchy_name = binding.get("Hierarchy") or binding.get("hierarchy") + level_name = binding.get("HierarchyLevel") or binding.get("hierarchyLevel") + + if isinstance(level_name, str) and level_name: + hierarchy = hierarchy_for(table, hierarchy_name or "") + hierarchy.setdefault("levels", []).append({"name": level_name, "include": include}) + elif isinstance(hierarchy_name, str) and hierarchy_name: + hierarchy_for(table, hierarchy_name)["include"] = include + elif isinstance(property_name, str) and property_name: + table.setdefault("columns", []).append({"name": property_name, "include": include}) + else: + table["include"] = include + + table_list = list(tables.values()) + for table in table_list: + if not table.get("columns"): + table.pop("columns", None) + if not table.get("hierarchies"): + table.pop("hierarchies", None) + else: + for hierarchy in table["hierarchies"]: + if not hierarchy.get("levels"): + hierarchy.pop("levels", None) + return {"tables": table_list} + + +def hierarchy_for(table: dict[str, Any], name: str) -> dict[str, Any]: + hierarchies = table.setdefault("hierarchies", []) + for hierarchy in hierarchies: + if hierarchy.get("name") == name: + return hierarchy + hierarchy = {"name": name, "include": True, "levels": []} + hierarchies.append(hierarchy) + return hierarchy + + +def entity_included(entity: dict[str, Any]) -> bool: + state = str(entity.get("State") or entity.get("state") or "Generated").lower() + return state not in {"deleted", "hidden", "disabled"} + + +def add_ai_schema_objects(result: dict[str, Any], source_path: str, schema: dict[str, Any]) -> None: + for table_key, table in collection_entries(schema.get("tables") or schema.get("Tables")): + table_name = schema_object_name(table_key, table) + if not table_name: + continue + push_schema_object(result, source_path, table, {"type": "table", "table": table_name}) + + for column_key, column in collection_entries(get_child(table, "columns")): + column_name = schema_object_name(column_key, column) + if column_name: + push_schema_object(result, source_path, column, {"type": "column", "table": table_name, "property": column_name}) + + for measure_key, measure in collection_entries(get_child(table, "measures")): + measure_name = schema_object_name(measure_key, measure) + if measure_name: + push_schema_object(result, source_path, measure, {"type": "measure", "table": table_name, "property": measure_name}) + + for hierarchy_key, hierarchy in collection_entries(get_child(table, "hierarchies")): + hierarchy_name = schema_object_name(hierarchy_key, hierarchy) + if not hierarchy_name: + continue + push_schema_object(result, source_path, hierarchy, {"type": "hierarchy", "table": table_name, "hierarchy": hierarchy_name}) + for level_key, level in collection_entries(get_child(hierarchy, "levels")): + level_name = schema_object_name(level_key, level) + if level_name: + push_schema_object( + result, + source_path, + level, + {"type": "level", "table": table_name, "hierarchy": hierarchy_name, "level": level_name}, + ) + + +def push_schema_object(result: dict[str, Any], source_path: str, value: Any, obj: dict[str, Any]) -> None: + result["aiSchemaObjects"].append( + { + "sourcePath": source_path, + "object": obj, + "include": schema_include(value), + "visibility": schema_property(value, "visibility"), + "index": schema_property(value, "index"), + } + ) + + +def collection_entries(value: Any) -> list[tuple[str | None, Any]]: + if isinstance(value, list): + return [(schema_object_name(None, item), item) for item in value] + if isinstance(value, dict): + return list(value.items()) + return [] + + +def get_child(value: Any, name: str) -> Any: + if not isinstance(value, dict): + return None + return value.get(name) or value.get(name[:1].upper() + name[1:]) + + +def schema_object_name(key: str | None, value: Any) -> str | None: + if isinstance(value, dict): + candidate = value.get("name") or value.get("Name") or value.get("id") or value.get("Id") + if isinstance(candidate, str): + return candidate + return key + + +def schema_include(value: Any) -> bool | None: + if isinstance(value, bool): + return value + include = schema_property(value, "include") + if isinstance(include, bool): + return include + enabled = schema_property(value, "enabled") + if isinstance(enabled, bool): + return enabled + selected = schema_property(value, "selected") + if isinstance(selected, bool): + return selected + visibility = schema_property(value, "visibility") + if isinstance(visibility, str): + if visibility.lower() == "hidden": + return False + if visibility.lower() == "visible": + return True + return None + + +def schema_property(value: Any, name: str) -> Any: + if not isinstance(value, dict): + return None + return value.get(name) if name in value else value.get(name[:1].upper() + name[1:]) + + +def try_parse_json(text: str) -> Any: + stripped = text.lstrip("\ufeff").strip() + if not stripped.startswith(("{", "[")): + return None + try: + return json.loads(stripped) + except json.JSONDecodeError: + return None + + +def extract_instruction_text(payload: Any) -> str | None: + if isinstance(payload, str): + return payload.strip() + if not isinstance(payload, dict): + return None + for key in ["instructions", "aiInstructions", "systemInstructions", "copilotInstructions", "prompt"]: + value = payload.get(key) + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return None + + +def path_tokens(lower_path: str) -> set[str]: + return {token for token in re.split(r"[^a-z0-9]+", lower_path) if token} + + +def is_ai_instruction_path(lower_path: str) -> bool: + if "copilot/instructions/version.json" in lower_path: + return False + if lower_path.endswith("copilot/instructions/instructions.md"): + return True + tokens = path_tokens(lower_path) + return bool(tokens & {"instruction", "instructions", "prompt", "prompts"}) and bool(tokens & {"ai", "copilot"}) + + +def is_ai_schema_path(lower_path: str) -> bool: + return ( + "ai-schema" in lower_path + or "/ai/schema" in lower_path + or "copilot/schema" in lower_path + ) + + +def is_linguistic_metadata_path(lower_path: str) -> bool: + return lower_path.endswith(".lsdl") or lower_path.endswith(".lsdl.json") or ( + lower_path.endswith(".tmdl") and ("/cultures/" in lower_path or lower_path.startswith("cultures/")) + ) + + +def write_outputs(result: dict[str, Any], instructions_out: Path | None, schema_out: Path | None) -> None: + if instructions_out: + if result["aiInstructions"]: + instructions_out.parent.mkdir(parents=True, exist_ok=True) + instructions_out.write_text(result["aiInstructions"][0]["text"], encoding="utf-8") + else: + result["warnings"].append(f"No AI instructions found; not writing {instructions_out}.") + if schema_out: + if result["aiSchemas"]: + schema_out.parent.mkdir(parents=True, exist_ok=True) + schema_out.write_text(json.dumps(result["aiSchemas"][0]["schema"], indent=2) + "\n", encoding="utf-8") + else: + result["warnings"].append(f"No AI schema found; not writing {schema_out}.") + + +def main() -> None: + configure_output_streams() + parser = argparse.ArgumentParser(description="Retrieve semantic model AI instructions and AI schema with fab.") + parser.add_argument("model", nargs="?", help='Fabric path: "Workspace.Workspace/Model.SemanticModel"') + parser.add_argument("--definition-file", type=Path, help="Parse a saved fab definition JSON instead of calling fab.") + parser.add_argument("--culture", help="Culture to use when multiple TMDL culture metadata files exist.") + parser.add_argument("--instructions-out", type=Path, help="Write the first AI instructions payload to this file.") + parser.add_argument("--schema-out", type=Path, help="Write the first AI schema payload to this JSON file.") + parser.add_argument("--format", choices=["json", "text"], default="json", help="Output format. Default: json.") + args = parser.parse_args() + + if not args.definition_file and not args.model: + parser.error("provide a semantic model path or --definition-file") + + definition = load_definition_file(args.definition_file) if args.definition_file else get_definition(args.model) + decode_warnings: list[str] = [] + files = definition_files(definition, decode_warnings) + result = parse_metadata(files, args.culture) + result["warnings"].extend(decode_warnings) + result["model"] = args.model or str(args.definition_file) + result["partCount"] = len(files) + + write_outputs(result, args.instructions_out, args.schema_out) + + if args.format == "text": + for warning in result["warnings"]: + print(warning, file=sys.stderr) + if result["aiInstructions"]: + print(result["aiInstructions"][0]["text"]) + return + sys.exit(1) + + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata-interactive.csx b/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata-interactive.csx new file mode 100644 index 00000000..b9c5ec84 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata-interactive.csx @@ -0,0 +1,574 @@ +#r "System.Drawing" + +// Interactive TE3 macro for semantic model AI instructions and AI schema. +// It edits culture linguistic metadata: +// CustomInstructions -> Copilot/Instructions/instructions.md equivalent +// Entities -> Copilot/schema.json equivalent +// +// The UI is created through reflection so this script still compiles in the +// headless te CLI, where System.Windows.Forms is not available on macOS/Linux. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +if (Model.Cultures.Count == 0) +{ + Model.AddTranslation("en-US"); +} + +var ui = FormsUi.TryCreate(); +if (ui == null) +{ + Error("This interactive script requires Tabular Editor 3 Desktop with System.Windows.Forms. Use manage-ai-metadata.csx for te CLI automation."); + return; +} + +ScriptHelper.WaitFormVisible = false; + +var font = new Font("Segoe UI", 10); +var monoFont = new Font("Consolas", 10); + +dynamic form = ui.New("Form"); +form.Text = "Semantic Model AI Metadata"; +form.StartPosition = ui.Enum("FormStartPosition", "CenterScreen"); +form.Width = 980; +form.Height = 720; +form.MinimumSize = new Size(820, 520); + +dynamic cultureLabel = ui.New("Label"); +cultureLabel.Text = "Culture"; +cultureLabel.Left = 16; +cultureLabel.Top = 18; +cultureLabel.Width = 60; +cultureLabel.Font = font; + +dynamic cultureCombo = ui.New("ComboBox"); +cultureCombo.Left = 82; +cultureCombo.Top = 14; +cultureCombo.Width = 190; +cultureCombo.DropDownStyle = ui.Enum("ComboBoxStyle", "DropDownList"); +cultureCombo.Font = font; +foreach (var culture in Model.Cultures) cultureCombo.Items.Add(culture.Name); +var preferredCulture = Model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) ?? Model.Cultures.First(); +cultureCombo.SelectedItem = preferredCulture.Name; + +dynamic targetLabel = ui.New("Label"); +targetLabel.Text = "Target"; +targetLabel.Left = 288; +targetLabel.Top = 18; +targetLabel.Width = 52; +targetLabel.Font = font; + +dynamic targetCombo = ui.New("ComboBox"); +targetCombo.Left = 346; +targetCombo.Top = 14; +targetCombo.Width = 160; +targetCombo.DropDownStyle = ui.Enum("ComboBoxStyle", "DropDownList"); +targetCombo.Font = font; +targetCombo.Items.Add("Instructions"); +targetCombo.Items.Add("Schema JSON"); +targetCombo.SelectedIndex = 0; + +dynamic statusLabel = ui.New("Label"); +statusLabel.Left = 522; +statusLabel.Top = 18; +statusLabel.Width = 420; +statusLabel.Height = 24; +statusLabel.Font = font; +statusLabel.TextAlign = ContentAlignment.MiddleLeft; + +dynamic editor = ui.New("TextBox"); +editor.Left = 16; +editor.Top = 54; +editor.Width = 940; +editor.Height = 590; +editor.Multiline = true; +editor.ScrollBars = ui.Enum("ScrollBars", "Both"); +editor.WordWrap = false; +editor.AcceptsReturn = true; +editor.AcceptsTab = true; +editor.Font = monoFont; + +dynamic deleteButton = ui.New("Button"); +deleteButton.Text = "Delete"; +deleteButton.Left = 676; +deleteButton.Top = 652; +deleteButton.Width = 88; +deleteButton.Font = font; + +dynamic saveButton = ui.New("Button"); +saveButton.Text = "Save"; +saveButton.Left = 772; +saveButton.Top = 652; +saveButton.Width = 88; +saveButton.Font = font; + +dynamic closeButton = ui.New("Button"); +closeButton.Text = "Close"; +closeButton.Left = 868; +closeButton.Top = 652; +closeButton.Width = 88; +closeButton.Font = font; + +form.Controls.Add(cultureLabel); +form.Controls.Add(cultureCombo); +form.Controls.Add(targetLabel); +form.Controls.Add(targetCombo); +form.Controls.Add(statusLabel); +form.Controls.Add(editor); +form.Controls.Add(deleteButton); +form.Controls.Add(saveButton); +form.Controls.Add(closeButton); + +Func selectedCulture = () => Model.Cultures[(string)cultureCombo.SelectedItem]; +Func editingInstructions = () => ((string)targetCombo.SelectedItem) == "Instructions"; + +Action refreshStatus = () => +{ + if (editingInstructions()) + { + var count = ((string)editor.Text).Length; + statusLabel.Text = count + " / " + AiMetadataInteractive.InstructionsLimit + " characters"; + statusLabel.ForeColor = count > AiMetadataInteractive.InstructionsLimit ? Color.Firebrick : SystemColors.ControlText; + saveButton.Enabled = count <= AiMetadataInteractive.InstructionsLimit; + } + else + { + statusLabel.Text = "Copilot schema JSON"; + statusLabel.ForeColor = SystemColors.ControlText; + saveButton.Enabled = true; + } +}; + +Action loadEditor = () => +{ + var culture = selectedCulture(); + if (editingInstructions()) + { + editor.Text = AiMetadataInteractive.GetInstructions(culture); + } + else + { + editor.Text = AiMetadataInteractive.GetSchema(culture).ToString(Formatting.Indented); + } + refreshStatus(); +}; + +ui.On((object)cultureCombo, "SelectedIndexChanged", new EventHandler((sender, args) => loadEditor())); +ui.On((object)targetCombo, "SelectedIndexChanged", new EventHandler((sender, args) => loadEditor())); +ui.On((object)editor, "TextChanged", new EventHandler((sender, args) => refreshStatus())); + +ui.On((object)saveButton, "Click", new EventHandler((sender, args) => +{ + try + { + var culture = selectedCulture(); + var text = (string)editor.Text; + if (editingInstructions()) + { + if (text.Length > AiMetadataInteractive.InstructionsLimit) + { + statusLabel.Text = "AI instructions must be 10000 characters or fewer."; + statusLabel.ForeColor = Color.Firebrick; + return; + } + AiMetadataInteractive.SetInstructions(culture, text); + statusLabel.Text = "AI instructions saved to " + culture.Name + "."; + } + else + { + var schema = JObject.Parse(text); + AiMetadataInteractive.SetSchema(culture, schema); + editor.Text = AiMetadataInteractive.GetSchema(culture).ToString(Formatting.Indented); + statusLabel.Text = "AI schema saved to " + culture.Name + "."; + } + } + catch (Exception ex) + { + statusLabel.Text = ex.Message; + statusLabel.ForeColor = Color.Firebrick; + } +})); + +ui.On((object)deleteButton, "Click", new EventHandler((sender, args) => +{ + if (editingInstructions()) AiMetadataInteractive.DeleteInstructions(selectedCulture()); + else AiMetadataInteractive.DeleteSchema(selectedCulture()); + loadEditor(); +})); + +ui.On((object)closeButton, "Click", new EventHandler((sender, args) => form.Close())); + +loadEditor(); +form.ShowDialog(); + +public sealed class FormsUi +{ + private readonly Assembly _forms; + + private FormsUi(Assembly forms) + { + _forms = forms; + } + + public static FormsUi TryCreate() + { + var forms = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "System.Windows.Forms"); + if (forms == null) + { + try + { + forms = Assembly.Load("System.Windows.Forms"); + } + catch + { + return null; + } + } + return new FormsUi(forms); + } + + public dynamic New(string typeName) + { + var type = _forms.GetType("System.Windows.Forms." + typeName, true); + return Activator.CreateInstance(type); + } + + public object Enum(string typeName, string value) + { + var type = _forms.GetType("System.Windows.Forms." + typeName, true); + return System.Enum.Parse(type, value); + } + + public void On(object target, string eventName, EventHandler handler) + { + target.GetType().GetEvent(eventName).AddEventHandler(target, handler); + } +} + +public static class AiMetadataInteractive +{ + public const int InstructionsLimit = 10000; + + public static string GetInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + return (string)payload["CustomInstructions"] ?? ""; + } + + public static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + public static void DeleteInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + payload.Remove("CustomInstructions"); + SavePayload(culture, payload); + } + + public static JObject GetSchema(Culture culture) + { + var payload = GetPayload(culture, false); + return SchemaFromEntities(payload["Entities"] as JObject); + } + + public static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + public static void DeleteSchema(Culture culture) + { + var payload = GetPayload(culture, false); + payload.Remove("Entities"); + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) return new JObject { ["tables"] = orderedTables }; + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } +} diff --git a/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata.csx b/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata.csx new file mode 100644 index 00000000..91848721 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/scripts/manage-ai-metadata.csx @@ -0,0 +1,612 @@ +// Manage semantic model AI instructions and AI schema from te script. +// +// Non-interactive usage: +// TE_AI_ACTION=get TE_AI_TARGET=both te script -S manage-ai-metadata.csx -m ./model --output-format json +// TE_AI_ACTION=set TE_AI_TARGET=instructions TE_AI_INPUT_FILE=./instructions.md te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=set TE_AI_TARGET=schema TE_AI_INPUT_FILE=./schema.json te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=delete TE_AI_TARGET=schema te script -S manage-ai-metadata.csx -m ./model --save +// +// Environment variables: +// TE_AI_ACTION list | get | set | delete. Default: get. +// TE_AI_TARGET instructions | schema | both. Default: both for get/list, required for set/delete. +// TE_AI_CULTURE Culture name to use. Default: first culture with linguistic metadata, then first culture, then en-US on set. +// TE_AI_INPUT_FILE File to read for set. +// TE_AI_INPUT Inline payload to use for set when TE_AI_INPUT_FILE is not set. +// TE_AI_OUTPUT_FILE Optional file path for JSON/text output. +// TE_AI_ALLOW_OVER_LIMIT=true permits instructions longer than 10000 characters. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +var action = AiMetadata.Env("TE_AI_ACTION", "get").Trim().ToLowerInvariant(); +var target = AiMetadata.Env("TE_AI_TARGET", action == "set" || action == "delete" ? "" : "both").Trim().ToLowerInvariant(); +var cultureName = AiMetadata.Env("TE_AI_CULTURE", "").Trim(); +var outputFile = AiMetadata.Env("TE_AI_OUTPUT_FILE", "").Trim(); + +// Write a machine-readable error envelope (stdout / TE_AI_OUTPUT_FILE) before +// reporting the error, so failures never leave a stale success payload behind. +// The envelope write must never mask the original error with its own exception +// (e.g. an unwritable TE_AI_OUTPUT_FILE), so it is best-effort. +void Fail(string message) +{ + try + { + AiMetadata.WriteResult(new JObject + { + ["error"] = message, + ["action"] = action, + ["target"] = target, + ["culture"] = cultureName + }, outputFile); + } + catch (Exception writeEx) + { + Error("Failed to write error envelope: " + writeEx.Message); + } + Error(message); +} + +if (action != "list" && action != "get" && action != "set" && action != "delete") +{ + Fail("TE_AI_ACTION must be 'list', 'get', 'set', or 'delete'."); + return; +} + +try +{ + if (action == "list") + { + AiMetadata.WriteResult(AiMetadata.ListCultures(Model), outputFile); + return; + } + + if (target != "instructions" && target != "schema" && target != "both") + { + Fail("TE_AI_TARGET must be 'instructions', 'schema', or 'both'."); + return; + } + + var culture = AiMetadata.FindCulture(Model, cultureName, action == "set"); + if (culture == null) + { + Fail("No culture is available on this model. Add a culture before managing AI metadata."); + return; + } + + if (action == "get") + { + var result = AiMetadata.Read(Model, culture, target); + AiMetadata.WriteResult(result, outputFile); + return; + } + + if (action == "set") + { + var input = AiMetadata.ReadInput(); + if (target == "instructions") + { + if (input.Length > AiMetadata.InstructionsLimit && !AiMetadata.AllowOverLimit()) + { + Fail("AI instructions are " + input.Length + " characters. Limit is " + AiMetadata.InstructionsLimit + ". Set TE_AI_ALLOW_OVER_LIMIT=true to override."); + return; + } + AiMetadata.SetInstructions(culture, input); + } + else if (target == "schema") + { + var schema = AiMetadata.ResolveSchemaInput(JObject.Parse(input)); + if (schema == null) + { + Fail("No tables found in input. Expected {\"tables\": [...]} or the get output envelope."); + return; + } + AiMetadata.SetSchema(culture, schema); + } + else + { + Fail("TE_AI_TARGET=both is not valid for set. Set instructions and schema in separate calls."); + return; + } + + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } + + if (action == "delete") + { + if (target == "instructions" || target == "both") AiMetadata.DeleteInstructions(culture); + if (target == "schema" || target == "both") AiMetadata.DeleteSchema(culture); + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } +} +catch (Exception ex) +{ + Fail(ex.Message); +} + +public static class AiMetadata +{ + public const int InstructionsLimit = 10000; + + public static string Env(string name, string fallback) + { + var value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? fallback : value; + } + + public static bool AllowOverLimit() + { + return string.Equals(Env("TE_AI_ALLOW_OVER_LIMIT", ""), "true", StringComparison.OrdinalIgnoreCase); + } + + public static string ReadInput() + { + var inputFile = Env("TE_AI_INPUT_FILE", "").Trim(); + if (!string.IsNullOrWhiteSpace(inputFile)) return File.ReadAllText(inputFile); + + var input = Environment.GetEnvironmentVariable("TE_AI_INPUT"); + if (input != null) return input; + + throw new InvalidOperationException("Set TE_AI_INPUT_FILE or TE_AI_INPUT for TE_AI_ACTION=set."); + } + + public static void WriteResult(JToken result, string outputFile) + { + var text = result.ToString(Formatting.Indented); + if (!string.IsNullOrWhiteSpace(outputFile)) + { + var dir = Path.GetDirectoryName(Path.GetFullPath(outputFile)); + if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir); + File.WriteAllText(outputFile, text + Environment.NewLine); + Console.WriteLine("Wrote " + outputFile); + return; + } + + Console.WriteLine(text); + } + + public static Culture FindCulture(TabularEditor.TOMWrapper.Model model, string cultureName, bool createIfMissing) + { + if (!string.IsNullOrWhiteSpace(cultureName)) + { + if (!model.Cultures.Contains(cultureName)) + { + if (createIfMissing) return model.AddTranslation(cultureName); + throw new InvalidOperationException("Culture '" + cultureName + "' was not found."); + } + return model.Cultures[cultureName]; + } + + var withMetadata = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)); + if (withMetadata != null) return withMetadata; + var firstCulture = model.Cultures.FirstOrDefault(); + if (firstCulture != null) return firstCulture; + return createIfMissing ? model.AddTranslation("en-US") : null; + } + + public static JArray ListCultures(TabularEditor.TOMWrapper.Model model) + { + var cultures = new JArray(); + foreach (var culture in model.Cultures) + { + var payload = TryParsePayload(culture); + var entities = payload?["Entities"] as JObject; + cultures.Add(new JObject + { + ["name"] = culture.Name, + ["hasLinguisticMetadata"] = !string.IsNullOrWhiteSpace(culture.Content), + ["hasAiInstructions"] = payload?["CustomInstructions"] != null, + ["schemaObjectCount"] = entities == null ? 0 : entities.Properties().Count() + }); + } + return cultures; + } + + public static JObject Read(TabularEditor.TOMWrapper.Model model, Culture culture, string target) + { + var payload = GetPayload(culture, false); + var result = new JObject + { + ["model"] = model.Name, + ["culture"] = culture.Name, + ["storage"] = "culture.linguisticMetadata", + ["copilotTooling"] = HasCopilotTooling(model) + }; + + if (target == "instructions" || target == "both") + { + var instructions = (string)payload["CustomInstructions"]; + result["aiInstructions"] = new JObject + { + ["exists"] = instructions != null, + ["length"] = instructions == null ? 0 : instructions.Length, + ["limit"] = InstructionsLimit, + ["text"] = instructions ?? "" + }; + } + + if (target == "schema" || target == "both") + { + var schema = SchemaFromEntities(payload["Entities"] as JObject); + result["aiSchema"] = schema; + result["schemaObjectCount"] = CountSchemaObjects(schema); + } + + return result; + } + + public static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + public static void DeleteInstructions(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("CustomInstructions")) return; + SavePayload(culture, payload); + } + + public static JObject ResolveSchemaInput(JObject input) + { + if (input == null) return null; + if (HasTablesCollection(input)) return input; + if (input["aiSchema"] is JObject envelope && HasTablesCollection(envelope)) return envelope; + return null; + } + + private static bool HasTablesCollection(JObject candidate) + { + // A JSON null value parses to a JValue, not a reference null, so a + // bare null check would let {"tables": null} through and wipe Entities. + return candidate["tables"] is JContainer || candidate["Tables"] is JContainer; + } + + public static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + public static void DeleteSchema(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("Entities")) return; + SavePayload(culture, payload); + } + + private static bool HasCopilotTooling(TabularEditor.TOMWrapper.Model model) + { + var value = model.GetAnnotation("PBI_ProTooling"); + return value != null && value.IndexOf("CopilotTooling", StringComparison.OrdinalIgnoreCase) >= 0; + } + + private static JObject TryParsePayload(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return null; + try + { + return JObject.Parse(culture.Content); + } + catch + { + return null; + } + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + try + { + return JObject.Parse(culture.Content); + } + catch (Exception ex) + { + throw new InvalidOperationException("Culture '" + culture.Name + "' linguistic metadata is not valid JSON: " + ex.Message); + } + } + + if (!create) + { + return new JObject(); + } + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) + { + return new JObject { ["tables"] = orderedTables }; + } + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static int CountSchemaObjects(JObject schema) + { + var count = 0; + foreach (var table in (schema["tables"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (table["columns"] as JArray ?? new JArray()).Count; + count += (table["measures"] as JArray ?? new JArray()).Count; + foreach (var hierarchy in (table["hierarchies"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (hierarchy["levels"] as JArray ?? new JArray()).Count; + } + } + return count; + } +} diff --git a/plugins/tabular-editor/skills/te-cli/SKILL.md b/plugins/tabular-editor/skills/te-cli/SKILL.md index 37924f6f..7614428a 100644 --- a/plugins/tabular-editor/skills/te-cli/SKILL.md +++ b/plugins/tabular-editor/skills/te-cli/SKILL.md @@ -162,6 +162,18 @@ For build scripts that issue many `te` calls, set `te config set bpa.onSave fals Gate any cross-tool refactor with `te validate` before touching the report or the service, and remember every `te` mutation stages in memory until `--save`. +## Bundled scripts + +- `scripts/manage-ai-metadata.csx` - non-interactive `te script` CRUD for + semantic model AI instructions (`CustomInstructions`) and AI schema + (`Entities`) stored in culture linguistic metadata. +- `scripts/edit-ai-instructions-interactive.csx` - TE3 Desktop GUI editor for + semantic model AI instructions. +- `scripts/edit-ai-schema-interactive.csx` - TE3 Desktop GUI editor for + semantic model AI schema. +- `scripts/manage-ai-metadata-interactive.csx` - original combined TE3 Desktop + editor prototype. + ## References Bundled (load as needed): diff --git a/plugins/tabular-editor/skills/te-cli/scripts/README.md b/plugins/tabular-editor/skills/te-cli/scripts/README.md new file mode 100644 index 00000000..f3bec706 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/scripts/README.md @@ -0,0 +1,56 @@ +# TE CLI Scripts + +Utility C# scripts for `te script`. Pass `--output-format json` for agent use, +and add `--save` when setting or deleting metadata. + +## Semantic model AI metadata + +### manage-ai-metadata.csx + +Read, set, list, or delete semantic model AI instructions and AI schema stored +in culture linguistic metadata: + +- `CustomInstructions` maps to Copilot instructions. +- `Entities` maps to the semantic model AI schema. + +```bash +TE_AI_ACTION=get TE_AI_TARGET=both \ + te script -s "workspace" -d "model" \ + -S scripts/manage-ai-metadata.csx \ + --output-format json --non-interactive +``` + +```bash +TE_AI_ACTION=set TE_AI_TARGET=instructions TE_AI_INPUT_FILE=instructions.md \ + te script -s "workspace" -d "model" \ + -S scripts/manage-ai-metadata.csx \ + --save --output-format json --non-interactive +``` + +Environment variables: + +- `TE_AI_ACTION`: `list`, `get`, `set`, or `delete`. Default: `get`. +- `TE_AI_TARGET`: `instructions`, `schema`, or `both`. +- `TE_AI_CULTURE`: optional culture name. Defaults to the best available + culture and creates `en-US` on `set` when needed. +- `TE_AI_INPUT_FILE`: payload file for `set`. +- `TE_AI_INPUT`: inline payload for `set`. +- `TE_AI_OUTPUT_FILE`: optional output file. +- `TE_AI_ALLOW_OVER_LIMIT=true`: allow instructions over 10000 characters. + +### edit-ai-instructions-interactive.csx + +TE3 Desktop GUI editor for AI instructions. It uses the connected model, +defaults to `en-US`, does not require a selected object, and enforces the +10000 character guard. + +### edit-ai-schema-interactive.csx + +TE3 Desktop GUI editor for AI schema. It opens on a perspective-editor-style +object tree and includes a JSON tab for exact schema roundtrips. + +### manage-ai-metadata-interactive.csx + +Original combined TE3 Desktop prototype for editing both AI instructions and +AI schema in one dialog. Prefer the two focused GUI editors above for normal +interactive work. diff --git a/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-instructions-interactive.csx b/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-instructions-interactive.csx new file mode 100644 index 00000000..f7437ad5 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-instructions-interactive.csx @@ -0,0 +1,517 @@ +#r "System.Drawing" + +// TE3 Desktop GUI editor for semantic model AI instructions. +// Uses Model.Cultures["en-US"].Content -> CustomInstructions. + +using System; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiInstructionsEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiInstructionsEditor.RootMessage(ex), "Semantic Model AI Instructions"); +} + +public static class AiInstructionsEditor +{ + private const string DefaultCultureName = "en-US"; + private const int InstructionsLimit = 10000; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Instructions"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Instructions"); + return; + } + + using (var form = new Form()) + using (var editor = ScriptTextEditor.Create("markdown", true)) + { + form.Text = "Semantic Model AI Instructions"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 980; + form.Height = 760; + form.MinimumSize = new Size(760, 520); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + + (editor.IsScintilla ? " Editor: Scintilla" : " Editor: TextBox") + }; + + var editorPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + editorPanel.Controls.Add(editor.Control); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Margin = new Padding(0, 0, 8, 0) + }; + + var wrapButton = NewFooterButton("Wrap", font); + var reloadButton = NewFooterButton("Reload", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(wrapButton); + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(editorPanel, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + Action refreshStatus = () => + { + var count = NormalizeForStorage(editor.Text).Length; + status.Text = count + " / " + InstructionsLimit + " characters" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + status.ForeColor = count > InstructionsLimit ? Color.Firebrick : + string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + saveButton.Enabled = count <= InstructionsLimit; + }; + + Action load = () => + { + try + { + editor.Text = NormalizeForEditor(GetInstructions(culture)); + editor.SelectStart(); + refreshStatus(); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + editor.TextChanged += (sender, args) => refreshStatus(); + wrapButton.Click += (sender, args) => editor.WordWrap = !editor.WordWrap; + reloadButton.Click += (sender, args) => load(); + saveButton.Click += (sender, args) => + { + try + { + var text = NormalizeForStorage(editor.Text); + if (text.Length > InstructionsLimit) + { + status.Text = "AI instructions must be 10000 characters or fewer."; + status.ForeColor = Color.Firebrick; + return; + } + + SetInstructions(culture, text); + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + load(); + form.Shown += (sender, args) => editor.Focus(); + form.ShowDialog(); + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static string GetInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + return (string)payload["CustomInstructions"] ?? ""; + } + + private static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + private static string NormalizeForStorage(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n"); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly object scintilla; + private readonly TextBox textBox; + private bool wordWrap; + + public Control Control { get; private set; } + public bool IsScintilla { get { return scintilla != null; } } + public event EventHandler TextChanged; + + private ScriptTextEditor(object scintillaControl, TextBox fallbackTextBox, bool initialWordWrap) + { + scintilla = scintillaControl; + textBox = fallbackTextBox; + Control = (Control)(scintillaControl ?? (object)fallbackTextBox); + wordWrap = initialWordWrap; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(string lexerName, bool wordWrap) + { + try + { + var assembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, "ScintillaNET", StringComparison.OrdinalIgnoreCase)) + ?? Assembly.Load("ScintillaNET"); + var type = assembly.GetType("ScintillaNET.Scintilla", true); + var control = (Control)Activator.CreateInstance(type); + ConfigureScintilla(control, lexerName, wordWrap); + return new ScriptTextEditor(control, null, wordWrap); + } + catch + { + var fallback = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(null, fallback, wordWrap); + } + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return wordWrap; } + set + { + wordWrap = value; + if (textBox != null) + { + textBox.WordWrap = value; + return; + } + + SetEnumProperty(scintilla, "WrapMode", value ? "Word" : "None"); + SetProperty(scintilla, "HScrollBar", !value); + } + } + + public void Focus() + { + Control.Focus(); + } + + public void SelectStart() + { + if (textBox != null) + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + return; + } + + SetProperty(scintilla, "CurrentPosition", 0); + SetProperty(scintilla, "AnchorPosition", 0); + } + + public void Dispose() + { + Control?.Dispose(); + } + + private static void ConfigureScintilla(Control control, string lexerName, bool wordWrap) + { + control.Dock = DockStyle.Fill; + control.Font = new Font("Consolas", 10F); + control.BackColor = Color.White; + + var target = (object)control; + SetEnumProperty(target, "BorderStyle", "None"); + SetProperty(target, "LexerName", lexerName); + SetEnumProperty(target, "WrapMode", wordWrap ? "Word" : "None"); + SetEnumProperty(target, "WrapIndentMode", "Indent"); + SetProperty(target, "ScrollWidthTracking", true); + SetProperty(target, "MultipleSelection", true); + SetProperty(target, "AdditionalSelectionTyping", true); + SetProperty(target, "MouseSelectionRectangularSwitch", true); + SetProperty(target, "HScrollBar", !wordWrap); + SetProperty(target, "VScrollBar", true); + ConfigureMargins(target); + ConfigureContextMenu(control, target); + } + + private static void ConfigureMargins(object target) + { + var margins = GetProperty(target, "Margins"); + if (margins == null) return; + + var lineMargin = GetIndexerValue(margins, 0); + if (lineMargin != null) + { + SetProperty(lineMargin, "Width", 42); + SetEnumProperty(lineMargin, "Type", "Number"); + SetEnumProperty(lineMargin, "Cursor", "ReverseArrow"); + } + + var foldMargin = GetIndexerValue(margins, 2); + if (foldMargin != null) + { + SetProperty(foldMargin, "Width", 16); + SetProperty(foldMargin, "Sensitive", true); + SetEnumProperty(foldMargin, "Type", "Symbol"); + SetEnumProperty(foldMargin, "Cursor", "Arrow"); + } + } + + private static void ConfigureContextMenu(Control control, object target) + { + var menu = new ContextMenuStrip(); + AddMenuItem(menu, "Undo", () => InvokeNoArgs(target, "Undo")); + AddMenuItem(menu, "Redo", () => InvokeNoArgs(target, "Redo")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Cut", () => InvokeNoArgs(target, "Cut")); + AddMenuItem(menu, "Copy", () => InvokeNoArgs(target, "Copy")); + AddMenuItem(menu, "Paste", () => InvokeNoArgs(target, "Paste")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Select All", () => InvokeNoArgs(target, "SelectAll")); + control.ContextMenuStrip = menu; + } + + private static void AddMenuItem(ContextMenuStrip menu, string text, Action action) + { + var item = new ToolStripMenuItem(text); + item.Click += (sender, args) => + { + try { action(); } + catch { } + }; + menu.Items.Add(item); + } + + private static void InvokeNoArgs(object target, string methodName) + { + var method = target.GetType().GetMethod(methodName, BindingFlags.Instance | BindingFlags.Public); + if (method != null) method.Invoke(target, null); + } + + private static object GetProperty(object target, string propertyName) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + return prop == null ? null : prop.GetValue(target, null); + } + + private static object GetIndexerValue(object target, int index) + { + var prop = target.GetType().GetProperties() + .FirstOrDefault(p => p.GetIndexParameters().Length == 1 && p.GetIndexParameters()[0].ParameterType == typeof(int)); + return prop == null ? null : prop.GetValue(target, new object[] { index }); + } + + private static void SetProperty(object target, string propertyName, object value) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite) return; + prop.SetValue(target, value, null); + } + + private static void SetEnumProperty(object target, string propertyName, string enumValue) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite || !prop.PropertyType.IsEnum) return; + prop.SetValue(target, Enum.Parse(prop.PropertyType, enumValue), null); + } +} diff --git a/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-schema-interactive.csx b/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-schema-interactive.csx new file mode 100644 index 00000000..80b77368 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/scripts/edit-ai-schema-interactive.csx @@ -0,0 +1,1251 @@ +#r "System.Drawing" + +// TE3 Desktop GUI editor for semantic model AI schema. +// Uses Model.Cultures["en-US"].Content -> Entities. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiSchemaEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiSchemaEditor.RootMessage(ex), "Semantic Model AI Schema"); +} + +public static class AiSchemaEditor +{ + private const string DefaultCultureName = "en-US"; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Schema"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Schema"); + return; + } + + using (var form = new Form()) + using (var jsonEditor = ScriptTextEditor.Create("json", false)) + using (var stateImages = BuildStateImages()) + { + form.Text = "Semantic Model AI Schema"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 1040; + form.Height = 780; + form.MinimumSize = new Size(820, 560); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + }; + + var tabs = new TabControl + { + Dock = DockStyle.Fill, + Font = font + }; + var treePage = new TabPage("Objects"); + var jsonPage = new TabPage("JSON"); + tabs.TabPages.Add(treePage); + tabs.TabPages.Add(jsonPage); + + var tree = new TreeView + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.None, + Font = font, + HideSelection = false, + ShowLines = true, + ShowPlusMinus = true, + ShowRootLines = true, + StateImageList = stateImages + }; + + var treePanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle + }; + treePanel.Controls.Add(tree); + + var toolbar = new FlowLayoutPanel + { + Dock = DockStyle.Top, + Height = 44, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Padding = new Padding(0, 6, 0, 4) + }; + var showHidden = new CheckBox { Text = "Show hidden", Checked = true, AutoSize = true, Font = font, Padding = new Padding(0, 7, 12, 0) }; + var checkAllButton = NewToolbarButton("Check all", font); + var clearButton = NewToolbarButton("Clear", font); + var expandButton = NewToolbarButton("Expand", font); + var collapseButton = NewToolbarButton("Collapse", font); + toolbar.Controls.Add(showHidden); + toolbar.Controls.Add(checkAllButton); + toolbar.Controls.Add(clearButton); + toolbar.Controls.Add(expandButton); + toolbar.Controls.Add(collapseButton); + + var treeLayout = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0) }; + treeLayout.Controls.Add(treePanel); + treeLayout.Controls.Add(toolbar); + treePage.Controls.Add(treeLayout); + + var jsonPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + jsonPanel.Controls.Add(jsonEditor.Control); + jsonPage.Controls.Add(jsonPanel); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod, + Margin = new Padding(0, 0, 8, 0) + }; + + var reloadButton = NewFooterButton("Reload", font); + var formatButton = NewFooterButton("Format JSON", font); + var updateJsonButton = NewFooterButton("Update JSON", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(formatButton); + buttonStrip.Controls.Add(updateJsonButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(tabs, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + bool updatingTree = false; + + Action loadSchemaIntoUi = schema => + { + updatingTree = true; + try + { + BuildTree(tree, model, schema, showHidden.Checked); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + } + finally + { + updatingTree = false; + } + }; + + Action reload = () => + { + try + { + loadSchemaIntoUi(GetSchema(culture)); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + tree.NodeMouseClick += (sender, args) => + { + if (updatingTree) return; + try + { + ToggleNode(args.Node); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + showHidden.CheckedChanged += (sender, args) => + { + if (updatingTree) return; + try + { + var current = SchemaFromTree(tree); + loadSchemaIntoUi(current); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + checkAllButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, CheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + clearButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, UncheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + expandButton.Click += (sender, args) => tree.ExpandAll(); + collapseButton.Click += (sender, args) => tree.CollapseAll(); + + reloadButton.Click += (sender, args) => reload(); + formatButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(JObject.Parse(jsonEditor.Text).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + status.Text = "JSON formatted."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + updateJsonButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(SchemaFromTree(tree).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + jsonEditor.SelectStart(); + status.Text = "JSON updated from object tree."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + saveButton.Click += (sender, args) => + { + try + { + JObject schema; + if (tabs.SelectedTab == jsonPage) + { + schema = JObject.Parse(jsonEditor.Text); + SetSchema(culture, schema); + loadSchemaIntoUi(schema); + tabs.SelectedTab = jsonPage; + } + else + { + schema = SchemaFromTree(tree); + SetSchema(culture, schema); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + } + + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + reload(); + form.ShowDialog(); + } + } + + private const int UncheckedState = 0; + private const int CheckedState = 1; + private const int MixedState = 2; + + private static void BuildTree(TreeView tree, TabularEditor.TOMWrapper.Model model, JObject schema, bool showHidden) + { + tree.BeginUpdate(); + try + { + tree.Nodes.Clear(); + var index = BuildSchemaIndex(schema); + var hasSchema = index.Count > 0; + + foreach (var table in model.Tables.OrderBy(t => t.Name)) + { + if (!showHidden && !IsVisibleObject(table)) continue; + + var tableNode = NewNode(table.Name, new SchemaNode("table", table.Name, null, null, null)); + tree.Nodes.Add(tableNode); + + foreach (var column in table.Columns.OrderBy(c => c.Name)) + { + if (!showHidden && !IsVisibleObject(column)) continue; + var node = NewNode(column.Name + " column", new SchemaNode("column", table.Name, column.Name, null, null)); + node.ForeColor = IsVisibleObject(column) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(column)); + tableNode.Nodes.Add(node); + } + + foreach (var measure in table.Measures.OrderBy(m => m.Name)) + { + if (!showHidden && !IsVisibleObject(measure)) continue; + var node = NewNode(measure.Name + " measure", new SchemaNode("measure", table.Name, measure.Name, null, null)); + node.ForeColor = IsVisibleObject(measure) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(measure)); + tableNode.Nodes.Add(node); + } + + foreach (var hierarchy in table.Hierarchies.OrderBy(h => h.Name)) + { + if (!showHidden && !IsVisibleObject(hierarchy)) continue; + var hierarchyNode = NewNode(hierarchy.Name + " hierarchy", new SchemaNode("hierarchy", table.Name, null, hierarchy.Name, null)); + hierarchyNode.ForeColor = IsVisibleObject(hierarchy) ? SystemColors.WindowText : SystemColors.GrayText; + hierarchyNode.StateImageIndex = IncludeState(index, hierarchyNode.Tag as SchemaNode, hasSchema, IsVisibleObject(hierarchy)); + tableNode.Nodes.Add(hierarchyNode); + + foreach (var level in hierarchy.Levels.OrderBy(l => l.Name)) + { + var levelNode = NewNode(level.Name + " level", new SchemaNode("level", table.Name, null, hierarchy.Name, level.Name)); + levelNode.StateImageIndex = IncludeState(index, levelNode.Tag as SchemaNode, hasSchema, true); + hierarchyNode.Nodes.Add(levelNode); + } + + if (hierarchyNode.Nodes.Count > 0 && !index.ContainsKey(Key(hierarchyNode.Tag as SchemaNode))) + { + hierarchyNode.StateImageIndex = AggregateState(hierarchyNode); + } + } + + tableNode.ForeColor = IsVisibleObject(table) ? SystemColors.WindowText : SystemColors.GrayText; + tableNode.StateImageIndex = index.ContainsKey(Key(tableNode.Tag as SchemaNode)) + ? index[Key(tableNode.Tag as SchemaNode)] ? CheckedState : UncheckedState + : tableNode.Nodes.Count == 0 ? (IsVisibleObject(table) || !hasSchema ? CheckedState : UncheckedState) : AggregateState(tableNode); + tableNode.Expand(); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static Button NewToolbarButton(string text, Font font) + { + return new Button + { + Text = text, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Font = font, + Margin = new Padding(6, 0, 0, 0), + MinimumSize = new Size(96, 30), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static TreeNode NewNode(string text, SchemaNode tag) + { + return new TreeNode(text) + { + Tag = tag, + StateImageIndex = CheckedState + }; + } + + private static bool IsVisibleObject(object obj) + { + if (obj is IHideableObject hideable) return hideable.IsVisible; + return true; + } + + private static int IncludeState(Dictionary index, SchemaNode node, bool hasSchema, bool visible) + { + var key = Key(node); + if (index.ContainsKey(key)) return index[key] ? CheckedState : UncheckedState; + return !hasSchema || visible ? CheckedState : UncheckedState; + } + + private static Dictionary BuildSchemaIndex(JObject schema) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tableEntry in CollectionEntries(schema?["tables"] ?? schema?["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + result[Key(new SchemaNode("table", tableName, null, null, null))] = IncludeValue(tableEntry.Value); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var name = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("column", tableName, name, null, null))] = IncludeValue(columnEntry.Value); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var name = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("measure", tableName, name, null, null))] = IncludeValue(measureEntry.Value); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var name = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(name)) continue; + result[Key(new SchemaNode("hierarchy", tableName, null, name, null))] = IncludeValue(hierarchyEntry.Value); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var level = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(level)) result[Key(new SchemaNode("level", tableName, null, name, level))] = IncludeValue(levelEntry.Value); + } + } + } + return result; + } + + private static string Key(SchemaNode node) + { + if (node == null) return ""; + if (node.Kind == "table") return "T|" + node.Table; + if (node.Kind == "column" || node.Kind == "measure") return "P|" + node.Table + "|" + node.Property; + if (node.Kind == "hierarchy") return "H|" + node.Table + "|" + node.Hierarchy; + if (node.Kind == "level") return "L|" + node.Table + "|" + node.Hierarchy + "|" + node.Level; + return ""; + } + + private static void ToggleNode(TreeNode node) + { + var next = node.StateImageIndex == CheckedState ? UncheckedState : CheckedState; + SetNodeAndChildren(node, next); + RefreshAncestors(node.Parent); + } + + private static void SetAllTreeNodes(TreeView tree, int state) + { + tree.BeginUpdate(); + try + { + foreach (TreeNode node in tree.Nodes) + { + SetNodeAndChildren(node, state); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static void SetNodeAndChildren(TreeNode node, int state) + { + node.StateImageIndex = state; + foreach (TreeNode child in node.Nodes) + { + SetNodeAndChildren(child, state); + } + } + + private static void RefreshAncestors(TreeNode node) + { + while (node != null) + { + node.StateImageIndex = AggregateState(node); + node = node.Parent; + } + } + + private static int AggregateState(TreeNode node) + { + if (node.Nodes.Count == 0) return node.StateImageIndex == CheckedState ? CheckedState : UncheckedState; + + var checkedCount = 0; + var uncheckedCount = 0; + foreach (TreeNode child in node.Nodes) + { + if (child.StateImageIndex == CheckedState) checkedCount++; + else if (child.StateImageIndex == UncheckedState) uncheckedCount++; + else return MixedState; + } + + if (checkedCount == node.Nodes.Count) return CheckedState; + if (uncheckedCount == node.Nodes.Count) return UncheckedState; + return MixedState; + } + + private static JObject SchemaFromTree(TreeView tree) + { + var tables = new JArray(); + foreach (TreeNode tableNode in tree.Nodes) + { + var tableTag = tableNode.Tag as SchemaNode; + if (tableTag == null || tableTag.Kind != "table") continue; + + var table = new JObject + { + ["name"] = tableTag.Table, + ["include"] = tableNode.StateImageIndex != UncheckedState + }; + var columns = new JArray(); + var measures = new JArray(); + var hierarchies = new JArray(); + + foreach (TreeNode child in tableNode.Nodes) + { + var tag = child.Tag as SchemaNode; + if (tag == null) continue; + + if (tag.Kind == "column") + { + columns.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "measure") + { + measures.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "hierarchy") + { + var hierarchy = new JObject + { + ["name"] = tag.Hierarchy, + ["include"] = child.StateImageIndex != UncheckedState + }; + var levels = new JArray(); + foreach (TreeNode levelNode in child.Nodes) + { + var levelTag = levelNode.Tag as SchemaNode; + if (levelTag != null && levelTag.Kind == "level") + { + levels.Add(new JObject { ["name"] = levelTag.Level, ["include"] = levelNode.StateImageIndex == CheckedState }); + } + } + if (levels.Count > 0) hierarchy["levels"] = levels; + hierarchies.Add(hierarchy); + } + } + + if (columns.Count > 0) table["columns"] = columns; + if (measures.Count > 0) table["measures"] = measures; + if (hierarchies.Count > 0) table["hierarchies"] = hierarchies; + tables.Add(table); + } + + return new JObject { ["tables"] = tables }; + } + + private static string StatusText(TreeView tree, string startupWarning) + { + var total = 0; + var included = 0; + CountNodes(tree.Nodes, ref total, ref included); + return included + " included / " + total + " objects" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + } + + private static void CountNodes(TreeNodeCollection nodes, ref int total, ref int included) + { + foreach (TreeNode node in nodes) + { + if (node.Tag is SchemaNode) + { + total++; + if (node.StateImageIndex != UncheckedState) included++; + } + CountNodes(node.Nodes, ref total, ref included); + } + } + + private static ImageList BuildStateImages() + { + var list = new ImageList { ImageSize = new Size(16, 16), ColorDepth = ColorDepth.Depth32Bit }; + list.Images.Add(DrawStateImage(UncheckedState)); + list.Images.Add(DrawStateImage(CheckedState)); + list.Images.Add(DrawStateImage(MixedState)); + return list; + } + + private static Bitmap DrawStateImage(int state) + { + var bmp = new Bitmap(16, 16); + using (var g = Graphics.FromImage(bmp)) + using (var border = new Pen(Color.FromArgb(120, 120, 120))) + using (var fill = new SolidBrush(Color.White)) + using (var mark = new Pen(Color.FromArgb(0, 120, 215), 2F)) + using (var mixed = new SolidBrush(Color.FromArgb(0, 120, 215))) + { + g.Clear(Color.Transparent); + g.FillRectangle(fill, 2, 2, 12, 12); + g.DrawRectangle(border, 2, 2, 12, 12); + if (state == CheckedState) + { + g.DrawLines(mark, new[] { new Point(4, 8), new Point(7, 11), new Point(12, 5) }); + } + else if (state == MixedState) + { + g.FillRectangle(mixed, 5, 7, 6, 2); + } + } + return bmp; + } + + private sealed class SchemaNode + { + public readonly string Kind; + public readonly string Table; + public readonly string Property; + public readonly string Hierarchy; + public readonly string Level; + + public SchemaNode(string kind, string table, string property, string hierarchy, string level) + { + Kind = kind; + Table = table; + Property = property; + Hierarchy = hierarchy; + Level = level; + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static JObject GetSchema(Culture culture) + { + var payload = GetPayload(culture, false); + return SchemaFromEntities(payload["Entities"] as JObject); + } + + private static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) return new JObject { ["tables"] = orderedTables }; + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly object scintilla; + private readonly TextBox textBox; + private bool wordWrap; + + public Control Control { get; private set; } + public bool IsScintilla { get { return scintilla != null; } } + public event EventHandler TextChanged; + + private ScriptTextEditor(object scintillaControl, TextBox fallbackTextBox, bool initialWordWrap) + { + scintilla = scintillaControl; + textBox = fallbackTextBox; + Control = (Control)(scintillaControl ?? (object)fallbackTextBox); + wordWrap = initialWordWrap; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(string lexerName, bool wordWrap) + { + try + { + var assembly = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => string.Equals(a.GetName().Name, "ScintillaNET", StringComparison.OrdinalIgnoreCase)) + ?? Assembly.Load("ScintillaNET"); + var type = assembly.GetType("ScintillaNET.Scintilla", true); + var control = (Control)Activator.CreateInstance(type); + ConfigureScintilla(control, lexerName, wordWrap); + return new ScriptTextEditor(control, null, wordWrap); + } + catch + { + var fallback = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(null, fallback, wordWrap); + } + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return wordWrap; } + set + { + wordWrap = value; + if (textBox != null) + { + textBox.WordWrap = value; + return; + } + + SetEnumProperty(scintilla, "WrapMode", value ? "Word" : "None"); + SetProperty(scintilla, "HScrollBar", !value); + } + } + + public void SelectStart() + { + if (textBox != null) + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + return; + } + + SetProperty(scintilla, "CurrentPosition", 0); + SetProperty(scintilla, "AnchorPosition", 0); + } + + public void Dispose() + { + Control?.Dispose(); + } + + private static void ConfigureScintilla(Control control, string lexerName, bool wordWrap) + { + control.Dock = DockStyle.Fill; + control.Font = new Font("Consolas", 10F); + control.BackColor = Color.White; + + var target = (object)control; + SetEnumProperty(target, "BorderStyle", "None"); + SetProperty(target, "LexerName", lexerName); + SetEnumProperty(target, "WrapMode", wordWrap ? "Word" : "None"); + SetEnumProperty(target, "WrapIndentMode", "Indent"); + SetProperty(target, "ScrollWidthTracking", true); + SetProperty(target, "MultipleSelection", true); + SetProperty(target, "AdditionalSelectionTyping", true); + SetProperty(target, "MouseSelectionRectangularSwitch", true); + SetProperty(target, "HScrollBar", !wordWrap); + SetProperty(target, "VScrollBar", true); + ConfigureMargins(target); + ConfigureContextMenu(control, target); + } + + private static void ConfigureMargins(object target) + { + var margins = GetProperty(target, "Margins"); + if (margins == null) return; + + var lineMargin = GetIndexerValue(margins, 0); + if (lineMargin != null) + { + SetProperty(lineMargin, "Width", 42); + SetEnumProperty(lineMargin, "Type", "Number"); + SetEnumProperty(lineMargin, "Cursor", "ReverseArrow"); + } + + var foldMargin = GetIndexerValue(margins, 2); + if (foldMargin != null) + { + SetProperty(foldMargin, "Width", 16); + SetProperty(foldMargin, "Sensitive", true); + SetEnumProperty(foldMargin, "Type", "Symbol"); + SetEnumProperty(foldMargin, "Cursor", "Arrow"); + } + } + + private static void ConfigureContextMenu(Control control, object target) + { + var menu = new ContextMenuStrip(); + AddMenuItem(menu, "Undo", () => InvokeNoArgs(target, "Undo")); + AddMenuItem(menu, "Redo", () => InvokeNoArgs(target, "Redo")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Cut", () => InvokeNoArgs(target, "Cut")); + AddMenuItem(menu, "Copy", () => InvokeNoArgs(target, "Copy")); + AddMenuItem(menu, "Paste", () => InvokeNoArgs(target, "Paste")); + menu.Items.Add(new ToolStripSeparator()); + AddMenuItem(menu, "Select All", () => InvokeNoArgs(target, "SelectAll")); + control.ContextMenuStrip = menu; + } + + private static void AddMenuItem(ContextMenuStrip menu, string text, Action action) + { + var item = new ToolStripMenuItem(text); + item.Click += (sender, args) => + { + try { action(); } + catch { } + }; + menu.Items.Add(item); + } + + private static void InvokeNoArgs(object target, string methodName) + { + var method = target.GetType().GetMethod(methodName, BindingFlags.Instance | BindingFlags.Public); + if (method != null) method.Invoke(target, null); + } + + private static object GetProperty(object target, string propertyName) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + return prop == null ? null : prop.GetValue(target, null); + } + + private static object GetIndexerValue(object target, int index) + { + var prop = target.GetType().GetProperties() + .FirstOrDefault(p => p.GetIndexParameters().Length == 1 && p.GetIndexParameters()[0].ParameterType == typeof(int)); + return prop == null ? null : prop.GetValue(target, new object[] { index }); + } + + private static void SetProperty(object target, string propertyName, object value) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite) return; + prop.SetValue(target, value, null); + } + + private static void SetEnumProperty(object target, string propertyName, string enumValue) + { + var prop = target.GetType().GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public); + if (prop == null || !prop.CanWrite || !prop.PropertyType.IsEnum) return; + prop.SetValue(target, Enum.Parse(prop.PropertyType, enumValue), null); + } +} diff --git a/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata-interactive.csx b/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata-interactive.csx new file mode 100644 index 00000000..b9c5ec84 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata-interactive.csx @@ -0,0 +1,574 @@ +#r "System.Drawing" + +// Interactive TE3 macro for semantic model AI instructions and AI schema. +// It edits culture linguistic metadata: +// CustomInstructions -> Copilot/Instructions/instructions.md equivalent +// Entities -> Copilot/schema.json equivalent +// +// The UI is created through reflection so this script still compiles in the +// headless te CLI, where System.Windows.Forms is not available on macOS/Linux. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +if (Model.Cultures.Count == 0) +{ + Model.AddTranslation("en-US"); +} + +var ui = FormsUi.TryCreate(); +if (ui == null) +{ + Error("This interactive script requires Tabular Editor 3 Desktop with System.Windows.Forms. Use manage-ai-metadata.csx for te CLI automation."); + return; +} + +ScriptHelper.WaitFormVisible = false; + +var font = new Font("Segoe UI", 10); +var monoFont = new Font("Consolas", 10); + +dynamic form = ui.New("Form"); +form.Text = "Semantic Model AI Metadata"; +form.StartPosition = ui.Enum("FormStartPosition", "CenterScreen"); +form.Width = 980; +form.Height = 720; +form.MinimumSize = new Size(820, 520); + +dynamic cultureLabel = ui.New("Label"); +cultureLabel.Text = "Culture"; +cultureLabel.Left = 16; +cultureLabel.Top = 18; +cultureLabel.Width = 60; +cultureLabel.Font = font; + +dynamic cultureCombo = ui.New("ComboBox"); +cultureCombo.Left = 82; +cultureCombo.Top = 14; +cultureCombo.Width = 190; +cultureCombo.DropDownStyle = ui.Enum("ComboBoxStyle", "DropDownList"); +cultureCombo.Font = font; +foreach (var culture in Model.Cultures) cultureCombo.Items.Add(culture.Name); +var preferredCulture = Model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) ?? Model.Cultures.First(); +cultureCombo.SelectedItem = preferredCulture.Name; + +dynamic targetLabel = ui.New("Label"); +targetLabel.Text = "Target"; +targetLabel.Left = 288; +targetLabel.Top = 18; +targetLabel.Width = 52; +targetLabel.Font = font; + +dynamic targetCombo = ui.New("ComboBox"); +targetCombo.Left = 346; +targetCombo.Top = 14; +targetCombo.Width = 160; +targetCombo.DropDownStyle = ui.Enum("ComboBoxStyle", "DropDownList"); +targetCombo.Font = font; +targetCombo.Items.Add("Instructions"); +targetCombo.Items.Add("Schema JSON"); +targetCombo.SelectedIndex = 0; + +dynamic statusLabel = ui.New("Label"); +statusLabel.Left = 522; +statusLabel.Top = 18; +statusLabel.Width = 420; +statusLabel.Height = 24; +statusLabel.Font = font; +statusLabel.TextAlign = ContentAlignment.MiddleLeft; + +dynamic editor = ui.New("TextBox"); +editor.Left = 16; +editor.Top = 54; +editor.Width = 940; +editor.Height = 590; +editor.Multiline = true; +editor.ScrollBars = ui.Enum("ScrollBars", "Both"); +editor.WordWrap = false; +editor.AcceptsReturn = true; +editor.AcceptsTab = true; +editor.Font = monoFont; + +dynamic deleteButton = ui.New("Button"); +deleteButton.Text = "Delete"; +deleteButton.Left = 676; +deleteButton.Top = 652; +deleteButton.Width = 88; +deleteButton.Font = font; + +dynamic saveButton = ui.New("Button"); +saveButton.Text = "Save"; +saveButton.Left = 772; +saveButton.Top = 652; +saveButton.Width = 88; +saveButton.Font = font; + +dynamic closeButton = ui.New("Button"); +closeButton.Text = "Close"; +closeButton.Left = 868; +closeButton.Top = 652; +closeButton.Width = 88; +closeButton.Font = font; + +form.Controls.Add(cultureLabel); +form.Controls.Add(cultureCombo); +form.Controls.Add(targetLabel); +form.Controls.Add(targetCombo); +form.Controls.Add(statusLabel); +form.Controls.Add(editor); +form.Controls.Add(deleteButton); +form.Controls.Add(saveButton); +form.Controls.Add(closeButton); + +Func selectedCulture = () => Model.Cultures[(string)cultureCombo.SelectedItem]; +Func editingInstructions = () => ((string)targetCombo.SelectedItem) == "Instructions"; + +Action refreshStatus = () => +{ + if (editingInstructions()) + { + var count = ((string)editor.Text).Length; + statusLabel.Text = count + " / " + AiMetadataInteractive.InstructionsLimit + " characters"; + statusLabel.ForeColor = count > AiMetadataInteractive.InstructionsLimit ? Color.Firebrick : SystemColors.ControlText; + saveButton.Enabled = count <= AiMetadataInteractive.InstructionsLimit; + } + else + { + statusLabel.Text = "Copilot schema JSON"; + statusLabel.ForeColor = SystemColors.ControlText; + saveButton.Enabled = true; + } +}; + +Action loadEditor = () => +{ + var culture = selectedCulture(); + if (editingInstructions()) + { + editor.Text = AiMetadataInteractive.GetInstructions(culture); + } + else + { + editor.Text = AiMetadataInteractive.GetSchema(culture).ToString(Formatting.Indented); + } + refreshStatus(); +}; + +ui.On((object)cultureCombo, "SelectedIndexChanged", new EventHandler((sender, args) => loadEditor())); +ui.On((object)targetCombo, "SelectedIndexChanged", new EventHandler((sender, args) => loadEditor())); +ui.On((object)editor, "TextChanged", new EventHandler((sender, args) => refreshStatus())); + +ui.On((object)saveButton, "Click", new EventHandler((sender, args) => +{ + try + { + var culture = selectedCulture(); + var text = (string)editor.Text; + if (editingInstructions()) + { + if (text.Length > AiMetadataInteractive.InstructionsLimit) + { + statusLabel.Text = "AI instructions must be 10000 characters or fewer."; + statusLabel.ForeColor = Color.Firebrick; + return; + } + AiMetadataInteractive.SetInstructions(culture, text); + statusLabel.Text = "AI instructions saved to " + culture.Name + "."; + } + else + { + var schema = JObject.Parse(text); + AiMetadataInteractive.SetSchema(culture, schema); + editor.Text = AiMetadataInteractive.GetSchema(culture).ToString(Formatting.Indented); + statusLabel.Text = "AI schema saved to " + culture.Name + "."; + } + } + catch (Exception ex) + { + statusLabel.Text = ex.Message; + statusLabel.ForeColor = Color.Firebrick; + } +})); + +ui.On((object)deleteButton, "Click", new EventHandler((sender, args) => +{ + if (editingInstructions()) AiMetadataInteractive.DeleteInstructions(selectedCulture()); + else AiMetadataInteractive.DeleteSchema(selectedCulture()); + loadEditor(); +})); + +ui.On((object)closeButton, "Click", new EventHandler((sender, args) => form.Close())); + +loadEditor(); +form.ShowDialog(); + +public sealed class FormsUi +{ + private readonly Assembly _forms; + + private FormsUi(Assembly forms) + { + _forms = forms; + } + + public static FormsUi TryCreate() + { + var forms = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "System.Windows.Forms"); + if (forms == null) + { + try + { + forms = Assembly.Load("System.Windows.Forms"); + } + catch + { + return null; + } + } + return new FormsUi(forms); + } + + public dynamic New(string typeName) + { + var type = _forms.GetType("System.Windows.Forms." + typeName, true); + return Activator.CreateInstance(type); + } + + public object Enum(string typeName, string value) + { + var type = _forms.GetType("System.Windows.Forms." + typeName, true); + return System.Enum.Parse(type, value); + } + + public void On(object target, string eventName, EventHandler handler) + { + target.GetType().GetEvent(eventName).AddEventHandler(target, handler); + } +} + +public static class AiMetadataInteractive +{ + public const int InstructionsLimit = 10000; + + public static string GetInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + return (string)payload["CustomInstructions"] ?? ""; + } + + public static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + public static void DeleteInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + payload.Remove("CustomInstructions"); + SavePayload(culture, payload); + } + + public static JObject GetSchema(Culture culture) + { + var payload = GetPayload(culture, false); + return SchemaFromEntities(payload["Entities"] as JObject); + } + + public static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + public static void DeleteSchema(Culture culture) + { + var payload = GetPayload(culture, false); + payload.Remove("Entities"); + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) return new JObject { ["tables"] = orderedTables }; + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } +} diff --git a/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata.csx b/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata.csx new file mode 100644 index 00000000..91848721 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/scripts/manage-ai-metadata.csx @@ -0,0 +1,612 @@ +// Manage semantic model AI instructions and AI schema from te script. +// +// Non-interactive usage: +// TE_AI_ACTION=get TE_AI_TARGET=both te script -S manage-ai-metadata.csx -m ./model --output-format json +// TE_AI_ACTION=set TE_AI_TARGET=instructions TE_AI_INPUT_FILE=./instructions.md te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=set TE_AI_TARGET=schema TE_AI_INPUT_FILE=./schema.json te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=delete TE_AI_TARGET=schema te script -S manage-ai-metadata.csx -m ./model --save +// +// Environment variables: +// TE_AI_ACTION list | get | set | delete. Default: get. +// TE_AI_TARGET instructions | schema | both. Default: both for get/list, required for set/delete. +// TE_AI_CULTURE Culture name to use. Default: first culture with linguistic metadata, then first culture, then en-US on set. +// TE_AI_INPUT_FILE File to read for set. +// TE_AI_INPUT Inline payload to use for set when TE_AI_INPUT_FILE is not set. +// TE_AI_OUTPUT_FILE Optional file path for JSON/text output. +// TE_AI_ALLOW_OVER_LIMIT=true permits instructions longer than 10000 characters. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +var action = AiMetadata.Env("TE_AI_ACTION", "get").Trim().ToLowerInvariant(); +var target = AiMetadata.Env("TE_AI_TARGET", action == "set" || action == "delete" ? "" : "both").Trim().ToLowerInvariant(); +var cultureName = AiMetadata.Env("TE_AI_CULTURE", "").Trim(); +var outputFile = AiMetadata.Env("TE_AI_OUTPUT_FILE", "").Trim(); + +// Write a machine-readable error envelope (stdout / TE_AI_OUTPUT_FILE) before +// reporting the error, so failures never leave a stale success payload behind. +// The envelope write must never mask the original error with its own exception +// (e.g. an unwritable TE_AI_OUTPUT_FILE), so it is best-effort. +void Fail(string message) +{ + try + { + AiMetadata.WriteResult(new JObject + { + ["error"] = message, + ["action"] = action, + ["target"] = target, + ["culture"] = cultureName + }, outputFile); + } + catch (Exception writeEx) + { + Error("Failed to write error envelope: " + writeEx.Message); + } + Error(message); +} + +if (action != "list" && action != "get" && action != "set" && action != "delete") +{ + Fail("TE_AI_ACTION must be 'list', 'get', 'set', or 'delete'."); + return; +} + +try +{ + if (action == "list") + { + AiMetadata.WriteResult(AiMetadata.ListCultures(Model), outputFile); + return; + } + + if (target != "instructions" && target != "schema" && target != "both") + { + Fail("TE_AI_TARGET must be 'instructions', 'schema', or 'both'."); + return; + } + + var culture = AiMetadata.FindCulture(Model, cultureName, action == "set"); + if (culture == null) + { + Fail("No culture is available on this model. Add a culture before managing AI metadata."); + return; + } + + if (action == "get") + { + var result = AiMetadata.Read(Model, culture, target); + AiMetadata.WriteResult(result, outputFile); + return; + } + + if (action == "set") + { + var input = AiMetadata.ReadInput(); + if (target == "instructions") + { + if (input.Length > AiMetadata.InstructionsLimit && !AiMetadata.AllowOverLimit()) + { + Fail("AI instructions are " + input.Length + " characters. Limit is " + AiMetadata.InstructionsLimit + ". Set TE_AI_ALLOW_OVER_LIMIT=true to override."); + return; + } + AiMetadata.SetInstructions(culture, input); + } + else if (target == "schema") + { + var schema = AiMetadata.ResolveSchemaInput(JObject.Parse(input)); + if (schema == null) + { + Fail("No tables found in input. Expected {\"tables\": [...]} or the get output envelope."); + return; + } + AiMetadata.SetSchema(culture, schema); + } + else + { + Fail("TE_AI_TARGET=both is not valid for set. Set instructions and schema in separate calls."); + return; + } + + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } + + if (action == "delete") + { + if (target == "instructions" || target == "both") AiMetadata.DeleteInstructions(culture); + if (target == "schema" || target == "both") AiMetadata.DeleteSchema(culture); + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } +} +catch (Exception ex) +{ + Fail(ex.Message); +} + +public static class AiMetadata +{ + public const int InstructionsLimit = 10000; + + public static string Env(string name, string fallback) + { + var value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? fallback : value; + } + + public static bool AllowOverLimit() + { + return string.Equals(Env("TE_AI_ALLOW_OVER_LIMIT", ""), "true", StringComparison.OrdinalIgnoreCase); + } + + public static string ReadInput() + { + var inputFile = Env("TE_AI_INPUT_FILE", "").Trim(); + if (!string.IsNullOrWhiteSpace(inputFile)) return File.ReadAllText(inputFile); + + var input = Environment.GetEnvironmentVariable("TE_AI_INPUT"); + if (input != null) return input; + + throw new InvalidOperationException("Set TE_AI_INPUT_FILE or TE_AI_INPUT for TE_AI_ACTION=set."); + } + + public static void WriteResult(JToken result, string outputFile) + { + var text = result.ToString(Formatting.Indented); + if (!string.IsNullOrWhiteSpace(outputFile)) + { + var dir = Path.GetDirectoryName(Path.GetFullPath(outputFile)); + if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir); + File.WriteAllText(outputFile, text + Environment.NewLine); + Console.WriteLine("Wrote " + outputFile); + return; + } + + Console.WriteLine(text); + } + + public static Culture FindCulture(TabularEditor.TOMWrapper.Model model, string cultureName, bool createIfMissing) + { + if (!string.IsNullOrWhiteSpace(cultureName)) + { + if (!model.Cultures.Contains(cultureName)) + { + if (createIfMissing) return model.AddTranslation(cultureName); + throw new InvalidOperationException("Culture '" + cultureName + "' was not found."); + } + return model.Cultures[cultureName]; + } + + var withMetadata = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)); + if (withMetadata != null) return withMetadata; + var firstCulture = model.Cultures.FirstOrDefault(); + if (firstCulture != null) return firstCulture; + return createIfMissing ? model.AddTranslation("en-US") : null; + } + + public static JArray ListCultures(TabularEditor.TOMWrapper.Model model) + { + var cultures = new JArray(); + foreach (var culture in model.Cultures) + { + var payload = TryParsePayload(culture); + var entities = payload?["Entities"] as JObject; + cultures.Add(new JObject + { + ["name"] = culture.Name, + ["hasLinguisticMetadata"] = !string.IsNullOrWhiteSpace(culture.Content), + ["hasAiInstructions"] = payload?["CustomInstructions"] != null, + ["schemaObjectCount"] = entities == null ? 0 : entities.Properties().Count() + }); + } + return cultures; + } + + public static JObject Read(TabularEditor.TOMWrapper.Model model, Culture culture, string target) + { + var payload = GetPayload(culture, false); + var result = new JObject + { + ["model"] = model.Name, + ["culture"] = culture.Name, + ["storage"] = "culture.linguisticMetadata", + ["copilotTooling"] = HasCopilotTooling(model) + }; + + if (target == "instructions" || target == "both") + { + var instructions = (string)payload["CustomInstructions"]; + result["aiInstructions"] = new JObject + { + ["exists"] = instructions != null, + ["length"] = instructions == null ? 0 : instructions.Length, + ["limit"] = InstructionsLimit, + ["text"] = instructions ?? "" + }; + } + + if (target == "schema" || target == "both") + { + var schema = SchemaFromEntities(payload["Entities"] as JObject); + result["aiSchema"] = schema; + result["schemaObjectCount"] = CountSchemaObjects(schema); + } + + return result; + } + + public static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + public static void DeleteInstructions(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("CustomInstructions")) return; + SavePayload(culture, payload); + } + + public static JObject ResolveSchemaInput(JObject input) + { + if (input == null) return null; + if (HasTablesCollection(input)) return input; + if (input["aiSchema"] is JObject envelope && HasTablesCollection(envelope)) return envelope; + return null; + } + + private static bool HasTablesCollection(JObject candidate) + { + // A JSON null value parses to a JValue, not a reference null, so a + // bare null check would let {"tables": null} through and wipe Entities. + return candidate["tables"] is JContainer || candidate["Tables"] is JContainer; + } + + public static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + public static void DeleteSchema(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("Entities")) return; + SavePayload(culture, payload); + } + + private static bool HasCopilotTooling(TabularEditor.TOMWrapper.Model model) + { + var value = model.GetAnnotation("PBI_ProTooling"); + return value != null && value.IndexOf("CopilotTooling", StringComparison.OrdinalIgnoreCase) >= 0; + } + + private static JObject TryParsePayload(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return null; + try + { + return JObject.Parse(culture.Content); + } + catch + { + return null; + } + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + try + { + return JObject.Parse(culture.Content); + } + catch (Exception ex) + { + throw new InvalidOperationException("Culture '" + culture.Name + "' linguistic metadata is not valid JSON: " + ex.Message); + } + } + + if (!create) + { + return new JObject(); + } + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) + { + return new JObject { ["tables"] = orderedTables }; + } + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static int CountSchemaObjects(JObject schema) + { + var count = 0; + foreach (var table in (schema["tables"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (table["columns"] as JArray ?? new JArray()).Count; + count += (table["measures"] as JArray ?? new JArray()).Count; + foreach (var hierarchy in (table["hierarchies"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (hierarchy["levels"] as JArray ?? new JArray()).Count; + } + } + return count; + } +}