diff --git a/frontend/src2/charts/components/NewMeasureSelectorDialog.vue b/frontend/src2/charts/components/NewMeasureSelectorDialog.vue index 6c0df73ba..d177ca8c5 100644 --- a/frontend/src2/charts/components/NewMeasureSelectorDialog.vue +++ b/frontend/src2/charts/components/NewMeasureSelectorDialog.vue @@ -4,6 +4,9 @@ import { COLUMN_TYPES, FIELDTYPES } from '../../helpers/constants' import ExpressionEditor from '../../query/components/ExpressionEditor.vue' import { expression } from '../../query/helpers' import { ColumnOption, ExpressionMeasure, MeasureDataType } from '../../types/query.types' +import { cachedCall } from '../../helpers' +import { TextInput } from 'frappe-ui' +import { SearchIcon } from 'lucide-vue-next' const props = defineProps<{ measure?: ExpressionMeasure @@ -13,7 +16,7 @@ const emit = defineEmits({ select: (measure: ExpressionMeasure) => true }) const showDialog = defineModel() const columnTypes = COLUMN_TYPES.map((t) => t.value).filter((t) => - FIELDTYPES.NUMBER.includes(t), + FIELDTYPES.NUMBER.includes(t) ) as MeasureDataType[] const newMeasure = ref( @@ -27,23 +30,50 @@ const newMeasure = ref( name: 'new_measure', type: columnTypes[0], expression: '', - }, + } ) const isValid = computed(() => { return newMeasure.value.name && newMeasure.value.type && newMeasure.value.expression.trim() }) -function confirmCalculation() { +const validationState = ref<'unknown' | 'validating' | 'valid' | 'invalid'>('unknown') +const validationErrors = ref>([]) + +async function confirmCalculation() { if (!isValid.value) return - emit('select', { - measure_name: newMeasure.value.name, - data_type: newMeasure.value.type, - expression: expression(newMeasure.value.expression), - }) - resetNewMeasure() - showDialog.value = false + validationState.value = 'validating' + validationErrors.value = [] + try { + const res: any = await cachedCall( + 'insights.insights.doctype.insights_data_source_v3.ibis.utils.validate_expression', + { + expression: newMeasure.value.expression, + column_options: JSON.stringify(props.columnOptions), + } + ) + + if (!res || !res.is_valid) { + validationState.value = 'invalid' + validationErrors.value = res?.errors || [{ message: 'Validation failed' }] + return + } + + validationState.value = 'valid' + emit('select', { + measure_name: newMeasure.value.name, + data_type: newMeasure.value.type, + expression: expression(newMeasure.value.expression), + }) + resetNewMeasure() + showDialog.value = false + } catch (e) { + console.error(e) + validationState.value = 'unknown' + validationErrors.value = [{ message: 'Unexpected validation error' }] + } } + function resetNewMeasure() { newMeasure.value = { name: 'new_measure', @@ -51,47 +81,184 @@ function resetNewMeasure() { expression: '', } } + +const functionList = ref([]) +const selectedFunction = ref('') + +const searchTerm = ref('') +const filteredFunctions = computed(() => { + const searchQuery = searchTerm.value.trim().toLowerCase() + if (!searchQuery) return functionList.value + return functionList.value.filter((fn) => fn.toLowerCase().includes(searchQuery)) +}) + +type FunctionSignature = { + name: string + definition: string + description: string + current_param: string + current_param_description: string + params: { name: string; description: string }[] +} +const functionDoc = ref(null) +const columns = props.columnOptions.map((c) => c.label) +cachedCall('insights.insights.doctype.insights_data_source_v3.ibis.utils.get_function_list').then( + (res: any) => { + const result = [...res, ...columns] + functionList.value = result + } +) + +function selectFunction(funcName: string) { + selectedFunction.value = funcName + + cachedCall( + 'insights.insights.doctype.insights_data_source_v3.ibis.utils.get_function_description', + { funcName } + ) + .then((res: any) => { + if (res) { + functionDoc.value = res + } + }) + .catch(console.error) +} + +function updateDocumentationFromEditor(currentFunction: any) { + if (currentFunction) { + functionDoc.value = currentFunction + selectedFunction.value = currentFunction.name + } +} + + diff --git a/frontend/src2/components/Code.vue b/frontend/src2/components/Code.vue index 8e19a801a..767e39c8d 100644 --- a/frontend/src2/components/Code.vue +++ b/frontend/src2/components/Code.vue @@ -23,7 +23,8 @@ import { javascript } from '@codemirror/lang-javascript' import { python } from '@codemirror/lang-python' import { MySQL, sql } from '@codemirror/lang-sql' import { syntaxTree } from '@codemirror/language' -import { EditorView } from '@codemirror/view' +import { linter } from '@codemirror/lint' +import { Decoration, EditorView, ViewPlugin } from '@codemirror/view' import { onMounted, ref, watch } from 'vue' import { Codemirror } from 'vue-codemirror' import { tomorrow } from 'thememirror' @@ -70,6 +71,14 @@ const props = defineProps({ type: Boolean, default: true, }, + columnNames: { + type: Array, + default: () => [], + }, + validationErrors: { + type: Array, + default: () => [], + }, }) const emit = defineEmits(['inputChange', 'viewUpdate', 'focus', 'blur']) @@ -101,18 +110,96 @@ const language = props.language === 'javascript' ? javascript() : props.language === 'python' - ? python() - : sql({ - dialect: MySQL, - upperCaseKeywords: true, - schema: props.schema, - tables: props.tables, - }) - -const extensions = [language, closeBrackets(), tomorrow] + ? python() + : sql({ + dialect: MySQL, + upperCaseKeywords: true, + schema: props.schema, + tables: props.tables, + }) + +const columnHighlighter = ViewPlugin.fromClass( + class { + decorations + + constructor(view) { + this.decorations = this.buildDecorations(view) + } + + update(update) { + if (update.docChanged || update.viewportChanged) { + this.decorations = this.buildDecorations(update.view) + } + } + + buildDecorations(view) { + if (!props.columnNames || props.columnNames.length === 0) { + return Decoration.none + } + + const decorations = [] + const columnSet = new Set(props.columnNames) + const doc = view.state.doc + + for (let i = 1; i <= doc.lines; i++) { + const line = doc.line(i) + const text = line.text + + // match only whole words ie `signups` but not `signups_today` + const wordRegex = /\b\w+\b/g + let match + + while ((match = wordRegex.exec(text)) !== null) { + const word = match[0] + if (columnSet.has(word)) { + const from = line.from + match.index + const to = from + word.length + decorations.push( + Decoration.mark({ + class: 'cm-column-highlight', + }).range(from, to) + ) + } + } + } + + return Decoration.set(decorations) + } + }, + { + decorations: (v) => v.decorations, + } +) + +const validationLinter = linter((view) => { + const diagnostics = [] + + for (const error of props.validationErrors) { + if (!error.line) continue + const line = view.state.doc.line(error.line) + const from = error.column ? line.from + error.column - 1 : line.from + const to = error.column ? from + 1 : line.to + + diagnostics.push({ + from: Math.max(0, from), + to: Math.min(view.state.doc.length, to), + severity: 'error', + message: error.message + (error.hint ? `\n${error.hint}` : ''), + }) + } + return diagnostics +}) + +const extensions = [language, closeBrackets(),tomorrow, validationLinter] + if (props.multiLine) { extensions.push(EditorView.lineWrapping) } + +if (props.columnNames && props.columnNames.length > 0) { + extensions.push(columnHighlighter) +} + const autocompletionOptions = { activateOnTyping: true, closeOnBlur: false, @@ -149,3 +236,17 @@ defineExpose({ }, }) + + diff --git a/frontend/src2/query/components/ExpressionEditor.vue b/frontend/src2/query/components/ExpressionEditor.vue index 50d88db19..2b918690b 100644 --- a/frontend/src2/query/components/ExpressionEditor.vue +++ b/frontend/src2/query/components/ExpressionEditor.vue @@ -1,9 +1,18 @@ diff --git a/insights/insights/doctype/insights_data_source_v3/ibis/utils.py b/insights/insights/doctype/insights_data_source_v3/ibis/utils.py index f4eb22927..eb5a1ecc6 100644 --- a/insights/insights/doctype/insights_data_source_v3/ibis/utils.py +++ b/insights/insights/doctype/insights_data_source_v3/ibis/utils.py @@ -1,3 +1,7 @@ +import ast +import json +import re + import frappe import ibis from ibis import selectors as s @@ -110,7 +114,7 @@ def get_function_list(): @frappe.whitelist() -def get_code_completions(code: str): +def get_code_completions(code: str, column_options=None): import_statement = """from insights.insights.doctype.insights_data_source_v3.ibis.functions import *\nfrom ibis import selectors as s""" code = f"{import_statement}\n\n{code}" @@ -119,6 +123,16 @@ def get_code_completions(code: str): column_pos = cursor_pos - code.rfind("\n", 0, cursor_pos) - 1 code = code.replace("|", "") + column_types = {} + if column_options: + try: + columns = json.loads(column_options) + column_types = { + col.get("value"): col.get("data_type", "Unknown") for col in columns if col.get("value") + } + except (json.JSONDecodeError, TypeError): + pass + current_function = None script = Script(code) @@ -151,6 +165,208 @@ def get_code_completions(code: str): current_function["current_param"] = current_param.name current_function["current_param_description"] = current_param.description + # add column type if the current parameter is a column + if current_param.name in column_types: + current_function["current_param_type"] = column_types[current_param.name] + return { "current_function": current_function, + "column_types": column_types, + } + + +@frappe.whitelist() +def get_function_description(funcName: str): + functions_dict = get_functions() + func_obj = functions_dict.get(funcName) + + if not func_obj: + return None + + docstring = getattr(func_obj, "__doc__", "") or "" + + if "def " in docstring: + lines = docstring.split("\n", 1) + definition = lines[0].replace("def ", "").strip() + description = lines[1].strip() if len(lines) > 1 else "" + + return {"name": funcName, "definition": definition, "description": description} + + +def parse_column_metadata(column_options: str): + columns = frappe.parse_json(column_options) or [] + meta = [col for col in columns if col.get("value")] + return meta + + +def create_error(line: int, column: int, message: str): + return {"line": line, "column": column, "message": message} + + +def get_ibis_dtype(columns: list[dict]): + type_mapping = { + "String": "string", + "Integer": "int64", + "Decimal": "float64", + "Date": "date", + "Datetime": "timestamp", + "Time": "time", + "Text": "string", + "JSON": "json", + "Array": "array", + "Auto": "", + } + return { + col.get("value"): type_mapping.get(col.get("description")) + for col in columns + if col.get("value") and col.get("description") } + + +def find_similar_names(name: str, names: set[str]): + name_lower = name.lower() + return [c for c in names if name_lower in c.lower() or c.lower() in name_lower] + + +def validate_syntax(expression: str): + try: + ast.parse(expression) + return {"is_valid": True, "errors": []} + except SyntaxError as e: + error = create_error(line=e.lineno or 1, column=e.offset or 0, message=f"Syntax error: {e.msg}") + return {"is_valid": False, "errors": [error]} + + +def is_function(node, tree): + for parent in ast.walk(tree): + for child in ast.iter_child_nodes(parent): + if child == node: + return isinstance(parent, ast.Call) and parent.func == node + return False + + +def validate_function_name(node, available_functions: set[str]): + func_name = node.func.id + if func_name in available_functions: + return None + + suggestions = find_similar_names(func_name, available_functions) + suggestion_text = f"Did you mean: {', '.join(suggestions[:2])}?" if suggestions else "" + + return create_error( + line=node.lineno, + column=node.col_offset, + message=f"Unknown function '{func_name}'. {suggestion_text}".strip(), + ) + + +def validate_variable_name(node, tree, available_functions: set[str], available_columns: set[str]): + var_name = node.id + + if var_name in available_functions or var_name in available_columns: + return None + + # pass if the variable is being assigned (LHS) + # perf: this is faster and more accurate than looping + # ctx is Store if the container is an assignment target + if isinstance(node.ctx, ast.Store): + return None + + if is_function(node, tree): + return None + + suggestions = find_similar_names(var_name, available_columns) + suggestion_text = f"Did you mean: {', '.join(suggestions[:2])}?" if suggestions else "" + + return create_error( + line=node.lineno, + column=node.col_offset, + message=f"Column '{var_name}' not found. {suggestion_text}".strip(), + ) + + +def validate_names(tree, columns: list[dict]): + functions = get_functions() + available_functions = set(functions.keys()) + available_columns = {col.get("value") for col in columns} + + errors = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + error = validate_function_name(node, available_functions) + if error: + errors.append(error) + + if isinstance(node, ast.Name): + error = validate_variable_name(node, tree, available_functions, available_columns) + if error: + errors.append(error) + + return {"is_valid": len(errors) == 0, "errors": errors} + + +def eval_script(table, schema: dict[str, str]): + script = get_functions() + for col_name in schema: + script[col_name] = getattr(table, col_name) + return script + + +# Functions that are not supported by certain column types like `DateColumn.sum()` +def handle_attribute_error(error: AttributeError): + error_msg = str(error) + match = re.search(r"'(\w+)Column' object has no attribute '(\w+)'", error_msg) + + if match: + column_type = match.group(1) + method_name = match.group(2) + message = f"Type error: {column_type} columns do not support {method_name}()" + else: + message = f"Type error: {error_msg}" + + return create_error(line=1, column=0, message=message) + + +def validate_types(expression: str, columns: list[dict]): + schema = get_ibis_dtype(columns) + if not schema: + return {"is_valid": True, "errors": []} + + try: + validation_table = ibis.table(schema, name="validation_table") + eval_context = eval_script(validation_table, schema) + exec(expression, {"__builtins__": {}}, eval_context) + return {"is_valid": True, "errors": []} + + except AttributeError as e: + error = handle_attribute_error(e) + return {"is_valid": False, "errors": [error]} + + except TypeError as e: + error = create_error(line=1, column=0, message=f"Type error: {str(e)}") + return {"is_valid": False, "errors": [error]} + + except Exception as e: + frappe.log_error(f"Unexpected validation error: {str(e)}") + return {"is_valid": False, "errors": []} + + +@frappe.whitelist() +def validate_expression(expression: str, column_options: str): + """Main function to validate expression/syntax""" + + if not expression.strip(): + return {"is_valid": True, "errors": []} + + columns = parse_column_metadata(column_options) + syntax_result = validate_syntax(expression) + if not syntax_result["is_valid"]: + return syntax_result + + tree = ast.parse(expression) + name_result = validate_names(tree, columns) + if not name_result["is_valid"]: + return name_result + + type_result = validate_types(expression, columns) + return type_result diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py index c64c79cda..19fd0dacf 100644 --- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py +++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py @@ -788,9 +788,9 @@ def exec_with_return( if isinstance(last_node, ast.Expr): output_expression = ast.unparse(last_node) elif isinstance(last_node, ast.Assign): - output_expression = ast.unparse(last_node.targets[0]) + output_expression = ast.unparse(last_node.value) elif isinstance(last_node, ast.AnnAssign | ast.AugAssign): - output_expression = ast.unparse(last_node.target) + output_expression = ast.unparse(last_node.value) _globals = _globals or {} _locals = _locals or {}