diff --git a/docs/custom_search_commands.md b/docs/custom_search_commands.md index 66a4ab46e1..0d367dda0a 100644 --- a/docs/custom_search_commands.md +++ b/docs/custom_search_commands.md @@ -9,7 +9,6 @@ There are 4 types of Custom search commands: - Transforming - Dataset processing -> Note: Currently UCC supports only three types of custom search command, that are `Generating`, `Streaming` and `Dataset processing`. > Note: Eventing commands are being referred as Dataset processing commands [reference](https://dev.splunk.com/enterprise/docs/devtools/customsearchcommands/). diff --git a/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py index 322b9d1e19..a0ccc5e53f 100644 --- a/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py +++ b/splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +import ast +from pathlib import Path from typing import Any, Dict, List, Optional - from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator from splunk_add_on_ucc_framework.global_config import GlobalConfig @@ -29,8 +31,25 @@ def __init__( self.commands_info = [] for command in global_config.custom_search_commands: argument_list: List[str] = [] + import_map = False imported_file_name = command["fileName"].replace(".py", "") template = command["commandType"].replace(" ", "_") + ".template" + if command["commandType"] == "transforming": + module_path = Path( + os.path.realpath( + os.path.join(self._input_dir, "bin", command["fileName"]) + ) + ) + + if not module_path.is_file(): + raise FileNotFoundError( + f"Module path '{module_path}' does not point to a valid file." + ) + module_content = ast.parse(module_path.read_text(encoding="utf-8")) + for node in module_content.body: + if isinstance(node, ast.FunctionDef) and node.name == "map": + import_map = True + for argument in command["arguments"]: argument_dict = { "name": argument["name"], @@ -48,6 +67,7 @@ def __init__( "syntax": command.get("syntax"), "template": template, "list_arg": argument_list, + "import_map": import_map, } ) @@ -109,6 +129,7 @@ def generate(self) -> Optional[List[Dict[str, str]]]: description=command_info["description"], syntax=command_info["syntax"], list_arg=command_info["list_arg"], + import_map=command_info["import_map"], ) generated_files.append( { diff --git a/splunk_add_on_ucc_framework/schema/schema.json b/splunk_add_on_ucc_framework/schema/schema.json index 2d2f043708..642e020919 100644 --- a/splunk_add_on_ucc_framework/schema/schema.json +++ b/splunk_add_on_ucc_framework/schema/schema.json @@ -448,7 +448,8 @@ "enum": [ "generating", "streaming", - "dataset processing" + "dataset processing", + "transforming" ] }, "requiredSearchAssistant": { diff --git a/splunk_add_on_ucc_framework/templates/custom_command/transforming.template b/splunk_add_on_ucc_framework/templates/custom_command/transforming.template new file mode 100644 index 0000000000..0fc7a46d82 --- /dev/null +++ b/splunk_add_on_ucc_framework/templates/custom_command/transforming.template @@ -0,0 +1,44 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, ReportingCommand, Configuration, Option, validators + +{% if import_map %} +from {{ imported_file_name }} import reduce, map +{% else %} +from {{ imported_file_name }} import reduce +{% endif %} + +@Configuration() +class {{class_name}}Command(ReportingCommand): + {% if syntax or description%} + """ + + {% if syntax %} + ##Syntax + {{syntax}} + {% endif %} + + {% if description %} + ##Description + {{description}} + {% endif %} + + """ + {% endif %} + + {% for arg in list_arg %} + {{arg}} + {% endfor %} + + {% if import_map %} + @Configuration() + def map(self, events): + return map(self, events) + {% endif %} + + def reduce(self, events): + return reduce(self, events) + +dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/appserver/static/js/build/globalConfig.json b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/appserver/static/js/build/globalConfig.json index f7dcfe8389..cbd7f700e9 100644 --- a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/appserver/static/js/build/globalConfig.json +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/appserver/static/js/build/globalConfig.json @@ -2427,5 +2427,106 @@ } ], "_uccVersion": "5.62.0" - } + }, + "customSearchCommand": [ + { + "commandName": "generatetextcommand", + "fileName": "generatetext.py", + "commandType": "generating", + "requiredSearchAssistant": true, + "description": " This command generates COUNT occurrences of a TEXT string.", + "syntax": "generatetextcommand count= text=", + "usage": "public", + "arguments": [ + { + "name": "count", + "required": true, + "validate": { + "type": "Integer", + "minimum": 5, + "maximum": 10 + } + }, + { + "name": "text", + "required": true + } + ] + }, + { + "commandName": "filtercommand", + "fileName": "filter.py", + "commandType": "dataset processing", + "requiredSearchAssistant": true, + "description": "It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`.", + "syntax": "| filtercommand contains='value1' replace='value to be replaced,value to replace with'", + "usage": "public", + "arguments": [ + { + "name": "contains" + }, + { + "name": "replace_array" + } + ] + }, + { + "commandName": "sumcommand", + "fileName": "sum.py", + "commandType": "transforming", + "requiredSearchAssistant": true, + "description": "The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.", + "syntax": "| sumcommand total=lines linecount", + "usage": "public", + "arguments": [ + { + "name": "total", + "validate": { + "type": "Fieldname" + }, + "required": true + } + ] + }, + { + "commandName": "sumtwocommand", + "fileName": "sum_without_map.py", + "commandType": "transforming", + "requiredSearchAssistant": true, + "description": "Computes sum(total, 1, N) and stores the result in 'total'", + "syntax": "| sumtwocommand total=lines linecount", + "usage": "public", + "arguments": [ + { + "name": "total", + "validate": { + "type": "Fieldname" + }, + "required": true + } + ] + }, + { + "commandName": "countmatchescommand", + "fileName": "countmatches.py", + "commandType": "streaming", + "requiredSearchAssistant": false, + "arguments": [ + { + "name": "fieldname", + "validate": { + "type": "Fieldname" + }, + "required": true + }, + { + "name": "pattern", + "validate": { + "type": "RegularExpression" + }, + "required": true + } + ] + } + ] } diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum.py new file mode 100644 index 0000000000..b8be634c28 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum.py @@ -0,0 +1,22 @@ +import logging + +def map(self, records): + """ Computes sum(fieldname, 1, n) and stores the result in 'total' """ + fieldnames = self.fieldnames + total = 0.0 + for record in records: + for fieldname in fieldnames: + total += float(record[fieldname]) + yield {self.total: total} + +def reduce(self, records): + """ Computes sum(total, 1, N) and stores the result in 'total' """ + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug(' could not convert %s value to float: %s', fieldname, repr(value)) + yield {self.total: total} diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum_without_map.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum_without_map.py new file mode 100644 index 0000000000..4e20c3b6d8 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum_without_map.py @@ -0,0 +1,14 @@ +import logging + +def reduce(self, records): + """ Computes sum(total, 1, N) and stores the result in 'total' """ + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug(' could not convert %s value to float: %s', fieldname, repr(value)) + yield {self.total: total} + \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumcommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumcommand.py new file mode 100644 index 0000000000..d01f33ea0c --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumcommand.py @@ -0,0 +1,30 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, ReportingCommand, Configuration, Option, validators + +from sum import reduce, map + +@Configuration() +class SumcommandCommand(ReportingCommand): + """ + + ##Syntax + | sumcommand total=lines linecount + + ##Description + The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records. + + """ + + total = Option(name='total', require=True, validate=validators.Fieldname()) + + @Configuration() + def map(self, events): + return map(self, events) + + def reduce(self, events): + return reduce(self, events) + +dispatch(SumcommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumtwocommand.py b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumtwocommand.py new file mode 100644 index 0000000000..bd9f85c3d8 --- /dev/null +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumtwocommand.py @@ -0,0 +1,27 @@ +import sys +import import_declare_test + +from splunklib.searchcommands import \ + dispatch, ReportingCommand, Configuration, Option, validators + +from sum_without_map import reduce + +@Configuration() +class SumtwocommandCommand(ReportingCommand): + """ + + ##Syntax + | sumtwocommand total=lines linecount + + ##Description + The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records. + + """ + + total = Option(name='total', require=True, validate=validators.Fieldname()) + + + def reduce(self, events): + return reduce(self, events) + +dispatch(SumtwocommandCommand, sys.argv, sys.stdin, sys.stdout, __name__) \ No newline at end of file diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf index ee84578dbf..d2ee56fd17 100644 --- a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf @@ -8,6 +8,16 @@ filename = filtercommand.py chunked = true python.version = python3 +[sumcommand] +filename = sumcommand.py +chunked = true +python.version = python3 + +[sumtwocommand] +filename = sumtwocommand.py +chunked = true +python.version = python3 + [countmatchescommand] filename = countmatchescommand.py chunked = true diff --git a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf index 2e3361648e..a310c21291 100644 --- a/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf +++ b/tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf @@ -7,3 +7,13 @@ usage = public syntax = filtercommand contains= replace= description = Filters records from the events stream. usage = public + +[sumcommand-command] +syntax = | sumcommand total=lines linecount +description = The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records. +usage = public + +[sumtwocommand-command] +syntax = | sumtwocommand total=lines linecount +description = Computes sum(total, 1, N) and stores the result in 'total' +usage = public \ No newline at end of file diff --git a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json index 2bcc308e29..68421c1d57 100644 --- a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json +++ b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json @@ -2550,6 +2550,42 @@ } ] }, + { + "commandName": "sumcommand", + "fileName": "sum.py", + "commandType": "transforming", + "requiredSearchAssistant": true, + "description": "The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.", + "syntax": "| sumcommand total=lines linecount", + "usage": "public", + "arguments": [ + { + "name": "total", + "validate": { + "type": "Fieldname" + }, + "required": true + } + ] + }, + { + "commandName": "sumtwocommand", + "fileName": "sum_without_map.py", + "commandType": "transforming", + "requiredSearchAssistant": true, + "description": "Computes sum(total, 1, N) and stores the result in 'total'", + "syntax": "| sumtwocommand total=lines linecount", + "usage": "public", + "arguments": [ + { + "name": "total", + "validate": { + "type": "Fieldname" + }, + "required": true + } + ] + }, { "commandName": "countmatchescommand", "fileName": "countmatches.py", diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/sum.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/sum.py new file mode 100644 index 0000000000..b8be634c28 --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/sum.py @@ -0,0 +1,22 @@ +import logging + +def map(self, records): + """ Computes sum(fieldname, 1, n) and stores the result in 'total' """ + fieldnames = self.fieldnames + total = 0.0 + for record in records: + for fieldname in fieldnames: + total += float(record[fieldname]) + yield {self.total: total} + +def reduce(self, records): + """ Computes sum(total, 1, N) and stores the result in 'total' """ + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug(' could not convert %s value to float: %s', fieldname, repr(value)) + yield {self.total: total} diff --git a/tests/testdata/test_addons/package_global_config_everything/package/bin/sum_without_map.py b/tests/testdata/test_addons/package_global_config_everything/package/bin/sum_without_map.py new file mode 100644 index 0000000000..4e20c3b6d8 --- /dev/null +++ b/tests/testdata/test_addons/package_global_config_everything/package/bin/sum_without_map.py @@ -0,0 +1,14 @@ +import logging + +def reduce(self, records): + """ Computes sum(total, 1, N) and stores the result in 'total' """ + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug(' could not convert %s value to float: %s', fieldname, repr(value)) + yield {self.total: total} + \ No newline at end of file diff --git a/tests/unit/generators/python_files/test_create_custom_command_python.py b/tests/unit/generators/python_files/test_create_custom_command_python.py index 6cec958b8f..1e4150b575 100644 --- a/tests/unit/generators/python_files/test_create_custom_command_python.py +++ b/tests/unit/generators/python_files/test_create_custom_command_python.py @@ -1,5 +1,8 @@ -from pytest import fixture +import shutil +import os +import pytest from splunk_add_on_ucc_framework.generators.python_files import CustomCommandPy +from tests.unit.helpers import get_testdata_file_path from textwrap import dedent @@ -7,7 +10,7 @@ def normalize_code(code: str) -> str: return dedent(code).replace("\\\n", " ").replace("\n", "").replace(" ", "") -@fixture +@pytest.fixture def custom_search_commands(): return [ { @@ -40,6 +43,110 @@ def custom_search_commands(): ] +@pytest.fixture +def transforming_custom_search_command(): + return [ + { + "commandName": "transformingcommand", + "commandType": "transforming", + "fileName": "transforming_with_map.py", + "description": "This is a transforming command", + "syntax": "transformingcommand action=", + "arguments": [ + { + "name": "action", + "required": True, + "validate": {"type": "Fieldname"}, + }, + { + "name": "test", + }, + ], + } + ] + + +def test_for_transforming_command_with_error( + transforming_custom_search_command, + global_config_all_json, + input_dir, + output_dir, +): + global_config_all_json._content[ + "customSearchCommand" + ] = transforming_custom_search_command + + with pytest.raises(FileNotFoundError): + CustomCommandPy(global_config_all_json, input_dir, output_dir) + + +def test_for_transforming_command( + transforming_custom_search_command, + global_config_all_json, + input_dir, + output_dir, +): + file_path = get_testdata_file_path("transforming_with_map.py") + bin_dir = os.path.join(input_dir, "bin") + os.makedirs(bin_dir, exist_ok=True) + + shutil.copy(file_path, bin_dir) + global_config_all_json._content[ + "customSearchCommand" + ] = transforming_custom_search_command + custom_command_py = CustomCommandPy(global_config_all_json, input_dir, output_dir) + + assert custom_command_py.commands_info == [ + { + "imported_file_name": "transforming_with_map", + "file_name": "transformingcommand", + "class_name": "Transformingcommand", + "description": "This is a transforming command", + "syntax": "transformingcommand action=", + "template": "transforming.template", + "list_arg": [ + "action = Option(name='action', require=True, validate=validators.Fieldname())", + "test = Option(name='test', require=False)", + ], + "import_map": True, + } + ] + + +def test_for_transforming_command_without_map( + global_config_all_json, + input_dir, + output_dir, + transforming_custom_search_command, +): + file_path = get_testdata_file_path("transforming_without_map.py") + bin_dir = os.path.join(input_dir, "bin") + os.makedirs(bin_dir, exist_ok=True) + + shutil.copy(file_path, bin_dir) + transforming_custom_search_command[0]["fileName"] = "transforming_without_map.py" + global_config_all_json._content[ + "customSearchCommand" + ] = transforming_custom_search_command + custom_command_py = CustomCommandPy(global_config_all_json, input_dir, output_dir) + + assert custom_command_py.commands_info == [ + { + "imported_file_name": "transforming_without_map", + "file_name": "transformingcommand", + "class_name": "Transformingcommand", + "description": "This is a transforming command", + "syntax": "transformingcommand action=", + "template": "transforming.template", + "list_arg": [ + "action = Option(name='action', require=True, validate=validators.Fieldname())", + "test = Option(name='test', require=False)", + ], + "import_map": False, + } + ] + + def test_init_without_custom_command( global_config_only_configuration, input_dir, @@ -82,6 +189,7 @@ def test_init( "contains = Option(name='contains', require=False)", "fieldname = Option(name='fieldname', require=False, validate=validators.Fieldname())", ], + "import_map": False, } ] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 8904ab600c..4190ae295a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -36,6 +36,7 @@ def test_get_j2_env(): "custom_command/dataset_processing.template", "custom_command/generating.template", "custom_command/streaming.template", + "custom_command/transforming.template", "conf_files/commands_conf.template", "conf_files/searchbnf_conf.template", ] diff --git a/tests/unit/testdata/transforming_with_map.py b/tests/unit/testdata/transforming_with_map.py new file mode 100644 index 0000000000..f0bca1d235 --- /dev/null +++ b/tests/unit/testdata/transforming_with_map.py @@ -0,0 +1,26 @@ +import logging + + +def map(self, records): + """Computes sum(fieldname, 1, n) and stores the result in 'total'""" + fieldnames = self.fieldnames + total = 0.0 + for record in records: + for fieldname in fieldnames: + total += float(record[fieldname]) + yield {self.total: total} + + +def reduce(self, records): + """Computes sum(total, 1, N) and stores the result in 'total'""" + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug( + "could not convert %s value to float: %s", fieldname, repr(value) + ) + yield {self.total: total} diff --git a/tests/unit/testdata/transforming_without_map.py b/tests/unit/testdata/transforming_without_map.py new file mode 100644 index 0000000000..e9f73e36cf --- /dev/null +++ b/tests/unit/testdata/transforming_without_map.py @@ -0,0 +1,16 @@ +import logging + + +def reduce(self, records): + """Computes sum(total, 1, N) and stores the result in 'total'""" + fieldname = self.total + total = 0.0 + for record in records: + value = record[fieldname] + try: + total += float(value) + except ValueError: + logging.debug( + "could not convert %s value to float: %s", fieldname, repr(value) + ) + yield {self.total: total}