feat: add support of transforming custom command

docs/custom_search_commands.md

@@ -9,7 +9,6 @@ There are 4 types of Custom search commands:
 - Transforming
 - Dataset processing
 
-> Note: Currently UCC supports only three types of custom search command, that are `Generating`, `Streaming` and `Dataset processing`.
 
 > Note: Eventing commands are being referred as Dataset processing commands [reference](https://dev.splunk.com/enterprise/docs/devtools/customsearchcommands/).
 

splunk_add_on_ucc_framework/generators/python_files/create_custom_command_python.py

@@ -13,8 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
+import ast
+from pathlib import Path
 from typing import Any, Dict, List, Optional
-
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
 from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
@@ -29,8 +31,25 @@ def __init__(
         self.commands_info = []
         for command in global_config.custom_search_commands:
             argument_list: List[str] = []
+            import_map = False
             imported_file_name = command["fileName"].replace(".py", "")
             template = command["commandType"].replace(" ", "_") + ".template"
+            if command["commandType"] == "transforming":
+                module_path = Path(
+                    os.path.realpath(
+                        os.path.join(self._input_dir, "bin", command["fileName"])
+                    )
+                )
+
+                if not module_path.is_file():
+                    raise FileNotFoundError(
+                        f"Module path '{module_path}' does not point to a valid file."
+                    )
+                module_content = ast.parse(module_path.read_text(encoding="utf-8"))
+                for node in module_content.body:
+                    if isinstance(node, ast.FunctionDef) and node.name == "map":
+                        import_map = True
+
             for argument in command["arguments"]:
                 argument_dict = {
                     "name": argument["name"],
@@ -48,6 +67,7 @@ def __init__(
                     "syntax": command.get("syntax"),
                     "template": template,
                     "list_arg": argument_list,
+                    "import_map": import_map,
                 }
             )
 
@@ -109,6 +129,7 @@ def generate(self) -> Optional[List[Dict[str, str]]]:
                 description=command_info["description"],
                 syntax=command_info["syntax"],
                 list_arg=command_info["list_arg"],
+                import_map=command_info["import_map"],
             )
             generated_files.append(
                 {

splunk_add_on_ucc_framework/schema/schema.json

@@ -448,7 +448,8 @@
           "enum": [
             "generating",
             "streaming",
-            "dataset processing"
+            "dataset processing",
+            "transforming"
           ]
         },
         "requiredSearchAssistant": {

splunk_add_on_ucc_framework/templates/custom_command/transforming.template

@@ -0,0 +1,44 @@
+import sys
+import import_declare_test
+
+from splunklib.searchcommands import \
+    dispatch, ReportingCommand, Configuration, Option, validators
+
+{% if import_map %}
+from {{ imported_file_name }} import reduce, map
+{% else %}
+from {{ imported_file_name }} import reduce
+{% endif %}
+
+@Configuration()
+class {{class_name}}Command(ReportingCommand):
+    {% if syntax or description%}
+    """
+
+    {% if syntax %}
+    ##Syntax
+    {{syntax}}
+    {% endif %}
+
+    {% if description %}
+    ##Description
+    {{description}}
+    {% endif %}
+
+    """
+    {% endif %}
+
+    {% for arg in list_arg %}
+    {{arg}}
+    {% endfor %}
+
+    {% if import_map %}
+    @Configuration()
+    def map(self, events):
+        return map(self, events)
+    {% endif %}
+
+    def reduce(self, events):
+        return reduce(self, events)
+
+dispatch({{class_name}}Command, sys.argv, sys.stdin, sys.stdout, __name__)

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/appserver/static/js/build/globalConfig.json

@@ -2427,5 +2427,106 @@
             }
         ],
         "_uccVersion": "5.62.0"
-    }
+    },
+    "customSearchCommand": [
+        {
+            "commandName": "generatetextcommand",
+            "fileName": "generatetext.py",
+            "commandType": "generating",
+            "requiredSearchAssistant": true,
+            "description": " This command generates COUNT occurrences of a TEXT string.",
+            "syntax": "generatetextcommand count=<event_count> text=<string>",
+            "usage": "public",
+            "arguments": [
+                {
+                    "name": "count",
+                    "required": true,
+                    "validate": {
+                        "type": "Integer",
+                        "minimum": 5,
+                        "maximum": 10
+                    }
+                },
+                {
+                    "name": "text",
+                    "required": true
+                }
+            ]
+        },
+        {
+            "commandName": "filtercommand",
+            "fileName": "filter.py",
+            "commandType": "dataset processing",
+            "requiredSearchAssistant": true,
+            "description": "It filters records from the events stream returning only those which has :code:`contains` in them and replaces :code:`replace_array[0]` with :code:`replace_array[1]`.",
+            "syntax": "| filtercommand contains='value1' replace='value to be replaced,value to replace with'",
+            "usage": "public",
+            "arguments": [
+                {
+                    "name": "contains"
+                },
+                {
+                    "name": "replace_array"
+                }
+            ]
+        },
+        {
+            "commandName": "sumcommand",
+            "fileName": "sum.py",
+            "commandType": "transforming",
+            "requiredSearchAssistant": true,
+            "description": "The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.",
+            "syntax": "| sumcommand total=lines linecount",
+            "usage": "public",
+            "arguments": [
+                {
+                    "name": "total",
+                    "validate": {
+                        "type": "Fieldname"
+                    },
+                    "required": true
+                }
+            ]
+        },
+        {
+            "commandName": "sumtwocommand",
+            "fileName": "sum_without_map.py",
+            "commandType": "transforming",
+            "requiredSearchAssistant": true,
+            "description": "Computes sum(total, 1, N) and stores the result in 'total'",
+            "syntax": "| sumtwocommand total=lines linecount",
+            "usage": "public",
+            "arguments": [
+                {
+                    "name": "total",
+                    "validate": {
+                        "type": "Fieldname"
+                    },
+                    "required": true
+                }
+            ]
+        },
+        {
+            "commandName": "countmatchescommand",
+            "fileName": "countmatches.py",
+            "commandType": "streaming",
+            "requiredSearchAssistant": false,
+            "arguments": [
+                {
+                    "name": "fieldname",
+                    "validate": {
+                        "type": "Fieldname"
+                    },
+                    "required": true
+                },
+                {
+                    "name": "pattern",
+                    "validate": {
+                        "type": "RegularExpression"
+                    },
+                    "required": true
+                }
+            ]
+        }
+    ]
 }

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum.py

@@ -0,0 +1,22 @@
+import logging
+
+def map(self, records):
+        """ Computes sum(fieldname, 1, n) and stores the result in 'total' """
+        fieldnames = self.fieldnames
+        total = 0.0
+        for record in records:
+            for fieldname in fieldnames:
+                total += float(record[fieldname])
+        yield {self.total: total}
+
+def reduce(self, records):
+    """ Computes sum(total, 1, N) and stores the result in 'total' """
+    fieldname = self.total
+    total = 0.0
+    for record in records:
+        value = record[fieldname]
+        try:
+            total += float(value)
+        except ValueError:
+            logging.debug('  could not convert %s value to float: %s', fieldname, repr(value))
+    yield {self.total: total}

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sum_without_map.py

@@ -0,0 +1,14 @@
+import logging
+
+def reduce(self, records):
+    """ Computes sum(total, 1, N) and stores the result in 'total' """
+    fieldname = self.total
+    total = 0.0
+    for record in records:
+        value = record[fieldname]
+        try:
+            total += float(value)
+        except ValueError:
+            logging.debug('  could not convert %s value to float: %s', fieldname, repr(value))
+    yield {self.total: total}
+

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumcommand.py

@@ -0,0 +1,30 @@
+import sys
+import import_declare_test
+
+from splunklib.searchcommands import \
+    dispatch, ReportingCommand, Configuration, Option, validators
+
+from sum import reduce, map
+
+@Configuration()
+class SumcommandCommand(ReportingCommand):
+    """
+
+    ##Syntax
+    | sumcommand total=lines linecount
+
+    ##Description
+    The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.
+
+    """
+
+    total = Option(name='total', require=True, validate=validators.Fieldname())
+
+    @Configuration()
+    def map(self, events):
+        return map(self, events)
+
+    def reduce(self, events):
+        return reduce(self, events)
+
+dispatch(SumcommandCommand, sys.argv, sys.stdin, sys.stdout, __name__)

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/bin/sumtwocommand.py

@@ -0,0 +1,27 @@
+import sys
+import import_declare_test
+
+from splunklib.searchcommands import \
+    dispatch, ReportingCommand, Configuration, Option, validators
+
+from sum_without_map import reduce
+
+@Configuration()
+class SumtwocommandCommand(ReportingCommand):
+    """
+
+    ##Syntax
+    | sumtwocommand total=lines linecount
+
+    ##Description
+    The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.
+
+    """
+
+    total = Option(name='total', require=True, validate=validators.Fieldname())
+
+
+    def reduce(self, events):
+        return reduce(self, events)
+
+dispatch(SumtwocommandCommand, sys.argv, sys.stdin, sys.stdout, __name__)

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/commands.conf

@@ -8,6 +8,16 @@ filename = filtercommand.py
 chunked = true
 python.version = python3
 
+[sumcommand]
+filename = sumcommand.py
+chunked = true
+python.version = python3
+
+[sumtwocommand]
+filename = sumtwocommand.py
+chunked = true
+python.version = python3
+
 [countmatchescommand]
 filename = countmatchescommand.py
 chunked = true

tests/testdata/expected_addons/expected_output_global_config_everything/Splunk_TA_UCCExample/default/searchbnf.conf

@@ -7,3 +7,13 @@ usage = public
 syntax = filtercommand contains=<string> replace=<string>
 description = Filters records from the events stream.
 usage = public
+
+[sumcommand-command]
+syntax = | sumcommand total=lines linecount
+description = The total produced is sum(sum(fieldname, 1, n), 1, N) where n = number of fields, N = number of records.
+usage = public
+
+[sumtwocommand-command]
+syntax = | sumtwocommand total=lines linecount
+description = Computes sum(total, 1, N) and stores the result in 'total'
+usage = public