Merge pull request #41 from fastly/1038-validation-changes

1038 validation changes
fastly · Mar 21, 2024 · 7b5b3e3 · 7b5b3e3
2 parents 0e14172 + 2bf8291
commit 7b5b3e3
Show file tree

Hide file tree

Showing 14 changed files with 110 additions and 97 deletions.
diff --git a/app_builder_import_file/sigsci_TA_for_splunk-1_0_38_export.tgz b/app_builder_import_file/sigsci_TA_for_splunk-1_0_38_export.tgz
diff --git a/sigsci_TA_for_splunk-1.0.38.tgz b/sigsci_TA_for_splunk-1.0.38.tgz
diff --git a/sigsci_TA_for_splunk/README/inputs.conf.spec b/sigsci_TA_for_splunk/README/inputs.conf.spec
@@ -1,21 +1,21 @@
-[SigsciEvent://<name>]
-site_api_name = This is the Site API Name. It should not be a URL.
+[SigsciRequests://<name>]
+site_api_name = This is the API Name of the site to pull request data from. This should not be a URL.
+request_limit = The amount of request objects returned in the array. Default: 100. Max:1000
 disable_catchup = Disables catch-up behavior. Events will always be ingested from now minus the delta (including an offset for the requests feed). Recommended to be left true. Default: True.
-twenty_hour_catchup = In the event the last time stored is >24Hours the TA will try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.
+twenty_hour_catchup = In the event the last time stored is >24hours the TA will try can try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.
+attack_and_anomaly_signals_only = Only retrieves requests that contain attack or anomaly signals. Please evaluate your signal configuration if there are overly inclusive signals creating excessive requests.
 request_timeout = Configures Request Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
-read_timeout = Configured Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
+read_timeout = Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
 
 [SigsciActivity://<name>]
 disable_catchup = Disables catch-up behavior. Events will always be ingested from now minus the delta (including an offset for the requests feed). Recommended to be left true. Default: True.
 twenty_hour_catchup = In the event the last time stored is >24Hours the TA will try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be false in order to work.
 request_timeout = Configures Request Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
 read_timeout = Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
 
-[SigsciRequests://<name>]
-site_api_name = This is the API Name of the site to pull request data from. This should not be a URL.
-request_limit = The amount of request objects returned in the array. Default: 100. Max:1000
+[SigsciEvent://<name>]
+site_api_name = This is the Site API Name. It should not be a URL.
 disable_catchup = Disables catch-up behavior. Events will always be ingested from now minus the delta (including an offset for the requests feed). Recommended to be left true. Default: True.
-twenty_hour_catchup = In the event the last time stored is >24hours the TA will try can try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.
-attack_and_anomaly_signals_only = Only retrieves requests that contain attack or anomaly signals. Please evaluate your signal configuration if there are overly inclusive signals creating excessive requests.
+twenty_hour_catchup = In the event the last time stored is >24Hours the TA will try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.
 request_timeout = Configures Request Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
 read_timeout = Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.
diff --git a/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json b/sigsci_TA_for_splunk/appserver/static/js/build/globalConfig.json
@@ -247,6 +247,10 @@
                         "field": "site_api_name",
                         "label": "Site API Name"
                     },
+                    {
+                        "field": "request_limit",
+                        "label": "Request Limit"
+                    },
                     {
                         "field": "disable_catchup",
                         "label": "Disable Catchup"
@@ -255,21 +259,17 @@
                         "field": "twenty_hour_catchup",
                         "label": "24 Hour Catchup"
                     },
+                    {
+                        "field": "attack_and_anomaly_signals_only",
+                        "label": "Attack & Anomaly Signals Only"
+                    },
                     {
                         "field": "request_timeout",
                         "label": "Request Timeout"
                     },
                     {
                         "field": "read_timeout",
-                        "label": "read_timeout"
-                    },
-                    {
-                        "field": "request_limit",
-                        "label": "Request Limit"
-                    },
-                    {
-                        "field": "attack_and_anomaly_signals_only",
-                        "label": "Attack & Anomaly Signals Only"
+                        "label": "Read Timeout"
                     }
                 ],
                 "actions": [
@@ -281,8 +281,8 @@
             },
             "services": [
                 {
-                    "name": "SigsciEvent",
-                    "title": "SigsciEvent",
+                    "name": "SigsciRequests",
+                    "title": "SigsciRequests",
                     "entity": [
                         {
                             "field": "name",
@@ -341,7 +341,7 @@
                         {
                             "field": "site_api_name",
                             "label": "Site API Name",
-                            "help": "This is the Site API Name. It should not be a URL.",
+                            "help": "This is the API Name of the site to pull request data from. This should not be a URL.",
                             "required": true,
                             "type": "text",
                             "validators": [
@@ -353,6 +353,22 @@
                                 }
                             ]
                         },
+                        {
+                            "field": "request_limit",
+                            "label": "Request Limit",
+                            "help": "The amount of request objects returned in the array. Default: 100. Max:1000",
+                            "required": true,
+                            "type": "text",
+                            "defaultValue": "1000",
+                            "validators": [
+                                {
+                                    "type": "string",
+                                    "minLength": 0,
+                                    "maxLength": 8192,
+                                    "errorMsg": "Max length of text input is 8192"
+                                }
+                            ]
+                        },
                         {
                             "field": "disable_catchup",
                             "label": "Disable Catchup",
@@ -364,7 +380,14 @@
                         {
                             "field": "twenty_hour_catchup",
                             "label": "24 Hour Catchup",
-                            "help": "In the event the last time stored is >24Hours the TA will try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work. ",
+                            "help": "In the event the last time stored is >24hours the TA will try can try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.",
+                            "required": false,
+                            "type": "checkbox"
+                        },
+                        {
+                            "field": "attack_and_anomaly_signals_only",
+                            "label": "Attack & Anomaly Signals Only",
+                            "help": "Only retrieves requests that contain attack or anomaly signals. Please evaluate your signal configuration if there are overly inclusive signals creating excessive requests.",
                             "required": false,
                             "type": "checkbox"
                         },
@@ -384,13 +407,13 @@
                                 }
                             ],
                             "options": {
-                                "placeholder": "60"
+                                "placeholder": "Request Timeout"
                             }
                         },
                         {
                             "field": "read_timeout",
-                            "label": "read_timeout",
-                            "help": "Configured Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.",
+                            "label": "Read Timeout",
+                            "help": "Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.",
                             "required": true,
                             "type": "text",
                             "defaultValue": "60",
@@ -401,10 +424,7 @@
                                     "maxLength": 8192,
                                     "errorMsg": "Max length of text input is 8192"
                                 }
-                            ],
-                            "options": {
-                                "placeholder": "60"
-                            }
+                            ]
                         }
                     ]
                 },
@@ -522,8 +542,8 @@
                     ]
                 },
                 {
-                    "name": "SigsciRequests",
-                    "title": "SigsciRequests",
+                    "name": "SigsciEvent",
+                    "title": "SigsciEvent",
                     "entity": [
                         {
                             "field": "name",
@@ -582,25 +602,9 @@
                         {
                             "field": "site_api_name",
                             "label": "Site API Name",
-                            "help": "This is the API Name of the site to pull request data from. This should not be a URL.",
-                            "required": true,
-                            "type": "text",
-                            "validators": [
-                                {
-                                    "type": "string",
-                                    "minLength": 0,
-                                    "maxLength": 8192,
-                                    "errorMsg": "Max length of text input is 8192"
-                                }
-                            ]
-                        },
-                        {
-                            "field": "request_limit",
-                            "label": "Request Limit",
-                            "help": "The amount of request objects returned in the array. Default: 100. Max:1000",
+                            "help": "This is the Site API Name. It should not be a URL.",
                             "required": true,
                             "type": "text",
-                            "defaultValue": "1000",
                             "validators": [
                                 {
                                     "type": "string",
@@ -621,14 +625,7 @@
                         {
                             "field": "twenty_hour_catchup",
                             "label": "24 Hour Catchup",
-                            "help": "In the event the last time stored is >24hours the TA will try can try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work.",
-                            "required": false,
-                            "type": "checkbox"
-                        },
-                        {
-                            "field": "attack_and_anomaly_signals_only",
-                            "label": "Attack & Anomaly Signals Only",
-                            "help": "Only retrieves requests that contain attack or anomaly signals. Please evaluate your signal configuration if there are overly inclusive signals creating excessive requests.",
+                            "help": "In the event the last time stored is >24Hours the TA will try and catch-up from exactly 24 hours ago, otherwise resets to now minus the delta. 'Disable Catchup' must be False in order to work. ",
                             "required": false,
                             "type": "checkbox"
                         },
@@ -648,12 +645,12 @@
                                 }
                             ],
                             "options": {
-                                "placeholder": "Request Timeout"
+                                "placeholder": "60"
                             }
                         },
                         {
                             "field": "read_timeout",
-                            "label": "Read Timeout",
+                            "label": "read_timeout",
                             "help": "Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.",
                             "required": true,
                             "type": "text",
@@ -665,7 +662,10 @@
                                     "maxLength": 8192,
                                     "errorMsg": "Max length of text input is 8192"
                                 }
-                            ]
+                            ],
+                            "options": {
+                                "placeholder": "60"
+                            }
                         }
                     ]
                 }

diff --git a/sigsci_TA_for_splunk/bin/SigsciEvent.py b/sigsci_TA_for_splunk/bin/SigsciEvent.py
@@ -63,7 +63,7 @@ def get_scheme(self):
                                          required_on_create=True,
                                          required_on_edit=False))
         scheme.add_argument(smi.Argument("read_timeout", title="read_timeout",
-                                         description="Configured Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.",
+                                         description="Configures Read Timeout for HTTP operations. Consider increasing if on a slow connection or pagination batches are large.",
                                          required_on_create=True,
                                          required_on_edit=False))
         return scheme

diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py b/sigsci_TA_for_splunk/bin/input_module_SigsciActivity.py
@@ -3,7 +3,7 @@
 import json
 import time
 from datetime import datetime
-from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts
+from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts, validate_catchup
 
 """
     IMPORTANT
@@ -22,11 +22,10 @@ def validate_input(helper, definition):
     read_timeout = definition.parameters.get("read_timeout", None)
     validate_timeouts(request_timeout, read_timeout)    
 
-    # Catchup Opts
-    twenty_hour_catchup = definition.parameters.get('twenty_hour_catchup', None)
-    disable_catchup = definition.parameters.get('disable_catchup', None)
-    if twenty_hour_catchup and disable_catchup is True:
-        raise ValueError(f"Catch up values are mutually exclusive")
+    # Catchup Behaviour Validation    
+    disable_catchup = definition.parameters.get("disable_catchup", None)
+    twenty_hour_catchup = definition.parameters.get("twenty_hour_catchup", None)
+    validate_catchup(disable_catchup, twenty_hour_catchup)
     pass
 
 

diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py b/sigsci_TA_for_splunk/bin/input_module_SigsciEvent.py
@@ -4,7 +4,7 @@
 import json
 import time
 from datetime import datetime
-from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts
+from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts, validate_catchup
 
 """
     IMPORTANT
@@ -41,11 +41,10 @@ def validate_input(helper, definition):
         )
         raise ValueError("InvalidSiteName", msg)
 
-    # Catchup Opts
-    twenty_hour_catchup = definition.parameters.get('twenty_hour_catchup', None)
-    disable_catchup = definition.parameters.get('disable_catchup', None)
-    if twenty_hour_catchup and disable_catchup is True:
-        raise ValueError(f"Catch up values are mutually exclusive")
+    # Catchup Behaviour Validation    
+    disable_catchup = definition.parameters.get("disable_catchup", None)
+    twenty_hour_catchup = definition.parameters.get("twenty_hour_catchup", None)
+    validate_catchup(disable_catchup, twenty_hour_catchup)
 
     pass
 

diff --git a/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py b/sigsci_TA_for_splunk/bin/input_module_SigsciRequests.py
@@ -2,7 +2,7 @@
 from timeit import default_timer as timer
 import time
 from datetime import datetime, timezone, timedelta
-from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts
+from sigsci_helper import get_from_and_until_times, Config, get_results, get_until_time, validate_timeouts, validate_catchup
 
 """
     IMPORTANT
@@ -30,10 +30,10 @@ def validate_input(helper,definition):
     read_timeout = definition.parameters.get("read_timeout", None)
     validate_timeouts(request_timeout, read_timeout)
 
-    twenty_hour_catchup = definition.parameters.get('twenty_hour_catchup', None)
-    disable_catchup = definition.parameters.get('disable_catchup', None)
-    if twenty_hour_catchup and disable_catchup is True:
-        raise ValueError(f"Catch up values are mutually exclusive")
+    # Catchup Behaviour Validation    
+    disable_catchup = definition.parameters.get("disable_catchup", None)
+    twenty_hour_catchup = definition.parameters.get("twenty_hour_catchup", None)
+    validate_catchup(disable_catchup, twenty_hour_catchup)
 
     site_name = definition.parameters.get("site_api_name", None)
     if site_name is None or site_name == "":

diff --git a/sigsci_TA_for_splunk/bin/sigsci_helper.py b/sigsci_TA_for_splunk/bin/sigsci_helper.py
@@ -29,6 +29,21 @@ def validate_timeouts(request_timeout, read_timeout):
     if read_timeout > 300.0 or read_timeout <= 0:
         raise ValueError(f"Read timeout must be between 0 and 300 seconds, got {read_timeout}")
 
+def validate_catchup(disable_catchup, twenty_hour_catchup):
+    ## definitions.parameters.get returns the defaultValue for a checkbox as a str on init with the value of `true`.
+    ## We have to accomodate for when a user tries and ticks both without changing the value.
+    if disable_catchup is not None:
+        if disable_catchup.lower() == 'true':
+            disable_catchup = 1
+        else:
+            disable_catchup = int(disable_catchup)
+
+        if twenty_hour_catchup is not None:
+            twenty_hour_catchup = int(twenty_hour_catchup)
+
+        if twenty_hour_catchup and disable_catchup:
+            raise ValueError("Catch up values are mutually exclusive")
+
 def check_response(
         code,
         response_text,

diff --git a/sigsci_TA_for_splunk/default/inputs.conf b/sigsci_TA_for_splunk/default/inputs.conf
@@ -1,10 +1,12 @@
-[SigsciEvent]
+[SigsciRequests]
 start_by_shell = false
 python.version = python3
-sourcetype = sigsci-event
+sourcetype = sigsci-requests
 interval = 300
+request_limit = 1000
 disable_catchup = True
 twenty_hour_catchup = False
+attack_and_anomaly_signals_only = False
 request_timeout = 60
 read_timeout = 60
 disabled = 0
@@ -20,15 +22,13 @@ request_timeout = 60
 read_timeout = 60
 disabled = 0
 
-[SigsciRequests]
+[SigsciEvent]
 start_by_shell = false
 python.version = python3
-sourcetype = sigsci-requests
+sourcetype = sigsci-event
 interval = 300
-request_limit = 1000
 disable_catchup = True
 twenty_hour_catchup = False
-attack_and_anomaly_signals_only = False
 request_timeout = 60
 read_timeout = 60
 disabled = 0