[Enhancement] Large incidents widgets - usability changes (#9366)

* Enhanced script
* Changed header Size->Size(MB)
* Changed header Size value to float
* Changed investigation IDs to Hyperlink
* Changed task IDs to Heyperlink
* GetLargestInvestigations now returns by default Text, but can also return table

* Added handling for playground investigation (link/name + size)

* Updated

Co-authored-by: Alex Fiedler <[email protected]>

Packs/CommonWidgets/ReleaseNotes/1_0_4.md

@@ -0,0 +1,27 @@
+
+#### Scripts
+##### GetLargestInputsAndOuputsInIncidents
+- Removed the MB suffix from the values of the **Size** column.
+- Changed the table header name **Size** to **Size(MB)**.
+- Changed the table header **IncidentID** to a hyperlink to the incident.
+- Changed the table header **TaskID** to a hyperlink to the task.
+- Changed the default result format to Markdown.
+##### GetLargestInvestigations
+- Removed the MB suffix from the values of the **Size** column.
+- Changed the table header name **Size** to **Size(MB)**.
+- Changed the table header **IncidentID** to a hyperlink to the incident.
+- Changed the default result format to Markdown.
+- Added handling for **Playground** investigation.
+- Added the *table_result* argument, which returns a result in either Markdown or in a format suitable for a table widget. By default,
+    the result is in Markdown.
+
+#### Widgets
+##### Largest Incidents by Storage Size
+- Removed the MB suffix from the values of the **Size** column.
+- Changed the table header name **Size** to **Size(MB)**.
+- Changed table header **IncidentID** to a hyperlink to the incident.
+##### Largest Inputs And Outputs In Incidents
+- Removed the MB suffix from the values of the **Size** column.
+- Changed the table header name **Size** to **Size(MB)**.
+- Changed the table header **IncidentID** to a hyperlink to the incident.
+- Changed the table header **TaskID** to a hyperlink to the task.

Packs/CommonWidgets/Scripts/GetLargestInputsAndOuputsInIncidents/GetLargestInputsAndOuputsInIncidents.py

@@ -1,15 +1,14 @@
-import traceback
-from operator import itemgetter
-
 import demistomock as demisto
 from CommonServerPython import *
 from CommonServerUserPython import *
 
+import traceback
+
 
 def find_largest_input_or_output(all_args_list):
-    max_arg = {'Size': 0}
+    max_arg = {'Size(MB)': 0}
     for arg in all_args_list:
-        if arg.get('Size') > max_arg.get('Size'):
+        if arg.get('Size(MB)') > max_arg.get('Size(MB)'):
             max_arg = arg
 
     return max_arg
@@ -18,31 +17,37 @@ def find_largest_input_or_output(all_args_list):
 def get_largest_inputs_and_outputs(inputs_and_outputs, largest_inputs_and_outputs, incident_id):
     inputs = []
     outputs = []
+    urls = demisto.demistoUrls()
+    server_url = urls.get('server', '')
+    incident_url = os.path.join(server_url, '#', 'incident', incident_id)
     if inputs_and_outputs:
         # In case no inputs and outputs are found a getInvPlaybookMetaData will return a string.
         # in that case we ignore the results and move on.
         if isinstance(inputs_and_outputs, str):
             return
         for task in inputs_and_outputs:
+            task_id = task.get('id')
             if 'outputs' in task:
                 for output in task.get('outputs'):
+                    task_url = os.path.join(server_url, '#', 'WorkPlan', incident_id, task_id)
                     outputs.append({
-                        'IncidentID': incident_id,
-                        'TaskID': task.get('id'),
+                        'IncidentID': f"[{incident_id}]({incident_url})",
+                        'TaskID': f"[{task_id}]({task_url})",
                         'TaskName': task.get('name'),
                         'Name': output.get('name'),
-                        'Size': output.get('size'),
+                        'Size(MB)': output.get('size'),
                         "InputOrOutput": 'Output'
                     })
 
             else:
                 for arg in task.get('args'):
+                    task_url = os.path.join(server_url, '#', 'WorkPlan', incident_id, task_id)
                     inputs.append({
-                        'IncidentID': incident_id,
-                        'TaskID': task.get('id'),
+                        'IncidentID': f"[{incident_id}]({incident_url})",
+                        'TaskID': f"[{task_id}]({task_url})",
                         'TaskName': task.get('name'),
                         'Name': arg.get('name'),
-                        'Size': arg.get('size'),
+                        'Size(MB)': arg.get('size'),
                         'InputOrOutput': "Input"
                     })
     if inputs:
@@ -52,15 +57,6 @@ def get_largest_inputs_and_outputs(inputs_and_outputs, largest_inputs_and_output
         largest_inputs_and_outputs.append(find_largest_input_or_output(outputs))
 
 
-def format_inputs_and_outputs_to_widget_table(largest_inputs_and_outputs):
-    widget_table = {
-        'data': sorted(largest_inputs_and_outputs, key=itemgetter('Size'), reverse=True),
-        'total': len(largest_inputs_and_outputs)
-    }
-
-    return widget_table
-
-
 def get_extra_data_from_investigations(investigations):
     largest_inputs_and_outputs: List = []
     for inv in investigations:
@@ -70,15 +66,17 @@ def get_extra_data_from_investigations(investigations):
                                                     })[0].get('Contents').get('tasks')
         get_largest_inputs_and_outputs(inputs_and_outputs, largest_inputs_and_outputs, inv.get('IncidentID'))
 
-    return format_inputs_and_outputs_to_widget_table(largest_inputs_and_outputs)
+    return largest_inputs_and_outputs
 
 
 def main():
     try:
-        raw_output = demisto.executeCommand('GetLargestInvestigations', args={'from': demisto.args().get('from'),
-                                                                              'to': demisto.args().get('to')})
+        raw_output = demisto.executeCommand('GetLargestInvestigations_copy', args={'from': demisto.args().get('from'),
+                                                                                   'to': demisto.args().get('to'),
+                                                                                   'table_result': 'true'})
         investigations = raw_output[0].get('Contents', {}).get('data')
-        demisto.results(get_extra_data_from_investigations(investigations))
+        demisto.results(tableToMarkdown('Largest Inputs And Outputs In Incidents',
+                                        get_extra_data_from_investigations(investigations)))
     except Exception:
         demisto.error(traceback.format_exc())  # print the traceback
         return_error(f'Failed to execute GetLargestInputsAndOuputsInIncidents. Error: {traceback.format_exc()}')

Packs/CommonWidgets/Scripts/GetLargestInputsAndOuputsInIncidents/GetLargestInputsAndOuputsInIncidents.yml

@@ -24,5 +24,5 @@ tags:
 - widget
 timeout: '0'
 type: python
-dockerimage: demisto/python3:3.8.5.11789
+dockerimage: demisto/python3:3.8.6.12176
 fromversion: 6.0.0

Packs/CommonWidgets/Scripts/GetLargestInputsAndOuputsInIncidents/GetLargestInputsAndOuputsInIncidents_test.py

@@ -24,20 +24,20 @@
 
 
 largest_input = {
-    'IncidentID': 1,
-    'TaskID': '10',
+    'IncidentID': '[1](https://test-address:8443/#/incident/1)',
+    'TaskID': '[10](https://test-address:8443/#/WorkPlan/1/10)',
     'TaskName': 'Extract indicators from incident again',
     'Name': 'text',
-    'Size': 11.44,
+    'Size(MB)': 11.44,
     'InputOrOutput': 'Input'
 }
 
 largest_output = {
-    'IncidentID': 1,
-    'TaskID': '200',
+    'IncidentID': '[1](https://test-address:8443/#/incident/1)',
+    'TaskID': '[200](https://test-address:8443/#/WorkPlan/1/200)',
     'TaskName': 'Malware Investigation 2',
     'Name': 'IP',
-    'Size': 200.692,
+    'Size(MB)': 200.692,
     'InputOrOutput': 'Output'
 }
 
@@ -53,7 +53,7 @@ def test_get_largest_inputs_and_outputs():
     """
     from GetLargestInputsAndOuputsInIncidents import get_largest_inputs_and_outputs
     res = []
-    get_largest_inputs_and_outputs(inputs_and_outputs, res, 1)
+    get_largest_inputs_and_outputs(inputs_and_outputs, res, '1')
     assert len(res) == 2
     assert largest_input in res
     assert largest_output in res
@@ -70,5 +70,5 @@ def test_get_largest_inputs_and_outputs__on_fail():
     """
     from GetLargestInputsAndOuputsInIncidents import get_largest_inputs_and_outputs
     res = []
-    get_largest_inputs_and_outputs('No data', res, 1)
+    get_largest_inputs_and_outputs('No data', res, '1')
     assert len(res) == 0

Packs/CommonWidgets/Scripts/GetLargestInvestigations/GetLargestInvestigations.py

@@ -1,13 +1,13 @@
+import demistomock as demisto
+from CommonServerPython import *
+from CommonServerUserPython import *
+
 import traceback
 from typing import List, Dict
 from operator import itemgetter
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
-import demistomock as demisto
-from CommonServerPython import *
-from CommonServerUserPython import *
-
 
 def get_investigations(raw_output, investigations):
     # in case getDBStatistics fails to fetch information it will return a message like so:
@@ -22,22 +22,33 @@ def get_investigations(raw_output, investigations):
                 investigations[entry].update({"Date": db.get('dbName')})
 
 
-def parse_investigations_to_table(investigations):
+def parse_investigations_to_table(investigations, is_table_result):
     data: List = []
     widget_table = {"total": len(investigations)}
+    urls = demisto.demistoUrls()
+    server_url = urls.get('server', '')
     for investigation in investigations.keys():
-        size = investigations[investigation].get('leafSize').split(' ')
-        if float(size[0]) >= 1.0 and size[1] == 'MB':
-            db_name = investigations[investigation].get('Date')
+        full_size = investigations[investigation].get('leafSize').split(' ')
+        db_name = investigations[investigation].get('Date')
+        size = float(full_size[0])
+        if size >= 1.0 and full_size[1] == 'MB':
+            if db_name.isdigit():
+                inv_id = investigation.split('-')[1]
+                inv_link = f"[{inv_id}]({os.path.join(server_url, '#', 'incident', inv_id)})"
+                date = db_name[:2] + "-" + db_name[2:]
+            else:
+                inv_id = "-".join(investigation.split('-')[1:])
+                inv_link = f"[playground]({os.path.join(server_url, '#', 'WarRoom', 'playground')})"
+                date = ""
+            inv_link = inv_id if is_table_result else inv_link
             data.append({
-                "IncidentID": investigation.split('-')[1],
-                "Size": investigations[investigation].get('leafSize'),
+                "IncidentID": inv_link,
+                "Size(MB)": int(size) if size == int(size) else size,
                 "AmountOfEntries": investigations[investigation].get('keyN'),
-                "Date": db_name[:2] + "-" + db_name[2:],
-                "SizeNumber": float(size[0])
+                "Date": date
             })
 
-    widget_table['data'] = sorted(data, key=itemgetter('SizeNumber'), reverse=True)  # type: ignore
+    widget_table['data'] = sorted(data, key=itemgetter('Size(MB)'), reverse=True)  # type: ignore
 
     return widget_table
 
@@ -48,13 +59,14 @@ def get_month_db_from_date(date):
     return month + year
 
 
-def get_time_object(timestring):
+def get_time_object(timestring, empty_res_as_now=True):
+    empty_res = datetime.now() if empty_res_as_now else None
     if timestring is None or timestring == '':
-        return datetime.now()
+        return empty_res
 
     date_object = parse(timestring)
     if date_object.year == 1:
-        return datetime.now()
+        return empty_res
     else:
         return date_object
 
@@ -75,10 +87,22 @@ def get_month_database_names():
 def main():
     try:
         investigations: Dict = {}
-        for db_name in get_month_database_names():
-            raw_output = demisto.executeCommand('getDBStatistics', args={"filter": db_name})
+        args: Dict = demisto.args()
+        from_date = args.get('from')
+        is_table_result = args.get('table_result') == 'true'
+        if not get_time_object(from_date, empty_res_as_now=False):
+            raw_output = demisto.executeCommand('getDBStatistics', args={})
             get_investigations(raw_output[0].get('Contents', {}), investigations)
-        demisto.results(parse_investigations_to_table(investigations))
+        else:
+            for db_name in get_month_database_names():
+                raw_output = demisto.executeCommand('getDBStatistics', args={"filter": db_name})
+                get_investigations(raw_output[0].get('Contents', {}), investigations)
+        result = parse_investigations_to_table(investigations, is_table_result)
+        if not is_table_result:
+            # change result to MD
+            result = tableToMarkdown('Largest Incidents by Storage Size', result.get("data"),
+                                     headers=["IncidentID", "Size(MB)", "AmountOfEntries", "Date"])
+        demisto.results(result)
     except Exception:
         demisto.error(traceback.format_exc())  # print the traceback
         return_error(f'Failed to execute GetLargestInvestigations. Error: {traceback.format_exc()}')

Packs/CommonWidgets/Scripts/GetLargestInvestigations/GetLargestInvestigations.yml

@@ -1,16 +1,33 @@
 args:
 - default: false
-  description: The end date for fetching incidents in ISO format. Incidents will be fetched until the last day of the specified month. For example, if you specify 2020-08-14, incidents created on the specified "from" date until  2020-08-31 will be returned.
+  description: The end date for fetching incidents in ISO format. Incidents will be
+    fetched until the last day of the specified month. For example, if you specify
+    2020-08-14, incidents created on the specified "from" date until  2020-08-31 will
+    be returned.
   isArray: false
   name: to
   required: false
   secret: false
 - default: false
-  description: The start date for fetching incidents in ISO format. Incidents will be fetched starting from the first day of the specified month. For example, if you specify 2020-08-14, incidents created on 2020-08-01 will be returned.
+  description: The start date for fetching incidents in ISO format. Incidents will
+    be fetched starting from the first day of the specified month. For example, if
+    you specify 2020-08-14, incidents created on 2020-08-01 will be returned.
   isArray: false
   name: from
   required: false
   secret: false
+- auto: PREDEFINED
+  default: false
+  defaultValue: 'false'
+  description: Change to true to return a result suitable for a table widget. By default
+    the return will be in Markdown.
+  isArray: false
+  name: table_result
+  predefined:
+  - 'true'
+  - 'false'
+  required: false
+  secret: false
 comment: Returns all investigations larger than 1 MB from all Cortex XSOAR.
 commonfields:
   id: GetLargestInvestigations
@@ -24,5 +41,5 @@ tags:
 - widget
 timeout: '0'
 type: python
-dockerimage: demisto/python3:3.8.5.11789
+dockerimage: demisto/python3:3.8.6.12176
 fromversion: 6.0.0

Packs/CommonWidgets/Scripts/GetLargestInvestigations/GetLargestInvestigations_test.py

@@ -20,9 +20,20 @@
                 'newInvPlaybooks': {
                     'keyN': 9,
                     'leafSize': '29 MB'
-                }
+                },
             },
         'dbName': '082020'
+    },
+    {
+        'buckets':
+            {
+                'investigations-playground': {
+                    'keyN': 1301,
+                    'leafSize': '4.8 MB',
+                    'Date': ''
+                }
+            },
+        'dbName': 'main'
     }
 ]
 
@@ -36,6 +47,11 @@
         'keyN': 1301,
         'leafSize': '4.8 MB',
         'Date': '082020'
+    },
+    'investigations-playground': {
+        'keyN': 1301,
+        'leafSize': '4.8 MB',
+        'Date': 'main'
     }
 }
 
@@ -81,12 +97,14 @@ def test_parse_investigations_to_table():
         order (sorted by size) and that the date is formatted correctly.
     """
     from GetLargestInvestigations import parse_investigations_to_table
-    table = parse_investigations_to_table(investigations)
-    assert table.get('total') == 2
+    table = parse_investigations_to_table(investigations, True)
+    assert table.get('total') == 3
     assert table.get('data')[0].get('IncidentID') == '4187'
-    assert table.get('data')[0].get('Size') == '4.8 MB'
-    assert table.get('data')[1].get('IncidentID') == '4185'
-    assert table.get('data')[1].get('Date') == '08-2020'
+    assert table.get('data')[0].get('Size(MB)') == 4.8
+    assert table.get('data')[1].get('Date') == ''
+    assert table.get('data')[1].get('IncidentID') == 'playground'
+    assert table.get('data')[2].get('IncidentID') == '4185'
+    assert table.get('data')[2].get('Date') == '08-2020'
 
 
 def test_get_month_database_names(mocker):