clarify submission with bots

Frontend: Clarify that the switch also applies to submissions. Longer
explanation text what happens when unchecked.
Backend: Do not send data to queue if bots were active (would be a
double-submit)
Documentation: Add documentation on the bots configuration parameter and
explain the behaviour of the Frontend's "Submit/Validate with Bots"
parameter

client/src/components/WebinputCSV.vue

@@ -265,7 +265,7 @@
                         @change="onDryRunChange"
                       ></b-form-checkbox>
                     </b-form-group>
-                    <b-form-group v-b-tooltip.hover label-cols=4 label="Validate with bots" title="Validate the data with all configured IntelMQ Bots included.">
+                    <b-form-group v-b-tooltip.hover label-cols=4 label="Validate/Submit with bots" title="Validate the data with all configured IntelMQ Bots included. If unchecked on submission, data will be submitted to the IntelMQ processing pipeline, not the configured output bot.">
                       <b-form-checkbox
                         v-model="validateWithBots"
                         switch

docs/user-guide.rst

@@ -28,7 +28,7 @@ Usual configuration parameters
    used to configure the pipeline, see the user-guide of intelmq for
    details.
 -  ``destination_pipeline_queue``: It is the queue/routing key to push
-   the messages into.
+   the messages into if the user does not check *Validate/Submit with Bots*.
 -  ``destination_pipeline_queue_formatted``: Optional, if true, the
    ``destination_pipeline_queue`` is formatted like a python format
    string with the event as ``ev``. E.g.
@@ -41,7 +41,48 @@ Usual configuration parameters
    list), all lines need to have these fields, otherwise they are marked
    as invalid and not submitted to IntelMQ. Example:
    ``required_fields: ["time.source", "source.ip"]``
--  ``bots``: FIXME
+-  ``bots``: An optional directory of bot definitions with the same structure as
+   IntelMQ's runtime configuration. If the Webinput user checks
+   *Validate/Submit with Bots* in the Frontend, these bots will be used for data
+   validation and submission. Needs to contain a processing chain of any number
+   of experts and exactly one output bot.
+   For validation, output bots are not called. For submission, only the output
+   bot writes the data to the destination, the data is not additionally pushed
+   to the redis queue as specified in the configuration parameter
+   ``destination_pipeline_queue``.
+   Example configuration:
+
+   .. code:: json
+
+      "bots": {
+          "taxonomy": {
+              "module": "intelmq.bots.experts.taxonomy.expert"
+          },
+          "url": {
+              "module": "intelmq.bots.experts.url.expert"
+          },
+          "gethostbyname": {
+              "module": "intelmq.bots.experts.gethostbyname.expert"
+          },
+          "sql": {
+              "module": "intelmq_webinput_csv.sql_output",
+              "parameters": {
+                  "autocommit": false,
+                  "engine": "postgresql",
+                  "database": "eventdb",
+                  "host": "intelmq-database",
+                  "port": 5432,
+                  "user": "intelmq",
+                  "password": "secret",
+                  "sslmode": "allow",
+                  "table": "events"
+              }
+          }
+      }
+
+   Warning: If no SQL output bot is configured, and the user uses validation/submission
+   with bots, no data will be written anywhere, neither to an IntelMQ pipeline,
+   nor to the database!
 
 Mailgen configuration parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

intelmq_webinput_csv/serve.py

@@ -314,27 +314,29 @@ def uploadCSV(body, request, response):
             bots_output = [event]
         output_lines += len(bots_output)
 
-        for event in bots_output:
-            try:
-                if CONFIG.get('destination_pipeline_queue_formatted', False):
-                    CONFIG['destination_pipeline_queue'].format(ev=event)
-            except Exception as exc:
-                retval[lineno][-1].append(f"Failed to generate destination_pipeline_queue {CONFIG['destination_pipeline_queue']}: {exc!s}")
-                input_line_valid = False
-            if not input_line_valid:
-                continue
-
-            if required_fields:
-                diff = set(required_fields) - event.keys()
-                if diff:
+        if not body.get('validate_with_bots', False):
+            for event in bots_output:
+                try:
+                    if CONFIG.get('destination_pipeline_queue_formatted', False):
+                        CONFIG['destination_pipeline_queue'].format(ev=event)
+                except Exception as exc:
+                    retval[lineno][-1].append(f"Failed to generate destination_pipeline_queue {CONFIG['destination_pipeline_queue']}: {exc!s}")
                     input_line_valid = False
-                    retval[lineno][-1].append(f"Line is missing these required fields: {', '.join(diff)}")
+                if not input_line_valid:
+                    continue
+
+                if required_fields:
+                    diff = set(required_fields) - event.keys()
+                    if diff:
+                        input_line_valid = False
+                        retval[lineno][-1].append(f"Line is missing these required fields: {', '.join(diff)}")
+
+                # if 'raw' not in event:
+                #     event.add('raw', ''.join(raw_header + [handle_rewindable.current_line]))
+                raw_message = MessageFactory.serialize(event)
+                if body.get('submit', True) and input_line_valid:
+                    destination_pipeline.send(raw_message)
 
-            # if 'raw' not in event:
-            #     event.add('raw', ''.join(raw_header + [handle_rewindable.current_line]))
-            raw_message = MessageFactory.serialize(event)
-            if body.get('submit', True) and input_line_valid:
-                destination_pipeline.send(raw_message)
         # if line was valid, increment the counter by 1
         input_lines_invalid += not input_line_valid