Merge pull request #2491 from e3rd/enh-smtp_batch-hardening

ENH: smtp_batch invalid row hardening
certtools · Apr 9, 2024 · d4ea2ad · d4ea2ad
2 parents 188f091 + 6e6c03b
commit d4ea2ad
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 8 deletions.
diff --git a/docs/user/bots.md b/docs/user/bots.md
@@ -4969,7 +4969,7 @@ Note: The field "raw" gets base64 decoded if possible. Bytes `\n` and `\r` are r
 
 Launch it like this:
 ```
-</usr/local/bin executable> <bot-id> cli [--tester tester's email]
+</usr/local/bin executable> <bot-id> --cli [--tester tester's email]
 ```
 Example:
 ```bash
@@ -4993,7 +4993,7 @@ You can schedule the batch sending easily with a cron script, I.E. put this into
 
 ```
 # Send the e-mails every day at 6 AM
-0 6 * * *  /usr/local/bin/intelmq.bots.outputs.smtp_batch.output smtp-batch-output-cz cli --ignore-older-than-days 4 --send > /tmp/intelmq-send.log
+0 6 * * *  /usr/local/bin/intelmq.bots.outputs.smtp_batch.output smtp-batch-output-cz cli --ignore-older-than-days 4 --send &> /tmp/intelmq-send.log
 ```
 
 **Module:** `intelmq.bots.outputs.smtp_batch.output`

diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py
@@ -237,9 +237,7 @@ def prepare_mails(self):
                         print("... failed ...", flush=True)
                         continue
                 else:
-                    # visible both warning and print
-                    print(f"Warning: {mail_record} timeout, too big to read from redis", flush=True)
-                    self.logger.warning(f"Warning: {mail_record} timeout, too big to read from redis")
+                    self.logger.warning("Warning: timeout, too big to read from redis: %r", mail_record)
                     self.timeout.append(mail_record)
                     continue
 
@@ -253,8 +251,11 @@ def prepare_mails(self):
             fieldnames = set()
             rows_output = []
             for row in lines:
-                if threshold and row["time.observation"][:19] < threshold.isoformat()[:19]:
-                    continue
+                try:
+                    if threshold and row["time.observation"][:19] < threshold.isoformat()[:19]:
+                        continue
+                except KeyError:
+                    self.logger.warning("Warning: row skipped due to time.observation error: %r", mail_record)
                 fieldnames = fieldnames | set(row.keys())
                 keys = set(self.allowed_fieldnames).intersection(row)
                 ordered_keys = []
@@ -290,7 +291,7 @@ def prepare_mails(self):
                     try:
                         zf.writestr(filename + ".csv", output.getvalue())
                     except Exception:
-                        self.logger.error(f"Cannot zip mail {mail_record}")
+                        self.logger.error("Error: Cannot zip mail: %r", mail_record)
                         continue
 
                 if email_to in self.alternative_mail: