diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f295e07 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203, E704 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ee8dbc..309ffe7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - uses: psf/black@stable + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..f43854b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.defaultFormatter": "ms-python.black-formatter" +} diff --git a/gcn_email/cli.py b/gcn_email/cli.py index af70d61..5b489c8 100644 --- a/gcn_email/cli.py +++ b/gcn_email/cli.py @@ -20,22 +20,31 @@ def host_port(host_port_str): # Parse netloc like it is done for HTTP URLs. # This ensures that we will get the correct behavior for hostname:port # splitting even for IPv6 addresses. - return urllib.parse.urlparse(f'http://{host_port_str}') + return urllib.parse.urlparse(f"http://{host_port_str}") @click.command() @click.option( - '--prometheus', type=host_port, default=':8000', show_default=True, - help='Hostname and port to listen on for Prometheus metric reporting') + "--prometheus", + type=host_port, + default=":8000", + show_default=True, + help="Hostname and port to listen on for Prometheus metric reporting", +) @click.option( - '--loglevel', type=click.Choice(logging._levelToName.values()), - default='INFO', show_default=True, help='Log level') + "--loglevel", + type=click.Choice(logging._levelToName.values()), + default="INFO", + show_default=True, + help="Log level", +) def main(prometheus, loglevel): logging.basicConfig(level=loglevel) - prometheus_client.start_http_server(prometheus.port, - prometheus.hostname or '0.0.0.0') - log.info('Prometheus listening on %s', prometheus.netloc) + prometheus_client.start_http_server( + prometheus.port, prometheus.hostname or "0.0.0.0" + ) + log.info("Prometheus listening on %s", prometheus.netloc) consumer = connect_as_consumer() subscribe_to_topics(consumer) diff --git a/gcn_email/core.py b/gcn_email/core.py index 7537575..b0dc8e1 100644 --- a/gcn_email/core.py +++ b/gcn_email/core.py @@ -18,21 +18,22 @@ from .helpers import periodic_task from . import metrics -SESV2 = boto3.client('sesv2') +SESV2 = boto3.client("sesv2") SENDER = f'GCN Notices <{os.environ["EMAIL_SENDER"]}>' # Maximum send rate -MAX_SENDS = boto3.client('ses').get_send_quota()['MaxSendRate'] +MAX_SENDS = boto3.client("ses").get_send_quota()["MaxSendRate"] log = logging.getLogger(__name__) def get_email_notification_subscription_table(): - client = boto3.client('ssm') + client = boto3.client("ssm") result = client.get_parameter( - Name='/RemixGcnProduction/tables/email_notification_subscription') - table_name = result['Parameter']['Value'] - return boto3.resource('dynamodb').Table(table_name) + Name="/RemixGcnProduction/tables/email_notification_subscription" + ) + table_name = result["Parameter"]["Value"] + return boto3.resource("dynamodb").Table(table_name) def query_and_project_subscribers(table, topic): @@ -43,20 +44,21 @@ def query_and_project_subscribers(table, topic): """ try: response = table.query( - IndexName='byTopic', + IndexName="byTopic", ProjectionExpression="#topic, recipient", ExpressionAttributeNames={"#topic": "topic"}, - KeyConditionExpression=(Key('topic').eq(topic))) + KeyConditionExpression=(Key("topic").eq(topic)), + ) except Exception: - log.exception('Failed to query recipients') + log.exception("Failed to query recipients") return [] else: - return [x['recipient'] for x in response['Items']] + return [x["recipient"] for x in response["Items"]] def connect_as_consumer(): - log.info('Connecting to Kafka') - return Consumer(config_from_env(), **{'enable.auto.commit': False}) + log.info("Connecting to Kafka") + return Consumer(config_from_env(), **{"enable.auto.commit": False}) @periodic_task(86400) @@ -65,26 +67,32 @@ def subscribe_to_topics(consumer: Consumer): # This may need to be updated if new topics have a format different than # 'gcn.classic.[text | voevent | binary].[topic]' topics = [ - topic for topic in consumer.list_topics().topics - if topic.startswith('gcn.')] - log.info('Subscribing to topics: %r', topics) + topic for topic in consumer.list_topics().topics if topic.startswith("gcn.") + ] + log.info("Subscribing to topics: %r", topics) consumer.subscribe(topics) def kafka_message_to_email(message): topic = message.topic() email_message = EmailMessage() - if topic.startswith('gcn.classic.text.'): + if topic.startswith("gcn.classic.text."): email_message.set_content(message.value().decode()) - elif topic.startswith('gcn.classic.voevent.'): + elif topic.startswith("gcn.classic.voevent."): email_message.add_attachment( - message.value(), filename='notice.xml', - maintype='application', subtype='xml') + message.value(), + filename="notice.xml", + maintype="application", + subtype="xml", + ) else: email_message.add_attachment( - message.value(), filename='notice.bin', - maintype='application', subtype='octet-stream') - email_message['Subject'] = topic + message.value(), + filename="notice.bin", + maintype="application", + subtype="octet-stream", + ) + email_message["Subject"] = topic return email_message.as_bytes() @@ -94,19 +102,19 @@ def recieve_alerts(consumer): for message in consumer.consume(timeout=1): error = message.error() if error is not None: - log.error('Error consuming message: %s', error) + log.error("Error consuming message: %s", error) continue topic = message.topic() metrics.received.labels(topic).inc() recipients = query_and_project_subscribers(table, topic) if recipients: - log.info('Sending message for topic %s', topic) + log.info("Sending message for topic %s", topic) email = kafka_message_to_email(message) for recipient in recipients: try: send_raw_ses_message_to_recipient(email, recipient) except Exception: - log.exception('Failed to send message') + log.exception("Failed to send message") else: metrics.sent.labels(topic, recipient).inc() consumer.commit(message) @@ -123,12 +131,13 @@ def recieve_alerts(consumer): SESV2.exceptions.SendingPausedException, SESV2.exceptions.TooManyRequestsException, ), - max_time=300 + max_time=300, ) @limits(calls=MAX_SENDS, period=1) @metrics.send_request_latency_seconds.time() def send_raw_ses_message_to_recipient(bytes, recipient): SESV2.send_email( FromEmailAddress=SENDER, - Destination={'ToAddresses': [recipient]}, - Content={'Raw': {'Data': bytes}}) + Destination={"ToAddresses": [recipient]}, + Content={"Raw": {"Data": bytes}}, + ) diff --git a/gcn_email/helpers.py b/gcn_email/helpers.py index 2796ca1..15c198f 100644 --- a/gcn_email/helpers.py +++ b/gcn_email/helpers.py @@ -15,11 +15,13 @@ def inner_wrap(): try: function(*args, **kwargs) except Exception: - log.exception('Periodic task failed') + log.exception("Periodic task failed") stop.wait(interval) t = threading.Thread(target=inner_wrap, daemon=True) t.start() return stop + return wrap + return outer_wrap diff --git a/gcn_email/metrics.py b/gcn_email/metrics.py index d42ef2a..07e94ac 100644 --- a/gcn_email/metrics.py +++ b/gcn_email/metrics.py @@ -9,20 +9,17 @@ import prometheus_client received = prometheus_client.Counter( - 'received', - 'Kafka messages received', - labelnames=['topic'], - namespace=__package__) + "received", "Kafka messages received", labelnames=["topic"], namespace=__package__ +) sent = prometheus_client.Counter( - 'sent', - 'Emails sent', - labelnames=['topic', 'address'], - namespace=__package__) + "sent", "Emails sent", labelnames=["topic", "address"], namespace=__package__ +) send_request_latency_seconds = prometheus_client.Histogram( - 'send_request_latency_seconds', - 'Time taken to send each message', - namespace=__package__) + "send_request_latency_seconds", + "Time taken to send each message", + namespace=__package__, +) diff --git a/poetry.lock b/poetry.lock index 52e5c94..0db626f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -25,6 +25,52 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +[[package]] +name = "black" +version = "24.1.1" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, + {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, + {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, + {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, + {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, + {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, + {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, + {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, + {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, + {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, + {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, + {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, + {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, + {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, + {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, + {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, + {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, + {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, + {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, + {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, + {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, + {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "boto3" version = "1.34.23" @@ -422,6 +468,54 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + [[package]] name = "prometheus-client" version = "0.17.1" @@ -542,6 +636,28 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + [[package]] name = "urllib3" version = "1.26.18" @@ -561,4 +677,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1ec7d2a8a55ecf8be5cedab2d52467790941e65955d501cb4c63c5d79f0949a6" +content-hash = "4d96a702f03e4cf54df9410bbbe4653c238ccda407505291fba41d254737f990" diff --git a/pyproject.toml b/pyproject.toml index c65e913..fd263ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ click = "^8.1.3" prometheus-client = "^0.17.0" [tool.poetry.group.dev.dependencies] +black = "*" flake8 = "*" [tool.poetry.scripts]