From 23314391a7e2221771068d61f350962254876bc5 Mon Sep 17 00:00:00 2001 From: Thomas Michelat Date: Fri, 13 Aug 2021 21:21:08 +0200 Subject: [PATCH 1/3] fix logging with loguru --- redmine_zulip/redmine.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/redmine_zulip/redmine.py b/redmine_zulip/redmine.py index acf915f..72e02ad 100644 --- a/redmine_zulip/redmine.py +++ b/redmine_zulip/redmine.py @@ -352,11 +352,8 @@ def _update_status(self, issue, ticket): notify_old_topic=False, notify_new_topic=False ) - log.info('Update status for issue #%s: %s -> %s', - issue["task_id"], - issue["status_name"], - ticket.status.name) - log.info('%s', res) + log.info(f'Update status for issue #{issue["task_id"]}: {issue["status_name"]} -> {ticket.status.name}') + log.info(res) # update DB entry data = {'task_id': ticket.id, From f4d96d1ceb82ce80d87410aa261ec53255856ab6 Mon Sep 17 00:00:00 2001 From: Thomas Michelat Date: Fri, 13 Aug 2021 21:41:18 +0200 Subject: [PATCH 2/3] refactor --- redmine_zulip/redmine.py | 94 ++++++++-------------------------------- redmine_zulip/utils.py | 51 ++++++++++++++++++++++ 2 files changed, 69 insertions(+), 76 deletions(-) create mode 100644 redmine_zulip/utils.py diff --git a/redmine_zulip/redmine.py b/redmine_zulip/redmine.py index 72e02ad..0548c0c 100644 --- a/redmine_zulip/redmine.py +++ b/redmine_zulip/redmine.py @@ -1,87 +1,24 @@ from argparse import ArgumentParser from datetime import datetime -from functools import lru_cache, partial, wraps +from functools import lru_cache from multiprocessing.dummy import Pool from pathlib import Path -import re import requests from textwrap import dedent from threading import Lock -from time import sleep from typing import Union import atoma import dataset from loguru import logger as log -import pypandoc as pandoc -from redminelib import Redmine as RedmineLib -from redminelib.resources.standard import Issue +from redminelib import Redmine import toml import zulip - -RESOLVED_TOPIC_PREFIX = b'\xe2\x9c\x94 '.decode('utf8') # = '✔ ' - - -def textile_to_md(text): - text = pandoc.convert_text(text, to='markdown_github', format='textile') - return re.sub(r'\\(.)', r'\1', text) - - -def indent(text, offset=3): - """Indent text with offset * 4 blank spaces - """ - def indented_lines(): - for ix, line in enumerate(text.splitlines(True)): - if ix == 0: - yield line - else: - yield ' ' * offset + line if line.strip() else line - return ''.join(indented_lines()) - - -def retry(func=None, *, attempts=1, delay=0, exc=(Exception,)): - """Re-execute decorated function. - :attemps int: number of tries, default 1 - :delay float: timeout between each tries in seconds, default 0 - :exc tuple: collection of exceptions to be caugth - """ - if func is None: - return partial(retry, attempts=attempts, delay=delay, exc=exc) - - @wraps(func) - def retried(*args, **kwargs): - retry._tries[func.__name__] = 0 - for i in reversed(range(attempts)): - retry._tries[func.__name__] += 1 - try: - ret = func(*args, *kwargs) - except exc: - if i <= 0: - raise - sleep(delay) - continue - else: - break - return ret - - retry._tries = {} - return retried +from .utils import indent, textile_to_md -def format_topic(issue: dict) -> str: - return f'Issue #{issue["task_id"]} - {issue["status_name"]}' - - -class Redmine: - def __init__(self, conf): - self.remote = RedmineLib(conf['url'], key=conf['token']) - self.project = self.remote.project.get(conf['project']) - - def get(self, issue: int) -> Issue: - """Get a redmine issue from it's ID - """ - return self.remote.issue.get(issue) +RESOLVED_TOPIC_PREFIX = b'\xe2\x9c\x94 '.decode('utf8') # = '✔ ' class Publisher: @@ -94,7 +31,8 @@ def __init__(self, configuration: Union[str, Path]): conf = toml.load(configuration) # logging - log.add(conf['LOGGING']['file']) + if conf['LOGGING'].get('file'): + log.add(conf['LOGGING']['file']) # database connection self.db_path = conf['DATABASE']['sql3_file'] @@ -106,9 +44,13 @@ def __init__(self, configuration: Union[str, Path]): self.zulip_admin = zulip.Client(config_file=conf['ZULIP']['admin']) self.stream = conf['ZULIP']['stream'] - self.redmine = Redmine(conf['REDMINE']) + self.redmine = Redmine(conf['REDMINE']['url'], key=conf['REDMINE']['token']) self.feed = conf['REDMINE']['rss_feed'] + @staticmethod + def format_topic(issue: dict) -> str: + return f'Issue #{issue["task_id"]} - {issue["status_name"]}' + def run(self): log.info('Polling Redmine for new tasks') self.poll() @@ -140,7 +82,7 @@ def poll(self): 'author': issue.authors[0].name, 'title': issue.title.value, } - issue = self.redmine.get(info['task_id']) + issue = self.redmine.issue.get(info['task_id']) assert issue.id == info['task_id'] info['status_name'] = issue.status.name info['status_id'] = issue.status.id @@ -169,7 +111,7 @@ def _track(self, data): issues = db['issues'] # log.debug(f'{n}/{len(issues)} - {issue}') - ticket = self.redmine.get(issue['task_id']) + ticket = self.redmine.issue.get(issue['task_id']) # check for new journal and attachments: add message per entry self._publish_journal(issue, ticket) @@ -227,7 +169,7 @@ def _publish_journal(self, issue, ticket): if not journal.notes: continue - url = f'{self.redmine.remote.url}/issues/{issue["task_id"]}#change-{journal.id}' + url = f'{self.redmine.url}/issues/{issue["task_id"]}#change-{journal.id}' msg = ( f'**{journal.user.name}** [said]({url}):\n' f'```quote\n{textile_to_md(journal.notes)}\n```' @@ -290,7 +232,7 @@ def upload_attachment(self, attachment): only publish images, other attachments are links to redmine """ - f = self.redmine.remote.file.get(attachment.id) + f = self.redmine.file.get(attachment.id) fpath = f.download(savepath='/tmp/') log.info("Redmine download file to: %s", fpath) @@ -308,7 +250,7 @@ def upload_attachment(self, attachment): return result def send(self, issue, content): - topic = format_topic(issue) + topic = self.format_topic(issue) if f'{RESOLVED_TOPIC_PREFIX}{topic}' in self.zulip_topic_names(): topic = f'{RESOLVED_TOPIC_PREFIX}{topic}' @@ -321,7 +263,7 @@ def send(self, issue, content): log.info("%s", reply) @lru_cache() - @retry(attempts=10) + # @retry(attempts=10) def zulip_topics(self): stream = self.zulip.get_stream_id(self.stream) stream = self.zulip.get_stream_topics(stream['stream_id']) @@ -366,7 +308,7 @@ def _maybe_resolve_topic(self, issue): if issue['status_name'] != 'Closed': return - title = format_topic(issue) + title = self.format_topic(issue) for topic in self.zulip_topics(): if topic['name'] == title: self.zulip.update_message({ diff --git a/redmine_zulip/utils.py b/redmine_zulip/utils.py new file mode 100644 index 0000000..9e83221 --- /dev/null +++ b/redmine_zulip/utils.py @@ -0,0 +1,51 @@ +from functools import partial, wraps +import re +from time import sleep + +import pypandoc + + +def textile_to_md(text): + text = pypandoc.convert_text(text, to='markdown_github', format='textile') + return re.sub(r'\\(.)', r'\1', text) + + +def indent(text, offset=3): + """Indent text with offset * 4 blank spaces + """ + def indented_lines(): + for ix, line in enumerate(text.splitlines(True)): + if ix == 0: + yield line + else: + yield ' ' * offset + line if line.strip() else line + return ''.join(indented_lines()) + + +def retry(func=None, *, attempts=1, delay=0, exc=(Exception,)): + """Re-execute decorated function. + :attemps int: number of tries, default 1 + :delay float: timeout between each tries in seconds, default 0 + :exc tuple: collection of exceptions to be caugth + """ + if func is None: + return partial(retry, attempts=attempts, delay=delay, exc=exc) + + @wraps(func) + def retried(*args, **kwargs): + retry._tries[func.__name__] = 0 + for i in reversed(range(attempts)): + retry._tries[func.__name__] += 1 + try: + ret = func(*args, *kwargs) + except exc: + if i <= 0: + raise + sleep(delay) + continue + else: + break + return ret + + retry._tries = {} + return retried From 810d4a64b88f328c0dd2fe8e7045db46b93a9f01 Mon Sep 17 00:00:00 2001 From: Thomas Michelat Date: Fri, 13 Aug 2021 22:13:20 +0200 Subject: [PATCH 3/3] lock writes to db --- redmine_zulip/redmine.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/redmine_zulip/redmine.py b/redmine_zulip/redmine.py index 0548c0c..ce546bc 100644 --- a/redmine_zulip/redmine.py +++ b/redmine_zulip/redmine.py @@ -6,9 +6,10 @@ import requests from textwrap import dedent from threading import Lock -from typing import Union +from typing import Dict, List, Set, Union import atoma +from atoma.atom import AtomEntry import dataset from loguru import logger as log from redminelib import Redmine @@ -35,8 +36,9 @@ def __init__(self, configuration: Union[str, Path]): log.add(conf['LOGGING']['file']) # database connection - self.db_path = conf['DATABASE']['sql3_file'] - self._db = dataset.connect(self.db_path, engine_kwargs={"connect_args": {"check_same_thread": False}}) + db_path = conf['DATABASE']['sql3_file'] + self._db = dataset.connect( + db_path, engine_kwargs={"connect_args": {"check_same_thread": False}}) self.issues = self._db['issues'] self.lock = Lock() @@ -107,10 +109,7 @@ def track(self): def _track(self, data): n, issue = data - db = dataset.connect(self.db_path) - issues = db['issues'] - - # log.debug(f'{n}/{len(issues)} - {issue}') + # log.debug(f'{n}/{len(self.issues)} - {issue}') ticket = self.redmine.issue.get(issue['task_id']) # check for new journal and attachments: add message per entry @@ -120,7 +119,7 @@ def _track(self, data): if ticket.status.id != issue['status_id']: # check for status: update the topic title self._update_status(issue, ticket) - issue = issues.find_one(task_id=issue['task_id']) + issue = self.issues.find_one(task_id=issue['task_id']) self._maybe_resolve_topic(issue) @@ -130,14 +129,16 @@ def _track(self, data): last_update = datetime.now() data = {'task_id': issue['task_id'], 'updated': last_update} - issues.update(data, ['task_id']) + with self.lock: + self.issues.update(data, ['task_id']) if ticket.status.name == 'Closed' and (datetime.now() - last_update).days >= 7: # ticket is closed, remove from DB log.info(f'ticket {ticket.id} closed and inactive for more than 7 days, stop tracking') - issues.delete(task_id=ticket.id) + with self.lock: + self.issues.delete(task_id=ticket.id) - def _get_feed(self): + def _get_feed(self) -> List[AtomEntry]: """Get issues from rss url""" r = requests.get(self.feed) if r.status_code != requests.codes.ok: @@ -189,7 +190,8 @@ def _publish_journal(self, issue, ticket): 'journals': str([e for e in sorted(known_entries)]), 'updated': datetime.now() } - self.issues.update(data, ['task_id']) + with self.lock: + self.issues.update(data, ['task_id']) def _publish_attachment(self, issue, ticket): known_attachments = eval(issue.get('attachments', '[]') or '[]') @@ -225,7 +227,8 @@ def _publish_attachment(self, issue, ticket): 'attachments': str([e for e in sorted(known_attachments)]), 'updated': datetime.now() } - self.issues.update(data, ['task_id']) + with self.lock: + self.issues.update(data, ['task_id']) def upload_attachment(self, attachment): """Download attachment from Redmine and upload it on Zulip @@ -263,13 +266,12 @@ def send(self, issue, content): log.info("%s", reply) @lru_cache() - # @retry(attempts=10) - def zulip_topics(self): + def zulip_topics(self) -> List[Dict]: stream = self.zulip.get_stream_id(self.stream) stream = self.zulip.get_stream_topics(stream['stream_id']) return [s for s in stream['topics']] - def zulip_topic_names(self): + def zulip_topic_names(self) -> Set[str]: return {s['name'] for s in self.zulip_topics()} def _update_status(self, issue, ticket): @@ -302,7 +304,8 @@ def _update_status(self, issue, ticket): 'status_id': ticket.status.id, 'status_name': ticket.status.name, 'updated': datetime.now()} - self.issues.update(data, ['task_id']) + with self.lock: + self.issues.update(data, ['task_id']) def _maybe_resolve_topic(self, issue): if issue['status_name'] != 'Closed':