From 72e8cd6616466ab34530336086af346462d453c1 Mon Sep 17 00:00:00 2001 From: gisly Date: Sun, 12 Mar 2023 18:36:49 +0300 Subject: [PATCH 01/18] Added a more detailed error in case the user has a wrong json structure --- src/tg/handlers/access_config_handler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tg/handlers/access_config_handler.py b/src/tg/handlers/access_config_handler.py index 4f845b9..7618920 100644 --- a/src/tg/handlers/access_config_handler.py +++ b/src/tg/handlers/access_config_handler.py @@ -1,5 +1,6 @@ import json import logging +from json import JSONDecodeError from ... import consts, jobs from ...app_context import AppContext @@ -68,6 +69,10 @@ def reload_config_jobs(update, tg_context): ) job_scheduler = JobScheduler() job_scheduler.reschedule_jobs() + except JSONDecodeError as json_e: + reply(f"Error in JSON structure: {json_e}", update) + logger.warning(f"Failed to reload jobs config: {json_e}") + return except Exception as e: reply(load("access_config_handler__reload_config_jobs_usage_example"), update) logger.warning(f"Failed to reload jobs config: {e}") From 0798f2a8f5af8078b1af186617dbe44e344cd9d9 Mon Sep 17 00:00:00 2001 From: Alex Kulikov <7394728+alexeyqu@users.noreply.github.com> Date: Sun, 10 Sep 2023 14:28:24 +0100 Subject: [PATCH 02/18] hr pt acquisition job (#295) * hr pt acquisition job * works --- config.json | 1 + src/bot.py | 6 ++ src/jobs/__init__.py | 1 + src/jobs/hr_acquisition_pt_job.py | 108 ++++++++++++++++++++++++++++++ src/sheets/sheets_client.py | 7 ++ src/sheets/sheets_objects.py | 25 +++++++ 6 files changed, 148 insertions(+) create mode 100644 src/jobs/hr_acquisition_pt_job.py diff --git a/config.json b/config.json index 7598333..a90fa52 100644 --- a/config.json +++ b/config.json @@ -23,6 +23,7 @@ "authors_sheet_key": "do_not_set_here_please_go_to_config_override", "curators_sheet_key": "do_not_set_here_please_go_to_config_override", "hr_sheet_key": "do_not_set_here_please_go_to_config_override", + "hr_pt_sheet_key": "do_not_set_here_please_go_to_config_override", "post_registry_sheet_key": "do_not_set_here_please_go_to_config_override", "postcards_sheet_key": "do_not_set_here_please_go_to_config_override", "rubrics_registry_sheet_key": "do_not_set_here_please_go_to_config_override", diff --git a/src/bot.py b/src/bot.py index dbd1118..66676ab 100644 --- a/src/bot.py +++ b/src/bot.py @@ -130,6 +130,12 @@ def init_handlers(self): self.manager_reply_handler("hr_acquisition_job"), "обработать новые анкеты", ) + self.add_admin_handler( + "hr_acquisition_pt", + CommandCategories.HR, + self.manager_reply_handler("hr_acquisition_pt_job"), + "обработать новые анкеты Пишу Тебе", + ) self.add_manager_handler( "get_hr_status", CommandCategories.HR, diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py index 957f129..1423261 100644 --- a/src/jobs/__init__.py +++ b/src/jobs/__init__.py @@ -18,6 +18,7 @@ from .fb_analytics_report_job import FBAnalyticsReportJob from .fill_posts_list_job import FillPostsListJob from .hr_acquisition_job import HRAcquisitionJob +from .hr_acquisition_pt_job import HRAcquisitionPTJob from .hr_check_chat_consistency_frozen_job import HRCheckChatConsistencyFrozenJob from .hr_check_chat_consistency_job import HRCheckChatConsistencyJob from .hr_check_trello_consistency_frozen_job import HRCheckTrelloConsistencyFrozenJob diff --git a/src/jobs/hr_acquisition_pt_job.py b/src/jobs/hr_acquisition_pt_job.py new file mode 100644 index 0000000..c92a4c4 --- /dev/null +++ b/src/jobs/hr_acquisition_pt_job.py @@ -0,0 +1,108 @@ +import logging +from typing import Callable, List + +from sheetfu import Table + +from ..app_context import AppContext +from ..sheets.sheets_objects import HRPersonPTProcessed, HRPersonPTRaw +from ..strings import load +from ..tg.sender import pretty_send +from .base_job import BaseJob + +logger = logging.getLogger(__name__) + + +class HRAcquisitionPTJob(BaseJob): + @staticmethod + def _execute( + app_context: AppContext, send: Callable[[str], None], called_from_handler=False + ): + paragraphs = [load("hr_acquisition_job__hello")] # list of paragraph strings + + new_people = HRAcquisitionPTJob._process_new_people(app_context) + + if not new_people: + return + + paragraphs += [ + HRAcquisitionPTJob._get_new_person_paragraph(item) for item in new_people + ] + + pretty_send(paragraphs, send) + + @staticmethod + def _process_new_people( + app_context: AppContext, + ) -> List[HRPersonPTProcessed]: + forms_raw = app_context.sheets_client.fetch_hr_pt_forms_raw() + forms_processed = app_context.sheets_client.fetch_hr_pt_forms_processed() + + new_items = HRAcquisitionPTJob._process_raw_forms(forms_raw, forms_processed) + + try: + forms_raw.commit() + forms_processed.commit() + except Exception as e: + logger.error(f"failed to export data: {e}") + + return new_items + + @staticmethod + def _process_raw_forms( + forms_raw: Table, forms_processed: Table + ) -> List[HRPersonPTProcessed]: + people = [HRPersonPTRaw(item) for item in forms_raw] + existing_people = [person for person in people if person.status] + new_people = [person for person in people if not person.status] + new_items = [] + + for person in new_people: + # filter out incomplete responses + if not person.telegram and not person.other_contacts: + person.status = load("sheets__hr__pt__raw__status_rejection") + continue + if person.telegram and ( + person.telegram in {person.telegram for person in existing_people} + or person.telegram in {person.telegram for person in new_items} + ): + person.status = load("sheets__hr__pt__raw__status_double") + continue + + # move good ones to another sheet + person.status = load("sheets__hr__pt__raw__status_processed") + # TODO: PR to sheetfu which will allow better API here + person_dict = { + "id": len(forms_processed) + + 2, # 1 for starting with 1 and 1 for the header + "name": person.name, + "interests": person.interests, + "about": person.about, + "date_submitted": person.ts, + "referral": person.referral, + "telegram": person.telegram, + "status": "TODO", # this is a legitimate value, not an actual TODO + } + new_items.append( + HRPersonPTProcessed.add_one_to_table(forms_processed, person_dict) + ) + + return new_items + + @staticmethod + def _get_new_person_paragraph(item: HRPersonPTProcessed) -> str: + name = load( + "hr_acquisition_job__name", + name=item.name, + telegram=item.telegram, + date=item.date_submitted.split(" ")[0], + ) + interests = load("hr_acquisition_job__interests", interests=item.interests) + about = load("hr_acquisition_job__about", description=item.about) + paragraph = load( + "hr_acquisition_job__person", + name=name, + interests=interests, + about=about, + contacts="", # no other contacts field in PT + ) + return paragraph diff --git a/src/sheets/sheets_client.py b/src/sheets/sheets_client.py index 044352b..4da645a 100644 --- a/src/sheets/sheets_client.py +++ b/src/sheets/sheets_client.py @@ -29,6 +29,7 @@ def _update_from_config(self): self.authors_sheet_key = self._sheets_config["authors_sheet_key"] self.curators_sheet_key = self._sheets_config["curators_sheet_key"] self.hr_sheet_key = self._sheets_config["hr_sheet_key"] + self.hr_pt_sheet_key = self._sheets_config["hr_pt_sheet_key"] self.post_registry_sheet_key = self._sheets_config["post_registry_sheet_key"] self.rubrics_registry_sheet_key = self._sheets_config[ "rubrics_registry_sheet_key" @@ -57,6 +58,12 @@ def fetch_hr_forms_raw(self) -> Table: def fetch_hr_forms_processed(self) -> Table: return self._fetch_table(self.hr_sheet_key, "Анкеты") + def fetch_hr_pt_forms_raw(self) -> Table: + return self._fetch_table(self.hr_pt_sheet_key, "Ответы Главный сайт") + + def fetch_hr_pt_forms_processed(self) -> Table: + return self._fetch_table(self.hr_pt_sheet_key, "Анкеты") + def fetch_hr_team(self) -> Table: return self._fetch_table(self.hr_sheet_key, "Команда (с заморозкой)") diff --git a/src/sheets/sheets_objects.py b/src/sheets/sheets_objects.py index ef97cca..69ff0a4 100644 --- a/src/sheets/sheets_objects.py +++ b/src/sheets/sheets_objects.py @@ -244,6 +244,31 @@ class HRPersonProcessed(SheetsItem): } +class HRPersonPTRaw(SheetsItem): + field_alias = { + "ts": "sheets__hr__pt__raw__timestamp", + "name": "sheets__hr__pt__raw__name", + "interests": "sheets__hr__pt__raw__interests", + "about": "sheets__hr__pt__raw__about", + "telegram": "sheets__hr__pt__raw__telegram", + "referral": "sheets__hr__pt__raw__referral", + "status": "sheets__hr__raw__status", + } + + +class HRPersonPTProcessed(SheetsItem): + field_alias = { + "id": "sheets__hr__pt__processed__id", + "name": "sheets__hr__pt__processed__name", + "interests": "sheets__hr__pt__processed__interests", + "about": "sheets__hr__pt__processed__about", + "referral": "sheets__hr__pt__processed__referral", + "date_submitted": "sheets__hr__pt__processed__date_submitted", + "telegram": "sheets__hr__pt__processed__telegram", + "status": "sheets__hr__pt__processed__status", + } + + class PostRegistryItem(SheetsItem): field_alias = { "name": "sheets__post_registry__column_name", From af51fda54b6e72cf870e03cf4402c27b8fae3409 Mon Sep 17 00:00:00 2001 From: Alexey Kulikov Date: Sun, 10 Sep 2023 14:31:44 +0100 Subject: [PATCH 03/18] type comparisons have failed for some reason --- src/sheets/sheets_objects.py | 2 +- src/tg/handlers/access_config_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sheets/sheets_objects.py b/src/sheets/sheets_objects.py index 69ff0a4..fcb3241 100644 --- a/src/sheets/sheets_objects.py +++ b/src/sheets/sheets_objects.py @@ -188,7 +188,7 @@ def __getattr__(self, name): # Excel time format, http://www.cpearson.com/excel/datetime.htm # 40000 is around 2009 # 50000 is around 2040, ph I hope Sysblok will thrive in 2040 - if type(value) == float and 40000 <= value <= 50000: + if type(value) is float and 40000 <= value <= 50000: return convert_excel_datetime_to_string(value) else: return value diff --git a/src/tg/handlers/access_config_handler.py b/src/tg/handlers/access_config_handler.py index 7618920..1b3a339 100644 --- a/src/tg/handlers/access_config_handler.py +++ b/src/tg/handlers/access_config_handler.py @@ -164,7 +164,7 @@ def _set_config(update, config_path: str, new_value, config_manager: ConfigManag update, ) return - if type(current_config) != type(new_value): + if type(current_config) is not type(new_value): reply( load( "access_config_handler__set_config_type_mismatch", From beb9ec9287f890763bf99c8ddbcf65bb8c25e7e3 Mon Sep 17 00:00:00 2001 From: Alexey Kulikov Date: Sun, 10 Sep 2023 14:58:29 +0100 Subject: [PATCH 04/18] upd PyYAML to 6.0.1 per https://stackoverflow.com/a/76710304 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17ce7ec..a20a2ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -73,7 +73,7 @@ python-dateutil==2.8.1 python-telegram-bot==12.6.1 pytoml==0.1.21 pytz==2020.1 -PyYAML==6.0 +PyYAML==6.0.1 regex==2020.4.4 requests==2.28.1 requests-oauthlib==1.3.0 From 2c0aa7cb68bd6d0f7915346e495b6e724f3953ac Mon Sep 17 00:00:00 2001 From: Alexey Kulikov Date: Sun, 10 Sep 2023 16:13:01 +0100 Subject: [PATCH 05/18] fix --- src/jobs/hr_acquisition_pt_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/hr_acquisition_pt_job.py b/src/jobs/hr_acquisition_pt_job.py index c92a4c4..4812f78 100644 --- a/src/jobs/hr_acquisition_pt_job.py +++ b/src/jobs/hr_acquisition_pt_job.py @@ -58,7 +58,7 @@ def _process_raw_forms( for person in new_people: # filter out incomplete responses - if not person.telegram and not person.other_contacts: + if not person.telegram: person.status = load("sheets__hr__pt__raw__status_rejection") continue if person.telegram and ( From c8bb18f87f039df070ee4972f648cfbdbfe5ae6f Mon Sep 17 00:00:00 2001 From: gisly Date: Sun, 1 Oct 2023 19:29:10 +0300 Subject: [PATCH 06/18] Checking if the web site is accessible at all --- src/jobs/site_health_check_job.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/jobs/site_health_check_job.py b/src/jobs/site_health_check_job.py index e6e7f6e..c825f6e 100644 --- a/src/jobs/site_health_check_job.py +++ b/src/jobs/site_health_check_job.py @@ -41,7 +41,18 @@ def _execute( kwargs = schedule[KWARGS] url = kwargs.get("index_url") logger.debug(f"Checking site health for {kwargs.get('name')}: {url}") - page = requests.get(url) + try: + page = requests.get(url) + except Exception as e: + send( + load( + "site_health_check_job__connection_error", + url=url, + ) + ) + logger.error(f"Connection error for {url}") + return + if page.status_code != 200: send( load( From 6871a091d09f7de0a38095c5682abae022440fb6 Mon Sep 17 00:00:00 2001 From: gisly Date: Sun, 1 Oct 2023 19:47:08 +0300 Subject: [PATCH 07/18] Added an empty paragraph --- src/jobs/site_health_check_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jobs/site_health_check_job.py b/src/jobs/site_health_check_job.py index c825f6e..fa36e1a 100644 --- a/src/jobs/site_health_check_job.py +++ b/src/jobs/site_health_check_job.py @@ -41,6 +41,7 @@ def _execute( kwargs = schedule[KWARGS] url = kwargs.get("index_url") logger.debug(f"Checking site health for {kwargs.get('name')}: {url}") + try: page = requests.get(url) except Exception as e: From b3f2c2742f48a16c0a34abdc2f92e5fabfe8ac3a Mon Sep 17 00:00:00 2001 From: gisly Date: Sun, 1 Oct 2023 20:00:44 +0300 Subject: [PATCH 08/18] Removed whitespace --- src/jobs/site_health_check_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jobs/site_health_check_job.py b/src/jobs/site_health_check_job.py index fa36e1a..c825f6e 100644 --- a/src/jobs/site_health_check_job.py +++ b/src/jobs/site_health_check_job.py @@ -41,7 +41,6 @@ def _execute( kwargs = schedule[KWARGS] url = kwargs.get("index_url") logger.debug(f"Checking site health for {kwargs.get('name')}: {url}") - try: page = requests.get(url) except Exception as e: From 0ab895ef122b5f5c0f6e6be5a81e84c654ce701b Mon Sep 17 00:00:00 2001 From: gisly Date: Sat, 11 Nov 2023 00:44:49 +0300 Subject: [PATCH 09/18] Update site_health_check_job.py --- src/jobs/site_health_check_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/site_health_check_job.py b/src/jobs/site_health_check_job.py index c825f6e..88e02d9 100644 --- a/src/jobs/site_health_check_job.py +++ b/src/jobs/site_health_check_job.py @@ -79,7 +79,7 @@ def _execute( logger.error(f"Bad body contents for {url}") logger.warning(f"Html:\n\n{body_contents}") return - logger.debug("Site contents look healthy") + logger.debug("Site content looks healthy") if called_from_handler: send( load( From 216f8d398698330f06cde2bd8962eef9742180bc Mon Sep 17 00:00:00 2001 From: gisly Date: Sat, 11 Nov 2023 23:49:36 +0300 Subject: [PATCH 10/18] Added automatic string fetching when reloading the job config --- src/tg/handlers/access_config_handler.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tg/handlers/access_config_handler.py b/src/tg/handlers/access_config_handler.py index 1b3a339..8564deb 100644 --- a/src/tg/handlers/access_config_handler.py +++ b/src/tg/handlers/access_config_handler.py @@ -77,6 +77,14 @@ def reload_config_jobs(update, tg_context): reply(load("access_config_handler__reload_config_jobs_usage_example"), update) logger.warning(f"Failed to reload jobs config: {e}") return + num_strings = 0 + try: + num_strings = AppContext().strings_db_client.fetch_strings_sheet( + AppContext().sheets_client + ) + except Exception as e: + reply(load("access_config_handler__reload_config_jobs_usage_example"), update) + logger.warning(f"Failed to reload jobs config when fetching strings: {e}") reply( load( "common__code_wrapper", @@ -84,6 +92,10 @@ def reload_config_jobs(update, tg_context): ), update, ) + reply( + load("db_fetch_strings_sheet_job__success", num_strings=num_strings), + update, + ) @admin_only From 5330c809b8b48b467c949d77a55b3e0f311e15b3 Mon Sep 17 00:00:00 2001 From: Alex Kulikov Date: Mon, 4 Dec 2023 01:17:56 +0000 Subject: [PATCH 11/18] fix: rm legacy workflow --- .github/workflows/publish_master_lobanov.yml | 137 ------------------- 1 file changed, 137 deletions(-) delete mode 100644 .github/workflows/publish_master_lobanov.yml diff --git a/.github/workflows/publish_master_lobanov.yml b/.github/workflows/publish_master_lobanov.yml deleted file mode 100644 index 04175b8..0000000 --- a/.github/workflows/publish_master_lobanov.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Lobanov production - -on: - push: - # Publish `master` as Docker `prod` image. - branches: - - master - - # Publish `v1.2.3` tags as releases. - tags: - - v* - -env: - IMAGE_NAME: lobanovbot - -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v2 - with: - ref: master - - - name: Generate build args - id: args - run: | - echo "::set-output name=commit_hash::$(git rev-parse HEAD)" - echo "::set-output name=commit_hash_short::$(git rev-parse --short HEAD)" - - - name: Publish to Github Packages Registry with cache - uses: whoan/docker-build-with-cache-action@v5 - env: - IMAGE_NAME: ${{ env.IMAGE_NAME }} - COMMIT_HASH: "${{ steps.args.outputs.commit_hash }}" - COMMIT_HASH_SHORT: "${{ steps.args.outputs.commit_hash_short }}" - with: - image_name: ${{ github.repository }}/${{ env.IMAGE_NAME }} - registry: docker.pkg.github.com - username: sysblok - password: ${{ secrets.GITHUB_TOKEN }} - dockerfile: Dockerfile - image_tag: "prod,latest" - build_extra_args: "--build-arg=COMMIT_HASH --build-arg=COMMIT_HASH_SHORT" - - - name: Notify us about failure - if: ${{ failure() }} - env: - TELEGRAM_PROD_TOKEN: ${{ secrets.TELEGRAM_LOBANOV_TOKEN }} - TELEGRAM_LOBANOV_LOG_CHAT_ID: ${{ secrets.TELEGRAM_LOBANOV_LOG_CHAT_ID }} - run: | - curl -X POST \ - -H 'Content-Type: application/json' \ - -d '{"parse_mode": "markdown", "chat_id": ${{ env.TELEGRAM_LOBANOV_LOG_CHAT_ID }}, "text": "[github CI] build [failed](https://github.com/sysblok/sysblokbot/actions/runs/${{github.run_id}})"}' \ - https://api.telegram.org/bot${{env.TELEGRAM_LOBANOV_TOKEN}}/sendMessage - - - name: Deploy package to digitalocean - uses: appleboy/ssh-action@master - env: - GITHUB_USERNAME: sysblok - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_NAME: ${{ env.IMAGE_NAME }} - ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/lobanov - with: - host: ${{ secrets.MASTER_HOST }} - username: ${{ secrets.MASTER_DO_USER }} - passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }} - key: ${{ secrets.MASTER_DO_SSH_KEY }} - port: ${{ secrets.MASTER_PORT }} - envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR - script: | - export CONTAINER_ID=$(docker ps -aq --filter name=lobanov) - export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/lobanov:prod') - docker stop --time=30 $CONTAINER_ID - docker rm $CONTAINER_ID - docker rmi $IMAGE_ID - docker login docker.pkg.github.com -u $GITHUB_USERNAME -p $GITHUB_TOKEN - touch ${{ env.ROOT_DIR }}/sysblokbot.sqlite - touch ${{ env.ROOT_DIR }}/strings.sqlite - docker run -dit --name lobanov \ - --env APP_SOURCE="github CI" --restart unless-stopped \ - -v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \ - -v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \ - -v ${{ env.ROOT_DIR }}/sysblokbot.sqlite:/app/sysblokbot.sqlite \ - -v ${{ env.ROOT_DIR }}/strings.sqlite:/app/strings.sqlite \ - -v ${{ env.ROOT_DIR }}/persistent_storage.pickle:/app/persistent_storage.pickle \ - docker.pkg.github.com/sysblok/sysblokbot/${{ env.IMAGE_NAME }}:prod - - - name: Notify us about failure - if: ${{ failure() }} - env: - TELEGRAM_PROD_TOKEN: ${{ secrets.TELEGRAM_LOBANOV_TOKEN }} - TELEGRAM_LOBANOV_LOG_CHAT_ID: ${{ secrets.TELEGRAM_LOBANOV_LOG_CHAT_ID }} - run: | - curl -X POST \ - -H 'Content-Type: application/json' \ - -d '{"parse_mode": "markdown", "chat_id": ${{ env.TELEGRAM_LOBANOV_LOG_CHAT_ID }}, "text": "[github CI] deploy [failed](https://github.com/sysblok/sysblokbot/actions/runs/${{github.run_id}})"}' \ - https://api.telegram.org/bot${{env.TELEGRAM_LOBANOV_TOKEN}}/sendMessage - - - name: Sleep for 30 seconds - uses: jakejarvis/wait-action@master - with: - time: '30s' - - - name: Check it has started properly - uses: appleboy/ssh-action@master - env: - TELEGRAM_PROD_TOKEN: ${{ secrets.TELEGRAM_LOBANOV_TOKEN }} - TELEGRAM_LOBANOV_LOG_CHAT_ID: ${{ secrets.TELEGRAM_LOBANOV_LOG_CHAT_ID }} - with: - host: ${{ secrets.MASTER_HOST }} - username: ${{ secrets.MASTER_DO_USER }} - passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }} - key: ${{ secrets.MASTER_DO_SSH_KEY }} - port: ${{ secrets.MASTER_PORT }} - envs: TELEGRAM_PROD_TOKEN, TELEGRAM_LOBANOV_LOG_CHAT_ID - script: | - export CONTAINER_ID=$(docker ps -aq --filter name=lobanov) - export IS_RUNNING=$(docker container inspect -f '{{.State.Running}}' $CONTAINER_ID) - if [ $IS_RUNNING == "false" ]; then - export DOCKER_LOGS=$(docker logs $CONTAINER_ID) - curl -X POST \ - -H 'Content-Type: application/json' \ - -d '{"parse_mode": "markdown", "chat_id": ${{ env.TELEGRAM_LOBANOV_LOG_CHAT_ID }}, "text": "[github CI] deploy was ok but the bot [failed](https://github.com/sysblok/sysblokbot/actions/runs/${{github.run_id}})."}' \ - https://api.telegram.org/bot${{env.TELEGRAM_PROD_TOKEN}}/sendMessage - curl -X POST -H 'Content-Type: application/json' \ - -d '{"parse_mode": "markdown", "chat_id": ${{ env.TELEGRAM_LOBANOV_LOG_CHAT_ID }}, "text": "'"$(printf \'%q\n\' $DOCKER_LOGS)"'"}' \ - https://api.telegram.org/bot${{env.TELEGRAM_PROD_TOKEN}}/sendMessage - fi \ No newline at end of file From 27900e0ca34ba755a3d385ed2c423271d5a8d798 Mon Sep 17 00:00:00 2001 From: River <79055927+riveriswild@users.noreply.github.com> Date: Fri, 29 Dec 2023 23:08:16 +0300 Subject: [PATCH 12/18] feat: set up adding manager by @ (#303) feat: set up adding manager by @ --- src/tg/handlers/access_config_handler.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tg/handlers/access_config_handler.py b/src/tg/handlers/access_config_handler.py index 1b3a339..4d1dace 100644 --- a/src/tg/handlers/access_config_handler.py +++ b/src/tg/handlers/access_config_handler.py @@ -105,10 +105,12 @@ def add_manager(update, tg_context): try: tokens = update.message.text.strip().split(maxsplit=2) assert len(tokens) == 2 - manager_id = json.loads(tokens[1]) - assert isinstance(manager_id, int) or ( - isinstance(manager_id, str) and not manager_id.startswith("@") - ) + try: + manager_id = json.loads(tokens[1]) + except Exception: + manager_id = tokens[1] + manager_id = manager_id.lstrip("@") + assert isinstance(manager_id, int) or isinstance(manager_id, str) config_manager = ConfigManager() manager_ids = config_manager.get_telegram_config()[consts.TELEGRAM_MANAGER_IDS][ : From d9de9c8b092ab300866a8548fcab55d776708b40 Mon Sep 17 00:00:00 2001 From: River <79055927+riveriswild@users.noreply.github.com> Date: Fri, 29 Dec 2023 23:24:21 +0300 Subject: [PATCH 13/18] feat: add get_managers handler (#302) feat: add get_managers handler --- src/bot.py | 6 ++++++ src/tg/handlers/__init__.py | 1 + src/tg/handlers/get_managers_handler.py | 10 ++++++++++ 3 files changed, 17 insertions(+) create mode 100644 src/tg/handlers/get_managers_handler.py diff --git a/src/bot.py b/src/bot.py index 66676ab..38633d9 100644 --- a/src/bot.py +++ b/src/bot.py @@ -389,6 +389,12 @@ def init_handlers(self): handlers.clean_chat_data, "clean_chat_data", ) + self.add_admin_handler( + "get_managers", + CommandCategories.MOST_USED, + handlers.get_managers, + "get_managers", + ) # sample handler self.add_handler( diff --git a/src/tg/handlers/__init__.py b/src/tg/handlers/__init__.py index 836efcc..a2aaa78 100644 --- a/src/tg/handlers/__init__.py +++ b/src/tg/handlers/__init__.py @@ -20,6 +20,7 @@ from .error_handler import error from .get_chat_data_handler import get_chat_data from .get_chat_id_handler import get_chat_id_handler as get_chat_id +from .get_managers_handler import get_managers from .get_members_for_role_handler import get_members_for_role # Admin (developer) handlers diff --git a/src/tg/handlers/get_managers_handler.py b/src/tg/handlers/get_managers_handler.py new file mode 100644 index 0000000..6d35e35 --- /dev/null +++ b/src/tg/handlers/get_managers_handler.py @@ -0,0 +1,10 @@ +from ... import consts +from ...config_manager import ConfigManager +from .utils import admin_only, reply + + +@admin_only +def get_managers(update, tg_context): + config_manager = ConfigManager() + manager_ids = config_manager.get_telegram_config()[consts.TELEGRAM_MANAGER_IDS][:] + reply(str(manager_ids), update) From f648cebc1b71106105540f8c82cc88eb530340e9 Mon Sep 17 00:00:00 2001 From: ovrsun <115068324+ovrsun@users.noreply.github.com> Date: Sun, 31 Dec 2023 15:20:53 +0100 Subject: [PATCH 14/18] feat: add fetching all team members (#304) --- .gitignore | 3 ++ src/bot.py | 6 ++++ src/jobs/__init__.py | 1 + src/jobs/db_fetch_all_team_members_job.py | 34 +++++++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 src/jobs/db_fetch_all_team_members_job.py diff --git a/.gitignore b/.gitignore index bac8fec..c584ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ config_override.json src/sheets/sysblokbot.json persistent_storage.pickle *.sqlite-journal +config_gs.json + +*.sqlite *.token diff --git a/src/bot.py b/src/bot.py index 38633d9..1c98fa7 100644 --- a/src/bot.py +++ b/src/bot.py @@ -427,6 +427,12 @@ def init_handlers(self): self.admin_reply_handler("db_fetch_strings_sheet_job"), "обновить таблицу со строками из Google Sheets", ) + self.add_admin_handler( + "db_fetch_all_team_members", + CommandCategories.DATA_SYNC, + self.admin_reply_handler("db_fetch_all_team_members_job"), + "обновить таблицы всех пользователей (авторов, кураторов, команда) из Google Sheets", + ) # general purpose cmds self.add_admin_handler( diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py index 1423261..c213fa1 100644 --- a/src/jobs/__init__.py +++ b/src/jobs/__init__.py @@ -8,6 +8,7 @@ from .config_updater_job import ConfigUpdaterJob from .create_folders_for_illustrators_job import CreateFoldersForIllustratorsJob +from .db_fetch_all_team_members_job import DBFetchAllTeamMembersJob from .db_fetch_authors_sheet_job import DBFetchAuthorsSheetJob from .db_fetch_curators_sheet_job import DBFetchCuratorsSheetJob from .db_fetch_strings_sheet_job import DBFetchStringsSheetJob diff --git a/src/jobs/db_fetch_all_team_members_job.py b/src/jobs/db_fetch_all_team_members_job.py new file mode 100644 index 0000000..4ca968e --- /dev/null +++ b/src/jobs/db_fetch_all_team_members_job.py @@ -0,0 +1,34 @@ +import logging +from typing import Callable + +from ..app_context import AppContext +from ..strings import load +from .base_job import BaseJob + +logger = logging.getLogger(__name__) + + +class DBFetchAllTeamMembersJob(BaseJob): + @staticmethod + def _execute( + app_context: AppContext, send: Callable[[str], None], called_from_handler=False + ): + num_authors = app_context.db_client.fetch_authors_sheet( + app_context.sheets_client + ) + logger.info(f"Fetched {num_authors} authors") + send(load("db_fetch_authors_sheet_job__success", num_authors=num_authors)) + + num_curators = app_context.db_client.fetch_curators_sheet( + app_context.sheets_client + ) + logger.info(f"Fetched {num_curators} curators") + send(load("db_fetch_curators_sheet_job__success", num_curators=num_curators)) + + team_size = app_context.db_client.fetch_team_sheet( + app_context.sheets_client + ) + # after we fetch the team, we need to recalculate the roles + app_context.role_manager.calculate_db_roles() + logger.info(f"Fetched {team_size} team members") + send(load("db_fetch_team_sheet_job__success", team_size=team_size)) From 3c4ae4743f4807f44145f59e1297f0c660ead02f Mon Sep 17 00:00:00 2001 From: River <79055927+riveriswild@users.noreply.github.com> Date: Wed, 10 Jan 2024 19:06:52 +0300 Subject: [PATCH 15/18] fix:simplify get_tasks_report command & move extra features (#305) fix:simplify get_tasks_report command & move extra features to new command --- src/bot.py | 6 +++ src/tg/handlers/__init__.py | 2 +- src/tg/handlers/get_tasks_report_handler.py | 21 +++++++++-- src/tg/handlers/user_message_handler.py | 42 ++++++++++++++------- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/bot.py b/src/bot.py index 1c98fa7..6cf724f 100644 --- a/src/bot.py +++ b/src/bot.py @@ -178,6 +178,12 @@ def init_handlers(self): direct_message_only(handlers.get_tasks_report), "получить список задач из Trello", ) + self.add_manager_handler( + "get_tasks_report_advanced", + CommandCategories.SUMMARY, + direct_message_only(handlers.get_tasks_report_advanced), + "получить список задач из Trello (расширенный)", + ) self.add_manager_handler( "get_articles_arts", CommandCategories.SUMMARY, diff --git a/src/tg/handlers/__init__.py b/src/tg/handlers/__init__.py index a2aaa78..a5f0a73 100644 --- a/src/tg/handlers/__init__.py +++ b/src/tg/handlers/__init__.py @@ -25,7 +25,7 @@ # Admin (developer) handlers from .get_roles_for_member_handler import get_roles_for_member -from .get_tasks_report_handler import get_tasks_report +from .get_tasks_report_handler import get_tasks_report, get_tasks_report_advanced from .help_handler import help from .list_chats_handler import list_chats from .list_job_handler import list_jobs diff --git a/src/tg/handlers/get_tasks_report_handler.py b/src/tg/handlers/get_tasks_report_handler.py index ddc2b67..2af8292 100644 --- a/src/tg/handlers/get_tasks_report_handler.py +++ b/src/tg/handlers/get_tasks_report_handler.py @@ -19,7 +19,23 @@ @manager_only def get_tasks_report(update: telegram.Update, tg_context: telegram.ext.CallbackContext): - # set initial dialogue data + _get_task_report_base(update, tg_context, advanced=False) + + return + + +@manager_only +def get_tasks_report_advanced( + update: telegram.Update, tg_context: telegram.ext.CallbackContext +): + _get_task_report_base(update, tg_context, advanced=True) + + return + + +def _get_task_report_base( + update: telegram.Update, tg_context: telegram.ext.CallbackContext, advanced: bool +): app_context = AppContext() boards_list = app_context.trello_client.get_boards_for_user() @@ -34,6 +50,7 @@ def get_tasks_report(update: telegram.Update, tg_context: telegram.ext.CallbackC tg_context.chat_data[TASK_NAME] = { consts.NEXT_ACTION: consts.PlainTextUserAction.GET_TASKS_REPORT__ENTER_BOARD_NUMBER.value } + tg_context.chat_data["advanced"] = advanced reply( load( "get_tasks_report_handler__choose_trello_board", lists=boards_list_formatted @@ -41,8 +58,6 @@ def get_tasks_report(update: telegram.Update, tg_context: telegram.ext.CallbackC update, ) - return - def generate_report_messages( board_id: str, list_id: str, introduction: str, add_labels: bool diff --git a/src/tg/handlers/user_message_handler.py b/src/tg/handlers/user_message_handler.py index 8364bdf..40526a3 100644 --- a/src/tg/handlers/user_message_handler.py +++ b/src/tg/handlers/user_message_handler.py @@ -90,6 +90,7 @@ def handle_user_message( assert 0 <= list_idx < len(board_list) board_id = board_list[list_idx]["id"] trello_lists = trello_client.get_lists(board_id) + trello_lists = trello_lists[::-1] except Exception as e: logger.warning(e) reply( @@ -107,7 +108,10 @@ def handle_user_message( ] trello_lists_formatted = "\n".join( - [f"{i + 1}) {lst.name}" for i, lst in enumerate(trello_lists)] + [ + f"{len(trello_lists) - i}) {lst.name}" + for i, lst in enumerate(trello_lists) + ] ) reply( load( @@ -123,8 +127,8 @@ def handle_user_message( elif next_action == PlainTextUserAction.GET_TASKS_REPORT__ENTER_LIST_NUMBER: try: trello_lists = command_data.get(consts.GetTasksReportData.LISTS, []) - list_idx = int(user_input) - 1 - assert 0 <= list_idx < len(trello_lists) + list_idx = -int(user_input) + assert 0 > list_idx >= -len(trello_lists) list_id = trello_lists[list_idx]["id"] except Exception as e: logger.warning(e) @@ -148,6 +152,12 @@ def handle_user_message( ] ] ) + if not tg_context.chat_data.get("advanced"): + add_labels = button == ButtonValues.GET_TASKS_REPORT__LABELS__NO + command_data[consts.GetTasksReportData.INTRO_TEXT] = None + handle_task_report(command_data, add_labels, update) + return + reply( load("get_tasks_report_handler__enter_intro"), update, @@ -188,17 +198,7 @@ def handle_user_message( reply(load("user_message_handler__press_button_please"), update) return add_labels = button == ButtonValues.GET_TASKS_REPORT__LABELS__YES - board_id = command_data[consts.GetTasksReportData.BOARD_ID] - list_id = command_data[consts.GetTasksReportData.LIST_ID] - introduction = command_data[consts.GetTasksReportData.INTRO_TEXT] - messages = get_tasks_report_handler.generate_report_messages( - board_id, list_id, introduction, add_labels - ) - for message in messages: - reply(message, update) - # finished with last action for /trello_client_get_lists - set_next_action(command_data, None) - return + handle_task_report(command_data, add_labels, update) elif next_action == PlainTextUserAction.MANAGE_REMINDERS__CHOOSE_ACTION: if button is None: reply(load("user_message_handler__press_button_please"), update) @@ -546,3 +546,17 @@ def handle_new_members( # writes chat_id and chat name to db when anybody (including the bot) is added to a new chat # very heuristic solution DBClient().set_chat_name(get_chat_id(update), get_chat_name(update)) + + +def handle_task_report(command_data, add_labels, update): + board_id = command_data[consts.GetTasksReportData.BOARD_ID] + list_id = command_data[consts.GetTasksReportData.LIST_ID] + introduction = command_data[consts.GetTasksReportData.INTRO_TEXT] + messages = get_tasks_report_handler.generate_report_messages( + board_id, list_id, introduction, add_labels + ) + for message in messages: + reply(message, update) + # finished with last action for /trello_client_get_lists + set_next_action(command_data, None) + return From eb8b9030863228816c65fc29b9cb31063f2f63c6 Mon Sep 17 00:00:00 2001 From: River Date: Wed, 10 Jan 2024 19:20:49 +0300 Subject: [PATCH 16/18] style: remove trailing whitespaces in user_message_handler --- src/tg/handlers/user_message_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tg/handlers/user_message_handler.py b/src/tg/handlers/user_message_handler.py index 40526a3..72a0380 100644 --- a/src/tg/handlers/user_message_handler.py +++ b/src/tg/handlers/user_message_handler.py @@ -157,7 +157,7 @@ def handle_user_message( command_data[consts.GetTasksReportData.INTRO_TEXT] = None handle_task_report(command_data, add_labels, update) return - + reply( load("get_tasks_report_handler__enter_intro"), update, @@ -546,7 +546,7 @@ def handle_new_members( # writes chat_id and chat name to db when anybody (including the bot) is added to a new chat # very heuristic solution DBClient().set_chat_name(get_chat_id(update), get_chat_name(update)) - + def handle_task_report(command_data, add_labels, update): board_id = command_data[consts.GetTasksReportData.BOARD_ID] From cd200d7132bad71ed2cf2a4e2e843fdd4177614a Mon Sep 17 00:00:00 2001 From: ovrsun <115068324+ovrsun@users.noreply.github.com> Date: Sun, 21 Jan 2024 21:03:42 +0100 Subject: [PATCH 17/18] fix: disable integration tests (#306) --- .github/workflows/publish_dev.yml | 38 ++++++++++++++-------------- .github/workflows/publish_master.yml | 38 ++++++++++++++-------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index 0eec3a0..76835dd 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -156,22 +156,22 @@ jobs: }' \ https://api.telegram.org/bot${{env.TELEGRAM_TEST_TOKEN}}/sendMessage - integration_tests: - needs: push - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - ref: dev - - - name: Integration tests - uses: fylein/python-pytest-github-action@v2 - env: - CONFIG_OVERRIDE: ${{ secrets.CONFIG_OVERRIDE_TESTING }} - with: - args: | - apt-get update && apt-get install -y git && \ - pip3 install -e . && pip3 install -r requirements.txt && \ - pytest tests/integration -vs + # integration_tests: + # needs: push + + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v2 + # with: + # ref: dev + + # - name: Integration tests + # uses: fylein/python-pytest-github-action@v2 + # env: + # CONFIG_OVERRIDE: ${{ secrets.CONFIG_OVERRIDE_TESTING }} + # with: + # args: | + # apt-get update && apt-get install -y git && \ + # pip3 install -e . && pip3 install -r requirements.txt && \ + # pytest tests/integration -vs diff --git a/.github/workflows/publish_master.yml b/.github/workflows/publish_master.yml index fd6281c..6e5df23 100644 --- a/.github/workflows/publish_master.yml +++ b/.github/workflows/publish_master.yml @@ -158,22 +158,22 @@ jobs: -d '{"parse_mode": "markdown", "chat_id": ${{ env.TELEGRAM_ERROR_CHAT_ID }}, "text": "[github CI] deploy [failed](https://github.com/sysblok/sysblokbot/actions/runs/${{github.run_id}})"}' \ https://api.telegram.org/bot${{env.TELEGRAM_TOKEN}}/sendMessage - integration_tests: - needs: push - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - ref: master - - - name: Integration tests - uses: fylein/python-pytest-github-action@v2 - env: - CONFIG_OVERRIDE: ${{ secrets.CONFIG_OVERRIDE_PROD }} - with: - args: | - apt-get update && apt-get install -y git && \ - pip3 install -e . && pip3 install -r requirements.txt && \ - pytest tests/integration -vs + # integration_tests: + # needs: push + + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v2 + # with: + # ref: master + + # - name: Integration tests + # uses: fylein/python-pytest-github-action@v2 + # env: + # CONFIG_OVERRIDE: ${{ secrets.CONFIG_OVERRIDE_PROD }} + # with: + # args: | + # apt-get update && apt-get install -y git && \ + # pip3 install -e . && pip3 install -r requirements.txt && \ + # pytest tests/integration -vs From bd846688e7a1f891c5125e4643f0aaf30d184678 Mon Sep 17 00:00:00 2001 From: Alex Kulikov Date: Wed, 24 Jan 2024 00:43:27 +0000 Subject: [PATCH 18/18] feat: add MVP for calling Facebook API via requests --- src/facebook/facebook_client.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/facebook/facebook_client.py b/src/facebook/facebook_client.py index 82c1abf..eaa2ecc 100644 --- a/src/facebook/facebook_client.py +++ b/src/facebook/facebook_client.py @@ -6,12 +6,15 @@ import dateutil.parser as dateparser import facebook +import requests from ..consts import ReportPeriod from ..utils.singleton import Singleton from .facebook_objects import FacebookPage logger = logging.getLogger(__name__) +BASE_URL = 'https://graph.facebook.com' +API_VERSION = 'v10.0' class FacebookClient(Singleton): @@ -32,13 +35,20 @@ def _update_from_config(self): self._api_client = facebook.GraphAPI(self._facebook_config["token"], 7.0) self._page_id = self._facebook_config["page_id"] + def _make_graph_api_call(self, uri: str, params: dict) -> dict: + params['access_token'] = self._facebook_config["token"] + response = requests.get( + '/'.join([BASE_URL, API_VERSION, uri]) + '?' + + '&'.join(f"{key}={value}" for key, value in params.items())) + return response.json() + def get_page(self) -> FacebookPage: """ Get facebook page """ - page_dict = self._api_client.get_object( - self._page_id, fields="link,name,followers_count,fan_count" - ) + page_dict = self._make_graph_api_call(str(self._page_id), { + 'fields': 'link,name,followers_count,fan_count' + }) return FacebookPage.from_dict(page_dict) def get_new_posts_count(self, since: datetime, until: datetime) -> int: