From 15313ab5f291c8a5528714d9d616c9b8c9009aa6 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 23 Aug 2021 14:38:25 -0400 Subject: [PATCH 001/100] plugins: restructure plugin utils to be in an addons module --- modmail/bot.py | 2 +- modmail/extensions/plugin_manager.py | 2 +- modmail/utils/addons/__init__.py | 0 modmail/utils/{ => addons}/plugins.py | 0 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 modmail/utils/addons/__init__.py rename modmail/utils/{ => addons}/plugins.py (100%) diff --git a/modmail/bot.py b/modmail/bot.py index 1ba67b52..a03ca213 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -12,8 +12,8 @@ from modmail.config import CONFIG from modmail.log import ModmailLogger +from modmail.utils.addons.plugins import PLUGINS, walk_plugins from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions -from modmail.utils.plugins import PLUGINS, walk_plugins REQUIRED_INTENTS = Intents( guilds=True, diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 4661a7ae..4ba94909 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -3,8 +3,8 @@ from modmail.bot import ModmailBot from modmail.extensions.extension_manager import ExtensionConverter, ExtensionManager +from modmail.utils.addons.plugins import PLUGINS, walk_plugins from modmail.utils.cogs import BotModes, ExtMetadata -from modmail.utils.plugins import PLUGINS, walk_plugins EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION) diff --git a/modmail/utils/addons/__init__.py b/modmail/utils/addons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modmail/utils/plugins.py b/modmail/utils/addons/plugins.py similarity index 100% rename from modmail/utils/plugins.py rename to modmail/utils/addons/plugins.py From 0c67d0d6fed5269278e64c94f8dbf47c68466cbe Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 23 Aug 2021 17:06:00 -0400 Subject: [PATCH 002/100] feat: add rudimentary download from zip URLS --- modmail/extensions/plugin_manager.py | 36 ++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 4ba94909..ddcc1755 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -1,13 +1,23 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + from discord.ext import commands from discord.ext.commands import Context -from modmail.bot import ModmailBot from modmail.extensions.extension_manager import ExtensionConverter, ExtensionManager -from modmail.utils.addons.plugins import PLUGINS, walk_plugins +from modmail.utils.addons.plugins import BASE_PATH, PLUGINS, walk_plugins from modmail.utils.cogs import BotModes, ExtMetadata +if TYPE_CHECKING: + from modmail.bot import ModmailBot + from modmail.log import ModmailLogger + EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION) +logger: ModmailLogger = logging.getLogger(__name__) + class PluginConverter(ExtensionConverter): """ @@ -89,6 +99,28 @@ async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) + @plugins_group.command(name="install", aliases=("",)) + async def install_plugins(self, ctx: Context, name: str, url: str) -> None: + """Install plugins from provided repo.""" + # TODO: ensure path is a valid link and whatnot + # TODO: also to support providing normal github and gitlab links and convert to zip + + logger.debug(f"Received command to download plugin {name} from {url}") + async with self.bot.http_session.get(url) as resp: + if resp.status != 200: + await ctx.send(f"Downloading {url} did not give a 200") + zip = await resp.read() + + # TODO: make this use a regex to get the name of the plugin, or make it provided in the inital arg + zip_path = BASE_PATH / ".cache" / "onerandomusername" / f"{name}.zip" + + if not zip_path.exists(): + zip_path.parent.mkdir(parents=True, exist_ok=True) + + with zip_path.open("wb") as f: + f.write(zip) + await ctx.send(f"Downloaded {zip_path}") + # TODO: Implement install/enable/disable/etc # This cannot be static (must have a __func__ attribute). From f59ce83735e07e9a5fcaea75c095234c2fe72b84 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 24 Aug 2021 02:58:27 -0400 Subject: [PATCH 003/100] chore: have coverage.py ignore type_checking ifs --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f37252c6..4c017b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ branch = true source_pkgs = ["modmail"] omit = ["modmail/plugins/**.*"] +[tool.coverage.report] +exclude_lines = ["if TYPE_CHECKING", "pragma: no cover"] + [tool.pytest.ini_options] addopts = "--cov -ra" minversion = "6.0" From 98abe99c2197e20b303f5110c677408ccf198989 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 24 Aug 2021 02:59:40 -0400 Subject: [PATCH 004/100] addons: create addon source converter --- modmail/utils/addons/sources.py | 66 ++++++++++++++++++++++ tests/docs.md | 12 ++++ tests/modmail/utils/addons/__init__.py | 0 tests/modmail/utils/addons/test_sources.py | 60 ++++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 modmail/utils/addons/sources.py create mode 100644 tests/modmail/utils/addons/__init__.py create mode 100644 tests/modmail/utils/addons/test_sources.py diff --git a/modmail/utils/addons/sources.py b/modmail/utils/addons/sources.py new file mode 100644 index 00000000..a8f788bb --- /dev/null +++ b/modmail/utils/addons/sources.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import re +from enum import Enum +from re import Pattern +from typing import TYPE_CHECKING + +from discord.ext import commands + +if TYPE_CHECKING: + from discord.ext.commands import Context + +ZIP_REGEX: Pattern = re.compile(r"^(?P^(?:https?:\/\/)?.*\..+\/.*\.zip) ?(?P[^@\s]+)?$") +REPO_REGEX: Pattern = re.compile( + r"^(?:(?:https?:\/\/)?(?Pgithub|gitlab)(?:\.com\/| )?)?" + # github allows usernames from 1 to 39 characters, and projects of 1 to 100 characters + # gitlab allows 255 characters in the username, and 255 max in a project path + # see https://gitlab.com/gitlab-org/gitlab/-/issues/197976 for more details + r"(?P[a-zA-Z0-9][a-zA-Z0-9\-]{0,254})\/(?P[\w\-\.]{1,100}) " + r"(?P[^@\s]+) ?(?P\@[\w\.\s]*)?$" +) + + +class AddonSourceEnum(Enum): + """Source Types.""" + + GITHUB = "github" + GITLAB = "gitlab" + LOCAL = ".local" + ZIP = ".zip" + + +class AddonSource: + """ + Represents an AddonSource. + + These could be from github, gitlab, a hosted zip file, or local. + """ + + def __init__(self, type: AddonSourceEnum, match: re.Match = None, url: str = None) -> AddonSource: + """Initialize the AddonSource.""" + self.type = type + self.url = url + if match is not None and (type == AddonSourceEnum.GITHUB or type == AddonSourceEnum.GITLAB): + # this is a repository, so we have extra metadata + self.user = match.group("user") + self.repo = match.group("repo") + self.githost = match.group("githost") or "github" + self.ref = match.group("reflike") + self.base_link = f"https://{self.githost}.com/{self.user}/{self.repo}" + if url is None: + self.url = ... + + +class AddonSourceConverter(commands.Converter): + """A converter that takes an addon source, and gets a Source object from it.""" + + async def convert(self, ctx: Context, argument: str) -> AddonSource: + """Convert a string in to an AddonSource.""" + match = ZIP_REGEX.fullmatch(argument) + if match is not None: + # we've matched, so its a zip + ... + match = REPO_REGEX.fullmatch(argument) + if match is None: + raise commands.BadArgument(f"{argument} is not a valid source.") diff --git a/tests/docs.md b/tests/docs.md index 1719ff47..dc445fca 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -67,3 +67,15 @@ Test creating an embed with extra parameters errors properly. **Markers:** - dependency (depends_on=patch_embed) +# tests.modmail.utils.addons.test_sources +## +### test_converter +Convert a user input into a Source. + +**Markers:** +- xfail (reason=Not implemented) +- skip +### test_zip_regex +Test the zip regex correctly gets zip and not the other. +### test_repo_regex +Test the repo regex to ensure that it matches what it should and none of what it shouldn't. diff --git a/tests/modmail/utils/addons/__init__.py b/tests/modmail/utils/addons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/modmail/utils/addons/test_sources.py b/tests/modmail/utils/addons/test_sources.py new file mode 100644 index 00000000..11a761cf --- /dev/null +++ b/tests/modmail/utils/addons/test_sources.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from re import Match +from textwrap import dedent + +import pytest + +import modmail.utils.addons.sources +from modmail.utils.addons.sources import REPO_REGEX, ZIP_REGEX, AddonSourceConverter + +ZIP_TEST_CASES_PASS = [ + "https://github.com/onerandomusername/modmail-addons/archive/main.zip", + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", + "https://example.com/bleeeep.zip", + "http://github.com/discord-modmail/addons/archive/bast.zip", + "rtfd.io/plugs.zip", + "pages.dev/hiy.zip", + "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", + "https://example.com/bleeeep.zip myanmar", + "http://github.com/discord-modmail/addons/archive/bast.zip thebot", + "rtfd.io/plugs.zip documentation", + "pages.dev/hiy.zip", +] +REPO_TEST_CASES_PASS = [ + "onerandomusername/repo planet", + "github onerandomusername/repo planet @master", + "gitlab onerandomusername/repo planet @v1.0.2", + "github onerandomusername/repo planet @master", + "gitlab onerandomusername/repo planet @main", + "https://github.com/onerandomusername/repo planet", + "https://gitlab.com/onerandomusername/repo planet", +] + + +@pytest.mark.skip +@pytest.mark.xfail(reason="Not implemented") +def test_converter() -> None: + """Convert a user input into a Source.""" + source = AddonSourceConverter().convert(None, "github") # noqa: F841 + + +def test_zip_regex() -> None: + """Test the zip regex correctly gets zip and not the other.""" + for case in ZIP_TEST_CASES_PASS: + print(case) + assert isinstance(ZIP_REGEX.fullmatch(case), Match) + for case in REPO_TEST_CASES_PASS: + print(case) + assert ZIP_REGEX.fullmatch(case) is None + + +def test_repo_regex() -> None: + """Test the repo regex to ensure that it matches what it should and none of what it shouldn't.""" + for case in REPO_TEST_CASES_PASS: + print(case) + assert isinstance(REPO_REGEX.fullmatch(case), Match) + for case in ZIP_TEST_CASES_PASS: + print(case) + assert REPO_REGEX.fullmatch(case) is None From 90643a0207b3f9b7217fc6efe9da74175595de93 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 24 Aug 2021 14:09:12 -0400 Subject: [PATCH 005/100] add addon models file --- modmail/extensions/plugin_manager.py | 14 +++--- modmail/utils/addons/models.py | 34 +++++++++++++ modmail/utils/addons/sources.py | 55 ++++++++++++---------- tests/modmail/utils/addons/test_sources.py | 4 +- 4 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 modmail/utils/addons/models.py diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index ddcc1755..b0e60143 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -7,7 +7,9 @@ from discord.ext.commands import Context from modmail.extensions.extension_manager import ExtensionConverter, ExtensionManager +from modmail.utils.addons.models import Addon from modmail.utils.addons.plugins import BASE_PATH, PLUGINS, walk_plugins +from modmail.utils.addons.sources import AddonWithSourceConverter from modmail.utils.cogs import BotModes, ExtMetadata if TYPE_CHECKING: @@ -100,19 +102,19 @@ async def resync_plugins(self, ctx: Context) -> None: await self.resync_extensions.callback(self, ctx) @plugins_group.command(name="install", aliases=("",)) - async def install_plugins(self, ctx: Context, name: str, url: str) -> None: + async def install_plugins(self, ctx: Context, *, plugin: AddonWithSourceConverter) -> None: """Install plugins from provided repo.""" # TODO: ensure path is a valid link and whatnot # TODO: also to support providing normal github and gitlab links and convert to zip - - logger.debug(f"Received command to download plugin {name} from {url}") - async with self.bot.http_session.get(url) as resp: + plugin: Addon = plugin + logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.url}") + async with self.bot.http_session.get(plugin.source.url) as resp: if resp.status != 200: - await ctx.send(f"Downloading {url} did not give a 200") + await ctx.send(f"Downloading {plugin.source.url} did not give a 200") zip = await resp.read() # TODO: make this use a regex to get the name of the plugin, or make it provided in the inital arg - zip_path = BASE_PATH / ".cache" / "onerandomusername" / f"{name}.zip" + zip_path = BASE_PATH / ".cache" / f"{plugin.source.name}.zip" if not zip_path.exists(): zip_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py new file mode 100644 index 00000000..fa067ee0 --- /dev/null +++ b/modmail/utils/addons/models.py @@ -0,0 +1,34 @@ +from enum import Enum, auto +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from modmail.utils.addons.sources import AddonSource + + +class AddonType(Enum): + """Supported addon types.""" + + PLUGIN = auto() + + +class Addon: + """Base class of an addon which make the bot extendable.""" + + name: str + description: Optional[str] + source: AddonSource + min_version: str + + def __init__( + self, + name: str, + source: AddonSource, + type: AddonType, + description: str = None, + min_version: str = None, + ) -> None: + self.name = name + self.source = source + self.description = description + self.min_version = min_version + self.type: AddonType = type diff --git a/modmail/utils/addons/sources.py b/modmail/utils/addons/sources.py index a8f788bb..0ec25779 100644 --- a/modmail/utils/addons/sources.py +++ b/modmail/utils/addons/sources.py @@ -1,17 +1,15 @@ from __future__ import annotations import re -from enum import Enum -from re import Pattern -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from discord.ext import commands if TYPE_CHECKING: from discord.ext.commands import Context -ZIP_REGEX: Pattern = re.compile(r"^(?P^(?:https?:\/\/)?.*\..+\/.*\.zip) ?(?P[^@\s]+)?$") -REPO_REGEX: Pattern = re.compile( +ZIP_REGEX: re.Pattern = re.compile(r"^(?P^(?:https?:\/\/)?.*\..+\/.*\.zip) ?(?P[^@\s]+)?$") +REPO_REGEX: re.Pattern = re.compile( r"^(?:(?:https?:\/\/)?(?Pgithub|gitlab)(?:\.com\/| )?)?" # github allows usernames from 1 to 39 characters, and projects of 1 to 100 characters # gitlab allows 255 characters in the username, and 255 max in a project path @@ -21,39 +19,47 @@ ) -class AddonSourceEnum(Enum): - """Source Types.""" +class GitHost: + """Base class for git hosts.""" - GITHUB = "github" - GITLAB = "gitlab" - LOCAL = ".local" - ZIP = ".zip" + pass + + +class Github(GitHost): + """Github's api.""" + + headers = {"Accept": "application/vnd.github.v3+json"} + base_api_url = "https://api.github.com" + repo_api_url = f"{base_api_url}/repos/{{user}}/{{repo}}" + zip_archive_api_url = f"{repo_api_url}/zipball" + + +class Gitlab(GitHost): + """Gitlab's api.""" + + headers = {} + base_api_url = "https://gitlab.com/api/v4" + repo_api_url = f"{base_api_url}/projects/{{user}}%2F{{repo}}" + zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" class AddonSource: """ Represents an AddonSource. - These could be from github, gitlab, a hosted zip file, or local. + These could be from github, gitlab, or hosted zip file. """ - def __init__(self, type: AddonSourceEnum, match: re.Match = None, url: str = None) -> AddonSource: + def __init__(self, type: Any, url: str) -> AddonSource: """Initialize the AddonSource.""" self.type = type self.url = url - if match is not None and (type == AddonSourceEnum.GITHUB or type == AddonSourceEnum.GITLAB): - # this is a repository, so we have extra metadata - self.user = match.group("user") - self.repo = match.group("repo") - self.githost = match.group("githost") or "github" - self.ref = match.group("reflike") - self.base_link = f"https://{self.githost}.com/{self.user}/{self.repo}" - if url is None: - self.url = ... + if self.type == "github": + ... -class AddonSourceConverter(commands.Converter): - """A converter that takes an addon source, and gets a Source object from it.""" +class AddonConverter(commands.Converter): + """A converter that takes an addon source, and gets a AddonWithSource object from it.""" async def convert(self, ctx: Context, argument: str) -> AddonSource: """Convert a string in to an AddonSource.""" @@ -61,6 +67,7 @@ async def convert(self, ctx: Context, argument: str) -> AddonSource: if match is not None: # we've matched, so its a zip ... + match = REPO_REGEX.fullmatch(argument) if match is None: raise commands.BadArgument(f"{argument} is not a valid source.") diff --git a/tests/modmail/utils/addons/test_sources.py b/tests/modmail/utils/addons/test_sources.py index 11a761cf..e60a9c6b 100644 --- a/tests/modmail/utils/addons/test_sources.py +++ b/tests/modmail/utils/addons/test_sources.py @@ -6,7 +6,7 @@ import pytest import modmail.utils.addons.sources -from modmail.utils.addons.sources import REPO_REGEX, ZIP_REGEX, AddonSourceConverter +from modmail.utils.addons.sources import REPO_REGEX, ZIP_REGEX, AddonConverter ZIP_TEST_CASES_PASS = [ "https://github.com/onerandomusername/modmail-addons/archive/main.zip", @@ -37,7 +37,7 @@ @pytest.mark.xfail(reason="Not implemented") def test_converter() -> None: """Convert a user input into a Source.""" - source = AddonSourceConverter().convert(None, "github") # noqa: F841 + addon = AddonConverter().convert(None, "github") # noqa: F841 def test_zip_regex() -> None: From 07976cc63084d32aa567d28f5ed7cb5f1949dc46 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 02:49:36 -0400 Subject: [PATCH 006/100] major: refactor plugin and addon structure plugins can now be created as a type of addon adds regexes to convert from a repo or zip to a plugin with a source tests nearly all of the added code. --- modmail/utils/addons/converters.py | 50 ++++ modmail/utils/addons/models.py | 127 +++++++-- modmail/utils/addons/sources.py | 73 ------ tests/docs.md | 240 +++++++++++++++++- tests/modmail/utils/addons/test_converters.py | 125 +++++++++ tests/modmail/utils/addons/test_models.py | 195 ++++++++++++++ tests/modmail/utils/addons/test_sources.py | 60 ----- 7 files changed, 715 insertions(+), 155 deletions(-) create mode 100644 modmail/utils/addons/converters.py delete mode 100644 modmail/utils/addons/sources.py create mode 100644 tests/modmail/utils/addons/test_converters.py create mode 100644 tests/modmail/utils/addons/test_models.py delete mode 100644 tests/modmail/utils/addons/test_sources.py diff --git a/modmail/utils/addons/converters.py b/modmail/utils/addons/converters.py new file mode 100644 index 00000000..741bc441 --- /dev/null +++ b/modmail/utils/addons/converters.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Type + +from discord.ext import commands + +from modmail.utils.addons.models import Addon, Plugin + +if TYPE_CHECKING: + from discord.ext.commands import Context + +ZIP_REGEX: re.Pattern = re.compile( + r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip)) (?P[^@\s]+)$" +) +REPO_REGEX: re.Pattern = re.compile( + r"^(?:(?:https?:\/\/)?(?Pgithub|gitlab)(?:\.com\/| )?)?" + # github allows usernames from 1 to 39 characters, and projects of 1 to 100 characters + # gitlab allows 255 characters in the username, and 255 max in a project path + # see https://gitlab.com/gitlab-org/gitlab/-/issues/197976 for more details + r"(?P[a-zA-Z0-9][a-zA-Z0-9\-]{0,254})\/(?P[\w\-\.]{1,100}) " + r"(?P[^@\s]+)(?: \@(?P[\w\.\s]*))?$" +) + + +AddonClass = Type[Addon] + + +class AddonConverter(commands.Converter): + """A converter that takes an addon source, and gets a Addon object from it.""" + + async def convert(self, ctx: Context, argument: str, cls: AddonClass) -> Addon: + """Convert a string in to an Addon.""" + match = ZIP_REGEX.fullmatch(argument) + if match is not None: + # we've matched, so its a zip + ... + + match = REPO_REGEX.fullmatch(argument) + if match is None: + raise commands.BadArgument(f"{argument} is not a valid source.") + return ... + + +class PluginWithSourceConverter(AddonConverter): + """A plugin converter that takes a source, addon name, and returns a Plugin.""" + + async def convert(self, ctx: Context, argument: str) -> Plugin: + """Convert a provided plugin and source to a Plugin.""" + super().convert(ctx, argument, cls=Plugin) diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py index fa067ee0..f12bbe3f 100644 --- a/modmail/utils/addons/models.py +++ b/modmail/utils/addons/models.py @@ -1,14 +1,89 @@ -from enum import Enum, auto -from typing import TYPE_CHECKING, Optional +from __future__ import annotations -if TYPE_CHECKING: - from modmail.utils.addons.sources import AddonSource +import re +from enum import Enum +from typing import TYPE_CHECKING, Literal, Optional -class AddonType(Enum): - """Supported addon types.""" +class SourceTypeEnum(Enum): + """Which source an addon is from.""" - PLUGIN = auto() + ZIP = 0 + REPO = 1 + + +class GitHost: + """Base class for git hosts.""" + + headers = {} + base_api_url: str + repo_api_url: str + zip_archive_api_url: str + + +class Github(GitHost): + """Github's api.""" + + headers = {"Accept": "application/vnd.github.v3+json"} + base_api_url = "https://api.github.com" + repo_api_url = f"{base_api_url}/repos/{{user}}/{{repo}}" + zip_archive_api_url = f"{repo_api_url}/zipball" + + +class Gitlab(GitHost): + """Gitlab's api.""" + + base_api_url = "https://gitlab.com/api/v4" + repo_api_url = f"{base_api_url}/projects/{{user}}%2F{{repo}}" + zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" + + +class AddonSource: + """ + Represents an AddonSource. + + These could be from github, gitlab, or hosted zip file. + """ + + if TYPE_CHECKING: + repo: Optional[str] + user: Optional[str] + reflike: Optional[str] + githost: Optional[Literal["github", "gitlab"]] + githost_api = Optional[GitHost] + + def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: + """Initialize the AddonSource.""" + self.zip_url = zip_url + self.source_type = type + + @classmethod + def from_repo( + cls, user: str, repo: str, reflike: str = None, githost: Literal["github", "gitlab"] = "github" + ) -> AddonSource: + """Create an AddonSource from a repo.""" + if githost == "github": + Host = Github # noqa: N806 + elif githost == "gitlab": + Host = Gitlab # noqa: N806 + else: + raise TypeError(f"{githost} is not a valid host.") + zip_url = Host.zip_archive_api_url.format(user=user, repo=repo) + + source = cls(zip_url, SourceTypeEnum.REPO) + source.repo = repo + source.user = user + source.reflike = reflike + source.githost = githost + source.githost_api = Host + return source + + @classmethod + def from_zip(cls, url: str) -> AddonSource: + """Create an AddonSource from a zip file.""" + match = re.match(r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip))", url) + source = cls(match.group("url"), SourceTypeEnum.ZIP) + return source class Addon: @@ -19,16 +94,32 @@ class Addon: source: AddonSource min_version: str - def __init__( - self, - name: str, - source: AddonSource, - type: AddonType, - description: str = None, - min_version: str = None, - ) -> None: + def __init__(self): + raise NotImplementedError("Inheriting classes need to implement their own init") + + +class Plugin(Addon): + """An addon which is a plugin.""" + + def __init__(self, name: str, source: AddonSource, **kw) -> Plugin: self.name = name self.source = source - self.description = description - self.min_version = min_version - self.type: AddonType = type + self.description = kw.get("description", None) + self.min_version = kw.get("min_version", None) + self.enabled = kw.get("enabled", True) + + @classmethod + def from_repo_match(cls, match: re.Match) -> Plugin: + """Create a Plugin from a repository regex match.""" + name = match.group("addon") + source = AddonSource.from_repo( + match.group("user"), match.group("repo"), match.group("reflike"), match.group("githost") + ) + return cls(name, source) + + @classmethod + def from_zip_match(cls, match: re.Match) -> Plugin: + """Create a Plugin from a zip regex match.""" + name = match.group("addon") + source = AddonSource.from_zip(match.group("url")) + return cls(name, source) diff --git a/modmail/utils/addons/sources.py b/modmail/utils/addons/sources.py deleted file mode 100644 index 0ec25779..00000000 --- a/modmail/utils/addons/sources.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import re -from typing import TYPE_CHECKING, Any - -from discord.ext import commands - -if TYPE_CHECKING: - from discord.ext.commands import Context - -ZIP_REGEX: re.Pattern = re.compile(r"^(?P^(?:https?:\/\/)?.*\..+\/.*\.zip) ?(?P[^@\s]+)?$") -REPO_REGEX: re.Pattern = re.compile( - r"^(?:(?:https?:\/\/)?(?Pgithub|gitlab)(?:\.com\/| )?)?" - # github allows usernames from 1 to 39 characters, and projects of 1 to 100 characters - # gitlab allows 255 characters in the username, and 255 max in a project path - # see https://gitlab.com/gitlab-org/gitlab/-/issues/197976 for more details - r"(?P[a-zA-Z0-9][a-zA-Z0-9\-]{0,254})\/(?P[\w\-\.]{1,100}) " - r"(?P[^@\s]+) ?(?P\@[\w\.\s]*)?$" -) - - -class GitHost: - """Base class for git hosts.""" - - pass - - -class Github(GitHost): - """Github's api.""" - - headers = {"Accept": "application/vnd.github.v3+json"} - base_api_url = "https://api.github.com" - repo_api_url = f"{base_api_url}/repos/{{user}}/{{repo}}" - zip_archive_api_url = f"{repo_api_url}/zipball" - - -class Gitlab(GitHost): - """Gitlab's api.""" - - headers = {} - base_api_url = "https://gitlab.com/api/v4" - repo_api_url = f"{base_api_url}/projects/{{user}}%2F{{repo}}" - zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" - - -class AddonSource: - """ - Represents an AddonSource. - - These could be from github, gitlab, or hosted zip file. - """ - - def __init__(self, type: Any, url: str) -> AddonSource: - """Initialize the AddonSource.""" - self.type = type - self.url = url - if self.type == "github": - ... - - -class AddonConverter(commands.Converter): - """A converter that takes an addon source, and gets a AddonWithSource object from it.""" - - async def convert(self, ctx: Context, argument: str) -> AddonSource: - """Convert a string in to an AddonSource.""" - match = ZIP_REGEX.fullmatch(argument) - if match is not None: - # we've matched, so its a zip - ... - - match = REPO_REGEX.fullmatch(argument) - if match is None: - raise commands.BadArgument(f"{argument} is not a valid source.") diff --git a/tests/docs.md b/tests/docs.md index dc445fca..1f1bcc7b 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -67,7 +67,7 @@ Test creating an embed with extra parameters errors properly. **Markers:** - dependency (depends_on=patch_embed) -# tests.modmail.utils.addons.test_sources +# tests.modmail.utils.addons.test_converters ## ### test_converter Convert a user input into a Source. @@ -75,7 +75,239 @@ Convert a user input into a Source. **Markers:** - xfail (reason=Not implemented) - skip -### test_zip_regex -Test the zip regex correctly gets zip and not the other. ### test_repo_regex -Test the repo regex to ensure that it matches what it should and none of what it shouldn't. +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_repo_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_zip_regex +Test the repo regex to ensure that it matches what it should. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +# tests.modmail.utils.addons.test_models +## +### test_addon_model +All addons will be of a specific type, so we should not be able to create a generic addon. +### test_addonsource_init +Test the AddonSource init sets class vars appropiately. + +**Markers:** +- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', )]) +### test_addonsource_init +Test the AddonSource init sets class vars appropiately. + +**Markers:** +- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', )]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_repo +Test an addon source is properly made from repository information. + +**Markers:** +- parametrize (user, repo, reflike, githost[('onerandomusername', 'addons', None, 'github'), ('onerandomusername', 'addons', 'master', 'github'), ('onerandomusername', 'repo', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'master', 'github'), ('onerandomusername', 'repo', 'main', 'gitlab'), ('onerandomusername', 'repo', None, 'github'), ('onerandomusername', 'repo', None, 'gitlab'), ('psf', 'black', '21.70b', 'github')]) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +### test_addonsource_from_zip +Test an addon source is properly made from a zip url. + +**Markers:** +- parametrize (url['github.com/onerandomusername/modmail-addons/archive/main.zip', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'example.com/bleeeep.zip', 'github.com/discord-modmail/addons/archive/bast.zip', 'rtfd.io/plugs.zip', 'pages.dev/hiy.zip']) +## TestPlugin +Test the Plugin class creation. +### test_plugin_init +Create a plugin model, and ensure it has the right properties. + +**Markers:** +- parametrize (name['earth', 'mona-lisa']) +### test_plugin_init +Create a plugin model, and ensure it has the right properties. + +**Markers:** +- parametrize (name['earth', 'mona-lisa']) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_repo_match +Test that a plugin can be created from a repo. + +**Markers:** +- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_from_zip_match +Test that a plugin can be created from a zip url. + +**Markers:** +- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) diff --git a/tests/modmail/utils/addons/test_converters.py b/tests/modmail/utils/addons/test_converters.py new file mode 100644 index 00000000..59d3a685 --- /dev/null +++ b/tests/modmail/utils/addons/test_converters.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from re import Match +from textwrap import dedent + +import pytest + +from modmail.utils.addons.converters import REPO_REGEX, ZIP_REGEX, AddonConverter + + +@pytest.mark.skip +@pytest.mark.xfail(reason="Not implemented") +def test_converter() -> None: + """Convert a user input into a Source.""" + addon = AddonConverter().convert(None, "github") # noqa: F841 + + +# fmt: off +@pytest.mark.parametrize( + "entry, user, repo, addon, reflike, githost", + [ + ( + "onerandomusername/addons planet", + "onerandomusername", "addons", "planet", None, None, + ), + ( + "github onerandomusername/addons planet @master", + "onerandomusername", "addons", "planet", "master", "github", + ), + ( + "gitlab onerandomusername/repo planet @v1.0.2", + "onerandomusername", "repo", "planet", "v1.0.2", "gitlab", + ), + ( + "github onerandomusername/repo planet @master", + "onerandomusername", "repo", "planet", "master", "github", + ), + ( + "gitlab onerandomusername/repo planet @main", + "onerandomusername", "repo", "planet", "main", "gitlab", + ), + ( + "https://github.com/onerandomusername/repo planet", + "onerandomusername", "repo", "planet", None, "github", + ), + ( + "https://gitlab.com/onerandomusername/repo planet", + "onerandomusername", "repo", "planet", None, "gitlab", + ), + ( + "https://github.com/psf/black black @21.70b", + "psf", "black", "black", "21.70b", "github", + ) + ], +) +# fmt: on +def test_repo_regex(entry, user, repo, addon, reflike, githost) -> None: + """Test the repo regex to ensure that it matches what it should.""" + match = REPO_REGEX.fullmatch(entry) + assert match is not None + assert match.group("user") == user + assert match.group("repo") == repo + assert match.group("addon") == addon + assert match.group("reflike") or None == reflike # noqa: E711 + assert match.group("githost") == githost + + +# fmt: off +@pytest.mark.parametrize( + "entry, url, domain, path, addon", + [ + ( + "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", + "github.com/onerandomusername/modmail-addons/archive/main.zip", + "github.com", + "onerandomusername/modmail-addons/archive/main.zip", + "planet", + ), + ( + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", # noqa: E501 + "gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", + "gitlab.com", + "onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", + "earth", + ), + ( + "https://example.com/bleeeep.zip myanmar", + "example.com/bleeeep.zip", + "example.com", + "bleeeep.zip", + "myanmar", + + ), + ( + "http://github.com/discord-modmail/addons/archive/bast.zip thebot", + "github.com/discord-modmail/addons/archive/bast.zip", + "github.com", + "discord-modmail/addons/archive/bast.zip", + "thebot", + ), + ( + "rtfd.io/plugs.zip documentation", + "rtfd.io/plugs.zip", + "rtfd.io", + "plugs.zip", + "documentation", + ), + ( + "pages.dev/hiy.zip black", + "pages.dev/hiy.zip", + "pages.dev", + "hiy.zip", + "black", + ), + ] +) +# fmt: on +def test_zip_regex(entry, url, domain, path, addon) -> None: + """Test the repo regex to ensure that it matches what it should.""" + match = ZIP_REGEX.fullmatch(entry) + assert match is not None + assert match.group("url") == url + assert match.group("domain") == domain + assert match.group("path") == path + assert match.group("addon") == addon diff --git a/tests/modmail/utils/addons/test_models.py b/tests/modmail/utils/addons/test_models.py new file mode 100644 index 00000000..7508498f --- /dev/null +++ b/tests/modmail/utils/addons/test_models.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import pytest + +from modmail.utils.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum + + +def test_addon_model(): + """All addons will be of a specific type, so we should not be able to create a generic addon.""" + with pytest.raises(NotImplementedError, match="Inheriting classes need to implement their own init"): + Addon() + + +@pytest.mark.parametrize( + "zip_url, source_type", + [ + ("github.com/bast0006.zip", SourceTypeEnum.ZIP), + ("gitlab.com/onerandomusername.zip", SourceTypeEnum.REPO), + ], +) +def test_addonsource_init(zip_url, source_type): + """Test the AddonSource init sets class vars appropiately.""" + addonsrc = AddonSource(zip_url, source_type) + assert addonsrc.zip_url == zip_url + assert addonsrc.source_type == source_type + + +@pytest.mark.parametrize( + "user, repo, reflike, githost", + [ + ("onerandomusername", "addons", None, "github"), + ("onerandomusername", "addons", "master", "github"), + ("onerandomusername", "repo", "v1.0.2", "gitlab"), + ("onerandomusername", "repo", "master", "github"), + ("onerandomusername", "repo", "main", "gitlab"), + ("onerandomusername", "repo", None, "github"), + ("onerandomusername", "repo", None, "gitlab"), + ("psf", "black", "21.70b", "github"), + ], +) +def test_addonsource_from_repo(user, repo, reflike, githost): + """Test an addon source is properly made from repository information.""" + src = AddonSource.from_repo(user, repo, reflike, githost) + assert src.user == user + assert src.repo == repo + assert src.reflike == reflike + assert src.githost == githost + assert src.source_type == SourceTypeEnum.REPO + + +@pytest.mark.parametrize( + "url", + [ + ("github.com/onerandomusername/modmail-addons/archive/main.zip"), + ("gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip"), + ("example.com/bleeeep.zip"), + ("github.com/discord-modmail/addons/archive/bast.zip"), + ("rtfd.io/plugs.zip"), + ("pages.dev/hiy.zip"), + ], +) +def test_addonsource_from_zip(url): + """Test an addon source is properly made from a zip url.""" + src = AddonSource.from_zip(url) + assert src.zip_url == url + assert src.source_type == SourceTypeEnum.ZIP + + +@pytest.fixture(name="source_fixture") +def addonsource_fixture(): + """Addonsource fixture for tests. The contents of this source do not matter, as long as they are valid.""" + return AddonSource("github.com/bast0006.zip", SourceTypeEnum.ZIP) + + +class TestPlugin: + """Test the Plugin class creation.""" + + @pytest.mark.parametrize("name", [("earth"), ("mona-lisa")]) + def test_plugin_init(self, name, source_fixture): + """Create a plugin model, and ensure it has the right properties.""" + plugin = Plugin(name, source_fixture) + assert isinstance(plugin, Plugin) + assert plugin.name == name + + # fmt: off + @pytest.mark.parametrize( + "entry, user, repo, name, reflike, githost", + [ + ( + "github onerandomusername/addons planet", + "onerandomusername", "addons", "planet", None, "github", + ), + ( + "github onerandomusername/addons planet @master", + "onerandomusername", "addons", "planet", "master", "github", + ), + ( + "gitlab onerandomusername/repo planet @v1.0.2", + "onerandomusername", "repo", "planet", "v1.0.2", "gitlab", + ), + ( + "github onerandomusername/repo planet @master", + "onerandomusername", "repo", "planet", "master", "github", + ), + ( + "gitlab onerandomusername/repo planet @main", + "onerandomusername", "repo", "planet", "main", "gitlab", + ), + ( + "https://github.com/onerandomusername/repo planet", + "onerandomusername", "repo", "planet", None, "github", + ), + ( + "https://gitlab.com/onerandomusername/repo planet", + "onerandomusername", "repo", "planet", None, "gitlab", + ), + ( + "https://github.com/psf/black black @21.70b", + "psf", "black", "black", "21.70b", "github", + ) + ], + ) + # fmt: on + def test_plugin_from_repo_match(self, entry, user, repo, name, reflike, githost): + """Test that a plugin can be created from a repo.""" + from modmail.utils.addons.converters import REPO_REGEX + + match = REPO_REGEX.match(entry) + plug = Plugin.from_repo_match(match) + assert plug.name == name + assert plug.source.user == user + assert plug.source.repo == repo + assert plug.source.reflike == reflike + assert plug.source.githost == githost + assert plug.source.source_type == SourceTypeEnum.REPO + + # fmt: off + @pytest.mark.parametrize( + "entry, url, domain, path, addon", + [ + ( + "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", + "github.com/onerandomusername/modmail-addons/archive/main.zip", + "github.com", + "onerandomusername/modmail-addons/archive/main.zip", + "planet", + ), + ( + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", # noqa: E501 + "gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", + "gitlab.com", + "onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", + "earth", + ), + ( + "https://example.com/bleeeep.zip myanmar", + "example.com/bleeeep.zip", + "example.com", + "bleeeep.zip", + "myanmar", + + ), + ( + "http://github.com/discord-modmail/addons/archive/bast.zip thebot", + "github.com/discord-modmail/addons/archive/bast.zip", + "github.com", + "discord-modmail/addons/archive/bast.zip", + "thebot", + ), + ( + "rtfd.io/plugs.zip documentation", + "rtfd.io/plugs.zip", + "rtfd.io", + "plugs.zip", + "documentation", + ), + ( + "pages.dev/hiy.zip black", + "pages.dev/hiy.zip", + "pages.dev", + "hiy.zip", + "black", + ), + ] + ) + # fmt: on + def test_plugin_from_zip_match(self, entry, url, domain, path, addon): + """Test that a plugin can be created from a zip url.""" + from modmail.utils.addons.converters import ZIP_REGEX + + match = ZIP_REGEX.match(entry) + plug = Plugin.from_zip_match(match) + assert plug.name == addon + assert plug.source.zip_url == url + assert plug.source.source_type == SourceTypeEnum.ZIP diff --git a/tests/modmail/utils/addons/test_sources.py b/tests/modmail/utils/addons/test_sources.py deleted file mode 100644 index e60a9c6b..00000000 --- a/tests/modmail/utils/addons/test_sources.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from re import Match -from textwrap import dedent - -import pytest - -import modmail.utils.addons.sources -from modmail.utils.addons.sources import REPO_REGEX, ZIP_REGEX, AddonConverter - -ZIP_TEST_CASES_PASS = [ - "https://github.com/onerandomusername/modmail-addons/archive/main.zip", - "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", - "https://example.com/bleeeep.zip", - "http://github.com/discord-modmail/addons/archive/bast.zip", - "rtfd.io/plugs.zip", - "pages.dev/hiy.zip", - "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", - "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", - "https://example.com/bleeeep.zip myanmar", - "http://github.com/discord-modmail/addons/archive/bast.zip thebot", - "rtfd.io/plugs.zip documentation", - "pages.dev/hiy.zip", -] -REPO_TEST_CASES_PASS = [ - "onerandomusername/repo planet", - "github onerandomusername/repo planet @master", - "gitlab onerandomusername/repo planet @v1.0.2", - "github onerandomusername/repo planet @master", - "gitlab onerandomusername/repo planet @main", - "https://github.com/onerandomusername/repo planet", - "https://gitlab.com/onerandomusername/repo planet", -] - - -@pytest.mark.skip -@pytest.mark.xfail(reason="Not implemented") -def test_converter() -> None: - """Convert a user input into a Source.""" - addon = AddonConverter().convert(None, "github") # noqa: F841 - - -def test_zip_regex() -> None: - """Test the zip regex correctly gets zip and not the other.""" - for case in ZIP_TEST_CASES_PASS: - print(case) - assert isinstance(ZIP_REGEX.fullmatch(case), Match) - for case in REPO_TEST_CASES_PASS: - print(case) - assert ZIP_REGEX.fullmatch(case) is None - - -def test_repo_regex() -> None: - """Test the repo regex to ensure that it matches what it should and none of what it shouldn't.""" - for case in REPO_TEST_CASES_PASS: - print(case) - assert isinstance(REPO_REGEX.fullmatch(case), Match) - for case in ZIP_TEST_CASES_PASS: - print(case) - assert REPO_REGEX.fullmatch(case) is None From b7ab4c3dcdb0e39649c2f3dc66e7da571203e2f8 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 11:52:04 -0400 Subject: [PATCH 007/100] major: don't require regex matches to be passed to create a Plugin --- modmail/utils/addons/models.py | 33 ++++--- tests/docs.md | 40 ++++---- tests/modmail/utils/addons/test_models.py | 113 ++++------------------ 3 files changed, 60 insertions(+), 126 deletions(-) diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py index f12bbe3f..774802d3 100644 --- a/modmail/utils/addons/models.py +++ b/modmail/utils/addons/models.py @@ -38,6 +38,9 @@ class Gitlab(GitHost): zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" +Host = Literal["Github", "Gitlab"] + + class AddonSource: """ Represents an AddonSource. @@ -49,7 +52,7 @@ class AddonSource: repo: Optional[str] user: Optional[str] reflike: Optional[str] - githost: Optional[Literal["github", "gitlab"]] + githost: Optional[Host] githost_api = Optional[GitHost] def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: @@ -58,9 +61,7 @@ def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: self.source_type = type @classmethod - def from_repo( - cls, user: str, repo: str, reflike: str = None, githost: Literal["github", "gitlab"] = "github" - ) -> AddonSource: + def from_repo(cls, user: str, repo: str, reflike: str = None, githost: Host = "github") -> AddonSource: """Create an AddonSource from a repo.""" if githost == "github": Host = Github # noqa: N806 @@ -85,6 +86,9 @@ def from_zip(cls, url: str) -> AddonSource: source = cls(match.group("url"), SourceTypeEnum.ZIP) return source + def __repr__(self) -> str: + return f"" + class Addon: """Base class of an addon which make the bot extendable.""" @@ -109,17 +113,18 @@ def __init__(self, name: str, source: AddonSource, **kw) -> Plugin: self.enabled = kw.get("enabled", True) @classmethod - def from_repo_match(cls, match: re.Match) -> Plugin: + def from_repo( + cls, addon: str, user: str, repo: str, reflike: str = None, githost: Host = "github" + ) -> Plugin: """Create a Plugin from a repository regex match.""" - name = match.group("addon") - source = AddonSource.from_repo( - match.group("user"), match.group("repo"), match.group("reflike"), match.group("githost") - ) - return cls(name, source) + source = AddonSource.from_repo(user, repo, reflike, githost) + return cls(addon, source) @classmethod - def from_zip_match(cls, match: re.Match) -> Plugin: + def from_zip(cls, addon: str, url: str) -> Plugin: """Create a Plugin from a zip regex match.""" - name = match.group("addon") - source = AddonSource.from_zip(match.group("url")) - return cls(name, source) + source = AddonSource.from_zip(url) + return cls(addon, source) + + def __repr__(self) -> str: + return f"" diff --git a/tests/docs.md b/tests/docs.md index 1f1bcc7b..28af2ff0 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -245,69 +245,69 @@ Create a plugin model, and ensure it has the right properties. Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) ### test_plugin_from_repo_match Test that a plugin can be created from a repo. **Markers:** -- parametrize (entry, user, repo, name, reflike, githost[('github onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, 'github'), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_zip_match +- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) -### test_plugin_from_zip_match +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) -### test_plugin_from_zip_match +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) -### test_plugin_from_zip_match +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) -### test_plugin_from_zip_match +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) -### test_plugin_from_zip_match +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +### test_plugin_from_zip Test that a plugin can be created from a zip url. **Markers:** -- parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) diff --git a/tests/modmail/utils/addons/test_models.py b/tests/modmail/utils/addons/test_models.py index 7508498f..74d50372 100644 --- a/tests/modmail/utils/addons/test_models.py +++ b/tests/modmail/utils/addons/test_models.py @@ -82,51 +82,22 @@ def test_plugin_init(self, name, source_fixture): assert isinstance(plugin, Plugin) assert plugin.name == name - # fmt: off @pytest.mark.parametrize( - "entry, user, repo, name, reflike, githost", + "user, repo, name, reflike, githost", [ - ( - "github onerandomusername/addons planet", - "onerandomusername", "addons", "planet", None, "github", - ), - ( - "github onerandomusername/addons planet @master", - "onerandomusername", "addons", "planet", "master", "github", - ), - ( - "gitlab onerandomusername/repo planet @v1.0.2", - "onerandomusername", "repo", "planet", "v1.0.2", "gitlab", - ), - ( - "github onerandomusername/repo planet @master", - "onerandomusername", "repo", "planet", "master", "github", - ), - ( - "gitlab onerandomusername/repo planet @main", - "onerandomusername", "repo", "planet", "main", "gitlab", - ), - ( - "https://github.com/onerandomusername/repo planet", - "onerandomusername", "repo", "planet", None, "github", - ), - ( - "https://gitlab.com/onerandomusername/repo planet", - "onerandomusername", "repo", "planet", None, "gitlab", - ), - ( - "https://github.com/psf/black black @21.70b", - "psf", "black", "black", "21.70b", "github", - ) + ("onerandomusername", "addons", "planet", None, "github"), + ("onerandomusername", "addons", "planet", "master", "github"), + ("onerandomusername", "repo", "planet", "v1.0.2", "gitlab"), + ("onerandomusername", "repo", "planet", "master", "github"), + ("onerandomusername", "repo", "planet", "main", "gitlab"), + ("onerandomusername", "repo", "planet", None, "github"), + ("onerandomusername", "repo", "planet", None, "gitlab"), + ("psf", "black", "black", "21.70b", "github"), ], ) - # fmt: on - def test_plugin_from_repo_match(self, entry, user, repo, name, reflike, githost): + def test_plugin_from_repo_match(self, user, repo, name, reflike, githost): """Test that a plugin can be created from a repo.""" - from modmail.utils.addons.converters import REPO_REGEX - - match = REPO_REGEX.match(entry) - plug = Plugin.from_repo_match(match) + plug = Plugin.from_repo(name, user, repo, reflike, githost) assert plug.name == name assert plug.source.user == user assert plug.source.repo == repo @@ -134,62 +105,20 @@ def test_plugin_from_repo_match(self, entry, user, repo, name, reflike, githost) assert plug.source.githost == githost assert plug.source.source_type == SourceTypeEnum.REPO - # fmt: off @pytest.mark.parametrize( - "entry, url, domain, path, addon", + "url, addon", [ - ( - "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", - "github.com/onerandomusername/modmail-addons/archive/main.zip", - "github.com", - "onerandomusername/modmail-addons/archive/main.zip", - "planet", - ), - ( - "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", # noqa: E501 - "gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", - "gitlab.com", - "onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", - "earth", - ), - ( - "https://example.com/bleeeep.zip myanmar", - "example.com/bleeeep.zip", - "example.com", - "bleeeep.zip", - "myanmar", - - ), - ( - "http://github.com/discord-modmail/addons/archive/bast.zip thebot", - "github.com/discord-modmail/addons/archive/bast.zip", - "github.com", - "discord-modmail/addons/archive/bast.zip", - "thebot", - ), - ( - "rtfd.io/plugs.zip documentation", - "rtfd.io/plugs.zip", - "rtfd.io", - "plugs.zip", - "documentation", - ), - ( - "pages.dev/hiy.zip black", - "pages.dev/hiy.zip", - "pages.dev", - "hiy.zip", - "black", - ), - ] + ("github.com/onerandomusername/modmail-addons/archive/main.zip", "planet"), + ("gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", "earth"), + ("example.com/bleeeep.zip", "myanmar"), + ("github.com/discord-modmail/addons/archive/bast.zip", "thebot"), + ("rtfd.io/plugs.zip", "documentation"), + ("pages.dev/hiy.zip", "black"), + ], ) - # fmt: on - def test_plugin_from_zip_match(self, entry, url, domain, path, addon): + def test_plugin_from_zip(self, url, addon): """Test that a plugin can be created from a zip url.""" - from modmail.utils.addons.converters import ZIP_REGEX - - match = ZIP_REGEX.match(entry) - plug = Plugin.from_zip_match(match) + plug = Plugin.from_zip(addon, url) assert plug.name == addon assert plug.source.zip_url == url assert plug.source.source_type == SourceTypeEnum.ZIP From 5899a1dc1442ea7873837366720d225c0907c39d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 12:36:43 -0400 Subject: [PATCH 008/100] feat: make plugin converter work again --- modmail/extensions/plugin_manager.py | 21 +- modmail/utils/addons/converters.py | 36 ++-- modmail/utils/addons/models.py | 8 +- tests/docs.md | 198 +++++++++++------- tests/modmail/utils/addons/test_converters.py | 39 +++- 5 files changed, 203 insertions(+), 99 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index b0e60143..84786507 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -7,9 +7,9 @@ from discord.ext.commands import Context from modmail.extensions.extension_manager import ExtensionConverter, ExtensionManager -from modmail.utils.addons.models import Addon +from modmail.utils.addons.converters import PluginWithSourceConverter +from modmail.utils.addons.models import Plugin from modmail.utils.addons.plugins import BASE_PATH, PLUGINS, walk_plugins -from modmail.utils.addons.sources import AddonWithSourceConverter from modmail.utils.cogs import BotModes, ExtMetadata if TYPE_CHECKING: @@ -21,7 +21,7 @@ logger: ModmailLogger = logging.getLogger(__name__) -class PluginConverter(ExtensionConverter): +class PluginPathConverter(ExtensionConverter): """ Fully qualify the name of a plugin and ensure it exists. @@ -58,7 +58,7 @@ async def plugins_group(self, ctx: Context) -> None: await ctx.send_help(ctx.command) @plugins_group.command(name="load", aliases=("l",)) - async def load_plugin(self, ctx: Context, *plugins: PluginConverter) -> None: + async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None: """ Load plugins given their fully qualified or unqualified names. @@ -67,7 +67,7 @@ async def load_plugin(self, ctx: Context, *plugins: PluginConverter) -> None: await self.load_extensions.callback(self, ctx, *plugins) @plugins_group.command(name="unload", aliases=("ul",)) - async def unload_plugins(self, ctx: Context, *plugins: PluginConverter) -> None: + async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: """ Unload currently loaded plugins given their fully qualified or unqualified names. @@ -76,7 +76,7 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginConverter) -> None: await self.unload_extensions.callback(self, ctx, *plugins) @plugins_group.command(name="reload", aliases=("r", "rl")) - async def reload_plugins(self, ctx: Context, *plugins: PluginConverter) -> None: + async def reload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: """ Reload plugins given their fully qualified or unqualified names. @@ -101,12 +101,17 @@ async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) + @plugins_group.command("convert") + async def plugin_convert_test(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: + """Convert a plugin and given its source information.""" + await ctx.send(f"```py\n{plugin.__repr__()}```") + @plugins_group.command(name="install", aliases=("",)) - async def install_plugins(self, ctx: Context, *, plugin: AddonWithSourceConverter) -> None: + async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: """Install plugins from provided repo.""" # TODO: ensure path is a valid link and whatnot # TODO: also to support providing normal github and gitlab links and convert to zip - plugin: Addon = plugin + plugin: Plugin = plugin logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.url}") async with self.bot.http_session.get(plugin.source.url) as resp: if resp.status != 200: diff --git a/modmail/utils/addons/converters.py b/modmail/utils/addons/converters.py index 741bc441..94a7c6d6 100644 --- a/modmail/utils/addons/converters.py +++ b/modmail/utils/addons/converters.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re from typing import TYPE_CHECKING, Type @@ -10,6 +11,8 @@ if TYPE_CHECKING: from discord.ext.commands import Context + from modmail.log import ModmailLogger + ZIP_REGEX: re.Pattern = re.compile( r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip)) (?P[^@\s]+)$" ) @@ -22,6 +25,7 @@ r"(?P[^@\s]+)(?: \@(?P[\w\.\s]*))?$" ) +logger: ModmailLogger = logging.getLogger(__name__) AddonClass = Type[Addon] @@ -29,22 +33,28 @@ class AddonConverter(commands.Converter): """A converter that takes an addon source, and gets a Addon object from it.""" - async def convert(self, ctx: Context, argument: str, cls: AddonClass) -> Addon: - """Convert a string in to an Addon.""" - match = ZIP_REGEX.fullmatch(argument) - if match is not None: - # we've matched, so its a zip - ... - - match = REPO_REGEX.fullmatch(argument) - if match is None: - raise commands.BadArgument(f"{argument} is not a valid source.") - return ... + async def convert(self, ctx: Context, argument: str) -> None: + """Convert an argument into an Addon.""" + raise NotImplementedError("Inheriting classes must overwrite this method.") class PluginWithSourceConverter(AddonConverter): """A plugin converter that takes a source, addon name, and returns a Plugin.""" - async def convert(self, ctx: Context, argument: str) -> Plugin: + async def convert(self, _: Context, argument: str) -> Plugin: """Convert a provided plugin and source to a Plugin.""" - super().convert(ctx, argument, cls=Plugin) + match = ZIP_REGEX.fullmatch(argument) + if match is not None: + logger.debug("Matched as a zip, creating a Plugin from zip.") + return Plugin.from_zip(match.group("addon"), match.group("url")) + + match = REPO_REGEX.fullmatch(argument) + if match is None: + raise commands.BadArgument(f"{argument} is not a valid source and plugin.") + return Plugin.from_repo( + match.group("addon"), + match.group("user"), + match.group("repo"), + match.group("reflike"), + match.group("githost") or "github", + ) diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py index 774802d3..499f7e16 100644 --- a/modmail/utils/addons/models.py +++ b/modmail/utils/addons/models.py @@ -38,7 +38,7 @@ class Gitlab(GitHost): zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" -Host = Literal["Github", "Gitlab"] +Host = Literal["github", "gitlab"] class AddonSource: @@ -86,7 +86,7 @@ def from_zip(cls, url: str) -> AddonSource: source = cls(match.group("url"), SourceTypeEnum.ZIP) return source - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: no cover return f"" @@ -114,7 +114,7 @@ def __init__(self, name: str, source: AddonSource, **kw) -> Plugin: @classmethod def from_repo( - cls, addon: str, user: str, repo: str, reflike: str = None, githost: Host = "github" + cls, addon: str, user: str, repo: str, reflike: str = None, githost: Optional[Host] = "github" ) -> Plugin: """Create a Plugin from a repository regex match.""" source = AddonSource.from_repo(user, repo, reflike, githost) @@ -126,5 +126,5 @@ def from_zip(cls, addon: str, url: str) -> Plugin: source = AddonSource.from_zip(url) return cls(addon, source) - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: no cover return f"" diff --git a/tests/docs.md b/tests/docs.md index 28af2ff0..a1f5a0a9 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -1,72 +1,3 @@ -# tests.test_bot - -Test modmail basics. - -- import module -- create a bot object - -## -### test_bot_creation -Ensure we can make a ModmailBot instance. - -**Markers:** -- asyncio -- dependency (name=create_bot) -### test_bot_close -Ensure bot closes without error. - -**Markers:** -- asyncio -- dependency (depends=['create_bot']) -### test_bot_main -Import modmail.__main__. - -**Markers:** -- dependency (depends=['create_bot']) -# tests.test_logs -## -### test_create_logging -Modmail logging is importable and sets root logger correctly. - -**Markers:** -- dependency (name=create_logger) -### test_notice_level -Test notice logging level prints a notice response. - -**Markers:** -- dependency (depends=['create_logger']) -### test_trace_level -Test trace logging level prints a trace response. - -**Markers:** -- skip -- dependency (depends=['create_logger']) -# tests.modmail.utils.test_embeds -## -### test_patch_embed -Ensure that the function changes init only after the patch is called. - -**Markers:** -- dependency (name=patch_embed) -### test_create_embed -Test creating an embed with patched parameters works properly. - -**Markers:** -- dependency (depends_on=patch_embed) -### test_create_embed_with_extra_params -Test creating an embed with extra parameters errors properly. - -**Markers:** -- dependency (depends_on=patch_embed) -### test_create_embed_with_description_and_content - - Create an embed while providing both description and content parameters. - - Providing both is ambiguous and should error. - - -**Markers:** -- dependency (depends_on=patch_embed) # tests.modmail.utils.addons.test_converters ## ### test_converter @@ -74,77 +5,204 @@ Convert a user input into a Source. **Markers:** - xfail (reason=Not implemented) -- skip +- asyncio ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_repo_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=repo_regex) - parametrize (entry, user, repo, addon, reflike, githost[('onerandomusername/addons planet', 'onerandomusername', 'addons', 'planet', None, None), ('github onerandomusername/addons planet @master', 'onerandomusername', 'addons', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @v1.0.2', 'onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('github onerandomusername/repo planet @master', 'onerandomusername', 'repo', 'planet', 'master', 'github'), ('gitlab onerandomusername/repo planet @main', 'onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('https://github.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'github'), ('https://gitlab.com/onerandomusername/repo planet', 'onerandomusername', 'repo', 'planet', None, 'gitlab'), ('https://github.com/psf/black black @21.70b', 'psf', 'black', 'black', '21.70b', 'github')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) ### test_zip_regex Test the repo regex to ensure that it matches what it should. **Markers:** +- dependency (name=zip_regex) - parametrize (entry, url, domain, path, addon[('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'github.com/onerandomusername/modmail-addons/archive/main.zip', 'github.com', 'onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'gitlab.com', 'onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('https://example.com/bleeeep.zip myanmar', 'example.com/bleeeep.zip', 'example.com', 'bleeeep.zip', 'myanmar'), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'github.com/discord-modmail/addons/archive/bast.zip', 'github.com', 'discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip documentation', 'rtfd.io/plugs.zip', 'rtfd.io', 'plugs.zip', 'documentation'), ('pages.dev/hiy.zip black', 'pages.dev/hiy.zip', 'pages.dev', 'hiy.zip', 'black')]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- xfail # tests.modmail.utils.addons.test_models ## ### test_addon_model diff --git a/tests/modmail/utils/addons/test_converters.py b/tests/modmail/utils/addons/test_converters.py index 59d3a685..9ebda7ce 100644 --- a/tests/modmail/utils/addons/test_converters.py +++ b/tests/modmail/utils/addons/test_converters.py @@ -5,14 +5,14 @@ import pytest -from modmail.utils.addons.converters import REPO_REGEX, ZIP_REGEX, AddonConverter +from modmail.utils.addons.converters import REPO_REGEX, ZIP_REGEX, AddonConverter, PluginWithSourceConverter -@pytest.mark.skip +@pytest.mark.asyncio @pytest.mark.xfail(reason="Not implemented") -def test_converter() -> None: +async def test_converter() -> None: """Convert a user input into a Source.""" - addon = AddonConverter().convert(None, "github") # noqa: F841 + addon = await AddonConverter().convert(None, "github") # noqa: F841 # fmt: off @@ -54,6 +54,7 @@ def test_converter() -> None: ], ) # fmt: on +@pytest.mark.dependency(name="repo_regex") def test_repo_regex(entry, user, repo, addon, reflike, githost) -> None: """Test the repo regex to ensure that it matches what it should.""" match = REPO_REGEX.fullmatch(entry) @@ -115,6 +116,7 @@ def test_repo_regex(entry, user, repo, addon, reflike, githost) -> None: ] ) # fmt: on +@pytest.mark.dependency(name="zip_regex") def test_zip_regex(entry, url, domain, path, addon) -> None: """Test the repo regex to ensure that it matches what it should.""" match = ZIP_REGEX.fullmatch(entry) @@ -123,3 +125,32 @@ def test_zip_regex(entry, url, domain, path, addon) -> None: assert match.group("domain") == domain assert match.group("path") == path assert match.group("addon") == addon + + +@pytest.mark.parametrize( + "arg", + [ + "github.com/onerandomusername/modmail-addons/archive/main.zip earth", + "onerandomusername/addons planet", + "github onerandomusername/addons planet @master", + "gitlab onerandomusername/repo planet @v1.0.2", + "github onerandomusername/repo planet @master", + "gitlab onerandomusername/repo planet @main", + "https://github.com/onerandomusername/repo planet", + "https://gitlab.com/onerandomusername/repo planet", + "https://github.com/psf/black black @21.70b", + "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", + "https://example.com/bleeeep.zip myanmar", + "http://github.com/discord-modmail/addons/archive/bast.zip thebot", + "rtfd.io/plugs.zip documentation", + "pages.dev/hiy.zip black", + pytest.param("the world exists.", marks=pytest.mark.xfail) + + ] +) +@pytest.mark.dependency(depends_on=["repo_regex", "zip_regex"]) +@pytest.mark.asyncio +async def test_plugin_with_source_converter(arg: str) -> None: + """Test the Plugin converter works, and successfully converts a plugin with its source.""" + await PluginWithSourceConverter().convert(None, arg) # noqa: F841 From 3544367df77bf6e9c586f7a835d7af1bda816ea2 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 13:31:52 -0400 Subject: [PATCH 009/100] plugins: add @local plugin source --- modmail/utils/addons/converters.py | 8 +- modmail/utils/addons/models.py | 1 + tests/docs.md | 117 +++++++++++++++--- tests/modmail/utils/addons/test_converters.py | 101 +++++++++++---- tests/modmail/utils/addons/test_models.py | 1 + 5 files changed, 187 insertions(+), 41 deletions(-) diff --git a/modmail/utils/addons/converters.py b/modmail/utils/addons/converters.py index 94a7c6d6..78b7c881 100644 --- a/modmail/utils/addons/converters.py +++ b/modmail/utils/addons/converters.py @@ -6,13 +6,14 @@ from discord.ext import commands -from modmail.utils.addons.models import Addon, Plugin +from modmail.utils.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum if TYPE_CHECKING: from discord.ext.commands import Context from modmail.log import ModmailLogger +LOCAL_REGEX: re.Pattern = re.compile(r"^\@local (?P[^@\s]+)$") ZIP_REGEX: re.Pattern = re.compile( r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip)) (?P[^@\s]+)$" ) @@ -43,6 +44,11 @@ class PluginWithSourceConverter(AddonConverter): async def convert(self, _: Context, argument: str) -> Plugin: """Convert a provided plugin and source to a Plugin.""" + match = LOCAL_REGEX.match(argument) + if match is not None: + logger.debug("Matched as a local file, creating a Plugin without a source url.") + source = AddonSource(None, SourceTypeEnum.LOCAL) + return Plugin(name=match.group("addon"), source=source) match = ZIP_REGEX.fullmatch(argument) if match is not None: logger.debug("Matched as a zip, creating a Plugin from zip.") diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py index 499f7e16..f789a3c6 100644 --- a/modmail/utils/addons/models.py +++ b/modmail/utils/addons/models.py @@ -10,6 +10,7 @@ class SourceTypeEnum(Enum): ZIP = 0 REPO = 1 + LOCAL = 2 class GitHost: diff --git a/tests/docs.md b/tests/docs.md index a1f5a0a9..435e24c0 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -1,3 +1,72 @@ +# tests.test_bot + +Test modmail basics. + +- import module +- create a bot object + +## +### test_bot_creation +Ensure we can make a ModmailBot instance. + +**Markers:** +- asyncio +- dependency (name=create_bot) +### test_bot_close +Ensure bot closes without error. + +**Markers:** +- asyncio +- dependency (depends=['create_bot']) +### test_bot_main +Import modmail.__main__. + +**Markers:** +- dependency (depends=['create_bot']) +# tests.test_logs +## +### test_create_logging +Modmail logging is importable and sets root logger correctly. + +**Markers:** +- dependency (name=create_logger) +### test_notice_level +Test notice logging level prints a notice response. + +**Markers:** +- dependency (depends=['create_logger']) +### test_trace_level +Test trace logging level prints a trace response. + +**Markers:** +- skip +- dependency (depends=['create_logger']) +# tests.modmail.utils.test_embeds +## +### test_patch_embed +Ensure that the function changes init only after the patch is called. + +**Markers:** +- dependency (name=patch_embed) +### test_create_embed +Test creating an embed with patched parameters works properly. + +**Markers:** +- dependency (depends_on=patch_embed) +### test_create_embed_with_extra_params +Test creating an embed with extra parameters errors properly. + +**Markers:** +- dependency (depends_on=patch_embed) +### test_create_embed_with_description_and_content + + Create an embed while providing both description and content parameters. + + Providing both is ambiguous and should error. + + +**Markers:** +- dependency (depends_on=patch_embed) # tests.modmail.utils.addons.test_converters ## ### test_converter @@ -96,112 +165,119 @@ Test the Plugin converter works, and successfully converts a plugin with its sou **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) ### test_plugin_with_source_converter Test the Plugin converter works, and successfully converts a plugin with its source. **Markers:** - asyncio - dependency (depends_on=['repo_regex', 'zip_regex']) -- parametrize (arg['github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'onerandomusername/addons planet', 'github onerandomusername/addons planet @master', 'gitlab onerandomusername/repo planet @v1.0.2', 'github onerandomusername/repo planet @master', 'gitlab onerandomusername/repo planet @main', 'https://github.com/onerandomusername/repo planet', 'https://gitlab.com/onerandomusername/repo planet', 'https://github.com/psf/black black @21.70b', 'https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'https://example.com/bleeeep.zip myanmar', 'http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'rtfd.io/plugs.zip documentation', 'pages.dev/hiy.zip black', ParameterSet(values=('the world exists.',), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) +### test_plugin_with_source_converter +Test the Plugin converter works, and successfully converts a plugin with its source. + +**Markers:** +- asyncio +- dependency (depends_on=['repo_regex', 'zip_regex']) +- parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) - xfail # tests.modmail.utils.addons.test_models ## @@ -211,12 +287,17 @@ All addons will be of a specific type, so we should not be able to create a gene Test the AddonSource init sets class vars appropiately. **Markers:** -- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', )]) +- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', ), (None, )]) +### test_addonsource_init +Test the AddonSource init sets class vars appropiately. + +**Markers:** +- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', ), (None, )]) ### test_addonsource_init Test the AddonSource init sets class vars appropiately. **Markers:** -- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', )]) +- parametrize (zip_url, source_type[('github.com/bast0006.zip', ), ('gitlab.com/onerandomusername.zip', ), (None, )]) ### test_addonsource_from_repo Test an addon source is properly made from repository information. diff --git a/tests/modmail/utils/addons/test_converters.py b/tests/modmail/utils/addons/test_converters.py index 9ebda7ce..0d346b58 100644 --- a/tests/modmail/utils/addons/test_converters.py +++ b/tests/modmail/utils/addons/test_converters.py @@ -5,7 +5,12 @@ import pytest -from modmail.utils.addons.converters import REPO_REGEX, ZIP_REGEX, AddonConverter, PluginWithSourceConverter +# fmt: off +from modmail.utils.addons.converters import ( + REPO_REGEX, ZIP_REGEX, AddonConverter, PluginWithSourceConverter, SourceTypeEnum, +) + +# fmt: on @pytest.mark.asyncio @@ -127,30 +132,82 @@ def test_zip_regex(entry, url, domain, path, addon) -> None: assert match.group("addon") == addon +# fmt: off @pytest.mark.parametrize( - "arg", + "entry, name, source_type", [ - "github.com/onerandomusername/modmail-addons/archive/main.zip earth", - "onerandomusername/addons planet", - "github onerandomusername/addons planet @master", - "gitlab onerandomusername/repo planet @v1.0.2", - "github onerandomusername/repo planet @master", - "gitlab onerandomusername/repo planet @main", - "https://github.com/onerandomusername/repo planet", - "https://gitlab.com/onerandomusername/repo planet", - "https://github.com/psf/black black @21.70b", - "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", - "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", - "https://example.com/bleeeep.zip myanmar", - "http://github.com/discord-modmail/addons/archive/bast.zip thebot", - "rtfd.io/plugs.zip documentation", - "pages.dev/hiy.zip black", - pytest.param("the world exists.", marks=pytest.mark.xfail) - - ] + ( + "onerandomusername/addons planet", + "planet", SourceTypeEnum.REPO + ), + ( + "github onerandomusername/addons planet @master", + "planet", SourceTypeEnum.REPO + ), + ( + "gitlab onerandomusername/repo planet @v1.0.2", + "planet", SourceTypeEnum.REPO + ), + ( + "github onerandomusername/repo planet @master", + "planet", SourceTypeEnum.REPO + ), + ( + "gitlab onerandomusername/repo planet @main", + "planet", SourceTypeEnum.REPO + ), + ( + "https://github.com/onerandomusername/repo planet", + "planet", SourceTypeEnum.REPO + ), + ( + "https://gitlab.com/onerandomusername/repo planet", + "planet", SourceTypeEnum.REPO + ), + ( + "https://github.com/psf/black black @21.70b", + "black", SourceTypeEnum.REPO + ), + ( + "github.com/onerandomusername/modmail-addons/archive/main.zip earth", + "earth", SourceTypeEnum.ZIP + ), + ( + "https://github.com/onerandomusername/modmail-addons/archive/main.zip planet", + "planet", SourceTypeEnum.ZIP + ), + ( + "https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth", # noqa: E501 + "earth", SourceTypeEnum.ZIP + ), + ( + "https://example.com/bleeeep.zip myanmar", + "myanmar", SourceTypeEnum.ZIP + ), + ( + "http://github.com/discord-modmail/addons/archive/bast.zip thebot", + "thebot", SourceTypeEnum.ZIP + ), + ( + "rtfd.io/plugs.zip documentation", + "documentation", SourceTypeEnum.ZIP + ), + ( + "pages.dev/hiy.zip black", + "black", SourceTypeEnum.ZIP + ), + ( + "@local earth", + "earth", SourceTypeEnum.LOCAL + ), + pytest.param("the world exists.", None, None, marks=pytest.mark.xfail), + ], ) +# fmt: on @pytest.mark.dependency(depends_on=["repo_regex", "zip_regex"]) @pytest.mark.asyncio -async def test_plugin_with_source_converter(arg: str) -> None: +async def test_plugin_with_source_converter(entry: str, name: str, source_type: SourceTypeEnum) -> None: """Test the Plugin converter works, and successfully converts a plugin with its source.""" - await PluginWithSourceConverter().convert(None, arg) # noqa: F841 + plugin = await PluginWithSourceConverter().convert(None, entry) + assert plugin.name == name + assert plugin.source.source_type == source_type diff --git a/tests/modmail/utils/addons/test_models.py b/tests/modmail/utils/addons/test_models.py index 74d50372..1e75de8c 100644 --- a/tests/modmail/utils/addons/test_models.py +++ b/tests/modmail/utils/addons/test_models.py @@ -16,6 +16,7 @@ def test_addon_model(): [ ("github.com/bast0006.zip", SourceTypeEnum.ZIP), ("gitlab.com/onerandomusername.zip", SourceTypeEnum.REPO), + (None, SourceTypeEnum.LOCAL), ], ) def test_addonsource_init(zip_url, source_type): From 3ae5c0e18abc57124ba8e17c5f34d0658f840a4d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 20:08:49 -0400 Subject: [PATCH 010/100] chore: ensure zip_url does not contain https?:// --- modmail/utils/addons/models.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modmail/utils/addons/models.py b/modmail/utils/addons/models.py index f789a3c6..4b110945 100644 --- a/modmail/utils/addons/models.py +++ b/modmail/utils/addons/models.py @@ -54,11 +54,23 @@ class AddonSource: user: Optional[str] reflike: Optional[str] githost: Optional[Host] - githost_api = Optional[GitHost] + githost_api: Optional[GitHost] + + domain: Optional[str] + path: Optional[str] def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: """Initialize the AddonSource.""" self.zip_url = zip_url + if self.zip_url is not None: + match = re.match(r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*))$", self.zip_url) + self.zip_url = match.group("url") + self.domain = match.group("domain") + self.path = match.group("path") + else: + self.domain = None + self.path = None + self.source_type = type @classmethod @@ -83,7 +95,7 @@ def from_repo(cls, user: str, repo: str, reflike: str = None, githost: Host = "g @classmethod def from_zip(cls, url: str) -> AddonSource: """Create an AddonSource from a zip file.""" - match = re.match(r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip))", url) + match = re.match(r"^(?P(?:https?:\/\/)?(?P.*\..+?)\/(?P.*\.zip))$", url) source = cls(match.group("url"), SourceTypeEnum.ZIP) return source From 40c5b252f9d9f8ef16f4229b966c971e9660a6a4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 20:10:22 -0400 Subject: [PATCH 011/100] minor: seperate syncing logic from a command --- modmail/extensions/extension_manager.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 2f5c5a41..56bc2ff2 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -188,13 +188,8 @@ async def list_extensions(self, ctx: Context) -> None: # TODO: since we currently don't have a paginator. await ctx.send("".join(lines) or f"There are no {self.type}s installed.") - @extensions_group.command(name="refresh", aliases=("rewalk", "rescan")) - async def resync_extensions(self, ctx: Context) -> None: - """ - Refreshes the list of extensions from disk, but do not unload any currently active. - - Typical use case is in the event that the existing extensions have changed while the bot is running. - """ + def _resync_extensions(self) -> None: + """Resyncs extensions. Useful for when the files are dynamically updated.""" log.debug(f"Refreshing list of {self.type}s.") # make sure the new walk contains all currently loaded extensions, so they can be unloaded @@ -209,6 +204,15 @@ async def resync_extensions(self, ctx: Context) -> None: self.all_extensions.update(loaded_extensions) # now we can re-walk the extensions self.all_extensions.update(self.refresh_method()) + + @extensions_group.command(name="refresh", aliases=("rewalk", "rescan")) + async def resync_extensions(self, ctx: Context) -> None: + """ + Refreshes the list of extensions from disk, but do not unload any currently active. + + Typical use case is in the event that the existing extensions have changed while the bot is running. + """ + self._resync_extensions() await ctx.send(f":ok_hand: Refreshed list of {self.type}s.") def group_extension_statuses(self) -> t.Mapping[str, str]: From f0cb103e15b5f9cd500c8f3af5cef5d499671a71 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 21:00:00 -0400 Subject: [PATCH 012/100] feat: add downloading plugins --- modmail/extensions/plugin_manager.py | 133 +++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 17 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 84786507..5ec9ad84 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -1,14 +1,17 @@ from __future__ import annotations +import asyncio import logging +import os +import zipfile from typing import TYPE_CHECKING from discord.ext import commands from discord.ext.commands import Context -from modmail.extensions.extension_manager import ExtensionConverter, ExtensionManager +from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.addons.converters import PluginWithSourceConverter -from modmail.utils.addons.models import Plugin +from modmail.utils.addons.models import Plugin, SourceTypeEnum from modmail.utils.addons.plugins import BASE_PATH, PLUGINS, walk_plugins from modmail.utils.cogs import BotModes, ExtMetadata @@ -17,7 +20,7 @@ from modmail.log import ModmailLogger EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION) - +VALID_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] logger: ModmailLogger = logging.getLogger(__name__) @@ -101,7 +104,7 @@ async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) - @plugins_group.command("convert") + @plugins_group.command("convert", hidden=True) async def plugin_convert_test(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: """Convert a plugin and given its source information.""" await ctx.send(f"```py\n{plugin.__repr__()}```") @@ -109,24 +112,120 @@ async def plugin_convert_test(self, ctx: Context, *, plugin: PluginWithSourceCon @plugins_group.command(name="install", aliases=("",)) async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: """Install plugins from provided repo.""" - # TODO: ensure path is a valid link and whatnot - # TODO: also to support providing normal github and gitlab links and convert to zip plugin: Plugin = plugin - logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.url}") - async with self.bot.http_session.get(plugin.source.url) as resp: + if plugin.source.source_type is SourceTypeEnum.LOCAL: + # TODO: check the path of a local plugin + await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") + return + logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.zip_url}") + async with self.bot.http_session.get(f"https://{plugin.source.zip_url}") as resp: if resp.status != 200: - await ctx.send(f"Downloading {plugin.source.url} did not give a 200") - zip = await resp.read() + await ctx.send(f"Downloading {plugin.source.zip_url} did not give a 200") + return + raw_bytes = await resp.read() + if plugin.source.source_type is SourceTypeEnum.REPO: + file_name = f"{plugin.source.githost}/{plugin.source.user}/{plugin.source.repo}" + elif plugin.source.source_type is SourceTypeEnum.ZIP: + file_name = plugin.source.path.rstrip(".zip") + else: + raise TypeError("Unsupported source detected.") - # TODO: make this use a regex to get the name of the plugin, or make it provided in the inital arg - zip_path = BASE_PATH / ".cache" / f"{plugin.source.name}.zip" + zipfile_path = BASE_PATH / ".cache" / f"{file_name}.zip" - if not zip_path.exists(): - zip_path.parent.mkdir(parents=True, exist_ok=True) + plugin.source.cache_file = zipfile_path - with zip_path.open("wb") as f: - f.write(zip) - await ctx.send(f"Downloaded {zip_path}") + if not zipfile_path.exists(): + zipfile_path.parent.mkdir(parents=True, exist_ok=True) + else: + # overwriting an existing file + logger.info("Zip file already exists, overwriting it.") + + with zipfile_path.open("wb") as f: + f.write(raw_bytes) + await ctx.send(f"Downloaded {zipfile_path}") + + file = zipfile.ZipFile(zipfile_path) + print(file.namelist()) + file.printdir() + print(file.infolist()) + print("-" * 50) + _temp_direct_children = [p for p in zipfile.Path(file).iterdir()] + if len(_temp_direct_children) == 1: + # only one folder, so we probably want to recurse into it + _folder = _temp_direct_children[0] + if _folder.is_dir(): + # the zip folder needs to have the extra directory removed, + # and everything moved up a directory. + temp_archive = BASE_PATH / ".restructure.zip" + temp_archive = zipfile.ZipFile(temp_archive, mode="w") + for path in file.infolist(): + logger.trace(f"File name: {path.filename}") + if (new_name := path.filename.split("/", 1)[-1]) == "": + continue + temp_archive.writestr(new_name, file.read(path)) + # given that we are writing to disk, we want to ensure + # that we yield to anything that needs the event loop + await asyncio.sleep(0) + temp_archive.close() + os.replace(temp_archive.filename, file.filename) + + # reset the file so we ensure we have the new archive open + file.close() + print(zipfile_path) + print(file.filename) + file = zipfile.ZipFile(zipfile_path) + + # TODO: REMOVE THIS SECTION + # extract the archive + file.extractall(BASE_PATH / ".extraction") + + # determine plugins in the archive + archive_plugin_directory = None + for dir in VALID_PLUGIN_DIRECTORIES: + dir = dir + "/" + if dir in file.namelist(): + archive_plugin_directory = dir + break + if archive_plugin_directory is None: + await ctx.send("Looks like there isn't a valid plugin here.") + return + + archive_plugin_directory = zipfile.Path(file, at=archive_plugin_directory) + print(archive_plugin_directory) + lil_pluggies = [] + for path in archive_plugin_directory.iterdir(): + logger.debug(f"archive_plugin_directory: {path}") + if path.is_dir(): + lil_pluggies.append(archive_plugin_directory.name + "/" + path.name + "/") + + logger.debug(f"Plugins detected: {lil_pluggies}") + all_lil_pluggies = lil_pluggies + files = file.namelist() + for pluggy in all_lil_pluggies: + for f in files: + if f == pluggy: + continue + if f.startswith(pluggy): + all_lil_pluggies.append(f) + print(f) + + # extract the drive + these_plugins_dir = BASE_PATH / file_name + print(file.namelist()) + file.extractall(these_plugins_dir, all_lil_pluggies) + + # TODO: rewrite this only need to scan the new directory + self._resync_extensions() + temp_new_plugins = [x.strip("/").rsplit("/", 1)[1] for x in lil_pluggies] + new_plugins = [] + for p in temp_new_plugins: + logger.debug(p) + try: + new_plugins.append(await PluginPathConverter().convert(None, p)) + except commands.BadArgument: + pass + + self.batch_manage(Action.LOAD, *new_plugins) # TODO: Implement install/enable/disable/etc From 75bae576b1e2180e859f901043626176bebdce11 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 28 Aug 2021 21:42:43 -0400 Subject: [PATCH 013/100] minor: move addons out of utils --- modmail/{utils => }/addons/__init__.py | 0 modmail/{utils => }/addons/converters.py | 2 +- modmail/{utils => }/addons/models.py | 0 modmail/{utils => }/addons/plugins.py | 0 modmail/bot.py | 2 +- modmail/extensions/plugin_manager.py | 6 +- tests/docs.md | 60 +++++++++---------- tests/modmail/{utils => }/addons/__init__.py | 0 .../{utils => }/addons/test_converters.py | 2 +- .../modmail/{utils => }/addons/test_models.py | 2 +- tests/{ => modmail}/test_bot.py | 0 tests/{ => modmail}/test_logs.py | 0 12 files changed, 37 insertions(+), 37 deletions(-) rename modmail/{utils => }/addons/__init__.py (100%) rename modmail/{utils => }/addons/converters.py (96%) rename modmail/{utils => }/addons/models.py (100%) rename modmail/{utils => }/addons/plugins.py (100%) rename tests/modmail/{utils => }/addons/__init__.py (100%) rename tests/modmail/{utils => }/addons/test_converters.py (99%) rename tests/modmail/{utils => }/addons/test_models.py (98%) rename tests/{ => modmail}/test_bot.py (100%) rename tests/{ => modmail}/test_logs.py (100%) diff --git a/modmail/utils/addons/__init__.py b/modmail/addons/__init__.py similarity index 100% rename from modmail/utils/addons/__init__.py rename to modmail/addons/__init__.py diff --git a/modmail/utils/addons/converters.py b/modmail/addons/converters.py similarity index 96% rename from modmail/utils/addons/converters.py rename to modmail/addons/converters.py index 78b7c881..14f38378 100644 --- a/modmail/utils/addons/converters.py +++ b/modmail/addons/converters.py @@ -6,7 +6,7 @@ from discord.ext import commands -from modmail.utils.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum +from modmail.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum if TYPE_CHECKING: from discord.ext.commands import Context diff --git a/modmail/utils/addons/models.py b/modmail/addons/models.py similarity index 100% rename from modmail/utils/addons/models.py rename to modmail/addons/models.py diff --git a/modmail/utils/addons/plugins.py b/modmail/addons/plugins.py similarity index 100% rename from modmail/utils/addons/plugins.py rename to modmail/addons/plugins.py diff --git a/modmail/bot.py b/modmail/bot.py index a03ca213..2833d347 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -10,9 +10,9 @@ from discord.client import _cleanup_loop from discord.ext import commands +from modmail.addons.plugins import PLUGINS, walk_plugins from modmail.config import CONFIG from modmail.log import ModmailLogger -from modmail.utils.addons.plugins import PLUGINS, walk_plugins from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions REQUIRED_INTENTS = Intents( diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 5ec9ad84..fee010a6 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -9,10 +9,10 @@ from discord.ext import commands from discord.ext.commands import Context +from modmail.addons.converters import PluginWithSourceConverter +from modmail.addons.models import Plugin, SourceTypeEnum +from modmail.addons.plugins import BASE_PATH, PLUGINS, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager -from modmail.utils.addons.converters import PluginWithSourceConverter -from modmail.utils.addons.models import Plugin, SourceTypeEnum -from modmail.utils.addons.plugins import BASE_PATH, PLUGINS, walk_plugins from modmail.utils.cogs import BotModes, ExtMetadata if TYPE_CHECKING: diff --git a/tests/docs.md b/tests/docs.md index 435e24c0..75812f76 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -1,4 +1,4 @@ -# tests.test_bot +# tests.modmail.test_bot Test modmail basics. @@ -23,7 +23,7 @@ Import modmail.__main__. **Markers:** - dependency (depends=['create_bot']) -# tests.test_logs +# tests.modmail.test_logs ## ### test_create_logging Modmail logging is importable and sets root logger correctly. @@ -41,33 +41,7 @@ Test trace logging level prints a trace response. **Markers:** - skip - dependency (depends=['create_logger']) -# tests.modmail.utils.test_embeds -## -### test_patch_embed -Ensure that the function changes init only after the patch is called. - -**Markers:** -- dependency (name=patch_embed) -### test_create_embed -Test creating an embed with patched parameters works properly. - -**Markers:** -- dependency (depends_on=patch_embed) -### test_create_embed_with_extra_params -Test creating an embed with extra parameters errors properly. - -**Markers:** -- dependency (depends_on=patch_embed) -### test_create_embed_with_description_and_content - - Create an embed while providing both description and content parameters. - - Providing both is ambiguous and should error. - - -**Markers:** -- dependency (depends_on=patch_embed) -# tests.modmail.utils.addons.test_converters +# tests.modmail.addons.test_converters ## ### test_converter Convert a user input into a Source. @@ -279,7 +253,7 @@ Test the Plugin converter works, and successfully converts a plugin with its sou - dependency (depends_on=['repo_regex', 'zip_regex']) - parametrize (entry, name, source_type[('onerandomusername/addons planet', 'planet', ), ('github onerandomusername/addons planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @v1.0.2', 'planet', ), ('github onerandomusername/repo planet @master', 'planet', ), ('gitlab onerandomusername/repo planet @main', 'planet', ), ('https://github.com/onerandomusername/repo planet', 'planet', ), ('https://gitlab.com/onerandomusername/repo planet', 'planet', ), ('https://github.com/psf/black black @21.70b', 'black', ), ('github.com/onerandomusername/modmail-addons/archive/main.zip earth', 'earth', ), ('https://github.com/onerandomusername/modmail-addons/archive/main.zip planet', 'planet', ), ('https://gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip earth', 'earth', ), ('https://example.com/bleeeep.zip myanmar', 'myanmar', ), ('http://github.com/discord-modmail/addons/archive/bast.zip thebot', 'thebot', ), ('rtfd.io/plugs.zip documentation', 'documentation', ), ('pages.dev/hiy.zip black', 'black', ), ('@local earth', 'earth', ), ParameterSet(values=('the world exists.', None, None), marks=(MarkDecorator(mark=Mark(name='xfail', args=(), kwargs={})),), id=None)]) - xfail -# tests.modmail.utils.addons.test_models +# tests.modmail.addons.test_models ## ### test_addon_model All addons will be of a specific type, so we should not be able to create a generic addon. @@ -450,3 +424,29 @@ Test that a plugin can be created from a zip url. **Markers:** - parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) +# tests.modmail.utils.test_embeds +## +### test_patch_embed +Ensure that the function changes init only after the patch is called. + +**Markers:** +- dependency (name=patch_embed) +### test_create_embed +Test creating an embed with patched parameters works properly. + +**Markers:** +- dependency (depends_on=patch_embed) +### test_create_embed_with_extra_params +Test creating an embed with extra parameters errors properly. + +**Markers:** +- dependency (depends_on=patch_embed) +### test_create_embed_with_description_and_content + + Create an embed while providing both description and content parameters. + + Providing both is ambiguous and should error. + + +**Markers:** +- dependency (depends_on=patch_embed) diff --git a/tests/modmail/utils/addons/__init__.py b/tests/modmail/addons/__init__.py similarity index 100% rename from tests/modmail/utils/addons/__init__.py rename to tests/modmail/addons/__init__.py diff --git a/tests/modmail/utils/addons/test_converters.py b/tests/modmail/addons/test_converters.py similarity index 99% rename from tests/modmail/utils/addons/test_converters.py rename to tests/modmail/addons/test_converters.py index 0d346b58..556b8156 100644 --- a/tests/modmail/utils/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -6,7 +6,7 @@ import pytest # fmt: off -from modmail.utils.addons.converters import ( +from modmail.addons.converters import ( REPO_REGEX, ZIP_REGEX, AddonConverter, PluginWithSourceConverter, SourceTypeEnum, ) diff --git a/tests/modmail/utils/addons/test_models.py b/tests/modmail/addons/test_models.py similarity index 98% rename from tests/modmail/utils/addons/test_models.py rename to tests/modmail/addons/test_models.py index 1e75de8c..1114dfc9 100644 --- a/tests/modmail/utils/addons/test_models.py +++ b/tests/modmail/addons/test_models.py @@ -2,7 +2,7 @@ import pytest -from modmail.utils.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum +from modmail.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum def test_addon_model(): diff --git a/tests/test_bot.py b/tests/modmail/test_bot.py similarity index 100% rename from tests/test_bot.py rename to tests/modmail/test_bot.py diff --git a/tests/test_logs.py b/tests/modmail/test_logs.py similarity index 100% rename from tests/test_logs.py rename to tests/modmail/test_logs.py From 4605b1459b87f953c2ce9e8f3299332f5cbd09cc Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 00:52:51 -0400 Subject: [PATCH 014/100] ci: make isort not fight with black --- tests/modmail/addons/test_converters.py | 6 +++++- tox.ini | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index 556b8156..51b05497 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -7,7 +7,11 @@ # fmt: off from modmail.addons.converters import ( - REPO_REGEX, ZIP_REGEX, AddonConverter, PluginWithSourceConverter, SourceTypeEnum, + REPO_REGEX, + ZIP_REGEX, + AddonConverter, + PluginWithSourceConverter, + SourceTypeEnum, ) # fmt: on diff --git a/tox.ini b/tox.ini index d06c9a68..ca7d64e2 100644 --- a/tox.ini +++ b/tox.ini @@ -29,10 +29,11 @@ per-file-ignores= docs.py:B008 [isort] -profile = black -multi_line_output=5 +profile=black +multi_line_output=3 include_trailing_comma=True force_grid_wrap=0 use_parentheses=True ensure_newline_before_comments=True line_length=110 +atomic=True From 8ede9db83e5f301b57f6ff655f40b184e4e00076 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 02:19:45 -0400 Subject: [PATCH 015/100] major: refactor plugin installation --- modmail/addons/errors.py | 10 +++ modmail/addons/models.py | 9 +- modmail/addons/plugins.py | 57 +++++++++++-- modmail/addons/utils.py | 71 ++++++++++++++++ modmail/errors.py | 8 ++ modmail/extensions/plugin_manager.py | 119 ++++++--------------------- 6 files changed, 172 insertions(+), 102 deletions(-) create mode 100644 modmail/addons/errors.py create mode 100644 modmail/addons/utils.py create mode 100644 modmail/errors.py diff --git a/modmail/addons/errors.py b/modmail/addons/errors.py new file mode 100644 index 00000000..c0a9810e --- /dev/null +++ b/modmail/addons/errors.py @@ -0,0 +1,10 @@ +class AddonError(Exception): + """Base Addon utils and extension exception.""" + + pass + + +class NoPluginDirectoryError(AddonError): + """No plugin directory exists.""" + + pass diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 4b110945..11921bd1 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -2,7 +2,11 @@ import re from enum import Enum -from typing import TYPE_CHECKING, Literal, Optional +from typing import TYPE_CHECKING, Literal, Optional, Union + +if TYPE_CHECKING: + import pathlib + import zipfile class SourceTypeEnum(Enum): @@ -59,6 +63,9 @@ class AddonSource: domain: Optional[str] path: Optional[str] + addon_directory: Optional[str] + cache_file: Optional[Union[zipfile.Path, pathlib.Path]] + def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: """Initialize the AddonSource.""" self.zip_url = zip_url diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index e6f681ac..91569dc3 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -14,21 +14,62 @@ import inspect import logging import typing as t +import zipfile from pathlib import Path +from zipfile import ZipFile from modmail import plugins +from modmail.addons.errors import NoPluginDirectoryError from modmail.log import ModmailLogger from modmail.utils.cogs import ExtMetadata from modmail.utils.extensions import BOT_MODE, unqualify -log: ModmailLogger = logging.getLogger(__name__) - +logger: ModmailLogger = logging.getLogger(__name__) +VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] BASE_PATH = Path(plugins.__file__).parent.resolve() PLUGIN_MODULE = "modmail.plugins" PLUGINS: t.Dict[str, t.Tuple[bool, bool]] = dict() +def find_plugins_in_zip(zip_path: t.Union[str, Path]) -> t.Tuple[t.List[str], t.List[str]]: + """ + Find the plugins that are in a zip file. + + All plugins in a zip folder will be located at either `Plugins/` or `plugins/` + """ + archive_plugin_directory = None + file = ZipFile(zip_path) + for dir in VALID_ZIP_PLUGIN_DIRECTORIES: + dir = dir + "/" + if dir in file.namelist(): + archive_plugin_directory = dir + break + if archive_plugin_directory is None: + raise NoPluginDirectoryError(f"No {' or '.join(VALID_ZIP_PLUGIN_DIRECTORIES)} directory exists.") + archive_plugin_directory = zipfile.Path(file, at=archive_plugin_directory) + lil_pluggies = [] + for path in archive_plugin_directory.iterdir(): + logger.debug(f"archive_plugin_directory: {path}") + if path.is_dir(): + lil_pluggies.append(archive_plugin_directory.name + "/" + path.name + "/") + + logger.debug(f"Plugins detected: {lil_pluggies}") + all_lil_pluggies = lil_pluggies.copy() + files = file.namelist() + for pluggy in all_lil_pluggies: + for f in files: + if f == pluggy: + continue + if f.startswith(pluggy): + all_lil_pluggies.append(f) + print(f) + logger.trace(f"lil_pluggies: {lil_pluggies}") + logger.trace(f"all_lil_pluggies: {all_lil_pluggies}") + + return lil_pluggies, all_lil_pluggies + + def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder @@ -38,13 +79,13 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: # support following symlinks, see: https://bugs.python.org/issue33428 for path in glob.iglob(f"{BASE_PATH}/**/*.py", recursive=True): - log.trace("Path: {0}".format(path)) + logger.trace("Path: {0}".format(path)) # calculate the module name, dervived from the relative path relative_path = Path(path).relative_to(BASE_PATH) name = relative_path.__str__().rstrip(".py").replace("/", ".") name = PLUGIN_MODULE + "." + name - log.trace("Module name: {0}".format(name)) + logger.trace("Module name: {0}".format(name)) if unqualify(name.split(".")[-1]).startswith("_"): # Ignore module/package names starting with an underscore. @@ -61,7 +102,7 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: imported = importlib.util.module_from_spec(spec) spec.loader.exec_module(imported) except Exception: - log.error( + logger.error( "Failed to import {0}. As a result, this plugin is not considered installed.".format(name), exc_info=True, ) @@ -69,18 +110,18 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: if not inspect.isfunction(getattr(imported, "setup", None)): # If it lacks a setup function, it's not a plugin. This is enforced by dpy. - log.trace("{0} does not have a setup function. Skipping.".format(name)) + logger.trace("{0} does not have a setup function. Skipping.".format(name)) continue ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) if ext_metadata is not None: # check if this plugin is dev only or plugin dev only load_cog = bool(int(ext_metadata.load_if_mode) & BOT_MODE) - log.trace(f"Load plugin {imported.__name__!r}?: {load_cog}") + logger.trace(f"Load plugin {imported.__name__!r}?: {load_cog}") yield imported.__name__, load_cog continue - log.info( + logger.info( f"Plugin {imported.__name__!r} is missing a EXT_METADATA variable. Assuming its a normal plugin." ) diff --git a/modmail/addons/utils.py b/modmail/addons/utils.py new file mode 100644 index 00000000..137d3874 --- /dev/null +++ b/modmail/addons/utils.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import logging +import os +import pathlib +import zipfile +from typing import TYPE_CHECKING, Union + +from modmail.addons.models import SourceTypeEnum +from modmail.addons.plugins import BASE_PATH +from modmail.errors import HTTPError + +if TYPE_CHECKING: + from aiohttp import ClientSession + + from modmail.addons.models import AddonSource + from modmail.log import ModmailLogger +logger: ModmailLogger = logging.getLogger(__name__) + + +def move_zip_contents_up_a_level(zip_path: Union[str, pathlib.Path], folder: str = None) -> None: + """ + Assuming that there is only one folder, move everything up a level. + + If a folder is provided, it will attempt to use that folder. This folder *must* be located in the root + of the zip folder. + """ + file = zipfile.ZipFile(zip_path) + temp_archive = BASE_PATH / ".tmp.zip" # temporary folder for moving + temp_archive = zipfile.ZipFile(temp_archive, mode="w") + for path in file.infolist(): + logger.trace(f"File name: {path.filename}") + if (new_name := path.filename.split("/", 1)[-1]) == "": + continue + temp_archive.writestr(new_name, file.read(path)) + temp_archive.close() + os.replace(temp_archive.filename, file.filename) + + +async def download_zip_from_source(source: AddonSource, session: ClientSession) -> zipfile.ZipFile: + """ + Download a zip file from a source. + + It is currently required to provide an http session. + """ + async with session.get(f"https://{source.zip_url}") as resp: + if resp.status != 200: + raise HTTPError(resp) + raw_bytes = await resp.read() + if source.source_type is SourceTypeEnum.REPO: + file_name = f"{source.githost}/{source.user}/{source.repo}" + elif source.source_type is SourceTypeEnum.ZIP: + file_name = source.path.rstrip(".zip") + else: + raise TypeError("Unsupported source detected.") + + zipfile_path = BASE_PATH / ".cache" / f"{file_name}.zip" + + source.addon_directory = file_name + source.cache_file = zipfile_path + + if not zipfile_path.exists(): + zipfile_path.parent.mkdir(parents=True, exist_ok=True) + else: + # overwriting an existing file + logger.info("Zip file already exists, overwriting it.") + + with zipfile_path.open("wb") as f: + f.write(raw_bytes) + + return zipfile.ZipFile(zipfile_path) diff --git a/modmail/errors.py b/modmail/errors.py new file mode 100644 index 00000000..e60400fe --- /dev/null +++ b/modmail/errors.py @@ -0,0 +1,8 @@ +from aiohttp import ClientResponse + + +class HTTPError(Exception): + """Response from an http request was not desired.""" + + def __init__(self, response: ClientResponse) -> None: + self.response = response diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index fee010a6..cc82c303 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -2,16 +2,17 @@ import asyncio import logging -import os import zipfile from typing import TYPE_CHECKING from discord.ext import commands from discord.ext.commands import Context +import modmail.addons.utils as addon_utils +from modmail import errors from modmail.addons.converters import PluginWithSourceConverter from modmail.addons.models import Plugin, SourceTypeEnum -from modmail.addons.plugins import BASE_PATH, PLUGINS, walk_plugins +from modmail.addons.plugins import BASE_PATH, PLUGINS, find_plugins_in_zip, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModes, ExtMetadata @@ -20,7 +21,7 @@ from modmail.log import ModmailLogger EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION) -VALID_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] + logger: ModmailLogger = logging.getLogger(__name__) @@ -118,105 +119,36 @@ async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConvert await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") return logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.zip_url}") - async with self.bot.http_session.get(f"https://{plugin.source.zip_url}") as resp: - if resp.status != 200: - await ctx.send(f"Downloading {plugin.source.zip_url} did not give a 200") - return - raw_bytes = await resp.read() - if plugin.source.source_type is SourceTypeEnum.REPO: - file_name = f"{plugin.source.githost}/{plugin.source.user}/{plugin.source.repo}" - elif plugin.source.source_type is SourceTypeEnum.ZIP: - file_name = plugin.source.path.rstrip(".zip") - else: - raise TypeError("Unsupported source detected.") - - zipfile_path = BASE_PATH / ".cache" / f"{file_name}.zip" - - plugin.source.cache_file = zipfile_path - - if not zipfile_path.exists(): - zipfile_path.parent.mkdir(parents=True, exist_ok=True) + try: + file = await addon_utils.download_zip_from_source(plugin.source, self.bot.http_session) + except errors.HTTPException: + await ctx.send(f"Downloading {plugin.source.zip_url} did not give a 200 response code.") + return else: - # overwriting an existing file - logger.info("Zip file already exists, overwriting it.") - - with zipfile_path.open("wb") as f: - f.write(raw_bytes) - await ctx.send(f"Downloaded {zipfile_path}") - - file = zipfile.ZipFile(zipfile_path) - print(file.namelist()) - file.printdir() - print(file.infolist()) - print("-" * 50) - _temp_direct_children = [p for p in zipfile.Path(file).iterdir()] - if len(_temp_direct_children) == 1: - # only one folder, so we probably want to recurse into it - _folder = _temp_direct_children[0] - if _folder.is_dir(): - # the zip folder needs to have the extra directory removed, - # and everything moved up a directory. - temp_archive = BASE_PATH / ".restructure.zip" - temp_archive = zipfile.ZipFile(temp_archive, mode="w") - for path in file.infolist(): - logger.trace(f"File name: {path.filename}") - if (new_name := path.filename.split("/", 1)[-1]) == "": - continue - temp_archive.writestr(new_name, file.read(path)) - # given that we are writing to disk, we want to ensure - # that we yield to anything that needs the event loop - await asyncio.sleep(0) - temp_archive.close() - os.replace(temp_archive.filename, file.filename) - - # reset the file so we ensure we have the new archive open + file = zipfile.ZipFile(file.filename) + await ctx.send(f"Downloaded {file.filename}") + + temp_direct_children = [p for p in zipfile.Path(file).iterdir()] + if len(temp_direct_children) == 1: + folder = temp_direct_children[0] + if folder.is_dir(): + addon_utils.move_zip_contents_up_a_level(file.filename, temp_direct_children) file.close() - print(zipfile_path) - print(file.filename) - file = zipfile.ZipFile(zipfile_path) - - # TODO: REMOVE THIS SECTION - # extract the archive - file.extractall(BASE_PATH / ".extraction") + file = zipfile.ZipFile(file.filename) # determine plugins in the archive - archive_plugin_directory = None - for dir in VALID_PLUGIN_DIRECTORIES: - dir = dir + "/" - if dir in file.namelist(): - archive_plugin_directory = dir - break - if archive_plugin_directory is None: - await ctx.send("Looks like there isn't a valid plugin here.") - return + top_level_plugins, all_plugin_files = find_plugins_in_zip(file.filename) - archive_plugin_directory = zipfile.Path(file, at=archive_plugin_directory) - print(archive_plugin_directory) - lil_pluggies = [] - for path in archive_plugin_directory.iterdir(): - logger.debug(f"archive_plugin_directory: {path}") - if path.is_dir(): - lil_pluggies.append(archive_plugin_directory.name + "/" + path.name + "/") - - logger.debug(f"Plugins detected: {lil_pluggies}") - all_lil_pluggies = lil_pluggies - files = file.namelist() - for pluggy in all_lil_pluggies: - for f in files: - if f == pluggy: - continue - if f.startswith(pluggy): - all_lil_pluggies.append(f) - print(f) + # yield to any coroutines that need to run + await asyncio.sleep(0) # extract the drive - these_plugins_dir = BASE_PATH / file_name - print(file.namelist()) - file.extractall(these_plugins_dir, all_lil_pluggies) + file.extractall(BASE_PATH / plugin.source.addon_directory, all_plugin_files) - # TODO: rewrite this only need to scan the new directory + # TODO: rewrite this as it only needs to (and should) scan the new directory self._resync_extensions() - temp_new_plugins = [x.strip("/").rsplit("/", 1)[1] for x in lil_pluggies] + + temp_new_plugins = [x.strip("/").rsplit("/", 1)[1] for x in all_plugin_files] new_plugins = [] for p in temp_new_plugins: logger.debug(p) @@ -226,6 +158,7 @@ async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConvert pass self.batch_manage(Action.LOAD, *new_plugins) + await ctx.reply("Installed plugins: \n" + "\n".join(top_level_plugins)) # TODO: Implement install/enable/disable/etc From ca919dff4d919e7532bf17bbce6ca7c686097aab Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 17:03:52 -0400 Subject: [PATCH 016/100] dependencies: use atoml --- poetry.lock | 18 +++++++++++++++--- pyproject.toml | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9b45d696..c996c292 100644 --- a/poetry.lock +++ b/poetry.lock @@ -66,6 +66,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "atoml" +version = "1.0.3" +description = "Yet another style preserving TOML library" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "attrs" version = "21.2.0" @@ -252,7 +260,7 @@ toml = ["toml"] [[package]] name = "discord.py" -version = "2.0.0a3470+gfeae059c" +version = "2.0.0a3575+g45d498c1" description = "A Python wrapper for the Discord API" category = "main" optional = false @@ -271,7 +279,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] type = "git" url = "https://github.com/Rapptz/discord.py.git" reference = "master" -resolved_reference = "feae059c6858e419552ec4096f1ad2692bb4c484" +resolved_reference = "45d498c1b76deaf3b394d17ccf56112fa691d160" [[package]] name = "distlib" @@ -1198,7 +1206,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b2fe2e490de66438a4a582dbe69e4b9f1417af90f47d928dc59ef6686a22c01a" +content-hash = "bb6e3b11d7d3353fb87a7c3152506415b0ffe09aa978e12652db92c3b411b6fb" [metadata.files] aiodns = [ @@ -1260,6 +1268,10 @@ atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] +atoml = [ + {file = "atoml-1.0.3-py3-none-any.whl", hash = "sha256:944c0e9043ca4e0729d4125132841ef1110677b8d015a624892d63cdc4988655"}, + {file = "atoml-1.0.3.tar.gz", hash = "sha256:5dd70efcafde94a6aa5db2e8c6af5d832bf95b38f47d3283ee3779e920218e94"}, +] attrs = [ {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, diff --git a/pyproject.toml b/pyproject.toml index 4c017b9a..1b02aeb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ packages = [{ include = "modmail" }] python = "^3.8" aiohttp = { extras = ["speedups"], version = "^3.7.4" } arrow = "^1.1.1" +atoml = "^1.0.3" colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { git = "https://github.com/Rapptz/discord.py.git", rev = "master" } From 5be54b0e9847c25c166e0625bc288aca8fb6134b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 22:40:31 -0400 Subject: [PATCH 017/100] rework: remove sources from being part of a plugin --- modmail/addons/converters.py | 14 ++--- modmail/addons/models.py | 24 +++------ modmail/extensions/plugin_manager.py | 24 +++++---- tests/docs.md | 70 ------------------------- tests/modmail/addons/test_converters.py | 9 ++-- tests/modmail/addons/test_models.py | 51 +----------------- 6 files changed, 31 insertions(+), 161 deletions(-) diff --git a/modmail/addons/converters.py b/modmail/addons/converters.py index 14f38378..1f10c2e7 100644 --- a/modmail/addons/converters.py +++ b/modmail/addons/converters.py @@ -2,7 +2,7 @@ import logging import re -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING, Tuple, Type from discord.ext import commands @@ -39,26 +39,26 @@ async def convert(self, ctx: Context, argument: str) -> None: raise NotImplementedError("Inheriting classes must overwrite this method.") -class PluginWithSourceConverter(AddonConverter): +class SourceAndPluginConverter(AddonConverter): """A plugin converter that takes a source, addon name, and returns a Plugin.""" - async def convert(self, _: Context, argument: str) -> Plugin: + async def convert(self, _: Context, argument: str) -> Tuple[Plugin, AddonSource]: """Convert a provided plugin and source to a Plugin.""" match = LOCAL_REGEX.match(argument) if match is not None: logger.debug("Matched as a local file, creating a Plugin without a source url.") source = AddonSource(None, SourceTypeEnum.LOCAL) - return Plugin(name=match.group("addon"), source=source) + return Plugin(name=match.group("addon")), source match = ZIP_REGEX.fullmatch(argument) if match is not None: logger.debug("Matched as a zip, creating a Plugin from zip.") - return Plugin.from_zip(match.group("addon"), match.group("url")) + source = AddonSource.from_zip(match.group("url")) + return Plugin(match.group("addon")), source match = REPO_REGEX.fullmatch(argument) if match is None: raise commands.BadArgument(f"{argument} is not a valid source and plugin.") - return Plugin.from_repo( - match.group("addon"), + return Plugin(match.group("addon")), AddonSource.from_repo( match.group("user"), match.group("repo"), match.group("reflike"), diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 11921bd1..c9f298a2 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -115,7 +115,6 @@ class Addon: name: str description: Optional[str] - source: AddonSource min_version: str def __init__(self): @@ -125,26 +124,15 @@ def __init__(self): class Plugin(Addon): """An addon which is a plugin.""" - def __init__(self, name: str, source: AddonSource, **kw) -> Plugin: + if TYPE_CHECKING: + folder: Union[str, pathlib.Path, zipfile.Path] + + def __init__(self, name: str, **kw) -> Plugin: self.name = name - self.source = source self.description = kw.get("description", None) + self.folder = kw.get("folder", None) self.min_version = kw.get("min_version", None) self.enabled = kw.get("enabled", True) - @classmethod - def from_repo( - cls, addon: str, user: str, repo: str, reflike: str = None, githost: Optional[Host] = "github" - ) -> Plugin: - """Create a Plugin from a repository regex match.""" - source = AddonSource.from_repo(user, repo, reflike, githost) - return cls(addon, source) - - @classmethod - def from_zip(cls, addon: str, url: str) -> Plugin: - """Create a Plugin from a zip regex match.""" - source = AddonSource.from_zip(url) - return cls(addon, source) - def __repr__(self) -> str: # pragma: no cover - return f"" + return f"" diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index cc82c303..78083740 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -10,8 +10,8 @@ import modmail.addons.utils as addon_utils from modmail import errors -from modmail.addons.converters import PluginWithSourceConverter -from modmail.addons.models import Plugin, SourceTypeEnum +from modmail.addons.converters import SourceAndPluginConverter +from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum from modmail.addons.plugins import BASE_PATH, PLUGINS, find_plugins_in_zip, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModes, ExtMetadata @@ -106,23 +106,25 @@ async def resync_plugins(self, ctx: Context) -> None: await self.resync_extensions.callback(self, ctx) @plugins_group.command("convert", hidden=True) - async def plugin_convert_test(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: + async def plugin_convert_test(self, ctx: Context, *, plugin: SourceAndPluginConverter) -> None: """Convert a plugin and given its source information.""" await ctx.send(f"```py\n{plugin.__repr__()}```") @plugins_group.command(name="install", aliases=("",)) - async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConverter) -> None: + async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPluginConverter) -> None: """Install plugins from provided repo.""" - plugin: Plugin = plugin - if plugin.source.source_type is SourceTypeEnum.LOCAL: + plugin: Plugin + source: AddonSource + plugin, source = source_and_plugin + if source.source_type is SourceTypeEnum.LOCAL: # TODO: check the path of a local plugin await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") return - logger.debug(f"Received command to download plugin {plugin.name} from {plugin.source.zip_url}") + logger.debug(f"Received command to download plugin {plugin.name} from {source.zip_url}") try: - file = await addon_utils.download_zip_from_source(plugin.source, self.bot.http_session) - except errors.HTTPException: - await ctx.send(f"Downloading {plugin.source.zip_url} did not give a 200 response code.") + file = await addon_utils.download_zip_from_source(source, self.bot.http_session) + except errors.HTTPError: + await ctx.send(f"Downloading {source.zip_url} did not give a 200 response code.") return else: file = zipfile.ZipFile(file.filename) @@ -143,7 +145,7 @@ async def install_plugins(self, ctx: Context, *, plugin: PluginWithSourceConvert await asyncio.sleep(0) # extract the drive - file.extractall(BASE_PATH / plugin.source.addon_directory, all_plugin_files) + file.extractall(BASE_PATH / source.addon_directory, all_plugin_files) # TODO: rewrite this as it only needs to (and should) scan the new directory self._resync_extensions() diff --git a/tests/docs.md b/tests/docs.md index 75812f76..5a45183d 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -354,76 +354,6 @@ Create a plugin model, and ensure it has the right properties. **Markers:** - parametrize (name['earth', 'mona-lisa']) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_repo_match -Test that a plugin can be created from a repo. - -**Markers:** -- parametrize (user, repo, name, reflike, githost[('onerandomusername', 'addons', 'planet', None, 'github'), ('onerandomusername', 'addons', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'v1.0.2', 'gitlab'), ('onerandomusername', 'repo', 'planet', 'master', 'github'), ('onerandomusername', 'repo', 'planet', 'main', 'gitlab'), ('onerandomusername', 'repo', 'planet', None, 'github'), ('onerandomusername', 'repo', 'planet', None, 'gitlab'), ('psf', 'black', 'black', '21.70b', 'github')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) -### test_plugin_from_zip -Test that a plugin can be created from a zip url. - -**Markers:** -- parametrize (url, addon[('github.com/onerandomusername/modmail-addons/archive/main.zip', 'planet'), ('gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip', 'earth'), ('example.com/bleeeep.zip', 'myanmar'), ('github.com/discord-modmail/addons/archive/bast.zip', 'thebot'), ('rtfd.io/plugs.zip', 'documentation'), ('pages.dev/hiy.zip', 'black')]) # tests.modmail.utils.test_embeds ## ### test_patch_embed diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index 51b05497..95369673 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -5,17 +5,14 @@ import pytest -# fmt: off from modmail.addons.converters import ( REPO_REGEX, ZIP_REGEX, AddonConverter, - PluginWithSourceConverter, + SourceAndPluginConverter, SourceTypeEnum, ) -# fmt: on - @pytest.mark.asyncio @pytest.mark.xfail(reason="Not implemented") @@ -212,6 +209,6 @@ def test_zip_regex(entry, url, domain, path, addon) -> None: @pytest.mark.asyncio async def test_plugin_with_source_converter(entry: str, name: str, source_type: SourceTypeEnum) -> None: """Test the Plugin converter works, and successfully converts a plugin with its source.""" - plugin = await PluginWithSourceConverter().convert(None, entry) + plugin, source = await SourceAndPluginConverter().convert(None, entry) assert plugin.name == name - assert plugin.source.source_type == source_type + assert source.source_type == source_type diff --git a/tests/modmail/addons/test_models.py b/tests/modmail/addons/test_models.py index 1114dfc9..c5807db4 100644 --- a/tests/modmail/addons/test_models.py +++ b/tests/modmail/addons/test_models.py @@ -67,59 +67,12 @@ def test_addonsource_from_zip(url): assert src.source_type == SourceTypeEnum.ZIP -@pytest.fixture(name="source_fixture") -def addonsource_fixture(): - """Addonsource fixture for tests. The contents of this source do not matter, as long as they are valid.""" - return AddonSource("github.com/bast0006.zip", SourceTypeEnum.ZIP) - - class TestPlugin: """Test the Plugin class creation.""" @pytest.mark.parametrize("name", [("earth"), ("mona-lisa")]) - def test_plugin_init(self, name, source_fixture): + def test_plugin_init(self, name): """Create a plugin model, and ensure it has the right properties.""" - plugin = Plugin(name, source_fixture) + plugin = Plugin(name) assert isinstance(plugin, Plugin) assert plugin.name == name - - @pytest.mark.parametrize( - "user, repo, name, reflike, githost", - [ - ("onerandomusername", "addons", "planet", None, "github"), - ("onerandomusername", "addons", "planet", "master", "github"), - ("onerandomusername", "repo", "planet", "v1.0.2", "gitlab"), - ("onerandomusername", "repo", "planet", "master", "github"), - ("onerandomusername", "repo", "planet", "main", "gitlab"), - ("onerandomusername", "repo", "planet", None, "github"), - ("onerandomusername", "repo", "planet", None, "gitlab"), - ("psf", "black", "black", "21.70b", "github"), - ], - ) - def test_plugin_from_repo_match(self, user, repo, name, reflike, githost): - """Test that a plugin can be created from a repo.""" - plug = Plugin.from_repo(name, user, repo, reflike, githost) - assert plug.name == name - assert plug.source.user == user - assert plug.source.repo == repo - assert plug.source.reflike == reflike - assert plug.source.githost == githost - assert plug.source.source_type == SourceTypeEnum.REPO - - @pytest.mark.parametrize( - "url, addon", - [ - ("github.com/onerandomusername/modmail-addons/archive/main.zip", "planet"), - ("gitlab.com/onerandomusername/modmail-addons/-/archive/main/modmail-addons-main.zip", "earth"), - ("example.com/bleeeep.zip", "myanmar"), - ("github.com/discord-modmail/addons/archive/bast.zip", "thebot"), - ("rtfd.io/plugs.zip", "documentation"), - ("pages.dev/hiy.zip", "black"), - ], - ) - def test_plugin_from_zip(self, url, addon): - """Test that a plugin can be created from a zip url.""" - plug = Plugin.from_zip(addon, url) - assert plug.name == addon - assert plug.source.zip_url == url - assert plug.source.source_type == SourceTypeEnum.ZIP From 7ada61cb8fd3ed4d4c0e35ac2d37b0a51a36d79a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 22:42:43 -0400 Subject: [PATCH 018/100] plugins: add method to get list of plugins from toml --- modmail/addons/plugins.py | 41 +++++++++++++++++++-------- tests/docs.md | 7 +++++ tests/modmail/addons/test_plugins.py | 42 ++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 tests/modmail/addons/test_plugins.py diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 91569dc3..7c4dd662 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -6,20 +6,23 @@ TODO: Expand file to download plugins from github and gitlab from a list that is passed. """ - +from __future__ import annotations import glob import importlib import importlib.util import inspect import logging +import pathlib import typing as t import zipfile -from pathlib import Path -from zipfile import ZipFile +from typing import List + +import atoml from modmail import plugins from modmail.addons.errors import NoPluginDirectoryError +from modmail.addons.models import Plugin from modmail.log import ModmailLogger from modmail.utils.cogs import ExtMetadata from modmail.utils.extensions import BOT_MODE, unqualify @@ -27,19 +30,35 @@ logger: ModmailLogger = logging.getLogger(__name__) VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] -BASE_PATH = Path(plugins.__file__).parent.resolve() +BASE_PATH = pathlib.Path(plugins.__file__).parent.resolve() PLUGIN_MODULE = "modmail.plugins" PLUGINS: t.Dict[str, t.Tuple[bool, bool]] = dict() -def find_plugins_in_zip(zip_path: t.Union[str, Path]) -> t.Tuple[t.List[str], t.List[str]]: +def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plugin]: + """Parses a plugin toml, given the string loaded in.""" + doc = atoml.parse(unparsed_plugin_toml_str) + found_plugins: List[Plugin] = [] + for plug_entry in doc["plugins"]: + found_plugins.append( + Plugin( + plug_entry["name"], + folder=plug_entry["folder"], + description=plug_entry["description"], + min_bot_version=plug_entry["min_bot_version"], + ) + ) + return found_plugins + + +def find_plugins_in_zip(zip_path: t.Union[str, pathlib.Path]) -> t.Tuple[t.List[str], t.List[str]]: """ Find the plugins that are in a zip file. All plugins in a zip folder will be located at either `Plugins/` or `plugins/` """ archive_plugin_directory = None - file = ZipFile(zip_path) + file = zipfile.ZipFile(zip_path) for dir in VALID_ZIP_PLUGIN_DIRECTORIES: dir = dir + "/" if dir in file.namelist(): @@ -79,13 +98,13 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: # support following symlinks, see: https://bugs.python.org/issue33428 for path in glob.iglob(f"{BASE_PATH}/**/*.py", recursive=True): - logger.trace("Path: {0}".format(path)) + logger.trace(f"Path: {path}") # calculate the module name, dervived from the relative path - relative_path = Path(path).relative_to(BASE_PATH) + relative_path = pathlib.Path(path).relative_to(BASE_PATH) name = relative_path.__str__().rstrip(".py").replace("/", ".") name = PLUGIN_MODULE + "." + name - logger.trace("Module name: {0}".format(name)) + logger.trace(f"Module name: {name}") if unqualify(name.split(".")[-1]).startswith("_"): # Ignore module/package names starting with an underscore. @@ -103,14 +122,14 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: spec.loader.exec_module(imported) except Exception: logger.error( - "Failed to import {0}. As a result, this plugin is not considered installed.".format(name), + f"Failed to import {name}. As a result, this plugin is not considered installed.", exc_info=True, ) continue if not inspect.isfunction(getattr(imported, "setup", None)): # If it lacks a setup function, it's not a plugin. This is enforced by dpy. - logger.trace("{0} does not have a setup function. Skipping.".format(name)) + logger.trace(f"{name} does not have a setup function. Skipping.") continue ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) diff --git a/tests/docs.md b/tests/docs.md index 5a45183d..194f7a83 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -354,6 +354,13 @@ Create a plugin model, and ensure it has the right properties. **Markers:** - parametrize (name['earth', 'mona-lisa']) +# tests.modmail.addons.test_plugins +## +### test_parse_plugin_toml_from_string +Make sure that a plugin toml file is correctly parsed. + +**Markers:** +- parametrize (toml, name, folder, description, min_bot_version[('\n[[plugins]]\nname = "Planet"\nfolder = "planet"\ndescription = "Planet. Tells you which planet you are probably on."\nmin_bot_version = "v0.2.0"\n', 'Planet', 'planet', 'Planet. Tells you which planet you are probably on.', 'v0.2.0')]) # tests.modmail.utils.test_embeds ## ### test_patch_embed diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py new file mode 100644 index 00000000..3ce230c5 --- /dev/null +++ b/tests/modmail/addons/test_plugins.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from textwrap import dedent +from typing import List + +import pytest + +from modmail.addons.models import Plugin +from modmail.addons.plugins import parse_plugin_toml_from_string + +VALID_PLUGIN_TOML = """ +[[plugins]] +name = "Planet" +folder = "planet" +description = "Planet. Tells you which planet you are probably on." +min_bot_version = "v0.2.0" +""" + + +@pytest.mark.parametrize( + "toml, name, folder, description, min_bot_version", + [ + ( + VALID_PLUGIN_TOML, + "Planet", + "planet", + "Planet. Tells you which planet you are probably on.", + "v0.2.0", + ) + ], +) +def test_parse_plugin_toml_from_string( + toml: str, name: str, folder: str, description: str, min_bot_version: str +): + """Make sure that a plugin toml file is correctly parsed.""" + plugs = parse_plugin_toml_from_string(VALID_PLUGIN_TOML) + plug = plugs[0] + print(plug.__repr__()) + assert isinstance(plug, Plugin) + assert plug.name == name + assert plug.folder == folder + assert plug.description == description From 8720942ddf92b8ed2b85904fd3cd56b2d69165ba Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 29 Aug 2021 23:13:52 -0400 Subject: [PATCH 019/100] tests: remove skips on implemented tests --- tests/docs.md | 2 -- tests/modmail/addons/test_converters.py | 4 ++-- tests/modmail/test_logs.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/docs.md b/tests/docs.md index 194f7a83..2abb0e47 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -39,7 +39,6 @@ Test notice logging level prints a notice response. Test trace logging level prints a trace response. **Markers:** -- skip - dependency (depends=['create_logger']) # tests.modmail.addons.test_converters ## @@ -47,7 +46,6 @@ Test trace logging level prints a trace response. Convert a user input into a Source. **Markers:** -- xfail (reason=Not implemented) - asyncio ### test_repo_regex Test the repo regex to ensure that it matches what it should. diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index 95369673..b42806e8 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -15,10 +15,10 @@ @pytest.mark.asyncio -@pytest.mark.xfail(reason="Not implemented") async def test_converter() -> None: """Convert a user input into a Source.""" - addon = await AddonConverter().convert(None, "github") # noqa: F841 + with pytest.raises(NotImplementedError): + addon = await AddonConverter().convert(None, "github") # noqa: F841 # fmt: off diff --git a/tests/modmail/test_logs.py b/tests/modmail/test_logs.py index 9f03fce0..0a596a50 100644 --- a/tests/modmail/test_logs.py +++ b/tests/modmail/test_logs.py @@ -44,7 +44,6 @@ def test_notice_level(log): @pytest.mark.dependency(depends=["create_logger"]) -@pytest.mark.skip() def test_trace_level(log): """Test trace logging level prints a trace response.""" trace_test_phrase = "Getting in the weeds" From 9a69e88a2e94963467674922c74ec43a6b5ab76e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 30 Aug 2021 00:34:59 -0400 Subject: [PATCH 020/100] chore: annotate as much of tests as viable --- tests/modmail/addons/test_converters.py | 13 +++++++------ tests/modmail/addons/test_models.py | 12 +++++++----- tests/modmail/addons/test_plugins.py | 5 +---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index b42806e8..c0283c0b 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -1,7 +1,6 @@ from __future__ import annotations -from re import Match -from textwrap import dedent +from typing import Optional import pytest @@ -59,9 +58,11 @@ async def test_converter() -> None: ) ], ) -# fmt: on @pytest.mark.dependency(name="repo_regex") -def test_repo_regex(entry, user, repo, addon, reflike, githost) -> None: +# fmt: on +def test_repo_regex( + entry: str, user: str, repo: str, addon: str, reflike: Optional[str], githost: Optional[str] +) -> None: """Test the repo regex to ensure that it matches what it should.""" match = REPO_REGEX.fullmatch(entry) assert match is not None @@ -123,7 +124,7 @@ def test_repo_regex(entry, user, repo, addon, reflike, githost) -> None: ) # fmt: on @pytest.mark.dependency(name="zip_regex") -def test_zip_regex(entry, url, domain, path, addon) -> None: +def test_zip_regex(entry: str, url: str, domain: str, path: str, addon: str) -> None: """Test the repo regex to ensure that it matches what it should.""" match = ZIP_REGEX.fullmatch(entry) assert match is not None @@ -204,9 +205,9 @@ def test_zip_regex(entry, url, domain, path, addon) -> None: pytest.param("the world exists.", None, None, marks=pytest.mark.xfail), ], ) -# fmt: on @pytest.mark.dependency(depends_on=["repo_regex", "zip_regex"]) @pytest.mark.asyncio +# fmt: on async def test_plugin_with_source_converter(entry: str, name: str, source_type: SourceTypeEnum) -> None: """Test the Plugin converter works, and successfully converts a plugin with its source.""" plugin, source = await SourceAndPluginConverter().convert(None, entry) diff --git a/tests/modmail/addons/test_models.py b/tests/modmail/addons/test_models.py index c5807db4..9e2f2c02 100644 --- a/tests/modmail/addons/test_models.py +++ b/tests/modmail/addons/test_models.py @@ -1,11 +1,13 @@ from __future__ import annotations +from typing import Optional + import pytest from modmail.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum -def test_addon_model(): +def test_addon_model() -> None: """All addons will be of a specific type, so we should not be able to create a generic addon.""" with pytest.raises(NotImplementedError, match="Inheriting classes need to implement their own init"): Addon() @@ -19,7 +21,7 @@ def test_addon_model(): (None, SourceTypeEnum.LOCAL), ], ) -def test_addonsource_init(zip_url, source_type): +def test_addonsource_init(zip_url: str, source_type: SourceTypeEnum) -> None: """Test the AddonSource init sets class vars appropiately.""" addonsrc = AddonSource(zip_url, source_type) assert addonsrc.zip_url == zip_url @@ -39,7 +41,7 @@ def test_addonsource_init(zip_url, source_type): ("psf", "black", "21.70b", "github"), ], ) -def test_addonsource_from_repo(user, repo, reflike, githost): +def test_addonsource_from_repo(user: str, repo: str, reflike: Optional[str], githost: str) -> None: """Test an addon source is properly made from repository information.""" src = AddonSource.from_repo(user, repo, reflike, githost) assert src.user == user @@ -60,7 +62,7 @@ def test_addonsource_from_repo(user, repo, reflike, githost): ("pages.dev/hiy.zip"), ], ) -def test_addonsource_from_zip(url): +def test_addonsource_from_zip(url: str) -> None: """Test an addon source is properly made from a zip url.""" src = AddonSource.from_zip(url) assert src.zip_url == url @@ -71,7 +73,7 @@ class TestPlugin: """Test the Plugin class creation.""" @pytest.mark.parametrize("name", [("earth"), ("mona-lisa")]) - def test_plugin_init(self, name): + def test_plugin_init(self, name: str) -> None: """Create a plugin model, and ensure it has the right properties.""" plugin = Plugin(name) assert isinstance(plugin, Plugin) diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py index 3ce230c5..f1b9e47f 100644 --- a/tests/modmail/addons/test_plugins.py +++ b/tests/modmail/addons/test_plugins.py @@ -1,8 +1,5 @@ from __future__ import annotations -from textwrap import dedent -from typing import List - import pytest from modmail.addons.models import Plugin @@ -31,7 +28,7 @@ ) def test_parse_plugin_toml_from_string( toml: str, name: str, folder: str, description: str, min_bot_version: str -): +) -> None: """Make sure that a plugin toml file is correctly parsed.""" plugs = parse_plugin_toml_from_string(VALID_PLUGIN_TOML) plug = plugs[0] From 32aa1701be65da7f0ec1212bba808ca21899111f Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 30 Aug 2021 00:39:41 -0400 Subject: [PATCH 021/100] plugins: fix min_bot_version --- modmail/addons/models.py | 9 +++++---- tests/modmail/addons/test_plugins.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index c9f298a2..3825bfa2 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -113,9 +113,10 @@ def __repr__(self) -> str: # pragma: no cover class Addon: """Base class of an addon which make the bot extendable.""" - name: str - description: Optional[str] - min_version: str + if TYPE_CHECKING: + name: str + description: Optional[str] + min_bot_version: str def __init__(self): raise NotImplementedError("Inheriting classes need to implement their own init") @@ -131,7 +132,7 @@ def __init__(self, name: str, **kw) -> Plugin: self.name = name self.description = kw.get("description", None) self.folder = kw.get("folder", None) - self.min_version = kw.get("min_version", None) + self.min_bot_version = kw.get("min_bot_version", None) self.enabled = kw.get("enabled", True) def __repr__(self) -> str: # pragma: no cover diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py index f1b9e47f..766d7fb3 100644 --- a/tests/modmail/addons/test_plugins.py +++ b/tests/modmail/addons/test_plugins.py @@ -30,10 +30,11 @@ def test_parse_plugin_toml_from_string( toml: str, name: str, folder: str, description: str, min_bot_version: str ) -> None: """Make sure that a plugin toml file is correctly parsed.""" - plugs = parse_plugin_toml_from_string(VALID_PLUGIN_TOML) + plugs = parse_plugin_toml_from_string(toml) plug = plugs[0] print(plug.__repr__()) assert isinstance(plug, Plugin) assert plug.name == name assert plug.folder == folder assert plug.description == description + assert plug.min_bot_version == min_bot_version From 9a32c0f1c440d32f7394283d443d97ed25820ec2 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 30 Aug 2021 01:30:24 -0400 Subject: [PATCH 022/100] tests: start testing addon utils --- tests/docs.md | 7 +++++++ tests/modmail/addons/test_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/modmail/addons/test_utils.py diff --git a/tests/docs.md b/tests/docs.md index 2abb0e47..9992a602 100644 --- a/tests/docs.md +++ b/tests/docs.md @@ -359,6 +359,13 @@ Make sure that a plugin toml file is correctly parsed. **Markers:** - parametrize (toml, name, folder, description, min_bot_version[('\n[[plugins]]\nname = "Planet"\nfolder = "planet"\ndescription = "Planet. Tells you which planet you are probably on."\nmin_bot_version = "v0.2.0"\n', 'Planet', 'planet', 'Planet. Tells you which planet you are probably on.', 'v0.2.0')]) +# tests.modmail.addons.test_utils +## +### test_download_zip_from_source + +**Markers:** +- asyncio +- parametrize (source[>]) # tests.modmail.utils.test_embeds ## ### test_patch_embed diff --git a/tests/modmail/addons/test_utils.py b/tests/modmail/addons/test_utils.py new file mode 100644 index 00000000..5a66ce28 --- /dev/null +++ b/tests/modmail/addons/test_utils.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import zipfile + +import pytest +from aiohttp import ClientSession + +from modmail.addons.models import AddonSource, SourceTypeEnum +from modmail.addons.utils import download_zip_from_source + +# https://github.com/discord-modmail/modmail/archive/main.zip + + +@pytest.fixture() +@pytest.mark.asyncio +async def session() -> ClientSession: + """Fixture function for a aiohttp.ClientSession.""" + return ClientSession() + + +@pytest.mark.parametrize( + "source", [AddonSource.from_zip("https://github.com/discord-modmail/modmail/archive/main.zip")] +) +@pytest.mark.asyncio +async def test_download_zip_from_source(source: AddonSource, session: ClientSession): + """Test that a zip can be successfully downloaded and everything is safe inside.""" + file = await download_zip_from_source(source, session) + assert isinstance(file, zipfile.ZipFile) + assert file.testzip() is None From 68ed6123fb808536e94937c04d8c49766a162a85 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 31 Aug 2021 15:41:49 -0400 Subject: [PATCH 023/100] fix: close session after test --- tests/modmail/addons/test_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/modmail/addons/test_utils.py b/tests/modmail/addons/test_utils.py index 5a66ce28..7ba23e98 100644 --- a/tests/modmail/addons/test_utils.py +++ b/tests/modmail/addons/test_utils.py @@ -15,7 +15,10 @@ @pytest.mark.asyncio async def session() -> ClientSession: """Fixture function for a aiohttp.ClientSession.""" - return ClientSession() + sess = ClientSession() + yield sess + + await sess.close() @pytest.mark.parametrize( From 0aa4a9fa2c5996d27c668c4ac49c58f620d2b970 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 31 Aug 2021 22:12:39 -0400 Subject: [PATCH 024/100] chore: move plugin_helpers to modmail/plugins Moving plugin_helpers to a file within the plugin class means that it can be used with `from modmail.plugins` rather than use the weird plugin_helpers name that it previously had. --- modmail/__init__.py | 5 +++++ modmail/plugins/.gitignore | 4 +++- modmail/plugins/__init__.py | 1 + modmail/{plugin_helpers.py => plugins/helpers.py} | 6 +++--- tox.ini | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) rename modmail/{plugin_helpers.py => plugins/helpers.py} (76%) diff --git a/modmail/__init__.py b/modmail/__init__.py index 14cd73a5..64769450 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -57,3 +57,8 @@ logging.getLogger("websockets").setLevel(logging.ERROR) # Set asyncio logging back to the default of INFO even if asyncio's debug mode is enabled. logging.getLogger("asyncio").setLevel(logging.INFO) + +# now that the logger is configured, we can import the bot for using as all and typing +from modmail.bot import ModmailBot # noqa: E402 + +__all__ = [ModmailBot, ModmailLogger] diff --git a/modmail/plugins/.gitignore b/modmail/plugins/.gitignore index 4be6c40b..676818ea 100644 --- a/modmail/plugins/.gitignore +++ b/modmail/plugins/.gitignore @@ -8,5 +8,7 @@ local/** !local/ !local/README.md -# ensure this file is uploaded so `plugins` is considered a module +# ensure __init__.py is uploaded so `plugins` is considered a module !/__init__.py +# keep our helper file in here +!/helpers.py diff --git a/modmail/plugins/__init__.py b/modmail/plugins/__init__.py index e69de29b..43bbaeae 100644 --- a/modmail/plugins/__init__.py +++ b/modmail/plugins/__init__.py @@ -0,0 +1 @@ +from .helpers import BotModes, ExtMetadata, PluginCog diff --git a/modmail/plugin_helpers.py b/modmail/plugins/helpers.py similarity index 76% rename from modmail/plugin_helpers.py rename to modmail/plugins/helpers.py index d20abf8e..b5b89323 100644 --- a/modmail/plugin_helpers.py +++ b/modmail/plugins/helpers.py @@ -1,8 +1,8 @@ -from modmail.bot import ModmailBot -from modmail.log import ModmailLogger +from __future__ import annotations + from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog -__all__ = ["PluginCog", ModmailBot, ModmailLogger, BotModes, ExtMetadata] +__all__ = ["PluginCog", BotModes, ExtMetadata] class PluginCog(ModmailCog): diff --git a/tox.ini b/tox.ini index dc78f84a..15974e42 100644 --- a/tox.ini +++ b/tox.ini @@ -26,8 +26,9 @@ ignore= # Whitespace Before E203 per-file-ignores= - tests/*:,ANN,S101,F401 + tests/*:ANN,S101,F401 docs.py:B008 + modmail/**/__init__.py:F401 [isort] profile=black From 5052b61528d6993d207192db518dc0fcd0d0d920 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 1 Sep 2021 17:23:57 -0400 Subject: [PATCH 025/100] dependencies: drop toml for atoml --- modmail/config.py | 9 ++++++--- poetry.lock | 2 +- pyproject.toml | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modmail/config.py b/modmail/config.py index 6f139626..16ae81df 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any, Dict, Optional, Tuple +import atoml import discord -import toml from discord.ext.commands import BadArgument from pydantic import BaseModel from pydantic import BaseSettings as PydanticBaseSettings @@ -54,7 +54,9 @@ def toml_default_config_source(settings: PydanticBaseSettings) -> Dict[str, Any] Here we happen to choose to use the `env_file_encoding` from Config when reading `config-default.toml` """ - return dict(**toml.load(DEFAULT_CONFIG_PATH)) + with open(DEFAULT_CONFIG_PATH) as f: + + return atoml.loads(f.read()).value def toml_user_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: @@ -66,7 +68,8 @@ def toml_user_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: when reading `config-default.toml` """ if USER_CONFIG_PATH: - return dict(**toml.load(USER_CONFIG_PATH)) + with open(USER_CONFIG_PATH) as f: + return atoml.loads(f.read()).value else: return dict() diff --git a/poetry.lock b/poetry.lock index d15fe66b..dbe549b7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1134,7 +1134,7 @@ test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybi name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" diff --git a/pyproject.toml b/pyproject.toml index 2868c5a4..01688fd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } pydantic = { version = "^1.8.2", extras = ["dotenv"] } -toml = "^0.10.2" [tool.poetry.extras] From dbb15115406588f244d91001323b73c27a9ecd37 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 1 Sep 2021 19:47:36 -0400 Subject: [PATCH 026/100] fix: don't use None when typing __init__ return --- modmail/addons/models.py | 8 ++++---- modmail/addons/plugins.py | 3 ++- modmail/errors.py | 2 +- modmail/extensions/plugin_manager.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 3825bfa2..038cc9da 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -2,7 +2,7 @@ import re from enum import Enum -from typing import TYPE_CHECKING, Literal, Optional, Union +from typing import TYPE_CHECKING, Literal, NoReturn, Optional, Union if TYPE_CHECKING: import pathlib @@ -66,7 +66,7 @@ class AddonSource: addon_directory: Optional[str] cache_file: Optional[Union[zipfile.Path, pathlib.Path]] - def __init__(self, zip_url: str, type: SourceTypeEnum) -> AddonSource: + def __init__(self, zip_url: str, type: SourceTypeEnum): """Initialize the AddonSource.""" self.zip_url = zip_url if self.zip_url is not None: @@ -118,7 +118,7 @@ class Addon: description: Optional[str] min_bot_version: str - def __init__(self): + def __init__(self) -> NoReturn: raise NotImplementedError("Inheriting classes need to implement their own init") @@ -128,7 +128,7 @@ class Plugin(Addon): if TYPE_CHECKING: folder: Union[str, pathlib.Path, zipfile.Path] - def __init__(self, name: str, **kw) -> Plugin: + def __init__(self, name: str, **kw): self.name = name self.description = kw.get("description", None) self.folder = kw.get("folder", None) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 7c4dd662..f513c00a 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -29,10 +29,11 @@ logger: ModmailLogger = logging.getLogger(__name__) -VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] + BASE_PATH = pathlib.Path(plugins.__file__).parent.resolve() PLUGIN_MODULE = "modmail.plugins" PLUGINS: t.Dict[str, t.Tuple[bool, bool]] = dict() +VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plugin]: diff --git a/modmail/errors.py b/modmail/errors.py index e60400fe..788eed92 100644 --- a/modmail/errors.py +++ b/modmail/errors.py @@ -4,5 +4,5 @@ class HTTPError(Exception): """Response from an http request was not desired.""" - def __init__(self, response: ClientResponse) -> None: + def __init__(self, response: ClientResponse): self.response = response diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index df1a51de..68e40a9d 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -43,7 +43,7 @@ class PluginManager(ExtensionManager, name="Plugin Manager"): type = "plugin" module_name = "plugins" # modmail/plugins - def __init__(self, bot: ModmailBot) -> None: + def __init__(self, bot: ModmailBot): super().__init__(bot) self.all_extensions = PLUGINS self.refresh_method = walk_plugins From 5adac3a0821a3df2c5280660a58d4edea69562da Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 1 Sep 2021 20:08:57 -0400 Subject: [PATCH 027/100] minor: don't use kwargs --- modmail/addons/models.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 038cc9da..f6934b4a 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -2,7 +2,7 @@ import re from enum import Enum -from typing import TYPE_CHECKING, Literal, NoReturn, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Literal, NoReturn, Optional, Union if TYPE_CHECKING: import pathlib @@ -127,13 +127,28 @@ class Plugin(Addon): if TYPE_CHECKING: folder: Union[str, pathlib.Path, zipfile.Path] - - def __init__(self, name: str, **kw): + extra_kwargs = Dict[str, Any] + + def __init__( + self, + name: str, + description: Optional[str] = None, + *, + min_bot_version: Optional[str] = None, + folder: Optional[str] = None, + enabled: bool = True, + **kw, + ): self.name = name - self.description = kw.get("description", None) - self.folder = kw.get("folder", None) - self.min_bot_version = kw.get("min_bot_version", None) - self.enabled = kw.get("enabled", True) + self.description = description + self.folder = folder or name + self.min_bot_version = min_bot_version + self.enabled = enabled + + # store any extra kwargs here + # this is to ensure backwards compatiablilty with plugins that support older versions, + # but want to use newer toml options + self.extra_kwargs = kw def __repr__(self) -> str: # pragma: no cover return f"" From f2ce9f3f7d74a40b6221d0103b53e74670146926 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 4 Sep 2021 00:30:21 -0400 Subject: [PATCH 028/100] refactor plugin downloading to be easier to read and use --- modmail/addons/plugins.py | 55 ++++++++++++++++------------ modmail/extensions/plugin_manager.py | 34 +++++++++++------ 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index f513c00a..38247537 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -14,9 +14,8 @@ import inspect import logging import pathlib -import typing as t import zipfile -from typing import List +from typing import Dict, Iterator, List, Tuple, Union import atoml @@ -29,11 +28,11 @@ logger: ModmailLogger = logging.getLogger(__name__) +VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] BASE_PATH = pathlib.Path(plugins.__file__).parent.resolve() -PLUGIN_MODULE = "modmail.plugins" -PLUGINS: t.Dict[str, t.Tuple[bool, bool]] = dict() -VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] + +PLUGINS: Dict[str, Tuple[bool, bool]] = dict() def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plugin]: @@ -52,45 +51,53 @@ def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plug return found_plugins -def find_plugins_in_zip(zip_path: t.Union[str, pathlib.Path]) -> t.Tuple[t.List[str], t.List[str]]: +def find_plugins_in_zip(zip_path: Union[str, pathlib.Path]) -> Dict[str, List[str]]: """ Find the plugins that are in a zip file. All plugins in a zip folder will be located at either `Plugins/` or `plugins/` """ - archive_plugin_directory = None file = zipfile.ZipFile(zip_path) - for dir in VALID_ZIP_PLUGIN_DIRECTORIES: - dir = dir + "/" - if dir in file.namelist(): - archive_plugin_directory = dir + + # figure out which directory plugins are in. Both Plugins and plugins are supported. + # default is plugins. + archive_plugin_directory = None + for dir_ in VALID_ZIP_PLUGIN_DIRECTORIES: + dir_ = dir_ + "/" # zipfile require `/` after the path if its a directory + if dir_ in file.namelist(): + archive_plugin_directory = dir_ break + if archive_plugin_directory is None: raise NoPluginDirectoryError(f"No {' or '.join(VALID_ZIP_PLUGIN_DIRECTORIES)} directory exists.") + + # convert archive_plugin_directory into a zip file object from the path it contains. archive_plugin_directory = zipfile.Path(file, at=archive_plugin_directory) - lil_pluggies = [] + + all_plugins: Dict[str, List[str]] = {} + for path in archive_plugin_directory.iterdir(): logger.debug(f"archive_plugin_directory: {path}") if path.is_dir(): - lil_pluggies.append(archive_plugin_directory.name + "/" + path.name + "/") + plugin_name = archive_plugin_directory.name + "/" + path.name + "/" + all_plugins[plugin_name] = list() - logger.debug(f"Plugins detected: {lil_pluggies}") - all_lil_pluggies = lil_pluggies.copy() + logger.debug(f"Plugins detected: {all_plugins.keys()}") files = file.namelist() - for pluggy in all_lil_pluggies: + for pluggy in all_plugins.keys(): for f in files: - if f == pluggy: + if f == pluggy: # don't include files that are plugin directories continue if f.startswith(pluggy): - all_lil_pluggies.append(f) - print(f) - logger.trace(f"lil_pluggies: {lil_pluggies}") - logger.trace(f"all_lil_pluggies: {all_lil_pluggies}") + all_plugins[pluggy].append(f) + logger.trace(f"{f = }") + logger.debug(f"{all_plugins.keys() = }") + logger.debug(f"{all_plugins.values() = }") - return lil_pluggies, all_lil_pluggies + return all_plugins -def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: +def walk_plugins() -> Iterator[Tuple[str, bool]]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder # this is to ensure folder symlinks are supported, @@ -104,7 +111,7 @@ def walk_plugins() -> t.Iterator[t.Tuple[str, bool]]: # calculate the module name, dervived from the relative path relative_path = pathlib.Path(path).relative_to(BASE_PATH) name = relative_path.__str__().rstrip(".py").replace("/", ".") - name = PLUGIN_MODULE + "." + name + name = plugins.__name__ + "." + name logger.trace(f"Module name: {name}") if unqualify(name.split(".")[-1]).startswith("_"): diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 68e40a9d..6950ed24 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -3,7 +3,7 @@ import asyncio import logging import zipfile -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List from discord.ext import commands from discord.ext.commands import Context @@ -140,28 +140,40 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu file = zipfile.ZipFile(file.filename) # determine plugins in the archive - top_level_plugins, all_plugin_files = find_plugins_in_zip(file.filename) + plugins = find_plugins_in_zip(file.filename) # yield to any coroutines that need to run + # its not possible to do this with aiofiles, so when we export the zip, + # its important to yield right after await asyncio.sleep(0) # extract the drive - file.extractall(BASE_PATH / source.addon_directory, all_plugin_files) + all_files = [] + for k, v in plugins.items(): + all_files.append(k) + all_files.extend(v) + file.extractall(BASE_PATH / source.addon_directory, all_files) - # TODO: rewrite this as it only needs to (and should) scan the new directory + # TODO: rewrite this method as it only needs to (and should) scan the new directory self._resync_extensions() - temp_new_plugins = [x.strip("/").rsplit("/", 1)[1] for x in all_plugin_files] - new_plugins = [] - for p in temp_new_plugins: - logger.debug(p) + # figure out all of the files + temp_new_plugin_files: List[str] = [p.strip("/").rsplit("/", 1)[1] for p in all_files] + new_plugin_files: List[str] = [] + logger.debug(f"{temp_new_plugin_files = }") + for plug in temp_new_plugin_files: + logger.trace(f"{plug = }") try: - new_plugins.append(await PluginPathConverter().convert(None, p)) + plug = await PluginPathConverter().convert(None, plug) except commands.BadArgument: pass + else: + if plug in PLUGINS: + new_plugin_files.append(plug) - self.batch_manage(Action.LOAD, *new_plugins) - await ctx.reply("Installed plugins: \n" + "\n".join(top_level_plugins)) + logger.debug(f"{new_plugin_files = }") + self.batch_manage(Action.LOAD, *new_plugin_files) + await ctx.reply("Installed plugins: \n" + "\n".join(plugins.keys())) # TODO: Implement install/enable/disable/etc From cf098178412b80276ecddc734ec6ab447e2ae8cf Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 4 Sep 2021 01:13:36 -0400 Subject: [PATCH 029/100] chore: use temp directory for caching zips --- modmail/addons/plugins.py | 6 +++--- modmail/addons/utils.py | 16 ++++++++++++---- modmail/extensions/extension_manager.py | 2 -- modmail/extensions/plugin_manager.py | 4 ++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 38247537..6204785e 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -30,7 +30,7 @@ VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] -BASE_PATH = pathlib.Path(plugins.__file__).parent.resolve() +BASE_PLUGIN_PATH = pathlib.Path(plugins.__file__).parent.resolve() PLUGINS: Dict[str, Tuple[bool, bool]] = dict() @@ -104,12 +104,12 @@ def walk_plugins() -> Iterator[Tuple[str, bool]]: # which are important for ease of development. # NOTE: We are not using Pathlib's glob utility as it doesn't # support following symlinks, see: https://bugs.python.org/issue33428 - for path in glob.iglob(f"{BASE_PATH}/**/*.py", recursive=True): + for path in glob.iglob(f"{BASE_PLUGIN_PATH}/**/*.py", recursive=True): logger.trace(f"Path: {path}") # calculate the module name, dervived from the relative path - relative_path = pathlib.Path(path).relative_to(BASE_PATH) + relative_path = pathlib.Path(path).relative_to(BASE_PLUGIN_PATH) name = relative_path.__str__().rstrip(".py").replace("/", ".") name = plugins.__name__ + "." + name logger.trace(f"Module name: {name}") diff --git a/modmail/addons/utils.py b/modmail/addons/utils.py index 137d3874..7735d236 100644 --- a/modmail/addons/utils.py +++ b/modmail/addons/utils.py @@ -3,11 +3,12 @@ import logging import os import pathlib +import random +import tempfile import zipfile from typing import TYPE_CHECKING, Union from modmail.addons.models import SourceTypeEnum -from modmail.addons.plugins import BASE_PATH from modmail.errors import HTTPError if TYPE_CHECKING: @@ -17,6 +18,8 @@ from modmail.log import ModmailLogger logger: ModmailLogger = logging.getLogger(__name__) +TEMP_DIR = pathlib.Path(tempfile.gettempdir()) + def move_zip_contents_up_a_level(zip_path: Union[str, pathlib.Path], folder: str = None) -> None: """ @@ -26,7 +29,7 @@ def move_zip_contents_up_a_level(zip_path: Union[str, pathlib.Path], folder: str of the zip folder. """ file = zipfile.ZipFile(zip_path) - temp_archive = BASE_PATH / ".tmp.zip" # temporary folder for moving + temp_archive = TEMP_DIR / f"modmail-{random.getrandbits(8)}.zip" temp_archive = zipfile.ZipFile(temp_archive, mode="w") for path in file.infolist(): logger.trace(f"File name: {path.filename}") @@ -37,7 +40,9 @@ def move_zip_contents_up_a_level(zip_path: Union[str, pathlib.Path], folder: str os.replace(temp_archive.filename, file.filename) -async def download_zip_from_source(source: AddonSource, session: ClientSession) -> zipfile.ZipFile: +async def download_zip_from_source( + source: AddonSource, session: ClientSession, zipfile_path: Union[str, pathlib.Path] = None +) -> zipfile.ZipFile: """ Download a zip file from a source. @@ -54,7 +59,10 @@ async def download_zip_from_source(source: AddonSource, session: ClientSession) else: raise TypeError("Unsupported source detected.") - zipfile_path = BASE_PATH / ".cache" / f"{file_name}.zip" + if zipfile_path is None: + zipfile_path = TEMP_DIR / "modmail-zips" / f"{hash(file_name)}.zip" + else: + zipfile_path = pathlib.Path(zipfile_path) source.addon_directory = file_name source.cache_file = zipfile_path diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index c072a806..86ab90db 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -20,8 +20,6 @@ log: ModmailLogger = logging.getLogger(__name__) -BASE_PATH_LEN = __name__.count(".") - EXT_METADATA = ExtMetadata(load_if_mode=BotModes.DEVELOP, no_unload=True) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 6950ed24..634bdd95 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -12,7 +12,7 @@ from modmail import errors from modmail.addons.converters import SourceAndPluginConverter from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum -from modmail.addons.plugins import BASE_PATH, PLUGINS, find_plugins_in_zip, walk_plugins +from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_zip, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModes, ExtMetadata @@ -152,7 +152,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu for k, v in plugins.items(): all_files.append(k) all_files.extend(v) - file.extractall(BASE_PATH / source.addon_directory, all_files) + file.extractall(BASE_PLUGIN_PATH / source.addon_directory, all_files) # TODO: rewrite this method as it only needs to (and should) scan the new directory self._resync_extensions() From afd4b186e1fed53a93017808cbbe436327f30e1e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 4 Sep 2021 21:32:37 -0400 Subject: [PATCH 030/100] plugin-install: refactor to use zip stream and only install the provided plugin --- modmail/addons/errors.py | 14 ++++- modmail/addons/models.py | 1 - modmail/addons/plugins.py | 87 ++++++++++++++++------------ modmail/addons/utils.py | 69 +++++++++------------- modmail/extensions/plugin_manager.py | 71 +++++++++++++---------- 5 files changed, 128 insertions(+), 114 deletions(-) diff --git a/modmail/addons/errors.py b/modmail/addons/errors.py index c0a9810e..d8d79ed0 100644 --- a/modmail/addons/errors.py +++ b/modmail/addons/errors.py @@ -4,7 +4,19 @@ class AddonError(Exception): pass -class NoPluginDirectoryError(AddonError): +class PluginError(AddonError): + """General Plugin error.""" + + pass + + +class NoPluginDirectoryError(PluginError): """No plugin directory exists.""" pass + + +class PluginNotFoundError(PluginError): + """Plugins are not found and can therefore not be actioned on.""" + + pass diff --git a/modmail/addons/models.py b/modmail/addons/models.py index f6934b4a..93d84698 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -63,7 +63,6 @@ class AddonSource: domain: Optional[str] path: Optional[str] - addon_directory: Optional[str] cache_file: Optional[Union[zipfile.Path, pathlib.Path]] def __init__(self, zip_url: str, type: SourceTypeEnum): diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 6204785e..d9e3712d 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -13,9 +13,9 @@ import importlib.util import inspect import logging +import os import pathlib -import zipfile -from typing import Dict, Iterator, List, Tuple, Union +from typing import Dict, Iterator, List, Tuple import atoml @@ -51,46 +51,59 @@ def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plug return found_plugins -def find_plugins_in_zip(zip_path: Union[str, pathlib.Path]) -> Dict[str, List[str]]: +def find_plugins_in_dir(addon_repo_path: pathlib.Path) -> Dict[pathlib.Path, List[pathlib.Path]]: """ - Find the plugins that are in a zip file. + Find the plugins that are in a directory. All plugins in a zip folder will be located at either `Plugins/` or `plugins/` - """ - file = zipfile.ZipFile(zip_path) + Returns a dict containing all of the plugin folders as keys + and the values as lists of the files within those folders. + """ + temp_direct_children = [p for p in addon_repo_path.iterdir()] + if len(temp_direct_children) == 1: + folder = temp_direct_children[0] + if folder.is_dir(): + addon_repo_path = addon_repo_path / folder + del temp_direct_children # figure out which directory plugins are in. Both Plugins and plugins are supported. # default is plugins. - archive_plugin_directory = None - for dir_ in VALID_ZIP_PLUGIN_DIRECTORIES: - dir_ = dir_ + "/" # zipfile require `/` after the path if its a directory - if dir_ in file.namelist(): - archive_plugin_directory = dir_ + plugin_directory = None + direct_children = [p for p in addon_repo_path.iterdir()] + logger.debug(f"{direct_children = }") + for path_ in direct_children: + if path_.name.rsplit("/", 1)[-1] in VALID_ZIP_PLUGIN_DIRECTORIES: + plugin_directory = path_ break - if archive_plugin_directory is None: + if plugin_directory is None: + logger.debug(f"{direct_children = }") raise NoPluginDirectoryError(f"No {' or '.join(VALID_ZIP_PLUGIN_DIRECTORIES)} directory exists.") - # convert archive_plugin_directory into a zip file object from the path it contains. - archive_plugin_directory = zipfile.Path(file, at=archive_plugin_directory) + plugin_directory = addon_repo_path / plugin_directory - all_plugins: Dict[str, List[str]] = {} + all_plugins: Dict[pathlib.Path, List[pathlib.Path]] = {} - for path in archive_plugin_directory.iterdir(): - logger.debug(f"archive_plugin_directory: {path}") + for path in plugin_directory.iterdir(): + logger.debug(f"plugin_directory: {path}") if path.is_dir(): - plugin_name = archive_plugin_directory.name + "/" + path.name + "/" - all_plugins[plugin_name] = list() - - logger.debug(f"Plugins detected: {all_plugins.keys()}") - files = file.namelist() - for pluggy in all_plugins.keys(): - for f in files: - if f == pluggy: # don't include files that are plugin directories - continue - if f.startswith(pluggy): - all_plugins[pluggy].append(f) - logger.trace(f"{f = }") + all_plugins[path] = list() + + logger.debug(f"Plugins detected: {[p.name for p in all_plugins.keys()]}") + + for plugin_path in all_plugins.keys(): + logger.trace(f"{plugin_path =}") + for dirpath, dirnames, filenames in os.walk(plugin_path): + logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") + for list_ in dirnames, filenames: + logger.trace(f"{list_ =}") + for file in list_: + logger.trace(f"{file =}") + if file == dirpath: # don't include files that are plugin directories + continue + + all_plugins[plugin_path].append(pathlib.Path(file)) + logger.debug(f"{all_plugins.keys() = }") logger.debug(f"{all_plugins.values() = }") @@ -106,15 +119,15 @@ def walk_plugins() -> Iterator[Tuple[str, bool]]: # support following symlinks, see: https://bugs.python.org/issue33428 for path in glob.iglob(f"{BASE_PLUGIN_PATH}/**/*.py", recursive=True): - logger.trace(f"Path: {path}") + logger.trace(f"{path =}") # calculate the module name, dervived from the relative path relative_path = pathlib.Path(path).relative_to(BASE_PLUGIN_PATH) - name = relative_path.__str__().rstrip(".py").replace("/", ".") - name = plugins.__name__ + "." + name - logger.trace(f"Module name: {name}") + module_name = relative_path.__str__().rstrip(".py").replace("/", ".") + module_name = plugins.__name__ + "." + module_name + logger.trace(f"{module_name =}") - if unqualify(name.split(".")[-1]).startswith("_"): + if unqualify(module_name.split(".")[-1]).startswith("_"): # Ignore module/package names starting with an underscore. continue @@ -125,19 +138,19 @@ def walk_plugins() -> Iterator[Tuple[str, bool]]: # load the plugins using importlib # this needs to be done like this, due to the fact that # its possible a plugin will not have an __init__.py file - spec = importlib.util.spec_from_file_location(name, path) + spec = importlib.util.spec_from_file_location(module_name, path) imported = importlib.util.module_from_spec(spec) spec.loader.exec_module(imported) except Exception: logger.error( - f"Failed to import {name}. As a result, this plugin is not considered installed.", + f"Failed to import {module_name}. As a result, this plugin is not considered installed.", exc_info=True, ) continue if not inspect.isfunction(getattr(imported, "setup", None)): # If it lacks a setup function, it's not a plugin. This is enforced by dpy. - logger.trace(f"{name} does not have a setup function. Skipping.") + logger.trace(f"{module_name} does not have a setup function. Skipping.") continue ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) diff --git a/modmail/addons/utils.py b/modmail/addons/utils.py index 7735d236..79154f98 100644 --- a/modmail/addons/utils.py +++ b/modmail/addons/utils.py @@ -1,12 +1,11 @@ from __future__ import annotations +import io import logging -import os import pathlib -import random import tempfile import zipfile -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from modmail.addons.models import SourceTypeEnum from modmail.errors import HTTPError @@ -21,59 +20,43 @@ TEMP_DIR = pathlib.Path(tempfile.gettempdir()) -def move_zip_contents_up_a_level(zip_path: Union[str, pathlib.Path], folder: str = None) -> None: +def unpack_zip(zip_file: zipfile.ZipFile, path: pathlib.Path = None) -> pathlib.Path: """ - Assuming that there is only one folder, move everything up a level. + Unpack a zip file and return its new path. - If a folder is provided, it will attempt to use that folder. This folder *must* be located in the root - of the zip folder. + If path is provided, the zip will be unpacked to that path. + If path is not provided, a file in the platform's temp directory will be used. """ - file = zipfile.ZipFile(zip_path) - temp_archive = TEMP_DIR / f"modmail-{random.getrandbits(8)}.zip" - temp_archive = zipfile.ZipFile(temp_archive, mode="w") - for path in file.infolist(): - logger.trace(f"File name: {path.filename}") - if (new_name := path.filename.split("/", 1)[-1]) == "": - continue - temp_archive.writestr(new_name, file.read(path)) - temp_archive.close() - os.replace(temp_archive.filename, file.filename) - - -async def download_zip_from_source( - source: AddonSource, session: ClientSession, zipfile_path: Union[str, pathlib.Path] = None -) -> zipfile.ZipFile: + if path is None: + path = TEMP_DIR / "modmail-addons" / f"zip-{hash(zip_file)}" + + zip_file.extractall(path=path) + return path + + +async def download_zip_from_source(source: AddonSource, session: ClientSession) -> zipfile.ZipFile: """ Download a zip file from a source. It is currently required to provide an http session. """ + if source.source_type not in (SourceTypeEnum.REPO, SourceTypeEnum.ZIP): + raise TypeError("Unsupported source detected.") + async with session.get(f"https://{source.zip_url}") as resp: if resp.status != 200: raise HTTPError(resp) raw_bytes = await resp.read() - if source.source_type is SourceTypeEnum.REPO: - file_name = f"{source.githost}/{source.user}/{source.repo}" - elif source.source_type is SourceTypeEnum.ZIP: - file_name = source.path.rstrip(".zip") - else: - raise TypeError("Unsupported source detected.") - - if zipfile_path is None: - zipfile_path = TEMP_DIR / "modmail-zips" / f"{hash(file_name)}.zip" - else: - zipfile_path = pathlib.Path(zipfile_path) - source.addon_directory = file_name - source.cache_file = zipfile_path + zip_stream = io.BytesIO(raw_bytes) + zip_stream.write(raw_bytes) - if not zipfile_path.exists(): - zipfile_path.parent.mkdir(parents=True, exist_ok=True) - else: - # overwriting an existing file - logger.info("Zip file already exists, overwriting it.") + return zipfile.ZipFile(zip_stream) - with zipfile_path.open("wb") as f: - f.write(raw_bytes) - return zipfile.ZipFile(zipfile_path) +async def download_and_unpack_source( + source: AddonSource, session: ClientSession, path: pathlib.Path = None +) -> pathlib.Path: + """Downloads and unpacks source.""" + zip_file = await download_zip_from_source(source, session) + return unpack_zip(zip_file, path) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 634bdd95..0d9903ee 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -2,7 +2,7 @@ import asyncio import logging -import zipfile +import shutil from typing import TYPE_CHECKING, List from discord.ext import commands @@ -11,8 +11,9 @@ import modmail.addons.utils as addon_utils from modmail import errors from modmail.addons.converters import SourceAndPluginConverter +from modmail.addons.errors import PluginNotFoundError from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum -from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_zip, walk_plugins +from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModes, ExtMetadata @@ -114,68 +115,74 @@ async def plugin_convert_test(self, ctx: Context, *, plugin: SourceAndPluginConv @plugins_group.command(name="install", aliases=("",)) async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPluginConverter) -> None: """Install plugins from provided repo.""" + # this could take a while + await ctx.trigger_typing() + + # create variables for the user input, typehint them, then assign them from the converter tuple plugin: Plugin source: AddonSource plugin, source = source_and_plugin + if source.source_type is SourceTypeEnum.LOCAL: # TODO: check the path of a local plugin await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") return logger.debug(f"Received command to download plugin {plugin.name} from {source.zip_url}") try: - file = await addon_utils.download_zip_from_source(source, self.bot.http_session) + directory = await addon_utils.download_and_unpack_source(source, self.bot.http_session) except errors.HTTPError: await ctx.send(f"Downloading {source.zip_url} did not give a 200 response code.") return - else: - file = zipfile.ZipFile(file.filename) - await ctx.send(f"Downloaded {file.filename}") - temp_direct_children = [p for p in zipfile.Path(file).iterdir()] - if len(temp_direct_children) == 1: - folder = temp_direct_children[0] - if folder.is_dir(): - addon_utils.move_zip_contents_up_a_level(file.filename, temp_direct_children) - file.close() - file = zipfile.ZipFile(file.filename) + source.cache_file = directory # determine plugins in the archive - plugins = find_plugins_in_zip(file.filename) + plugins = find_plugins_in_dir(directory) # yield to any coroutines that need to run # its not possible to do this with aiofiles, so when we export the zip, # its important to yield right after await asyncio.sleep(0) - # extract the drive - all_files = [] - for k, v in plugins.items(): - all_files.append(k) - all_files.extend(v) - file.extractall(BASE_PLUGIN_PATH / source.addon_directory, all_files) + # copy the requested plugin over to the new folder + plugin_path = None + for p in plugins.keys(): + if p.name == plugin.folder: + plugin_path = p + try: + shutil.copytree(p, BASE_PLUGIN_PATH / p.name, dirs_exist_ok=True) + except shutil.FileExistsError: + await ctx.send( + "Plugin already seems to be installed. " + "This could be caused by the plugin already existing, " + "or a plugin of the same name existing." + ) + return + break + + if plugin_path is None: + raise PluginNotFoundError(f"Could not find plugin {plugin}") + logger.trace(f"{BASE_PLUGIN_PATH = }") # TODO: rewrite this method as it only needs to (and should) scan the new directory self._resync_extensions() - - # figure out all of the files - temp_new_plugin_files: List[str] = [p.strip("/").rsplit("/", 1)[1] for p in all_files] - new_plugin_files: List[str] = [] - logger.debug(f"{temp_new_plugin_files = }") - for plug in temp_new_plugin_files: + files_to_load: List[str] = [] + for plug in plugins[plugin_path]: logger.trace(f"{plug = }") try: - plug = await PluginPathConverter().convert(None, plug) + plug = await PluginPathConverter().convert(None, plug.name.rstrip(".py")) except commands.BadArgument: pass else: if plug in PLUGINS: - new_plugin_files.append(plug) + files_to_load.append(plug) + + logger.debug(f"{files_to_load = }") + self.batch_manage(Action.LOAD, *files_to_load) - logger.debug(f"{new_plugin_files = }") - self.batch_manage(Action.LOAD, *new_plugin_files) - await ctx.reply("Installed plugins: \n" + "\n".join(plugins.keys())) + await ctx.reply(f"Installed plugin {plugin_path.name}.") - # TODO: Implement install/enable/disable/etc + # TODO: Implement enable/disable/etc # This cannot be static (must have a __func__ attribute). async def cog_check(self, ctx: Context) -> bool: From 45037976c748214da1505c8fdfc7fd03edfe721a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 4 Sep 2021 23:37:53 -0400 Subject: [PATCH 031/100] minor: remove admins from being able to manage plugins This permission can be restored once it is able to check permissions on a designated guild This could be used by people adding the bot if a user makes the bot public by mistake --- modmail/extensions/plugin_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 0d9903ee..07189acd 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -190,7 +190,7 @@ async def cog_check(self, ctx: Context) -> bool: if ctx.guild is None: return await self.bot.is_owner(ctx.author) else: - return ctx.author.guild_permissions.administrator or await self.bot.is_owner(ctx.author) + return await self.bot.is_owner(ctx.author) # HACK: Delete the commands from ExtensionManager that PluginManager has inherited From 8501636341d4b6efd9d5c279058674f2c6799690 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 02:06:22 -0400 Subject: [PATCH 032/100] plugins: allow spaces in plugin names, and dashes in refs --- modmail/addons/converters.py | 2 +- tests/modmail/addons/test_converters.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modmail/addons/converters.py b/modmail/addons/converters.py index 1f10c2e7..dab3e74d 100644 --- a/modmail/addons/converters.py +++ b/modmail/addons/converters.py @@ -23,7 +23,7 @@ # gitlab allows 255 characters in the username, and 255 max in a project path # see https://gitlab.com/gitlab-org/gitlab/-/issues/197976 for more details r"(?P[a-zA-Z0-9][a-zA-Z0-9\-]{0,254})\/(?P[\w\-\.]{1,100}) " - r"(?P[^@\s]+)(?: \@(?P[\w\.\s]*))?$" + r"(?P[^@]+[^\s@])(?: \@(?P[\w\.\-\S]*))?" ) logger: ModmailLogger = logging.getLogger(__name__) diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index c0283c0b..e61e9f6d 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -40,6 +40,10 @@ async def test_converter() -> None: "github onerandomusername/repo planet @master", "onerandomusername", "repo", "planet", "master", "github", ), + ( + "github onerandomusername/repo planet @bad-toml", + "onerandomusername", "repo", "planet", "bad-toml", "github", + ), ( "gitlab onerandomusername/repo planet @main", "onerandomusername", "repo", "planet", "main", "gitlab", From eb73921acc4c81f926c167b265923e389b8e9a18 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 02:08:20 -0400 Subject: [PATCH 033/100] parse plugin toml for plugin metadata --- modmail/addons/errors.py | 4 ++ modmail/addons/models.py | 19 +++++++-- modmail/addons/plugins.py | 57 +++++++++++++++++++------ modmail/extensions/extension_manager.py | 2 + modmail/extensions/plugin_manager.py | 15 +++---- modmail/utils/__init__.py | 15 +++++++ tests/modmail/addons/test_plugins.py | 2 +- 7 files changed, 89 insertions(+), 25 deletions(-) diff --git a/modmail/addons/errors.py b/modmail/addons/errors.py index d8d79ed0..cddf4115 100644 --- a/modmail/addons/errors.py +++ b/modmail/addons/errors.py @@ -20,3 +20,7 @@ class PluginNotFoundError(PluginError): """Plugins are not found and can therefore not be actioned on.""" pass + + +class NoPluginTomlFoundError(PluginError): + """Raised when a plugin.toml file is expected to exist but does not exist.""" diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 93d84698..86db1aca 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -4,6 +4,8 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Literal, NoReturn, Optional, Union +from modmail.utils import MISSING + if TYPE_CHECKING: import pathlib import zipfile @@ -125,7 +127,8 @@ class Plugin(Addon): """An addon which is a plugin.""" if TYPE_CHECKING: - folder: Union[str, pathlib.Path, zipfile.Path] + folder_name: Optional[str] + folder_path: Optional[pathlib.Path] extra_kwargs = Dict[str, Any] def __init__( @@ -134,13 +137,21 @@ def __init__( description: Optional[str] = None, *, min_bot_version: Optional[str] = None, - folder: Optional[str] = None, + folder: Optional[str] = MISSING, + folder_path: Optional[pathlib.Path] = None, enabled: bool = True, **kw, ): self.name = name self.description = description - self.folder = folder or name + if folder is MISSING: + if folder_path is not None: + self.folder_name = folder or folder_path.name or name + else: + self.folder_name = folder or name + else: + self.folder_name = folder + self.folder_path = folder_path self.min_bot_version = min_bot_version self.enabled = enabled @@ -150,4 +161,4 @@ def __init__( self.extra_kwargs = kw def __repr__(self) -> str: # pragma: no cover - return f"" + return f"" diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index d9e3712d..a468571a 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -20,7 +20,7 @@ import atoml from modmail import plugins -from modmail.addons.errors import NoPluginDirectoryError +from modmail.addons.errors import NoPluginDirectoryError, NoPluginTomlFoundError from modmail.addons.models import Plugin from modmail.log import ModmailLogger from modmail.utils.cogs import ExtMetadata @@ -34,29 +34,39 @@ PLUGINS: Dict[str, Tuple[bool, bool]] = dict() +PLUGIN_TOML = "plugin.toml" + def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plugin]: """Parses a plugin toml, given the string loaded in.""" doc = atoml.parse(unparsed_plugin_toml_str) found_plugins: List[Plugin] = [] for plug_entry in doc["plugins"]: + found_plugins.append( Plugin( plug_entry["name"], - folder=plug_entry["folder"], - description=plug_entry["description"], - min_bot_version=plug_entry["min_bot_version"], + folder=plug_entry.get("folder"), + description=plug_entry.get("description"), + min_bot_version=plug_entry.get("min_bot_version"), ) ) return found_plugins -def find_plugins_in_dir(addon_repo_path: pathlib.Path) -> Dict[pathlib.Path, List[pathlib.Path]]: +def find_plugins_in_dir( + addon_repo_path: pathlib.Path, + *, + parse_toml: bool = True, + no_toml_exist_ok: bool = True, +) -> Dict[Plugin, List[pathlib.Path]]: """ Find the plugins that are in a directory. All plugins in a zip folder will be located at either `Plugins/` or `plugins/` + If parse_toml is true, if the plugin.toml file is found, it will be parsed. + Returns a dict containing all of the plugin folders as keys and the values as lists of the files within those folders. """ @@ -82,18 +92,41 @@ def find_plugins_in_dir(addon_repo_path: pathlib.Path) -> Dict[pathlib.Path, Lis plugin_directory = addon_repo_path / plugin_directory - all_plugins: Dict[pathlib.Path, List[pathlib.Path]] = {} - + all_plugins: Dict[Plugin, List[pathlib.Path]] = {} + + toml_plugins: List[Plugin] = [] + if parse_toml: + toml_path = plugin_directory / PLUGIN_TOML + if toml_path.exists(): + # parse the toml + with open(toml_path) as toml_file: + toml_plugins = parse_plugin_toml_from_string(toml_file.read()) + elif no_toml_exist_ok: + # toml does not exist but the caller does not care + pass + else: + raise NoPluginTomlFoundError(toml_path, "does not exist") + + logger.debug(f"{toml_plugins =}") + toml_plugin_names = [p.folder_name for p in toml_plugins] for path in plugin_directory.iterdir(): logger.debug(f"plugin_directory: {path}") if path.is_dir(): - all_plugins[path] = list() + # use an existing toml plugin object + if path.name in toml_plugin_names: + for p in toml_plugins: + if p.folder_name == path.name: + p.folder_path = path + all_plugins[p] = list() + else: + temp_plugin = Plugin(path.name, folder_path=path) + all_plugins[temp_plugin] = list() logger.debug(f"Plugins detected: {[p.name for p in all_plugins.keys()]}") - for plugin_path in all_plugins.keys(): - logger.trace(f"{plugin_path =}") - for dirpath, dirnames, filenames in os.walk(plugin_path): + for plugin_ in all_plugins.keys(): + logger.trace(f"{plugin_.folder_path =}") + for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") for list_ in dirnames, filenames: logger.trace(f"{list_ =}") @@ -102,7 +135,7 @@ def find_plugins_in_dir(addon_repo_path: pathlib.Path) -> Dict[pathlib.Path, Lis if file == dirpath: # don't include files that are plugin directories continue - all_plugins[plugin_path].append(pathlib.Path(file)) + all_plugins[plugin_].append(pathlib.Path(file)) logger.debug(f"{all_plugins.keys() = }") logger.debug(f"{all_plugins.values() = }") diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 86ab90db..7b283120 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -305,6 +305,8 @@ async def cog_command_error(self, ctx: Context, error: Exception) -> None: if isinstance(error, commands.BadArgument): await ctx.send(str(error), allowed_mentions=AllowedMentions.none()) error.handled = True + else: + raise error def setup(bot: ModmailBot) -> None: diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 07189acd..71a24291 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -145,29 +145,28 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu await asyncio.sleep(0) # copy the requested plugin over to the new folder - plugin_path = None for p in plugins.keys(): - if p.name == plugin.folder: - plugin_path = p + if p.name == plugin.name: try: - shutil.copytree(p, BASE_PLUGIN_PATH / p.name, dirs_exist_ok=True) - except shutil.FileExistsError: + shutil.copytree(p.folder_path, BASE_PLUGIN_PATH / p.folder_path.name, dirs_exist_ok=True) + except FileExistsError: await ctx.send( "Plugin already seems to be installed. " "This could be caused by the plugin already existing, " "or a plugin of the same name existing." ) return + plugin = p break - if plugin_path is None: + if plugin.folder_path is None: raise PluginNotFoundError(f"Could not find plugin {plugin}") logger.trace(f"{BASE_PLUGIN_PATH = }") # TODO: rewrite this method as it only needs to (and should) scan the new directory self._resync_extensions() files_to_load: List[str] = [] - for plug in plugins[plugin_path]: + for plug in plugins[plugin]: logger.trace(f"{plug = }") try: plug = await PluginPathConverter().convert(None, plug.name.rstrip(".py")) @@ -180,7 +179,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu logger.debug(f"{files_to_load = }") self.batch_manage(Action.LOAD, *files_to_load) - await ctx.reply(f"Installed plugin {plugin_path.name}.") + await ctx.reply(f"Installed plugin {plugin.name}.") # TODO: Implement enable/disable/etc diff --git a/modmail/utils/__init__.py b/modmail/utils/__init__.py index e69de29b..259de944 100644 --- a/modmail/utils/__init__.py +++ b/modmail/utils/__init__.py @@ -0,0 +1,15 @@ +from typing import Any + + +class _MissingSentinel: + def __eq__(self, other: Any): + return False + + def __bool__(self): + return False + + def __repr__(self): + return "..." + + +MISSING: Any = _MissingSentinel() diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py index 766d7fb3..e319f1b5 100644 --- a/tests/modmail/addons/test_plugins.py +++ b/tests/modmail/addons/test_plugins.py @@ -35,6 +35,6 @@ def test_parse_plugin_toml_from_string( print(plug.__repr__()) assert isinstance(plug, Plugin) assert plug.name == name - assert plug.folder == folder + assert plug.folder_name == folder assert plug.description == description assert plug.min_bot_version == min_bot_version From 005fdb51c159ff396423dfdb7af6847c8d81b18d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 02:10:13 -0400 Subject: [PATCH 034/100] chore: prevent plugin manager from being unloaded --- modmail/extensions/plugin_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 71a24291..d049a2df 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -21,7 +21,7 @@ from modmail.bot import ModmailBot from modmail.log import ModmailLogger -EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION) +EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION, no_unload=True) logger: ModmailLogger = logging.getLogger(__name__) From fd252c6f27abe6a5fc729ae5fcae9c524bcdd860 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 14:35:35 -0400 Subject: [PATCH 035/100] remove toml from requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a009c8d0..e32cd079 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,5 @@ pyreadline==2.1; python_version >= "2.7" and python_full_version < "3.0.0" and s python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" python-dotenv==0.19.0; python_full_version >= "3.6.1" and python_version >= "3.5" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") -typing-extensions==3.10.0.0; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" +typing-extensions==3.10.0.2; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" yarl==1.6.3; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" From 52e2fe09d46ff11579953ae4c221cbfea02eae5e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 14:39:18 -0400 Subject: [PATCH 036/100] tests: use pytest.mark.raises instead of xfail --- poetry.lock | 46 +++++++++++++++++++++---- pyproject.toml | 5 +-- tests/modmail/addons/test_converters.py | 7 +++- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 59778184..0eb9c7d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -277,6 +277,7 @@ voice = ["PyNaCl (>=1.3.0,<1.5)"] [package.source] type = "url" url = "https://github.com/Rapptz/discord.py/archive/master.zip" + [[package]] name = "distlib" version = "0.3.2" @@ -951,6 +952,20 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" py = "*" pytest = ">=3.10" +[[package]] +name = "pytest-raises" +version = "0.11" +description = "An implementation of pytest.raises as a pytest.mark fixture" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pytest = ">=3.2.2" + +[package.extras] +develop = ["pylint", "pytest-cov"] + [[package]] name = "pytest-sugar" version = "0.9.4" @@ -1213,7 +1228,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "13d426f2143ed7a3fb374ff734603e7dd6e16bf259834232f9fb6c056f11f2bb" +content-hash = "b31abbde606fe9e5b9e54afc101b5dc4f244d9b1b9f5e7af615fa5df079a8194" [metadata.files] aiodns = [ @@ -1373,11 +1388,6 @@ cffi = [ {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, - {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, - {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, - {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, - {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, @@ -1590,12 +1600,22 @@ markdown = [ {file = "Markdown-3.3.4.tar.gz", hash = "sha256:31b5b491868dcc87d6c24b7e3d19a0d730d59d3e46f4eea6430a321bed387a49"}, ] markupsafe = [ + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -1604,14 +1624,21 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -1621,6 +1648,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, @@ -1873,6 +1903,10 @@ pytest-forked = [ {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, ] +pytest-raises = [ + {file = "pytest-raises-0.11.tar.gz", hash = "sha256:f64a4dbcb5f89c100670fe83d87a5cd9d956586db461c5c628f7eb94b749c90b"}, + {file = "pytest_raises-0.11-py2.py3-none-any.whl", hash = "sha256:33a1351f2debb9f74ca6ef70e374899f608a1217bf13ca4a0767f37b49e9cdda"}, +] pytest-sugar = [ {file = "pytest-sugar-0.9.4.tar.gz", hash = "sha256:b1b2186b0a72aada6859bea2a5764145e3aaa2c1cfbb23c3a19b5f7b697563d3"}, ] diff --git a/pyproject.toml b/pyproject.toml index 368b70a5..84144745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ pytest = "^6.2.4" pytest-asyncio = "^0.15.1" pytest-cov = "^2.12.1" pytest-dependency = "^0.5.1" +pytest-raises = "^0.11" pytest-sugar = "^0.9.4" pytest-xdist = { version = "^2.3.0", extras = ["psutil"] } # Documentation @@ -93,7 +94,7 @@ help = "Export installed packages in requirements.txt format" [tool.taskipy.tasks] # Documentation -docs = { cmd = "mkdocs serve", help = "Run the docs on a local automatically reloading server"} +docs = { cmd = "mkdocs serve", help = "Run the docs on a local automatically reloading server" } # Bot start = { cmd = "python -m modmail", help = "Run bot" } @@ -106,7 +107,7 @@ lint = { cmd = "pre-commit run --all-files", help = "Checks all files for CI err # Testing codecov-validate = { cmd = "curl --data-binary @.codecov.yml https://codecov.io/validate", help = "Validate `.codecov.yml` with their api." } -cov-server = { cmd = "coverage html", help = "Start an http.server for viewing coverage data."} +cov-server = { cmd = "coverage html", help = "Start an http.server for viewing coverage data." } post_cov-server = "python -m http.server 8012 --bind 127.0.0.1 --directory htmlcov" report = { cmd = "coverage report", help = "Show coverage report from previously run tests." } test = { cmd = "pytest -n auto --dist loadfile", help = "Runs tests and save results to a coverage report" } diff --git a/tests/modmail/addons/test_converters.py b/tests/modmail/addons/test_converters.py index e61e9f6d..75803e54 100644 --- a/tests/modmail/addons/test_converters.py +++ b/tests/modmail/addons/test_converters.py @@ -3,6 +3,7 @@ from typing import Optional import pytest +from discord.ext.commands.errors import BadArgument from modmail.addons.converters import ( REPO_REGEX, @@ -206,7 +207,11 @@ def test_zip_regex(entry: str, url: str, domain: str, path: str, addon: str) -> "@local earth", "earth", SourceTypeEnum.LOCAL ), - pytest.param("the world exists.", None, None, marks=pytest.mark.xfail), + pytest.param( + "the world exists.", + None, None, + marks=pytest.mark.raises(exception=BadArgument) + ), ], ) @pytest.mark.dependency(depends_on=["repo_regex", "zip_regex"]) From b032c744c34e689ce110fcd7890c3010b0771835 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 16:54:45 -0400 Subject: [PATCH 037/100] fix[plugins]: allow installation by name or folder name --- modmail/extensions/plugin_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 231e251b..16d91ebd 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -147,7 +147,8 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu # copy the requested plugin over to the new folder for p in plugins.keys(): - if p.name == plugin.name: + # check if user-provided plugin matches either plugin name or folder name + if plugin.name in (p.name, p.folder_name): try: shutil.copytree(p.folder_path, BASE_PLUGIN_PATH / p.folder_path.name, dirs_exist_ok=True) except FileExistsError: From 795a611bce05ac5e1ecc2a2641ec156f5826d4e3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 19:58:43 -0400 Subject: [PATCH 038/100] add a parameter to walk only a sub folder --- modmail/addons/plugins.py | 4 ++-- modmail/extensions/plugin_manager.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index dd6b0565..931eb37d 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -144,14 +144,14 @@ def find_plugins_in_dir( return all_plugins -def walk_plugins() -> Iterator[Tuple[str, bool]]: +def walk_plugins(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterator[Tuple[str, bool]]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder # this is to ensure folder symlinks are supported, # which are important for ease of development. # NOTE: We are not using Pathlib's glob utility as it doesn't # support following symlinks, see: https://bugs.python.org/issue33428 - for path in glob.iglob(f"{BASE_PLUGIN_PATH}/**/*.py", recursive=True): + for path in glob.iglob(f"{detection_path}/**/*.py", recursive=True): logger.trace(f"{path =}") diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 16d91ebd..a68167bb 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -165,8 +165,8 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu raise PluginNotFoundError(f"Could not find plugin {plugin}") logger.trace(f"{BASE_PLUGIN_PATH = }") - # TODO: rewrite this method as it only needs to (and should) scan the new directory - self._resync_extensions() + PLUGINS.update(walk_plugins(BASE_PLUGIN_PATH / p.folder_path.name)) + files_to_load: List[str] = [] for plug in plugins[plugin]: logger.trace(f"{plug = }") From d8a059c6bd2c5eb4b6b192142d16e6b71402481d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 22:07:54 -0400 Subject: [PATCH 039/100] minor: BotModes -> BotModeEnum --- modmail/extensions/extension_manager.py | 4 ++-- modmail/extensions/plugin_manager.py | 4 ++-- modmail/plugins/__init__.py | 2 +- modmail/plugins/helpers.py | 4 ++-- modmail/utils/cogs.py | 8 ++++---- modmail/utils/extensions.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index cd33dbab..5a2755b2 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -13,7 +13,7 @@ from modmail.bot import ModmailBot from modmail.log import ModmailLogger -from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog +from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator @@ -21,7 +21,7 @@ log: ModmailLogger = logging.getLogger(__name__) -EXT_METADATA = ExtMetadata(load_if_mode=BotModes.DEVELOP, no_unload=True) +EXT_METADATA = ExtMetadata(load_if_mode=BotModeEnum.DEVELOP, no_unload=True) class Action(Enum): diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index a68167bb..33cce4bb 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -15,14 +15,14 @@ from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager -from modmail.utils.cogs import BotModes, ExtMetadata +from modmail.utils.cogs import BotModeEnum, ExtMetadata if TYPE_CHECKING: from modmail.bot import ModmailBot from modmail.log import ModmailLogger -EXT_METADATA = ExtMetadata(load_if_mode=BotModes.PRODUCTION, no_unload=True) +EXT_METADATA = ExtMetadata(load_if_mode=BotModeEnum.PRODUCTION, no_unload=True) logger: ModmailLogger = logging.getLogger(__name__) diff --git a/modmail/plugins/__init__.py b/modmail/plugins/__init__.py index 43bbaeae..3125625e 100644 --- a/modmail/plugins/__init__.py +++ b/modmail/plugins/__init__.py @@ -1 +1 @@ -from .helpers import BotModes, ExtMetadata, PluginCog +from .helpers import BotModeEnum, ExtMetadata, PluginCog diff --git a/modmail/plugins/helpers.py b/modmail/plugins/helpers.py index 054fd905..0de98e98 100644 --- a/modmail/plugins/helpers.py +++ b/modmail/plugins/helpers.py @@ -1,9 +1,9 @@ from __future__ import annotations -from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog +from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog -__all__ = ["PluginCog", BotModes, ExtMetadata] +__all__ = ["PluginCog", BotModeEnum, ExtMetadata] class PluginCog(ModmailCog): diff --git a/modmail/utils/cogs.py b/modmail/utils/cogs.py index 5d5a92b8..5fcfb7c5 100644 --- a/modmail/utils/cogs.py +++ b/modmail/utils/cogs.py @@ -12,7 +12,7 @@ def _generate_next_value_(name, start, count, last_values) -> int: # noqa: ANN0 return 1 << count -class BotModes(BitwiseAutoEnum): +class BotModeEnum(BitwiseAutoEnum): """ Valid modes for the bot. @@ -24,18 +24,18 @@ class BotModes(BitwiseAutoEnum): PLUGIN_DEV = auto() -BOT_MODES = BotModes +BOT_MODES = BotModeEnum @dataclass() class ExtMetadata: """Ext metadata class to determine if extension should load at runtime depending on bot configuration.""" - load_if_mode: int = BotModes.PRODUCTION + load_if_mode: int = BotModeEnum.PRODUCTION # this is to determine if the cog is allowed to be unloaded. no_unload: bool = False - def __init__(self, load_if_mode: int = BotModes.PRODUCTION, no_unload: bool = False) -> "ExtMetadata": + def __init__(self, load_if_mode: int = BotModeEnum.PRODUCTION, no_unload: bool = False) -> "ExtMetadata": self.load_if_mode = load_if_mode self.no_unload = no_unload diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index be547c50..44348f26 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -10,7 +10,7 @@ from modmail import extensions from modmail.config import CONFIG from modmail.log import ModmailLogger -from modmail.utils.cogs import BOT_MODES, BotModes, ExtMetadata +from modmail.utils.cogs import BOT_MODES, BotModeEnum, ExtMetadata log: ModmailLogger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def determine_bot_mode() -> int: The configuration system uses true/false values, so we need to turn them into an integer for bitwise. """ bot_mode = 0 - for mode in BotModes: + for mode in BotModeEnum: if getattr(CONFIG.dev.mode, unqualify(str(mode)).lower(), True): bot_mode += mode.value return bot_mode From 2053f309f346bfb25c3c6657f0580f7a79e29379 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 22:31:19 -0400 Subject: [PATCH 040/100] remove test command --- modmail/extensions/plugin_manager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 33cce4bb..bb71ce89 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -108,11 +108,6 @@ async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) - @plugins_group.command("convert", hidden=True) - async def plugin_convert_test(self, ctx: Context, *, plugin: SourceAndPluginConverter) -> None: - """Convert a plugin and given its source information.""" - await ctx.send(f"```py\n{plugin.__repr__()}```") - @plugins_group.command(name="install", aliases=("",)) async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPluginConverter) -> None: """Install plugins from provided repo.""" From b3520718aee319963e7f8557b1494b215d6d8758 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 23:06:43 -0400 Subject: [PATCH 041/100] changes: document plugin install and uninstall system --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 14225c04..d55c3178 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Running the bot after configuring the env vars is now as simple as `docker-compose up` - Automatic docker image creation: `ghcr.io/discord-modmail/modmail` (#19) - Dockerfile support for all supported hosting providers. (#58) +- Plugin installation and uninstall system (#69) ### Changed From 997a20cd15bffb7636a78c61f6046bad0a4751e4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 5 Sep 2021 23:07:27 -0400 Subject: [PATCH 042/100] plugins: disable specific manager commands if plugin dev mode is disabled --- docs/changelog.md | 1 + modmail/extensions/plugin_manager.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d55c3178..436322a9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Running the bot is still the same method, but it loads extensions and plugins now. - `bot.start()` can also be used if already in a running event loop. Keep in mind using it will require handling loop errors, as run() does this automatically. +- Disabled some plugin management commands if PLUGIN_DEV mode is not set (#69) ### Internal diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index bb71ce89..6a17ef68 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -16,6 +16,7 @@ from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugins from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModeEnum, ExtMetadata +from modmail.utils.extensions import BOT_MODE if TYPE_CHECKING: @@ -26,6 +27,8 @@ logger: ModmailLogger = logging.getLogger(__name__) +PLUGIN_DEV_ENABLED = BOT_MODE & BotModeEnum.PLUGIN_DEV + class PluginPathConverter(ExtensionConverter): """ @@ -64,7 +67,7 @@ async def plugins_group(self, ctx: Context) -> None: """Install, uninstall, disable, update, and enable installed plugins.""" await ctx.send_help(ctx.command) - @plugins_group.command(name="load", aliases=("l",)) + @plugins_group.command(name="load", aliases=("l",), enabled=PLUGIN_DEV_ENABLED) async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Load plugins given their fully qualified or unqualified names. @@ -73,7 +76,7 @@ async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None """ await self.load_extensions.callback(self, ctx, *plugins) - @plugins_group.command(name="unload", aliases=("ul",)) + @plugins_group.command(name="unload", aliases=("ul",), enabled=PLUGIN_DEV_ENABLED) async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Unload currently loaded plugins given their fully qualified or unqualified names. @@ -82,7 +85,7 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> N """ await self.unload_extensions.callback(self, ctx, *plugins) - @plugins_group.command(name="reload", aliases=("r", "rl")) + @plugins_group.command(name="reload", aliases=("r", "rl"), enabled=PLUGIN_DEV_ENABLED) async def reload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Reload plugins given their fully qualified or unqualified names. @@ -103,7 +106,7 @@ async def list_plugins(self, ctx: Context) -> None: """ await self.list_extensions.callback(self, ctx) - @plugins_group.command(name="refresh", aliases=("rewalk", "rescan")) + @plugins_group.command(name="refresh", aliases=("rewalk", "rescan"), enabled=PLUGIN_DEV_ENABLED) async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) From fb9d20cc066ea0c146c77d6e98c731daba60124d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 6 Sep 2021 01:02:10 -0400 Subject: [PATCH 043/100] chore: rename to @ local to match the ref format --- modmail/plugins/.gitignore | 6 +++--- modmail/plugins/{local => @local}/README.md | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename modmail/plugins/{local => @local}/README.md (100%) diff --git a/modmail/plugins/.gitignore b/modmail/plugins/.gitignore index 676818ea..d316dad9 100644 --- a/modmail/plugins/.gitignore +++ b/modmail/plugins/.gitignore @@ -4,9 +4,9 @@ !/.gitignore # ignore the local folder, but not the readme -local/** -!local/ -!local/README.md +/@local/** +!/@local/ +!/@local/README.md # ensure __init__.py is uploaded so `plugins` is considered a module !/__init__.py diff --git a/modmail/plugins/local/README.md b/modmail/plugins/@local/README.md similarity index 100% rename from modmail/plugins/local/README.md rename to modmail/plugins/@local/README.md From ac46ac91bba4e357aa774cbe463a8169742bf1c0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 6 Sep 2021 01:55:22 -0400 Subject: [PATCH 044/100] add list of installed plugins to bot instance --- modmail/addons/models.py | 14 ++++++++++++-- modmail/bot.py | 16 ++++++++++++---- modmail/extensions/plugin_manager.py | 2 ++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index b79b43e9..c090321f 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -2,7 +2,7 @@ import re from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, Literal, NoReturn, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, NoReturn, Optional, Union from modmail.utils import MISSING @@ -123,14 +123,18 @@ class Addon: def __init__(self) -> NoReturn: raise NotImplementedError("Inheriting classes need to implement their own init") + def __hash__(self): + return hash(self.name) + class Plugin(Addon): """An addon which is a plugin.""" if TYPE_CHECKING: - folder_name: Optional[str] + folder_name: str folder_path: Optional[pathlib.Path] extra_kwargs = Dict[str, Any] + extension_files = List[pathlib.Path] def __init__( self, @@ -163,3 +167,9 @@ def __init__( def __repr__(self) -> str: # pragma: no cover return f"" + + def __hash__(self): + return hash(self.folder_name) + + def __eq__(self, other: Any): + return hash(self) == hash(other) diff --git a/modmail/bot.py b/modmail/bot.py index f4eebebb..0685b2f5 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -1,8 +1,7 @@ import asyncio import logging import signal -import typing as t -from typing import Any +from typing import TYPE_CHECKING, Any, Dict, List, Optional import arrow import discord @@ -17,6 +16,12 @@ from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions +if TYPE_CHECKING: + import pathlib + + from modmail.addons.models import Plugin + + REQUIRED_INTENTS = Intents( guilds=True, messages=True, @@ -38,8 +43,11 @@ class ModmailBot(commands.Bot): def __init__(self, **kwargs): self.config = CONFIG - self.start_time: t.Optional[arrow.Arrow] = None # arrow.utcnow() - self.http_session: t.Optional[ClientSession] = None + self.start_time: Optional[arrow.Arrow] = None # arrow.utcnow() + self.http_session: Optional[ClientSession] = None + + # keys: plugins, list values: all plugin files + self.installed_plugins: Dict["Plugin", List["pathlib.Path"]] = {} status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 6a17ef68..fb088853 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -179,6 +179,8 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu logger.debug(f"{files_to_load = }") self.batch_manage(Action.LOAD, *files_to_load) + self.bot.installed_plugins[plugin] = files_to_load + await ctx.reply(f"Installed plugin {plugin.name}.") # TODO: Implement enable/disable/etc From ece505b1ac6f3b1fbef1a0d63fa744e8a9977723 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 6 Sep 2021 14:23:26 -0400 Subject: [PATCH 045/100] move dev commands to a dev group --- modmail/extensions/plugin_manager.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index fb088853..2ad844de 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -67,7 +67,14 @@ async def plugins_group(self, ctx: Context) -> None: """Install, uninstall, disable, update, and enable installed plugins.""" await ctx.send_help(ctx.command) - @plugins_group.command(name="load", aliases=("l",), enabled=PLUGIN_DEV_ENABLED) + @plugins_group.group( + "dev", aliases=("developer", "d"), invoke_without_command=True, enabled=PLUGIN_DEV_ENABLED + ) + async def plugin_dev_group(self, ctx: Context) -> None: + """Manage plugin files directly, rather than whole plugin objects.""" + await ctx.send_help(ctx.command) + + @plugin_dev_group.command(name="load", aliases=("l",)) async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Load plugins given their fully qualified or unqualified names. @@ -76,7 +83,7 @@ async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None """ await self.load_extensions.callback(self, ctx, *plugins) - @plugins_group.command(name="unload", aliases=("ul",), enabled=PLUGIN_DEV_ENABLED) + @plugin_dev_group.command(name="unload", aliases=("ul",)) async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Unload currently loaded plugins given their fully qualified or unqualified names. @@ -85,7 +92,7 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> N """ await self.unload_extensions.callback(self, ctx, *plugins) - @plugins_group.command(name="reload", aliases=("r", "rl"), enabled=PLUGIN_DEV_ENABLED) + @plugin_dev_group.command(name="reload", aliases=("r", "rl")) async def reload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: r""" Reload plugins given their fully qualified or unqualified names. @@ -106,7 +113,7 @@ async def list_plugins(self, ctx: Context) -> None: """ await self.list_extensions.callback(self, ctx) - @plugins_group.command(name="refresh", aliases=("rewalk", "rescan"), enabled=PLUGIN_DEV_ENABLED) + @plugin_dev_group.command(name="refresh", aliases=("rewalk", "rescan")) async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) From a67522e11da080cfdf6d9bb7449c441ef6ddcfad Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 6 Sep 2021 15:51:58 -0400 Subject: [PATCH 046/100] fix: actually use provided refs --- modmail/addons/models.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index c090321f..e8ebccc4 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -27,6 +27,7 @@ class GitHost: base_api_url: str repo_api_url: str zip_archive_api_url: str + zip_archive_api_url_with_ref: str class Github(GitHost): @@ -36,6 +37,7 @@ class Github(GitHost): base_api_url = "https://api.github.com" repo_api_url = f"{base_api_url}/repos/{{user}}/{{repo}}" zip_archive_api_url = f"{repo_api_url}/zipball" + zip_archive_api_url_with_ref = f"{zip_archive_api_url}/{{ref}}" class Gitlab(GitHost): @@ -44,6 +46,7 @@ class Gitlab(GitHost): base_api_url = "https://gitlab.com/api/v4" repo_api_url = f"{base_api_url}/projects/{{user}}%2F{{repo}}" zip_archive_api_url = f"{repo_api_url}/repository/archive.zip" + zip_archive_api_url_with_ref = f"{zip_archive_api_url}?sha={{ref}}" Host = Literal["github", "gitlab"] @@ -86,12 +89,15 @@ def __init__(self, zip_url: str, type: SourceTypeEnum): def from_repo(cls, user: str, repo: str, reflike: str = None, githost: Host = "github") -> AddonSource: """Create an AddonSource from a repo.""" if githost == "github": - Host = Github # noqa: N806 + Host = Github() # noqa: N806 elif githost == "gitlab": - Host = Gitlab # noqa: N806 + Host = Gitlab() # noqa: N806 else: raise TypeError(f"{githost} is not a valid host.") - zip_url = Host.zip_archive_api_url.format(user=user, repo=repo) + if reflike is not None: + zip_url = Host.zip_archive_api_url_with_ref.format(user=user, repo=repo, ref=reflike) + else: + zip_url = Host.zip_archive_api_url.format(user=user, repo=repo) source = cls(zip_url, SourceTypeEnum.REPO) source.repo = repo From 781f625c9b04a90015651f9cea880deb2499c0f2 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 6 Sep 2021 16:25:03 -0400 Subject: [PATCH 047/100] feat: add enabling, disabling, uninstalling plugins --- modmail/addons/models.py | 5 +- modmail/bot.py | 4 +- modmail/extensions/plugin_manager.py | 86 +++++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index e8ebccc4..195b8970 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -139,8 +139,9 @@ class Plugin(Addon): if TYPE_CHECKING: folder_name: str folder_path: Optional[pathlib.Path] - extra_kwargs = Dict[str, Any] - extension_files = List[pathlib.Path] + extra_kwargs: Dict[str, Any] + installed_path: Optional[pathlib.Path] + extension_files: List[pathlib.Path] def __init__( self, diff --git a/modmail/bot.py b/modmail/bot.py index 0685b2f5..dacf39d3 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -17,8 +17,6 @@ if TYPE_CHECKING: - import pathlib - from modmail.addons.models import Plugin @@ -47,7 +45,7 @@ def __init__(self, **kwargs): self.http_session: Optional[ClientSession] = None # keys: plugins, list values: all plugin files - self.installed_plugins: Dict["Plugin", List["pathlib.Path"]] = {} + self.installed_plugins: Dict["Plugin", List[str]] = {} status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 2ad844de..e644e4fc 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -3,7 +3,7 @@ import asyncio import logging import shutil -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Dict, List from discord.ext import commands from discord.ext.commands import Context @@ -30,7 +30,7 @@ PLUGIN_DEV_ENABLED = BOT_MODE & BotModeEnum.PLUGIN_DEV -class PluginPathConverter(ExtensionConverter): +class PluginDevPathConverter(ExtensionConverter): """ Fully qualify the name of a plugin and ensure it exists. @@ -42,6 +42,39 @@ class PluginPathConverter(ExtensionConverter): NO_UNLOAD = None +class PluginConverter(commands.Converter): + """Convert a plugin name into a full plugin with path and related args.""" + + async def convert(self, ctx: Context, argument: str) -> List[str]: + """Converts a plugin into a full plugin with a path and all other attributes.""" + loaded_plugs: Dict[Plugin, List[str]] = ctx.bot.installed_plugins + + for plug in loaded_plugs: + if argument in (plug.name, plug.folder_name): + return plug + + raise commands.BadArgument(f"{argument} is not in list of installed plugins.") + + +class PluginFilesConverter(commands.Converter): + """ + Convert a name of a plugin into a full plugin. + + In this case Plugins are group of extensions, as if they have multiple files in their directory, + they will be treated as one plugin for the sake of managing. + """ + + async def convert(self, ctx: Context, argument: str) -> List[str]: + """Converts a provided plugin into a list of its paths.""" + loaded_plugs: Dict[Plugin, List[str]] = ctx.bot.installed_plugins + + for plug in loaded_plugs: + if argument in (plug.name, plug.folder_name): + return loaded_plugs[plug] + + raise commands.BadArgument(f"{argument} is not an installed plugin.") + + class PluginManager(ExtensionManager, name="Plugin Manager"): """Plugin management commands.""" @@ -75,7 +108,7 @@ async def plugin_dev_group(self, ctx: Context) -> None: await ctx.send_help(ctx.command) @plugin_dev_group.command(name="load", aliases=("l",)) - async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None: + async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Load plugins given their fully qualified or unqualified names. @@ -84,7 +117,7 @@ async def load_plugin(self, ctx: Context, *plugins: PluginPathConverter) -> None await self.load_extensions.callback(self, ctx, *plugins) @plugin_dev_group.command(name="unload", aliases=("ul",)) - async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: + async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Unload currently loaded plugins given their fully qualified or unqualified names. @@ -93,7 +126,7 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> N await self.unload_extensions.callback(self, ctx, *plugins) @plugin_dev_group.command(name="reload", aliases=("r", "rl")) - async def reload_plugins(self, ctx: Context, *plugins: PluginPathConverter) -> None: + async def reload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Reload plugins given their fully qualified or unqualified names. @@ -133,7 +166,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu # TODO: check the path of a local plugin await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") return - logger.debug(f"Received command to download plugin {plugin.name} from {source.zip_url}") + logger.debug(f"Received command to download plugin {plugin.name} from https://{source.zip_url}") try: directory = await addon_utils.download_and_unpack_source(source, self.bot.http_session) except errors.HTTPError: @@ -154,8 +187,9 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu for p in plugins.keys(): # check if user-provided plugin matches either plugin name or folder name if plugin.name in (p.name, p.folder_name): + install_path = BASE_PLUGIN_PATH / p.folder_path.name try: - shutil.copytree(p.folder_path, BASE_PLUGIN_PATH / p.folder_path.name, dirs_exist_ok=True) + shutil.copytree(p.folder_path, install_path, dirs_exist_ok=True) except FileExistsError: await ctx.send( "Plugin already seems to be installed. " @@ -163,6 +197,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu "or a plugin of the same name existing." ) return + p.installed_path = install_path plugin = p break @@ -176,7 +211,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu for plug in plugins[plugin]: logger.trace(f"{plug = }") try: - plug = await PluginPathConverter().convert(None, plug.name.rstrip(".py")) + plug = await PluginDevPathConverter().convert(None, plug.name.rstrip(".py")) except commands.BadArgument: pass else: @@ -190,6 +225,41 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu await ctx.reply(f"Installed plugin {plugin.name}.") + @plugins_group.command(name="uninstall", aliases=("rm",)) + async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + """Uninstall a provided plugin, given the name of the plugin.""" + plugin: Plugin = plugin + + # plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + await self.disable_plugin.callback(self, ctx, plugin=plugin) + + shutil.rmtree(plugin.installed_path) + + plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + for file_ in plugin_files: + del PLUGINS[file_] + del self.bot.installed_plugins[plugin] + + await ctx.send(plugin.installed_path) + + @plugins_group.command(name="enable") + async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + """Enable a provided plugin, given the name or folder of the plugin.""" + plugin: Plugin = plugin + + plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + + await self.load_plugins.callback(self, ctx, *plugin_files) + + @plugins_group.command(name="disable") + async def disable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + """Disable a provided plugin, given the name or folder of the plugin.""" + plugin: Plugin = plugin + + plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + + await self.unload_plugins.callback(self, ctx, *plugin_files) + # TODO: Implement enable/disable/etc # This cannot be static (must have a __func__ attribute). From dd463ecb4005bb51ce4d8bb1389ec15eb99c10c0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 7 Sep 2021 02:00:48 -0400 Subject: [PATCH 048/100] plugins: refactor to allow local plugins local plugins now act mostly like normal plugins. they do not have any special names, and exist as if they were installed normally to add a local plugin, put the files in the plugins folder, in a normal folder, and create a local.toml file in the same format as plugins.toml for addon repositories. --- modmail/addons/models.py | 4 +- modmail/addons/plugins.py | 65 ++++++++++++++++++++++++- modmail/bot.py | 13 +++-- modmail/extensions/plugin_manager.py | 71 ++++++++++++++++++++++++---- modmail/plugins/.gitignore | 5 -- modmail/plugins/@local/README.md | 12 ----- 6 files changed, 137 insertions(+), 33 deletions(-) delete mode 100644 modmail/plugins/@local/README.md diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 195b8970..52e3478e 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -151,7 +151,7 @@ def __init__( min_bot_version: Optional[str] = None, folder: Optional[str] = MISSING, folder_path: Optional[pathlib.Path] = None, - enabled: bool = True, + local: bool = False, **kw, ): self.name = name @@ -165,7 +165,7 @@ def __init__( self.folder_name = folder self.folder_path = folder_path self.min_bot_version = min_bot_version - self.enabled = enabled + self.local = local # store any extra kwargs here # this is to ensure backwards compatiablilty with plugins that support older versions, diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 931eb37d..772ce1e2 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -144,7 +144,70 @@ def find_plugins_in_dir( return all_plugins -def walk_plugins(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterator[Tuple[str, bool]]: +def find_local_plugins( + detection_path: pathlib.Path = BASE_PLUGIN_PATH, / # noqa: W504 +) -> Dict[Plugin, List[pathlib.Path]]: + """ + Walks the local path, and determines which files are local plugins. + + Yields a list of plugins, + """ + all_plugins: Dict[Plugin, List[pathlib.Path]] = {} + + toml_plugins: List[Plugin] = [] + toml_path = detection_path / "local.toml" + if toml_path.exists(): + # parse the toml + with open(toml_path) as toml_file: + toml_plugins = parse_plugin_toml_from_string(toml_file.read()) + else: + raise NoPluginTomlFoundError(toml_path, "does not exist") + + logger.debug(f"{toml_plugins =}") + toml_plugin_names = [p.folder_name for p in toml_plugins] + for path in detection_path.iterdir(): + logger.debug(f"detection_path / path: {path}") + if path.is_dir(): + # use an existing toml plugin object + if path.name in toml_plugin_names: + for p in toml_plugins: + if p.folder_name == path.name: + p.folder_path = path + all_plugins[p] = list() + else: + if path.name != "__pycache__": + temp_plugin = Plugin(path.name, folder_path=path) + all_plugins[temp_plugin] = list() + + logger.debug(f"Local plugins detected: {[p.name for p in all_plugins.keys()]}") + + for plugin_ in all_plugins.keys(): + logger.trace(f"{plugin_.folder_path =}") + plugin_.local = True # take this as an opportunity to configure local to True on all plugins + for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): + logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") + for list_ in dirnames, filenames: + logger.trace(f"{list_ =}") + for file in list_: + logger.trace(f"{file =}") + if file == dirpath: # don't include files that are plugin directories + continue + if "__pycache__" in file or "__pycache__" in dirpath: + continue + + relative_path = pathlib.Path(dirpath + "/" + file).relative_to(BASE_PLUGIN_PATH) + module_name = relative_path.__str__().rstrip(".py").replace("/", ".") + module_name = plugins.__name__ + "." + module_name + + all_plugins[plugin_].append(module_name) + + logger.debug(f"{all_plugins.keys() = }") + logger.debug(f"{all_plugins.values() = }") + + return all_plugins + + +def walk_plugin_files(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterator[Tuple[str, bool]]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder # this is to ensure folder symlinks are supported, diff --git a/modmail/bot.py b/modmail/bot.py index dacf39d3..7c0b24a2 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -10,7 +10,8 @@ from discord.client import _cleanup_loop from discord.ext import commands -from modmail.addons.plugins import PLUGINS, walk_plugins +from modmail.addons.errors import NoPluginTomlFoundError +from modmail.addons.plugins import PLUGINS, find_local_plugins, walk_plugin_files from modmail.config import CONFIG from modmail.log import ModmailLogger from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions @@ -186,8 +187,7 @@ def load_extensions(self) -> None: def load_plugins(self) -> None: """Load all enabled plugins.""" - PLUGINS.update(walk_plugins()) - + PLUGINS.update(walk_plugin_files()) for plugin, should_load in PLUGINS.items(): if should_load: self.logger.debug(f"Loading plugin {plugin}") @@ -198,6 +198,13 @@ def load_plugins(self) -> None: except Exception: self.logger.error("Failed to load plugin {0}".format(plugin), exc_info=True) + try: + plugins = find_local_plugins() + except NoPluginTomlFoundError: + pass + else: + self.installed_plugins.update(plugins) + def add_cog(self, cog: commands.Cog, *, override: bool = False) -> None: """ Load a given cog. diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index e644e4fc..0f2b186a 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -3,8 +3,10 @@ import asyncio import logging import shutil -from typing import TYPE_CHECKING, Dict, List +from collections import defaultdict +from typing import TYPE_CHECKING, Dict, List, Mapping +from discord import Colour, Embed from discord.ext import commands from discord.ext.commands import Context @@ -13,10 +15,11 @@ from modmail.addons.converters import SourceAndPluginConverter from modmail.addons.errors import PluginNotFoundError from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum -from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugins +from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugin_files from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModeEnum, ExtMetadata from modmail.utils.extensions import BOT_MODE +from modmail.utils.pagination import ButtonPaginator if TYPE_CHECKING: @@ -84,7 +87,7 @@ class PluginManager(ExtensionManager, name="Plugin Manager"): def __init__(self, bot: ModmailBot): super().__init__(bot) self.all_extensions = PLUGINS - self.refresh_method = walk_plugins + self.refresh_method = walk_plugin_files def get_black_listed_extensions(self) -> list: """ @@ -136,13 +139,13 @@ async def reload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) - """ await self.reload_extensions.callback(self, ctx, *plugins) - @plugins_group.command(name="list", aliases=("all", "ls")) - async def list_plugins(self, ctx: Context) -> None: + @plugin_dev_group.command(name="list", aliases=("all", "ls")) + async def dev_list_plugins(self, ctx: Context) -> None: """ - Get a list of all plugins, including their loaded status. + Get a list of all plugin files, including their loaded status. - Red indicates that the plugin is unloaded. - Green indicates that the plugin is currently loaded. + Red indicates that the plugin file is unloaded. + Green indicates that the plugin file is currently loaded. """ await self.list_extensions.callback(self, ctx) @@ -205,7 +208,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu raise PluginNotFoundError(f"Could not find plugin {plugin}") logger.trace(f"{BASE_PLUGIN_PATH = }") - PLUGINS.update(walk_plugins(BASE_PLUGIN_PATH / p.folder_path.name)) + PLUGINS.update(walk_plugin_files(BASE_PLUGIN_PATH / p.folder_path.name)) files_to_load: List[str] = [] for plug in plugins[plugin]: @@ -260,7 +263,55 @@ async def disable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None await self.unload_plugins.callback(self, ctx, *plugin_files) - # TODO: Implement enable/disable/etc + def group_plugin_statuses(self) -> Mapping[str, str]: + """Return a mapping of plugin names and statuses to their module.""" + plugins = defaultdict(str) + + for plug, files in self.bot.installed_plugins.items(): + plug_status = [] + for ext in files: + if ext in self.bot.extensions: + status = True + else: + status = False + plug_status.append(status) + + if all(plug_status): + status = ":green_circle:" + elif any(plug_status): + status = ":yellow_circle:" + else: + status = ":red_circle:" + + plugins[plug.name] = status + + return dict(plugins) + + @plugins_group.command(name="list", aliases=("all", "ls")) + async def list_plugins(self, ctx: Context) -> None: + """ + Get a list of all plugins, including their loaded status. + + Green indicates that the extension is fully loaded. + Yellow indicates that the plugin is partially loaded. + Red indicates that the plugin is fully unloaded. + """ + embed = Embed(colour=Colour.blurple()) + embed.set_author( + name=f"{self.type.capitalize()} List", + ) + + lines = [] + plugin_statuses = self.group_plugin_statuses() + for plugin_name, status in sorted(plugin_statuses.items()): + # plugin_name = plugin_name.replace("_", " ").title() + lines.append(f"{status} **{plugin_name}**") + + logger.debug(f"{ctx.author} requested a list of all {self.type}s. " "Returning a paginated list.") + + await ButtonPaginator.paginate( + lines or f"There are no {self.type}s installed.", ctx.message, embed=embed + ) # This cannot be static (must have a __func__ attribute). async def cog_check(self, ctx: Context) -> bool: diff --git a/modmail/plugins/.gitignore b/modmail/plugins/.gitignore index d316dad9..60a54109 100644 --- a/modmail/plugins/.gitignore +++ b/modmail/plugins/.gitignore @@ -3,11 +3,6 @@ # don't ignore this file !/.gitignore -# ignore the local folder, but not the readme -/@local/** -!/@local/ -!/@local/README.md - # ensure __init__.py is uploaded so `plugins` is considered a module !/__init__.py # keep our helper file in here diff --git a/modmail/plugins/@local/README.md b/modmail/plugins/@local/README.md deleted file mode 100644 index 817f6aba..00000000 --- a/modmail/plugins/@local/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Plugins - -This folder is where local plugins can be put for developing. - -Plugins should be like normal discord cogs, but should subclass `PluginCog` from `modmail.plugin_helpers` - -```py -from modmail.plugin_helpers import PluginCog - -class MyPlugin(PluginCog): - pass -``` From 3ce64a6740b8fe58dd554f48e09502f7bc4c7253 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 7 Sep 2021 02:15:34 -0400 Subject: [PATCH 049/100] rewrite plugin models to require folder_name instead of name --- modmail/addons/converters.py | 2 +- modmail/addons/models.py | 22 ++++++++++------------ modmail/addons/plugins.py | 4 ++-- tests/modmail/addons/test_models.py | 8 ++++---- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/modmail/addons/converters.py b/modmail/addons/converters.py index 5197c6db..78ab98ce 100644 --- a/modmail/addons/converters.py +++ b/modmail/addons/converters.py @@ -49,7 +49,7 @@ async def convert(self, _: Context, argument: str) -> Tuple[Plugin, AddonSource] if match is not None: logger.debug("Matched as a local file, creating a Plugin without a source url.") source = AddonSource(None, SourceTypeEnum.LOCAL) - return Plugin(name=match.group("addon")), source + return Plugin(match.group("addon")), source match = ZIP_REGEX.fullmatch(argument) if match is not None: logger.debug("Matched as a zip, creating a Plugin from zip.") diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 52e3478e..82afd00f 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -4,8 +4,6 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Literal, NoReturn, Optional, Union -from modmail.utils import MISSING - if TYPE_CHECKING: import pathlib @@ -145,24 +143,21 @@ class Plugin(Addon): def __init__( self, - name: str, + folder: str, description: Optional[str] = None, *, min_bot_version: Optional[str] = None, - folder: Optional[str] = MISSING, + name: Optional[str] = None, folder_path: Optional[pathlib.Path] = None, local: bool = False, **kw, ): - self.name = name + self.folder_name = folder self.description = description - if folder is MISSING: - if folder_path is not None: - self.folder_name = folder or folder_path.name or name - else: - self.folder_name = folder or name + if name is None: + self.name = self.folder_name else: - self.folder_name = folder + self.name = name self.folder_path = folder_path self.min_bot_version = min_bot_version self.local = local @@ -173,7 +168,10 @@ def __init__( self.extra_kwargs = kw def __repr__(self) -> str: # pragma: no cover - return f"" + return ( + f"" + ) def __hash__(self): return hash(self.folder_name) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 772ce1e2..290e788f 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -46,8 +46,8 @@ def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plug found_plugins.append( Plugin( - plug_entry["name"], - folder=plug_entry.get("folder"), + plug_entry["folder"], + name=plug_entry.get("name"), description=plug_entry.get("description"), min_bot_version=plug_entry.get("min_bot_version"), ) diff --git a/tests/modmail/addons/test_models.py b/tests/modmail/addons/test_models.py index 9e2f2c02..e1814cac 100644 --- a/tests/modmail/addons/test_models.py +++ b/tests/modmail/addons/test_models.py @@ -72,9 +72,9 @@ def test_addonsource_from_zip(url: str) -> None: class TestPlugin: """Test the Plugin class creation.""" - @pytest.mark.parametrize("name", [("earth"), ("mona-lisa")]) - def test_plugin_init(self, name: str) -> None: + @pytest.mark.parametrize("folder", [("earth"), ("mona-lisa")]) + def test_plugin_init(self, folder: str) -> None: """Create a plugin model, and ensure it has the right properties.""" - plugin = Plugin(name) + plugin = Plugin(folder) assert isinstance(plugin, Plugin) - assert plugin.name == name + assert plugin.folder_name == folder From a9f99dd444adc8377b23a89aa152c9238b881621 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 7 Sep 2021 02:21:54 -0400 Subject: [PATCH 050/100] fix[local-plugins]: don't add plugins not in local.toml --- modmail/addons/plugins.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 290e788f..4210416b 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -174,10 +174,6 @@ def find_local_plugins( if p.folder_name == path.name: p.folder_path = path all_plugins[p] = list() - else: - if path.name != "__pycache__": - temp_plugin = Plugin(path.name, folder_path=path) - all_plugins[temp_plugin] = list() logger.debug(f"Local plugins detected: {[p.name for p in all_plugins.keys()]}") From da19a070c2856d374e9a8545eb567d75047ec92d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 7 Sep 2021 02:22:26 -0400 Subject: [PATCH 051/100] plugin-toml: add directory as an alias for folder --- modmail/addons/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 4210416b..cebea0b8 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -46,7 +46,7 @@ def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plug found_plugins.append( Plugin( - plug_entry["folder"], + plug_entry.get("directory") or plug_entry["folder"], name=plug_entry.get("name"), description=plug_entry.get("description"), min_bot_version=plug_entry.get("min_bot_version"), From c9fcf01feca5aba603574659cad5c48e790182fc Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 7 Sep 2021 23:10:02 -0400 Subject: [PATCH 052/100] minor: allow local plugins to be disabled in local.toml and fix some typehints --- modmail/addons/models.py | 7 ++- modmail/addons/plugins.py | 76 +++++++++++++++++++++------- modmail/bot.py | 25 ++++++--- modmail/extensions/plugin_manager.py | 16 +++++- modmail/utils/cogs.py | 4 +- modmail/utils/extensions.py | 2 +- 6 files changed, 100 insertions(+), 30 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 82afd00f..1debeef8 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -150,6 +150,7 @@ def __init__( name: Optional[str] = None, folder_path: Optional[pathlib.Path] = None, local: bool = False, + enabled: bool = True, **kw, ): self.folder_name = folder @@ -161,13 +162,17 @@ def __init__( self.folder_path = folder_path self.min_bot_version = min_bot_version self.local = local + self.enabled = enabled # store any extra kwargs here # this is to ensure backwards compatiablilty with plugins that support older versions, # but want to use newer toml options self.extra_kwargs = kw - def __repr__(self) -> str: # pragma: no cover + def __str__(self): + return self.name + + def __repr__(self): # pragma: no cover return ( f"" diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index cebea0b8..2e0e3fb9 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -37,24 +37,69 @@ PLUGIN_TOML = "plugin.toml" +LOCAL_PLUGIN_TOML = BASE_PLUGIN_PATH / "local.toml" -def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /) -> List[Plugin]: + +def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /, local: bool = False) -> List[Plugin]: """Parses a plugin toml, given the string loaded in.""" doc = atoml.parse(unparsed_plugin_toml_str) found_plugins: List[Plugin] = [] for plug_entry in doc["plugins"]: - + if local: + enabled = plug_entry.get("enabled", True) + else: + enabled = None found_plugins.append( Plugin( plug_entry.get("directory") or plug_entry["folder"], name=plug_entry.get("name"), description=plug_entry.get("description"), min_bot_version=plug_entry.get("min_bot_version"), + enabled=enabled, ) ) return found_plugins +def update_local_toml_enable_or_disable(plugin: Plugin, /) -> None: + """ + Updates the local toml so local plugins stay disabled or enabled. + + This is the local implementation for disabling and enabling to actually disable and enable plugins. + Non local plugins are saved in the database. + """ + if not LOCAL_PLUGIN_TOML.exists(): + raise NoPluginTomlFoundError + + with LOCAL_PLUGIN_TOML.open("r") as f: + doc = atoml.loads(f.read()) + plugs = doc["plugins"] + + plug_found = False + for plug_entry in plugs: + folder_name = plug_entry.get("directory") or plug_entry["folder"] + if folder_name == plugin.folder_name: + plug_entry["enabled"] = plugin.enabled + plug_found = True + break + + if not plug_found: + # need to write a new entry + logger.trace(f"Local plugin toml does not contain an entry for {plugin}") + + plugin_table = atoml.table() + if plugin.name != plugin.folder_name: + plugin_table.add("name", atoml.item(plugin.name)) + + plugin_table.add("directory", atoml.item(plugin.folder_name)) + plug_entry["enabled"] = plugin.enabled + plugs.append(plugin_table) + print(plugs) + + with open(LOCAL_PLUGIN_TOML, "w") as f: + f.write(doc.as_string()) + + def find_plugins_in_dir( addon_repo_path: pathlib.Path, *, @@ -146,20 +191,20 @@ def find_plugins_in_dir( def find_local_plugins( detection_path: pathlib.Path = BASE_PLUGIN_PATH, / # noqa: W504 -) -> Dict[Plugin, List[pathlib.Path]]: +) -> Dict[Plugin, List[str]]: """ Walks the local path, and determines which files are local plugins. Yields a list of plugins, """ - all_plugins: Dict[Plugin, List[pathlib.Path]] = {} + all_plugins: Dict[Plugin, List[str]] = {} toml_plugins: List[Plugin] = [] - toml_path = detection_path / "local.toml" + toml_path = LOCAL_PLUGIN_TOML if toml_path.exists(): # parse the toml with open(toml_path) as toml_file: - toml_plugins = parse_plugin_toml_from_string(toml_file.read()) + toml_plugins = parse_plugin_toml_from_string(toml_file.read(), local=True) else: raise NoPluginTomlFoundError(toml_path, "does not exist") @@ -182,20 +227,17 @@ def find_local_plugins( plugin_.local = True # take this as an opportunity to configure local to True on all plugins for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") - for list_ in dirnames, filenames: + for list_ in dirnames, [dirpath]: logger.trace(f"{list_ =}") - for file in list_: - logger.trace(f"{file =}") - if file == dirpath: # don't include files that are plugin directories - continue - if "__pycache__" in file or "__pycache__" in dirpath: + for dir_ in list_: + logger.trace(f"{dir_ =}") + + if "__pycache__" in dir_ or "__pycache__" in dirpath: continue - relative_path = pathlib.Path(dirpath + "/" + file).relative_to(BASE_PLUGIN_PATH) - module_name = relative_path.__str__().rstrip(".py").replace("/", ".") - module_name = plugins.__name__ + "." + module_name + modules = [x for x, y in walk_plugin_files(dirpath)] - all_plugins[plugin_].append(module_name) + all_plugins[plugin_].extend(modules) logger.debug(f"{all_plugins.keys() = }") logger.debug(f"{all_plugins.values() = }") @@ -259,4 +301,4 @@ def walk_plugin_files(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterat ) # Presume Production Mode/Metadata defaults if metadata var does not exist. - yield imported.__name__, ExtMetadata.load_if_mode + yield imported.__name__, bool(ExtMetadata.load_if_mode) diff --git a/modmail/bot.py b/modmail/bot.py index 7c0b24a2..ee928c3c 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -188,8 +188,22 @@ def load_extensions(self) -> None: def load_plugins(self) -> None: """Load all enabled plugins.""" PLUGINS.update(walk_plugin_files()) + try: + plugins = find_local_plugins() + except NoPluginTomlFoundError: + dont_load_at_start = [] + else: + self.installed_plugins.update(plugins) + + dont_load_at_start = [] + for plug, modules in self.installed_plugins.items(): + if plug.enabled: + continue + self.logger.debug(f"Not loading {plug.__str__()} on start since it's not enabled.") + dont_load_at_start.extend(modules) + for plugin, should_load in PLUGINS.items(): - if should_load: + if should_load and plugin not in dont_load_at_start: self.logger.debug(f"Loading plugin {plugin}") try: # since we're loading user generated content, @@ -197,13 +211,8 @@ def load_plugins(self) -> None: self.load_extension(plugin) except Exception: self.logger.error("Failed to load plugin {0}".format(plugin), exc_info=True) - - try: - plugins = find_local_plugins() - except NoPluginTomlFoundError: - pass - else: - self.installed_plugins.update(plugins) + else: + self.logger.debug(f"SKIPPED loading plugin {plugin}") def add_cog(self, cog: commands.Cog, *, override: bool = False) -> None: """ diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 0f2b186a..1413e771 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -15,7 +15,13 @@ from modmail.addons.converters import SourceAndPluginConverter from modmail.addons.errors import PluginNotFoundError from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum -from modmail.addons.plugins import BASE_PLUGIN_PATH, PLUGINS, find_plugins_in_dir, walk_plugin_files +from modmail.addons.plugins import ( + BASE_PLUGIN_PATH, + PLUGINS, + find_plugins_in_dir, + update_local_toml_enable_or_disable, + walk_plugin_files, +) from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager from modmail.utils.cogs import BotModeEnum, ExtMetadata from modmail.utils.extensions import BOT_MODE @@ -249,18 +255,26 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: """Enable a provided plugin, given the name or folder of the plugin.""" plugin: Plugin = plugin + plugin.enabled = True plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + if plugin.local: + update_local_toml_enable_or_disable(plugin) + await self.load_plugins.callback(self, ctx, *plugin_files) @plugins_group.command(name="disable") async def disable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: """Disable a provided plugin, given the name or folder of the plugin.""" plugin: Plugin = plugin + plugin.enabled = False plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + if plugin.local: + update_local_toml_enable_or_disable(plugin) + await self.unload_plugins.callback(self, ctx, *plugin_files) def group_plugin_statuses(self) -> Mapping[str, str]: diff --git a/modmail/utils/cogs.py b/modmail/utils/cogs.py index 5fcfb7c5..75554e95 100644 --- a/modmail/utils/cogs.py +++ b/modmail/utils/cogs.py @@ -31,11 +31,11 @@ class BotModeEnum(BitwiseAutoEnum): class ExtMetadata: """Ext metadata class to determine if extension should load at runtime depending on bot configuration.""" - load_if_mode: int = BotModeEnum.PRODUCTION + load_if_mode: BotModeEnum = BotModeEnum.PRODUCTION # this is to determine if the cog is allowed to be unloaded. no_unload: bool = False - def __init__(self, load_if_mode: int = BotModeEnum.PRODUCTION, no_unload: bool = False) -> "ExtMetadata": + def __init__(self, load_if_mode: BotModeEnum = BotModeEnum.PRODUCTION, no_unload: bool = False): self.load_if_mode = load_if_mode self.no_unload = no_unload diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index 44348f26..ce94fc1f 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -68,7 +68,7 @@ def on_error(name: str) -> t.NoReturn: ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) if ext_metadata is not None: # check if this cog is dev only or plugin dev only - load_cog = bool(int(ext_metadata.load_if_mode) & BOT_MODE) + load_cog = bool(ext_metadata.load_if_mode.value & BOT_MODE) log.trace(f"Load cog {module.name!r}?: {load_cog}") no_unload = ext_metadata.no_unload yield module.name, (load_cog, no_unload) From c01185ab0bc24ddf9f70e011a17fa1f883682836 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 8 Sep 2021 02:51:31 -0400 Subject: [PATCH 053/100] major: plugin loading error handling and proper replies plugin management handles more errors and validation helper class in modmail.utils.responses to send success or fail messages --- modmail/addons/plugins.py | 5 +- modmail/extensions/extension_manager.py | 83 ++++++++++---- modmail/extensions/plugin_manager.py | 112 +++++++++++++------ modmail/utils/responses.py | 143 ++++++++++++++++++++++++ 4 files changed, 284 insertions(+), 59 deletions(-) create mode 100644 modmail/utils/responses.py diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 2e0e3fb9..832f320f 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -69,7 +69,10 @@ def update_local_toml_enable_or_disable(plugin: Plugin, /) -> None: Non local plugins are saved in the database. """ if not LOCAL_PLUGIN_TOML.exists(): - raise NoPluginTomlFoundError + raise NoPluginTomlFoundError( + f"The required file at {LOCAL_PLUGIN_TOML!s} does not exist to deal with local plugins.\n" + "You may need to create it." + ) with LOCAL_PLUGIN_TOML.open("r") as f: doc = atoml.loads(f.read()) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 5a2755b2..b4cd3ebe 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -3,11 +3,11 @@ # MIT License 2021 Python Discord import functools import logging -import typing as t from collections import defaultdict from enum import Enum +from typing import Mapping, Optional, Tuple -from discord import AllowedMentions, Colour, Embed +from discord import Colour, Embed from discord.ext import commands from discord.ext.commands import Context @@ -16,6 +16,7 @@ from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator +from modmail.utils.responses import Response log: ModmailLogger = logging.getLogger(__name__) @@ -24,6 +25,15 @@ EXT_METADATA = ExtMetadata(load_if_mode=BotModeEnum.DEVELOP, no_unload=True) +class StatusEmojis: + """Status emojis for extension statuses.""" + + fully_loaded: str = ":green_circle:" + partially_loaded: str = ":yellow_circle:" + unloaded: str = ":red_circle:" + disabled: str = ":brown_circle:" + + class Action(Enum): """Represents an action to perform on an extension.""" @@ -32,6 +42,10 @@ class Action(Enum): UNLOAD = functools.partial(ModmailBot.unload_extension) RELOAD = functools.partial(ModmailBot.reload_extension) + # for plugins + ENABLE = functools.partial(ModmailBot.load_extension) + DISABLE = functools.partial(ModmailBot.unload_extension) + class ExtensionConverter(commands.Converter): """ @@ -114,8 +128,8 @@ async def load_extensions(self, ctx: Context, *extensions: ExtensionConverter) - if "*" in extensions: extensions = sorted(ext for ext in self.all_extensions if ext not in self.bot.extensions.keys()) - msg = self.batch_manage(Action.LOAD, *extensions) - await ctx.send(msg) + msg, is_error = self.batch_manage(Action.LOAD, *extensions) + await Response.send_response(ctx, msg, not is_error) @extensions_group.command(name="unload", aliases=("ul",)) async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -132,7 +146,9 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if blacklisted: bl_msg = "\n".join(blacklisted) - await ctx.send(f":x: The following {self.type}(s) may not be unloaded:```\n{bl_msg}```") + await Response.send_negatory( + ctx, f":x: The following {self.type}(s) may not be unloaded:```\n{bl_msg}```" + ) return if "*" in extensions: @@ -142,7 +158,8 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if ext not in (self.get_black_listed_extensions()) ) - await ctx.send(self.batch_manage(Action.UNLOAD, *extensions)) + msg, is_error = self.batch_manage(Action.UNLOAD, *extensions) + await Response.send_response(ctx, msg, not is_error) @extensions_group.command(name="reload", aliases=("r", "rl")) async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -160,7 +177,8 @@ async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if "*" in extensions: extensions = self.bot.extensions.keys() & self.all_extensions.keys() - await ctx.send(self.batch_manage(Action.RELOAD, *extensions)) + msg, is_error = self.batch_manage(Action.RELOAD, *extensions) + await Response.send_response(ctx, msg, not is_error) @extensions_group.command(name="list", aliases=("all", "ls")) async def list_extensions(self, ctx: Context) -> None: @@ -216,17 +234,17 @@ async def resync_extensions(self, ctx: Context) -> None: Typical use case is in the event that the existing extensions have changed while the bot is running. """ self._resync_extensions() - await ctx.send(f":ok_hand: Refreshed list of {self.type}s.") + await Response.send_positive(ctx, f":ok_hand: Refreshed list of {self.type}s.") - def group_extension_statuses(self) -> t.Mapping[str, str]: + def group_extension_statuses(self) -> Mapping[str, str]: """Return a mapping of extension names and statuses to their categories.""" categories = defaultdict(list) for ext in self.all_extensions: if ext in self.bot.extensions: - status = ":green_circle:" + status = StatusEmojis.fully_loaded else: - status = ":red_circle:" + status = StatusEmojis.unloaded root, name = ext.rsplit(".", 1) if root.split(".", 1)[1] == self.module_name: @@ -237,21 +255,26 @@ def group_extension_statuses(self) -> t.Mapping[str, str]: return dict(categories) - def batch_manage(self, action: Action, *extensions: str) -> str: + def batch_manage( + self, + action: Action, + *extensions: str, + **kw, + ) -> Tuple[str, bool]: """ Apply an action to multiple extensions and return a message with the results. - If only one extension is given, it is deferred to `manage()`. + Any extra kwargs are passed to `manage()` which handles all passed modules. """ if len(extensions) == 1: - msg, _ = self.manage(action, extensions[0]) - return msg + msg, failures = self.manage(action, extensions[0], **kw) + return msg, bool(failures) verb = action.name.lower() failures = {} for extension in sorted(extensions): - _, error = self.manage(action, extension) + _, error = self.manage(action, extension, **kw) if error: failures[extension] = error @@ -264,21 +287,34 @@ def batch_manage(self, action: Action, *extensions: str) -> str: log.debug(f"Batch {verb}ed {self.type}s.") - return msg + return msg, bool(failures) - def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: + def manage( + self, + action: Action, + ext: str, + *, + is_plugin: bool = False, + suppress_already_error: bool = False, + ) -> Tuple[str, Optional[str]]: """Apply an action to an extension and return the status message and any error message.""" verb = action.name.lower() error_msg = None - + msg = None + not_quite = False try: action.value(self.bot, ext) except (commands.ExtensionAlreadyLoaded, commands.ExtensionNotLoaded): - if action is Action.RELOAD: + if suppress_already_error: + pass + elif action is Action.RELOAD: # When reloading, have a special error. msg = f":x: {self.type.capitalize()} `{ext}` is not loaded, so it was not {verb}ed." + not_quite = True + else: msg = f":x: {self.type.capitalize()} `{ext}` is already {verb}ed." + not_quite = True except Exception as e: if hasattr(e, "original"): # If original exception is present, then utilize it @@ -288,11 +324,12 @@ def manage(self, action: Action, ext: str) -> t.Tuple[str, t.Optional[str]]: error_msg = f"{e.__class__.__name__}: {e}" msg = f":x: Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" - else: + + if msg is None: msg = f":thumbsup: {self.type.capitalize()} successfully {verb}ed: `{ext}`." log.debug(error_msg or msg) - return msg, error_msg + return msg, error_msg or not_quite # This cannot be static (must have a __func__ attribute). async def cog_check(self, ctx: Context) -> bool: @@ -304,7 +341,7 @@ async def cog_check(self, ctx: Context) -> bool: async def cog_command_error(self, ctx: Context, error: Exception) -> None: """Handle BadArgument errors locally to prevent the help command from showing.""" if isinstance(error, commands.BadArgument): - await ctx.send(str(error), allowed_mentions=AllowedMentions.none()) + await Response.send_negatory(ctx, str(error)) error.handled = True else: raise error diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 1413e771..eb885bea 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -6,14 +6,16 @@ from collections import defaultdict from typing import TYPE_CHECKING, Dict, List, Mapping +from atoml.exceptions import ParseError from discord import Colour, Embed +from discord.abc import Messageable from discord.ext import commands from discord.ext.commands import Context import modmail.addons.utils as addon_utils from modmail import errors from modmail.addons.converters import SourceAndPluginConverter -from modmail.addons.errors import PluginNotFoundError +from modmail.addons.errors import NoPluginTomlFoundError from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum from modmail.addons.plugins import ( BASE_PLUGIN_PATH, @@ -22,10 +24,11 @@ update_local_toml_enable_or_disable, walk_plugin_files, ) -from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager +from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager, StatusEmojis from modmail.utils.cogs import BotModeEnum, ExtMetadata from modmail.utils.extensions import BOT_MODE from modmail.utils.pagination import ButtonPaginator +from modmail.utils.responses import Response if TYPE_CHECKING: @@ -54,7 +57,7 @@ class PluginDevPathConverter(ExtensionConverter): class PluginConverter(commands.Converter): """Convert a plugin name into a full plugin with path and related args.""" - async def convert(self, ctx: Context, argument: str) -> List[str]: + async def convert(self, ctx: Context, argument: str) -> Plugin: """Converts a plugin into a full plugin with a path and all other attributes.""" loaded_plugs: Dict[Plugin, List[str]] = ctx.bot.installed_plugins @@ -110,7 +113,7 @@ async def plugins_group(self, ctx: Context) -> None: await ctx.send_help(ctx.command) @plugins_group.group( - "dev", aliases=("developer", "d"), invoke_without_command=True, enabled=PLUGIN_DEV_ENABLED + "dev", aliases=("developer",), invoke_without_command=True, enabled=PLUGIN_DEV_ENABLED ) async def plugin_dev_group(self, ctx: Context) -> None: """Manage plugin files directly, rather than whole plugin objects.""" @@ -125,7 +128,7 @@ async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> """ await self.load_extensions.callback(self, ctx, *plugins) - @plugin_dev_group.command(name="unload", aliases=("ul",)) + @plugin_dev_group.command(name="unload", aliases=("u", "ul")) async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Unload currently loaded plugins given their fully qualified or unqualified names. @@ -155,15 +158,17 @@ async def dev_list_plugins(self, ctx: Context) -> None: """ await self.list_extensions.callback(self, ctx) - @plugin_dev_group.command(name="refresh", aliases=("rewalk", "rescan")) + @plugin_dev_group.command(name="refresh", aliases=("rewalk", "rescan", "resync")) async def resync_plugins(self, ctx: Context) -> None: """Refreshes the list of plugins from disk, but do not unload any currently active.""" await self.resync_extensions.callback(self, ctx) - @plugins_group.command(name="install", aliases=("",)) + @commands.max_concurrency(1, per=commands.BucketType.default, wait=True) + @plugins_group.command(name="install", aliases=("add",)) async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPluginConverter) -> None: """Install plugins from provided repo.""" # this could take a while + # I'm aware this should be a context manager, but do not want to indent almost the entire command await ctx.trigger_typing() # create variables for the user input, typehint them, then assign them from the converter tuple @@ -173,13 +178,19 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu if source.source_type is SourceTypeEnum.LOCAL: # TODO: check the path of a local plugin - await ctx.send("This plugin is a local plugin, and likely can be loaded with the load command.") + await Response.send_negatory( + ctx, + "This plugin seems to be a local plugin, and therefore can probably be " + "loaded with the load command, if it isn't loaded already.", + ) return logger.debug(f"Received command to download plugin {plugin.name} from https://{source.zip_url}") try: directory = await addon_utils.download_and_unpack_source(source, self.bot.http_session) except errors.HTTPError: - await ctx.send(f"Downloading {source.zip_url} did not give a 200 response code.") + await Response.send_negatory( + ctx, f"Downloading {source.zip_url} did not give a 200 response code." + ) return source.cache_file = directory @@ -200,10 +211,11 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu try: shutil.copytree(p.folder_path, install_path, dirs_exist_ok=True) except FileExistsError: - await ctx.send( + await Response.send_negatory( + ctx, "Plugin already seems to be installed. " "This could be caused by the plugin already existing, " - "or a plugin of the same name existing." + "or a plugin of the same name existing.", ) return p.installed_path = install_path @@ -211,7 +223,8 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu break if plugin.folder_path is None: - raise PluginNotFoundError(f"Could not find plugin {plugin}") + await Response.send_negatory(ctx, f"Could not find plugin {plugin}") + return logger.trace(f"{BASE_PLUGIN_PATH = }") PLUGINS.update(walk_plugin_files(BASE_PLUGIN_PATH / p.folder_path.name)) @@ -232,15 +245,25 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu self.bot.installed_plugins[plugin] = files_to_load - await ctx.reply(f"Installed plugin {plugin.name}.") + await Response.send_positive(ctx, f"Installed plugin {plugin}.") @plugins_group.command(name="uninstall", aliases=("rm",)) async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: """Uninstall a provided plugin, given the name of the plugin.""" plugin: Plugin = plugin - # plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) - await self.disable_plugin.callback(self, ctx, plugin=plugin) + if plugin.local: + await Response.send_negatory( + ctx, "You may not uninstall a local plugin.\nUse the disable command to stop using it." + ) + return + + plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) + + _, err = self.batch_manage(Action.UNLOAD, *plugin_files, is_plugin=True, suppress_already_error=True) + if err: + await Response.send_negatory(ctx, "There was a problem unloading the plugin from the bot.") + return shutil.rmtree(plugin.installed_path) @@ -249,33 +272,49 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No del PLUGINS[file_] del self.bot.installed_plugins[plugin] - await ctx.send(plugin.installed_path) + await Response.send_positive(ctx, f"Successfully uninstalled plugin {plugin}") + + async def _enable_or_disable_plugin( + self, + ctx: Messageable, + plugin: Plugin, + action: Action, + enable: bool, + ) -> None: + """Enables or disables a provided plugin.""" + verb = action.name.lower() + if plugin.enabled == enable: + await Response.send_negatory(ctx, f"Plugin {plugin!s} is already {verb}d.") + return - @plugins_group.command(name="enable") - async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: - """Enable a provided plugin, given the name or folder of the plugin.""" - plugin: Plugin = plugin - plugin.enabled = True + plugin.enabled = enable plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) if plugin.local: - update_local_toml_enable_or_disable(plugin) + try: + update_local_toml_enable_or_disable(plugin) + except (NoPluginTomlFoundError, ParseError) as e: + plugin.enabled = not plugin.enabled # reverse the state + await Response.send_negatory(ctx, e.args[0]) + + msg, err = self.batch_manage(Action.LOAD, *plugin_files, is_plugin=True, suppress_already_error=True) + if err: + await Response.send_negatory( + ctx, "Er, something went wrong.\n" f":x: {plugin!s} was unable to be {verb}d properly!" + ) + else: + await Response.send_positive(ctx, f":thumbsup: Plugin {plugin!s} successfully {verb}d.") - await self.load_plugins.callback(self, ctx, *plugin_files) + @plugins_group.command(name="enable") + async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + """Enable a provided plugin, given the name or folder of the plugin.""" + await self._enable_or_disable_plugin(ctx, plugin, Action.ENABLE, True) @plugins_group.command(name="disable") async def disable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: """Disable a provided plugin, given the name or folder of the plugin.""" - plugin: Plugin = plugin - plugin.enabled = False - - plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) - - if plugin.local: - update_local_toml_enable_or_disable(plugin) - - await self.unload_plugins.callback(self, ctx, *plugin_files) + await self._enable_or_disable_plugin(ctx, plugin, Action.DISABLE, False) def group_plugin_statuses(self) -> Mapping[str, str]: """Return a mapping of plugin names and statuses to their module.""" @@ -291,11 +330,14 @@ def group_plugin_statuses(self) -> Mapping[str, str]: plug_status.append(status) if all(plug_status): - status = ":green_circle:" + status = StatusEmojis.fully_loaded elif any(plug_status): - status = ":yellow_circle:" + status = StatusEmojis.partially_loaded else: - status = ":red_circle:" + if plug.enabled: + status = StatusEmojis.unloaded + else: + status = StatusEmojis.disabled plugins[plug.name] = status diff --git a/modmail/utils/responses.py b/modmail/utils/responses.py new file mode 100644 index 00000000..4366a6db --- /dev/null +++ b/modmail/utils/responses.py @@ -0,0 +1,143 @@ +import logging +from random import choice +from typing import List, final + +import discord + +from modmail.log import ModmailLogger + + +__all__ = ("Response",) + +_UNSET = object() + +logger: ModmailLogger = logging.getLogger() + + +@final +class Response: + """Responses from the bot to the user.""" + + success_color = discord.Colour.green() + success_headers: List[str] = [ + "You got it.", + "Done.", + "Affirmative.", + "As you wish.", + "Okay.", + "Fine by me.", + "There we go.", + "Sure!", + "Your wish is my command.", + ] + error_color = discord.Colour.red() + error_headers: List[str] = [ + "Abort!", + "FAIL.", + "I cannot do that.", + "I'm leaving you.", + "Its not me, its you.", + "Hold up!", + "Mistakes were made.", + "Nope.", + "Not happening.", + "Oops.", + "Something went wrong.", + "Sorry, no.", + "This will never work.", + "Uh. No.", + "\U0001f914", + "That is not happening.", + "Whups.", + ] + + @classmethod + async def send_positive( + cls, + channel: discord.abc.Messageable, + response: str, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, + ) -> discord.Message: + """ + Send an affirmative response. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) + + logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") + + if embed is None: + return await channel.send(response, **kwargs) + + if colour is _UNSET: + colour = cls.success_color + + if embed is _UNSET: + embed = discord.Embed(colour=colour) + embed.title = choice(cls.success_headers) + embed.description = response + + return await channel.send(embed=embed, **kwargs) + + @classmethod + async def send_negatory( + cls, + channel: discord.abc.Messageable, + response: str, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, + ) -> discord.Message: + """ + Send a negatory response. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) + + logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") + + if embed is None: + return await channel.send(response, **kwargs) + + if colour is _UNSET: + colour = cls.error_color + + if embed is _UNSET: + embed = discord.Embed(colour=colour) + embed.title = choice(cls.error_headers) + embed.description = response + + return await channel.send(embed=embed, **kwargs) + + @classmethod + async def send_response( + cls, + channel: discord.abc.Messageable, + response: str, + success: bool, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, + ) -> discord.Message: + """ + Send a response based on success or failure. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + if success: + return await cls.send_positive(channel, response, embed, colour, **kwargs) + else: + return await cls.send_negatory(channel, response, embed, colour, **kwargs) From 0b73254e1bf1085340a198b3304d3eaff2120304 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 9 Sep 2021 21:30:08 -0400 Subject: [PATCH 054/100] chore: remove the Responses class and switch to using the responses module --- modmail/extensions/extension_manager.py | 14 +- modmail/extensions/plugin_manager.py | 28 +-- modmail/utils/responses.py | 265 ++++++++++++------------ 3 files changed, 158 insertions(+), 149 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index b4cd3ebe..035c66a3 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -13,10 +13,10 @@ from modmail.bot import ModmailBot from modmail.log import ModmailLogger +from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator -from modmail.utils.responses import Response log: ModmailLogger = logging.getLogger(__name__) @@ -129,7 +129,7 @@ async def load_extensions(self, ctx: Context, *extensions: ExtensionConverter) - extensions = sorted(ext for ext in self.all_extensions if ext not in self.bot.extensions.keys()) msg, is_error = self.batch_manage(Action.LOAD, *extensions) - await Response.send_response(ctx, msg, not is_error) + await responses.send_response(ctx, msg, not is_error) @extensions_group.command(name="unload", aliases=("ul",)) async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -146,7 +146,7 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) if blacklisted: bl_msg = "\n".join(blacklisted) - await Response.send_negatory( + await responses.send_negatory_response( ctx, f":x: The following {self.type}(s) may not be unloaded:```\n{bl_msg}```" ) return @@ -159,7 +159,7 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) ) msg, is_error = self.batch_manage(Action.UNLOAD, *extensions) - await Response.send_response(ctx, msg, not is_error) + await responses.send_response(ctx, msg, not is_error) @extensions_group.command(name="reload", aliases=("r", "rl")) async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -178,7 +178,7 @@ async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) extensions = self.bot.extensions.keys() & self.all_extensions.keys() msg, is_error = self.batch_manage(Action.RELOAD, *extensions) - await Response.send_response(ctx, msg, not is_error) + await responses.send_response(ctx, msg, not is_error) @extensions_group.command(name="list", aliases=("all", "ls")) async def list_extensions(self, ctx: Context) -> None: @@ -234,7 +234,7 @@ async def resync_extensions(self, ctx: Context) -> None: Typical use case is in the event that the existing extensions have changed while the bot is running. """ self._resync_extensions() - await Response.send_positive(ctx, f":ok_hand: Refreshed list of {self.type}s.") + await responses.send_positive_response(ctx, f":ok_hand: Refreshed list of {self.type}s.") def group_extension_statuses(self) -> Mapping[str, str]: """Return a mapping of extension names and statuses to their categories.""" @@ -341,7 +341,7 @@ async def cog_check(self, ctx: Context) -> bool: async def cog_command_error(self, ctx: Context, error: Exception) -> None: """Handle BadArgument errors locally to prevent the help command from showing.""" if isinstance(error, commands.BadArgument): - await Response.send_negatory(ctx, str(error)) + await responses.send_negatory_response(ctx, str(error)) error.handled = True else: raise error diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index eb885bea..735d57f5 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -25,10 +25,10 @@ walk_plugin_files, ) from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager, StatusEmojis +from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata from modmail.utils.extensions import BOT_MODE from modmail.utils.pagination import ButtonPaginator -from modmail.utils.responses import Response if TYPE_CHECKING: @@ -178,7 +178,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu if source.source_type is SourceTypeEnum.LOCAL: # TODO: check the path of a local plugin - await Response.send_negatory( + await responses.send_negatory_response( ctx, "This plugin seems to be a local plugin, and therefore can probably be " "loaded with the load command, if it isn't loaded already.", @@ -188,7 +188,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu try: directory = await addon_utils.download_and_unpack_source(source, self.bot.http_session) except errors.HTTPError: - await Response.send_negatory( + await responses.send_negatory_response( ctx, f"Downloading {source.zip_url} did not give a 200 response code." ) return @@ -211,7 +211,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu try: shutil.copytree(p.folder_path, install_path, dirs_exist_ok=True) except FileExistsError: - await Response.send_negatory( + await responses.send_negatory_response( ctx, "Plugin already seems to be installed. " "This could be caused by the plugin already existing, " @@ -223,7 +223,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu break if plugin.folder_path is None: - await Response.send_negatory(ctx, f"Could not find plugin {plugin}") + await responses.send_negatory_response(ctx, f"Could not find plugin {plugin}") return logger.trace(f"{BASE_PLUGIN_PATH = }") @@ -245,7 +245,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu self.bot.installed_plugins[plugin] = files_to_load - await Response.send_positive(ctx, f"Installed plugin {plugin}.") + await responses.send_positive_response(ctx, f"Installed plugin {plugin}.") @plugins_group.command(name="uninstall", aliases=("rm",)) async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: @@ -253,7 +253,7 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No plugin: Plugin = plugin if plugin.local: - await Response.send_negatory( + await responses.send_negatory_response( ctx, "You may not uninstall a local plugin.\nUse the disable command to stop using it." ) return @@ -262,7 +262,9 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No _, err = self.batch_manage(Action.UNLOAD, *plugin_files, is_plugin=True, suppress_already_error=True) if err: - await Response.send_negatory(ctx, "There was a problem unloading the plugin from the bot.") + await responses.send_negatory_response( + ctx, "There was a problem unloading the plugin from the bot." + ) return shutil.rmtree(plugin.installed_path) @@ -272,7 +274,7 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No del PLUGINS[file_] del self.bot.installed_plugins[plugin] - await Response.send_positive(ctx, f"Successfully uninstalled plugin {plugin}") + await responses.send_positive_response(ctx, f"Successfully uninstalled plugin {plugin}") async def _enable_or_disable_plugin( self, @@ -284,7 +286,7 @@ async def _enable_or_disable_plugin( """Enables or disables a provided plugin.""" verb = action.name.lower() if plugin.enabled == enable: - await Response.send_negatory(ctx, f"Plugin {plugin!s} is already {verb}d.") + await responses.send_negatory_response(ctx, f"Plugin {plugin!s} is already {verb}d.") return plugin.enabled = enable @@ -296,15 +298,15 @@ async def _enable_or_disable_plugin( update_local_toml_enable_or_disable(plugin) except (NoPluginTomlFoundError, ParseError) as e: plugin.enabled = not plugin.enabled # reverse the state - await Response.send_negatory(ctx, e.args[0]) + await responses.send_negatory_response(ctx, e.args[0]) msg, err = self.batch_manage(Action.LOAD, *plugin_files, is_plugin=True, suppress_already_error=True) if err: - await Response.send_negatory( + await responses.send_negatory_response( ctx, "Er, something went wrong.\n" f":x: {plugin!s} was unable to be {verb}d properly!" ) else: - await Response.send_positive(ctx, f":thumbsup: Plugin {plugin!s} successfully {verb}d.") + await responses.send_positive_response(ctx, f":thumbsup: Plugin {plugin!s} successfully {verb}d.") @plugins_group.command(name="enable") async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: diff --git a/modmail/utils/responses.py b/modmail/utils/responses.py index 4366a6db..649df924 100644 --- a/modmail/utils/responses.py +++ b/modmail/utils/responses.py @@ -1,143 +1,150 @@ +""" +Helper methods for responses from the bot to the user. + +These help ensure consistency between errors, as they will all be consistent between different uses. +""" import logging from random import choice -from typing import List, final +from typing import List import discord from modmail.log import ModmailLogger -__all__ = ("Response",) +__all__ = ( + "default_success_color", + "success_headers", + "default_error_color", + "error_headers", + "send_positive_response", + "send_negatory_response", + "send_response", +) _UNSET = object() logger: ModmailLogger = logging.getLogger() -@final -class Response: - """Responses from the bot to the user.""" - - success_color = discord.Colour.green() - success_headers: List[str] = [ - "You got it.", - "Done.", - "Affirmative.", - "As you wish.", - "Okay.", - "Fine by me.", - "There we go.", - "Sure!", - "Your wish is my command.", - ] - error_color = discord.Colour.red() - error_headers: List[str] = [ - "Abort!", - "FAIL.", - "I cannot do that.", - "I'm leaving you.", - "Its not me, its you.", - "Hold up!", - "Mistakes were made.", - "Nope.", - "Not happening.", - "Oops.", - "Something went wrong.", - "Sorry, no.", - "This will never work.", - "Uh. No.", - "\U0001f914", - "That is not happening.", - "Whups.", - ] - - @classmethod - async def send_positive( - cls, - channel: discord.abc.Messageable, - response: str, - embed: discord.Embed = _UNSET, - colour: discord.Colour = _UNSET, - **kwargs, - ) -> discord.Message: - """ - Send an affirmative response. - - Requires a messageable, and a response. - If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. - If embed is provided, this method will send a response using the provided embed, edited in place. - Extra kwargs are passed to Messageable.send() - """ - kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) - - logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") - - if embed is None: - return await channel.send(response, **kwargs) - - if colour is _UNSET: - colour = cls.success_color - - if embed is _UNSET: - embed = discord.Embed(colour=colour) - embed.title = choice(cls.success_headers) - embed.description = response - - return await channel.send(embed=embed, **kwargs) - - @classmethod - async def send_negatory( - cls, - channel: discord.abc.Messageable, - response: str, - embed: discord.Embed = _UNSET, - colour: discord.Colour = _UNSET, - **kwargs, - ) -> discord.Message: - """ - Send a negatory response. - - Requires a messageable, and a response. - If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. - If embed is provided, this method will send a response using the provided embed, edited in place. - Extra kwargs are passed to Messageable.send() - """ - kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) - - logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") - - if embed is None: - return await channel.send(response, **kwargs) - - if colour is _UNSET: - colour = cls.error_color - - if embed is _UNSET: - embed = discord.Embed(colour=colour) - embed.title = choice(cls.error_headers) - embed.description = response - - return await channel.send(embed=embed, **kwargs) - - @classmethod - async def send_response( - cls, - channel: discord.abc.Messageable, - response: str, - success: bool, - embed: discord.Embed = _UNSET, - colour: discord.Colour = _UNSET, - **kwargs, - ) -> discord.Message: - """ - Send a response based on success or failure. - - Requires a messageable, and a response. - If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. - If embed is provided, this method will send a response using the provided embed, edited in place. - Extra kwargs are passed to Messageable.send() - """ - if success: - return await cls.send_positive(channel, response, embed, colour, **kwargs) - else: - return await cls.send_negatory(channel, response, embed, colour, **kwargs) +default_success_color = discord.Colour.green() +success_headers: List[str] = [ + "You got it.", + "Done.", + "Affirmative.", + "As you wish.", + "Okay.", + "Fine by me.", + "There we go.", + "Sure!", + "Your wish is my command.", +] + +default_error_color = discord.Colour.red() +error_headers: List[str] = [ + "Abort!", + "FAIL.", + "I cannot do that.", + "I'm leaving you.", + "Its not me, its you.", + "Hold up!", + "Mistakes were made.", + "Nope.", + "Not happening.", + "Oops.", + "Something went wrong.", + "Sorry, no.", + "This will never work.", + "Uh. No.", + "\U0001f914", + "That is not happening.", + "Whups.", +] + + +async def send_positive_response( + channel: discord.abc.Messageable, + response: str, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, +) -> discord.Message: + """ + Send an affirmative response. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) + + logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") + + if embed is None: + return await channel.send(response, **kwargs) + + if colour is _UNSET: + colour = default_success_color + + if embed is _UNSET: + embed = discord.Embed(colour=colour) + embed.title = choice(success_headers) + embed.description = response + + return await channel.send(embed=embed, **kwargs) + + +async def send_negatory_response( + channel: discord.abc.Messageable, + response: str, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, +) -> discord.Message: + """ + Send a negatory response. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) + + logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") + + if embed is None: + return await channel.send(response, **kwargs) + + if colour is _UNSET: + colour = default_error_color + + if embed is _UNSET: + embed = discord.Embed(colour=colour) + embed.title = choice(error_headers) + embed.description = response + + return await channel.send(embed=embed, **kwargs) + + +async def send_response( + channel: discord.abc.Messageable, + response: str, + success: bool, + embed: discord.Embed = _UNSET, + colour: discord.Colour = _UNSET, + **kwargs, +) -> discord.Message: + """ + Send a response based on success or failure. + + Requires a messageable, and a response. + If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. + If embed is provided, this method will send a response using the provided embed, edited in place. + Extra kwargs are passed to Messageable.send() + """ + if success: + return await send_positive_response(channel, response, embed, colour, **kwargs) + else: + return await send_negatory_response(channel, response, embed, colour, **kwargs) From e98ddaea641beedd7c2d4617543ffe2757e22390 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 10 Sep 2021 03:12:28 -0400 Subject: [PATCH 055/100] breaking: move the plugin helpers file --- modmail/{plugins => addons}/helpers.py | 0 modmail/plugins/__init__.py | 1 - 2 files changed, 1 deletion(-) rename modmail/{plugins => addons}/helpers.py (100%) diff --git a/modmail/plugins/helpers.py b/modmail/addons/helpers.py similarity index 100% rename from modmail/plugins/helpers.py rename to modmail/addons/helpers.py diff --git a/modmail/plugins/__init__.py b/modmail/plugins/__init__.py index 3125625e..e69de29b 100644 --- a/modmail/plugins/__init__.py +++ b/modmail/plugins/__init__.py @@ -1 +0,0 @@ -from .helpers import BotModeEnum, ExtMetadata, PluginCog From c874b5a6f14c113dbbe6c9f5f299614057159f9b Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 10 Sep 2021 03:37:00 -0400 Subject: [PATCH 056/100] minor: fix responses.py logger --- modmail/utils/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/utils/responses.py b/modmail/utils/responses.py index 649df924..e22af0e4 100644 --- a/modmail/utils/responses.py +++ b/modmail/utils/responses.py @@ -24,7 +24,7 @@ _UNSET = object() -logger: ModmailLogger = logging.getLogger() +logger: ModmailLogger = logging.getLogger(__name__) default_success_color = discord.Colour.green() From 68490dbedfbc593fe1bb4bf86215e3b1b21c4dad Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 10 Sep 2021 03:37:32 -0400 Subject: [PATCH 057/100] fix: handle status of plugins with no files --- modmail/extensions/extension_manager.py | 1 + modmail/extensions/plugin_manager.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 035c66a3..d56f49de 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -32,6 +32,7 @@ class StatusEmojis: partially_loaded: str = ":yellow_circle:" unloaded: str = ":red_circle:" disabled: str = ":brown_circle:" + unknown: str = ":black_circle:" class Action(Enum): diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 735d57f5..0ba5dbe5 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -331,7 +331,9 @@ def group_plugin_statuses(self) -> Mapping[str, str]: status = False plug_status.append(status) - if all(plug_status): + if len(plug_status) == 0: + status = StatusEmojis.unknown + elif all(plug_status): status = StatusEmojis.fully_loaded elif any(plug_status): status = StatusEmojis.partially_loaded From 7f5341ce539a46ba8c44ab527a6b3c0f6419ffa3 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 10 Sep 2021 22:53:58 -0400 Subject: [PATCH 058/100] breaking: restructure how extensions are stored major EXT_METADATA improvements --- modmail/bot.py | 19 ++++--- modmail/extensions/extension_manager.py | 22 ++++---- modmail/extensions/utils/paginator_manager.py | 4 +- modmail/utils/cogs.py | 5 +- modmail/utils/extensions.py | 54 ++++++++++++------- 5 files changed, 60 insertions(+), 44 deletions(-) diff --git a/modmail/bot.py b/modmail/bot.py index ee928c3c..400c3051 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -14,7 +14,7 @@ from modmail.addons.plugins import PLUGINS, find_local_plugins, walk_plugin_files from modmail.config import CONFIG from modmail.log import ModmailLogger -from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, walk_extensions +from modmail.utils.extensions import BOT_MODE, EXTENSIONS, NO_UNLOAD, walk_extensions if TYPE_CHECKING: @@ -39,6 +39,7 @@ class ModmailBot(commands.Bot): """ logger: ModmailLogger = logging.getLogger(__name__) + mode: int def __init__(self, **kwargs): self.config = CONFIG @@ -173,17 +174,19 @@ async def close(self) -> None: def load_extensions(self) -> None: """Load all enabled extensions.""" + self.mode = BOT_MODE EXTENSIONS.update(walk_extensions()) - # set up no_unload global too - for ext, value in EXTENSIONS.items(): - if value[1]: + for ext, metadata in EXTENSIONS.items(): + # set up no_unload global too + if metadata.no_unload: NO_UNLOAD.append(ext) - for extension, value in EXTENSIONS.items(): - if value[0]: - self.logger.debug(f"Loading extension {extension}") - self.load_extension(extension) + if metadata.load_if_mode & BOT_MODE: + self.logger.info(f"Loading extension {ext}") + self.load_extension(ext) + else: + self.logger.debug(f"SKIPPING load of extension {ext} due to BOT_MODE.") def load_plugins(self) -> None: """Load all enabled plugins.""" diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index d56f49de..8b33ba15 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from enum import Enum -from typing import Mapping, Optional, Tuple +from typing import Dict, Mapping, Optional, Tuple from discord import Colour, Embed from discord.ext import commands @@ -15,7 +15,7 @@ from modmail.log import ModmailLogger from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog -from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, unqualify, walk_extensions +from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, ModuleName, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator @@ -176,7 +176,7 @@ async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) return if "*" in extensions: - extensions = self.bot.extensions.keys() & self.all_extensions.keys() + extensions = self.bot.extensions.keys() & self.all_extensions msg, is_error = self.batch_manage(Action.RELOAD, *extensions) await responses.send_response(ctx, msg, not is_error) @@ -215,17 +215,17 @@ def _resync_extensions(self) -> None: log.debug(f"Refreshing list of {self.type}s.") # make sure the new walk contains all currently loaded extensions, so they can be unloaded - loaded_extensions = {} - for name, should_load in self.all_extensions.items(): + all_exts: Dict[ModuleName, ExtMetadata] = {} + for name, metadata in self.all_extensions.items(): if name in self.bot.extensions: - loaded_extensions[name] = should_load + all_exts[name] = metadata + + # re-walk the extensions + for name, metadata in self.refresh_method(): + all_exts[name] = metadata - # now that we know what the list was, we can clear it self.all_extensions.clear() - # put the loaded extensions back in - self.all_extensions.update(loaded_extensions) - # now we can re-walk the extensions - self.all_extensions.update(self.refresh_method()) + self.all_extensions.update(all_exts) @extensions_group.command(name="refresh", aliases=("rewalk", "rescan")) async def resync_extensions(self, ctx: Context) -> None: diff --git a/modmail/extensions/utils/paginator_manager.py b/modmail/extensions/utils/paginator_manager.py index 899c0d4f..7bd501a2 100644 --- a/modmail/extensions/utils/paginator_manager.py +++ b/modmail/extensions/utils/paginator_manager.py @@ -6,7 +6,7 @@ from discord import InteractionType -from modmail.utils.cogs import ModmailCog +from modmail.utils.cogs import ExtMetadata, ModmailCog if TYPE_CHECKING: @@ -17,6 +17,8 @@ logger: ModmailLogger = logging.getLogger(__name__) +EXT_METADATA = ExtMetadata + class PaginatorManager(ModmailCog): """Handles paginators that were still active when the bot shut down.""" diff --git a/modmail/utils/cogs.py b/modmail/utils/cogs.py index 75554e95..5e0dc3f2 100644 --- a/modmail/utils/cogs.py +++ b/modmail/utils/cogs.py @@ -24,9 +24,6 @@ class BotModeEnum(BitwiseAutoEnum): PLUGIN_DEV = auto() -BOT_MODES = BotModeEnum - - @dataclass() class ExtMetadata: """Ext metadata class to determine if extension should load at runtime depending on bot configuration.""" @@ -35,7 +32,7 @@ class ExtMetadata: # this is to determine if the cog is allowed to be unloaded. no_unload: bool = False - def __init__(self, load_if_mode: BotModeEnum = BotModeEnum.PRODUCTION, no_unload: bool = False): + def __init__(self, *, load_if_mode: BotModeEnum = BotModeEnum.PRODUCTION, no_unload: bool = False): self.load_if_mode = load_if_mode self.no_unload = no_unload diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index ce94fc1f..00ea2acf 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -1,25 +1,26 @@ -# original source: +# initial source: # https://github.com/python-discord/bot/blob/a8869b4d60512b173871c886321b261cbc4acca9/bot/utils/extensions.py # MIT License 2021 Python Discord import importlib import inspect import logging import pkgutil -import typing as t +from typing import Dict, Generator, List, NewType, NoReturn, Tuple from modmail import extensions from modmail.config import CONFIG from modmail.log import ModmailLogger -from modmail.utils.cogs import BOT_MODES, BotModeEnum, ExtMetadata +from modmail.utils.cogs import BotModeEnum, ExtMetadata log: ModmailLogger = logging.getLogger(__name__) EXT_METADATA = ExtMetadata +ModuleName = NewType("ModuleName", str) -EXTENSIONS: t.Dict[str, t.Tuple[bool, bool]] = dict() -NO_UNLOAD: t.List[str] = list() +EXTENSIONS: Dict[ModuleName, ExtMetadata] = dict() +NO_UNLOAD: List[ModuleName] = list() def unqualify(name: str) -> str: @@ -44,14 +45,14 @@ def determine_bot_mode() -> int: log.trace(f"BOT_MODE value: {BOT_MODE}") -log.debug(f"Dev mode status: {bool(BOT_MODE & BOT_MODES.DEVELOP)}") -log.debug(f"Plugin dev mode status: {bool(BOT_MODE & BOT_MODES.PLUGIN_DEV)}") +log.debug(f"Dev mode status: {bool(BOT_MODE & BotModeEnum.DEVELOP)}") +log.debug(f"Plugin dev mode status: {bool(BOT_MODE & BotModeEnum.PLUGIN_DEV)}") -def walk_extensions() -> t.Iterator[t.Tuple[str, t.Tuple[bool, bool]]]: +def walk_extensions() -> Generator[Tuple[ModuleName, ExtMetadata], None, None]: """Yield extension names from the modmail.exts subpackage.""" - def on_error(name: str) -> t.NoReturn: + def on_error(name: str) -> NoReturn: raise ImportError(name=name) # pragma: no cover for module in pkgutil.walk_packages(extensions.__path__, f"{extensions.__name__}.", onerror=on_error): @@ -60,21 +61,34 @@ def on_error(name: str) -> t.NoReturn: continue imported = importlib.import_module(module.name) - if module.ispkg: - if not inspect.isfunction(getattr(imported, "setup", None)): - # If it lacks a setup function, it's not an extension. - continue + if not inspect.isfunction(getattr(imported, "setup", None)): + # If it lacks a setup function, it's not an extension. + continue ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) if ext_metadata is not None: - # check if this cog is dev only or plugin dev only - load_cog = bool(ext_metadata.load_if_mode.value & BOT_MODE) - log.trace(f"Load cog {module.name!r}?: {load_cog}") - no_unload = ext_metadata.no_unload - yield module.name, (load_cog, no_unload) + if not isinstance(ext_metadata, ExtMetadata): + if ext_metadata == ExtMetadata: + log.info( + f"{module.name!r} seems to have passed the ExtMetadata class directly to " + "EXT_METADATA. Using defaults." + ) + else: + log.error( + f"Extension {module.name!r} contains an invalid EXT_METADATA variable. " + "Loading with metadata defaults. Please report this bug to the developers." + ) + yield module.name, ExtMetadata() + continue + + log.debug(f"{module.name!r} contains a EXT_METADATA variable. Loading it.") + + yield module.name, ext_metadata continue - log.notice(f"Cog {module.name!r} is missing an EXT_METADATA variable. Assuming its a normal cog.") + log.notice( + f"Extension {module.name!r} is missing an EXT_METADATA variable. Assuming its a normal extension." + ) # Presume Production Mode/Metadata defaults if metadata var does not exist. - yield module.name, (ExtMetadata.load_if_mode, ExtMetadata.no_unload) + yield module.name, ExtMetadata() From fff1e903868c7b7687b717d665e258ba2323b8d0 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 11 Sep 2021 01:10:14 -0400 Subject: [PATCH 059/100] implement partial plugin restructure restructure plugin variables to be a global list of plugins with the extensions as an attribute --- modmail/addons/models.py | 3 ++ modmail/addons/plugins.py | 60 +++++++++++++++---------- modmail/bot.py | 60 +++++++++++++------------ modmail/extensions/extension_manager.py | 6 +-- modmail/extensions/plugin_manager.py | 6 +-- modmail/utils/extensions.py | 3 +- 6 files changed, 80 insertions(+), 58 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 1debeef8..2fa1bf63 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -9,6 +9,8 @@ import pathlib import zipfile + from modmail.utils.extensions import ModuleDict + class SourceTypeEnum(Enum): """Which source an addon is from.""" @@ -140,6 +142,7 @@ class Plugin(Addon): extra_kwargs: Dict[str, Any] installed_path: Optional[pathlib.Path] extension_files: List[pathlib.Path] + modules: ModuleDict def __init__( self, diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 832f320f..33d6b59b 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -15,7 +15,8 @@ import logging import os import pathlib -from typing import Dict, Iterator, List, Tuple +from collections.abc import Generator, MutableSet +from typing import Dict, List, Tuple import atoml @@ -24,7 +25,7 @@ from modmail.addons.models import Plugin from modmail.log import ModmailLogger from modmail.utils.cogs import ExtMetadata -from modmail.utils.extensions import BOT_MODE, unqualify +from modmail.utils.extensions import ModuleName, unqualify logger: ModmailLogger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ BASE_PLUGIN_PATH = pathlib.Path(plugins.__file__).parent.resolve() -PLUGINS: Dict[str, Tuple[bool, bool]] = dict() +PLUGINS: MutableSet[Plugin] = set() PLUGIN_TOML = "plugin.toml" @@ -194,13 +195,13 @@ def find_plugins_in_dir( def find_local_plugins( detection_path: pathlib.Path = BASE_PLUGIN_PATH, / # noqa: W504 -) -> Dict[Plugin, List[str]]: +) -> Generator[Plugin, None, None]: """ Walks the local path, and determines which files are local plugins. Yields a list of plugins, """ - all_plugins: Dict[Plugin, List[str]] = {} + all_plugins: MutableSet[Plugin] = set() toml_plugins: List[Plugin] = [] toml_path = LOCAL_PLUGIN_TOML @@ -221,11 +222,11 @@ def find_local_plugins( for p in toml_plugins: if p.folder_name == path.name: p.folder_path = path - all_plugins[p] = list() + all_plugins.add(p) - logger.debug(f"Local plugins detected: {[p.name for p in all_plugins.keys()]}") + logger.debug(f"Local plugins detected: {[p.name for p in all_plugins]}") - for plugin_ in all_plugins.keys(): + for plugin_ in all_plugins: logger.trace(f"{plugin_.folder_path =}") plugin_.local = True # take this as an opportunity to configure local to True on all plugins for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): @@ -238,17 +239,16 @@ def find_local_plugins( if "__pycache__" in dir_ or "__pycache__" in dirpath: continue - modules = [x for x, y in walk_plugin_files(dirpath)] + plugin_.modules = {} + plugin_.modules.update(walk_plugin_files(dirpath)) + yield plugin_ - all_plugins[plugin_].extend(modules) - - logger.debug(f"{all_plugins.keys() = }") - logger.debug(f"{all_plugins.values() = }") - - return all_plugins + logger.debug(f"{all_plugins = }") -def walk_plugin_files(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterator[Tuple[str, bool]]: +def walk_plugin_files( + detection_path: pathlib.Path = BASE_PLUGIN_PATH, +) -> Generator[Tuple[ModuleName, ExtMetadata], None, None]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder # this is to ensure folder symlinks are supported, @@ -293,15 +293,29 @@ def walk_plugin_files(detection_path: pathlib.Path = BASE_PLUGIN_PATH) -> Iterat ext_metadata: ExtMetadata = getattr(imported, "EXT_METADATA", None) if ext_metadata is not None: - # check if this plugin is dev only or plugin dev only - load_cog = bool(int(ext_metadata.load_if_mode) & BOT_MODE) - logger.trace(f"Load plugin {imported.__name__!r}?: {load_cog}") - yield imported.__name__, load_cog + if not isinstance(ext_metadata, ExtMetadata): + if ext_metadata == ExtMetadata: + logger.info( + f"{imported.__name__!r} seems to have passed the ExtMetadata class directly to " + "EXT_METADATA. Using defaults." + ) + else: + logger.error( + f"Plugin extension {imported.__name__!r} contains an invalid EXT_METADATA variable. " + "Loading with metadata defaults. Please report this bug to the developers." + ) + yield imported.__name__, ExtMetadata() + continue + + logger.debug(f"{imported.__name__!r} contains a EXT_METADATA variable. Loading it.") + + yield imported.__name__, ext_metadata continue - logger.info( - f"Plugin {imported.__name__!r} is missing a EXT_METADATA variable. Assuming its a normal plugin." + logger.notice( + f"Plugin extension {imported.__name__!r} is missing an EXT_METADATA variable. " + "Assuming its a normal plugin extension." ) # Presume Production Mode/Metadata defaults if metadata var does not exist. - yield imported.__name__, bool(ExtMetadata.load_if_mode) + yield imported.__name__, ExtMetadata() diff --git a/modmail/bot.py b/modmail/bot.py index 400c3051..7236144c 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -1,7 +1,8 @@ import asyncio import logging import signal -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from collections.abc import MutableSet +from typing import Any, Optional import arrow import discord @@ -11,16 +12,13 @@ from discord.ext import commands from modmail.addons.errors import NoPluginTomlFoundError -from modmail.addons.plugins import PLUGINS, find_local_plugins, walk_plugin_files +from modmail.addons.models import Plugin +from modmail.addons.plugins import PLUGINS, find_local_plugins from modmail.config import CONFIG from modmail.log import ModmailLogger from modmail.utils.extensions import BOT_MODE, EXTENSIONS, NO_UNLOAD, walk_extensions -if TYPE_CHECKING: - from modmail.addons.models import Plugin - - REQUIRED_INTENTS = Intents( guilds=True, messages=True, @@ -47,7 +45,7 @@ def __init__(self, **kwargs): self.http_session: Optional[ClientSession] = None # keys: plugins, list values: all plugin files - self.installed_plugins: Dict["Plugin", List[str]] = {} + self.installed_plugins: MutableSet[Plugin] = {} status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") @@ -148,7 +146,12 @@ def stop_loop_on_completion(f: Any) -> None: async def close(self) -> None: """Safely close HTTP session, unload plugins and extensions when the bot is shutting down.""" - plugins = self.extensions & PLUGINS.keys() + plugins = [] + for plug in PLUGINS: + plugins.extend([mod for mod in plug.modules]) + + plugins = self.extensions.keys() & plugins + for plug in list(plugins): try: self.unload_extension(plug) @@ -190,32 +193,33 @@ def load_extensions(self) -> None: def load_plugins(self) -> None: """Load all enabled plugins.""" - PLUGINS.update(walk_plugin_files()) + self.installed_plugins = PLUGINS + dont_load_at_start = [] try: - plugins = find_local_plugins() + for p in find_local_plugins(): + PLUGINS.add(p) except NoPluginTomlFoundError: - dont_load_at_start = [] + # no local plugins + pass else: - self.installed_plugins.update(plugins) - - dont_load_at_start = [] - for plug, modules in self.installed_plugins.items(): + for plug in self.installed_plugins: if plug.enabled: continue self.logger.debug(f"Not loading {plug.__str__()} on start since it's not enabled.") - dont_load_at_start.extend(modules) - - for plugin, should_load in PLUGINS.items(): - if should_load and plugin not in dont_load_at_start: - self.logger.debug(f"Loading plugin {plugin}") - try: - # since we're loading user generated content, - # any errors here will take down the entire bot - self.load_extension(plugin) - except Exception: - self.logger.error("Failed to load plugin {0}".format(plugin), exc_info=True) - else: - self.logger.debug(f"SKIPPED loading plugin {plugin}") + dont_load_at_start.extend(plug.modules) + + for plug in PLUGINS: + for mod, metadata in plug.modules.items(): + if metadata.load_if_mode & self.mode and mod not in dont_load_at_start: + self.logger.debug(f"Loading plugin {mod}") + try: + # since we're loading user generated content, + # any errors here will take down the entire bot + self.load_extension(mod) + except Exception: + self.logger.error(f"Failed to load plugin {mod!s}", exc_info=True) + else: + self.logger.debug(f"SKIPPED loading plugin {mod}") def add_cog(self, cog: commands.Cog, *, override: bool = False) -> None: """ diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 8b33ba15..e3696c21 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from enum import Enum -from typing import Dict, Mapping, Optional, Tuple +from typing import Mapping, Optional, Tuple from discord import Colour, Embed from discord.ext import commands @@ -15,7 +15,7 @@ from modmail.log import ModmailLogger from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog -from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, ModuleName, unqualify, walk_extensions +from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, ModuleDict, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator @@ -215,7 +215,7 @@ def _resync_extensions(self) -> None: log.debug(f"Refreshing list of {self.type}s.") # make sure the new walk contains all currently loaded extensions, so they can be unloaded - all_exts: Dict[ModuleName, ExtMetadata] = {} + all_exts: ModuleDict = {} for name, metadata in self.all_extensions.items(): if name in self.bot.extensions: all_exts[name] = metadata diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 0ba5dbe5..0e582ae9 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -4,7 +4,7 @@ import logging import shutil from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Mapping +from typing import TYPE_CHECKING, Dict, List, Mapping, MutableSet from atoml.exceptions import ParseError from discord import Colour, Embed @@ -35,7 +35,7 @@ from modmail.bot import ModmailBot from modmail.log import ModmailLogger -EXT_METADATA = ExtMetadata(load_if_mode=BotModeEnum.PRODUCTION, no_unload=True) +EXT_METADATA = ExtMetadata(no_unload=True) logger: ModmailLogger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ class PluginConverter(commands.Converter): async def convert(self, ctx: Context, argument: str) -> Plugin: """Converts a plugin into a full plugin with a path and all other attributes.""" - loaded_plugs: Dict[Plugin, List[str]] = ctx.bot.installed_plugins + loaded_plugs: MutableSet[Plugin] = ctx.bot.installed_plugins for plug in loaded_plugs: if argument in (plug.name, plug.folder_name): diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index 00ea2acf..ff20a1bf 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -18,8 +18,9 @@ EXT_METADATA = ExtMetadata ModuleName = NewType("ModuleName", str) +ModuleDict = Dict[ModuleName, ExtMetadata] -EXTENSIONS: Dict[ModuleName, ExtMetadata] = dict() +EXTENSIONS: ModuleDict = dict() NO_UNLOAD: List[ModuleName] = list() From c163723baebe96bbf9b3b5f9db547ce56cc27e2e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 11 Sep 2021 03:30:14 -0400 Subject: [PATCH 060/100] breaking: finish refactoring PLUGINS global --- modmail/addons/models.py | 2 + modmail/addons/plugins.py | 76 ++++++----- modmail/bot.py | 10 +- modmail/extensions/extension_manager.py | 10 +- modmail/extensions/plugin_manager.py | 170 +++++++++++++----------- 5 files changed, 145 insertions(+), 123 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 2fa1bf63..a005bdfd 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -167,6 +167,8 @@ def __init__( self.local = local self.enabled = enabled + self.modules = dict() + # store any extra kwargs here # this is to ensure backwards compatiablilty with plugins that support older versions, # but want to use newer toml options diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 33d6b59b..40255efd 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -15,8 +15,8 @@ import logging import os import pathlib -from collections.abc import Generator, MutableSet -from typing import Dict, List, Tuple +from collections.abc import Generator +from typing import List, Optional, Set, Tuple import atoml @@ -28,13 +28,27 @@ from modmail.utils.extensions import ModuleName, unqualify +__all__ = [ + "VALID_ZIP_PLUGIN_DIRECTORIES", + "BASE_PLUGIN_PATH", + "PLUGINS", + "PLUGIN_TOML", + "LOCAL_PLUGIN_TOML", + "parse_plugin_toml_from_string", + "update_local_toml_enable_or_disable", + "find_partial_plugins_from_dir", + "find_plugins", + "walk_plugin_files", +] + + logger: ModmailLogger = logging.getLogger(__name__) VALID_ZIP_PLUGIN_DIRECTORIES = ["plugins", "Plugins"] BASE_PLUGIN_PATH = pathlib.Path(plugins.__file__).parent.resolve() -PLUGINS: MutableSet[Plugin] = set() +PLUGINS: Set[Plugin] = set() PLUGIN_TOML = "plugin.toml" @@ -104,12 +118,12 @@ def update_local_toml_enable_or_disable(plugin: Plugin, /) -> None: f.write(doc.as_string()) -def find_plugins_in_dir( +def find_partial_plugins_from_dir( addon_repo_path: pathlib.Path, *, parse_toml: bool = True, no_toml_exist_ok: bool = True, -) -> Dict[Plugin, List[pathlib.Path]]: +) -> Generator[Plugin, None, None]: """ Find the plugins that are in a directory. @@ -142,66 +156,48 @@ def find_plugins_in_dir( plugin_directory = addon_repo_path / plugin_directory - all_plugins: Dict[Plugin, List[pathlib.Path]] = {} + all_plugins: Set[Plugin] = set() - toml_plugins: List[Plugin] = [] if parse_toml: toml_path = plugin_directory / PLUGIN_TOML if toml_path.exists(): # parse the toml with open(toml_path) as toml_file: - toml_plugins = parse_plugin_toml_from_string(toml_file.read()) + all_plugins.update(parse_plugin_toml_from_string(toml_file.read())) + elif no_toml_exist_ok: # toml does not exist but the caller does not care pass else: raise NoPluginTomlFoundError(toml_path, "does not exist") - logger.debug(f"{toml_plugins =}") - toml_plugin_names = [p.folder_name for p in toml_plugins] + logger.debug(f"{all_plugins =}") for path in plugin_directory.iterdir(): logger.debug(f"plugin_directory: {path}") if path.is_dir(): # use an existing toml plugin object - if path.name in toml_plugin_names: - for p in toml_plugins: + if path.name in all_plugins: + for p in all_plugins: if p.folder_name == path.name: p.folder_path = path - all_plugins[p] = list() + yield p + break else: - temp_plugin = Plugin(path.name, folder_path=path) - all_plugins[temp_plugin] = list() - - logger.debug(f"Plugins detected: {[p.name for p in all_plugins.keys()]}") - - for plugin_ in all_plugins.keys(): - logger.trace(f"{plugin_.folder_path =}") - for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): - logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") - for list_ in dirnames, filenames: - logger.trace(f"{list_ =}") - for file in list_: - logger.trace(f"{file =}") - if file == dirpath: # don't include files that are plugin directories - continue - - all_plugins[plugin_].append(pathlib.Path(file)) - - logger.debug(f"{all_plugins.keys() = }") - logger.debug(f"{all_plugins.values() = }") - - return all_plugins + logger.debug( + f"Plugin in {addon_repo_path!s} is not provided in toml. Creating new plugin object." + ) + yield Plugin(path.name, folder_path=path) -def find_local_plugins( - detection_path: pathlib.Path = BASE_PLUGIN_PATH, / # noqa: W504 +def find_plugins( + detection_path: pathlib.Path = BASE_PLUGIN_PATH, /, *, local: Optional[bool] = True ) -> Generator[Plugin, None, None]: """ Walks the local path, and determines which files are local plugins. Yields a list of plugins, """ - all_plugins: MutableSet[Plugin] = set() + all_plugins: Set[Plugin] = set() toml_plugins: List[Plugin] = [] toml_path = LOCAL_PLUGIN_TOML @@ -228,7 +224,9 @@ def find_local_plugins( for plugin_ in all_plugins: logger.trace(f"{plugin_.folder_path =}") - plugin_.local = True # take this as an opportunity to configure local to True on all plugins + if local is not None: + # configure all plugins with the provided local variable + plugin_.local = local for dirpath, dirnames, filenames in os.walk(plugin_.folder_path): logger.trace(f"{dirpath =}, {dirnames =}, {filenames =}") for list_ in dirnames, [dirpath]: diff --git a/modmail/bot.py b/modmail/bot.py index 7236144c..6f73bc0b 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -1,8 +1,7 @@ import asyncio import logging import signal -from collections.abc import MutableSet -from typing import Any, Optional +from typing import Any, Optional, Set import arrow import discord @@ -13,7 +12,7 @@ from modmail.addons.errors import NoPluginTomlFoundError from modmail.addons.models import Plugin -from modmail.addons.plugins import PLUGINS, find_local_plugins +from modmail.addons.plugins import PLUGINS, find_plugins from modmail.config import CONFIG from modmail.log import ModmailLogger from modmail.utils.extensions import BOT_MODE, EXTENSIONS, NO_UNLOAD, walk_extensions @@ -45,7 +44,7 @@ def __init__(self, **kwargs): self.http_session: Optional[ClientSession] = None # keys: plugins, list values: all plugin files - self.installed_plugins: MutableSet[Plugin] = {} + self.installed_plugins: Set[Plugin] = {} status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") @@ -196,8 +195,7 @@ def load_plugins(self) -> None: self.installed_plugins = PLUGINS dont_load_at_start = [] try: - for p in find_local_plugins(): - PLUGINS.add(p) + PLUGINS.update(find_plugins()) except NoPluginTomlFoundError: # no local plugins pass diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index e3696c21..688b8b56 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -15,7 +15,7 @@ from modmail.log import ModmailLogger from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog -from modmail.utils.extensions import EXTENSIONS, NO_UNLOAD, ModuleDict, unqualify, walk_extensions +from modmail.utils.extensions import BOT_MODE, EXTENSIONS, NO_UNLOAD, ModuleDict, unqualify, walk_extensions from modmail.utils.pagination import ButtonPaginator @@ -100,11 +100,11 @@ class ExtensionManager(ModmailCog, name="Extension Manager"): type = "extension" module_name = "extensions" # modmail/extensions + all_extensions: ModuleDict def __init__(self, bot: ModmailBot): self.bot = bot self.all_extensions = EXTENSIONS - self.refresh_method = walk_extensions def get_black_listed_extensions(self) -> list: """Returns a list of all unload blacklisted extensions.""" @@ -221,7 +221,7 @@ def _resync_extensions(self) -> None: all_exts[name] = metadata # re-walk the extensions - for name, metadata in self.refresh_method(): + for name, metadata in walk_extensions(): all_exts[name] = metadata self.all_extensions.clear() @@ -241,9 +241,11 @@ def group_extension_statuses(self) -> Mapping[str, str]: """Return a mapping of extension names and statuses to their categories.""" categories = defaultdict(list) - for ext in self.all_extensions: + for ext, metadata in self.all_extensions.items(): if ext in self.bot.extensions: status = StatusEmojis.fully_loaded + elif metadata.load_if_mode & BOT_MODE: + status = StatusEmojis.disabled else: status = StatusEmojis.unloaded diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 0e582ae9..24f82611 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -4,7 +4,7 @@ import logging import shutil from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Mapping, MutableSet +from typing import TYPE_CHECKING, Mapping, Set from atoml.exceptions import ParseError from discord import Colour, Embed @@ -20,14 +20,15 @@ from modmail.addons.plugins import ( BASE_PLUGIN_PATH, PLUGINS, - find_plugins_in_dir, + find_partial_plugins_from_dir, + find_plugins, update_local_toml_enable_or_disable, walk_plugin_files, ) from modmail.extensions.extension_manager import Action, ExtensionConverter, ExtensionManager, StatusEmojis from modmail.utils import responses from modmail.utils.cogs import BotModeEnum, ExtMetadata -from modmail.utils.extensions import BOT_MODE +from modmail.utils.extensions import BOT_MODE, ModuleDict from modmail.utils.pagination import ButtonPaginator @@ -53,13 +54,22 @@ class PluginDevPathConverter(ExtensionConverter): type = "plugin" NO_UNLOAD = None + def __init__(self): + """Properly set the source_list.""" + super().__init__() + PluginDevPathConverter.source_list + modules: ModuleDict = {} + for plug in PluginDevPathConverter.source_list: + modules.update({k: v for k, v in plug.modules.items()}) + self.source_list = modules + class PluginConverter(commands.Converter): """Convert a plugin name into a full plugin with path and related args.""" async def convert(self, ctx: Context, argument: str) -> Plugin: """Converts a plugin into a full plugin with a path and all other attributes.""" - loaded_plugs: MutableSet[Plugin] = ctx.bot.installed_plugins + loaded_plugs: Set[Plugin] = ctx.bot.installed_plugins for plug in loaded_plugs: if argument in (plug.name, plug.folder_name): @@ -68,25 +78,6 @@ async def convert(self, ctx: Context, argument: str) -> Plugin: raise commands.BadArgument(f"{argument} is not in list of installed plugins.") -class PluginFilesConverter(commands.Converter): - """ - Convert a name of a plugin into a full plugin. - - In this case Plugins are group of extensions, as if they have multiple files in their directory, - they will be treated as one plugin for the sake of managing. - """ - - async def convert(self, ctx: Context, argument: str) -> List[str]: - """Converts a provided plugin into a list of its paths.""" - loaded_plugs: Dict[Plugin, List[str]] = ctx.bot.installed_plugins - - for plug in loaded_plugs: - if argument in (plug.name, plug.folder_name): - return loaded_plugs[plug] - - raise commands.BadArgument(f"{argument} is not an installed plugin.") - - class PluginManager(ExtensionManager, name="Plugin Manager"): """Plugin management commands.""" @@ -95,8 +86,11 @@ class PluginManager(ExtensionManager, name="Plugin Manager"): def __init__(self, bot: ModmailBot): super().__init__(bot) - self.all_extensions = PLUGINS - self.refresh_method = walk_plugin_files + + modules: ModuleDict = {} + for plug in PLUGINS: + modules.update({k: v for k, v in plug.modules.items()}) + self.all_extensions = modules def get_black_listed_extensions(self) -> list: """ @@ -122,7 +116,7 @@ async def plugin_dev_group(self, ctx: Context) -> None: @plugin_dev_group.command(name="load", aliases=("l",)) async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" - Load plugins given their fully qualified or unqualified names. + Load singular plugin files given their fully qualified or unqualified names. If '\*' is given as the name, all unloaded plugins will be loaded. """ @@ -131,7 +125,7 @@ async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> @plugin_dev_group.command(name="unload", aliases=("u", "ul")) async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" - Unload currently loaded plugins given their fully qualified or unqualified names. + Unoad singular plugin files given their fully qualified or unqualified names. If '\*' is given as the name, all loaded plugins will be unloaded. """ @@ -140,23 +134,51 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) - @plugin_dev_group.command(name="reload", aliases=("r", "rl")) async def reload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" - Reload plugins given their fully qualified or unqualified names. + Reload singular plugin files given their fully qualified or unqualified names. - If an plugin fails to be reloaded, it will be rolled-back to the prior working state. + If a plugin file fails to be reloaded, it will be rolled-back to the prior working state. If '\*' is given as the name, all currently loaded plugins will be reloaded. """ await self.reload_extensions.callback(self, ctx, *plugins) - @plugin_dev_group.command(name="list", aliases=("all", "ls")) - async def dev_list_plugins(self, ctx: Context) -> None: - """ - Get a list of all plugin files, including their loaded status. + def group_extension_statuses(self) -> Mapping[str, str]: + """Return a mapping of plugin names and statuses to their categories.""" + categories = defaultdict(list) - Red indicates that the plugin file is unloaded. - Green indicates that the plugin file is currently loaded. - """ - await self.list_extensions.callback(self, ctx) + for plug in PLUGINS: + for mod, metadata in plug.modules.items(): + if mod in self.bot.extensions: + status = StatusEmojis.fully_loaded + elif metadata.load_if_mode & BOT_MODE: + status = StatusEmojis.disabled + else: + status = StatusEmojis.unloaded + + name = mod.split(".", 2)[-1] + categories[plug.name].append(f"{status} `{name}`") + + return dict(categories) + + def _resync_extensions(self) -> None: + """Resyncs plugin. Useful for when the files are dynamically updated.""" + logger.debug(f"Refreshing list of {self.type}s.") + + # remove all fully unloaded plugins from the list + for plug in PLUGINS: + safe_to_remove = [] + for mod in plug.modules: + safe_to_remove.append(mod not in self.bot.extensions) + if all(safe_to_remove): + PLUGINS.remove(plug) + + for plug in find_plugins(): + PLUGINS.update(plug) + + modules: ModuleDict = {} + for plug in PLUGINS: + modules.update({k: v for k, v in plug.modules.items()}) + self.all_extensions = modules @plugin_dev_group.command(name="refresh", aliases=("rewalk", "rescan", "resync")) async def resync_plugins(self, ctx: Context) -> None: @@ -196,15 +218,15 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu source.cache_file = directory # determine plugins in the archive - plugins = find_plugins_in_dir(directory) + archive_plugins = {x for x in find_partial_plugins_from_dir(directory)} # yield to any coroutines that need to run - # its not possible to do this with aiofiles, so when we export the zip, + # afaik its not possible to do this with aiofiles, so when we export the zip, # its important to yield right after await asyncio.sleep(0) # copy the requested plugin over to the new folder - for p in plugins.keys(): + for p in archive_plugins: # check if user-provided plugin matches either plugin name or folder name if plugin.name in (p.name, p.folder_name): install_path = BASE_PLUGIN_PATH / p.folder_path.name @@ -227,23 +249,11 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu return logger.trace(f"{BASE_PLUGIN_PATH = }") - PLUGINS.update(walk_plugin_files(BASE_PLUGIN_PATH / p.folder_path.name)) - - files_to_load: List[str] = [] - for plug in plugins[plugin]: - logger.trace(f"{plug = }") - try: - plug = await PluginDevPathConverter().convert(None, plug.name.rstrip(".py")) - except commands.BadArgument: - pass - else: - if plug in PLUGINS: - files_to_load.append(plug) + plugin.modules.update(walk_plugin_files(BASE_PLUGIN_PATH / plugin.folder_name)) - logger.debug(f"{files_to_load = }") - self.batch_manage(Action.LOAD, *files_to_load) + PLUGINS.add(plugin) - self.bot.installed_plugins[plugin] = files_to_load + self.batch_manage(Action.LOAD, *plugin.modules.keys()) await responses.send_positive_response(ctx, f"Installed plugin {plugin}.") @@ -258,9 +268,10 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No ) return - plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) - - _, err = self.batch_manage(Action.UNLOAD, *plugin_files, is_plugin=True, suppress_already_error=True) + plugin = await PluginConverter().convert(ctx, plugin.folder_name) + _, err = self.batch_manage( + Action.UNLOAD, *plugin.modules.keys(), is_plugin=True, suppress_already_error=True + ) if err: await responses.send_negatory_response( ctx, "There was a problem unloading the plugin from the bot." @@ -269,10 +280,8 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No shutil.rmtree(plugin.installed_path) - plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) - for file_ in plugin_files: - del PLUGINS[file_] - del self.bot.installed_plugins[plugin] + plugin = await PluginConverter().convert(ctx, plugin.folder_name) + PLUGINS.remove(plugin) await responses.send_positive_response(ctx, f"Successfully uninstalled plugin {plugin}") @@ -291,8 +300,6 @@ async def _enable_or_disable_plugin( plugin.enabled = enable - plugin_files: List[str] = await PluginFilesConverter().convert(ctx, plugin.folder_name) - if plugin.local: try: update_local_toml_enable_or_disable(plugin) @@ -300,7 +307,9 @@ async def _enable_or_disable_plugin( plugin.enabled = not plugin.enabled # reverse the state await responses.send_negatory_response(ctx, e.args[0]) - msg, err = self.batch_manage(Action.LOAD, *plugin_files, is_plugin=True, suppress_already_error=True) + msg, err = self.batch_manage( + action, *plugin.modules.keys(), is_plugin=True, suppress_already_error=True + ) if err: await responses.send_negatory_response( ctx, "Er, something went wrong.\n" f":x: {plugin!s} was unable to be {verb}d properly!" @@ -322,13 +331,13 @@ def group_plugin_statuses(self) -> Mapping[str, str]: """Return a mapping of plugin names and statuses to their module.""" plugins = defaultdict(str) - for plug, files in self.bot.installed_plugins.items(): + for plug in self.bot.installed_plugins: plug_status = [] - for ext in files: - if ext in self.bot.extensions: - status = True - else: - status = False + for mod, metadata in plug.modules.items(): + status = mod in self.bot.extensions + # check that the file is supposed to be loaded + if not status and not metadata.load_if_mode & self.bot.mode: + continue plug_status.append(status) if len(plug_status) == 0: @@ -347,7 +356,7 @@ def group_plugin_statuses(self) -> Mapping[str, str]: return dict(plugins) - @plugins_group.command(name="list", aliases=("all", "ls")) + @plugins_group.group(name="list", aliases=("all", "ls"), invoke_without_command=True) async def list_plugins(self, ctx: Context) -> None: """ Get a list of all plugins, including their loaded status. @@ -368,11 +377,24 @@ async def list_plugins(self, ctx: Context) -> None: lines.append(f"{status} **{plugin_name}**") logger.debug(f"{ctx.author} requested a list of all {self.type}s. " "Returning a paginated list.") - + if PLUGIN_DEV_ENABLED: + kw = {"footer_text": "Tip: use the detailed command to see all plugin files"} + else: + kw = {} await ButtonPaginator.paginate( - lines or f"There are no {self.type}s installed.", ctx.message, embed=embed + lines or f"There are no {self.type}s installed.", ctx.message, embed=embed, **kw ) + @list_plugins.command(name="detailed", aliases=("files", "-a"), hidden=not PLUGIN_DEV_ENABLED) + async def dev_list_plugins(self, ctx: Context) -> None: + """ + Get a list of all plugin files, including their loaded status. + + Red indicates that the plugin file is unloaded. + Green indicates that the plugin file is currently loaded. + """ + await self.list_extensions.callback(self, ctx) + # This cannot be static (must have a __func__ attribute). async def cog_check(self, ctx: Context) -> bool: """Only allow server admins and bot owners to invoke the commands in this cog.""" From 5c9e61e291dda121c187c6aa23d5fcdcb51199ea Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 11 Sep 2021 14:43:51 -0400 Subject: [PATCH 061/100] minor: add plugin suggesting on plugin converter --- modmail/extensions/plugin_manager.py | 46 +++++++++++- poetry.lock | 106 ++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 149 insertions(+), 4 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 24f82611..e3b7f31b 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -4,13 +4,14 @@ import logging import shutil from collections import defaultdict -from typing import TYPE_CHECKING, Mapping, Set +from typing import TYPE_CHECKING, Dict, Mapping, Set from atoml.exceptions import ParseError from discord import Colour, Embed from discord.abc import Messageable from discord.ext import commands from discord.ext.commands import Context +from rapidfuzz import fuzz, process import modmail.addons.utils as addon_utils from modmail import errors @@ -69,13 +70,52 @@ class PluginConverter(commands.Converter): async def convert(self, ctx: Context, argument: str) -> Plugin: """Converts a plugin into a full plugin with a path and all other attributes.""" - loaded_plugs: Set[Plugin] = ctx.bot.installed_plugins + loaded_plugs: Set[Plugin] = PLUGINS for plug in loaded_plugs: if argument in (plug.name, plug.folder_name): return plug - raise commands.BadArgument(f"{argument} is not in list of installed plugins.") + # Determine close plugins + # Using a set to prevent duplicates + # all_possible_args: Set[str] = set() + arg_mapping: Dict[str, Plugin] = dict() + for plug in loaded_plugs: + for name in plug.name, plug.folder_name: + # all_possible_args.add(name) + arg_mapping[name] = plug + + result = process.extract( + argument, + arg_mapping.keys(), + scorer=fuzz.ratio, + score_cutoff=86, + ) + logger.debug(f"{result = }") + + if not len(result): + raise commands.BadArgument(f"`{argument}` is not in list of installed plugins.") + + all_fully_matched_plugins: Set[Plugin] = set() + all_partially_matched_plugins: Dict[Plugin, float] = dict() + for res in result: + all_partially_matched_plugins[arg_mapping[res[0]]] = res[1] + + if res[1] == 100: + all_fully_matched_plugins.add(arg_mapping[res[0]]) + + if len(all_fully_matched_plugins) != 1: + suggested = "" + for plug, percent in all_partially_matched_plugins.items(): + suggested += f"`{plug.name}` ({round(percent)}%)\n" + raise commands.BadArgument( + f"`{argument}` is not in list of installed plugins." + f"\n\n**Suggested plugins**:\n{suggested}" + if len(suggested) + else "" + ) + + return await self.convert(ctx, all_fully_matched_plugins.pop().name) class PluginManager(ExtensionManager, name="Plugin Manager"): diff --git a/poetry.lock b/poetry.lock index 0eb9c7d0..f71923a0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -677,6 +677,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "numpy" +version = "1.21.1" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "packaging" version = "21.0" @@ -1038,6 +1046,17 @@ python-versions = ">=3.6" [package.dependencies] pyyaml = "*" +[[package]] +name = "rapidfuzz" +version = "1.6.1" +description = "rapid fuzzy string matching" +category = "main" +optional = false +python-versions = ">=2.7" + +[package.dependencies] +numpy = "*" + [[package]] name = "regex" version = "2021.8.28" @@ -1228,7 +1247,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b31abbde606fe9e5b9e54afc101b5dc4f244d9b1b9f5e7af615fa5df079a8194" +content-hash = "b2198b4b46511db8863b73739e75f132d5cd834ba59f4fe11205d5b3b52f7ded" [metadata.files] aiodns = [ @@ -1730,6 +1749,36 @@ nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] +numpy = [ + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, +] packaging = [ {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, @@ -1957,6 +2006,61 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] +rapidfuzz = [ + {file = "rapidfuzz-1.6.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ef307be084a6022f07fbcdb2614d3d92818750c4df97f38cf12551e029543ea4"}, + {file = "rapidfuzz-1.6.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1b6afbde4ac6cab1d5162bcf90d56b95b67632fbbeec77c923f0628381f12a4e"}, + {file = "rapidfuzz-1.6.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e6fc27ab404ee918dd92bd3200aae98e4539f3383e2fec39b6fea7ed70e9eb61"}, + {file = "rapidfuzz-1.6.1-cp27-cp27m-win32.whl", hash = "sha256:c266e7de2f3d29648a06ae15d650c029c2021e6cdee3bc67fdaabeff47385e49"}, + {file = "rapidfuzz-1.6.1-cp27-cp27m-win_amd64.whl", hash = "sha256:dc133cfed87dadf620d780a2f9334c6209b1a6eed030e10bd3f7c5d9d52bbce1"}, + {file = "rapidfuzz-1.6.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:8888ffc8f2504dcc5943bd2b016f20b69d6c94695fcb6626cd193703c0667c67"}, + {file = "rapidfuzz-1.6.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:23b6ae916278b3b4591f8749a264ef41fa11cb6abd736eadf9a625d45f800e72"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:02ccc031bb18dad5834e4e22795b6980471522ef6557e6ced1685c5cb43d411c"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ca2178f56017a117afe76dd8192c07e4f18df116516893f7218e3f8689680489"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:fe3eba011c105d4d69cbbed9e7d64edea9077c45f3c2fdfa06cc8747b55f2e56"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:317fafb2e725490721e76ee6b660dff1c10e959199dd525c447a1044ecfd2eaf"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:9ef4d63990c8ed1fe57f402302e612da4ffa712d5ff8f760a83f18ed72cc54dd"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-win32.whl", hash = "sha256:24b4fed0e31f975d7325ec26552eb71c2a0f17dfa7867fd8c9d59aadc749a960"}, + {file = "rapidfuzz-1.6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9ae20b5687121966b67658911f48fd938565b18b510633e6f56e3d04d1d15409"}, + {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75503a5c57c4353ddf99615557836b4dc7da78443099f269a4c7e504ddc6e9eb"}, + {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9e0594fd26b01ab81c25acd0d35e6f1ec91a0bd7bbb03568247db5501401f5f"}, + {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4c81df7e4f9a3ed9983ff0ec13b5e88d31596d996b919eeca8b1002e8367045"}, + {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:df8fcb829ac760b6743626143d33470b65a7a758558646c396fa7334fd9dd995"}, + {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a9e7f82401a66b40b3b0588761495aadd53c9c24d36f3543f0d82e5cce126964"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:17c23bc209f02d7481a3ad0d9fe72674a3fb8c7b3b72477d4aae311d67924f7f"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a764b8a8eca7980fe21e95fe54b506273a6f62b5083234de3133877b69c8482"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7394d4624b181346654fd0c2f1a94bd24aff3b68dc42044c4c6954c17822ed31"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589baf8444719e8466fef7db9854fb190af871c1b97538d2e249a3c4f621d972"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca7989d42c9c14ad576e8a655d5f0ae5d10777332e1cc9384267a423f93b498a"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8dea0e89791c5ac55ff314a6ac2aa699b69bf0c802e7253628baa730d80f68b"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:65e331eefee945ba5c9c00aa0af0eb8db94515f6f46090e5991152ef77b92e53"}, + {file = "rapidfuzz-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4f10c8a1656eccddad84c9380715d6a5acfa1652c2a261fff166ef786904ba20"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:61900b8e4574277056b5de177c194e628ce2eb750fad94c8ba5995ae1a356fc1"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:375703ba1e5928cc6dae755e716bd13ce6dce8c51e0dae64d900adf5ccaf8d24"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7cdd8a82c4ff7871f4dad3b45940ebc12aee8fcd32182d5cbafbad607f34d72"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9acb9fa62b34b839c69372e60d52bca24f994ede1e8d37bea4b905e197009c7a"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c1ced861667713655ce2ec5300945d42b33e22b14d373ce1a37a9ac883007a50"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5a8001dc1d83b904fba421df6d715a0c5f2a15396c48ee1792b4e00081460d9"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5e1dad95996cc2dc382ccb2762e42061494805aac749d63140d62623ee254d64"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:48c0da783ce1b604f647db8d0060dc987a1e429abf66afcb932b70179525db0c"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-win32.whl", hash = "sha256:8076188fc13379d05f26c06337a4b1d45ea4ca7c872d77f47cdebb0d06d8e409"}, + {file = "rapidfuzz-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e0290e09c8dcac11e9631ab59351b67efd50f5eb2f23c0b2faf413cbe393dfa"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d9292132cc011680954bf5ea8535660e90c16fb3af2584e56b7327dacde53e23"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:32262729c0280567b52df5f755a05cbd0689b2f9103e67d67b1538297988f45b"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:58a97922a65c3c336a5c8613b70956cf8bb8ecc39b8177581249df3f92144d89"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3d7602067d38f0da988f84aa0694ff277d0ab228b4ea849f10d5a6c797f1ebe"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba36dcd6add2e5e9e41e1b016f2fe11a32beabdfd5987f2df4cc3f9322fe2a22"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c734fd58bb80b0930f5bfc2aba46538088bc06e10e9c66a6a32ac609f14fed27"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f8b42e07efa816413eb58a53feed300898b302cc2180c824a18518c7fdae6124"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24e378ce0629852f55a6c7cefbe3de5d477023d195da44c3bdd224c3cd9400a6"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-win32.whl", hash = "sha256:2af0bf8f4bef68fc56966c9f9c01b6d5412ba063ea670c14bf769081f86bf022"}, + {file = "rapidfuzz-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:ce68d063f97e3ac69c6bf77b366d088db3ba0cad60659dfea1add2ed33ed41ba"}, + {file = "rapidfuzz-1.6.1-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:e6c93b726666db391275f554a7fc9b20bb7f5fbdd028d94cef3d3f19cd1d33ce"}, + {file = "rapidfuzz-1.6.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:edefe618fa6b4bf1ba684c88a2253b7a475cc85edd2020616287e1619cbe3284"}, + {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e0b57bf21e69636da99092406d5effcd4b8ba804c5e1f3b6a075a2f1e2c077d3"}, + {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3ec0ece8ba8c9a1785a7ec3d2ba0d402cccf8199e99d9c33e98cbeab08be7f18"}, + {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ddd972f15cc1bd5c95854f0f7ebad55e46f56bdc6545cdbf6fcf71390993785c"}, + {file = "rapidfuzz-1.6.1.tar.gz", hash = "sha256:5cc007251bc6b5d5d8e7e66e6f8e1b36033d3e240845a2e5721a025ef850d690"}, +] regex = [ {file = "regex-2021.8.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2"}, {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a"}, diff --git a/pyproject.toml b/pyproject.toml index 84144745..96be6386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } pydantic = { version = "^1.8.2", extras = ["dotenv"] } +rapidfuzz = "^1.6.1" [tool.poetry.extras] From 49c3d597248dfe78dcb1e432da9808c6856e7780 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sat, 11 Sep 2021 15:11:53 -0400 Subject: [PATCH 062/100] priortise plugins folder names over names --- modmail/extensions/plugin_manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index e3b7f31b..bddb5113 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -72,9 +72,16 @@ async def convert(self, ctx: Context, argument: str) -> Plugin: """Converts a plugin into a full plugin with a path and all other attributes.""" loaded_plugs: Set[Plugin] = PLUGINS + # its possible to have a plugin with the same name as a folder of a plugin + # folder names are the priority + secondary_names = dict() for plug in loaded_plugs: - if argument in (plug.name, plug.folder_name): + if argument == plug.name: return plug + secondary_names[plug.folder_name] = plug + + if argument in secondary_names: + return secondary_names[argument] # Determine close plugins # Using a set to prevent duplicates From 1cf6233ec21c5c1d2a3a0da77cf96d994679ec29 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 01:48:47 -0400 Subject: [PATCH 063/100] minor: add message param to responses to allow editing a previous response --- modmail/utils/responses.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/modmail/utils/responses.py b/modmail/utils/responses.py index e22af0e4..fd233255 100644 --- a/modmail/utils/responses.py +++ b/modmail/utils/responses.py @@ -67,6 +67,7 @@ async def send_positive_response( response: str, embed: discord.Embed = _UNSET, colour: discord.Colour = _UNSET, + message: discord.Message = None, **kwargs, ) -> discord.Message: """ @@ -76,13 +77,18 @@ async def send_positive_response( If embed is set to None, this will send response as a plaintext message, with no allowed_mentions. If embed is provided, this method will send a response using the provided embed, edited in place. Extra kwargs are passed to Messageable.send() + + If message is provided, it will attempt to edit that message rather than sending a new one. """ kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") if embed is None: - return await channel.send(response, **kwargs) + if message is None: + return await channel.send(response, **kwargs) + else: + return await message.edit(response, **kwargs) if colour is _UNSET: colour = default_success_color @@ -92,7 +98,10 @@ async def send_positive_response( embed.title = choice(success_headers) embed.description = response - return await channel.send(embed=embed, **kwargs) + if message is None: + return await channel.send(embed=embed, **kwargs) + else: + return await message.edit(embed=embed, **kwargs) async def send_negatory_response( @@ -100,6 +109,7 @@ async def send_negatory_response( response: str, embed: discord.Embed = _UNSET, colour: discord.Colour = _UNSET, + message: discord.Message = None, **kwargs, ) -> discord.Message: """ @@ -115,7 +125,10 @@ async def send_negatory_response( logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") if embed is None: - return await channel.send(response, **kwargs) + if message is None: + return await channel.send(response, **kwargs) + else: + return await message.edit(response, **kwargs) if colour is _UNSET: colour = default_error_color @@ -125,7 +138,10 @@ async def send_negatory_response( embed.title = choice(error_headers) embed.description = response - return await channel.send(embed=embed, **kwargs) + if message is None: + return await channel.send(embed=embed, **kwargs) + else: + return await message.edit(embed=embed, **kwargs) async def send_response( @@ -134,6 +150,7 @@ async def send_response( success: bool, embed: discord.Embed = _UNSET, colour: discord.Colour = _UNSET, + message: discord.Message = None, **kwargs, ) -> discord.Message: """ @@ -145,6 +162,6 @@ async def send_response( Extra kwargs are passed to Messageable.send() """ if success: - return await send_positive_response(channel, response, embed, colour, **kwargs) + return await send_positive_response(channel, response, embed, colour, message, **kwargs) else: - return await send_negatory_response(channel, response, embed, colour, **kwargs) + return await send_negatory_response(channel, response, embed, colour, message, **kwargs) From 43eb5162f9f698fc58f48efd5a7bfc56d6384c40 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 02:22:39 -0400 Subject: [PATCH 064/100] allow plugins to declare dependencies --- modmail/addons/models.py | 9 ++++-- modmail/addons/plugins.py | 42 ++++++++++++++++++++++++++++ modmail/extensions/plugin_manager.py | 19 ++++++++++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index a005bdfd..d3004bd4 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -149,11 +149,12 @@ def __init__( folder: str, description: Optional[str] = None, *, - min_bot_version: Optional[str] = None, - name: Optional[str] = None, + dependencies: Optional[List[str]] = None, + enabled: bool = True, folder_path: Optional[pathlib.Path] = None, local: bool = False, - enabled: bool = True, + min_bot_version: Optional[str] = None, + name: Optional[str] = None, **kw, ): self.folder_name = folder @@ -167,6 +168,8 @@ def __init__( self.local = local self.enabled = enabled + self.dependencies = dependencies or [] + self.modules = dict() # store any extra kwargs here diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 40255efd..acce089a 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import asyncio import glob import importlib import importlib.util @@ -15,6 +16,8 @@ import logging import os import pathlib +import sys +from asyncio import subprocess from collections.abc import Generator from typing import List, Optional, Set, Tuple @@ -54,6 +57,44 @@ LOCAL_PLUGIN_TOML = BASE_PLUGIN_PATH / "local.toml" +PYTHON_INTERPRETER: Optional[str] = sys.executable + + +async def install_dependencies(plugin: Plugin) -> str: + """Installs provided dependencies from a plugin.""" + # check if there are any plugins to install + if not len(plugin.dependencies): + return + + if PYTHON_INTERPRETER is None: + raise FileNotFoundError("Could not locate python interpreter.") + + # This uses the check argument with our exported requirements.txt + # to make pip promise that anything it installs won't change + # the packages that the bot requires to have installed. + pip_install_args = [ + "-m", + "pip", + "--no-input", + "--no-color", + "install", + "--constraint", + "requirements.txt", + ] + proc = await asyncio.create_subprocess_exec( + f"{PYTHON_INTERPRETER}", + *pip_install_args, + *plugin.dependencies, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + logger.debug(f"{stdout.decode() = }") + if stderr: + logger.error(f"Received stderr: {stderr.decode()}") + raise Exception("Daquack?") + return stdout.decode() + def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /, local: bool = False) -> List[Plugin]: """Parses a plugin toml, given the string loaded in.""" @@ -71,6 +112,7 @@ def parse_plugin_toml_from_string(unparsed_plugin_toml_str: str, /, local: bool description=plug_entry.get("description"), min_bot_version=plug_entry.get("min_bot_version"), enabled=enabled, + dependencies=plug_entry.get("dependencies"), ) ) return found_plugins diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index bddb5113..a4b13be2 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -23,6 +23,7 @@ PLUGINS, find_partial_plugins_from_dir, find_plugins, + install_dependencies, update_local_toml_enable_or_disable, walk_plugin_files, ) @@ -240,6 +241,9 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu # I'm aware this should be a context manager, but do not want to indent almost the entire command await ctx.trigger_typing() + # if we send a preliminary action message this gets set and is edited upon success. + message = None + # create variables for the user input, typehint them, then assign them from the converter tuple plugin: Plugin source: AddonSource @@ -294,6 +298,19 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu if plugin.folder_path is None: await responses.send_negatory_response(ctx, f"Could not find plugin {plugin}") return + + if plugin.dependencies and len(plugin.dependencies): + # install dependencies since they exist + message = await ctx.send( + embed=Embed("Installing dependencies.", title="Pending install", colour=Colour.yellow()) + ) + try: + await install_dependencies(plugin) + except Exception: + await responses.send_negatory_response( + ctx, "Could not successfully install plugin dependencies.", message=message + ) + logger.trace(f"{BASE_PLUGIN_PATH = }") plugin.modules.update(walk_plugin_files(BASE_PLUGIN_PATH / plugin.folder_name)) @@ -302,7 +319,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu self.batch_manage(Action.LOAD, *plugin.modules.keys()) - await responses.send_positive_response(ctx, f"Installed plugin {plugin}.") + await responses.send_positive_response(ctx, f"Installed plugin {plugin}.", message=message) @plugins_group.command(name="uninstall", aliases=("rm",)) async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: From c9a3c9f0782c6d41b15c6c5244def97453a6f238 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 04:04:52 -0400 Subject: [PATCH 065/100] commit: fix: use pip to install in dockerfile instead --- Dockerfile | 19 ++++++++++--------- requirements.txt | 2 ++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8eef8de6..05f3114c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,23 @@ FROM python:3.9-slim # Set pip to have cleaner logs and no saved cache -ENV PIP_NO_CACHE_DIR=false \ - POETRY_VIRTUALENVS_CREATE=false +ENV PIP_NO_CACHE_DIR=false -# Install poetry -RUN pip install -U poetry - -# See https://github.com/python-poetry/poetry/issues/3336 -RUN poetry config experimental.new-installer false +# Update pip +RUN pip install -U pip # Create the working directory WORKDIR /modmail +# Copy requirements so they can be installed +COPY ./requirements.txt ./requirements.txt + +# Install dependencies +RUN pip install -r ./requirements.txt + + # Copy the source code in last to optimize rebuilding the image COPY . . -# Install project dependencies -RUN poetry install --no-dev CMD ["python", "-m", "modmail"] diff --git a/requirements.txt b/requirements.txt index e32cd079..8eaae06f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,12 +17,14 @@ discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_fu humanfriendly==9.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" idna==3.2; python_version >= "3.6" multidict==5.1.0; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" +numpy==1.21.1; python_version >= "3.7" pycares==4.0.0; python_version >= "3.6" pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" pydantic==1.8.2; python_full_version >= "3.6.1" pyreadline==2.1; python_version >= "2.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and sys_platform == "win32" python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" python-dotenv==0.19.0; python_full_version >= "3.6.1" and python_version >= "3.5" +rapidfuzz==1.6.1; python_version >= "2.7" six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" typing-extensions==3.10.0.2; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" yarl==1.6.3; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" From 8a2194dd0e05a16e4f9a9e7e9eec28e1ef43e1d8 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 04:17:11 -0400 Subject: [PATCH 066/100] chore: fix comments and lower score cutoff for plugin suggestions --- modmail/extensions/plugin_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index a4b13be2..2b92c7ac 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -85,19 +85,17 @@ async def convert(self, ctx: Context, argument: str) -> Plugin: return secondary_names[argument] # Determine close plugins - # Using a set to prevent duplicates - # all_possible_args: Set[str] = set() + # Using a dict to prevent duplicates arg_mapping: Dict[str, Plugin] = dict() for plug in loaded_plugs: for name in plug.name, plug.folder_name: - # all_possible_args.add(name) arg_mapping[name] = plug result = process.extract( argument, arg_mapping.keys(), scorer=fuzz.ratio, - score_cutoff=86, + score_cutoff=69, ) logger.debug(f"{result = }") From 9ea3be4618d0744c6913d27b9de5e57c2d7214ed Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 04:36:48 -0400 Subject: [PATCH 067/100] fix: don't send a success message on error --- modmail/extensions/extension_manager.py | 6 ++++++ modmail/extensions/plugin_manager.py | 24 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 688b8b56..c9e1a009 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -47,6 +47,8 @@ class Action(Enum): ENABLE = functools.partial(ModmailBot.load_extension) DISABLE = functools.partial(ModmailBot.unload_extension) + INSTALL = functools.partial(ModmailBot.reload_extension) + class ExtensionConverter(commands.Converter): """ @@ -314,6 +316,10 @@ def manage( # When reloading, have a special error. msg = f":x: {self.type.capitalize()} `{ext}` is not loaded, so it was not {verb}ed." not_quite = True + elif action is Action.INSTALL: + # extension wasn't loaded, so load it + # this is used for plugins + Action.LOAD.value(self.bot, ext) else: msg = f":x: {self.type.capitalize()} `{ext}` is already {verb}ed." diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 2b92c7ac..8623dbd9 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -308,6 +308,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu await responses.send_negatory_response( ctx, "Could not successfully install plugin dependencies.", message=message ) + return logger.trace(f"{BASE_PLUGIN_PATH = }") @@ -315,9 +316,28 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu PLUGINS.add(plugin) - self.batch_manage(Action.LOAD, *plugin.modules.keys()) + self.batch_manage(Action.INSTALL, *plugin.modules.keys()) - await responses.send_positive_response(ctx, f"Installed plugin {plugin}.", message=message) + # check if the manage was successful + failed = [] + for mod, metadata in plugin.modules.items(): + if mod in self.bot.extensions: + fail = False + elif metadata.load_if_mode & BOT_MODE: + fail = False + else: + fail = True + + failed.append(fail) + + if any(failed): + await responses.send_negatory_response( + ctx, f"Failed to fully install plugin {plugin}.", message=message + ) + else: + await responses.send_positive_response( + ctx, f"Successfully installed plugin {plugin}.", message=message + ) @plugins_group.command(name="uninstall", aliases=("rm",)) async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: From 44e2e44a20945b19986315611dd0d2a8f5b820b7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 17:58:16 -0400 Subject: [PATCH 068/100] chore: add plugin as an alias --- modmail/extensions/plugin_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 8623dbd9..1b02538d 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -147,7 +147,7 @@ def get_black_listed_extensions(self) -> list: """ return [] - @commands.group("plugins", aliases=("plug", "plugs"), invoke_without_command=True) + @commands.group("plugins", aliases=("plug", "plugs", "plugin"), invoke_without_command=True) async def plugins_group(self, ctx: Context) -> None: """Install, uninstall, disable, update, and enable installed plugins.""" await ctx.send_help(ctx.command) From 063e55a60ce5d4e71c0578b704b67329599d05cd Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 19:59:23 -0400 Subject: [PATCH 069/100] docs: add addon and plugin documentation --- docs/addons/README.md | 39 ++++++++++++++++++++++ docs/addons/installation.md | 53 +++++++++++++++++++++++++++++ docs/addons/plugins.md | 66 +++++++++++++++++++++++++++++++++++++ mkdocs.yml | 14 +++++--- 4 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 docs/addons/README.md create mode 100644 docs/addons/installation.md create mode 100644 docs/addons/plugins.md diff --git a/docs/addons/README.md b/docs/addons/README.md new file mode 100644 index 00000000..8751fab8 --- /dev/null +++ b/docs/addons/README.md @@ -0,0 +1,39 @@ +# Addons + +Addons are our built-in system to extend the features of the bot in an officially supported manner. + +Currently supported systems are plugins, which are slightly modified discord.py extensions. + +!!!note + This guide is for those who want to **write** addons. If you are looking to use an addon, please view our guide [on installing them][installation]. + + +## Guides +- [Installation][installation] +- [Repo Setup](#repo-setup) +- [Making Plugins][making-plugins] + + +## Repo Setup +In order to be able to install addons, a few things are required. +Each addon type will have its own requirements in addition to the following. + +### Overall File Structure +At the base of the addon system is the source. Sources have a folder structure like the following: + +```sh +. +├── Plugins/ +└── README.md +``` + +In this structure, this repository is holding addons of a plugin type. The structure of the Plugins folder itself is detailed in the [creating plugins guide][making-plugins]. + +### Hosting +All addons must be hosted on either github or gitlab as of now. + +!!!note + Addons currently do not automatically update, and need to be re-installed on each run. This will be fixed once the database client exists. + +[making-plugins]: ./plugins.md +[installation]: ./installation.md diff --git a/docs/addons/installation.md b/docs/addons/installation.md new file mode 100644 index 00000000..333ddf4b --- /dev/null +++ b/docs/addons/installation.md @@ -0,0 +1,53 @@ +# Installation + +!!!note + If you are looking to write addons, check our writing addons guide. + + + +## Plugins + +Plugins are discord.py extensions which expand the functionality of the bot beyond its core feature: relaying messages back and forth between staff and members. + +We've done our best to make this system easy to use for both novice and experienced developers--installing plugins should require no programming knowledge at all. + +By default, modmail will install plugins hosted on [github.com](https://github.com), but also supports installing from github. + +This may look complex, but it supports a wide variety of options. + +```fix +?plugin install [git-host] / [@ref] +?plugin install [@ref] +``` + + +### Git-host (Optional) +Valid options are: + +- `github` +- `gitlab` + +Default: + +- `github` + + +### User/Repo + +This is the user and the respository hosted on a valid git-host. + +In the link , the user and repo are `discord-modmail/addons`. + + +### Name + +This is the addon name, it is not allowed to contain `@`. +By default, this is the plugin folder name, unless it is defined in the plugin.toml file. +A repository should provide a list of their plugins either in a plugin readme, or the full repository readme. + + +### Ref + +This is the git reference, leave blank to use the repository default. +If you would like to use a specific commit, branch, or tag, then provide it preceeded by a `@`. +For example, to use tagged version 1.2, `@v1.2` would install from that tag. diff --git a/docs/addons/plugins.md b/docs/addons/plugins.md new file mode 100644 index 00000000..f70baaee --- /dev/null +++ b/docs/addons/plugins.md @@ -0,0 +1,66 @@ +# Creating Plugins + +## File Structure Overview + +This details the structure of a plugin addon. +!!!note + This builds on the [addon structure documentation][addon-guide]. + +!!!note + This guide is **not** how to install plugins, please view our [installation guide][installation] for that. +```sh +Plugins/ +├── react_to_contact +│ ├── listener.py +│ └── react_to_contact.py +├── verify_contact +│ └── verify_contact.py +└── plugin.toml +``` + +Even though there are three .py files, this repository contains two plugins. Each top level folder in the Plugins folder contains one plugin. +The number of py files in each plugins folder does not matter, there are still two plugins here. + +One plugin here is named `react_to_contact`, the other is `verify_contact` + +However, those are not user friendly names. It would be a lot easier for the end user to reference with `React to Contact`, and for the user interface to refer to it as such. + +To do so, a name can be provided in the plugin.toml file. + +## plugin.toml + +There are several variables which can be configured by providing a plugin.toml file. + +If you don't already know what toml is, [check out their docs](https://toml.io/) + + +!!!warning + `plugin.toml` is supplemental to the list of folders. This means that all plugins in the repository are installable at any time. Providing a plugin.toml does not mean that any plugins *not* in the toml are not included anymore. + + This has the advantage of being able to use `plugin.toml` to change the name of one plugin, without having to add all other plugins to the toml. + + +A full plugin.toml for the above repository may look like this: + +```toml +[[plugins]] +name = 'React to Contact' +description = 'Provides a permanent message where the user can react to open a thread' +directory = 'react_to_contact' + +[[plugins]] +name = 'Verify Contact' +description = 'Prevents the user from accidently opening a thread by asking if they are sure.' +directory = 'verify_contact' +``` + +The name and directory are the only keys in use today, +the description is not yet used. +`directory` is required, name is optional, and defaults to the directory if not provided. + + + + +[addon-guide]: ./README.md +[addon-repo-structure]: ./README.md#initial-setup +[installation]: ./installation.md#plugins diff --git a/mkdocs.yml b/mkdocs.yml index 1712cdc8..0f5d730d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,10 +56,16 @@ plugins: # Page tree nav: -- Home: README.md -- Contributing: contributing.md -- Security: security.md -- Changelog: changelog.md + - Home: README.md + - Contributing: contributing.md + - Security: security.md + - Changelog: changelog.md + - Addons: + - Overview: addons/ + - Installation: addons/installation.md + - Plugins: addons/plugins.md + + # Extensions markdown_extensions: From abab87ce911a90933b82f00f4fc016480b3bb5d6 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 20:04:02 -0400 Subject: [PATCH 070/100] minor: add missing link --- docs/addons/installation.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/addons/installation.md b/docs/addons/installation.md index 333ddf4b..1b1ca1e9 100644 --- a/docs/addons/installation.md +++ b/docs/addons/installation.md @@ -1,7 +1,7 @@ # Installation !!!note - If you are looking to write addons, check our writing addons guide. + If you are looking to write addons, check our [writing addons][addon-guide] guide. @@ -51,3 +51,5 @@ A repository should provide a list of their plugins either in a plugin readme, o This is the git reference, leave blank to use the repository default. If you would like to use a specific commit, branch, or tag, then provide it preceeded by a `@`. For example, to use tagged version 1.2, `@v1.2` would install from that tag. + +[addon-guide]: ./README.md From 5a6f8897cc2e3f38c4887567e226b88be98d334a Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 12 Sep 2021 23:34:35 -0400 Subject: [PATCH 071/100] minor: move bot mode determine to utils.cogs --- modmail/addons/helpers.py | 18 ++++++++++++++---- modmail/utils/cogs.py | 27 +++++++++++++++++++++++++++ modmail/utils/extensions.py | 19 +------------------ 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/modmail/addons/helpers.py b/modmail/addons/helpers.py index 0de98e98..c9da6507 100644 --- a/modmail/addons/helpers.py +++ b/modmail/addons/helpers.py @@ -1,14 +1,24 @@ from __future__ import annotations -from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog +from modmail.bot import ModmailBot +from modmail.log import ModmailLogger +from modmail.utils.cogs import BOT_MODE, BotModeEnum, ExtMetadata +from modmail.utils.cogs import ModmailCog as _ModmailCog -__all__ = ["PluginCog", BotModeEnum, ExtMetadata] +__all__ = [ + "PluginCog", + BOT_MODE, + BotModeEnum, + ExtMetadata, + ModmailBot, + ModmailLogger, +] -class PluginCog(ModmailCog): +class PluginCog(_ModmailCog): """ - The base class that all cogs must inherit from. + The base class that all Plugin cogs must inherit from. A cog is a collection of commands, listeners, and optional state to help group commands together. More information on them can be found on diff --git a/modmail/utils/cogs.py b/modmail/utils/cogs.py index 5e0dc3f2..ebd3822c 100644 --- a/modmail/utils/cogs.py +++ b/modmail/utils/cogs.py @@ -3,6 +3,17 @@ from discord.ext import commands +from modmail.config import CONFIG + + +__all__ = ( + "BitwiseAutoEnum", + "BotModeEnum", + "ExtMetadata", + "BOT_MODE", + "ModmailCog", +) + class BitwiseAutoEnum(IntEnum): """Enum class which generates binary value for each item.""" @@ -37,6 +48,22 @@ def __init__(self, *, load_if_mode: BotModeEnum = BotModeEnum.PRODUCTION, no_unl self.no_unload = no_unload +def determine_bot_mode() -> int: + """ + Figure out the bot mode from the configuration system. + + The configuration system uses true/false values, so we need to turn them into an integer for bitwise. + """ + bot_mode = 0 + for mode in BotModeEnum: + if getattr(CONFIG.dev.mode, str(mode).rsplit(".", maxsplit=1)[-1].lower(), True): + bot_mode += mode.value + return bot_mode + + +BOT_MODE = determine_bot_mode() + + class ModmailCog(commands.Cog): """ The base class that all cogs must inherit from. diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index ff20a1bf..e8bf2946 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -8,9 +8,8 @@ from typing import Dict, Generator, List, NewType, NoReturn, Tuple from modmail import extensions -from modmail.config import CONFIG from modmail.log import ModmailLogger -from modmail.utils.cogs import BotModeEnum, ExtMetadata +from modmail.utils.cogs import BOT_MODE, BotModeEnum, ExtMetadata log: ModmailLogger = logging.getLogger(__name__) @@ -29,22 +28,6 @@ def unqualify(name: str) -> str: return name.rsplit(".", maxsplit=1)[-1] -def determine_bot_mode() -> int: - """ - Figure out the bot mode from the configuration system. - - The configuration system uses true/false values, so we need to turn them into an integer for bitwise. - """ - bot_mode = 0 - for mode in BotModeEnum: - if getattr(CONFIG.dev.mode, unqualify(str(mode)).lower(), True): - bot_mode += mode.value - return bot_mode - - -BOT_MODE = determine_bot_mode() - - log.trace(f"BOT_MODE value: {BOT_MODE}") log.debug(f"Dev mode status: {bool(BOT_MODE & BotModeEnum.DEVELOP)}") log.debug(f"Plugin dev mode status: {bool(BOT_MODE & BotModeEnum.PLUGIN_DEV)}") From dafb22ccd5ef87b0b0b9b2001a9d082523cf6017 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 13 Sep 2021 00:19:00 -0400 Subject: [PATCH 072/100] docs: add guide on ExtMetadata and BOT_MODE --- docs/contributing/creating_an_extension.md | 83 ++++++++++++++++++++++ mkdocs.yml | 5 +- 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 docs/contributing/creating_an_extension.md diff --git a/docs/contributing/creating_an_extension.md b/docs/contributing/creating_an_extension.md new file mode 100644 index 00000000..411728f1 --- /dev/null +++ b/docs/contributing/creating_an_extension.md @@ -0,0 +1,83 @@ +# Creating an Extension + +Welcome! + +Please note that extensions are cogs are different things. Extensions are files which add features to the bot, +and a cog is way to group commands and features within a file. + +This is an addendum to the [discord.py guide](https://discordpy.readthedocs.io/en/master/ext/commands/extensions.html) on how to write extensions. +This guide below details additional information which is not part of discord.py. + +**There is one major change from discord.py**: +Cogs **must** inherit from `ModmailCog`. +If this does not happen, the bot will let you know. + +ModmailCog is defined in `modmail/utils/cogs.py`. + +## BOT_MODE and `ExtMetadata` + +In general, an extension does not need to use the feature of an extension metadata. + +On every extension of the bot, an `EXT_METADATA` constant should exist, and it should be an instance of `ExtMetadata`. +The `ExtMetadata` class is defined in `modmail/utils/cogs.py`, along with `BotModeEnum`. + +It should be sufficent to have an EXT_METADATA variable declared at the top of the file as an instance of ExtMetadata. + +```python +from modmail.utils.cogs import ExtMetadata + +EXT_METADATA = ExtMetadata() +``` + +### `ExtMetadata` + +The purpose of ExtMetadata is to define metadata about an extension. Currently, it supports two items of metadata. + +- `load_if_mode` + - used to determine if this extension should be loaded at runtime. +- `no_unload` (Not supported by plugins) + - prevents an extension from being unloaded by the `?ext unload` command. This is mainly used to keep the extension manager from unloading itself. + +`no_unload` is pretty self explanatory, pass either True or False and the extension will either be blocked from being unloaded, or allowed to unload. +This only has an impact if the current bot mode is DEVELOP. Note that this does prevent the developer from *reloading* the extension. + +### `load_if_mode` + +`load_if_mode` currently has three modes, which each have their own uses.: + +- `PRODUCTION` + - The default mode, the bot is always in this mode. +- `DEVELOP` + - Bot developer. Enables the extension management commands. +- `PLUGIN_DEV` + - Plugin developer. Enables lower-level plugin commands. + +!!!tip + To enable these modes, set the corresponding environment variable to a truthy value. eg `DEVELOP=1` in your project `.env` file will enable the bot developer mode. + +To set an extension to only load on one cog, set the load_if_mode param when initialising a ExtMetadata object. + +```python +from modmail.utils.cogs import BotModeEnum, ExtMetadata + +EXT_METADATA = ExtMetadata(load_if_mode=BotModeEnum.DEVELOP) +``` + +*This is not a complete extension and will not run if tested!* + +This `EXT_METADATA` variable above declares the extension will only run if a bot developer is running the bot. + +However, we may want to load our extension normally but have a command or two which only load in specific modes. + +### `BOT_MODE` + +The bot exposes a BOT_MODE variable which contains a bitmask of the current mode. This is created with the BotModeEnum. +This allows code like this to determine if the bot mode is a specific mode. + +```python +from modmail.utils.cogs import BOT_MODE, BotModeEnum + +is_plugin_dev_enabled = BOT_MODE & BotModeEnum.PLUGIN_DEV +``` + +This is used in the plugin_manager extension to determine if the lower-level commands which manage plugin extensions directly should be enabled. diff --git a/mkdocs.yml b/mkdocs.yml index 0f5d730d..bb939ab1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -57,13 +57,14 @@ plugins: # Page tree nav: - Home: README.md - - Contributing: contributing.md - Security: security.md - Changelog: changelog.md - Addons: - Overview: addons/ - Installation: addons/installation.md - - Plugins: addons/plugins.md + - Contributing: + - Guidelines: contributing.md + - Creating an Extension: contributing/creating_an_extension.md From dc5e38d99dfb8279da92dc4a39d54ece77e458a1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 13 Sep 2021 00:31:09 -0400 Subject: [PATCH 073/100] docs: enhance plugin documentation --- docs/addons/README.md | 20 ++++++--- docs/addons/installation.md | 32 +++++++++----- docs/addons/plugins.md | 84 ++++++++++++++++++++++++++++++++----- mkdocs.yml | 3 +- 4 files changed, 111 insertions(+), 28 deletions(-) diff --git a/docs/addons/README.md b/docs/addons/README.md index 8751fab8..b546dcb5 100644 --- a/docs/addons/README.md +++ b/docs/addons/README.md @@ -2,23 +2,30 @@ Addons are our built-in system to extend the features of the bot in an officially supported manner. -Currently supported systems are plugins, which are slightly modified discord.py extensions. +Modmail, in its simplest form, is simple: relay messages to and from users to staff members. +However, we acknowledge that its not a one size fits all solution. Some communities need a bit more than most. +That's where the addon system fills the void between additional features that aren't core to our main goal. + +The addon system currently supports one kind of addon, plugins. +This guide will help you set up a respository to create your own addons. +Once its set up, please refer to the [plugin creation guide][making-plugins] for more details. !!!note This guide is for those who want to **write** addons. If you are looking to use an addon, please view our guide [on installing them][installation]. - ## Guides -- [Installation][installation] -- [Repo Setup](#repo-setup) -- [Making Plugins][making-plugins] +- [Installation] +- [Repo Setup](#repo-setup) +- [Creating Plugins][making-plugins] ## Repo Setup + In order to be able to install addons, a few things are required. Each addon type will have its own requirements in addition to the following. ### Overall File Structure + At the base of the addon system is the source. Sources have a folder structure like the following: ```sh @@ -30,10 +37,11 @@ At the base of the addon system is the source. Sources have a folder structure l In this structure, this repository is holding addons of a plugin type. The structure of the Plugins folder itself is detailed in the [creating plugins guide][making-plugins]. ### Hosting + All addons must be hosted on either github or gitlab as of now. !!!note Addons currently do not automatically update, and need to be re-installed on each run. This will be fixed once the database client exists. -[making-plugins]: ./plugins.md [installation]: ./installation.md +[making-plugins]: ./plugins.md diff --git a/docs/addons/installation.md b/docs/addons/installation.md index 1b1ca1e9..77b6bd2b 100644 --- a/docs/addons/installation.md +++ b/docs/addons/installation.md @@ -3,25 +3,27 @@ !!!note If you are looking to write addons, check our [writing addons][addon-guide] guide. - - ## Plugins Plugins are discord.py extensions which expand the functionality of the bot beyond its core feature: relaying messages back and forth between staff and members. We've done our best to make this system easy to use for both novice and experienced developers--installing plugins should require no programming knowledge at all. -By default, modmail will install plugins hosted on [github.com](https://github.com), but also supports installing from github. +By default, modmail will install plugins hosted on [github.com](https://github.com), but also supports installing from [gitlab](https://gitlab.com). -This may look complex, but it supports a wide variety of options. +This may look complex, but it supports a wide variety of options, as demonstrated below ```fix ?plugin install [git-host] / [@ref] ?plugin install [@ref] ``` +### Git-host style + +> `[git-host] / [@ref]` + +#### Git-host (Optional) -### Git-host (Optional) Valid options are: - `github` @@ -31,25 +33,33 @@ Default: - `github` - -### User/Repo +#### User/Repo This is the user and the respository hosted on a valid git-host. In the link , the user and repo are `discord-modmail/addons`. - -### Name +#### Name This is the addon name, it is not allowed to contain `@`. By default, this is the plugin folder name, unless it is defined in the plugin.toml file. A repository should provide a list of their plugins either in a plugin readme, or the full repository readme. - -### Ref +#### Ref This is the git reference, leave blank to use the repository default. If you would like to use a specific commit, branch, or tag, then provide it preceeded by a `@`. For example, to use tagged version 1.2, `@v1.2` would install from that tag. +### Link + +> ` [@ref]` + +If the above githost format seems too complicated, its possible to just copy the url to the repo +(ex. https://github.com/discord-modmail/addons) and use that for the link. + +The name of the plugin still must be provided, however. +The @ref can also be provided, if installating a specific version is desired. + + [addon-guide]: ./README.md diff --git a/docs/addons/plugins.md b/docs/addons/plugins.md index f70baaee..a8d257a3 100644 --- a/docs/addons/plugins.md +++ b/docs/addons/plugins.md @@ -1,13 +1,17 @@ # Creating Plugins -## File Structure Overview +Modmail, in its simplest form, is small: relay messages to and from users to staff members. However, we acknowledge that its not a one size fits all solution. Some communities need a bit more than most. That's where the addon system comes in to play. -This details the structure of a plugin addon. -!!!note - This builds on the [addon structure documentation][addon-guide]. +!!!Tip + This builds on the [addon structure documentation][addon-guide]. Please ensure you have a solid understanding of the basic repository structure beforehand. !!!note This guide is **not** how to install plugins, please view our [installation guide][installation] for that. + +## File Structure Overview + +This details the structure of a plugin addon. + ```sh Plugins/ ├── react_to_contact @@ -18,7 +22,7 @@ Plugins/ └── plugin.toml ``` -Even though there are three .py files, this repository contains two plugins. Each top level folder in the Plugins folder contains one plugin. +Even though there are three `.py` files, this repository contains two plugins. Each top level folder in the Plugins folder contains one plugin. The number of py files in each plugins folder does not matter, there are still two plugins here. One plugin here is named `react_to_contact`, the other is `verify_contact` @@ -33,14 +37,15 @@ There are several variables which can be configured by providing a plugin.toml f If you don't already know what toml is, [check out their docs](https://toml.io/) - -!!!warning +!!!tip `plugin.toml` is supplemental to the list of folders. This means that all plugins in the repository are installable at any time. Providing a plugin.toml does not mean that any plugins *not* in the toml are not included anymore. This has the advantage of being able to use `plugin.toml` to change the name of one plugin, without having to add all other plugins to the toml. -A full plugin.toml for the above repository may look like this: +### Options + +A full `plugin.toml` for the above repository may look like this: ```toml [[plugins]] @@ -56,11 +61,70 @@ directory = 'verify_contact' The name and directory are the only keys in use today, the description is not yet used. -`directory` is required, name is optional, and defaults to the directory if not provided. +The `directory` key is required, if wanting to set any other settings for a plugin. + +!!!tip + `directory` is aliased to `folder`. Both keys are valid, but if the `directory` key exists it will be used and `folder` will be ignored. + +Name is optional, and defaults to the directory if not provided. + +!!!warning + Capitals matter. Both the `plugin.toml` file and `[[plugins]]` table ***must*** be lowercase. + This also goes for all keys and directory arguments--they must match the capitials of the existing directory. + +### Dependencies + +If the dependencies that the bot is installed with, it is possible to declare a dependency and it will be installed when installing the plugin. + +!!! Waring + For the most part, you won't need to use this. But if you absolutely must use an additional dependency which isn't part of the bot, put it in this array. + +This is an array of arguments which should be just like they are being passed to pip. + +```toml +[[plugins]] +directory = 'solar_system' +dependencies = ['earthlib==0.2.2'] +``` + +This will install earthlib 0.2.2. + +## Code + +Now that we have an understanding of where the plugin files go, and how to configure them, its time to write their code. + +### `PluginCog` + +All plugin cogs ***must*** inherit from `PluginCog`. + +If plugin cogs do not inherit from this class, they will fail to load. + +A majority of the needed modmail classes have been imported into helpers for your convinence. + +```python +from modmail.addons import helpers + +# Cog +helpers.PluginCog + +# Extension Metadata +helpers.ExtMetadata + +### For Typehints +# bot class +helpers.ModmailBot + +# logger +helpers.ModmailLogger +``` + +### `ExtMetadata` +There is a system where extensions can declare load modes. +There is a longer write up on it [here][ext_metadata]. [addon-guide]: ./README.md -[addon-repo-structure]: ./README.md#initial-setup +[ext_metadata]: /contributing/creating_an_extension/#bot_mode-and-extmetadata [installation]: ./installation.md#plugins diff --git a/mkdocs.yml b/mkdocs.yml index bb939ab1..f5404be9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -60,8 +60,9 @@ nav: - Security: security.md - Changelog: changelog.md - Addons: - - Overview: addons/ + - Overview: addons/README.md - Installation: addons/installation.md + - Creating Plugins: addons/plugins.md - Contributing: - Guidelines: contributing.md - Creating an Extension: contributing/creating_an_extension.md From 6bc98dd0d949bec9c630f7f7e1250052702ca1e5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 13 Sep 2021 00:49:04 -0400 Subject: [PATCH 074/100] chore: fix invalid attr bug --- modmail/bot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modmail/bot.py b/modmail/bot.py index 6f73bc0b..7684ff53 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -15,6 +15,7 @@ from modmail.addons.plugins import PLUGINS, find_plugins from modmail.config import CONFIG from modmail.log import ModmailLogger +from modmail.utils.cogs import ModmailCog from modmail.utils.extensions import BOT_MODE, EXTENSIONS, NO_UNLOAD, walk_extensions @@ -226,12 +227,10 @@ def add_cog(self, cog: commands.Cog, *, override: bool = False) -> None: Utilizes the default discord.py loader beneath, but also checks so we can warn when we're loading a non-ModmailCog cog. """ - from modmail.utils.cogs import ModmailCog - if not isinstance(cog, ModmailCog): self.logger.warning( - f"Cog {cog.name} is not a ModmailCog. All loaded cogs should always be" - f" instances of ModmailCog." + f"Cog {cog.qualified_name} is not a ModmailCog. All loaded cogs should always be" + " instances of ModmailCog." ) super().add_cog(cog, override=override) self.logger.info(f"Cog loaded: {cog.qualified_name}") From 72fd8a70764db55cd724d474a009baeb28ef5f34 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 13 Sep 2021 00:49:54 -0400 Subject: [PATCH 075/100] docs: move changelog above security --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index f5404be9..c08b80a7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -57,8 +57,8 @@ plugins: # Page tree nav: - Home: README.md - - Security: security.md - Changelog: changelog.md + - Security: security.md - Addons: - Overview: addons/README.md - Installation: addons/installation.md From 978464817b255b27bcbadfa79daa971e05695997 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 15 Sep 2021 15:38:01 -0400 Subject: [PATCH 076/100] fix: logging typos --- modmail/extensions/extension_manager.py | 2 +- modmail/utils/responses.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index c9e1a009..6c661e36 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -335,7 +335,7 @@ def manage( msg = f":x: Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" if msg is None: - msg = f":thumbsup: {self.type.capitalize()} successfully {verb}ed: `{ext}`." + msg = f":thumbsup: {self.type.capitalize()} successfully {verb.rstrip('e')}d: `{ext}`." log.debug(error_msg or msg) return msg, error_msg or not_quite diff --git a/modmail/utils/responses.py b/modmail/utils/responses.py index fd233255..135ac23d 100644 --- a/modmail/utils/responses.py +++ b/modmail/utils/responses.py @@ -8,6 +8,7 @@ from typing import List import discord +from discord.ext.commands import Context from modmail.log import ModmailLogger @@ -82,6 +83,9 @@ async def send_positive_response( """ kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) + if isinstance(channel, Context): + channel = channel.channel + logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") if embed is None: @@ -122,7 +126,10 @@ async def send_negatory_response( """ kwargs["allowed_mentions"] = kwargs.get("allowed_mentions", discord.AllowedMentions.none()) - logger.debug(f"Requested to send affirmative message to {channel!s}. Response: {response!s}") + if isinstance(channel, Context): + channel = channel.channel + + logger.debug(f"Requested to send negatory message to {channel!s}. Response: {response!s}") if embed is None: if message is None: From d70cd3da34cc0ca0429e1d2a7e377a31d3eaa3c4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 15 Sep 2021 16:11:04 -0400 Subject: [PATCH 077/100] fix: get rid of requirements.txt, switch to constraints.txt --- Dockerfile | 2 +- modmail/addons/plugins.py | 6 +++--- requirements.txt => modmail/constraints.txt | 0 pyproject.toml | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) rename requirements.txt => modmail/constraints.txt (100%) diff --git a/Dockerfile b/Dockerfile index 05f3114c..7dc2eeb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN pip install -U pip WORKDIR /modmail # Copy requirements so they can be installed -COPY ./requirements.txt ./requirements.txt +COPY ./modmail/constraints.txt ./requirements.txt # Install dependencies RUN pip install -r ./requirements.txt diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index acce089a..5d70d780 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -79,10 +79,10 @@ async def install_dependencies(plugin: Plugin) -> str: "--no-color", "install", "--constraint", - "requirements.txt", + str(BASE_PLUGIN_PATH.parent / "constraints.txt"), ] proc = await asyncio.create_subprocess_exec( - f"{PYTHON_INTERPRETER}", + PYTHON_INTERPRETER, *pip_install_args, *plugin.dependencies, stdout=subprocess.PIPE, @@ -92,7 +92,7 @@ async def install_dependencies(plugin: Plugin) -> str: logger.debug(f"{stdout.decode() = }") if stderr: logger.error(f"Received stderr: {stderr.decode()}") - raise Exception("Daquack?") + raise Exception("Something went wrong when installing.") return stdout.decode() diff --git a/requirements.txt b/modmail/constraints.txt similarity index 100% rename from requirements.txt rename to modmail/constraints.txt diff --git a/pyproject.toml b/pyproject.toml index 96be6386..4be22145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,15 +83,15 @@ include = '\.pyi?$' [tool.taskipy.tasks.export] cmd = """ -echo 'Exporting installed packages to requirements.txt.\n\ +echo 'Exporting installed packages to modmail/constraints.txt.\n\ This task automatically relocks the lock file using "poetry lock --no-update"' && \ poetry lock --no-update && \ -echo '# Do not manually edit.\n# Generate with "poetry run task export"\n' > requirements.txt && \ +echo '# Do not manually edit.\n# Generate with "poetry run task export"\n' > modmail/constraints.txt && \ echo "Exporting..." && \ -poetry export --without-hashes >> requirements.txt && \ +poetry export --without-hashes >> modmail/constraints.txt && \ echo "Done exporting." """ -help = "Export installed packages in requirements.txt format" +help = "Export installed packages in requirements.txt format to modmail/constraints.txt." [tool.taskipy.tasks] # Documentation From ece99be973a4b4a7ea2c2d29d8c283735defe913 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 16 Sep 2021 12:20:56 -0400 Subject: [PATCH 078/100] fix: ignore root warning when installing dependencies with pip --- modmail/addons/plugins.py | 13 +++++++++++-- modmail/extensions/plugin_manager.py | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 5d70d780..68dc6473 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -59,6 +59,12 @@ PYTHON_INTERPRETER: Optional[str] = sys.executable +PIP_NO_ROOT_WARNING = ( + "WARNING: Running pip as the 'root' user can result in broken permissions and " + "conflicting behaviour with the system package manager. " + "It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv" +).encode() + async def install_dependencies(plugin: Plugin) -> str: """Installs provided dependencies from a plugin.""" @@ -90,9 +96,12 @@ async def install_dependencies(plugin: Plugin) -> str: ) stdout, stderr = await proc.communicate() logger.debug(f"{stdout.decode() = }") + if stderr: - logger.error(f"Received stderr: {stderr.decode()}") - raise Exception("Something went wrong when installing.") + stderr = stderr.replace(PIP_NO_ROOT_WARNING, b"").strip() + if len(stderr.decode()) > 0: + logger.error(f"Received stderr: '{stderr.decode()}'") + raise Exception("Something went wrong when installing.") return stdout.decode() diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 1b02538d..ba12e688 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -304,7 +304,8 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu ) try: await install_dependencies(plugin) - except Exception: + except Exception as e: + logger.error(e, exc_info=True) await responses.send_negatory_response( ctx, "Could not successfully install plugin dependencies.", message=message ) From 141db4a18c102701f66d5a595178e717ddb59e9f Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 17 Sep 2021 16:23:31 -0400 Subject: [PATCH 079/100] fix a bug with set changing size during iteration --- modmail/extensions/plugin_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index ba12e688..230d71b0 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -211,15 +211,14 @@ def _resync_extensions(self) -> None: logger.debug(f"Refreshing list of {self.type}s.") # remove all fully unloaded plugins from the list - for plug in PLUGINS: + for plug in PLUGINS.copy(): safe_to_remove = [] for mod in plug.modules: safe_to_remove.append(mod not in self.bot.extensions) if all(safe_to_remove): PLUGINS.remove(plug) - for plug in find_plugins(): - PLUGINS.update(plug) + PLUGINS.update(find_plugins()) modules: ModuleDict = {} for plug in PLUGINS: From c7e5376ba9cab35604df2e978941194b5d0bf604 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 17 Sep 2021 16:25:13 -0400 Subject: [PATCH 080/100] minor: add a missing 'e' for grammar --- modmail/extensions/extension_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 6c661e36..2174b292 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -322,7 +322,7 @@ def manage( Action.LOAD.value(self.bot, ext) else: - msg = f":x: {self.type.capitalize()} `{ext}` is already {verb}ed." + msg = f":x: {self.type.capitalize()} `{ext}` is already {verb.rstrip('e')}ed." not_quite = True except Exception as e: if hasattr(e, "original"): @@ -335,7 +335,7 @@ def manage( msg = f":x: Failed to {verb} {self.type} `{ext}`:\n```\n{error_msg}```" if msg is None: - msg = f":thumbsup: {self.type.capitalize()} successfully {verb.rstrip('e')}d: `{ext}`." + msg = f":thumbsup: {self.type.capitalize()} successfully {verb.rstrip('e')}ed: `{ext}`." log.debug(error_msg or msg) return msg, error_msg or not_quite From 6ff682d65d5ca1aff0923c00596e511b556871aa Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Sep 2021 00:32:12 -0400 Subject: [PATCH 081/100] docs: improve wording of opening sentences and don't duplicate myself --- docs/addons/README.md | 9 +++++---- docs/addons/installation.md | 2 +- docs/addons/plugins.md | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/addons/README.md b/docs/addons/README.md index b546dcb5..3cfbd208 100644 --- a/docs/addons/README.md +++ b/docs/addons/README.md @@ -2,11 +2,12 @@ Addons are our built-in system to extend the features of the bot in an officially supported manner. -Modmail, in its simplest form, is simple: relay messages to and from users to staff members. -However, we acknowledge that its not a one size fits all solution. Some communities need a bit more than most. -That's where the addon system fills the void between additional features that aren't core to our main goal. +Modmail, in its most basic form, is simple: relay messages to and from users to staff members. +However, we acknowledge that its not a one-size-fits-all solution. +Some communities need a few more features than others. +That's where the addon system fills the void. -The addon system currently supports one kind of addon, plugins. +The addon system currently supports only one kind of addon, plugins. This guide will help you set up a respository to create your own addons. Once its set up, please refer to the [plugin creation guide][making-plugins] for more details. diff --git a/docs/addons/installation.md b/docs/addons/installation.md index 77b6bd2b..3056174c 100644 --- a/docs/addons/installation.md +++ b/docs/addons/installation.md @@ -1,7 +1,7 @@ # Installation !!!note - If you are looking to write addons, check our [writing addons][addon-guide] guide. + If you are looking to write addons, check out our [writing addons][addon-guide] guide. ## Plugins diff --git a/docs/addons/plugins.md b/docs/addons/plugins.md index a8d257a3..46fe7fbc 100644 --- a/docs/addons/plugins.md +++ b/docs/addons/plugins.md @@ -1,6 +1,10 @@ # Creating Plugins -Modmail, in its simplest form, is small: relay messages to and from users to staff members. However, we acknowledge that its not a one size fits all solution. Some communities need a bit more than most. That's where the addon system comes in to play. +If you are looking to write a feature to extend the functionality of your modmail bot, plugins are *the* +supported way to add additional code to modmail. + +In short, plugins are discord.py extensions which expand the functionality of the bot beyond its built-in duties. + !!!Tip This builds on the [addon structure documentation][addon-guide]. Please ensure you have a solid understanding of the basic repository structure beforehand. From 4401f348bdb72d1486f846a5ae0ae776306bb251 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 23 Sep 2021 00:01:12 -0400 Subject: [PATCH 082/100] fix: declare required arguments as required --- modmail/extensions/extension_manager.py | 20 +++++--------------- modmail/extensions/plugin_manager.py | 6 +++--- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 2174b292..9c8dae63 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -117,34 +117,26 @@ async def extensions_group(self, ctx: Context) -> None: """Load, unload, reload, and list loaded extensions.""" await ctx.send_help(ctx.command) - @extensions_group.command(name="load", aliases=("l",)) + @extensions_group.command(name="load", aliases=("l",), require_var_positional=True) async def load_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: r""" Load extensions given their fully qualified or unqualified names. If '\*' is given as the name, all unloaded extensions will be loaded. """ - if not extensions: - await ctx.send_help(ctx.command) - return - if "*" in extensions: extensions = sorted(ext for ext in self.all_extensions if ext not in self.bot.extensions.keys()) msg, is_error = self.batch_manage(Action.LOAD, *extensions) await responses.send_response(ctx, msg, not is_error) - @extensions_group.command(name="unload", aliases=("ul",)) + @extensions_group.command(name="unload", aliases=("ul",), require_var_positional=True) async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: r""" Unload currently loaded extensions given their fully qualified or unqualified names. If '\*' is given as the name, all loaded extensions will be unloaded. """ - if not extensions: - await ctx.send_help(ctx.command) - return - blacklisted = [ext for ext in self.get_black_listed_extensions() if ext in extensions] if blacklisted: @@ -164,7 +156,7 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) msg, is_error = self.batch_manage(Action.UNLOAD, *extensions) await responses.send_response(ctx, msg, not is_error) - @extensions_group.command(name="reload", aliases=("r", "rl")) + @extensions_group.command(name="reload", aliases=("r", "rl"), require_var_positional=True) async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: r""" Reload extensions given their fully qualified or unqualified names. @@ -173,10 +165,6 @@ async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) If '\*' is given as the name, all currently loaded extensions will be reloaded. """ - if not extensions: - await ctx.send_help(ctx.command) - return - if "*" in extensions: extensions = self.bot.extensions.keys() & self.all_extensions @@ -352,6 +340,8 @@ async def cog_command_error(self, ctx: Context, error: Exception) -> None: if isinstance(error, commands.BadArgument): await responses.send_negatory_response(ctx, str(error)) error.handled = True + elif isinstance(error, commands.MissingRequiredArgument): + await ctx.send_help(ctx.command) else: raise error diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 230d71b0..2f778879 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -159,7 +159,7 @@ async def plugin_dev_group(self, ctx: Context) -> None: """Manage plugin files directly, rather than whole plugin objects.""" await ctx.send_help(ctx.command) - @plugin_dev_group.command(name="load", aliases=("l",)) + @plugin_dev_group.command(name="load", aliases=("l",), require_var_positional=True) async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Load singular plugin files given their fully qualified or unqualified names. @@ -168,7 +168,7 @@ async def load_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> """ await self.load_extensions.callback(self, ctx, *plugins) - @plugin_dev_group.command(name="unload", aliases=("u", "ul")) + @plugin_dev_group.command(name="unload", aliases=("u", "ul"), require_var_positional=True) async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Unoad singular plugin files given their fully qualified or unqualified names. @@ -177,7 +177,7 @@ async def unload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) - """ await self.unload_extensions.callback(self, ctx, *plugins) - @plugin_dev_group.command(name="reload", aliases=("r", "rl")) + @plugin_dev_group.command(name="reload", aliases=("r", "rl"), require_var_positional=True) async def reload_plugins(self, ctx: Context, *plugins: PluginDevPathConverter) -> None: r""" Reload singular plugin files given their fully qualified or unqualified names. From f7ffd7d9d7116db892b223f3a5ea8fe30f64e509 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Mon, 11 Oct 2021 21:29:23 -0400 Subject: [PATCH 083/100] fix: also export modmail/constraints.txt --- .pre-commit-config.yaml | 2 +- modmail/constraints.txt | 58 ++++++------- scripts/export_requirements.py | 143 ++++++++++++++++++++------------- 3 files changed, 119 insertions(+), 84 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd26af93..4a14f284 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: generate_requirements.txt name: Generate requirements.txt entry: python -m scripts.export_requirements - files: '(pyproject.toml|poetry.lock|requirements.txt|scripts\/export\_requirements\.py)$' + files: '(pyproject.toml|poetry.lock|requirements.txt|constraints.txt|scripts\/export\_requirements\.py)$' language: python pass_filenames: false require_serial: true diff --git a/modmail/constraints.txt b/modmail/constraints.txt index 8eaae06f..a27c0504 100644 --- a/modmail/constraints.txt +++ b/modmail/constraints.txt @@ -1,30 +1,30 @@ -# Do not manually edit. -# Generate with "poetry run task export" +# NOTICE: This file is automatically generated by scripts/export_requirements.py +# This is also automatically regenerated when an edit to pyproject.toml or poetry.lock is commited. -aiodns==3.0.0; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -aiohttp==3.7.4.post0; python_version >= "3.6" -arrow==1.1.1; python_version >= "3.6" -async-timeout==3.0.1; python_full_version >= "3.5.3" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -atoml==1.0.3; python_version >= "3.6" -attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -brotlipy==0.7.0; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -cchardet==2.1.7; python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -cffi==1.14.6; python_version >= "3.6" -chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.8.0" -colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") -coloredlogs==15.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") -discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_full_version >= "3.8.0" -humanfriendly==9.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" -idna==3.2; python_version >= "3.6" -multidict==5.1.0; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" -numpy==1.21.1; python_version >= "3.7" -pycares==4.0.0; python_version >= "3.6" -pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" -pydantic==1.8.2; python_full_version >= "3.6.1" -pyreadline==2.1; python_version >= "2.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and sys_platform == "win32" -python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -python-dotenv==0.19.0; python_full_version >= "3.6.1" and python_version >= "3.5" -rapidfuzz==1.6.1; python_version >= "2.7" -six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -typing-extensions==3.10.0.2; python_version >= "3.6" or python_full_version >= "3.6.1" or python_version >= "3.6" and python_full_version >= "3.8.0" -yarl==1.6.3; python_version >= "3.6" and python_full_version >= "3.8.0" or python_version >= "3.6" +aiodns==3.0.0 +aiohttp==3.7.4.post0 +arrow==1.1.1 +async-timeout==3.0.1 +atoml==1.0.3 +attrs==21.2.0 +brotlipy==0.7.0 +cchardet==2.1.7 +cffi==1.14.6 +chardet==4.0.0 +colorama==0.4.4 +coloredlogs==15.0.1 +discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip +humanfriendly==9.2 +idna==3.2 +multidict==5.1.0 +numpy==1.21.1 +pycares==4.0.0 +pycparser==2.20 +pydantic==1.8.2 +pyreadline==2.1 +python-dateutil==2.8.2 +python-dotenv==0.19.0 +rapidfuzz==1.6.1 +six==1.16.0 +typing-extensions==3.10.0.2 +yarl==1.6.3 diff --git a/scripts/export_requirements.py b/scripts/export_requirements.py index 367dfcda..fe99d177 100644 --- a/scripts/export_requirements.py +++ b/scripts/export_requirements.py @@ -19,6 +19,8 @@ GENERATED_FILE = pathlib.Path("requirements.txt") +CONSTRAINTS_FILE = pathlib.Path("modmail/constraints.txt") + VERSION_RESTRICTER_REGEX = re.compile(r"(?P[<>=!]{1,2})(?P\d+\.\d+?)(?P\.\d+?|\.\*)?") PLATFORM_MARKERS_REGEX = re.compile(r'sys_platform\s?==\s?"(?P\w+)"') @@ -55,7 +57,27 @@ def get_hash(content: dict) -> str: return hash == get_hash(content) -def main(req_path: os.PathLike, should_validate_hash: bool = True) -> typing.Optional[int]: +def _write_file(path: os.PathLike, contents: str, skip_if_identical: bool = True) -> bool: + """ + Write to a supplied file. + + If skip_if_equal is True, will not write if the contents will not change. (Default: True) + """ + path = pathlib.Path(path) + if path.exists(): + with open(path, "r") as f: + if contents == f.read(): + # nothing to edit + return False + + with open(path, "w") as f: + f.write(contents) + return True + + +def export( + req_path: os.PathLike, *, include_markers: bool = True, should_validate_hash: bool = True +) -> typing.Optional[int]: """Read and export all required packages to their pinned version in requirements.txt format.""" req_path = pathlib.Path(req_path) @@ -99,66 +121,75 @@ def main(req_path: os.PathLike, should_validate_hash: bool = True) -> typing.Opt line += "==" line += dep["version"] - if (pyvers := dep["python-versions"]) != "*": - # TODO: add support for platform and python combined version markers - line += " ; " - final_version_index = pyvers.count(", ") - for count, version in enumerate(pyvers.split(", ")): - match = VERSION_RESTRICTER_REGEX.match(version) - - if (patch := match.groupdict().get("patch", None)) is not None and not patch.endswith("*"): - version_kind = "python_full_version" - else: - version_kind = "python_version" - - patch = patch if patch is not None else "" - patch = patch if not patch.endswith("*") else "" - line += version_kind + " " - line += match.group("sign") + " " - line += '"' + match.group("version") + patch + '"' - line += " " - if count < final_version_index: - line += "and " - - if (dep_deps := dep.get("dependencies", None)) is not None: - - for k, v in copy.copy(dep_deps).items(): - if hasattr(v, "get") and v.get("markers", None) is not None: - pass - else: - del dep_deps[k] - if len(dep_deps): - to_add_markers.update(dep_deps) + if include_markers: + if (pyvers := dep["python-versions"]) != "*": + # TODO: add support for platform and python combined version markers + line += " ; " + final_version_index = pyvers.count(", ") + for count, version in enumerate(pyvers.split(", ")): + match = VERSION_RESTRICTER_REGEX.match(version) + + if (patch := match.groupdict().get("patch", None)) is not None and not patch.endswith( + "*" + ): + version_kind = "python_full_version" + else: + version_kind = "python_version" + + patch = patch if patch is not None else "" + patch = patch if not patch.endswith("*") else "" + line += version_kind + " " + line += match.group("sign") + " " + line += '"' + match.group("version") + patch + '"' + line += " " + if count < final_version_index: + line += "and " + + if (dep_deps := dep.get("dependencies", None)) is not None: + + for k, v in copy.copy(dep_deps).items(): + if hasattr(v, "get") and v.get("markers", None) is not None: + pass + else: + del dep_deps[k] + if len(dep_deps): + to_add_markers.update(dep_deps) dependency_lines[dep["name"]] = line - # add the sys_platform lines - # platform markers only matter based on what requires the dependency - # in order to support these properly, they have to be added to an already existing line - # for example, humanfriendly requires pyreadline on windows only, - # so sys_platform == win needs to be added to pyreadline - for k, v in to_add_markers.items(): - line = dependency_lines[k] - markers = PLATFORM_MARKERS_REGEX.match(v["markers"]) - if markers is not None: - if ";" not in line: - line += " ; " - elif "python_" in line or "sys_platform" in line: - line += "and " - line += 'sys_platform == "' + markers.group("platform") + '"' - dependency_lines[k] = line + if include_markers: + # add the sys_platform lines + # platform markers only matter based on what requires the dependency + # in order to support these properly, they have to be added to an already existing line + # for example, humanfriendly requires pyreadline on windows only, + # so sys_platform == win needs to be added to pyreadline + for k, v in to_add_markers.items(): + line = dependency_lines[k] + markers = PLATFORM_MARKERS_REGEX.match(v["markers"]) + if markers is not None: + if ";" not in line: + line += " ; " + elif "python_" in line or "sys_platform" in line: + line += "and " + line += 'sys_platform == "' + markers.group("platform") + '"' + dependency_lines[k] = line req_txt += "\n".join(sorted(k + v.rstrip() for k, v in dependency_lines.items())) + "\n" - if req_path.exists(): - with open(req_path, "r") as f: - if req_txt == f.read(): - # nothing to edit - return 0 - with open(req_path, "w") as f: - f.write(req_txt) + if _write_file(req_path, req_txt): print(f"Updated {req_path} with new requirements.") return 1 + else: + print(f"No changes were made to {req_path}") + return 0 + + +def main(path: os.PathLike, include_markers: bool = True, **kwargs) -> int: + """Export a requirements.txt and constraints.txt file.""" + if not include_markers: + path = path or CONSTRAINTS_FILE + kwargs["include_markers"] = include_markers + return export(path, **kwargs) if __name__ == "__main__": @@ -183,4 +214,8 @@ def main(req_path: os.PathLike, should_validate_hash: bool = True) -> typing.Opt ) args = parser.parse_args() - sys.exit(main(args.output_file, should_validate_hash=not args.skip_hash_check)) + # I am aware that the second method will only run if the first method returns 0. This is intended. + sys.exit( + main(args.output_file, should_validate_hash=not args.skip_hash_check) + or main(CONSTRAINTS_FILE, include_markers=False, should_validate_hash=not args.skip_hash_check) + ) From 1f49b79045a35bb4ed2c5fd8518c317f40aac5ab Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 17 Nov 2021 19:06:19 -0500 Subject: [PATCH 084/100] chore: fix bugs arised from merge --- modmail/extensions/extension_manager.py | 15 ++++++++++++--- modmail/extensions/utils/error_handler.py | 4 ++-- tests/modmail/extensions/test_plugin_manager.py | 17 ++++++++--------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index 9c8dae63..b4d6cd8f 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -128,7 +128,10 @@ async def load_extensions(self, ctx: Context, *extensions: ExtensionConverter) - extensions = sorted(ext for ext in self.all_extensions if ext not in self.bot.extensions.keys()) msg, is_error = self.batch_manage(Action.LOAD, *extensions) - await responses.send_response(ctx, msg, not is_error) + if not is_error: + await responses.send_positive_response(ctx, msg) + else: + await responses.send_negatory_response(ctx, msg) @extensions_group.command(name="unload", aliases=("ul",), require_var_positional=True) async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -154,7 +157,10 @@ async def unload_extensions(self, ctx: Context, *extensions: ExtensionConverter) ) msg, is_error = self.batch_manage(Action.UNLOAD, *extensions) - await responses.send_response(ctx, msg, not is_error) + if not is_error: + await responses.send_positive_response(ctx, msg) + else: + await responses.send_negatory_response(ctx, msg) @extensions_group.command(name="reload", aliases=("r", "rl"), require_var_positional=True) async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) -> None: @@ -169,7 +175,10 @@ async def reload_extensions(self, ctx: Context, *extensions: ExtensionConverter) extensions = self.bot.extensions.keys() & self.all_extensions msg, is_error = self.batch_manage(Action.RELOAD, *extensions) - await responses.send_response(ctx, msg, not is_error) + if not is_error: + await responses.send_positive_response(ctx, msg) + else: + await responses.send_negatory_response(ctx, msg) @extensions_group.command(name="list", aliases=("all", "ls")) async def list_extensions(self, ctx: Context) -> None: diff --git a/modmail/extensions/utils/error_handler.py b/modmail/extensions/utils/error_handler.py index 181519c6..a71d7b9a 100644 --- a/modmail/extensions/utils/error_handler.py +++ b/modmail/extensions/utils/error_handler.py @@ -9,7 +9,7 @@ from modmail.bot import ModmailBot from modmail.log import ModmailLogger from modmail.utils import responses -from modmail.utils.cogs import BotModes, ExtMetadata, ModmailCog +from modmail.utils.cogs import BotModeEnum, ExtMetadata, ModmailCog from modmail.utils.extensions import BOT_MODE @@ -21,7 +21,7 @@ ERROR_TITLE_REGEX = re.compile(r"((?<=[a-z])[A-Z]|(?<=[a-zA-Z])[A-Z](?=[a-z]))") -ANY_DEV_MODE = BOT_MODE & (BotModes.DEVELOP.value + BotModes.PLUGIN_DEV.value) +ANY_DEV_MODE = BOT_MODE & (BotModeEnum.DEVELOP.value + BotModeEnum.PLUGIN_DEV.value) class ErrorHandler(ModmailCog, name="Error Handler"): diff --git a/tests/modmail/extensions/test_plugin_manager.py b/tests/modmail/extensions/test_plugin_manager.py index eddae16d..95ba0ae7 100644 --- a/tests/modmail/extensions/test_plugin_manager.py +++ b/tests/modmail/extensions/test_plugin_manager.py @@ -1,32 +1,31 @@ +import unittest.mock from copy import copy import pytest +from modmail.addons.plugins import PLUGINS as GLOBAL_PLUGINS +from modmail.addons.plugins import find_plugins from modmail.extensions.plugin_manager import PluginConverter -from modmail.utils.plugins import PLUGINS as GLOBAL_PLUGINS -from modmail.utils.plugins import walk_plugins # load EXTENSIONS PLUGINS = copy(GLOBAL_PLUGINS) -PLUGINS.update(walk_plugins()) +PLUGINS.update(find_plugins()) class TestPluginConverter: """Test the extension converter converts extensions properly.""" - all_plugins = {x: y for x, y in walk_plugins()} - @pytest.fixture(scope="class", name="converter") def converter(self) -> PluginConverter: """Fixture method for a PluginConverter object.""" return PluginConverter() @pytest.mark.asyncio - @pytest.mark.parametrize("plugin", [e.rsplit(".", 1)[-1] for e in all_plugins.keys()]) + @pytest.mark.parametrize("plugin", [e.name for e in PLUGINS]) async def test_conversion_success(self, plugin: str, converter: PluginConverter) -> None: """Test all plugins in the list are properly converted.""" - converter.source_list = self.all_plugins - converted = await converter.convert(None, plugin) + with unittest.mock.patch("modmail.extensions.plugin_manager.PLUGINS", PLUGINS): + converted = await converter.convert(None, plugin) - assert converted.endswith(plugin) + assert plugin == converted.name From 74c2d78045bf90bbe418e5f1be38e377e4328e4d Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 17 Nov 2021 21:12:26 -0500 Subject: [PATCH 085/100] fix: use discord.py advanced converters for Plugin models --- modmail/addons/models.py | 75 ++++++++++++++++++- modmail/extensions/plugin_manager.py | 71 ++---------------- tests/modmail/addons/test_plugins.py | 24 +++++- .../modmail/extensions/test_plugin_manager.py | 31 -------- 4 files changed, 102 insertions(+), 99 deletions(-) delete mode 100644 tests/modmail/extensions/test_plugin_manager.py diff --git a/modmail/addons/models.py b/modmail/addons/models.py index d3004bd4..1b72f96a 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -1,15 +1,25 @@ from __future__ import annotations +import logging import re from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, List, Literal, NoReturn, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, NoReturn, Optional, Set, Union + +from discord.ext import commands +from rapidfuzz import fuzz, process + +from modmail.utils.extensions import ModuleDict if TYPE_CHECKING: import pathlib import zipfile - from modmail.utils.extensions import ModuleDict + from modmail.log import ModmailLogger + +logger: ModmailLogger = logging.getLogger(__name__) + +PLUGINS = None class SourceTypeEnum(Enum): @@ -191,3 +201,64 @@ def __hash__(self): def __eq__(self, other: Any): return hash(self) == hash(other) + + @classmethod + async def convert(cls, ctx: commands.Context, argument: str) -> Plugin: + """Converts a plugin into a full plugin with a path and all other attributes.""" + # have to do this here to prevent a recursive import + global PLUGINS + if PLUGINS is None: + logger.debug("Lazy import of global PLUGINS from modmail.addons.plugins") + from modmail.addons.plugins import PLUGINS + + loaded_plugs: Set[Plugin] = PLUGINS + + # its possible to have a plugin with the same name as a folder of a plugin + # folder names are the priority + secondary_names = dict() + for plug in loaded_plugs: + if argument == plug.name: + return plug + secondary_names[plug.folder_name] = plug + + if argument in secondary_names: + return secondary_names[argument] + + # Determine close plugins + # Using a dict to prevent duplicates + arg_mapping: Dict[str, Plugin] = dict() + for plug in loaded_plugs: + for name in plug.name, plug.folder_name: + arg_mapping[name] = plug + + result = process.extract( + argument, + arg_mapping.keys(), + scorer=fuzz.ratio, + score_cutoff=69, + ) + logger.debug(f"{result = }") + + if not len(result): + raise commands.BadArgument(f"`{argument}` is not in list of installed plugins.") + + all_fully_matched_plugins: Set[Plugin] = set() + all_partially_matched_plugins: Dict[Plugin, float] = dict() + for res in result: + all_partially_matched_plugins[arg_mapping[res[0]]] = res[1] + + if res[1] == 100: + all_fully_matched_plugins.add(arg_mapping[res[0]]) + + if len(all_fully_matched_plugins) != 1: + suggested = "" + for plug, percent in all_partially_matched_plugins.items(): + suggested += f"`{plug.name}` ({round(percent)}%)\n" + raise commands.BadArgument( + f"`{argument}` is not in list of installed plugins." + f"\n\n**Suggested plugins**:\n{suggested}" + if len(suggested) + else "" + ) + + return await cls.convert(ctx, all_fully_matched_plugins.pop().name) diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 2f778879..6c53c839 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -4,14 +4,13 @@ import logging import shutil from collections import defaultdict -from typing import TYPE_CHECKING, Dict, Mapping, Set +from typing import TYPE_CHECKING, Mapping from atoml.exceptions import ParseError from discord import Colour, Embed from discord.abc import Messageable from discord.ext import commands from discord.ext.commands import Context -from rapidfuzz import fuzz, process import modmail.addons.utils as addon_utils from modmail import errors @@ -66,64 +65,6 @@ def __init__(self): self.source_list = modules -class PluginConverter(commands.Converter): - """Convert a plugin name into a full plugin with path and related args.""" - - async def convert(self, ctx: Context, argument: str) -> Plugin: - """Converts a plugin into a full plugin with a path and all other attributes.""" - loaded_plugs: Set[Plugin] = PLUGINS - - # its possible to have a plugin with the same name as a folder of a plugin - # folder names are the priority - secondary_names = dict() - for plug in loaded_plugs: - if argument == plug.name: - return plug - secondary_names[plug.folder_name] = plug - - if argument in secondary_names: - return secondary_names[argument] - - # Determine close plugins - # Using a dict to prevent duplicates - arg_mapping: Dict[str, Plugin] = dict() - for plug in loaded_plugs: - for name in plug.name, plug.folder_name: - arg_mapping[name] = plug - - result = process.extract( - argument, - arg_mapping.keys(), - scorer=fuzz.ratio, - score_cutoff=69, - ) - logger.debug(f"{result = }") - - if not len(result): - raise commands.BadArgument(f"`{argument}` is not in list of installed plugins.") - - all_fully_matched_plugins: Set[Plugin] = set() - all_partially_matched_plugins: Dict[Plugin, float] = dict() - for res in result: - all_partially_matched_plugins[arg_mapping[res[0]]] = res[1] - - if res[1] == 100: - all_fully_matched_plugins.add(arg_mapping[res[0]]) - - if len(all_fully_matched_plugins) != 1: - suggested = "" - for plug, percent in all_partially_matched_plugins.items(): - suggested += f"`{plug.name}` ({round(percent)}%)\n" - raise commands.BadArgument( - f"`{argument}` is not in list of installed plugins." - f"\n\n**Suggested plugins**:\n{suggested}" - if len(suggested) - else "" - ) - - return await self.convert(ctx, all_fully_matched_plugins.pop().name) - - class PluginManager(ExtensionManager, name="Plugin Manager"): """Plugin management commands.""" @@ -340,7 +281,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu ) @plugins_group.command(name="uninstall", aliases=("rm",)) - async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + async def uninstall_plugin(self, ctx: Context, *, plugin: Plugin) -> None: """Uninstall a provided plugin, given the name of the plugin.""" plugin: Plugin = plugin @@ -350,7 +291,7 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No ) return - plugin = await PluginConverter().convert(ctx, plugin.folder_name) + plugin = await Plugin.convert(ctx, plugin.folder_name) _, err = self.batch_manage( Action.UNLOAD, *plugin.modules.keys(), is_plugin=True, suppress_already_error=True ) @@ -362,7 +303,7 @@ async def uninstall_plugin(self, ctx: Context, *, plugin: PluginConverter) -> No shutil.rmtree(plugin.installed_path) - plugin = await PluginConverter().convert(ctx, plugin.folder_name) + plugin = await Plugin.convert(ctx, plugin.folder_name) PLUGINS.remove(plugin) await responses.send_positive_response(ctx, f"Successfully uninstalled plugin {plugin}") @@ -400,12 +341,12 @@ async def _enable_or_disable_plugin( await responses.send_positive_response(ctx, f":thumbsup: Plugin {plugin!s} successfully {verb}d.") @plugins_group.command(name="enable") - async def enable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + async def enable_plugin(self, ctx: Context, *, plugin: Plugin) -> None: """Enable a provided plugin, given the name or folder of the plugin.""" await self._enable_or_disable_plugin(ctx, plugin, Action.ENABLE, True) @plugins_group.command(name="disable") - async def disable_plugin(self, ctx: Context, *, plugin: PluginConverter) -> None: + async def disable_plugin(self, ctx: Context, *, plugin: Plugin) -> None: """Disable a provided plugin, given the name or folder of the plugin.""" await self._enable_or_disable_plugin(ctx, plugin, Action.DISABLE, False) diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py index 41173e18..211b30df 100644 --- a/tests/modmail/addons/test_plugins.py +++ b/tests/modmail/addons/test_plugins.py @@ -1,9 +1,18 @@ from __future__ import annotations +import unittest.mock +from copy import copy + import pytest from modmail.addons.models import Plugin -from modmail.addons.plugins import parse_plugin_toml_from_string +from modmail.addons.plugins import PLUGINS as GLOBAL_PLUGINS +from modmail.addons.plugins import find_plugins, parse_plugin_toml_from_string + + +# load PLUGINS +PLUGINS = copy(GLOBAL_PLUGINS) +PLUGINS.update(find_plugins()) VALID_PLUGIN_TOML = """ @@ -39,3 +48,16 @@ def test_parse_plugin_toml_from_string( assert plug.folder_name == folder assert plug.description == description assert plug.min_bot_version == min_bot_version + + +class TestPluginConversion: + """Test the extension converter converts extensions properly.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("plugin", [e.name for e in PLUGINS]) + async def test_conversion_success(self, plugin: str) -> None: + """Test all plugins in the list are properly converted.""" + with unittest.mock.patch("modmail.addons.plugins.PLUGINS", PLUGINS): + converted = await Plugin.convert(None, plugin) + + assert plugin == converted.name diff --git a/tests/modmail/extensions/test_plugin_manager.py b/tests/modmail/extensions/test_plugin_manager.py deleted file mode 100644 index 95ba0ae7..00000000 --- a/tests/modmail/extensions/test_plugin_manager.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest.mock -from copy import copy - -import pytest - -from modmail.addons.plugins import PLUGINS as GLOBAL_PLUGINS -from modmail.addons.plugins import find_plugins -from modmail.extensions.plugin_manager import PluginConverter - - -# load EXTENSIONS -PLUGINS = copy(GLOBAL_PLUGINS) -PLUGINS.update(find_plugins()) - - -class TestPluginConverter: - """Test the extension converter converts extensions properly.""" - - @pytest.fixture(scope="class", name="converter") - def converter(self) -> PluginConverter: - """Fixture method for a PluginConverter object.""" - return PluginConverter() - - @pytest.mark.asyncio - @pytest.mark.parametrize("plugin", [e.name for e in PLUGINS]) - async def test_conversion_success(self, plugin: str, converter: PluginConverter) -> None: - """Test all plugins in the list are properly converted.""" - with unittest.mock.patch("modmail.extensions.plugin_manager.PLUGINS", PLUGINS): - converted = await converter.convert(None, plugin) - - assert plugin == converted.name From b3e30708c19d34bd67dbeb47a381d5a4f7f928e2 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 00:07:54 -0500 Subject: [PATCH 086/100] fix: patch plugin folder during tests fixes issue with testing environment differing if the user has custom plugins resolved by monkeypatching the modmail.plugin module to be believed to be in a different location this keeps the plugins consistent between environments --- modmail/addons/plugins.py | 8 +++++-- tests/modmail/addons/test_plugins.py | 22 ++++++++++++------- tests/modmail/conftest.py | 16 ++++++++++++++ tests/modmail/plugins/README.md | 4 ++++ tests/modmail/plugins/__init__.py | 1 + tests/modmail/plugins/test1.toml | 3 +++ .../plugins/working_plugin/working_plug.py | 21 ++++++++++++++++++ 7 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 tests/modmail/plugins/README.md create mode 100644 tests/modmail/plugins/__init__.py create mode 100644 tests/modmail/plugins/test1.toml create mode 100644 tests/modmail/plugins/working_plugin/working_plug.py diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 68dc6473..51cfa7e6 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -241,13 +241,15 @@ def find_partial_plugins_from_dir( def find_plugins( - detection_path: pathlib.Path = BASE_PLUGIN_PATH, /, *, local: Optional[bool] = True + detection_path: pathlib.Path = None, /, *, local: Optional[bool] = True ) -> Generator[Plugin, None, None]: """ Walks the local path, and determines which files are local plugins. Yields a list of plugins, """ + if detection_path is None: + detection_path = BASE_PLUGIN_PATH all_plugins: Set[Plugin] = set() toml_plugins: List[Plugin] = [] @@ -296,7 +298,7 @@ def find_plugins( def walk_plugin_files( - detection_path: pathlib.Path = BASE_PLUGIN_PATH, + detection_path: pathlib.Path = None, ) -> Generator[Tuple[ModuleName, ExtMetadata], None, None]: """Yield plugin names from the modmail.plugins subpackage.""" # walk all files in the plugins folder @@ -304,6 +306,8 @@ def walk_plugin_files( # which are important for ease of development. # NOTE: We are not using Pathlib's glob utility as it doesn't # support following symlinks, see: https://bugs.python.org/issue33428 + if detection_path is None: + detection_path = BASE_PLUGIN_PATH for path in glob.iglob(f"{detection_path}/**/*.py", recursive=True): logger.trace(f"{path =}") diff --git a/tests/modmail/addons/test_plugins.py b/tests/modmail/addons/test_plugins.py index 211b30df..e24b412d 100644 --- a/tests/modmail/addons/test_plugins.py +++ b/tests/modmail/addons/test_plugins.py @@ -8,11 +8,10 @@ from modmail.addons.models import Plugin from modmail.addons.plugins import PLUGINS as GLOBAL_PLUGINS from modmail.addons.plugins import find_plugins, parse_plugin_toml_from_string +from tests import mocks -# load PLUGINS -PLUGINS = copy(GLOBAL_PLUGINS) -PLUGINS.update(find_plugins()) +pytestmark = pytest.mark.usefixtures("reroute_plugins") VALID_PLUGIN_TOML = """ @@ -53,11 +52,18 @@ def test_parse_plugin_toml_from_string( class TestPluginConversion: """Test the extension converter converts extensions properly.""" + @classmethod + def setup_class(cls): + """Set the class plugins var to the scanned plugins.""" + cls.plugins = set(find_plugins()) + @pytest.mark.asyncio - @pytest.mark.parametrize("plugin", [e.name for e in PLUGINS]) - async def test_conversion_success(self, plugin: str) -> None: + async def test_conversion_success(self) -> None: """Test all plugins in the list are properly converted.""" - with unittest.mock.patch("modmail.addons.plugins.PLUGINS", PLUGINS): - converted = await Plugin.convert(None, plugin) + with unittest.mock.patch("modmail.addons.plugins.PLUGINS", self.plugins): + + for plugin in self.plugins: + print(f"Current plugin: {plugin}") + converted = await Plugin.convert(mocks.MockContext(), plugin) - assert plugin == converted.name + assert plugin.name == converted.name diff --git a/tests/modmail/conftest.py b/tests/modmail/conftest.py index 2e3b5139..1e699cc4 100644 --- a/tests/modmail/conftest.py +++ b/tests/modmail/conftest.py @@ -23,6 +23,22 @@ def _get_env(): return pathlib.Path(__file__).parent / "test.env" +@pytest.fixture(scope="package") +def reroute_plugins(): + """Reroute the plugin directory.""" + import modmail.plugins + from tests.modmail import plugins + + modmail.plugins.__file__ = plugins.__file__ + + import modmail.addons.plugins + + modmail.addons.plugins.BASE_PLUGIN_PATH = pathlib.Path(plugins.__file__).parent.resolve() + + modmail.addons.plugins.LOCAL_PLUGIN_TOML = modmail.addons.plugins.BASE_PLUGIN_PATH / "test1.toml" + yield + + def pytest_configure(): """Check that the test specific env file exists, and cancel the run if it does not exist.""" env = _get_env() diff --git a/tests/modmail/plugins/README.md b/tests/modmail/plugins/README.md new file mode 100644 index 00000000..2fba42e3 --- /dev/null +++ b/tests/modmail/plugins/README.md @@ -0,0 +1,4 @@ +These are where some plugins made for testing belong. + +The tests/modmail/conftest.py file overwrites modmail.plugins.__file__ +to redirect to the `[__init__.py](./__init__.py) located here.. diff --git a/tests/modmail/plugins/__init__.py b/tests/modmail/plugins/__init__.py new file mode 100644 index 00000000..5895363c --- /dev/null +++ b/tests/modmail/plugins/__init__.py @@ -0,0 +1 @@ +from modmail.plugins import * # noqa: F401 F403 diff --git a/tests/modmail/plugins/test1.toml b/tests/modmail/plugins/test1.toml new file mode 100644 index 00000000..ad7fc2cf --- /dev/null +++ b/tests/modmail/plugins/test1.toml @@ -0,0 +1,3 @@ +[[plugins]] +name = 'working plug' +directory = 'working_plugin' diff --git a/tests/modmail/plugins/working_plugin/working_plug.py b/tests/modmail/plugins/working_plugin/working_plug.py new file mode 100644 index 00000000..df5ca441 --- /dev/null +++ b/tests/modmail/plugins/working_plugin/working_plug.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import logging + +from modmail.addons.helpers import PluginCog +from modmail.bot import ModmailBot +from modmail.log import ModmailLogger + + +logger: ModmailLogger = logging.getLogger(__name__) + + +class WorkingPlugin(PluginCog): + """Demonstration plugin for testing.""" + + pass + + +def setup(bot: ModmailBot) -> None: + """Add the gateway logger to the bot.""" + bot.add_cog(WorkingPlugin(bot)) From 34031e4b30440ede1f57cc33925248fd5f6d5fa1 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 00:19:50 -0500 Subject: [PATCH 087/100] fix: use WindowsSelectorEventLoopPolicy on windows --- modmail/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modmail/__init__.py b/modmail/__init__.py index da261784..450d9a9a 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -1,5 +1,7 @@ +import asyncio import logging import logging.handlers +import os from pathlib import Path import coloredlogs @@ -7,6 +9,10 @@ from modmail.log import ModmailLogger +# On Windows, the selector event loop is required for aiodns. +if os.name == "nt": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + logging.TRACE = 5 logging.NOTICE = 25 logging.addLevelName(logging.TRACE, "TRACE") From 1c88421ee826491e2f45e5202ad78db1ca4a5c05 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 00:39:55 -0500 Subject: [PATCH 088/100] ci: run tests with verboseness, not quietness --- .github/workflows/lint_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index 6db38e31..a5655ec5 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -20,7 +20,6 @@ env: # Configure pip to cache dependencies and do a user install PIP_NO_CACHE_DIR: false PIP_USER: 1 - PYTHON_VERSION: 3.8 # Make sure package manager does not use virtualenv POETRY_VIRTUALENVS_CREATE: false @@ -162,7 +161,7 @@ jobs: # This is saved to ./.coverage to be used by codecov to link a # coverage report to github. - name: Run tests and generate coverage report - run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -q + run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -v --showlocals # This step will publish the coverage reports to codecov.io and # print a "job" link in the output of the GitHub Action From 8be233298773132d6963be44d542b61078971711 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 04:11:25 -0500 Subject: [PATCH 089/100] chore: fix windows dns lookup --- modmail/bot.py | 38 ++++++++++++++++++++++++++---- tests/conftest.py | 16 +++++++++++++ tests/modmail/addons/test_utils.py | 17 ++----------- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/modmail/bot.py b/modmail/bot.py index d9ff7e87..5eecbdac 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -1,11 +1,12 @@ import asyncio import logging import signal +import socket from typing import Any, Optional, Set +import aiohttp import arrow import discord -from aiohttp import ClientSession from discord import Activity, AllowedMentions, Intents from discord.client import _cleanup_loop from discord.ext import commands @@ -44,9 +45,12 @@ class ModmailBot(commands.Bot): def __init__(self, **kwargs): self.config = CONFIG self.start_time: Optional[arrow.Arrow] = None # arrow.utcnow() - self.http_session: Optional[ClientSession] = None + self.http_session: Optional[aiohttp.ClientSession] = None self.dispatcher = Dispatcher() + self._connector = None + self._resolver = None + # keys: plugins, list values: all plugin files self.installed_plugins: Set[Plugin] = {} @@ -71,6 +75,24 @@ def __init__(self, **kwargs): **kwargs, ) + async def create_connectors(self, *args, **kwargs) -> None: + """Re-create the connector and set up sessions before logging into Discord.""" + # Use asyncio for DNS resolution instead of threads so threads aren't spammed. + self._resolver = aiohttp.AsyncResolver() + + # Use AF_INET as its socket family to prevent HTTPS related problems both locally + # and in production. + self._connector = aiohttp.TCPConnector( + resolver=self._resolver, + family=socket.AF_INET, + ) + + # Client.login() will call HTTPClient.static_login() which will create a session using + # this connector attribute. + self.http.connector = self._connector + + self.http_session = aiohttp.ClientSession(connector=self._connector) + async def start(self, token: str, reconnect: bool = True) -> None: """ Start the bot. @@ -80,8 +102,8 @@ async def start(self, token: str, reconnect: bool = True) -> None: """ try: # create the aiohttp session - self.http_session = ClientSession(loop=self.loop) - self.logger.trace("Created ClientSession.") + await self.create_connectors() + self.logger.trace("Created aiohttp.ClientSession.") # set start time to when we started the bot. # This is now, since we're about to connect to the gateway. # This should also be before we load any extensions, since if they have a load time, it should @@ -175,10 +197,16 @@ async def close(self) -> None: except Exception: self.logger.error(f"Exception occured while removing cog {cog.name}", exc_info=True) + await super().close() + if self.http_session: await self.http_session.close() - await super().close() + if self._connector: + await self._connector.close() + + if self._resolver: + await self._resolver.close() def load_extensions(self) -> None: """Load all enabled extensions.""" diff --git a/tests/conftest.py b/tests/conftest.py index 5871ed8e..9ac88d6c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,17 @@ +import aiohttp import pytest + + +@pytest.fixture +@pytest.mark.asyncio +async def http_session() -> aiohttp.ClientSession: + """Fixture function for a aiohttp.ClientSession.""" + resolver = aiohttp.AsyncResolver() + connector = aiohttp.TCPConnector(resolver=resolver) + client_session = aiohttp.ClientSession(connector=connector) + + yield client_session + + await client_session.close() + await connector.close() + await resolver.close() diff --git a/tests/modmail/addons/test_utils.py b/tests/modmail/addons/test_utils.py index c1db672c..35913d0b 100644 --- a/tests/modmail/addons/test_utils.py +++ b/tests/modmail/addons/test_utils.py @@ -9,25 +9,12 @@ from modmail.addons.utils import download_zip_from_source -# https://github.com/discord-modmail/modmail/archive/main.zip - - -@pytest.fixture() -@pytest.mark.asyncio -async def session() -> ClientSession: - """Fixture function for a aiohttp.ClientSession.""" - sess = ClientSession() - yield sess - - await sess.close() - - @pytest.mark.parametrize( "source", [AddonSource.from_zip("https://github.com/discord-modmail/modmail/archive/main.zip")] ) @pytest.mark.asyncio -async def test_download_zip_from_source(source: AddonSource, session: ClientSession): +async def test_download_zip_from_source(source: AddonSource, http_session: ClientSession): """Test that a zip can be successfully downloaded and everything is safe inside.""" - file = await download_zip_from_source(source, session) + file = await download_zip_from_source(source, http_session) assert isinstance(file, zipfile.ZipFile) assert file.testzip() is None From 6e349943cc8bf2ee8df76d44d5427d2b7036e306 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 03:31:29 -0500 Subject: [PATCH 090/100] fix: use pathlib to determine module name --- modmail/addons/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 51cfa7e6..ba6936fb 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -314,7 +314,7 @@ def walk_plugin_files( # calculate the module name, dervived from the relative path relative_path = pathlib.Path(path).relative_to(BASE_PLUGIN_PATH) - module_name = relative_path.__str__().rstrip(".py").replace("/", ".") + module_name = ".".join(relative_path.parent.parts) + "." + relative_path.stem module_name = plugins.__name__ + "." + module_name logger.trace(f"{module_name =}") From c8e12bcd771474d062a4d64e0abfd310577291d9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 18 Nov 2021 21:22:08 -0500 Subject: [PATCH 091/100] chore: use a url parsing library instead of regex --- modmail/addons/models.py | 13 ++++++------- modmail/addons/utils.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 1b72f96a..2f9004d8 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -import re +import urllib.parse from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Literal, NoReturn, Optional, Set, Union @@ -85,10 +85,10 @@ def __init__(self, zip_url: str, type: SourceTypeEnum): """Initialize the AddonSource.""" self.zip_url = zip_url if self.zip_url is not None: - match = re.match(r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*))$", self.zip_url) - self.zip_url = match.group("url") - self.domain = match.group("domain") - self.path = match.group("path") + parsed_url = urllib.parse.urlparse(self.zip_url) + self.zip_url = urllib.parse.urlunparse(parsed_url) + self.domain = parsed_url.netloc + self.path = parsed_url.path else: self.domain = None self.path = None @@ -120,8 +120,7 @@ def from_repo(cls, user: str, repo: str, reflike: str = None, githost: Host = "g @classmethod def from_zip(cls, url: str) -> AddonSource: """Create an AddonSource from a zip file.""" - match = re.match(r"^(?P(?:https?:\/\/)?(?P.*\..+?)\/(?P.*\.zip))$", url) - source = cls(match.group("url"), SourceTypeEnum.ZIP) + source = cls(url, SourceTypeEnum.ZIP) return source def __repr__(self) -> str: # pragma: no cover diff --git a/modmail/addons/utils.py b/modmail/addons/utils.py index 600015d7..80a27abf 100644 --- a/modmail/addons/utils.py +++ b/modmail/addons/utils.py @@ -44,7 +44,7 @@ async def download_zip_from_source(source: AddonSource, session: ClientSession) if source.source_type not in (SourceTypeEnum.REPO, SourceTypeEnum.ZIP): raise TypeError("Unsupported source detected.") - async with session.get(f"https://{source.zip_url}") as resp: + async with session.get(source.zip_url, timeout=20) as resp: if resp.status != 200: raise HTTPError(resp) raw_bytes = await resp.read() From 0b89af0c11057cfc6499d6548fa4e90faacb3cce Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 01:22:35 -0500 Subject: [PATCH 092/100] tests: add aioresponses for aiohttp response mocking --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + tests/conftest.py | 16 ++++++++++++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0b1c4c64..fe4c124f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,6 +31,17 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +name = "aioresponses" +version = "0.7.2" +description = "Mock out requests made by ClientSession from aiohttp package" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +aiohttp = ">=2.0.0,<4.0.0" + [[package]] name = "arrow" version = "1.1.1" @@ -1247,7 +1258,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "ef8508a4932f302dcf69aa385a061c5ea6ec3e8f69b38134b1ca6bc01f6d1d1d" +content-hash = "da86ea6533f92352cf48dccb1404707a2aea1675279934626dc5f47e24bbc56b" [metadata.files] aiodns = [ @@ -1293,6 +1304,10 @@ aiohttp = [ {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"}, {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"}, ] +aioresponses = [ + {file = "aioresponses-0.7.2-py2.py3-none-any.whl", hash = "sha256:2f8ff624543066eb465b0238de68d29231e8488f41dc4b5a9dae190982cdae50"}, + {file = "aioresponses-0.7.2.tar.gz", hash = "sha256:82e495d118b74896aa5b4d47e17effb5e2cc783e510ae395ceade5e87cabe89a"}, +] arrow = [ {file = "arrow-1.1.1-py3-none-any.whl", hash = "sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510"}, {file = "arrow-1.1.1.tar.gz", hash = "sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a"}, diff --git a/pyproject.toml b/pyproject.toml index cee0b96d..eedab9cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ flake8-todo = "~=0.7" isort = "^5.9.2" pep8-naming = "~=0.11" # testing +aioresponses = "^0.7.2" codecov = "^2.1.11" coverage = { extras = ["toml"], version = "^6.0.2" } pytest = "^6.2.4" diff --git a/tests/conftest.py b/tests/conftest.py index 9ac88d6c..40c3ae9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,23 @@ import aiohttp +import aioresponses import pytest +@pytest.fixture +def aioresponse(): + """Fixture to mock aiohttp responses.""" + with aioresponses.aioresponses() as aioresponse: + yield aioresponse + + @pytest.fixture @pytest.mark.asyncio -async def http_session() -> aiohttp.ClientSession: - """Fixture function for a aiohttp.ClientSession.""" +async def http_session(aioresponse) -> aiohttp.ClientSession: + """ + Fixture function for a aiohttp.ClientSession. + + Requests fixture aioresponse to ensure that all client sessions do not make actual requests. + """ resolver = aiohttp.AsyncResolver() connector = aiohttp.TCPConnector(resolver=resolver) client_session = aiohttp.ClientSession(connector=connector) From d96d7a785a287e72835c953e16f40bb01bdf6f87 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 03:12:05 -0500 Subject: [PATCH 093/100] fix: mark failing test as xfail this is too involved to fix right now Solution involves returning zip files, but the changes to do that require commiting a zipfile, or zipping a zip file before the test. But the contents of that zipfile should be the expected format, so there needs to be several mock files for that. But then they should be in a test resources directory But then the test resources directory should have a few helper methods But then the test resources will need a large amount of review. So the short is, there is a fix for these tests, but it adds a subsystem and is out of scope of this pr. --- tests/modmail/addons/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/modmail/addons/test_utils.py b/tests/modmail/addons/test_utils.py index 35913d0b..2c31bbbe 100644 --- a/tests/modmail/addons/test_utils.py +++ b/tests/modmail/addons/test_utils.py @@ -9,6 +9,7 @@ from modmail.addons.utils import download_zip_from_source +@pytest.mark.xfail @pytest.mark.parametrize( "source", [AddonSource.from_zip("https://github.com/discord-modmail/modmail/archive/main.zip")] ) From 550764da2c76e1ebd14096be51c1787d445cb81e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 07:01:43 -0500 Subject: [PATCH 094/100] deps: upgrade rapidfuzz to wheel supported version --- modmail/constraints.txt | 3 +- poetry.lock | 153 +++++++++++++++------------------------- pyproject.toml | 2 +- requirements.txt | 3 +- 4 files changed, 61 insertions(+), 100 deletions(-) diff --git a/modmail/constraints.txt b/modmail/constraints.txt index 5db2a54f..0b853126 100644 --- a/modmail/constraints.txt +++ b/modmail/constraints.txt @@ -17,14 +17,13 @@ discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip humanfriendly==10.0 idna==3.2 multidict==5.1.0 -numpy==1.21.1 pycares==4.0.0 pycparser==2.20 pydantic==1.8.2 pyreadline3==3.3 python-dateutil==2.8.2 python-dotenv==0.19.0 -rapidfuzz==1.6.1 +rapidfuzz==1.8.3 six==1.16.0 typing-extensions==3.10.0.2 yarl==1.6.3 diff --git a/poetry.lock b/poetry.lock index fe4c124f..0d1131cd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -688,14 +688,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "numpy" -version = "1.21.1" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = false -python-versions = ">=3.7" - [[package]] name = "packaging" version = "21.0" @@ -1059,14 +1051,14 @@ pyyaml = "*" [[package]] name = "rapidfuzz" -version = "1.6.1" +version = "1.8.3" description = "rapid fuzzy string matching" category = "main" optional = false python-versions = ">=2.7" -[package.dependencies] -numpy = "*" +[package.extras] +full = ["numpy"] [[package]] name = "regex" @@ -1258,7 +1250,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "da86ea6533f92352cf48dccb1404707a2aea1675279934626dc5f47e24bbc56b" +content-hash = "550127246ec0ae33cfc4fda42a0c4ee57472a6ef01d476c51d3bcbe5a61d0782" [metadata.files] aiodns = [ @@ -1745,36 +1737,6 @@ nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] -numpy = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, -] packaging = [ {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, @@ -2002,59 +1964,60 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] rapidfuzz = [ - {file = "rapidfuzz-1.6.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ef307be084a6022f07fbcdb2614d3d92818750c4df97f38cf12551e029543ea4"}, - {file = "rapidfuzz-1.6.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1b6afbde4ac6cab1d5162bcf90d56b95b67632fbbeec77c923f0628381f12a4e"}, - {file = "rapidfuzz-1.6.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e6fc27ab404ee918dd92bd3200aae98e4539f3383e2fec39b6fea7ed70e9eb61"}, - {file = "rapidfuzz-1.6.1-cp27-cp27m-win32.whl", hash = "sha256:c266e7de2f3d29648a06ae15d650c029c2021e6cdee3bc67fdaabeff47385e49"}, - {file = "rapidfuzz-1.6.1-cp27-cp27m-win_amd64.whl", hash = "sha256:dc133cfed87dadf620d780a2f9334c6209b1a6eed030e10bd3f7c5d9d52bbce1"}, - {file = "rapidfuzz-1.6.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:8888ffc8f2504dcc5943bd2b016f20b69d6c94695fcb6626cd193703c0667c67"}, - {file = "rapidfuzz-1.6.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:23b6ae916278b3b4591f8749a264ef41fa11cb6abd736eadf9a625d45f800e72"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:02ccc031bb18dad5834e4e22795b6980471522ef6557e6ced1685c5cb43d411c"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ca2178f56017a117afe76dd8192c07e4f18df116516893f7218e3f8689680489"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:fe3eba011c105d4d69cbbed9e7d64edea9077c45f3c2fdfa06cc8747b55f2e56"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:317fafb2e725490721e76ee6b660dff1c10e959199dd525c447a1044ecfd2eaf"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:9ef4d63990c8ed1fe57f402302e612da4ffa712d5ff8f760a83f18ed72cc54dd"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-win32.whl", hash = "sha256:24b4fed0e31f975d7325ec26552eb71c2a0f17dfa7867fd8c9d59aadc749a960"}, - {file = "rapidfuzz-1.6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9ae20b5687121966b67658911f48fd938565b18b510633e6f56e3d04d1d15409"}, - {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75503a5c57c4353ddf99615557836b4dc7da78443099f269a4c7e504ddc6e9eb"}, - {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9e0594fd26b01ab81c25acd0d35e6f1ec91a0bd7bbb03568247db5501401f5f"}, - {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4c81df7e4f9a3ed9983ff0ec13b5e88d31596d996b919eeca8b1002e8367045"}, - {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:df8fcb829ac760b6743626143d33470b65a7a758558646c396fa7334fd9dd995"}, - {file = "rapidfuzz-1.6.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a9e7f82401a66b40b3b0588761495aadd53c9c24d36f3543f0d82e5cce126964"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:17c23bc209f02d7481a3ad0d9fe72674a3fb8c7b3b72477d4aae311d67924f7f"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a764b8a8eca7980fe21e95fe54b506273a6f62b5083234de3133877b69c8482"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7394d4624b181346654fd0c2f1a94bd24aff3b68dc42044c4c6954c17822ed31"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589baf8444719e8466fef7db9854fb190af871c1b97538d2e249a3c4f621d972"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca7989d42c9c14ad576e8a655d5f0ae5d10777332e1cc9384267a423f93b498a"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8dea0e89791c5ac55ff314a6ac2aa699b69bf0c802e7253628baa730d80f68b"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:65e331eefee945ba5c9c00aa0af0eb8db94515f6f46090e5991152ef77b92e53"}, - {file = "rapidfuzz-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4f10c8a1656eccddad84c9380715d6a5acfa1652c2a261fff166ef786904ba20"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:61900b8e4574277056b5de177c194e628ce2eb750fad94c8ba5995ae1a356fc1"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:375703ba1e5928cc6dae755e716bd13ce6dce8c51e0dae64d900adf5ccaf8d24"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7cdd8a82c4ff7871f4dad3b45940ebc12aee8fcd32182d5cbafbad607f34d72"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9acb9fa62b34b839c69372e60d52bca24f994ede1e8d37bea4b905e197009c7a"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c1ced861667713655ce2ec5300945d42b33e22b14d373ce1a37a9ac883007a50"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5a8001dc1d83b904fba421df6d715a0c5f2a15396c48ee1792b4e00081460d9"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5e1dad95996cc2dc382ccb2762e42061494805aac749d63140d62623ee254d64"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:48c0da783ce1b604f647db8d0060dc987a1e429abf66afcb932b70179525db0c"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-win32.whl", hash = "sha256:8076188fc13379d05f26c06337a4b1d45ea4ca7c872d77f47cdebb0d06d8e409"}, - {file = "rapidfuzz-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e0290e09c8dcac11e9631ab59351b67efd50f5eb2f23c0b2faf413cbe393dfa"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d9292132cc011680954bf5ea8535660e90c16fb3af2584e56b7327dacde53e23"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:32262729c0280567b52df5f755a05cbd0689b2f9103e67d67b1538297988f45b"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:58a97922a65c3c336a5c8613b70956cf8bb8ecc39b8177581249df3f92144d89"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3d7602067d38f0da988f84aa0694ff277d0ab228b4ea849f10d5a6c797f1ebe"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba36dcd6add2e5e9e41e1b016f2fe11a32beabdfd5987f2df4cc3f9322fe2a22"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c734fd58bb80b0930f5bfc2aba46538088bc06e10e9c66a6a32ac609f14fed27"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f8b42e07efa816413eb58a53feed300898b302cc2180c824a18518c7fdae6124"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24e378ce0629852f55a6c7cefbe3de5d477023d195da44c3bdd224c3cd9400a6"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-win32.whl", hash = "sha256:2af0bf8f4bef68fc56966c9f9c01b6d5412ba063ea670c14bf769081f86bf022"}, - {file = "rapidfuzz-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:ce68d063f97e3ac69c6bf77b366d088db3ba0cad60659dfea1add2ed33ed41ba"}, - {file = "rapidfuzz-1.6.1-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:e6c93b726666db391275f554a7fc9b20bb7f5fbdd028d94cef3d3f19cd1d33ce"}, - {file = "rapidfuzz-1.6.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:edefe618fa6b4bf1ba684c88a2253b7a475cc85edd2020616287e1619cbe3284"}, - {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e0b57bf21e69636da99092406d5effcd4b8ba804c5e1f3b6a075a2f1e2c077d3"}, - {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3ec0ece8ba8c9a1785a7ec3d2ba0d402cccf8199e99d9c33e98cbeab08be7f18"}, - {file = "rapidfuzz-1.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ddd972f15cc1bd5c95854f0f7ebad55e46f56bdc6545cdbf6fcf71390993785c"}, - {file = "rapidfuzz-1.6.1.tar.gz", hash = "sha256:5cc007251bc6b5d5d8e7e66e6f8e1b36033d3e240845a2e5721a025ef850d690"}, + {file = "rapidfuzz-1.8.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0aa566e46bf1bf8e98e7a009fb0119c6601aece029af2e9566cfdf7662526c20"}, + {file = "rapidfuzz-1.8.3-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:6854b2399fa39dbf480a55fe359e1012590b29e683035645dd8d56c8d367ca9b"}, + {file = "rapidfuzz-1.8.3-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:7f79d77e2d20d2042c7fa00c07e979e28d684d875e5a523a51c06e8b1a2f579c"}, + {file = "rapidfuzz-1.8.3-cp27-cp27m-win32.whl", hash = "sha256:b896fc68897611354d78285262e475e387f539cef85d11983c0c06c7aa0ac20c"}, + {file = "rapidfuzz-1.8.3-cp27-cp27m-win_amd64.whl", hash = "sha256:39ec5cec3f9054a1176906972b4d900b5ed314d25dab709156d1e9b7f957de11"}, + {file = "rapidfuzz-1.8.3-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:c69e0bbbfb6e4add79fe6919dea7e6936401c7708ed76280223a954dfb8a3277"}, + {file = "rapidfuzz-1.8.3-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e0c30a631fac14469d18d19190ef8b53d97a95aceecb0ffa103d13a76d7bbac"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d15cb1176d77962ef9af567aa3d33459930f290a0bf06355ac7b6d3bfb001aa"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e77f537bc28af69de0066e09191be746600f3b51c1d1c820b3e82c9e1b0152bd"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8fa4ef5f82762274558a7afe2037b016aee2c81b3d5d2c749a25771875013091"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a83917d28f23d87f6ad1c6c201ff8385bd5dfd37d5da9c4cb5967e9e3a431da"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8a477f5d75aef642e14f0051fe5e7315730dff4df4f6c02e2ddb046d3ba94791"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e3c7d39c97414952ce687db2ef7966612511d23561222c04cb226e9871d0cdb"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-win32.whl", hash = "sha256:0ed81b389274736675a7815b8f65b0492be65548cf03b5cc81687c66188ff9dd"}, + {file = "rapidfuzz-1.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:ced1bf333f228c4fd31db8d55185366b090755c5c634c51afadf3c4a079fe1fc"}, + {file = "rapidfuzz-1.8.3-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:50c25c058616b9c3a3b5814db1560e9ecbdeec3d987e51b641dc3bc261c55bbc"}, + {file = "rapidfuzz-1.8.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:87056d8106cb3f118b5fcc4a7c8ab77e40dcf7e5b5904a83a344d8a916feefd4"}, + {file = "rapidfuzz-1.8.3-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:0a255c6bd346dbb3c0fc96daf5ed10473a2568365ab76de7d779732d3a304af2"}, + {file = "rapidfuzz-1.8.3-cp35-cp35m-win32.whl", hash = "sha256:60168de30ea1280884a2ebf83ff028966c670b0c56840095939b987e3a372aaa"}, + {file = "rapidfuzz-1.8.3-cp35-cp35m-win_amd64.whl", hash = "sha256:d7f9cd0836689a6a928c79005108475c9e95cf9ea3ec850b54017f49a3cc961d"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:31082c7cf4ba405c054d149cb04e32f68cfb13c736d09354dab81aa60d553194"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ee1149e2038e6ea6065a439b14e2f7a6939d3bc9fb19fa9d4e32161f678ca555"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:205cf9193aa1cc6c368e1a744a35e205f152ca2f63f516802ed9322764ece04a"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2985f9694f2732a968f8af8cb7e4ab0325a7d80d9e8fd29f3b2b4621da6ccef8"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-win32.whl", hash = "sha256:4bf7a88deade25cb91eff36f79e40b174b6dc1fdb467e50a3aca65ab8a951431"}, + {file = "rapidfuzz-1.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:8a443341fbc171df6eed302fcf1adf4975045565988edeaee4302636c0a7e6c1"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70efbce4e2c80f091ae5f7040c6afe4f6e04836a2b0d27ab554fd6fb56b46ed5"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5954565dbc0a376971c4b8a65f698d8f12226b9e275ce1bef7874c2fc5a3a433"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a0b820f3279253d9deae6bed82c699d43903a2676208ac4d849f54a00919c473"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3435b497bc78e548977a671b91f9655c20045dbbeee6ca4ce5219cac1411682"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-win32.whl", hash = "sha256:2fe0e9272e35a1f98fdbeef16f2e969e29a9226f187f540febfc064d82878668"}, + {file = "rapidfuzz-1.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:1df3455ffed5cdcc28b6e2b53dfc3ec068b298dceb3782e2e654f50ab16b2e34"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d856c8ab95df935636108868e0579a1d78f68222d79fd35853e6d8ba54ced617"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3fcab917846c1c28fec36e8cd22c1a072cfb5ce5a297c6bda2017c01e309a892"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6571adf845b4e464a3b748de0b1cdd4acc66c01a0e9fd51e5d43cbf0d4a85524"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:05d21cb420848838d6c2c2816181325ab1ae3109bfbe45df863635ef8f159714"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dfc18bb38085e1b4a4dd2fe99b17770dcbf286408510477ff542fbdd0ffbe017"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:496ac913d5917838d92965873b3d9540be44619a9693123f6fb0d1074f1c63b5"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-win32.whl", hash = "sha256:e0013c270b8c097a90b92b6a4664e410cfb2195b2573431b651634a28c13ee6f"}, + {file = "rapidfuzz-1.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:124cfe1a3cfd0fa5069873aaa8933df50d9c0a1a0db126739aa3a129e09024da"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a08453c1f5a6b25e4cc61b99e0601adbb1daed3a360b1270abf24625d83d52f0"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d5a4befaa266fc75c5d9bd414029dc89a19ad0ad475ac527f5505119647a914"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8da386372e1bf7579c7a32c28a263bc417b14fbc66c6c1df76baf30d6efa98ae"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:29a4ea3a779dd1c8fafdff241f3737c079d7905a1c33beab306e2179bb9bd6ff"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0f9310f3d651aa50d4cb023de727bf3f8a96a76082ca3478a01d7a63109e3fd3"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:915ed93f12f551b0670d8f0d5949c660e533046e9efdfb49016de6c2ddca793c"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-win32.whl", hash = "sha256:55c1772561900bf08fc15efa359f971723785d8b42419c4ea18eacd001bad5fc"}, + {file = "rapidfuzz-1.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:911fb926f0237b67b6f566c4e1b029dd38888675228ad9e1613b2f8deb94d8a3"}, + {file = "rapidfuzz-1.8.3-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:4fc3f4430ca680bc576a789914d029fa1f332cd5836ca954ef8e12b11fd48801"}, + {file = "rapidfuzz-1.8.3-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:5d45e00b29594e4a785f413869a43815bc29d977c940410255ea51adca61644d"}, + {file = "rapidfuzz-1.8.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d11a69e5a33cbcb665d03f63f77d46bd2d4f4e8fc10f48e734d2880bba0b3ab7"}, + {file = "rapidfuzz-1.8.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d9205db2beda1b83fbfaf968039fbbd05f1c278c6e13782c699ef1ad4d2c43af"}, + {file = "rapidfuzz-1.8.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cee3f7daab7813314d61c6d81ba32fdd3c75f7cf6910cc630c76905195c4a2a4"}, + {file = "rapidfuzz-1.8.3.tar.gz", hash = "sha256:e85fa8110dc1271b7f193f225e5c6c63be81c3cf1a48648d01ed5d55955fbc4c"}, ] regex = [ {file = "regex-2021.8.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2"}, diff --git a/pyproject.toml b/pyproject.toml index eedab9cb..bc2aa19a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ colorama = "^0.4.3" coloredlogs = "^15.0" "discord.py" = { url = "https://github.com/Rapptz/discord.py/archive/master.zip" } pydantic = { version = "^1.8.2", extras = ["dotenv"] } -rapidfuzz = "^1.6.1" +rapidfuzz = "^1.8.0" [tool.poetry.extras] diff --git a/requirements.txt b/requirements.txt index 7039c93e..86dfabd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,14 +17,13 @@ discord.py @ https://github.com/Rapptz/discord.py/archive/master.zip ; python_fu humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" multidict==5.1.0 ; python_version >= "3.6" -numpy==1.21.1 ; python_version >= "3.7" pycares==4.0.0 pycparser==2.20 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" pydantic==1.8.2 ; python_full_version >= "3.6.1" pyreadline3==3.3 ; sys_platform == "win32" python-dateutil==2.8.2 ; python_version != "3.0" python-dotenv==0.19.0 ; python_version >= "3.5" -rapidfuzz==1.6.1 ; python_version >= "2.7" +rapidfuzz==1.8.3 ; python_version >= "2.7" six==1.16.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" typing-extensions==3.10.0.2 yarl==1.6.3 ; python_version >= "3.6" From bb9d3f8c8eb090640d68f3a78f3f062c3c3de188 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Fri, 19 Nov 2021 08:43:09 -0500 Subject: [PATCH 095/100] scripts: add diff output to export_requirements --- scripts/_utils.py | 85 ++++++++++++++++++++++++++++++++++ scripts/export_requirements.py | 47 ++++++++++--------- 2 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 scripts/_utils.py diff --git a/scripts/_utils.py b/scripts/_utils.py new file mode 100644 index 00000000..5491fc00 --- /dev/null +++ b/scripts/_utils.py @@ -0,0 +1,85 @@ +"""Utility functions and variables which are useful for all scripts.""" +import difflib +import importlib.util +import os +import pathlib +import typing + + +MODMAIL_DIR = pathlib.Path(importlib.util.find_spec("modmail").origin).parent +PROJECT_DIR = MODMAIL_DIR.parent +try: + import pygments +except ModuleNotFoundError: + pygments = None +else: + from pygments.formatters import Terminal256Formatter + from pygments.lexers.diff import DiffLexer + + +class CheckFileEdit: + """Check if a file is edited within the body of this class.""" + + def __init__(self, *files: os.PathLike): + self.files: typing.List[pathlib.Path] = [] + for f in files: + self.files.append(pathlib.Path(f)) + self.return_value: typing.Optional[int] = None + self.edited_files: typing.Dict[pathlib.Path] = dict() + + def __enter__(self): + self.file_contents = {} + for file in self.files: + try: + with open(file, "r") as f: + self.file_contents[file] = f.readlines() + except FileNotFoundError: + self.file_contents[file] = None + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 + for file in self.files: + with open(file, "r") as f: + original_contents = self.file_contents[file] + new_contents = f.readlines() + if original_contents != new_contents: + # construct a diff + diff = difflib.unified_diff( + original_contents, new_contents, fromfile="before", tofile="after" + ) + try: + diff = "".join(diff) + except TypeError: + diff = None + else: + if pygments is not None: + diff = pygments.highlight(diff, DiffLexer(), Terminal256Formatter()) + self.edited_files[file] = diff + + def write(self, path: str, contents: typing.Union[str, bytes], *, force: bool = False, **kwargs) -> bool: + """ + Write to the provided path with contents. Must be within the context manager. + + Returns False if contents are not edited, True if they are. + If force is True, will modify the files even if the contents match. + + Any extras kwargs are passed to open() + """ + path = pathlib.Path(path) + if path not in self.files: + raise AssertionError(f"{path} must have been passed to __init__") + + if not force: + try: + with open(path, "r") as f: + if contents == f.read(): + return False + except FileNotFoundError: + pass + if isinstance(contents, str): + contents = contents.encode() + + with open(path, "wb") as f: + f.write(contents) + + return True diff --git a/scripts/export_requirements.py b/scripts/export_requirements.py index f2055b8f..a4eb1357 100644 --- a/scripts/export_requirements.py +++ b/scripts/export_requirements.py @@ -16,10 +16,12 @@ import tomli +from ._utils import PROJECT_DIR, CheckFileEdit -GENERATED_FILE = pathlib.Path("requirements.txt") -CONSTRAINTS_FILE = pathlib.Path("modmail/constraints.txt") -DOC_REQUIREMENTS = pathlib.Path("docs/.requirements.txt") + +GENERATED_FILE = PROJECT_DIR / "requirements.txt" +CONSTRAINTS_FILE = PROJECT_DIR / "modmail/constraints.txt" +DOC_REQUIREMENTS = PROJECT_DIR / "docs/.requirements.txt" VERSION_RESTRICTER_REGEX = re.compile(r"(?P[<>=!]{1,2})(?P\d+\.\d+?)(?P\.\d+?|\.\*)?") PLATFORM_MARKERS_REGEX = re.compile(r'sys_platform\s?==\s?"(?P\w+)"') @@ -128,6 +130,7 @@ def _export_doc_requirements(toml: dict, file: pathlib.Path, *packages) -> int: file = pathlib.Path(file) if not file.exists(): # file does not exist + print(f"{file.relative_to(PROJECT_DIR)!s} must exist to export doc requirements") return 2 with open(file) as f: @@ -149,14 +152,18 @@ def _export_doc_requirements(toml: dict, file: pathlib.Path, *packages) -> int: except AttributeError as e: print(e) return 3 - if new_contents == contents: - # don't write anything, just return 0 - return 0 - with open(file, "w") as f: - f.write(new_contents) + with CheckFileEdit(file) as check_file: + + check_file.write(file, new_contents) - return 1 + for file, diff in check_file.edited_files.items(): + print( + f"Exported new documentation requirements to {file.relative_to(PROJECT_DIR)!s}.", + file=sys.stderr, + ) + print(diff or "No diff to show.") + print() def export( @@ -269,19 +276,17 @@ def export( else: exit_code = 0 - if req_path.exists(): - with open(req_path, "r") as f: - if req_txt == f.read(): - # nothing to edit - # if exit_code is ever removed from here, this should return zero - return exit_code + with CheckFileEdit(req_path) as check_file: + check_file.write(req_path, req_txt) - if _write_file(req_path, req_txt): - print(f"Updated {req_path} with new requirements.") - return 1 - else: - print(f"No changes were made to {req_path}") - return 0 + for file, diff in check_file.edited_files.items(): + print( + f"Exported new requirements to {file.relative_to(PROJECT_DIR)}.", + file=sys.stderr, + ) + print(diff or "No diff to show.") + print() + return bool(len(check_file.edited_files)) or exit_code def main(path: os.PathLike, include_markers: bool = True, **kwargs) -> int: From 20d722f02b8584d0daab441edb49498015503d0e Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 12:12:03 -0500 Subject: [PATCH 096/100] Revert "ci: run tests with verboseness, not quietness" This reverts commit 1c88421ee826491e2f45e5202ad78db1ca4a5c05. Not part of the scope of this pr. Also should not have been here but a rebase would change more history than I want to change. --- .github/workflows/lint_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml index a5655ec5..6db38e31 100644 --- a/.github/workflows/lint_test.yml +++ b/.github/workflows/lint_test.yml @@ -20,6 +20,7 @@ env: # Configure pip to cache dependencies and do a user install PIP_NO_CACHE_DIR: false PIP_USER: 1 + PYTHON_VERSION: 3.8 # Make sure package manager does not use virtualenv POETRY_VIRTUALENVS_CREATE: false @@ -161,7 +162,7 @@ jobs: # This is saved to ./.coverage to be used by codecov to link a # coverage report to github. - name: Run tests and generate coverage report - run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -v --showlocals + run: python -m pytest -n auto --dist loadfile --cov --disable-warnings -q # This step will publish the coverage reports to codecov.io and # print a "job" link in the output of the GitHub Action From a773b447b2ff21fc49b4dbdd208fb7b7fa3c74f7 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 12:25:34 -0500 Subject: [PATCH 097/100] chore: don't use __all__ in __init__.py --- modmail/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modmail/__init__.py b/modmail/__init__.py index 450d9a9a..907068f7 100644 --- a/modmail/__init__.py +++ b/modmail/__init__.py @@ -65,8 +65,4 @@ # Set asyncio logging back to the default of INFO even if asyncio's debug mode is enabled. logging.getLogger("asyncio").setLevel(logging.INFO) -# now that the logger is configured, we can import the bot for using as all and typing -from modmail.bot import ModmailBot # noqa: E402 - - -__all__ = [ModmailBot, ModmailLogger] +__all__ = () From 9987b2665d86b0f3d550bd3837fb412c6f661f39 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Sun, 21 Nov 2021 13:54:41 -0500 Subject: [PATCH 098/100] chore: make requested changes and address review - don't require future annotations in modmail/addons/converters.py - make all `__all__` tuples - set score cutoff to a constant in modmail/addons/models.py - compress if statements - use {} in place of dict() - fix typos in some docstrings - use correct typehints where they are incorrect - decode the stderr stream only once in pip install - remove unnecessary logging debug statements - restructure finding plugins to be easy to be easier to read - use set() where previously it was {}. This would cause the variable to be of the wrong type --- modmail/addons/converters.py | 16 +++---- modmail/addons/errors.py | 2 + modmail/addons/helpers.py | 4 +- modmail/addons/models.py | 13 +++--- modmail/addons/plugins.py | 55 +++++++++++-------------- modmail/bot.py | 2 +- modmail/extensions/extension_manager.py | 4 +- modmail/utils/extensions.py | 4 +- 8 files changed, 44 insertions(+), 56 deletions(-) diff --git a/modmail/addons/converters.py b/modmail/addons/converters.py index 78ab98ce..ae9a1c41 100644 --- a/modmail/addons/converters.py +++ b/modmail/addons/converters.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import logging import re from typing import TYPE_CHECKING, Tuple, Type @@ -10,15 +8,13 @@ if TYPE_CHECKING: - from discord.ext.commands import Context - from modmail.log import ModmailLogger -LOCAL_REGEX: re.Pattern = re.compile(r"^\@local (?P[^@\s]+)$") -ZIP_REGEX: re.Pattern = re.compile( +LOCAL_REGEX = re.compile(r"^\@local (?P[^@\s]+)$") +ZIP_REGEX = re.compile( r"^(?:https?:\/\/)?(?P(?P.*\..+?)\/(?P.*\.zip)) (?P[^@\s]+)$" ) -REPO_REGEX: re.Pattern = re.compile( +REPO_REGEX = re.compile( r"^(?:(?:https?:\/\/)?(?Pgithub|gitlab)(?:\.com\/| )?)?" # github allows usernames from 1 to 39 characters, and projects of 1 to 100 characters # gitlab allows 255 characters in the username, and 255 max in a project path @@ -27,7 +23,7 @@ r"(?P[^@]+[^\s@])(?: \@(?P[\w\.\-\S]*))?" ) -logger: ModmailLogger = logging.getLogger(__name__) +logger: "ModmailLogger" = logging.getLogger(__name__) AddonClass = Type[Addon] @@ -35,7 +31,7 @@ class AddonConverter(commands.Converter): """A converter that takes an addon source, and gets a Addon object from it.""" - async def convert(self, ctx: Context, argument: str) -> None: + async def convert(self, ctx: commands.Context, argument: str) -> None: """Convert an argument into an Addon.""" raise NotImplementedError("Inheriting classes must overwrite this method.") @@ -43,7 +39,7 @@ async def convert(self, ctx: Context, argument: str) -> None: class SourceAndPluginConverter(AddonConverter): """A plugin converter that takes a source, addon name, and returns a Plugin.""" - async def convert(self, _: Context, argument: str) -> Tuple[Plugin, AddonSource]: + async def convert(self, _: commands.Context, argument: str) -> Tuple[Plugin, AddonSource]: """Convert a provided plugin and source to a Plugin.""" match = LOCAL_REGEX.match(argument) if match is not None: diff --git a/modmail/addons/errors.py b/modmail/addons/errors.py index cddf4115..d4dc319a 100644 --- a/modmail/addons/errors.py +++ b/modmail/addons/errors.py @@ -24,3 +24,5 @@ class PluginNotFoundError(PluginError): class NoPluginTomlFoundError(PluginError): """Raised when a plugin.toml file is expected to exist but does not exist.""" + + pass diff --git a/modmail/addons/helpers.py b/modmail/addons/helpers.py index c9da6507..e8b2f1a3 100644 --- a/modmail/addons/helpers.py +++ b/modmail/addons/helpers.py @@ -6,14 +6,14 @@ from modmail.utils.cogs import ModmailCog as _ModmailCog -__all__ = [ +__all__ = ( "PluginCog", BOT_MODE, BotModeEnum, ExtMetadata, ModmailBot, ModmailLogger, -] +) class PluginCog(_ModmailCog): diff --git a/modmail/addons/models.py b/modmail/addons/models.py index 2f9004d8..be63dc2d 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -21,6 +21,8 @@ PLUGINS = None +SCORE_CUTOFF = 69 + class SourceTypeEnum(Enum): """Which source an addon is from.""" @@ -168,10 +170,7 @@ def __init__( ): self.folder_name = folder self.description = description - if name is None: - self.name = self.folder_name - else: - self.name = name + self.name = self.folder_name if name is None else name self.folder_path = folder_path self.min_bot_version = min_bot_version self.local = local @@ -179,7 +178,7 @@ def __init__( self.dependencies = dependencies or [] - self.modules = dict() + self.modules = {} # store any extra kwargs here # this is to ensure backwards compatiablilty with plugins that support older versions, @@ -225,7 +224,7 @@ async def convert(cls, ctx: commands.Context, argument: str) -> Plugin: # Determine close plugins # Using a dict to prevent duplicates - arg_mapping: Dict[str, Plugin] = dict() + arg_mapping: Dict[str, Plugin] = {} for plug in loaded_plugs: for name in plug.name, plug.folder_name: arg_mapping[name] = plug @@ -234,7 +233,7 @@ async def convert(cls, ctx: commands.Context, argument: str) -> Plugin: argument, arg_mapping.keys(), scorer=fuzz.ratio, - score_cutoff=69, + score_cutoff=SCORE_CUTOFF, ) logger.debug(f"{result = }") diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index ba6936fb..51b4e2f4 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -2,7 +2,7 @@ # https://github.com/python-discord/bot/blob/a8869b4d60512b173871c886321b261cbc4acca9/bot/utils/extensions.py # MIT License 2021 Python Discord """ -Helper utililites for managing plugins. +Helper utilities for managing plugins. TODO: Expand file to download plugins from github and gitlab from a list that is passed. """ @@ -31,7 +31,7 @@ from modmail.utils.extensions import ModuleName, unqualify -__all__ = [ +__all__ = ( "VALID_ZIP_PLUGIN_DIRECTORIES", "BASE_PLUGIN_PATH", "PLUGINS", @@ -42,7 +42,7 @@ "find_partial_plugins_from_dir", "find_plugins", "walk_plugin_files", -] +) logger: ModmailLogger = logging.getLogger(__name__) @@ -66,11 +66,11 @@ ).encode() -async def install_dependencies(plugin: Plugin) -> str: +async def install_dependencies(plugin: Plugin) -> Optional[str]: """Installs provided dependencies from a plugin.""" # check if there are any plugins to install if not len(plugin.dependencies): - return + return None if PYTHON_INTERPRETER is None: raise FileNotFoundError("Could not locate python interpreter.") @@ -95,13 +95,10 @@ async def install_dependencies(plugin: Plugin) -> str: stderr=subprocess.PIPE, ) stdout, stderr = await proc.communicate() - logger.debug(f"{stdout.decode() = }") - - if stderr: - stderr = stderr.replace(PIP_NO_ROOT_WARNING, b"").strip() - if len(stderr.decode()) > 0: - logger.error(f"Received stderr: '{stderr.decode()}'") - raise Exception("Something went wrong when installing.") + stderr = stderr.replace(PIP_NO_ROOT_WARNING, b"").strip() + if len(decoded_stderr := stderr.decode()) > 0: + logger.error(f"Received stderr: '{decoded_stderr}'") + raise Exception("Something went wrong when installing.") return stdout.decode() @@ -195,14 +192,13 @@ def find_partial_plugins_from_dir( # default is plugins. plugin_directory = None direct_children = [p for p in addon_repo_path.iterdir()] - logger.debug(f"{direct_children = }") + for path_ in direct_children: if path_.name.rsplit("/", 1)[-1] in VALID_ZIP_PLUGIN_DIRECTORIES: plugin_directory = path_ break if plugin_directory is None: - logger.debug(f"{direct_children = }") raise NoPluginDirectoryError(f"No {' or '.join(VALID_ZIP_PLUGIN_DIRECTORIES)} directory exists.") plugin_directory = addon_repo_path / plugin_directory @@ -222,9 +218,7 @@ def find_partial_plugins_from_dir( else: raise NoPluginTomlFoundError(toml_path, "does not exist") - logger.debug(f"{all_plugins =}") for path in plugin_directory.iterdir(): - logger.debug(f"plugin_directory: {path}") if path.is_dir(): # use an existing toml plugin object if path.name in all_plugins: @@ -254,24 +248,22 @@ def find_plugins( toml_plugins: List[Plugin] = [] toml_path = LOCAL_PLUGIN_TOML - if toml_path.exists(): - # parse the toml - with open(toml_path) as toml_file: - toml_plugins = parse_plugin_toml_from_string(toml_file.read(), local=True) - else: + + if not toml_path.exists(): raise NoPluginTomlFoundError(toml_path, "does not exist") - logger.debug(f"{toml_plugins =}") + # parse the toml + with open(toml_path) as toml_file: + toml_plugins = parse_plugin_toml_from_string(toml_file.read(), local=True) + toml_plugin_names = [p.folder_name for p in toml_plugins] for path in detection_path.iterdir(): - logger.debug(f"detection_path / path: {path}") - if path.is_dir(): + if path.is_dir() and path.name in toml_plugin_names: # use an existing toml plugin object - if path.name in toml_plugin_names: - for p in toml_plugins: - if p.folder_name == path.name: - p.folder_path = path - all_plugins.add(p) + for p in toml_plugins: + if p.folder_name == path.name: + p.folder_path = path + all_plugins.add(p) logger.debug(f"Local plugins detected: {[p.name for p in all_plugins]}") @@ -294,8 +286,6 @@ def find_plugins( plugin_.modules.update(walk_plugin_files(dirpath)) yield plugin_ - logger.debug(f"{all_plugins = }") - def walk_plugin_files( detection_path: pathlib.Path = None, @@ -355,7 +345,8 @@ def walk_plugin_files( else: logger.error( f"Plugin extension {imported.__name__!r} contains an invalid EXT_METADATA variable. " - "Loading with metadata defaults. Please report this bug to the developers." + "Loading with metadata defaults. " + "Please report this error to the respective plugin developers." ) yield imported.__name__, ExtMetadata() continue diff --git a/modmail/bot.py b/modmail/bot.py index 5eecbdac..d4d8b57b 100644 --- a/modmail/bot.py +++ b/modmail/bot.py @@ -52,7 +52,7 @@ def __init__(self, **kwargs): self._resolver = None # keys: plugins, list values: all plugin files - self.installed_plugins: Set[Plugin] = {} + self.installed_plugins: Optional[Set[Plugin]] = None status = discord.Status.online activity = Activity(type=discord.ActivityType.listening, name="users dming me!") diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index b4d6cd8f..db8e59a2 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from enum import Enum -from typing import Mapping, Optional, Tuple +from typing import Mapping, Tuple, Union from discord import Colour, Embed from discord.ext import commands @@ -298,7 +298,7 @@ def manage( *, is_plugin: bool = False, suppress_already_error: bool = False, - ) -> Tuple[str, Optional[str]]: + ) -> Tuple[str, Union[str, bool, None]]: """Apply an action to an extension and return the status message and any error message.""" verb = action.name.lower() error_msg = None diff --git a/modmail/utils/extensions.py b/modmail/utils/extensions.py index e8bf2946..f22bc038 100644 --- a/modmail/utils/extensions.py +++ b/modmail/utils/extensions.py @@ -19,8 +19,8 @@ ModuleName = NewType("ModuleName", str) ModuleDict = Dict[ModuleName, ExtMetadata] -EXTENSIONS: ModuleDict = dict() -NO_UNLOAD: List[ModuleName] = list() +EXTENSIONS: ModuleDict = {} +NO_UNLOAD: List[ModuleName] = [] def unqualify(name: str) -> str: From 1d32910d75a5a276279aa9056954d1127c5d2a06 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Thu, 24 Feb 2022 06:18:56 -0500 Subject: [PATCH 099/100] review: address --- modmail/addons/converters.py | 36 ++++++++++++------------- modmail/addons/models.py | 2 +- modmail/addons/plugins.py | 1 + modmail/config.py | 2 +- modmail/extensions/extension_manager.py | 2 +- modmail/extensions/plugin_manager.py | 23 +++++++--------- scripts/_utils.py | 2 +- 7 files changed, 32 insertions(+), 36 deletions(-) diff --git a/modmail/addons/converters.py b/modmail/addons/converters.py index ae9a1c41..6e9edd2e 100644 --- a/modmail/addons/converters.py +++ b/modmail/addons/converters.py @@ -1,10 +1,10 @@ import logging import re -from typing import TYPE_CHECKING, Tuple, Type +from typing import TYPE_CHECKING, Tuple from discord.ext import commands -from modmail.addons.models import Addon, AddonSource, Plugin, SourceTypeEnum +from modmail.addons.models import AddonSource, Plugin, SourceTypeEnum if TYPE_CHECKING: @@ -25,8 +25,6 @@ logger: "ModmailLogger" = logging.getLogger(__name__) -AddonClass = Type[Addon] - class AddonConverter(commands.Converter): """A converter that takes an addon source, and gets a Addon object from it.""" @@ -41,23 +39,23 @@ class SourceAndPluginConverter(AddonConverter): async def convert(self, _: commands.Context, argument: str) -> Tuple[Plugin, AddonSource]: """Convert a provided plugin and source to a Plugin.""" - match = LOCAL_REGEX.match(argument) - if match is not None: + if match := LOCAL_REGEX.match(argument): logger.debug("Matched as a local file, creating a Plugin without a source url.") + addon = match.group("addon") source = AddonSource(None, SourceTypeEnum.LOCAL) - return Plugin(match.group("addon")), source - match = ZIP_REGEX.fullmatch(argument) - if match is not None: + elif match := ZIP_REGEX.fullmatch(argument): logger.debug("Matched as a zip, creating a Plugin from zip.") + addon = match.group("addon") source = AddonSource.from_zip(match.group("url")) - return Plugin(match.group("addon")), source - - match = REPO_REGEX.fullmatch(argument) - if match is None: + elif match := REPO_REGEX.fullmatch(argument): + addon = match.group("addon") + source = AddonSource.from_repo( + match.group("user"), + match.group("repo"), + match.group("reflike"), + match.group("githost") or "github", + ) + else: raise commands.BadArgument(f"{argument} is not a valid source and plugin.") - return Plugin(match.group("addon")), AddonSource.from_repo( - match.group("user"), - match.group("repo"), - match.group("reflike"), - match.group("githost") or "github", - ) + + return Plugin(addon), source diff --git a/modmail/addons/models.py b/modmail/addons/models.py index be63dc2d..809d1eec 100644 --- a/modmail/addons/models.py +++ b/modmail/addons/models.py @@ -213,7 +213,7 @@ async def convert(cls, ctx: commands.Context, argument: str) -> Plugin: # its possible to have a plugin with the same name as a folder of a plugin # folder names are the priority - secondary_names = dict() + secondary_names = {} for plug in loaded_plugs: if argument == plug.name: return plug diff --git a/modmail/addons/plugins.py b/modmail/addons/plugins.py index 51b4e2f4..84289598 100644 --- a/modmail/addons/plugins.py +++ b/modmail/addons/plugins.py @@ -41,6 +41,7 @@ "update_local_toml_enable_or_disable", "find_partial_plugins_from_dir", "find_plugins", + "install_dependencies", "walk_plugin_files", ) diff --git a/modmail/config.py b/modmail/config.py index 6c7a3d6c..da19dda1 100644 --- a/modmail/config.py +++ b/modmail/config.py @@ -72,7 +72,7 @@ def toml_user_config_source(settings: PydanticBaseSettings) -> Dict[str, Any]: with open(USER_CONFIG_PATH) as f: return atoml.loads(f.read()).value else: - return dict() + return {} class BaseSettings(PydanticBaseSettings): diff --git a/modmail/extensions/extension_manager.py b/modmail/extensions/extension_manager.py index db8e59a2..d57cc5d6 100644 --- a/modmail/extensions/extension_manager.py +++ b/modmail/extensions/extension_manager.py @@ -298,7 +298,7 @@ def manage( *, is_plugin: bool = False, suppress_already_error: bool = False, - ) -> Tuple[str, Union[str, bool, None]]: + ) -> Tuple[str, Union[str, bool]]: """Apply an action to an extension and return the status message and any error message.""" verb = action.name.lower() error_msg = None diff --git a/modmail/extensions/plugin_manager.py b/modmail/extensions/plugin_manager.py index 6c53c839..30bdbe0a 100644 --- a/modmail/extensions/plugin_manager.py +++ b/modmail/extensions/plugin_manager.py @@ -153,9 +153,7 @@ def _resync_extensions(self) -> None: # remove all fully unloaded plugins from the list for plug in PLUGINS.copy(): - safe_to_remove = [] - for mod in plug.modules: - safe_to_remove.append(mod not in self.bot.extensions) + safe_to_remove = [mod not in self.bot.extensions for mod in plug.modules] if all(safe_to_remove): PLUGINS.remove(plug) @@ -198,9 +196,9 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu logger.debug(f"Received command to download plugin {plugin.name} from https://{source.zip_url}") try: directory = await addon_utils.download_and_unpack_source(source, self.bot.http_session) - except errors.HTTPError: + except errors.HTTPError as e: await responses.send_negatory_response( - ctx, f"Downloading {source.zip_url} did not give a 200 response code." + ctx, f"Downloading {source.zip_url} expected 200, received {e.response.status}." ) return @@ -240,7 +238,11 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu if plugin.dependencies and len(plugin.dependencies): # install dependencies since they exist message = await ctx.send( - embed=Embed("Installing dependencies.", title="Pending install", colour=Colour.yellow()) + embed=Embed( + description="Installing dependencies.", + title="Pending install", + colour=Colour.yellow(), + ) ) try: await install_dependencies(plugin) @@ -262,12 +264,7 @@ async def install_plugins(self, ctx: Context, *, source_and_plugin: SourceAndPlu # check if the manage was successful failed = [] for mod, metadata in plugin.modules.items(): - if mod in self.bot.extensions: - fail = False - elif metadata.load_if_mode & BOT_MODE: - fail = False - else: - fail = True + fail = not (mod in self.bot.extensions or metadata.load_if_mode & BOT_MODE) failed.append(fail) @@ -363,7 +360,7 @@ def group_plugin_statuses(self) -> Mapping[str, str]: continue plug_status.append(status) - if len(plug_status) == 0: + if not plug_status: status = StatusEmojis.unknown elif all(plug_status): status = StatusEmojis.fully_loaded diff --git a/scripts/_utils.py b/scripts/_utils.py index 5491fc00..c305b405 100644 --- a/scripts/_utils.py +++ b/scripts/_utils.py @@ -25,7 +25,7 @@ def __init__(self, *files: os.PathLike): for f in files: self.files.append(pathlib.Path(f)) self.return_value: typing.Optional[int] = None - self.edited_files: typing.Dict[pathlib.Path] = dict() + self.edited_files: typing.Dict[pathlib.Path] = {} def __enter__(self): self.file_contents = {} From 6214cdd7a9f53c3ae3c1a49627881d9252a804c5 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Wed, 20 Apr 2022 22:48:32 -0400 Subject: [PATCH 100/100] fix: readd rapidfuzz --- modmail/constraints.txt | 2 + poetry.lock | 154 +++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + requirements.txt | 2 + 4 files changed, 158 insertions(+), 1 deletion(-) diff --git a/modmail/constraints.txt b/modmail/constraints.txt index 481316e5..1dae72b2 100644 --- a/modmail/constraints.txt +++ b/modmail/constraints.txt @@ -17,6 +17,7 @@ desert==2020.11.18 discord.py @ https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip humanfriendly==10.0 idna==3.2 +jarowinkler==1.0.2 marshmallow-enum==1.5.1 marshmallow==3.13.0 multidict==5.2.0 @@ -27,6 +28,7 @@ pyreadline3==3.3 python-dateutil==2.8.2 python-dotenv==0.19.0 pyyaml==5.4.1 +rapidfuzz==2.0.10 six==1.16.0 typing-extensions==3.10.0.2 typing-inspect==0.7.1 diff --git a/poetry.lock b/poetry.lock index d938cf8c..962dd82e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -518,6 +518,14 @@ requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] plugins = ["setuptools"] +[[package]] +name = "jarowinkler" +version = "1.0.2" +description = "library for fast approximate string matching using Jaro and Jaro-Winkler similarity" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "jinja2" version = "3.0.1" @@ -1019,6 +1027,20 @@ python-versions = ">=3.6" [package.dependencies] pyyaml = "*" +[[package]] +name = "rapidfuzz" +version = "2.0.10" +description = "rapid fuzzy string matching" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +jarowinkler = ">=1.0.2,<1.1.0" + +[package.extras] +full = ["numpy"] + [[package]] name = "requests" version = "2.26.0" @@ -1197,7 +1219,7 @@ yaml = ["PyYAML"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "f683cc260ce66461b3c20dbb7acdf91856600fa1340cf2648687fb554908ab62" +content-hash = "6b7f6a7250d80315ec9bbdc547ea255336c6d6d91dc19369e29d89879e0b527f" [metadata.files] aiodns = [ @@ -1581,6 +1603,88 @@ isort = [ {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, ] +jarowinkler = [ + {file = "jarowinkler-1.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:71772fcd787e0286b779de0f1bef1e0a25deb4578328c0fc633bc345f13ffd20"}, + {file = "jarowinkler-1.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:912ee0a465822a8d659413cebc1ab9937ac5850c9cd1e80be478ba209e7c8095"}, + {file = "jarowinkler-1.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0320f7187dced1ad413bf2c3631ec47567e65dfdea92c523aafb2c085ae15035"}, + {file = "jarowinkler-1.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58bc6a8f01b0dfdf3721f9a4954060addeccf8bbe5e72a71cf23a88ce0d30440"}, + {file = "jarowinkler-1.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:679ec7a42f70baa61f3a214d1b59cec90fc036021c759722075efcc8697e7b1f"}, + {file = "jarowinkler-1.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dde57d47962d6a4436d8a3b477bcc8233c6da28e675027eb3a490b0d6dc325be"}, + {file = "jarowinkler-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:657f50204970fac8f120c293e52a3451b742c9b26125010405ec7365cb6e2a49"}, + {file = "jarowinkler-1.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04f18a7398766b36ffbe4bcd26d34fcd6ed01f4f2f7eea13e316e6cca0e10c98"}, + {file = "jarowinkler-1.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33a24b380e2c076eabf2d3e12eee56b6bf10b1f326444e18c36a495387dbf0de"}, + {file = "jarowinkler-1.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e1d7d6e6c98fb785026584373240cc4076ad21033f508973faae05e846206e8c"}, + {file = "jarowinkler-1.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e50c750a45c800d91134200d8cbf746258ed357a663e97cc0348ee42a948386a"}, + {file = "jarowinkler-1.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:5b380afce6cdc25a4dafd86874f07a393800577c05335c6ad67ccda41db95c60"}, + {file = "jarowinkler-1.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e73712747ac5d2218af3ed3c1600377f18a0a45af95f22c39576165aea2908b4"}, + {file = "jarowinkler-1.0.2-cp310-cp310-win32.whl", hash = "sha256:9511f4e1f00c822e08dbffeb69e15c75eb294a5f24729815a97807ecf03d22eb"}, + {file = "jarowinkler-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a5c44f92e9ac6088286292ecb69e970adc2b98e139b8923bce9bbb9d484e6a0f"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:02b0bf34ffc2995b695d9b10d2f18c1c447fbbdb7c913a84a0a48c186ccca3b8"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7a8e45176298a1210c06f8b2328030cc3c93a45dab068ac1fbc9cf075cd95b"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da27a9c206249a50701bfa5cfbbb3a04236e1145b2b0967e825438acb14269bf"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43ea0155379df92021af0f4a32253be3953dfa0f050ec3515f314b8f48a96674"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f33b6b1687db1be1abba60850628ee71547501592fcf3504e021274bc5ccb7a"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff304de32ee6acd5387103a0ad584060d8d419aa19cbbeca95204de9c4f01171"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:662dd6f59cca536640be0cda32c901989504d95316b192e6aa41d098fa08c795"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:01f85abb75fa43e98db34853d35570d98495ee2fcbbf45a93838e0289c162f19"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5b9332dcc8130af4101c9752a03e977c54b8c12982a2a3ca4c2e4cc542accc00"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:af765b037404a536c372e33ddd4c430aea28f1d82a8ef51a2955442b8b690577"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:aea2c7d66b57c56d00f9c45ae7862d86e3ae84368ecea17f3552c0052a7f3bcf"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-win32.whl", hash = "sha256:8b1288a09a8d100e9bf7cf9ce1329433db73a0d0350d74c2c6f5c31ac69096cf"}, + {file = "jarowinkler-1.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:ed39199b0e806902347473c65e5c05933549cf7e55ba628c6812782f2c310b19"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:473b057d7e5a0f5e5b8c0e0f7960d3ca2f2954c3c93fd7a9fb2cc4bc3cc940fb"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdb892dbbbd77b3789a10b2ce5e8acfe5821cc6423e835bae2b489159f3c2211"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:012a8333328ce061cba1ff081843c8d80eb1afe8fa2889ad29d767ea3fdc7562"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3421120c07ee6d3f59c5adde32eb9a050cfd1b3666b0e2d8c337d934a9d091f9"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad57327cc90f8daa3afb98e2d274d7dd1b60651f32717449be95d3b3366d61a"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4fd1757eff43df97227fd63d9c8078582267a0b25cefef6f6a64d3e46e80ba2"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:32269ebbcb860f01c055d9bb145b4cc91990f62c7644a85b21458b4868621113"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3b5a0839e84f5ff914b01b5b94d0273954affce9cc2b2ee2c31fe2fcb9c8ae76"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6c9d3a9ef008428b5dce2855eebe2b6127ea7a7e433aedf240653fad4bd4baa6"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a3d7759d8a66ee05595bde012f93da8a63499f38205e2bb47022c52bd6c47108"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2ba1b1b0bf45042a9bbb95d272fd8b0c559fe8f6806f088ec0372899e1bc6224"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-win32.whl", hash = "sha256:4cb33f4343774d69abf8cf65ad57919e7a171c44ba6ad57b08147c3f0f06b073"}, + {file = "jarowinkler-1.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:0392b72ddb5ab5d6c1d5df94dbdac7bf229670e5e64b2b9a382d02d6158755e5"}, + {file = "jarowinkler-1.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:94f663ad85bc7a89d7e8b6048f93a46d2848a0570ab07fc895a239b9a5d97b93"}, + {file = "jarowinkler-1.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:895a10766ff3db15e7cf2b735e4277bee051eaafb437aaaef2c5de64a5c3f05c"}, + {file = "jarowinkler-1.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0c1a84e770b3ec7385a4f40efb30bdc96f96844564f91f8d3937d54a8969d82c"}, + {file = "jarowinkler-1.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27defe81d76e02b3929322baea999f5232837e7f308c2dc5b37de7568c2bc583"}, + {file = "jarowinkler-1.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:158f117481388f8d23fe4bd2567f37be0ccae0f4631c34e4b0345803147da207"}, + {file = "jarowinkler-1.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:427c675b4f3e83c79a4b6af7441f29e30a173c7a0ae72a54f51090eee7a8ae02"}, + {file = "jarowinkler-1.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90a7f3fd173339bc62e52c02f43d50c947cb3af9cda41646e218aea13547e0c2"}, + {file = "jarowinkler-1.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3975cbe8b6ae13fc63d74bcbed8dac1577078d8cd8728e60621fe75885d2a8c5"}, + {file = "jarowinkler-1.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:141840f33345b00abd611839080edc99d4d31abd2dcf701a3e50c90f9bfb2383"}, + {file = "jarowinkler-1.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f592f9f6179e347a5f518ca7feb9bf3ac068f2fad60ece5a0eef5e5e580d4c8b"}, + {file = "jarowinkler-1.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:30565d70396eb9d1eb622e1e707ddc2f3b7a9692558b8bf4ea49415a5ca2f854"}, + {file = "jarowinkler-1.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:35fc430c11b80a43ed826879c78c4197ec665d5150745b3668bec961acf8a757"}, + {file = "jarowinkler-1.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e4cf4b7090f0c4075bec1638717f54b22c3b0fe733dc87146a19574346ed3161"}, + {file = "jarowinkler-1.0.2-cp38-cp38-win32.whl", hash = "sha256:199f4f7edbc49439a97440caa1e244d2e33da3e16d7b0afce4e4dfd307e555c7"}, + {file = "jarowinkler-1.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:b587e8fdd96cc470d6bdf428129c65264731b09b5db442e2d092e983feec4aab"}, + {file = "jarowinkler-1.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4b233180b3e2f2d7967aa570d36984e9d2ec5a9067c0d1c44cd3b805d9da9363"}, + {file = "jarowinkler-1.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2220665a1f52262ae8b76e3baf474ebcd209bfcb6a7cada346ffd62818f5aa3e"}, + {file = "jarowinkler-1.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08c98387e04e749c84cc967db628e5047843f19f87bf515a35b72f7050bc28ad"}, + {file = "jarowinkler-1.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d710921657442ad3c942de684aba0bdf16b7de5feed3223b12f3b2517cf17f7c"}, + {file = "jarowinkler-1.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:401c02ac7245103826f54c816324274f53d50b638ab0f8b359a13055a7a6e793"}, + {file = "jarowinkler-1.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a1929a0029f208cc9244499dc93b4d52ee8e80d2849177d425cf6e0be1ea781"}, + {file = "jarowinkler-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ab25d147be9b04e7de2d28a18e72fadc152698c3e51683c6c61f73ffbae2f9e"}, + {file = "jarowinkler-1.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:465cfdff355ec9c55f65fd1e1315260ec20c8cff0eb90d9f1a0ad8d503dc002b"}, + {file = "jarowinkler-1.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:29ef1113697cc74c2f04bc15008abbd726cb2d5b01c040ba87c6cb7abd1d0e0d"}, + {file = "jarowinkler-1.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:61b57c8b36361ec889f99f761441bb0fa21b850a5eb3305dea25fef68f6a797b"}, + {file = "jarowinkler-1.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ee9d9af1bbf194d78f4b69c2139807c23451068b27a053a1400d683d6f36c61d"}, + {file = "jarowinkler-1.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a9b33b0ceb472bbc65683467189bd032c162256b2a137586ee3448a9f8f886ec"}, + {file = "jarowinkler-1.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:582f6e213a6744883ced44482a51efcc21ae632defac27f12f6430a8e99b1070"}, + {file = "jarowinkler-1.0.2-cp39-cp39-win32.whl", hash = "sha256:4d1c8f403016d5c0262de7a8588eee370c37a609e1f529f8407e99a70d020af7"}, + {file = "jarowinkler-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:ab50ffa66aa201616871c1b90ac0790f56666118db3c8a8fcb3a7a6e03971510"}, + {file = "jarowinkler-1.0.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8e59a289dcf93504ab92795666c39b2dbe98ac18655201992a7e6247de676bf4"}, + {file = "jarowinkler-1.0.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c36eccdc866f06a7b35da701bd8f91e0dfc83b35c07aba75ce8c906cbafaf184"}, + {file = "jarowinkler-1.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:123163f01a5c43f12e4294e7ce567607d859e1446b1a43bd6cd404b3403ffa07"}, + {file = "jarowinkler-1.0.2-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d41fdecd907189e47c7d478e558ad417da38bf3eb34cc20527035cb3fca3e2b8"}, + {file = "jarowinkler-1.0.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7829368fc91de225f37f6325f8d8ec7ad831dc5b0e9547f1977e2fdc85eccc1"}, + {file = "jarowinkler-1.0.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278595417974553a8fdf3c8cce5c2b4f859335344075b870ecb55cc416eb76cf"}, + {file = "jarowinkler-1.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:208fc49741db5d3e6bbd4a2f7b32d32644b462bf205e7510eca4e2d530225f03"}, + {file = "jarowinkler-1.0.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:924afcab6739c453f1c3492701d185d71dc0e5ba15692bd0bfa6d482c7e8f79e"}, + {file = "jarowinkler-1.0.2.tar.gz", hash = "sha256:788ac33e6ffdbd78fd913b481e37cfa149288575f087a1aae1a4ce219cb1c654"}, +] jinja2 = [ {file = "Jinja2-3.0.1-py3-none-any.whl", hash = "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4"}, {file = "Jinja2-3.0.1.tar.gz", hash = "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4"}, @@ -1959,6 +2063,54 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069"}, {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] +rapidfuzz = [ + {file = "rapidfuzz-2.0.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d7297eeeb8b50aeeef81167c3cb34994adcf86f8d6bf0c9ea06fb566540c878d"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:263b7b9b7f9f68a255f2dc37c28c2213ae03903f5650dbdd4a0e1b44609ed222"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6738210e4dec381f41513e41eaf3a87124188dfab836459c6b392b121244a0f"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3421c6b1dd36a4a5f9e4c323b9e3116b178430ab769c61bce77e7aa85c53575"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbbb8e0abd7944254f62e510ed13fa9b5189c11c8247a77d5c7dc12cd58c20f1"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f8d5c2d593557da37632bc5c84b05cff69b14bb8255210fa27183e35b848542"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-win32.whl", hash = "sha256:db88723b83436b7188ad3f02f53d67ff78fbdb0e6a0b129cd7f51d18ebf52da6"}, + {file = "rapidfuzz-2.0.10-cp310-cp310-win_amd64.whl", hash = "sha256:270194fc82f055fc4fb63ce0550d9bb384540aef699218df330a30c24ce7546f"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cea0dea5e79219777347a7c83c7953bc6ed3fc73d4ede0a931ea3362e99de0bd"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0457b4a652484dc8b39b52e56b0d7ada2550b262df4e52a504db3b34f060ea6"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d9dd138c0f7edc48fd16ac8ad5b14c0c47f502e40f86979d54205b9b24e4d9f"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68a28b4cb76c422c30aff221a14009bbfed7134b3305966817970a0ad83ca1ca"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-win32.whl", hash = "sha256:1dc06e50fb5410d2b3f607767ab6fc1dd8b9a559d40e0099a8f8f73d9d4d3db3"}, + {file = "rapidfuzz-2.0.10-cp36-cp36m-win_amd64.whl", hash = "sha256:8d5ebda761193087d19606cd8026c7d3aa528ed13f4bc98ceecdd6da1d55fb20"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ca08437f42047a3e8b1aecd39ba06debf063bc171d8375f0ddfa9b6a507853e8"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fe107338825eadcda36ad3f50fe042e9e26018592af7c8ff3b4d16275f5fd01"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:86fbbabd4176efb3e289cff65192a71e74475866c5738ae2039212c3b2db25cd"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8156f7d8d7441c2bcb84ed9b5a873f5eee044fbdb3c1f599926a36663d323447"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-win32.whl", hash = "sha256:89482d7436b3a1768e330c952c5346bb777f1969858284f2a6dcfb1c7d47f51d"}, + {file = "rapidfuzz-2.0.10-cp37-cp37m-win_amd64.whl", hash = "sha256:db21778d648fa1805cea122b156c4727d3c6d2baf6ff0794af1794d17941512b"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:48b92a056246adac535d66e34ae7f5b9ed52654962f90d39c94fcb11dbeb6f0c"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de02ce4d7e241f3fcfba3227e1f9665f82b9af38c5d36190df3a247fb2461411"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8748f89974916b15e8d70c0ff7097e2656f3aa89cbeaa810e69b07819481f84c"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1400e029195f511788675144f1aab01de43aae7d3f5ec683f263ee13b47f6b16"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd8fdd75ad347b35eef562f39f5f8ad8c9784c5d3890bf49ecc24f5c1e3d16c1"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84391cd3fa17a6655abd83961f4720806b64f090dbc46426ed4841b410dbc841"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-win32.whl", hash = "sha256:3a0dd9a837288a65a74a819b0d6f0d139daeb7f6155c3158f6eedd0af1e6d021"}, + {file = "rapidfuzz-2.0.10-cp38-cp38-win_amd64.whl", hash = "sha256:16e69fcc7009659ee8459a9ad4605651b1cc7071e64698da1a5568f473c0ee3f"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9f0daa98b6f9d69811d64cb2277209c56ba7b68e5f50d6903795a2b0a2a4d9c2"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c3e7e5489fe1915950a663c8f6c572aa390765db96a171f36215af2d4bb19a6b"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a5d9bf9d03bc64720dc0ad4a10b8c1fb0326bc6883d3411181a519a3ccdf779"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8495ec199402ffa5b6b8c724675e1c0fb7e5a6617ea3c90323bb56449df6b36d"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13e584cd7879e8528913a30d69d83cf88198287a7557435361f31f794a349878"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7aa0510c2291751d3bd90b38cf11e9c60cda41766927a25b169698fc2c2689"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-win32.whl", hash = "sha256:a6fd46fe173a5bf7ec85819a1d2bb343303bd5b28a80671ce886b97f3c669ea9"}, + {file = "rapidfuzz-2.0.10-cp39-cp39-win_amd64.whl", hash = "sha256:be743ec34a7f88255c6735780b35578c03a6192ee2f9b325493ed736b0ab2cf3"}, + {file = "rapidfuzz-2.0.10-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:148e931c28aa09532c99db2f30f01a88eed5a065c9f9ed119c5b915994582054"}, + {file = "rapidfuzz-2.0.10-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16fbc2686eb9310ebcd77eb819743b541cd1dd2b83f555e0eaf356615452eb89"}, + {file = "rapidfuzz-2.0.10-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84a49b857c1d521691d64bbe085cc31f7b397853901acf0eb06b799f570dfbd3"}, + {file = "rapidfuzz-2.0.10-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ac7864b52714ef183d37c9fe50da806ad81bdb47f72bbe3e7c629932af62c66"}, + {file = "rapidfuzz-2.0.10-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5d72407122e7c4131aaf8ddb37cd05496d80418796a57bf90db976d511a74c"}, + {file = "rapidfuzz-2.0.10-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2402b91631c5c8e48055a8551692b313f6422fece403e2a8020ecbcafef140a7"}, + {file = "rapidfuzz-2.0.10-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:628cfa67d48d2fcc9a97ed2612ae166395862fb2aea3a810d5d341c3d3490f29"}, + {file = "rapidfuzz-2.0.10-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4fea35a888cdd8f2db5d7ebb02436ac4892ce1eaa33e2b090b29bdead4cc41f6"}, + {file = "rapidfuzz-2.0.10-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c00164fc33f2b64cb7cc33f1fb714924e1eaecd0ce92b8f68b2891072910082"}, + {file = "rapidfuzz-2.0.10.tar.gz", hash = "sha256:6c8fe3051dce837c4deb080b438b38efc8268e1c14b9e6a64b173b35f4e32773"}, +] requests = [ {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, diff --git a/pyproject.toml b/pyproject.toml index b9211ad5..2a3a9a87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ coloredlogs = "^15.0" attrs = "^21.2.0" desert = "^2020.11.18" marshmallow = "~=3.13.0" +rapidfuzz = "^2.0.10" python-dotenv = "^0.19.0" PyYAML = { version = "^5.4.1", optional = true } typing-extensions = "^3.10.0.2" diff --git a/requirements.txt b/requirements.txt index fc3854a3..9d885baf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ desert==2020.11.18 ; python_version >= "3.6" discord.py @ https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip ; python_full_version >= "3.8.0" humanfriendly==10.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" idna==3.2 ; python_version >= "3.5" +jarowinkler==1.0.2 ; python_version >= "3.6" marshmallow-enum==1.5.1 marshmallow==3.13.0 ; python_version >= "3.5" multidict==5.2.0 ; python_version >= "3.6" @@ -27,6 +28,7 @@ pyreadline3==3.3 ; sys_platform == "win32" python-dateutil==2.8.2 ; python_version != "3.0" python-dotenv==0.19.0 ; python_version >= "3.5" pyyaml==5.4.1 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" and python_version != "3.3" and python_version != "3.4" and python_version != "3.5" +rapidfuzz==2.0.10 ; python_version >= "3.6" six==1.16.0 ; python_version >= "2.7" and python_version != "3.0" and python_version != "3.1" and python_version != "3.2" typing-extensions==3.10.0.2 typing-inspect==0.7.1