From 2ffa54dfa3f70ff79dfce72be9bbe33bf882fbee Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 28 Jun 2025 10:33:17 +0200 Subject: [PATCH 1/2] Fix usage of deprecated libraries --- hivemind/p2p/p2p_daemon.py | 5 +-- pyproject.toml | 55 ++++++++++++++++++++++++++ requirements.txt | 2 +- setup.py | 72 +++------------------------------- tests/test_auth.py | 8 ++-- tests/test_utils/p2p_daemon.py | 5 +-- 6 files changed, 70 insertions(+), 77 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a6c9484c9..3840efc3b 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -8,7 +8,7 @@ from contextlib import closing, suppress from dataclasses import dataclass from datetime import datetime -from importlib.resources import path +from importlib.resources import files from typing import Any, AsyncIterator, Awaitable, Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union from google.protobuf.message import Message @@ -158,8 +158,7 @@ async def create( ) self = cls() - with path(cli, P2PD_FILENAME) as p: - p2pd_path = p + p2pd_path = files(cli) / P2PD_FILENAME socket_uid = secrets.token_urlsafe(8) self._daemon_listen_maddr = Multiaddr(cls._UNIX_SOCKET_PREFIX + f"p2pd-{socket_uid}.sock") diff --git a/pyproject.toml b/pyproject.toml index 1bef4011b..6d521f16f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,58 @@ +[build-system] +requires = ["setuptools>=64", "grpcio-tools"] +build-backend = "setuptools.build_meta" + +[project] +name = "hivemind" +description = "Decentralized deep learning in PyTorch" +readme = "README.md" +authors = [{name = "Learning@home & contributors", email = "hivemind-team@hotmail.com"}] +license = "MIT" +requires-python = ">=3.9" +keywords = ["pytorch", "deep learning", "machine learning", "gpu", "distributed computing", "volunteer computing", "dht"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dynamic = ["version", "dependencies"] + +[project.optional-dependencies] +bitsandbytes = ["bitsandbytes~=0.45.2"] + +[project.scripts] +hivemind-dht = "hivemind.hivemind_cli.run_dht:main" +hivemind-server = "hivemind.hivemind_cli.run_server:main" + +[project.urls] +Homepage = "https://github.com/learning-at-home/hivemind" +Repository = "https://github.com/learning-at-home/hivemind" + +[tool.setuptools] +packages = {find = {}} +include-package-data = true + +[tool.setuptools.dynamic] +version = {attr = "hivemind.__version__"} +dependencies = {file = ["requirements.txt"]} +optional-dependencies.dev = {file = ["requirements-dev.txt"]} +optional-dependencies.docs = {file = ["requirements-docs.txt"]} +optional-dependencies.all = {file = ["requirements-all.txt"]} + +[tool.setuptools.package-data] +hivemind = ["proto/*", "hivemind_cli/*"] + [tool.coverage.run] concurrency = ["thread", "multiprocessing"] omit = ["hivemind/proto/*"] diff --git a/requirements.txt b/requirements.txt index 78aab21bd..84117fbcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ msgpack>=0.5.6 sortedcontainers uvloop>=0.14.0 grpcio-tools>=1.33.2 -protobuf>=5.29.0 +protobuf>=6.31.0 configargparse>=1.2.3 py-multihash>=0.2.3 cryptography>=3.4.6 diff --git a/setup.py b/setup.py index e1f90b274..78e50b372 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import codecs import glob import hashlib import os @@ -9,8 +8,7 @@ import tempfile import urllib.request -from pkg_resources import parse_requirements, parse_version -from setuptools import find_packages, setup +from setuptools import setup from setuptools.command.build_py import build_py from setuptools.command.develop import develop @@ -64,9 +62,11 @@ def build_p2p_daemon(): if m is None: raise FileNotFoundError("Could not find golang installation") - version = parse_version(m.group(1)) - if version < parse_version("1.13"): - raise OSError(f"Newer version of go required: must be >= 1.13, found {version}") + + go_version_str = m.group(1) + go_version_parts = [int(x) for x in go_version_str.split(".")] + if go_version_parts[0] < 1 or (go_version_parts[0] == 1 and go_version_parts[1] < 13): + raise OSError(f"Newer version of go required: must be >= 1.13, found {go_version_str}") with tempfile.TemporaryDirectory() as tempdir: dest = os.path.join(tempdir, "libp2p-daemon.tar.gz") @@ -140,66 +140,6 @@ def run(self): super().run() -with open("requirements.txt") as requirements_file: - install_requires = list(map(str, parse_requirements(requirements_file))) - -# loading version from setup.py -with codecs.open(os.path.join(here, "hivemind/__init__.py"), encoding="utf-8") as init_file: - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.MULTILINE) - version_string = version_match.group(1) - -extras = {} - -with open("requirements-dev.txt") as dev_requirements_file: - extras["dev"] = list(map(str, parse_requirements(dev_requirements_file))) - -with open("requirements-docs.txt") as docs_requirements_file: - extras["docs"] = list(map(str, parse_requirements(docs_requirements_file))) - -extras["bitsandbytes"] = ["bitsandbytes~=0.45.2"] - -extras["all"] = extras["dev"] + extras["docs"] + extras["bitsandbytes"] - setup( - name="hivemind", - version=version_string, cmdclass={"build_py": BuildPy, "develop": Develop}, - description="Decentralized deep learning in PyTorch", - long_description="Decentralized deep learning in PyTorch. Built to train models on thousands of volunteers " - "across the world.", - author="Learning@home & contributors", - author_email="hivemind-team@hotmail.com", - url="https://github.com/learning-at-home/hivemind", - packages=find_packages(exclude=["tests"]), - package_data={"hivemind": ["proto/*", "hivemind_cli/*"]}, - include_package_data=True, - license="MIT", - setup_requires=["grpcio-tools"], - install_requires=install_requires, - extras_require=extras, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - entry_points={ - "console_scripts": [ - "hivemind-dht = hivemind.hivemind_cli.run_dht:main", - "hivemind-server = hivemind.hivemind_cli.run_server:main", - ] - }, - # What does your project relate to? - keywords="pytorch, deep learning, machine learning, gpu, distributed computing, volunteer computing, dht", ) diff --git a/tests/test_auth.py b/tests/test_auth.py index 75b647995..c402ab2b8 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import Optional import pytest @@ -31,7 +31,7 @@ async def get_token(self) -> AccessToken: token = AccessToken( username=self._username, public_key=self.local_public_key.to_bytes(), - expiration_time=str(datetime.utcnow() + timedelta(minutes=1)), + expiration_time=str(datetime.now(UTC) + timedelta(minutes=1)), ) token.signature = MockAuthorizer._authority_private_key.sign(self._token_to_bytes(token)) return token @@ -52,7 +52,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool: if expiration_time.tzinfo is not None: logger.exception(f"Expected to have no timezone for expiration time: {access_token.expiration_time}") return False - if expiration_time < datetime.utcnow(): + if expiration_time < datetime.now(UTC): logger.exception("Access token has expired") return False @@ -62,7 +62,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool: def does_token_need_refreshing(self, access_token: AccessToken) -> bool: expiration_time = datetime.fromisoformat(access_token.expiration_time) - return expiration_time < datetime.utcnow() + self._MAX_LATENCY + return expiration_time < datetime.now(UTC) + self._MAX_LATENCY @staticmethod def _token_to_bytes(access_token: AccessToken) -> bytes: diff --git a/tests/test_utils/p2p_daemon.py b/tests/test_utils/p2p_daemon.py index db208cdd5..ab87d96e3 100644 --- a/tests/test_utils/p2p_daemon.py +++ b/tests/test_utils/p2p_daemon.py @@ -5,17 +5,16 @@ import time import uuid from contextlib import asynccontextmanager, suppress +from importlib.resources import files from typing import NamedTuple -from pkg_resources import resource_filename - from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client from hivemind.utils.multiaddr import Multiaddr, protocols from test_utils.networking import get_free_port TIMEOUT_DURATION = 5 # seconds -P2PD_PATH = resource_filename("hivemind", "hivemind_cli/p2pd") +P2PD_PATH = str(files("hivemind").joinpath("hivemind_cli/p2pd")) async def try_until_success(coro_func, timeout=TIMEOUT_DURATION): From fc2625e9f09b225fb70fa1458ba31d13dedad975 Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sun, 3 Aug 2025 20:00:09 +0200 Subject: [PATCH 2/2] Fix the mounting path --- modal_ci.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/modal_ci.py b/modal_ci.py index 43b977ee7..e1c948125 100644 --- a/modal_ci.py +++ b/modal_ci.py @@ -15,13 +15,13 @@ "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ", ] ) - .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind") - .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt") - .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt") - .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt") - .add_local_file("setup.py", remote_path="/root/hivemind/setup.py") - .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml") - .add_local_dir("tests", remote_path="/root/hivemind/tests") + .add_local_dir("hivemind", remote_path="/root/project/hivemind") + .add_local_file("requirements.txt", remote_path="/root/project/requirements.txt") + .add_local_file("requirements-dev.txt", remote_path="/root/project/requirements-dev.txt") + .add_local_file("requirements-docs.txt", remote_path="/root/project/requirements-docs.txt") + .add_local_file("setup.py", remote_path="/root/project/setup.py") + .add_local_file("pyproject.toml", remote_path="/root/project/pyproject.toml") + .add_local_dir("tests", remote_path="/root/project/tests") ) # Create an image with golang and other system dependencies @@ -41,13 +41,13 @@ "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ", ] ) - .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind") - .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt") - .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt") - .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt") - .add_local_file("setup.py", remote_path="/root/hivemind/setup.py") - .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml") - .add_local_dir("tests", remote_path="/root/hivemind/tests") + .add_local_dir("hivemind", remote_path="/root/project/hivemind") + .add_local_file("requirements.txt", remote_path="/root/project/requirements.txt") + .add_local_file("requirements-dev.txt", remote_path="/root/project/requirements-dev.txt") + .add_local_file("requirements-docs.txt", remote_path="/root/project/requirements-docs.txt") + .add_local_file("setup.py", remote_path="/root/project/setup.py") + .add_local_file("pyproject.toml", remote_path="/root/project/pyproject.toml") + .add_local_dir("tests", remote_path="/root/project/tests") ) @@ -64,7 +64,7 @@ def setup_environment(*, build_p2pd=False): - os.chdir("/root/hivemind") + os.chdir("/root/project") if build_p2pd: install_cmd = [ @@ -77,7 +77,7 @@ def setup_environment(*, build_p2pd=False): "--no-use-pep517", ] else: - install_cmd = ["pip", "install", "-e", ".", "--no-use-pep517"] + install_cmd = ["pip", "install", "-e", "."] subprocess.run(install_cmd, check=True)