From 61ba8afee6117810972083a1d300c587ff728bec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Jan 2023 13:51:41 +0200 Subject: [PATCH 01/98] Bump version to 6.1.7a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 08d95a7985..579701abe1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, 6) +VERSION = (6, 1, "7a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 8835a03cd9acb47fe027629d238e6f7c32439270 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 20:49:59 +0200 Subject: [PATCH 02/98] Rename "InternetIsOffline" exception to "InternetConnectionError" --- platformio/builder/tools/piolib.py | 8 ++++++-- platformio/package/manager/platform.py | 4 ++-- tests/misc/test_misc.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8e1bec15fa..159dec601f 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -29,7 +29,7 @@ from platformio import exception, fs from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -982,7 +982,11 @@ def _is_builtin(spec): try: lm.install(spec) did_install = True - except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc: + except ( + HTTPClientError, + UnknownPackageError, + InternetConnectionError, + ) as exc: click.secho("Warning! %s" % exc, fg="yellow") # reset cache diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index e5e948babf..6dbd480c7f 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -15,7 +15,7 @@ import os from platformio import util -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.core import get_installed_core_packages @@ -128,7 +128,7 @@ def get_all_boards(self): key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (HTTPClientError, InternetIsOffline): + except (HTTPClientError, InternetConnectionError): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index 52a6cac4c7..349e9c7920 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -35,7 +35,7 @@ def test_ping_internet_ips(): def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() - with pytest.raises(http.InternetIsOffline): + with pytest.raises(http.InternetConnectionError): regclient.fetch_json_data("get", "/v2/stats") From 4350c4ca484d764fe3addc6befd9160aa61014fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 20:50:38 +0200 Subject: [PATCH 03/98] Rename "InternetIsOffline" exception to "InternetConnectionError" --- platformio/http.py | 4 ++-- platformio/maintenance.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/http.py b/platformio/http.py index 9f4f44ddb2..501d10783c 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -37,7 +37,7 @@ def __str__(self): # pragma: no cover return self.message -class InternetIsOffline(UserSideException): +class InternetConnectionError(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" @@ -204,7 +204,7 @@ def _internet_on(): def ensure_internet_on(raise_exception=False): result = _internet_on() if raise_exception and not result: - raise InternetIsOffline() + raise InternetConnectionError() return result diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7893a23db4..5518495364 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -24,7 +24,7 @@ from platformio.cli import PlatformioCLI from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.http import HTTPClientError, InternetIsOffline, ensure_internet_on +from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec @@ -51,7 +51,7 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_prune_system() except ( HTTPClientError, - InternetIsOffline, + InternetConnectionError, exception.GetLatestVersionError, ): click.secho( From 380652eb528323d3ece885e94ed3b46e10a33415 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 20:51:37 +0200 Subject: [PATCH 04/98] Raise 5000 error for RPC calls --- platformio/home/rpc/handlers/account.py | 2 +- platformio/home/rpc/handlers/piocore.py | 2 +- platformio/home/rpc/handlers/registry.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index d857d58703..2987329eb5 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -25,5 +25,5 @@ def call_client(method, *args, **kwargs): return getattr(client, method)(*args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Account Call Error", data=str(exc) + code=5000, message="PIO Account Call Error", data=str(exc) ) from exc diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index df2521145a..95b7b4781b 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -94,7 +94,7 @@ async def call(args, options=None): return PIOCoreRPC._process_result(result, to_json) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Core Call Error", data=str(exc) + code=5000, message="PIO Core Call Error", data=str(exc) ) from exc @staticmethod diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index a6d5b8bff1..fa6f702ccc 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -25,5 +25,5 @@ def call_client(method, *args, **kwargs): return getattr(client, method)(*args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="Registry Call Error", data=str(exc) + code=5000, message="Registry Call Error", data=str(exc) ) from exc From 464b167e65166d815f77b3a51d742ce0e38600a4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 20:52:48 +0200 Subject: [PATCH 05/98] Wrap "NoInternetConnection" to 4008 RPC error --- platformio/home/rpc/server.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index eb5bd6a527..6c0abdb45d 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -13,11 +13,13 @@ # limitations under the License. import click +from ajsonrpc.core import JSONRPC20Error from ajsonrpc.dispatcher import Dispatcher -from ajsonrpc.manager import AsyncJSONRPCResponseManager +from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response from starlette.endpoints import WebSocketEndpoint from platformio.compat import aio_create_task, aio_get_running_loop +from platformio.http import InternetConnectionError from platformio.proc import force_exit @@ -94,4 +96,13 @@ async def _handle_rpc(self, websocket, data): response = await self.factory.manager.get_response_for_payload(data) if response.error and response.error.data: click.secho("Error: %s" % response.error.data, fg="red", err=True) + if InternetConnectionError.MESSAGE in response.error.data: + response = JSONRPC20Response( + id=response.id, + error=JSONRPC20Error( + code=4008, + message="No Internet Connection", + data=response.error.data, + ), + ) await websocket.send_text(self.factory.manager.serialize(response.body)) From 0d57a799b5f2322900d0b07c42ea0bc5fcad2437 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 20:53:43 +0200 Subject: [PATCH 06/98] Port RPC "registry.call_client" to async --- platformio/home/rpc/handlers/registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index fa6f702ccc..34d0974b21 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -13,16 +13,17 @@ # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException +from starlette.concurrency import run_in_threadpool from platformio.registry.client import RegistryClient class RegistryRPC: @staticmethod - def call_client(method, *args, **kwargs): + async def call_client(method, *args, **kwargs): try: client = RegistryClient() - return getattr(client, method)(*args, **kwargs) + return await run_in_threadpool(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="Registry Call Error", data=str(exc) From 15d53c95c06d03b648c18548d1cb3258338e417b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jan 2023 21:06:13 +0200 Subject: [PATCH 07/98] Prevent shell injection when converting INO file to CPP // Resolve #4532 --- HISTORY.rst | 5 +++++ platformio/builder/tools/pioino.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index b963df382e..cc5ee8f6d6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,11 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.7 (2023-??-??) +~~~~~~~~~~~~~~~~~~ + +* Prevented shell injection when converting INO file to CPP (`issue #4532 `_) + 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index de1f8cf899..939cf9a0b3 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -103,7 +103,7 @@ def merge(self, nodes): return "\n".join(["#include "] + lines) if lines else None def process(self, contents): - out_file = self._main_ino + ".cpp" + out_file = re.sub(r"[\"\'\;]+", "", self._main_ino, flags=re.I) + ".cpp" assert self._gcc_preprocess(contents, out_file) contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) From b2a04f265e533bd7f53e77456f72663f3d7cdd97 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Jan 2023 14:34:06 +0200 Subject: [PATCH 08/98] Code cleanup --- platformio/builder/tools/pioino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 939cf9a0b3..57b8650002 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -103,7 +103,7 @@ def merge(self, nodes): return "\n".join(["#include "] + lines) if lines else None def process(self, contents): - out_file = re.sub(r"[\"\'\;]+", "", self._main_ino, flags=re.I) + ".cpp" + out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp" assert self._gcc_preprocess(contents, out_file) contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) From 188c65ef7be20a34528a6fb08090d1bb48d17486 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Jan 2023 14:34:54 +0200 Subject: [PATCH 09/98] Show detailed library dependency tree only in the verbose mode // Resolve #4517 --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index cc5ee8f6d6..5c47efcd82 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 6 6.1.7 (2023-??-??) ~~~~~~~~~~~~~~~~~~ +* Show detailed library dependency tree only in the `verbose mode `__ (`issue #4517 `_) * Prevented shell injection when converting INO file to CPP (`issue #4532 `_) 6.1.6 (2023-01-23) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 159dec601f..c0e2be593e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1161,7 +1161,7 @@ def _print_deps_tree(root, level=0): click.echo("Path: %s" % lb.path, nl=False) click.echo(")", nl=False) click.echo("") - if lb.depbuilders: + if lb.verbose and lb.depbuilders: _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") From 097de2be98af533578671baa903a3ae825d90b94 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Jan 2023 14:05:15 +0200 Subject: [PATCH 10/98] Do not copy PIO Core Python PATH to the global env --- platformio/platform/_run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 79afff9435..91c909314b 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -102,7 +102,6 @@ def _run_scons(self, variables, targets, jobs): for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - proc.copy_pythonpath_to_osenv() # force SCons output to Unicode os.environ["PYTHONIOENCODING"] = "utf-8" From 18b6aad369e0d10cd1b575f86ea2f64eb71a0a14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 1 Feb 2023 23:55:02 +0200 Subject: [PATCH 11/98] Bump version to 6.1.7a2 --- docs | 2 +- examples | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs b/docs index 95c339a711..ba58a6e7cd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 95c339a71162dc5bd28090f263e641a563229888 +Subproject commit ba58a6e7cd4138cbcdcca801cc251bbd2871ced2 diff --git a/examples b/examples index f98cb5a9be..b7900244c8 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f98cb5a9be74b9c87b6e5f3cd530d8d7f0548825 +Subproject commit b7900244c8ec2844dfea80b78b1dafd67855a8fd diff --git a/platformio/__init__.py b/platformio/__init__.py index 579701abe1..6872e7608e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "7a1") +VERSION = (6, 1, "7a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5073313c33f8a759d30fb6cb8c80f2c09e77ae5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 17:43:38 +0200 Subject: [PATCH 12/98] PyLint fixes --- .pylintrc | 3 ++- Makefile | 2 +- platformio/check/defect.py | 3 ++- platformio/device/monitor/terminal.py | 5 ++--- platformio/home/rpc/handlers/piocore.py | 3 ++- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index 0b9ff1f84b..172b43a191 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,4 +8,5 @@ disable= invalid-name, too-few-public-methods, consider-using-f-string, - cyclic-import + cyclic-import, + use-dict-literal diff --git a/Makefile b/Makefile index 09c0b8386a..027af3cac7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ lint: - pylint --rcfile=./.pylintrc ./tests pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests isort: isort ./platformio diff --git a/platformio/check/defect.py b/platformio/check/defect.py index 15f9df70cb..9efcad9096 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -16,6 +16,7 @@ import click +from platformio.exception import PlatformioException from platformio.project.helpers import get_project_dir # pylint: disable=too-many-instance-attributes, redefined-builtin @@ -79,7 +80,7 @@ def severity_to_int(label): for key, value in DefectItem.SEVERITY_LABELS.items(): if label == value: return key - raise Exception("Unknown severity label -> %s" % label) + raise PlatformioException("Unknown severity label -> %s" % label) def as_dict(self): return { diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index bec825e982..7b2a9b91b9 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -144,9 +144,8 @@ def new_serial_instance(options): # pylint: disable=too-many-branches except KeyboardInterrupt as exc: click.echo("", err=True) raise UserSideException("User aborted and port is not given") from exc - else: - if not port: - raise UserSideException("Port is not given") + if not port: + raise UserSideException("Port is not given") try: serial_instance = serial.serial_for_url( port, diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 95b7b4781b..99a5325e57 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -25,6 +25,7 @@ from platformio import __main__, __version__, fs, proc from platformio.compat import get_locale_encoding, is_bytes from platformio.home import helpers +from platformio.exception import PlatformioException class MultiThreadingStdStream: @@ -132,7 +133,7 @@ def _process_result(result, to_json=False): err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: - raise Exception(text) + raise PlatformioException(text) if not to_json: return text try: From 1af508272bb7b9824e631a748ce04689582f5d4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 17:46:03 +0200 Subject: [PATCH 13/98] Format code with Black 23.1.0 --- platformio/account/client.py | 4 ---- platformio/builder/tools/pioino.py | 1 - platformio/builder/tools/piolib.py | 1 - platformio/check/defect.py | 1 - platformio/cli.py | 1 - platformio/commands/boards.py | 2 +- platformio/debug/config/blackmagic.py | 1 - platformio/debug/config/generic.py | 1 - platformio/debug/config/jlink.py | 1 - platformio/debug/config/mspdebug.py | 1 - platformio/debug/config/native.py | 1 - platformio/debug/config/qemu.py | 1 - platformio/debug/config/renode.py | 1 - platformio/debug/exception.py | 1 - platformio/debug/helpers.py | 1 - platformio/debug/process/base.py | 1 - platformio/debug/process/gdb.py | 1 - platformio/debug/process/server.py | 1 - platformio/exception.py | 14 -------------- platformio/fs.py | 2 +- platformio/home/rpc/handlers/app.py | 1 - platformio/home/rpc/handlers/ide.py | 1 - platformio/home/rpc/handlers/piocore.py | 2 +- platformio/home/rpc/server.py | 1 - platformio/http.py | 1 - platformio/package/commands/list.py | 2 +- platformio/package/exception.py | 3 --- platformio/package/manager/_download.py | 1 - platformio/package/manager/_install.py | 1 - platformio/package/meta.py | 2 -- platformio/package/unpack.py | 1 - platformio/package/vcsclient.py | 4 ---- platformio/platform/_run.py | 1 - platformio/platform/base.py | 1 - platformio/platform/exception.py | 6 ------ platformio/project/commands/init.py | 2 +- platformio/project/config.py | 2 -- platformio/project/exception.py | 6 ------ platformio/registry/mirror.py | 1 - platformio/remote/ac/base.py | 1 - platformio/remote/cli.py | 2 -- platformio/remote/client/async_base.py | 2 +- platformio/remote/client/base.py | 1 - platformio/remote/client/device_list.py | 2 +- platformio/remote/client/device_monitor.py | 3 +-- platformio/remote/client/run_or_test.py | 3 +-- platformio/remote/projectsync.py | 4 ++-- platformio/telemetry.py | 2 -- platformio/test/exception.py | 1 - platformio/test/helpers.py | 1 - platformio/test/runners/base.py | 1 - platformio/test/runners/doctest.py | 1 - platformio/test/runners/googletest.py | 2 -- platformio/test/runners/readers/serial.py | 1 - platformio/test/runners/unity.py | 1 - platformio/util.py | 1 - tests/commands/test_ci.py | 1 - 57 files changed, 11 insertions(+), 96 deletions(-) diff --git a/platformio/account/client.py b/platformio/account/client.py index 7aabb24dc0..366e4ad5b6 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -21,22 +21,18 @@ class AccountError(PlatformioException): - MESSAGE = "{0}" class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError): - MESSAGE = "You are already authorized with {0} account." class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods - SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 57b8650002..c44e72b74f 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -25,7 +25,6 @@ class InoToCPPConverter: - PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index c0e2be593e..f95f6735bf 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -109,7 +109,6 @@ def get_used_frameworks(env, path): class LibBuilderBase: - CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() # Max depth of nested includes: diff --git a/platformio/check/defect.py b/platformio/check/defect.py index 9efcad9096..5689a18478 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -24,7 +24,6 @@ class DefectItem: - SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 SEVERITY_LOW = 4 diff --git a/platformio/cli.py b/platformio/cli.py index bdb46a3d40..41aab522e1 100644 --- a/platformio/cli.py +++ b/platformio/cli.py @@ -19,7 +19,6 @@ class PlatformioCLI(click.MultiCommand): - leftover_args = [] def __init__(self, *args, **kwargs): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 8ca008f13a..ac8367da90 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -42,7 +42,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards[board["platform"]].append(board) terminal_width = shutil.get_terminal_size().columns - for (platform, boards) in sorted(grpboards.items()): + for platform, boards in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) click.secho(platform, bold=True) diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index 83d98d9c4c..370fdb7ebd 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -18,7 +18,6 @@ class BlackmagicDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target set language c diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py index 1f155ecbb9..faa5a3413e 100644 --- a/platformio/debug/config/generic.py +++ b/platformio/debug/config/generic.py @@ -16,7 +16,6 @@ class GenericDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset halt diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py index 03a1bd3ac3..a820e0aa7a 100644 --- a/platformio/debug/config/jlink.py +++ b/platformio/debug/config/jlink.py @@ -16,7 +16,6 @@ class JlinkDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 09266b3b9e..7f4d8e1abb 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -16,7 +16,6 @@ class MspdebugDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py index be15b5f46c..413a9c62df 100644 --- a/platformio/debug/config/native.py +++ b/platformio/debug/config/native.py @@ -17,7 +17,6 @@ class NativeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py index e9a57409aa..b998c78217 100644 --- a/platformio/debug/config/qemu.py +++ b/platformio/debug/config/qemu.py @@ -16,7 +16,6 @@ class QemuDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor system_reset diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py index 724be40726..f2f62d66fb 100644 --- a/platformio/debug/config/renode.py +++ b/platformio/debug/config/renode.py @@ -16,7 +16,6 @@ class RenodeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor machine Reset diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py index a1269b2fd4..7f4d0f4cf1 100644 --- a/platformio/debug/exception.py +++ b/platformio/debug/exception.py @@ -20,7 +20,6 @@ class DebugError(PlatformioException): class DebugSupportError(DebugError, UserSideException): - MESSAGE = ( "Currently, PlatformIO does not support debugging for `{0}`.\n" "Please request support at https://github.com/platformio/" diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index a2d89de328..23f4fff05c 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -31,7 +31,6 @@ class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods - STDOUT = sys.stdout def write(self, text): diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index 2c9280c26f..6ba8c70ec3 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -53,7 +53,6 @@ def process_exited(self): class DebugBaseProcess: - STDOUT_CHUNK_SIZE = 2048 LOG_FILE = None diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 4ce9aebe62..ce7e82c8bf 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -24,7 +24,6 @@ class GDBClientProcess(DebugClientProcess): - PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 89b4095ffc..693dfa445d 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -26,7 +26,6 @@ class DebugServerProcess(DebugBaseProcess): - STD_BUFFER_SIZE = 1024 def __init__(self, debug_config): diff --git a/platformio/exception.py b/platformio/exception.py index a8287c047f..c9afd751ea 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -14,7 +14,6 @@ class PlatformioException(Exception): - MESSAGE = None def __str__(self): # pragma: no cover @@ -26,7 +25,6 @@ def __str__(self): # pragma: no cover class ReturnErrorCode(PlatformioException): - MESSAGE = "{0}" @@ -35,7 +33,6 @@ class UserSideException(PlatformioException): class AbortedByUser(UserSideException): - MESSAGE = "Aborted by user" @@ -49,7 +46,6 @@ class InvalidUdevRules(UserSideException): class MissedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Please install `99-platformio-udev.rules`. \nMore details: " "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" @@ -57,7 +53,6 @@ class MissedUdevRules(InvalidUdevRules): class OutdatedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." "\nMore details: " @@ -71,32 +66,26 @@ class OutdatedUdevRules(InvalidUdevRules): class GetSerialPortsError(PlatformioException): - MESSAGE = "No implementation for your platform ('{0}') available" class GetLatestVersionError(PlatformioException): - MESSAGE = "Can not retrieve the latest PlatformIO version" class InvalidSettingName(UserSideException): - MESSAGE = "Invalid setting with the name '{0}'" class InvalidSettingValue(UserSideException): - MESSAGE = "Invalid value '{0}' for the setting '{1}'" class InvalidJSONFile(PlatformioException): - MESSAGE = "Could not load broken JSON: {0}" class CIBuildEnvsEmpty(UserSideException): - MESSAGE = ( "Can't find PlatformIO build environments.\n" "Please specify `--board` or path to `platformio.ini` with " @@ -105,7 +94,6 @@ class CIBuildEnvsEmpty(UserSideException): class UpgradeError(PlatformioException): - MESSAGE = """{0} * Upgrade using `pip install -U platformio` @@ -115,7 +103,6 @@ class UpgradeError(PlatformioException): class HomeDirPermissionsError(UserSideException): - MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " "current user and PlatformIO can not store configuration data.\n" @@ -126,7 +113,6 @@ class HomeDirPermissionsError(UserSideException): class CygwinEnvDetected(PlatformioException): - MESSAGE = ( "PlatformIO does not work within Cygwin environment. " "Use native Terminal instead." diff --git a/platformio/fs.py b/platformio/fs.py index bd17950b2b..00055299b7 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -181,7 +181,7 @@ def _find_candidates(pattern): result = set() # correct fs directory separator src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) - for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): + for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter): candidates = _find_candidates(pattern) if action == "+": result |= candidates diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 9d7fc59048..dbeafbdbd6 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -21,7 +21,6 @@ class AppRPC: - IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index 5ff9fe3321..d08a821534 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -21,7 +21,6 @@ class IDERPC: - COMMAND_TIMEOUT = 1.5 # in seconds def __init__(self): diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 99a5325e57..0917a5f4b0 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -24,8 +24,8 @@ from platformio import __main__, __version__, fs, proc from platformio.compat import get_locale_encoding, is_bytes -from platformio.home import helpers from platformio.exception import PlatformioException +from platformio.home import helpers class MultiThreadingStdStream: diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index 6c0abdb45d..88f06704f9 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -24,7 +24,6 @@ class JSONRPCServerFactoryBase: - connection_nums = 0 shutdown_timer = None diff --git a/platformio/http.py b/platformio/http.py index 501d10783c..48235f7422 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -38,7 +38,6 @@ def __str__(self): # pragma: no cover class InternetConnectionError(UserSideException): - MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index 9d538b7daa..dfd2559f76 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -137,7 +137,7 @@ def list_global_packages(options): only_packages = any( options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data ) - for (type_, pm) in data: + for type_, pm in data: skip_conds = [ only_packages and not options.get(type_) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 580137a00f..d38c0e602b 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -48,12 +48,10 @@ def __str__(self): class MissingPackageManifestError(ManifestException): - MESSAGE = "Could not find one of '{0}' manifest files in the package" class UnknownPackageError(UserSideException): - MESSAGE = ( "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() @@ -61,7 +59,6 @@ class UnknownPackageError(UserSideException): class NotGlobalLibDir(UserSideException): - MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 9d9c611819..d1ffe710ab 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -26,7 +26,6 @@ class PackageManagerDownloadMixin: - DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index c89d7b861d..fc4d5ddbc6 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -27,7 +27,6 @@ class PackageManagerInstallMixin: - _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod diff --git a/platformio/package/meta.py b/platformio/package/meta.py index fbd2b7343f..4db5310607 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -65,7 +65,6 @@ def from_archive(cls, path): class PackageCompatibility: - KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") @classmethod @@ -468,7 +467,6 @@ def load(path): class PackageItem: - METAFILE_NAME = ".piopm" def __init__(self, path, metadata=None): diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index e39222c29e..35e0da6879 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -24,7 +24,6 @@ class ExtractArchiveItemError(PackageException): - MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> https://bit.ly/faq-package-manager" diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index b4dafc0319..058a4983f7 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -58,7 +58,6 @@ def new(src_dir, remote_url, silent=False): class VCSClientBase: - command = None def __init__(self, src_dir, remote_url=None, tag=None, silent=False): @@ -128,7 +127,6 @@ def get_cmd_output(self, args, **kwargs): class GitClient(VCSClientBase): - command = "git" _configured = False @@ -232,7 +230,6 @@ def get_latest_revision(self): class HgClient(VCSClientBase): - command = "hg" def export(self): @@ -256,7 +253,6 @@ def get_latest_revision(self): class SvnClient(VCSClientBase): - command = "svn" def export(self): diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 91c909314b..0c157f8520 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -28,7 +28,6 @@ class PlatformRunMixin: - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) @staticmethod diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 49db0df45c..23a66929b7 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -29,7 +29,6 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods PlatformPackagesMixin, PlatformRunMixin ): - CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index 604c322876..f044d3b7a7 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -20,12 +20,10 @@ class PlatformException(PlatformioException): class UnknownPlatform(PlatformException): - MESSAGE = "Unknown development platform '{0}'" class IncompatiblePlatform(PlatformException): - MESSAGE = ( "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " "depends on PlatformIO Core {2}.\n" @@ -33,20 +31,16 @@ class IncompatiblePlatform(PlatformException): class UnknownBoard(PlatformException): - MESSAGE = "Unknown board ID '{0}'" class InvalidBoardManifest(PlatformException): - MESSAGE = "Invalid board JSON manifest '{0}'" class UnknownFramework(PlatformException): - MESSAGE = "Unknown framework '{0}'" class BuildScriptNotFound(PlatformException): - MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index ca57de34bd..6a545df469 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -166,7 +166,7 @@ def init_base_project(project_dir): (config.get("platformio", "lib_dir"), init_lib_readme), (config.get("platformio", "test_dir"), init_test_readme), ] - for (path, cb) in dir_to_readme: + for path, cb in dir_to_readme: if os.path.isdir(path): continue os.makedirs(path) diff --git a/platformio/project/config.py b/platformio/project/config.py index ee380bb943..43d15f3e26 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -39,7 +39,6 @@ class ProjectConfigBase: - INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -412,7 +411,6 @@ def get_optional_dir(self, name): class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): - _instances = {} @staticmethod diff --git a/platformio/project/exception.py b/platformio/project/exception.py index aa45eb07c4..8e598e038e 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -20,7 +20,6 @@ class ProjectError(PlatformioException): class NotPlatformIOProjectError(ProjectError, UserSideException): - MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " @@ -29,25 +28,20 @@ class NotPlatformIOProjectError(ProjectError, UserSideException): class InvalidProjectConfError(ProjectError, UserSideException): - MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" class UndefinedEnvPlatformError(ProjectError, UserSideException): - MESSAGE = "Please specify platform for '{0}' environment" class ProjectEnvsNotAvailableError(ProjectError, UserSideException): - MESSAGE = "Please setup environments in `platformio.ini` file" class UnknownEnvNamesError(ProjectError, UserSideException): - MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" class ProjectOptionValueError(ProjectError, UserSideException): - MESSAGE = "{0} for option `{1}` in section [{2}]" diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index 4235e88d4f..4b4508f662 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -22,7 +22,6 @@ class RegistryFileMirrorIterator: - HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index 8105062c70..0c631c64cc 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -17,7 +17,6 @@ class AsyncCommandBase: - MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb def __init__(self, options=None, on_end_callback=None): diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index bede41d160..91f5e29ddf 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -118,7 +118,6 @@ def remote_run( silent, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( @@ -211,7 +210,6 @@ def remote_test( # pylint: disable=redefined-builtin without_uploading, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( diff --git a/platformio/remote/client/async_base.py b/platformio/remote/client/async_base.py index 488802a583..2a67264132 100644 --- a/platformio/remote/client/async_base.py +++ b/platformio/remote/client/async_base.py @@ -34,7 +34,7 @@ def agent_pool_ready(self): def cb_async_result(self, result): if self._acs_total == 0: self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) self.acread_data(*value) diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py index fe2c4fb4c9..cf5c240537 100644 --- a/platformio/remote/client/base.py +++ b/platformio/remote/client/base.py @@ -33,7 +33,6 @@ class RemoteClientBase( # pylint: disable=too-many-instance-attributes pb.Referenceable ): - PING_DELAY = 60 PING_MAX_FAILURES = 3 DEBUG = False diff --git a/platformio/remote/client/device_list.py b/platformio/remote/client/device_list.py index a22911b461..f05bbfd27c 100644 --- a/platformio/remote/client/device_list.py +++ b/platformio/remote/client/device_list.py @@ -32,7 +32,7 @@ def agent_pool_ready(self): def _cbResult(self, result): data = {} - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py index 46ca19e4ad..1499dd5866 100644 --- a/platformio/remote/client/device_monitor.py +++ b/platformio/remote/client/device_monitor.py @@ -71,7 +71,6 @@ def send_to_server(self, data): class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes RemoteClientBase ): - MAX_BUFFER_SIZE = 1024 * 1024 def __init__(self, agents, **kwargs): @@ -96,7 +95,7 @@ def agent_pool_ready(self): def _cb_device_list(self, result): devices = [] hwid_devindexes = [] - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/run_or_test.py b/platformio/remote/client/run_or_test.py index 71065640f7..d23d92c737 100644 --- a/platformio/remote/client/run_or_test.py +++ b/platformio/remote/client/run_or_test.py @@ -28,7 +28,6 @@ class RunOrTestClient(AsyncClientBase): - MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb @@ -147,7 +146,7 @@ def psync_init(self): def cb_psync_init_result(self, result): self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) agent_id, ac_id = value diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py index 820034ae26..1715145fde 100644 --- a/platformio/remote/projectsync.py +++ b/platformio/remote/projectsync.py @@ -47,13 +47,13 @@ def get_items(self): def rebuild_dbindex(self): self._db = {} - for (path, relpath, cb_filter) in self.items: + for path, relpath, cb_filter in self.items: if cb_filter and not cb_filter(path): continue self._insert_to_db(path, relpath) if not isdir(path): continue - for (root, _, files) in os.walk(path, followlinks=True): + for root, _, files in os.walk(path, followlinks=True): for name in files: self._insert_to_db( join(root, name), join(relpath, root[len(path) + 1 :], name) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 211c0beac6..0798641955 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -54,7 +54,6 @@ def send(self, hittype): class MeasurementProtocol(TelemetryBase): - TID = "UA-1768265-9" PARAMS_MAP = { "screen_name": "cd", @@ -201,7 +200,6 @@ def send(self, hittype): @util.singleton class MPDataPusher: - MAX_WORKERS = 5 def __init__(self): diff --git a/platformio/test/exception.py b/platformio/test/exception.py index 12b76ea491..14cbe5acf3 100644 --- a/platformio/test/exception.py +++ b/platformio/test/exception.py @@ -20,7 +20,6 @@ class UnitTestError(PlatformioException): class TestDirNotExistsError(UnitTestError, UserSideException): - MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " "directory in the project root and put a test suite.\n" diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index 01650a0b1d..25551ef664 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -40,7 +40,6 @@ def list_test_suites(project_config, environments, filters, ignores): test_names = list_test_names(project_config) for env_name in project_config.envs(): for test_name in test_names: - # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) for key, value in patterns.items(): diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 8f9136f126..78425a766c 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -54,7 +54,6 @@ def __init__( # pylint: disable=too-many-arguments class TestRunnerBase: - NAME = None EXTRA_LIB_DEPS = None TESTCASE_PARSE_RE = None diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index 52a0916fa2..dd30bc925a 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -101,7 +101,6 @@ def _parse_assert(self, line): class DoctestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index b299002e3c..8a4d1b15ae 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -22,7 +22,6 @@ class GoogletestTestCaseParser: - # Examples: # [ RUN ] FooTest.Bar # ... @@ -89,7 +88,6 @@ def _parse_source_and_message(self, stdout): class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index ab35775b39..2991691578 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -22,7 +22,6 @@ class SerialTestOutputReader: - SERIAL_TIMEOUT = 600 def __init__(self, test_runner): diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index b7d8026be5..1c8ef795bf 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -26,7 +26,6 @@ class UnityTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.5.2"] # Examples: diff --git a/platformio/util.py b/platformio/util.py index 63a16d6ed6..c981384ad8 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -96,7 +96,6 @@ class RetryStopException(RetryException): class retry: - RetryNextException = RetryNextException RetryStopException = RetryStopException diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index ae63228001..f67bada4d9 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -112,7 +112,6 @@ def test_ci_keep_build_dir_single_src_dir( def test_ci_keep_build_dir_nested_src_dirs( clirunner, tmpdir_factory, validate_cliresult ): - build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) # Split default Arduino project in two parts From 1422b772985daa3cdfcb3d02e83fb77e8fea1084 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 17:46:27 +0200 Subject: [PATCH 14/98] Allow extra path when fetching package data --- platformio/registry/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/platformio/registry/client.py b/platformio/registry/client.py index a682db5720..1a89e345a2 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -142,12 +142,15 @@ def list_packages(self, query=None, qualifiers=None, page=None, sort=None): x_with_authorization=self.allowed_private_packages(), ) - def get_package(self, type_, owner, name, version=None): + def get_package(self, type_, owner, name, version=None, extra_path=None): try: return self.fetch_json_data( "get", - "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=name.lower() + "/v3/packages/{owner}/{type}/{name}{extra_path}".format( + type=type_, + owner=owner.lower(), + name=name.lower(), + extra_path=extra_path or "", ), params=dict(version=version) if version else None, x_cache_valid="1h", From 7c650c2c087646e03f88bd02cc4b124456846438 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 18:14:05 +0200 Subject: [PATCH 15/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ba58a6e7cd..7074fd6f31 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ba58a6e7cd4138cbcdcca801cc251bbd2871ced2 +Subproject commit 7074fd6f31215f759cbbc9ad51a88545e86807c4 From 9b4a04541323ccff6a77231895d5bfd15e1f9785 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 18:14:15 +0200 Subject: [PATCH 16/98] Restored project generator for NetBeans IDE --- HISTORY.rst | 1 + setup.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5c47efcd82..9d4b7277ab 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 6 * Show detailed library dependency tree only in the `verbose mode `__ (`issue #4517 `_) * Prevented shell injection when converting INO file to CPP (`issue #4532 `_) +* Restored project generator for `NetBeans IDE `__ 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index 2e1fa86c0e..c04b7fdbfe 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,8 @@ "assets/templates/ide-projects/*/*.tpl", "assets/templates/ide-projects/*/.*.tpl", # include hidden files "assets/templates/ide-projects/*/.*/*.tpl", # include hidden folders + "assets/templates/ide-projects/*/*/*.tpl", # NetBeans + "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans ] }, entry_points={ From c785c8c6f3838d5c68ad334d422bf1177190169d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 18:14:44 +0200 Subject: [PATCH 17/98] Bump version to 6.1.7a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6872e7608e..d2136573cb 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "7a2") +VERSION = (6, 1, "7a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From efbe3d4aa67db1997ab9b763bf9658ad0639abd7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Feb 2023 19:35:40 +0200 Subject: [PATCH 18/98] PyLint fix --- Makefile | 2 +- tests/package/test_manifest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 027af3cac7..09c0b8386a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ lint: - pylint --rcfile=./.pylintrc ./platformio pylint --rcfile=./.pylintrc ./tests + pylint --rcfile=./.pylintrc ./platformio isort: isort ./platformio diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 828412f4f6..a3279e051d 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -901,5 +901,5 @@ def test_broken_schemas(): ) # invalid package name - with pytest.raises(ManifestValidationError, match=("are not allowed")): + with pytest.raises(ManifestValidationError, match="are not allowed"): ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) From 2c4055d9e1e41ad9d6c78bdd46bf57bbc75368d5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Feb 2023 14:33:19 +0200 Subject: [PATCH 19/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7074fd6f31..f712eddd90 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7074fd6f31215f759cbbc9ad51a88545e86807c4 +Subproject commit f712eddd905d307793b1e91e615f8a83c30fb9ed From 4c4fb5029eaa01d07a66d770d18d1e178b02e251 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Feb 2023 15:55:33 +0200 Subject: [PATCH 20/98] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index f712eddd90..041dbd7174 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f712eddd905d307793b1e91e615f8a83c30fb9ed +Subproject commit 041dbd7174de184b27cbcc468bc0acb1679b1082 diff --git a/examples b/examples index b7900244c8..894dbd36d9 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit b7900244c8ec2844dfea80b78b1dafd67855a8fd +Subproject commit 894dbd36d97534267829dc2992e8253f34c2be45 From 598d2b24debc672f05424e2ace64bac15b6dfc33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Feb 2023 22:44:00 +0200 Subject: [PATCH 21/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 041dbd7174..763cdb1129 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 041dbd7174de184b27cbcc468bc0acb1679b1082 +Subproject commit 763cdb11297767c46c88cb2e5e0c661e8353f07e From 674d00183e020d5d80a81d295f78e600af7cb09b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 11 Feb 2023 22:07:48 +0200 Subject: [PATCH 22/98] Update deps --- .github/workflows/docs.yml | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2765b55d55..9a64d88295 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,7 +65,7 @@ jobs: mkdir ./${{ env.LATEST_DOCS_DIR }} tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} - name: Delete Artifact - uses: geekyeggo/delete-artifact@v1 + uses: geekyeggo/delete-artifact@v2 with: name: docs - name: Select Docs type diff --git a/setup.py b/setup.py index c04b7fdbfe..745bebce54 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,9 @@ ] home_requirements = [ - "aiofiles==%s" % ("0.8.0" if PY36 else "22.1.*"), + "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.23.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.24.*"), "uvicorn==%s" % ("0.16.0" if PY36 else "0.20.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From 5b13aeda5261f0048bbe9b257f5533708c3e713a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Feb 2023 13:48:20 +0200 Subject: [PATCH 23/98] Update deps --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 745bebce54..2c3d224523 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ home_requirements = [ "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.24.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.25.*"), "uvicorn==%s" % ("0.16.0" if PY36 else "0.20.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From 56cc7ce270493dcd056606278376b0e5dfded750 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Feb 2023 13:48:41 +0200 Subject: [PATCH 24/98] Update SPDX License List to v3.20 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 937bbf91df..2075ac0381 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -276,7 +276,7 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.19" + version = "3.20" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version From cabe7d8c112bd49bd3dad6a17f3f2487a4c5e527 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Feb 2023 10:02:59 -0700 Subject: [PATCH 25/98] Docs: recommend specific version for a dev platform --- scripts/docspregen.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/docspregen.py b/scripts/docspregen.py index f895c25d84..b0fe2adee5 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -473,11 +473,12 @@ def generate_platform(pkg, rst_dir): .. code-block:: ini - ; Latest stable version + ; Latest stable version, NOT recommended + ; Pin the version as shown below [env:latest_stable] platform = {name} {board} - ; Custom stable version + ; Specific version [env:custom_stable] platform = {name}@x.y.z {board} From 8794b2a3a1b1665dcf05f15a1fbabcc3f4f9de1b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Feb 2023 10:03:24 -0700 Subject: [PATCH 26/98] Docs: Update "sphinx-rtd-theme" to 1.2.0 --- docs | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 763cdb1129..27eae2bd13 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 763cdb11297767c46c88cb2e5e0c661e8353f07e +Subproject commit 27eae2bd13973d1b03f58c8397f6652742b7e5de diff --git a/tox.ini b/tox.ini index 0a51741a44..608328066a 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.1.1 + sphinx-rtd-theme==1.2.0 sphinx-notfound-page sphinx-copybutton restructuredtext-lint From c7060f93e8e0600b4384729c31036fa6739646ad Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Feb 2023 13:18:57 -0700 Subject: [PATCH 27/98] Docs: Add link to the Serial & UDP Plotter --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 27eae2bd13..4e6814d678 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 27eae2bd13973d1b03f58c8397f6652742b7e5de +Subproject commit 4e6814d6787f0a0cd92ba72795bc5838d91a6f89 From e39438791c641b6bf8d2490ee5fc9a5f982ce9bc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 Feb 2023 09:47:01 -0700 Subject: [PATCH 28/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4e6814d678..239b55f227 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4e6814d6787f0a0cd92ba72795bc5838d91a6f89 +Subproject commit 239b55f227a05e805c125462697727112829f9d1 From 3dfb936f3cd83a42fa46c666b35dfe785d68ae6c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Mar 2023 11:38:36 -0700 Subject: [PATCH 29/98] Use query params for DELETE request --- platformio/account/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/account/client.py b/platformio/account/client.py index 366e4ad5b6..a898de481d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -294,7 +294,7 @@ def remove_org_owner(self, orgname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/owners" % orgname, - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) @@ -347,6 +347,6 @@ def remove_team_member(self, orgname, teamname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) From 07bfa8ce4aa59b71207fb9fad3ba7712cfc144ab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Mar 2023 11:16:40 -0700 Subject: [PATCH 30/98] Docs: Sync dev-platforms --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 239b55f227..b33e6c0312 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 239b55f227a05e805c125462697727112829f9d1 +Subproject commit b33e6c03121dff17222d93331e5ad8d6ca7f3f7a From a387f9708ab779d723236004845146b3b4e143c7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Mar 2023 14:47:47 -0700 Subject: [PATCH 31/98] PyLint fix --- platformio/test/runners/readers/serial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index 2991691578..a9732d7fe6 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -46,7 +46,7 @@ def begin(self): ser.open() except serial.SerialException as exc: click.secho(str(exc), fg="red", err=True) - return None + return if not self.test_runner.options.no_reset: ser.flushInput() From c3e287672ecfd9b3878b34bb82028ce01b4b0676 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Mar 2023 12:51:13 -0700 Subject: [PATCH 32/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b33e6c0312..67c016f8b7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b33e6c03121dff17222d93331e5ad8d6ca7f3f7a +Subproject commit 67c016f8b7f79d2ca5c8cb8361887da8ff4fb240 From 3e6725bb5f7eaf05f20db0b5b4420d002468560a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Mar 2023 15:29:04 -0600 Subject: [PATCH 33/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 67c016f8b7..e9b309b593 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 67c016f8b7f79d2ca5c8cb8361887da8ff4fb240 +Subproject commit e9b309b59307a0da69a0d28b2e19fc5f25dc1b27 From d7c9dc24116f02c27f327cc8cac73535e746cc58 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Mar 2023 20:13:54 -0600 Subject: [PATCH 34/98] Update deps --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2c3d224523..6e1a243f11 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ home_requirements = [ "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.25.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.20.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), + "uvicorn==%s" % ("0.16.0" if PY36 else "0.21.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From f2d206ca54a5d5011fcb80b741797f3253c752b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Mar 2023 12:40:11 -0600 Subject: [PATCH 35/98] Fixed "RuntimeError: deque mutated during iteration" --- platformio/builder/tools/piobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 32a744e338..ba194eb076 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -239,7 +239,7 @@ def ProcessUnFlags(env, flags): for scope in unflag_scopes: for unflags in parsed.values(): for unflag in unflags: - for current in env.get(scope, []): + for current in list(env.get(scope, [])): conditions = [ unflag == current, not isinstance(unflag, (tuple, list)) From d9ff250f82dd15dcc6fe56f3ec3a79a4dc6eee27 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Sun, 19 Mar 2023 00:45:59 +0200 Subject: [PATCH 36/98] Improved file filtering for the Static Analysis feature (#4570) * Improved file filtering for the Static Analysis feature * Better handling of legacy "check_patterns" option * Rename "check_src_filter" to plural form to better represent functionality * Move to plural forms of filter variables --- HISTORY.rst | 1 + platformio/check/cli.py | 28 +++- platformio/check/tools/base.py | 44 ++++-- platformio/check/tools/clangtidy.py | 4 +- platformio/check/tools/cppcheck.py | 10 +- platformio/check/tools/pvsstudio.py | 2 +- platformio/project/options.py | 3 +- tests/commands/test_check.py | 208 +++++++++++++++++++++------- 8 files changed, 224 insertions(+), 76 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9d4b7277ab..57126d547b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 6 * Show detailed library dependency tree only in the `verbose mode `__ (`issue #4517 `_) * Prevented shell injection when converting INO file to CPP (`issue #4532 `_) * Restored project generator for `NetBeans IDE `__ +* Improved source file filtering functionality for `Static Code Analysis `__ feature 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/check/cli.py b/platformio/check/cli.py index e293e8e02d..215a6b8ade 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -49,7 +49,12 @@ exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True ), ) -@click.option("--pattern", multiple=True) +@click.option("--pattern", multiple=True, hidden=True) +@click.option( + "-f", + "--src-filters", + multiple=True +) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) @@ -67,6 +72,7 @@ def cli( environment, project_dir, project_conf, + src_filters, pattern, flags, severity, @@ -105,14 +111,24 @@ def cli( "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) ) - default_patterns = [ - config.get("platformio", "src_dir"), - config.get("platformio", "include_dir"), + default_src_filters = [ + "+<%s>" % os.path.basename(config.get("platformio", "src_dir")), + "+<%s>" % os.path.basename(config.get("platformio", "include_dir")), ] + + src_filters = ( + src_filters + or pattern + or env_options.get( + "check_src_filters", + env_options.get("check_patterns", default_src_filters), + ) + ) + tool_options = dict( verbose=verbose, silent=silent, - patterns=pattern or env_options.get("check_patterns", default_patterns), + src_filters=src_filters, flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent @@ -265,7 +281,7 @@ def print_defects_stats(results): tabular_data.append(total) headers = ["Component"] - headers.extend([l.upper() for l in severity_labels]) + headers.extend([label.upper() for label in severity_labels]) headers = [click.style(h, bold=True) for h in headers] click.echo(tabulate(tabular_data, headers=headers, numalign="center")) click.echo() diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index d51b315880..392a19af0d 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os import tempfile @@ -30,6 +29,7 @@ def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname self.options = options + self.project_dir = project_dir self.cc_flags = [] self.cxx_flags = [] self.cpp_includes = [] @@ -41,7 +41,7 @@ def __init__(self, project_dir, config, envname, options): self._defects = [] self._on_defect_callback = None self._bad_input = False - self._load_cpp_data(project_dir) + self._load_cpp_data() # detect all defects by default if not self.options.get("severity"): @@ -56,8 +56,8 @@ def __init__(self, project_dir, config, envname, options): for s in self.options["severity"] ] - def _load_cpp_data(self, project_dir): - data = load_build_metadata(project_dir, self.envname) + def _load_cpp_data(self): + data = load_build_metadata(self.project_dir, self.envname) if not data: return self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", "")) @@ -99,6 +99,13 @@ def _extract_defines(language, includes_file): includes_file, ) result = proc.exec_command(cmd, shell=True) + + if result["returncode"] != 0: + click.echo("Warning: Failed to extract toolchain defines!") + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": @@ -201,7 +208,7 @@ def execute_check_cmd(self, cmd): return result @staticmethod - def get_project_target_files(patterns): + def get_project_target_files(project_dir, src_filters): c_extension = (".c",) cpp_extensions = (".cc", ".cpp", ".cxx", ".ino") header_extensions = (".h", ".hh", ".hpp", ".hxx") @@ -216,13 +223,9 @@ def _add_file(path): elif path.endswith(cpp_extensions): result["c++"].append(os.path.abspath(path)) - for pattern in patterns: - for item in glob.glob(pattern, recursive=True): - if not os.path.isdir(item): - _add_file(item) - for root, _, files in os.walk(item, followlinks=True): - for f in files: - _add_file(os.path.join(root, f)) + src_filters = normalize_src_filters(src_filters) + for f in fs.match_src_files(project_dir, src_filters): + _add_file(f) return result @@ -243,3 +246,20 @@ def check(self, on_defect_callback=None): self.clean_up() return self._bad_input + + +# +# Helpers +# + + +def normalize_src_filters(src_filters): + def _normalize(src_filters): + return ( + src_filters if src_filters.startswith(("+<", "-<")) else "+<%s>" % src_filters + ) + + if isinstance(src_filters, (list, tuple)): + return " ".join([_normalize(f) for f in src_filters]) + + return _normalize(src_filters) diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 1dd4165eed..496acc7c89 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -64,7 +64,9 @@ def configure_command(self): ): cmd.append("--checks=*") - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files = [] for items in project_files.values(): diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index 0f8db40225..c3c595373c 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -96,7 +96,7 @@ def parse_defect(self, raw_line): # pylint: disable=arguments-differ ) click.echo() self._bad_input = True - self._buffer = "" + self._buffer = "" return None self._buffer = "" @@ -214,7 +214,9 @@ def clean_up(self): if not self.is_flag_set("--addon", self.get_flags("cppcheck")): return - for files in self.get_project_target_files(self.options["patterns"]).values(): + for files in self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ).values(): for f in files: dump_file = f + ".dump" if os.path.isfile(dump_file): @@ -243,7 +245,9 @@ def convert_language_standard(flag): def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files_scope = ("c", "c++") if not any(project_files[t] for t in src_files_scope): click.echo("Error: Nothing to check.") diff --git a/platformio/check/tools/pvsstudio.py b/platformio/check/tools/pvsstudio.py index ded65d1cb6..20979a0019 100644 --- a/platformio/check/tools/pvsstudio.py +++ b/platformio/check/tools/pvsstudio.py @@ -227,7 +227,7 @@ def is_check_successful(cmd_result): def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback for scope, files in self.get_project_target_files( - self.options["patterns"] + self.project_dir, self.options["src_filters"] ).items(): if scope not in ("c", "c++"): continue diff --git a/platformio/project/options.py b/platformio/project/options.py index 5e200629f4..53775b5d5d 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -649,7 +649,8 @@ def get_default_core_dir(): ), ConfigEnvOption( group="check", - name="check_patterns", + name="check_src_filters", + oldnames=["check_patterns"], description=( "Configure a list of target files or directories for checking " "(Unix shell-style wildcards)" diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index be6042ca18..cb6f77772c 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -15,8 +15,8 @@ # pylint: disable=redefined-outer-name import json +import os import sys -from os.path import isfile, join import pytest @@ -83,12 +83,12 @@ def check_dir(tmpdir_factory): def count_defects(output): error, warning, style = 0, 0, 0 - for l in output.split("\n"): - if "[high:error]" in l: + for line in output.split("\n"): + if "[high:error]" in line: error += 1 - elif "[medium:warning]" in l: + elif "[medium:warning]" in line: warning += 1 - elif "[low:style]" in l: + elif "[low:style]" in line: style += 1 return error, warning, style @@ -240,9 +240,9 @@ def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) inc_count = 0 - for l in result.output.split("\n"): - if l.startswith("Includes:"): - inc_count = l.count("-I") + for line in result.output.split("\n"): + if line.startswith("Includes:"): + inc_count = line.count("-I") # at least 1 include path for default mode assert inc_count > 0 @@ -259,46 +259,6 @@ def test_check_silent_mode(clirunner, validate_cliresult, check_dir): assert style == 0 -def test_check_custom_pattern_absolute_path( - clirunner, validate_cliresult, tmpdir_factory -): - project_dir = tmpdir_factory.mktemp("project") - project_dir.join("platformio.ini").write(DEFAULT_CONFIG) - - check_dir = tmpdir_factory.mktemp("custom_src_dir") - check_dir.join("main.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors == EXPECTED_ERRORS - assert warnings == EXPECTED_WARNINGS - assert style == EXPECTED_STYLE - - -def test_check_custom_pattern_relative_path( - clirunner, validate_cliresult, tmpdir_factory -): - tmpdir = tmpdir_factory.mktemp("project") - tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - - tmpdir.mkdir("app").join("main.cpp").write(TEST_CODE) - tmpdir.mkdir("prj").join("test.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors + warnings + style == EXPECTED_DEFECTS * 2 - - def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -427,7 +387,7 @@ def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, check_dir): validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output - assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) + assert not os.path.isfile(os.path.join(str(check_dir), "src", "main.cpp.dump")) def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): @@ -607,10 +567,10 @@ def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in result.output.split("\n"): - if not l.startswith("Includes:"): + for line in result.output.split("\n"): + if not line.startswith("Includes:"): continue - for inc in l.split(" "): + for inc in line.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) @@ -656,3 +616,147 @@ def test_check_handles_spaces_in_paths(clirunner, validate_cliresult, tmpdir_fac default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(default_result) + + +# +# Files filtering functionality +# + + +@pytest.mark.parametrize( + "src_filter,number_of_checked_files", + [ + (["+"], 1), + (["+"], 1), + (["+", "-"], 2), + (["-<*> + + +"], 3), + ], + ids=["Single file", "Glob pattern", "Exclude pattern", "Filter as string"], +) +def test_check_src_filter( + clirunner, + validate_cliresult, + tmpdir_factory, + src_filter, + number_of_checked_files, +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.join("app.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + cmd_args = ["--project-dir", str(tmpdir)] + [ + "--src-filters=%s" % f for f in src_filter + ] + + result = clirunner.invoke(cmd_check, cmd_args) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * number_of_checked_files + + +def test_check_src_filter_from_config(clirunner, validate_cliresult, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_src_filters = + + + + + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output + + +def test_check_custom_pattern_absolute_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + check_dir = tmpdir_factory.mktemp("custom_src_dir") + check_dir.join("main.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] + ) + + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == EXPECTED_WARNINGS + assert style == EXPECTED_STYLE + + +def test_check_custom_pattern_relative_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, + ["--project-dir", str(tmpdir), "--pattern=src/uart", "--pattern=src/spi"], + ) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + + +def test_check_src_filter_from_config_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_patterns = + src/spi/*.c* + tests/test.cpp + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output From 8d33a3d151af9a22dc4c26bd4961884e858f0fb3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Mar 2023 22:41:14 -0600 Subject: [PATCH 37/98] Do not build project when only "monitor" target is passed --- HISTORY.rst | 9 ++-- docs | 2 +- platformio/run/cli.py | 123 +++++++++++++++++++++--------------------- 3 files changed, 67 insertions(+), 67 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 57126d547b..317e9a4baa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,10 +17,11 @@ PlatformIO Core 6 6.1.7 (2023-??-??) ~~~~~~~~~~~~~~~~~~ -* Show detailed library dependency tree only in the `verbose mode `__ (`issue #4517 `_) -* Prevented shell injection when converting INO file to CPP (`issue #4532 `_) -* Restored project generator for `NetBeans IDE `__ -* Improved source file filtering functionality for `Static Code Analysis `__ feature +* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to +* Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) +* Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process +* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) +* Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index e9b309b593..ea46b38c80 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e9b309b59307a0da69a0d28b2e19fc5f25dc1b27 +Subproject commit ea46b38c806aff3f944ffba3de776cf7e1a03947 diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 6c64743156..bf237ac956 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -104,6 +104,8 @@ def cli( is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta results = [] + only_monitor = "monitor" in target and len(target) == 1 + command_failed = False with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) @@ -111,65 +113,75 @@ def cli( if list_targets: return print_target_list(list(environment) or config.envs()) - # clean obsolete build dir - if not disable_auto_clean: - build_dir = config.get("platformio", "build_dir") - try: - clean_build_dir(build_dir, config) - except ProjectError as exc: - raise exc - except: # pylint: disable=bare-except - click.secho( - "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % build_dir, - fg="yellow", - ) - - handle_legacy_libdeps(project_dir, config) + if not only_monitor: + # clean obsolete build dir + if not disable_auto_clean: + build_dir = config.get("platformio", "build_dir") + try: + clean_build_dir(build_dir, config) + except ProjectError as exc: + raise exc + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % build_dir, + fg="yellow", + ) - default_envs = config.default_envs() - for env in config.envs(): - skipenv = any( - [ - environment and env not in environment, - not environment and default_envs and env not in default_envs, - ] - ) - if skipenv: - results.append({"env": env}) - continue + handle_legacy_libdeps(project_dir, config) - # print empty line between multi environment project - if not silent and any(r.get("succeeded") is not None for r in results): - click.echo() - - results.append( - process_env( - ctx, - env, - config, - environment, - target, - upload_port, - monitor_port, - jobs, - program_args, - is_test_running, - silent, - verbose, + default_envs = config.default_envs() + for env in config.envs(): + skipenv = any( + [ + environment and env not in environment, + not environment and default_envs and env not in default_envs, + ] + ) + if skipenv: + results.append({"env": env}) + continue + + # print empty line between multi environment project + if not silent and any(r.get("succeeded") is not None for r in results): + click.echo() + + results.append( + process_env( + ctx, + env, + config, + target, + upload_port, + jobs, + program_args, + is_test_running, + silent, + verbose, + ) ) - ) - - command_failed = any(r.get("succeeded") is False for r in results) - if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results, verbose) + command_failed = any(r.get("succeeded") is False for r in results) + if ( + not is_test_running + and (command_failed or not silent) + and len(results) > 1 + ): + print_processing_summary(results, verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) if command_failed: raise exception.ReturnErrorCode(1) + + if "monitor" in target and "nobuild" not in target: + ctx.invoke( + device_monitor_cmd, + port=monitor_port, + environment=environment[0] if environment else None, + ) + return True @@ -177,10 +189,8 @@ def process_env( ctx, name, config, - environments, targets, upload_port, - monitor_port, jobs, program_args, is_test_running, @@ -208,17 +218,6 @@ def process_env( if not is_test_running and (not silent or not result["succeeded"]): print_processing_footer(result) - if ( - result["succeeded"] - and "monitor" in ep.get_build_targets() - and "nobuild" not in ep.get_build_targets() - ): - ctx.invoke( - device_monitor_cmd, - port=monitor_port, - environment=environments[0] if environments else None, - ) - return result From a24cf50413155a77c87e307e97ec67a8e4359180 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Mar 2023 23:13:24 -0600 Subject: [PATCH 38/98] Minor fixes --- platformio/project/options.py | 3 +-- platformio/run/cli.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/project/options.py b/platformio/project/options.py index 53775b5d5d..639cefe68a 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -652,8 +652,7 @@ def get_default_core_dir(): name="check_src_filters", oldnames=["check_patterns"], description=( - "Configure a list of target files or directories for checking " - "(Unix shell-style wildcards)" + "Configure a list of target files or directories for checking" ), multiple=True, ), diff --git a/platformio/run/cli.py b/platformio/run/cli.py index bf237ac956..96ea2b89c8 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -178,6 +178,7 @@ def cli( if "monitor" in target and "nobuild" not in target: ctx.invoke( device_monitor_cmd, + project_dir=project_dir, port=monitor_port, environment=environment[0] if environment else None, ) From 331ff2dc9c0e3a4e0cb5852031a3728c4cfbb9fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 12:38:40 -0600 Subject: [PATCH 39/98] Format code --- platformio/run/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 96ea2b89c8..f9f7a07a77 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -103,7 +103,6 @@ def cli( is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta - results = [] only_monitor = "monitor" in target and len(target) == 1 command_failed = False with fs.cd(project_dir): @@ -131,6 +130,7 @@ def cli( handle_legacy_libdeps(project_dir, config) default_envs = config.default_envs() + results = [] for env in config.envs(): skipenv = any( [ @@ -160,7 +160,6 @@ def cli( verbose, ) ) - command_failed = any(r.get("succeeded") is False for r in results) if ( not is_test_running From 269d5e0a3ea5823a5dc7eeff9902cb7062017430 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 15:00:22 -0600 Subject: [PATCH 40/98] Added validation for project working environment names --- HISTORY.rst | 1 + docs | 2 +- platformio/project/config.py | 22 +++++++++++++++++----- platformio/project/exception.py | 7 +++++++ tests/project/test_config.py | 22 +++++++++++++++++++--- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 317e9a4baa..5718e304a6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 6 * Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process +* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow diff --git a/docs b/docs index ea46b38c80..24b6ec7743 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ea46b38c806aff3f944ffba3de776cf7e1a03947 +Subproject commit 24b6ec7743d6a4553ae3c9ef6b7c991a63abdb65 diff --git a/platformio/project/config.py b/platformio/project/config.py index 43d15f3e26..bf5870c9f8 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -39,6 +39,7 @@ class ProjectConfigBase: + ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -388,16 +389,27 @@ def get_default_env(self): def validate(self, envs=None, silent=False): if not os.path.isfile(self.path): raise exception.NotPlatformIOProjectError(os.path.dirname(self.path)) + + known_envs = set(self.envs()) + # check envs - known = set(self.envs()) - if not known: + if not known_envs: raise exception.ProjectEnvsNotAvailableError() - unknown = set(list(envs or []) + self.default_envs()) - known - if unknown: - raise exception.UnknownEnvNamesError(", ".join(unknown), ", ".join(known)) + unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs + if unknown_envs: + raise exception.UnknownEnvNamesError( + ", ".join(unknown_envs), ", ".join(known_envs) + ) + + # check envs names + for env in known_envs: + if not self.ENVNAME_RE.match(env): + raise exception.InvalidEnvNameError(env) + if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") + return True diff --git a/platformio/project/exception.py b/platformio/project/exception.py index 8e598e038e..95681bc0df 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -43,5 +43,12 @@ class UnknownEnvNamesError(ProjectError, UserSideException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" +class InvalidEnvNameError(ProjectError, UserSideException): + MESSAGE = ( + "Invalid environment name '{0}'. The name can contain " + "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" + ) + + class ProjectOptionValueError(ProjectError, UserSideException): MESSAGE = "{0} for option `{1}` in section [{2}]" diff --git a/tests/project/test_config.py b/tests/project/test_config.py index 6076a7d6c7..f559502ed3 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -23,7 +23,11 @@ from platformio import fs from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError +from platformio.project.exception import ( + InvalidEnvNameError, + InvalidProjectConfError, + UnknownEnvNamesError, +) BASE_CONFIG = """ [platformio] @@ -662,9 +666,21 @@ def test_extends_order(tmp_path: Path): [c] upload_tool = three -[env:native] +[env:na_ti-ve13] extends = a, b, c """ ) config = ProjectConfig(str(project_conf)) - assert config.get("env:native", "upload_tool") == "three" + assert config.get("env:na_ti-ve13", "upload_tool") == "three" + + +def test_invalid_env_names(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app:1] + """ + ) + config = ProjectConfig(str(project_conf)) + with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): + config.validate() From 00c5d30ce991af5685775cd972fb5d0d9c620a41 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 15:00:59 -0600 Subject: [PATCH 41/98] Bump version to 6.1.7a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index d2136573cb..8668dd7eee 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "7a3") +VERSION = (6, 1, "7a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f43f41cc53055122a53a74dac0fb381e3fbcaea3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 15:39:51 -0600 Subject: [PATCH 42/98] Format code --- platformio/check/cli.py | 6 +----- platformio/check/tools/base.py | 4 +++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/platformio/check/cli.py b/platformio/check/cli.py index 215a6b8ade..d1abdfff03 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -50,11 +50,7 @@ ), ) @click.option("--pattern", multiple=True, hidden=True) -@click.option( - "-f", - "--src-filters", - multiple=True -) +@click.option("-f", "--src-filters", multiple=True) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 392a19af0d..77f0e55c16 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -256,7 +256,9 @@ def check(self, on_defect_callback=None): def normalize_src_filters(src_filters): def _normalize(src_filters): return ( - src_filters if src_filters.startswith(("+<", "-<")) else "+<%s>" % src_filters + src_filters + if src_filters.startswith(("+<", "-<")) + else "+<%s>" % src_filters ) if isinstance(src_filters, (list, tuple)): From 4b446b0d720da06b0e487906375471c90935b6aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 15:40:24 -0600 Subject: [PATCH 43/98] Fixed an issue when "build_cache_dir" was not honored across different environments // Resolve #4574 --- HISTORY.rst | 1 + platformio/builder/main.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5718e304a6..492fd204f4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 6 * Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow +* Fixed an issue when "build_cache_dir" was not honored across different environments (`issue #4574 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b56dc2a248..844118ed85 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -173,7 +173,8 @@ env.SConscriptChdir(0) env.SConsignFile( os.path.join( - "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]) + "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", + ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]), ) ) From 0bba598c615cf499a1fb05f7f7bcd971ed7db188 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 Mar 2023 18:56:39 -0600 Subject: [PATCH 44/98] Temporary fix for unreleased dev-platforms with broken env name --- HISTORY.rst | 2 +- tests/test_examples.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 492fd204f4..fd76135496 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,7 +23,7 @@ PlatformIO Core 6 * Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow -* Fixed an issue when "build_cache_dir" was not honored across different environments (`issue #4574 `_) +* Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/tests/test_examples.py b/tests/test_examples.py index 27164fd71d..c50285a563 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -69,6 +69,8 @@ def test_run(pioproject_dir): fs.rmtree(build_dir) env_names = config.envs() + # temporary fix for unreleased dev-platforms with broken env name + env_names = [name for name in env_names if " " not in name] result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) From 7d86eebe77e2f062662f170ae80a2479cfb063f5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Mar 2023 14:35:59 -0600 Subject: [PATCH 45/98] Upgraded the build engine to the latest version of SCons (4.5.2) --- HISTORY.rst | 1 + platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index fd76135496..0d2a5230bd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,7 @@ PlatformIO Core 6 * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process * Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index 8668dd7eee..fe0d0e7eb6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -47,7 +47,7 @@ __core_packages__ = { "contrib-piohome": "~3.4.2", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-scons": "~4.40400.0", + "tool-scons": "~4.40502.0", "tool-cppcheck": "~1.270.0", "tool-clangtidy": "~1.150005.0", "tool-pvs-studio": "~7.18.0", From 62e9589851547c563844cc828eeafb4e18e2f15c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Mar 2023 14:36:51 -0600 Subject: [PATCH 46/98] Use the built-in SCons "compile_db" tool --- platformio/builder/main.py | 9 +- platformio/builder/tools/compilation_db.py | 224 --------------------- 2 files changed, 8 insertions(+), 225 deletions(-) delete mode 100644 platformio/builder/tools/compilation_db.py diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 844118ed85..f80435f2f0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,7 +30,7 @@ from platformio import app, compat, fs from platformio.platform.base import PlatformBase -from platformio.proc import get_pythonexe_path +from platformio.proc import get_pythonexe_path, where_is_program from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) @@ -209,6 +209,13 @@ Default("checkprogsize") if "compiledb" in COMMAND_LINE_TARGETS: + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")) env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) # Print configured protocols diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py deleted file mode 100644 index 9fa11c8628..0000000000 --- a/platformio/builder/tools/compilation_db.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# Copyright 2020 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pylint: disable=unused-argument, protected-access, unused-variable, import-error -# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py - -import itertools -import json -import os - -import SCons - -from platformio.builder.tools.piobuild import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT -from platformio.proc import where_is_program - -# Implements the ability for SCons to emit a compilation database for the MongoDB project. See -# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation -# database is, and why you might want one. The only user visible entry point here is -# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that -# should hold the compilation database, otherwise, the file defaults to compile_commands.json, -# which is the name that most clang tools search for by default. - -# Is there a better way to do this than this global? Right now this exists so that the -# emitter we add can record all of the things it emits, so that the scanner for the top level -# compilation database can access the complete list, and also so that the writer has easy -# access to write all of the files. But it seems clunky. How can the emitter and the scanner -# communicate more gracefully? -__COMPILATION_DB_ENTRIES = [] - - -# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even -# integrate with the cache, but there doesn't seem to be much call for it. -class __CompilationDbNode(SCons.Node.Python.Value): - def __init__(self, value): - SCons.Node.Python.Value.__init__(self, value) - self.Decider(changed_since_last_build_node) - - -def changed_since_last_build_node(*args, **kwargs): - """Dummy decider to force always building""" - return True - - -def makeEmitCompilationDbEntry(comstr): - """ - Effectively this creates a lambda function to capture: - * command line - * source - * target - :param comstr: unevaluated command line - :return: an emitter which has captured the above - """ - user_action = SCons.Action.Action(comstr) - - def EmitCompilationDbEntry(target, source, env): - """ - This emitter will be added to each c/c++ object build to capture the info needed - for clang tools - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :return: target(s), source(s) - """ - - # Resolve absolute path of toolchain - for cmd in ("CC", "CXX", "AS"): - if cmd not in env: - continue - if os.path.isabs(env[cmd]): - continue - env[cmd] = where_is_program( - env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") - ) - - dbtarget = __CompilationDbNode(source) - - entry = env.__COMPILATIONDB_Entry( - target=dbtarget, - source=[], - __COMPILATIONDB_UTARGET=target, - __COMPILATIONDB_USOURCE=source, - __COMPILATIONDB_UACTION=user_action, - __COMPILATIONDB_ENV=env, - ) - - # Technically, these next two lines should not be required: it should be fine to - # cache the entries. However, they don't seem to update properly. Since they are quick - # to re-generate disable caching and sidestep this problem. - env.AlwaysBuild(entry) - env.NoCache(entry) - - __COMPILATION_DB_ENTRIES.append(dbtarget) - - return target, source - - return EmitCompilationDbEntry - - -def CompilationDbEntryAction(target, source, env, **kw): - """ - Create a dictionary with evaluated command line, target, source - and store that info as an attribute on the target - (Which has been stored in __COMPILATION_DB_ENTRIES array - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :param kw: - :return: None - """ - - command = env["__COMPILATIONDB_UACTION"].strfunction( - target=env["__COMPILATIONDB_UTARGET"], - source=env["__COMPILATIONDB_USOURCE"], - env=env["__COMPILATIONDB_ENV"], - ) - - entry = { - "directory": env.Dir("#").abspath, - "command": command, - "file": str(env["__COMPILATIONDB_USOURCE"][0]), - } - - target[0].write(entry) - - -def WriteCompilationDb(target, source, env): - entries = [] - - for s in __COMPILATION_DB_ENTRIES: - item = s.read() - item["file"] = os.path.abspath(item["file"]) - entries.append(item) - - with open(str(target[0]), mode="w", encoding="utf8") as target_file: - json.dump( - entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") - ) - - -def ScanCompilationDb(node, env, path): - return __COMPILATION_DB_ENTRIES - - -def generate(env, **kwargs): - static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - - env["COMPILATIONDB_COMSTR"] = kwargs.get( - "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" - ) - - components_by_suffix = itertools.chain( - itertools.product( - [".%s" % ext for ext in SRC_C_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_CXX_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_ASM_EXT], - [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")], - ), - ) - - for entry in components_by_suffix: - suffix = entry[0] - builder, base_emitter, command = entry[1] - - # Assumes a dictionary emitter - emitter = builder.emitter[suffix] - builder.emitter[suffix] = SCons.Builder.ListEmitter( - [emitter, makeEmitCompilationDbEntry(command)] - ) - - env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder( - action=SCons.Action.Action(CompilationDbEntryAction, None), - ) - - env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder( - action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), - target_scanner=SCons.Scanner.Scanner( - function=ScanCompilationDb, node_class=None - ), - ) - - def CompilationDatabase(env, target): - result = env.__COMPILATIONDB_Database(target=target, source=[]) - - env.AlwaysBuild(result) - env.NoCache(result) - - return result - - env.AddMethod(CompilationDatabase, "CompilationDatabase") - - -def exists(env): - return True From bea5e87543a727cbcaca0fd4cd5cb15916b55867 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Mar 2023 14:41:04 -0600 Subject: [PATCH 47/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 24b6ec7743..308d8c036d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 24b6ec7743d6a4553ae3c9ef6b7c991a63abdb65 +Subproject commit 308d8c036d3e78e9819ba9468a44a651036ccbdc From 0fb064eba3fc03847bfcdaee4f84f67c5b9f3f37 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 Mar 2023 18:59:06 -0600 Subject: [PATCH 48/98] Fixed an issue where organization details could not be updated --- HISTORY.rst | 1 + platformio/account/org/commands/update.py | 30 ++++++------ platformio/account/team/commands/create.py | 4 +- platformio/account/team/commands/update.py | 20 +++----- platformio/account/validate.py | 57 ++++++++++++---------- tests/test_examples.py | 10 +++- 6 files changed, 63 insertions(+), 59 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0d2a5230bd..9076270553 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,7 @@ PlatformIO Core 6 * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) +* Fixed an issue where organization details could not be updated using the `pio org update `__ command, ensuring that your organization's information remains up-to-date and accurate 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/account/org/commands/update.py b/platformio/account/org/commands/update.py index 9da9564c2f..86e7cffa8b 100644 --- a/platformio/account/org/commands/update.py +++ b/platformio/account/org/commands/update.py @@ -22,29 +22,27 @@ @click.argument("cur_orgname") @click.option( "--orgname", - callback=lambda _, __, value: validate_orgname(value), + callback=lambda _, __, value: validate_orgname(value) if value else value, help="A new orgname", ) -@click.option("--email") +@click.option( + "--email", + callback=lambda _, __, value: validate_email(value) if value else value, +) @click.option("--displayname") def org_update_cmd(cur_orgname, **kwargs): client = AccountClient() org = client.get_org(cur_orgname) - del org["owners"] - new_org = org.copy() + new_org = { + key: value if value is not None else org[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in org: - new_org[field] = click.prompt( - field.replace("_", " ").capitalize(), default=org[field] - ) - if field == "email": - validate_email(new_org[field]) - if field == "orgname": - validate_orgname(new_org[field]) - else: - new_org.update( - {key.replace("new_", ""): value for key, value in kwargs.items() if value} - ) + for key in kwargs: + new_org[key] = click.prompt(key.capitalize(), default=org[key]) + if key == "email": + validate_email(new_org[key]) + if key == "orgname": + validate_orgname(new_org[key]) client.update_org(cur_orgname, new_org) return click.secho( "The organization `%s` has been successfully updated." % cur_orgname, diff --git a/platformio/account/team/commands/create.py b/platformio/account/team/commands/create.py index 891d33b43b..164ca6acad 100644 --- a/platformio/account/team/commands/create.py +++ b/platformio/account/team/commands/create.py @@ -22,9 +22,7 @@ @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", - callback=lambda _, __, value: validate_orgname_teamname( - value, teamname_validate=True - ), + callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( "--description", diff --git a/platformio/account/team/commands/update.py b/platformio/account/team/commands/update.py index 3ead0feda1..c4c2f8b4f0 100644 --- a/platformio/account/team/commands/update.py +++ b/platformio/account/team/commands/update.py @@ -26,7 +26,7 @@ ) @click.option( "--name", - callback=lambda _, __, value: validate_teamname(value), + callback=lambda _, __, value: validate_teamname(value) if value else value, help="A new team name", ) @click.option( @@ -36,18 +36,14 @@ def team_update_cmd(orgname_teamname, **kwargs): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() team = client.get_team(orgname, teamname) - del team["id"] - del team["members"] - new_team = team.copy() + new_team = { + key: value if value is not None else team[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in team: - new_team[field] = click.prompt( - field.replace("_", " ").capitalize(), default=team[field] - ) - if field == "name": - validate_teamname(new_team[field]) - else: - new_team.update({key: value for key, value in kwargs.items() if value}) + for key in kwargs: + new_team[key] = click.prompt(key.capitalize(), default=team[key]) + if key == "name": + validate_teamname(new_team[key]) client.update_team(orgname, teamname, new_team) return click.secho( "The team %s has been successfully updated." % teamname, diff --git a/platformio/account/validate.py b/platformio/account/validate.py index 7fb7722224..60d3bb2e65 100644 --- a/platformio/account/validate.py +++ b/platformio/account/validate.py @@ -18,8 +18,10 @@ def validate_username(value, field="username"): - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I + ): raise click.BadParameter( "Invalid %s format. " "%s must contain only alphanumeric characters " @@ -30,16 +32,22 @@ def validate_username(value, field="username"): return value +def validate_orgname(value): + return validate_username(value, "Organization name") + + def validate_email(value): - value = str(value).strip() - if not re.match(r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I + ): raise click.BadParameter("Invalid email address") return value def validate_password(value): - value = str(value).strip() - if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): + value = str(value).strip() if value else None + if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): raise click.BadParameter( "Invalid password format. " "Password must contain at least 8 characters" @@ -48,27 +56,11 @@ def validate_password(value): return value -def validate_orgname(value): - return validate_username(value, "Organization name") - - -def validate_orgname_teamname(value, teamname_validate=False): - if ":" not in value: - raise click.BadParameter( - "Please specify organization and team name in the next" - " format - orgname:teamname. For example, mycompany:DreamTeam" - ) - teamname = str(value.strip().split(":", 1)[1]) - if teamname_validate: - validate_teamname(teamname) - return value - - def validate_teamname(value): - if not value: - return value - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I + ): raise click.BadParameter( "Invalid team name format. " "Team name must only contain alphanumeric characters, " @@ -77,3 +69,16 @@ def validate_teamname(value): " not be longer than 20 characters." ) return value + + +def validate_orgname_teamname(value): + value = str(value).strip() if value else None + if not value or ":" not in value: + raise click.BadParameter( + "Please specify organization and team name using the following" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + orgname, teamname = value.split(":", 1) + validate_orgname(orgname) + validate_teamname(teamname) + return value diff --git a/tests/test_examples.py b/tests/test_examples.py index c50285a563..852bb7aa6c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,6 +22,7 @@ from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError def pytest_generate_tests(metafunc): @@ -64,13 +65,18 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() + + # temporary fix for unreleased dev-platforms with broken env name + try: + config.validate() + except ProjectError as exc: + pytest.skip(str(exc)) + build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) env_names = config.envs() - # temporary fix for unreleased dev-platforms with broken env name - env_names = [name for name in env_names if " " not in name] result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) From faff0fb56ccd5f97b865a421b0132767b2afe80d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Mar 2023 08:29:52 -0600 Subject: [PATCH 49/98] Sync docs and examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 308d8c036d..2353c9975a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 308d8c036d3e78e9819ba9468a44a651036ccbdc +Subproject commit 2353c9975a2dcb1849ae8c85d776af59d40f1186 diff --git a/examples b/examples index 894dbd36d9..f1e0ecff5b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 894dbd36d97534267829dc2992e8253f34c2be45 +Subproject commit f1e0ecff5b8dccc04ee6ecbca67e9ffaf0617eb2 From 71bb84f3f27dd28dd873cb72e7a50e52e0d912fa Mon Sep 17 00:00:00 2001 From: Max Prokhorov Date: Fri, 7 Apr 2023 22:58:08 +0300 Subject: [PATCH 50/98] InoToCpp: add ':' to arguments regexp (#4586) InoToCpp: allow ':' in arguments --- platformio/builder/tools/pioino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index c44e72b74f..92e76ac124 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -30,7 +30,7 @@ class InoToCPPConverter: (?:template\<.*\>\s*)? # template ([a-z_\d\&]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype - \([a-z_,\.\*\&\[\]\s\d]*\) # arguments + \([a-z_:,\.\*\&\[\]\s\d]*\) # arguments )\s*(\{|;) # must end with `{` or `;` """, re.X | re.M | re.I, From ab15da4f4bc01e6b5577c88f466bb4cd4aa3bd42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Apr 2023 23:14:05 +0300 Subject: [PATCH 51/98] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2353c9975a..e3c5bb8597 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2353c9975a2dcb1849ae8c85d776af59d40f1186 +Subproject commit e3c5bb85973f41a48ad04e429531a7f9abb8d0ad From 58a1d5d96e6a1259dcb5d12b7b103eff4bbdd1fd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Apr 2023 14:18:33 +0300 Subject: [PATCH 52/98] Revert "InoToCpp: add ':' to arguments regexp" (#4595) Revert "InoToCpp: add ':' to arguments regexp (#4586)" This reverts commit 71bb84f3f27dd28dd873cb72e7a50e52e0d912fa. --- platformio/builder/tools/pioino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index 92e76ac124..c44e72b74f 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -30,7 +30,7 @@ class InoToCPPConverter: (?:template\<.*\>\s*)? # template ([a-z_\d\&]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype - \([a-z_:,\.\*\&\[\]\s\d]*\) # arguments + \([a-z_,\.\*\&\[\]\s\d]*\) # arguments )\s*(\{|;) # must end with `{` or `;` """, re.X | re.M | re.I, From 5c3ae15bee3470bfa4e8480644de41c6d8b5ba1a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Apr 2023 20:02:26 +0300 Subject: [PATCH 53/98] Store device monitor logs in the project "logs" directory // Resolve #4596 --- HISTORY.rst | 1 + docs | 2 +- platformio/device/monitor/command.py | 32 +++++++++---------- platformio/device/monitor/filters/log2file.py | 8 +++-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9076270553..d3218bbade 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 6 6.1.7 (2023-??-??) ~~~~~~~~~~~~~~~~~~ +* Implemented a new feature to store device monitor logs in the project's "logs" folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) * Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process diff --git a/docs b/docs index e3c5bb8597..2c2cc23f42 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e3c5bb85973f41a48ad04e429531a7f9abb8d0ad +Subproject commit 2c2cc23f42e9c8268223e9b3b4b53739d88bb38c diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 301de77296..3cf1357354 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -132,24 +132,24 @@ def device_monitor_cmd(**options): ensure_ready=True, ).find(initial_port=options["port"]) - if options["menu_char"] == options["exit_char"]: - raise exception.UserSideException( - "--exit-char can not be the same as --menu-char" - ) - - # check for unknown filters - if options["filters"]: - known_filters = set(get_available_filters()) - unknown_filters = set(options["filters"]) - known_filters - if unknown_filters: - options["filters"] = list(known_filters & set(options["filters"])) - click.secho( - ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") - % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), - fg="yellow", + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" ) - start_terminal(options) + # check for unknown filters + if options["filters"]: + known_filters = set(get_available_filters()) + unknown_filters = set(options["filters"]) - known_filters + if unknown_filters: + options["filters"] = list(known_filters & set(options["filters"])) + click.secho( + ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") + % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), + fg="yellow", + ) + + start_terminal(options) def get_project_options(environment=None): diff --git a/platformio/device/monitor/filters/log2file.py b/platformio/device/monitor/filters/log2file.py index bf97b5511f..8e00d29708 100644 --- a/platformio/device/monitor/filters/log2file.py +++ b/platformio/device/monitor/filters/log2file.py @@ -13,7 +13,7 @@ # limitations under the License. import io -import os.path +import os from datetime import datetime from platformio.device.monitor.filters.base import DeviceMonitorFilterBase @@ -27,8 +27,10 @@ def __init__(self, *args, **kwargs): self._log_fp = None def __call__(self): - log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( - "%y%m%d-%H%M%S" + if not os.path.isdir("logs"): + os.makedirs("logs") + log_file_name = os.path.join( + "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S") ) print("--- Logging an output to %s" % os.path.abspath(log_file_name)) # pylint: disable=consider-using-with From 66fe55668e8bab03e12377dd55613f6f8b11b9e5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Apr 2023 22:21:51 +0300 Subject: [PATCH 54/98] Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode // Resolve #4597 --- HISTORY.rst | 3 ++- docs | 2 +- .../templates/ide-projects/vscode/.vscode/launch.json.tpl | 4 ++-- platformio/project/integration/generator.py | 6 ++++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d3218bbade..8d526a6ca9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,7 +26,8 @@ PlatformIO Core 6 * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) -* Fixed an issue where organization details could not be updated using the `pio org update `__ command, ensuring that your organization's information remains up-to-date and accurate +* Fixed an issue where organization details could not be updated using the `pio org update `__ command +* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 2c2cc23f42..6647f6e074 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2c2cc23f42e9c8268223e9b3b4b53739d88bb38c +Subproject commit 6647f6e074168012e48e21666db6bd7a1df33130 diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl index 1575657537..c419d27ed7 100644 --- a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl +++ b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl @@ -16,7 +16,7 @@ % "request": "launch", % "name": "PIO Debug (skip Pre-Debug)", % "executable": _escape_path(prog_path), -% "projectEnvName": env_name, +% "projectEnvName": env_name if forced_env_name else default_debug_env_name, % "toolchainBinDir": _escape_path(os.path.dirname(gdb_path)), % "internalConsoleOptions": "openOnSessionStart", % } @@ -28,7 +28,7 @@ % debug["name"] = "PIO Debug" % debug["preLaunchTask"] = { % "type": "PlatformIO", -% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and original_env_name else "Pre-Debug", +% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug", % } % noloading = predebug.copy() % noloading["name"] = "PIO Debug (without uploading)" diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 4ea3bbf3f8..12f53ef09f 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -19,6 +19,7 @@ import bottle from platformio import fs, util +from platformio.debug.helpers import get_default_debug_env from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata @@ -27,7 +28,7 @@ class ProjectGenerator: def __init__(self, config, env_name, ide, board_ids=None): self.config = config self.project_dir = os.path.dirname(config.path) - self.original_env_name = env_name + self.forced_env_name = env_name self.env_name = str(env_name or self.get_best_envname(board_ids)) self.ide = str(ide) @@ -86,7 +87,8 @@ def _load_tplvars(self): "platformio", "name", os.path.basename(self.project_dir) ), "project_dir": self.project_dir, - "original_env_name": self.original_env_name, + "forced_env_name": self.forced_env_name, + "default_debug_env_name": get_default_debug_env(self.config), "env_name": self.env_name, "user_home_dir": os.path.abspath(fs.expanduser("~")), "platformio_path": sys.argv[0] From 0d9ee75b05909c4b1d7947828f345e501867c142 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Apr 2023 13:40:53 +0300 Subject: [PATCH 55/98] Resolved an issue where native tests would fail if a custom program name was specified // Resolve #4546 --- HISTORY.rst | 1 + platformio/test/runners/base.py | 8 ++++---- .../test/runners/readers/{program.py => native.py} | 10 +++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) rename platformio/test/runners/readers/{program.py => native.py} (91%) diff --git a/HISTORY.rst b/HISTORY.rst index 8d526a6ca9..64971a1ffe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,7 @@ PlatformIO Core 6 * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) * Fixed an issue where organization details could not be updated using the `pio org update `__ command * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) +* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 78425a766c..845a8c3771 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -18,7 +18,7 @@ from platformio.platform.factory import PlatformFactory from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestStatus -from platformio.test.runners.readers.program import ProgramTestOutputReader +from platformio.test.runners.readers.native import NativeTestOutputReader from platformio.test.runners.readers.serial import SerialTestOutputReader CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" @@ -160,7 +160,7 @@ def stage_testing(self): return None click.secho("Testing...", bold=True) test_port = self.get_test_port() - program_conds = [ + native_conds = [ not self.platform.is_embedded() and (not test_port or "://" not in test_port), self.project_config.get( @@ -168,8 +168,8 @@ def stage_testing(self): ), ] reader = ( - ProgramTestOutputReader(self) - if any(program_conds) + NativeTestOutputReader(self) + if any(native_conds) else SerialTestOutputReader(self) ) return reader.begin() diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/native.py similarity index 91% rename from platformio/test/runners/readers/program.py rename to platformio/test/runners/readers/native.py index d80d170d26..981febc375 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/native.py @@ -24,6 +24,7 @@ get_filesystem_encoding, get_locale_encoding, ) +from platformio.project.helpers import load_build_metadata from platformio.test.exception import UnitTestError EXITING_TIMEOUT = 5 # seconds @@ -56,7 +57,7 @@ def _stop_testing(self): self._exit_timer.cancel() -class ProgramTestOutputReader: +class NativeTestOutputReader: def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( @@ -78,6 +79,13 @@ def get_testing_command(self): "program.exe" if IS_WINDOWS else "program", ) ] + # if user changed PROGNAME + if not os.path.exists(cmd[0]): + build_data = load_build_metadata( + os.getcwd(), self.test_runner.test_suite.env_name, cache=True + ) + if build_data: + cmd[0] = build_data["prog_path"] if self.test_runner.options.program_args: cmd.extend(self.test_runner.options.program_args) return cmd From 7e9b637143bdfeca7d79a7b0309ad0bfe7f254cb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Apr 2023 19:21:46 +0300 Subject: [PATCH 56/98] Resolved an issue where the PlatformIO Debugging solution was not escaping the tool installation process into MI2 correctly // Resolve #4565 --- HISTORY.rst | 4 +- platformio/debug/cli.py | 99 ++++++++++++++------------------- platformio/debug/config/base.py | 6 +- 3 files changed, 47 insertions(+), 62 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 64971a1ffe..9ff6d5f4f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ Release Notes .. |LDF| replace:: `LDF `__ .. |INTERPOLATION| replace:: `Interpolation of Values `__ .. |UNITTESTING| replace:: `Unit Testing `__ +.. |DEBUGGING| replace:: `Debugging `__ .. _release_notes_6: @@ -26,9 +27,10 @@ PlatformIO Core 6 * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) -* Fixed an issue where organization details could not be updated using the `pio org update `__ command +* Resolved an issue where organization details could not be updated using the `pio org update `__ command * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) +* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 64bd1b4399..7137c98dd0 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -28,9 +28,9 @@ from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.gdb import GDBClientProcess +from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project from platformio.project.options import ProjectOptions @@ -81,61 +81,57 @@ def cli( project_dir = os.getenv(name) with fs.cd(project_dir): - return _debug_in_project_dir( + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=[environment] if environment else None) + env_name = environment or helpers.get_default_debug_env(project_config) + + if not interface: + return helpers.predebug_project( + ctx, project_dir, project_config, env_name, False, verbose + ) + + configure_args = ( ctx, - project_dir, - project_conf, - environment, + project_config, + env_name, load_mode, verbose, - interface, __unprocessed, ) + if helpers.is_gdbmi_mode(): + os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true" + stream = helpers.GDBMIConsoleStream() + with proc.capture_std_streams(stream): + debug_config = _configure(*configure_args) + stream.close() + else: + debug_config = _configure(*configure_args) + _run(project_dir, debug_config, __unprocessed) -def _debug_in_project_dir( - ctx, - project_dir, - project_conf, - environment, - load_mode, - verbose, - interface, - __unprocessed, -): - project_config = ProjectConfig.get_instance(project_conf) - project_config.validate(envs=[environment] if environment else None) - env_name = environment or helpers.get_default_debug_env(project_config) - - if not interface: - return helpers.predebug_project( - ctx, project_dir, project_config, env_name, False, verbose - ) + return None - env_options = project_config.items(env=env_name, as_dict=True) - if "platform" not in env_options: - raise ProjectEnvsNotAvailableError() +def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed): + platform = PlatformFactory.new( + project_config.get(f"env:{env_name}", "platform"), autoinstall=True + ) debug_config = DebugConfigFactory.new( - PlatformFactory.new(env_options["platform"], autoinstall=True), + platform, project_config, env_name, ) - if "--version" in __unprocessed: - return subprocess.run( - [debug_config.client_executable_path, "--version"], check=True + raise ReturnErrorCode( + subprocess.run( + [debug_config.client_executable_path, "--version"], check=True + ).returncode ) try: fs.ensure_udev_rules() except exception.InvalidUdevRules as exc: - click.echo( - helpers.escape_gdbmi_stream("~", str(exc) + "\n") - if helpers.is_gdbmi_mode() - else str(exc) + "\n", - nl=False, - ) + click.echo(str(exc)) rebuild_prog = False preload = debug_config.load_cmds == ["preload"] @@ -157,25 +153,10 @@ def _debug_in_project_dir( debug_config.load_cmds = [] if rebuild_prog: - if helpers.is_gdbmi_mode(): - click.echo( - helpers.escape_gdbmi_stream( - "~", "Preparing firmware for debugging...\n" - ), - nl=False, - ) - stream = helpers.GDBMIConsoleStream() - with proc.capture_std_streams(stream): - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - stream.close() - else: - click.echo("Preparing firmware for debugging...") - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - + click.echo("Preparing firmware for debugging...") + helpers.predebug_project( + ctx, os.getcwd(), project_config, env_name, preload, verbose + ) # save SHA sum of newly created prog if load_mode == "modified": helpers.is_prog_obsolete(debug_config.program_path) @@ -183,6 +164,10 @@ def _debug_in_project_dir( if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") + return debug_config + + +def _run(project_dir, debug_config, __unprocessed): loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) @@ -199,5 +184,3 @@ def _debug_in_project_dir( finally: client.close() loop.close() - - return True diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 24e20516d8..cb49658174 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -146,9 +146,9 @@ def server_ready_pattern(self): def _load_build_data(self): data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True) - if data: - return data - raise DebugInvalidOptionsError("Could not load a build configuration") + if not data: + raise DebugInvalidOptionsError("Could not load a build configuration") + return data def _configure_server(self): # user disabled server in platformio.ini From e8ffa244e537566e25f726b19cfb706ad7fbe8e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Apr 2023 13:15:00 +0300 Subject: [PATCH 57/98] Better handling of verbosity in debug mode --- platformio/debug/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index 23f4fff05c..8982c00f14 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -90,7 +90,7 @@ def predebug_project( TestSuite(env_name, debug_testname), project_config, TestRunnerOptions( - verbose=verbose, + verbose=3 if verbose else 0, without_building=False, without_debugging=False, without_uploading=not preload, From 4444a0db99dc5d67ae0b411f4abc9ff9f9a20d41 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Apr 2023 13:15:21 +0300 Subject: [PATCH 58/98] Implement PIO Home PlatformRPC --- platformio/home/rpc/handlers/platform.py | 60 ++++++++++++++++++++++++ platformio/home/run.py | 2 + 2 files changed, 62 insertions(+) create mode 100644 platformio/home/rpc/handlers/platform.py diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py new file mode 100644 index 0000000000..14383cd32e --- /dev/null +++ b/platformio/home/rpc/handlers/platform.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory + + +class PlatformRPC: + @staticmethod + def list_installed(options=None): + result = [] + options = options or {} + + def _matchSearchQuery(p): + searchQuery = options.get("searchQuery") + if not searchQuery: + return True + content_blocks = [p.name, p.title, p.description] + if p.frameworks: + content_blocks.append(" ".join(p.frameworks.keys())) + for board in p.get_boards().values(): + board_data = board.get_brief_data() + for key in ("id", "mcu", "vendor"): + content_blocks.append(board_data.get(key)) + return searchQuery.strip() in " ".join(content_blocks) + + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + if not _matchSearchQuery(p): + continue + result.append( + dict( + __pkg_path=pkg.path, + __pkg_meta=pkg.metadata.as_dict(), + name=p.name, + title=p.title, + description=p.description, + ) + ) + return result + + @staticmethod + def get_boards(spec): + p = PlatformFactory.new(spec) + return sorted( + [b.get_brief_data() for b in p.get_boards().values()], + key=lambda item: item["name"], + ) diff --git a/platformio/home/run.py b/platformio/home/run.py index 25f1081ec3..155e2926cf 100644 --- a/platformio/home/run.py +++ b/platformio/home/run.py @@ -32,6 +32,7 @@ from platformio.home.rpc.handlers.misc import MiscRPC from platformio.home.rpc.handlers.os import OSRPC from platformio.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.home.rpc.handlers.platform import PlatformRPC from platformio.home.rpc.handlers.project import ProjectRPC from platformio.home.rpc.handlers.registry import RegistryRPC from platformio.home.rpc.server import WebSocketJSONRPCServerFactory @@ -73,6 +74,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") + ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform") ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry") path = urlparse(home_url).path From f71317dad916e8aca9863330b85203a49af6ecab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Apr 2023 13:17:27 +0300 Subject: [PATCH 59/98] Bump version to 6.1.7b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index fe0d0e7eb6..66f390ed8d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "7a4") +VERSION = (6, 1, "7b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 7f38e222c99fdd77d5127815973d47b9c271b7d5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Apr 2023 20:16:59 +0300 Subject: [PATCH 60/98] Provide SECURITY.md --- SECURITY.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..3d71c1ec2a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +## Supported Versions + +We are committed to ensuring the security and protection of PlatformIO Core. +To this end, we support only the following versions: + +| Version | Supported | +| ------- | ------------------ | +| 6.1.x | :white_check_mark: | +| < 6.1 | :x: | + +Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. +Therefore, it is important that all developers use only supported versions of the PlatformIO Core. + +## Reporting a Vulnerability + +We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately. + +To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including: + +- A description of the vulnerability and how it can be exploited +- Steps to reproduce the vulnerability +- Any additional information that can help us understand and reproduce the vulnerability + +Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed. + +If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion. + +If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively. + +Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy. + +Thank you for helping us keep our systems and data secure. From 02a63a69543ef80b240931aaee16a4ebd796183f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Apr 2023 20:20:03 +0300 Subject: [PATCH 61/98] Provide CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..1958f6ced2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +See https://piolabs.com/legal/code-of-conduct.html From 97a7cdd2a21d76cf648266d59ba448dadef89b8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 18 Apr 2023 21:35:22 +0300 Subject: [PATCH 62/98] Check simultaneous use of `monitor_raw` and `monitor_filters` --- platformio/project/config.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index bf5870c9f8..c2b072ef16 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -401,11 +401,22 @@ def validate(self, envs=None, silent=False): ", ".join(unknown_envs), ", ".join(known_envs) ) - # check envs names for env in known_envs: + # check envs names if not self.ENVNAME_RE.match(env): raise exception.InvalidEnvNameError(env) + # check simultaneous use of `monitor_raw` and `monitor_filters` + if self.get(f"env:{env}", "monitor_raw", False) and self.get( + f"env:{env}", "monitor_filters", None + ): + self.warnings.append( + "The `monitor_raw` and `monitor_filters` options cannot be " + f"used simultaneously for the `{env}` environment in the " + "`platformio.ini` file. The `monitor_filters` option will " + "be disabled to avoid conflicts." + ) + if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") From fa255ff8b3ac774bf06a3e78b0877f6e39f7bd6b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Apr 2023 14:21:34 +0300 Subject: [PATCH 63/98] Provide verbose unpacking in non-terminal mode --- docs | 2 +- platformio/package/download.py | 2 +- platformio/package/manager/_download.py | 4 ++-- platformio/package/unpack.py | 31 +++++++++++++++++++++---- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs b/docs index 6647f6e074..437b1175bd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6647f6e074168012e48e21666db6bd7a1df33130 +Subproject commit 437b1175bdb310e143f1fe6312433ba4f34ef715 diff --git a/platformio/package/download.py b/platformio/package/download.py index 5045e8565a..17cc1f30eb 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -105,7 +105,7 @@ def start(self, with_progress=True, silent=False): label=label, update_min_steps=min( 256 * 1024, file_size / 100 - ), # every 256Kb or less, + ), # every 256Kb or less ) as pb: for chunk in pb: pb.update(len(chunk)) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index d1ffe710ab..bf844980ee 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -61,7 +61,7 @@ def download(self, url, checksum=None): self.set_download_utime(dl_path) return dl_path - with_progress = not silent and not app.is_disabled_progressbar() + with_progress = not app.is_disabled_progressbar() tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): @@ -70,7 +70,7 @@ def download(self, url, checksum=None): fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) except IOError as exc: - raise_error = not with_progress + raise_error = not silent if with_progress: try: fd = FileDownloader(url) diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index 35e0da6879..9d8919c32f 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -20,6 +20,7 @@ import click from platformio import fs +from platformio.compat import is_terminal from platformio.package.exception import PackageException @@ -158,18 +159,38 @@ def new_archiver(path): def unpack( self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False - ): + ): # pylint: disable=too-many-branches assert self._archiver + label = "Unpacking" + items = self._archiver.get_items() if not dest_dir: dest_dir = os.getcwd() + if not with_progress or silent: if not silent: - click.echo("Unpacking...") - for item in self._archiver.get_items(): + click.echo(f"{label}...") + for item in items: + self._archiver.extract_item(item, dest_dir) + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + unpacked_nums = 0 + for item in items: self._archiver.extract_item(item, dest_dir) + unpacked_nums += 1 + if (unpacked_nums / len(items) * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") else: - items = self._archiver.get_items() - with click.progressbar(items, label="Unpacking") as pb: + with click.progressbar( + items, + label=label, + update_min_steps=min(50, len(items) / 100), # every 50 files or less + ) as pb: for item in pb: self._archiver.extract_item(item, dest_dir) From 743fc8e6365bb0b87163d012a7de397e1cec4495 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Apr 2023 18:57:22 +0300 Subject: [PATCH 64/98] Improved support for projects located on Windows network drives // Resolve #3417 --- HISTORY.rst | 1 + platformio/builder/main.py | 15 +-------------- platformio/check/cli.py | 8 ++------ platformio/commands/ci.py | 6 ++---- platformio/commands/lib.py | 4 +--- platformio/debug/cli.py | 8 ++------ platformio/device/monitor/command.py | 2 +- platformio/fs.py | 21 +-------------------- platformio/package/commands/install.py | 4 ++-- platformio/package/commands/list.py | 4 ++-- platformio/package/commands/outdated.py | 2 +- platformio/package/commands/pack.py | 2 +- platformio/package/commands/publish.py | 2 +- platformio/package/commands/uninstall.py | 4 ++-- platformio/package/commands/update.py | 4 ++-- platformio/project/commands/config.py | 2 +- platformio/project/commands/init.py | 4 +--- platformio/project/commands/metadata.py | 4 ++-- platformio/project/helpers.py | 2 +- platformio/project/options.py | 2 +- platformio/remote/cli.py | 16 +++++----------- platformio/run/cli.py | 8 ++------ platformio/system/commands/completion.py | 4 ++-- platformio/test/cli.py | 12 ++++-------- tests/commands/test_test.py | 2 +- 25 files changed, 42 insertions(+), 101 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9ff6d5f4f5..786634ae70 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 6 ~~~~~~~~~~~~~~~~~~ * Implemented a new feature to store device monitor logs in the project's "logs" folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) +* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) * Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process diff --git a/platformio/builder/main.py b/platformio/builder/main.py index f80435f2f0..e73489d55a 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,7 +28,7 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import app, compat, fs +from platformio import app, fs from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path, where_is_program from platformio.project.helpers import get_project_dir @@ -139,19 +139,6 @@ # pylint: disable=protected-access click._compat.isatty = lambda stream: True -if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"): - click.secho("!!! WARNING !!!\t\t" * 3, fg="red") - click.secho( - "Your project is located on a mapped network drive but the " - "current command-line shell does not support the UNC paths.", - fg="yellow", - ) - click.secho( - "Please move your project to a physical drive or check this workaround: " - "https://bit.ly/3kuU5mP\n", - fg="yellow", - ) - if env.subst("$BUILD_CACHE_DIR"): if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): os.makedirs(env.subst("$BUILD_CACHE_DIR")) diff --git a/platformio/check/cli.py b/platformio/check/cli.py index d1abdfff03..d99da3137b 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -38,16 +38,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--pattern", multiple=True, hidden=True) @click.option("-f", "--src-filters", multiple=True) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index aebd9e7b09..de346afd1f 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -51,15 +51,13 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option( "--build-dir", default=tempfile.mkdtemp, - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) @click.option("--keep-build-dir", is_flag=True) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("-O", "--project-option", multiple=True) @click.option("-e", "--environment", "environments", multiple=True) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ed3c9e8d58..1c150c0914 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -65,9 +65,7 @@ def invoke_command(ctx, cmd, **kwargs): "--storage-dir", multiple=True, default=None, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), help="Manage custom library storage", ) @click.option( diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 7137c98dd0..12d6004af4 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -44,16 +44,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--environment", "-e", metavar="") @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type) diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 3cf1357354..3d3190fe48 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -104,7 +104,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", diff --git a/platformio/fs.py b/platformio/fs.py index 00055299b7..107e69828c 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -24,7 +24,7 @@ import click -from platformio import exception, proc +from platformio import exception from platformio.compat import IS_WINDOWS @@ -196,25 +196,6 @@ def to_unix_path(path): return re.sub(r"[\\]+", "/", path) -def normalize_path(path): - path = os.path.abspath(path) - if not IS_WINDOWS or not path.startswith("\\\\"): - return path - try: - result = proc.exec_command(["net", "use"]) - if result["returncode"] != 0: - return path - share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)") - for line in result["out"].split("\n"): - share = share_re.search(line) - if not share: - continue - path = path.replace(share.group(2), share.group(1)) - except OSError: - pass - return path - - def expanduser(path): """ Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 6041e3570f..e3eb05aa99 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -39,7 +39,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -55,7 +55,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index dfd2559f76..c2426c9c6c 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -31,7 +31,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -41,7 +41,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("--only-platforms", is_flag=True, help="List only platform packages") diff --git a/platformio/package/commands/outdated.py b/platformio/package/commands/outdated.py index 5a8c1ea406..18c3783aa1 100644 --- a/platformio/package/commands/outdated.py +++ b/platformio/package/commands/outdated.py @@ -58,7 +58,7 @@ def is_outdated(self): "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) def package_outdated_cmd(project_dir, environments): diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py index 038ffc157a..e9cd0f039b 100644 --- a/platformio/package/commands/pack.py +++ b/platformio/package/commands/pack.py @@ -26,7 +26,7 @@ "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 58230eefe2..228c1411ba 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -47,7 +47,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "--owner", diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py index e393d2a89e..2808d49107 100644 --- a/platformio/package/commands/uninstall.py +++ b/platformio/package/commands/uninstall.py @@ -33,7 +33,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py index 67fc5dd129..8788f7b596 100644 --- a/platformio/package/commands/update.py +++ b/platformio/package/commands/update.py @@ -33,7 +33,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py index b59ff005d1..9f218e8024 100644 --- a/platformio/project/commands/config.py +++ b/platformio/project/commands/config.py @@ -28,7 +28,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("--json-output", is_flag=True) def project_config_cmd(project_dir, json_output): diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index 6a545df469..d8cd5b8d2a 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -47,9 +47,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 "--project-dir", "-d", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index 0645757449..fbf71b8bbd 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -31,11 +31,11 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("--json-output", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): with fs.cd(project_dir): config = ProjectConfig.get_instance() diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 200321f26c..9dbd499cc9 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -24,7 +24,7 @@ def get_project_dir(): - return fs.normalize_path(os.getcwd()) + return os.getcwd() def is_platformio_project(project_dir=None): diff --git a/platformio/project/options.py b/platformio/project/options.py index 639cefe68a..2e13815ac4 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -114,7 +114,7 @@ def validate_dir(path): path = fs.expanduser(path) if "$" in path: path = expand_dir_templates(path) - return fs.normalize_path(path) + return os.path.abspath(path) def get_default_core_dir(): diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index 91f5e29ddf..f9462e0f31 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -56,7 +56,7 @@ def remote_agent(): "-d", "--working-dir", envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) def remote_agent_start(name, share, working_dir): from platformio.remote.client.agent_service import RemoteAgentService @@ -96,9 +96,7 @@ def remote_update(agents, only_check, dry_run): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @@ -186,9 +184,7 @@ def remote_run( "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @@ -334,7 +330,7 @@ def device_list(agents, json_output): "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", @@ -343,9 +339,7 @@ def device_list(agents, json_output): ) @click.option( "--sock", - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.pass_obj @click.pass_context diff --git a/platformio/run/cli.py b/platformio/run/cli.py index f9f7a07a77..ed2a0c7e98 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -47,16 +47,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option( "-j", diff --git a/platformio/system/commands/completion.py b/platformio/system/commands/completion.py index aac68592cc..5ff73bb163 100644 --- a/platformio/system/commands/completion.py +++ b/platformio/system/commands/completion.py @@ -33,7 +33,7 @@ def system_completion_cmd(): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) @@ -54,7 +54,7 @@ def system_completion_install(shell, path): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) diff --git a/platformio/test/cli.py b/platformio/test/cli.py index d0a6e53a4a..748580c8fb 100644 --- a/platformio/test/cli.py +++ b/platformio/test/cli.py @@ -48,16 +48,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -83,8 +79,8 @@ help="A program argument (multiple are allowed)", ) @click.option("--list-tests", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) -@click.option("--junit-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) +@click.option("--junit-output-path", type=click.Path()) @click.option( "--verbose", "-v", diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2608950049..29ff20ab7d 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -642,7 +642,7 @@ def test_googletest_framework(clirunner, tmp_path: Path): pio_test_cmd, [ "-d", - project_dir, + str(project_dir), "-e", "native", "--json-output-path", From df896ad40179c18dde99c3bafe498dabec43291f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Apr 2023 18:58:04 +0300 Subject: [PATCH 65/98] Fix normalizing path to unix // Resolve #4117 --- platformio/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/fs.py b/platformio/fs.py index 107e69828c..1f43a7a318 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -193,7 +193,7 @@ def _find_candidates(pattern): def to_unix_path(path): if not IS_WINDOWS or not path: return path - return re.sub(r"[\\]+", "/", path) + return path.replace("\\", "/") def expanduser(path): From a0c959be2877d36ba188ef5eb05329446061bbdb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Apr 2023 19:34:15 +0300 Subject: [PATCH 66/98] Improved support for projects located on Windows network drives // Issue #3417 --- platformio/builder/tools/piolib.py | 4 ++-- platformio/package/manager/_symlink.py | 2 +- platformio/package/meta.py | 4 ++-- platformio/project/integration/generator.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f95f6735bf..02d47ea667 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -170,8 +170,8 @@ def is_common_builder(self, root_path, child_path): ): return True # try to resolve paths - root_path = os.path.realpath(root_path) - child_path = os.path.realpath(child_path) + root_path = os.path.abspath(root_path) + child_path = os.path.abspath(child_path) return ( os.path.commonprefix([root_path + os.path.sep, child_path]) == root_path + os.path.sep diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py index 33ff03f1bc..f0337869a9 100644 --- a/platformio/package/manager/_symlink.py +++ b/platformio/package/manager/_symlink.py @@ -40,7 +40,7 @@ def get_symlinked_package(self, path): pkg_dir, spec = self.resolve_symlink(path) if not pkg_dir: return None - pkg = PackageItem(os.path.realpath(pkg_dir)) + pkg = PackageItem(os.path.abspath(pkg_dir)) if not pkg.metadata: pkg.metadata = self.build_metadata(pkg.path, spec) return pkg diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 4db5310607..5622e470da 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -482,7 +482,7 @@ def __repr__(self): def __eq__(self, other): conds = [ - os.path.realpath(self.path) == os.path.realpath(other.path) + os.path.abspath(self.path) == os.path.abspath(other.path) if self.path and other.path else self.path == other.path, self.metadata == other.metadata, @@ -490,7 +490,7 @@ def __eq__(self, other): return all(conds) def __hash__(self): - return hash(os.path.realpath(self.path)) + return hash(os.path.abspath(self.path)) def exists(self): return os.path.isdir(self.path) diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 12f53ef09f..40b8e87a0f 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -134,7 +134,7 @@ def get_src_files(self): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( - os.path.relpath(os.path.join(os.path.realpath(root), f)) + os.path.relpath(os.path.join(os.path.abspath(root), f)) ) return result From 6ee9cc04fbdeac8786dbd8c976ae6fccc523facd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Apr 2023 22:29:00 +0300 Subject: [PATCH 67/98] Minor SCons optimizations --- platformio/builder/main.py | 8 +++----- platformio/builder/tools/piolib.py | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e73489d55a..05470451b2 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -99,6 +99,7 @@ DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) +env.SConscriptChdir(False) # Load variables from CLI env.Replace( @@ -157,7 +158,6 @@ env.LoadProjectOptions() env.LoadPioPlatform() -env.SConscriptChdir(0) env.SConsignFile( os.path.join( "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", @@ -165,8 +165,7 @@ ) ) -for item in env.GetExtraScripts("pre"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("pre"), exports="env") if env.IsCleanTarget(): env.CleanProject("cleanall" in COMMAND_LINE_TARGETS) @@ -179,8 +178,7 @@ if env.GetProjectOption("upload_command"): env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) -for item in env.GetExtraScripts("post"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("post"), exports="env") ############################################################################## diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 02d47ea667..16aab05e91 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -297,11 +297,12 @@ def process_extra_options(self): with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: - self.env.SConscriptChdir(1) + self.env.SConscriptChdir(True) self.env.SConscript( os.path.abspath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) + self.env.SConscriptChdir(False) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): From 9344f3cd81226ceca8f707702b7a7fc37cf899ac Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Apr 2023 11:25:48 +0300 Subject: [PATCH 68/98] Revert "Improved support for projects located on Windows network drives // Issue #3417" This reverts commit a0c959be2877d36ba188ef5eb05329446061bbdb. --- platformio/builder/tools/piolib.py | 4 ++-- platformio/package/manager/_symlink.py | 2 +- platformio/package/meta.py | 4 ++-- platformio/project/integration/generator.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 16aab05e91..dee1d5bcef 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -170,8 +170,8 @@ def is_common_builder(self, root_path, child_path): ): return True # try to resolve paths - root_path = os.path.abspath(root_path) - child_path = os.path.abspath(child_path) + root_path = os.path.realpath(root_path) + child_path = os.path.realpath(child_path) return ( os.path.commonprefix([root_path + os.path.sep, child_path]) == root_path + os.path.sep diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py index f0337869a9..33ff03f1bc 100644 --- a/platformio/package/manager/_symlink.py +++ b/platformio/package/manager/_symlink.py @@ -40,7 +40,7 @@ def get_symlinked_package(self, path): pkg_dir, spec = self.resolve_symlink(path) if not pkg_dir: return None - pkg = PackageItem(os.path.abspath(pkg_dir)) + pkg = PackageItem(os.path.realpath(pkg_dir)) if not pkg.metadata: pkg.metadata = self.build_metadata(pkg.path, spec) return pkg diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 5622e470da..4db5310607 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -482,7 +482,7 @@ def __repr__(self): def __eq__(self, other): conds = [ - os.path.abspath(self.path) == os.path.abspath(other.path) + os.path.realpath(self.path) == os.path.realpath(other.path) if self.path and other.path else self.path == other.path, self.metadata == other.metadata, @@ -490,7 +490,7 @@ def __eq__(self, other): return all(conds) def __hash__(self): - return hash(os.path.abspath(self.path)) + return hash(os.path.realpath(self.path)) def exists(self): return os.path.isdir(self.path) diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 40b8e87a0f..12f53ef09f 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -134,7 +134,7 @@ def get_src_files(self): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( - os.path.relpath(os.path.join(os.path.abspath(root), f)) + os.path.relpath(os.path.join(os.path.realpath(root), f)) ) return result From 82d380d895e3a61ebd3d80841b17881639973e27 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Apr 2023 11:40:54 +0300 Subject: [PATCH 69/98] Improve docs for "build_flags" --- docs | 2 +- platformio/builder/tools/piobuild.py | 2 +- platformio/project/integration/generator.py | 2 +- tests/commands/test_run.py | 13 +++++++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs b/docs index 437b1175bd..890d8e20fe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 437b1175bdb310e143f1fe6312433ba4f34ef715 +Subproject commit 890d8e20fed40f653ac84ff366ac16d805b4bf01 diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index ba194eb076..79e2c39bc5 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -200,7 +200,7 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): - if os.path.isdir(p): + if not os.path.isabs(p): result[k][i] = os.path.abspath(p) # fix relative path for "-include" diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 12f53ef09f..40b8e87a0f 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -134,7 +134,7 @@ def get_src_files(self): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( - os.path.relpath(os.path.join(os.path.realpath(root), f)) + os.path.relpath(os.path.join(os.path.abspath(root), f)) ) return result diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 009f33c95f..cb9e3d859e 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -22,6 +22,7 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir): ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), + ("-Iextra_inc", "-Iextra_inc"), ] tmpdir.join("platformio.ini").write( @@ -58,8 +59,20 @@ def post_prog_action(source, target, env): """ ) + tmpdir.mkdir("extra_inc").join("foo.h").write( + """ +#define FOO + """ + ) + tmpdir.mkdir("src").join("main.cpp").write( """ +#include "foo.h" + +#ifndef FOO +#error "FOO" +#endif + #ifdef I_AM_ONLY_SRC_FLAG #include #else From b991d9f25cb263f8e84bc19ed9a3457f5e98cb3b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Apr 2023 12:52:39 +0300 Subject: [PATCH 70/98] Do not normalize path to abs if directory does not exist --- platformio/builder/tools/piobuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 79e2c39bc5..ba194eb076 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -200,7 +200,7 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): - if not os.path.isabs(p): + if os.path.isdir(p): result[k][i] = os.path.abspath(p) # fix relative path for "-include" From c4d178e50e3be9f3aac7bb39369dcf25747bd353 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Apr 2023 22:32:42 +0300 Subject: [PATCH 71/98] Bump version to 6.1.7b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 66f390ed8d..1f3229d9ba 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (6, 1, "7b1") +VERSION = (6, 1, "7b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1e5a728f3c507c36f5fbaa92520001053f974095 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Apr 2023 16:43:42 +0300 Subject: [PATCH 72/98] Return result for JSON RPC --- platformio/home/rpc/handlers/ide.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index d08a821534..d02b0627fc 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -55,6 +55,7 @@ def on_command_result(self, cmd_id, value): value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] + return True def _process_commands(self): for cmd_id in list(self._cmd_queue): From 0acf968b2d75d336a577b23700e2efcfe36524aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Apr 2023 18:20:15 +0300 Subject: [PATCH 73/98] Return result for JSON RPC --- platformio/home/rpc/handlers/ide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index d02b0627fc..c2eb5e063c 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -50,7 +50,7 @@ async def send_command(self, command, params=None): def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: - return + return False if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) From 981266646c895dd5a41d31833493f11dd9855a40 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Apr 2023 18:24:35 +0300 Subject: [PATCH 74/98] Force drive letter to lower case when calculating project checksum on Windows // Resolve #4600 --- platformio/project/helpers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 9dbd499cc9..aec12dd98a 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import subprocess from hashlib import sha1 @@ -94,7 +95,16 @@ def compute_project_checksum(config): checksum = sha1(hashlib_encode_data(__version__)) # configuration file state - checksum.update(hashlib_encode_data(config.to_json())) + config_data = config.to_json() + if IS_WINDOWS: + # issue #4600: fix drive letter + config_data = re.sub( + r"([A-Z]):\\", + lambda match: "%s:\\" % match.group(1).lower(), + config_data, + flags=re.I, + ) + checksum.update(hashlib_encode_data(config_data)) # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") From 53cd43b67673212ccf8f9365067a44f564ccb327 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Apr 2023 16:32:17 +0300 Subject: [PATCH 75/98] Remove obsolate code --- platformio/run/cli.py | 2 -- platformio/run/helpers.py | 23 ----------------------- 2 files changed, 25 deletions(-) diff --git a/platformio/run/cli.py b/platformio/run/cli.py index ed2a0c7e98..92a8057e27 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -123,8 +123,6 @@ def cli( fg="yellow", ) - handle_legacy_libdeps(project_dir, config) - default_envs = config.default_envs() results = [] for env in config.envs(): diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index 925b558484..cfa5165c85 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -15,33 +15,10 @@ from os import makedirs from os.path import isdir, isfile, join -import click - from platformio import fs from platformio.project.helpers import compute_project_checksum, get_project_dir -def handle_legacy_libdeps(project_dir, config): - legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get( - "platformio", "libdeps_dir" - ): - return - if not config.has_section("env"): - config.add_section("env") - lib_extra_dirs = config.get("env", "lib_extra_dirs", []) - lib_extra_dirs.append(legacy_libdeps_dir) - config.set("env", "lib_extra_dirs", lib_extra_dirs) - click.secho( - "DEPRECATED! A legacy library storage `{0}` has been found in a " - "project. \nPlease declare project dependencies in `platformio.ini`" - " file using `lib_deps` option and remove `{0}` folder." - "\nMore details -> https://docs.platformio.org/page/projectconf/" - "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow", - ) - - def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") From c8eea40dd0d296b34cb3e77a2141c168e96f64de Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Apr 2023 20:47:04 +0300 Subject: [PATCH 76/98] Better handling "clean" & "monitor" targets --- platformio/builder/main.py | 2 +- platformio/builder/tools/piotarget.py | 7 +- platformio/platform/_run.py | 18 ++- platformio/run/cli.py | 162 ++++++++++++++------------ platformio/run/helpers.py | 4 + platformio/run/processor.py | 39 +++---- 6 files changed, 125 insertions(+), 107 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 05470451b2..e3932be5e5 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -168,7 +168,7 @@ env.SConscript(env.GetExtraScripts("pre"), exports="env") if env.IsCleanTarget(): - env.CleanProject("cleanall" in COMMAND_LINE_TARGETS) + env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0))) env.Exit(0) env.SConscript("$BUILD_SCRIPT") diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index f2dc774d44..32bb5ffeb6 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -16,7 +16,6 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from platformio import compat, fs @@ -29,10 +28,10 @@ def VerboseAction(_, act, actstr): def IsCleanTarget(env): - return env.GetOption("clean") or ("cleanall" in COMMAND_LINE_TARGETS) + return env.GetOption("clean") -def CleanProject(env, clean_all=False): +def CleanProject(env, fullclean=False): def _relpath(path): if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() @@ -56,7 +55,7 @@ def _clean_dir(path): else: print("Build environment is clean") - if clean_all and os.path.isdir(libdeps_dir): + if fullclean and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) print("Done cleaning") diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 0c157f8520..4fc41f9c67 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -25,6 +25,7 @@ from platformio.compat import hashlib_encode_data from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound +from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS class PlatformRunMixin: @@ -56,9 +57,6 @@ def run( # pylint: disable=too-many-arguments self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") - if "clean" in targets: - targets = ["-c", "."] - variables["platform_manifest"] = self.manifest_path if "build_script" not in variables: @@ -92,10 +90,18 @@ def _run_scons(self, variables, targets, jobs): "--sconstruct", os.path.join(fs.get_source_dir(), "builder", "main.py"), ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + args.append("PIOVERBOSE=%d" % int(self.verbose)) # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets + args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) + + if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): + args.append("--clean") + args.append( + "FULLCLEAN=%d" + % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0) + ) + elif targets: + args.extend(targets) # encode and append variables for key, value in variables.items(): diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 92a8057e27..dc3eb2ee51 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -26,7 +26,7 @@ from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import find_project_dir_above, load_build_metadata -from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps +from platformio.run.helpers import clean_build_dir from platformio.run.processor import EnvironmentProcessor from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING @@ -97,10 +97,12 @@ def cli( if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) + targets = list(target) if target else [] + del target + only_monitor = targets == ["monitor"] is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta - - only_monitor = "monitor" in target and len(target) == 1 command_failed = False + with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) @@ -108,59 +110,60 @@ def cli( if list_targets: return print_target_list(list(environment) or config.envs()) - if not only_monitor: - # clean obsolete build dir - if not disable_auto_clean: - build_dir = config.get("platformio", "build_dir") - try: - clean_build_dir(build_dir, config) - except ProjectError as exc: - raise exc - except: # pylint: disable=bare-except - click.secho( - "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % build_dir, - fg="yellow", - ) - - default_envs = config.default_envs() - results = [] - for env in config.envs(): - skipenv = any( - [ - environment and env not in environment, - not environment and default_envs and env not in default_envs, - ] + # clean obsolete build dir + if not only_monitor and not disable_auto_clean: + build_dir = config.get("platformio", "build_dir") + try: + clean_build_dir(build_dir, config) + except ProjectError as exc: + raise exc + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % build_dir, + fg="yellow", ) - if skipenv: - results.append({"env": env}) - continue - - # print empty line between multi environment project - if not silent and any(r.get("succeeded") is not None for r in results): - click.echo() - - results.append( - process_env( - ctx, - env, - config, - target, - upload_port, - jobs, - program_args, - is_test_running, - silent, - verbose, - ) + + default_envs = config.default_envs() + results = [] + for env in config.envs(): + skipenv = any( + [ + environment and env not in environment, + not environment and default_envs and env not in default_envs, + ] + ) + if skipenv: + results.append({"env": env}) + continue + + # print empty line between multi environment project + if not silent and any(r.get("succeeded") is not None for r in results): + click.echo() + + results.append( + process_env( + ctx, + env, + config, + targets, + upload_port, + monitor_port, + jobs, + program_args, + is_test_running, + silent, + verbose, ) - command_failed = any(r.get("succeeded") is False for r in results) - if ( - not is_test_running - and (command_failed or not silent) - and len(results) > 1 - ): - print_processing_summary(results, verbose) + ) + command_failed = any(r.get("succeeded") is False for r in results) + if ( + not is_test_running + and not only_monitor + and (command_failed or not silent) + and len(results) > 1 + ): + print_processing_summary(results, verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) @@ -168,14 +171,6 @@ def cli( if command_failed: raise exception.ReturnErrorCode(1) - if "monitor" in target and "nobuild" not in target: - ctx.invoke( - device_monitor_cmd, - project_dir=project_dir, - port=monitor_port, - environment=environment[0] if environment else None, - ) - return True @@ -185,6 +180,7 @@ def process_env( config, targets, upload_port, + monitor_port, jobs, program_args, is_test_running, @@ -194,22 +190,38 @@ def process_env( if not is_test_running and not silent: print_processing_header(name, config, verbose) - ep = EnvironmentProcessor( - ctx, - name, - config, - targets, - upload_port, - jobs, - program_args, - silent, - verbose, - ) - result = {"env": name, "duration": time(), "succeeded": ep.process()} + targets = targets or config.get(f"env:{name}", "targets", []) + only_monitor = targets == ["monitor"] + result = {"env": name, "duration": time(), "succeeded": True} + + if not only_monitor: + result["succeeded"] = EnvironmentProcessor( + ctx, + name, + config, + [t for t in targets if t != "monitor"], + upload_port, + jobs, + program_args, + silent, + verbose, + ).process() + + if "monitor" in targets and "nobuild" not in targets: + ctx.invoke( + device_monitor_cmd, + port=monitor_port, + environment=name, + ) + result["duration"] = time() - result["duration"] # print footer on error or when is not unit testing - if not is_test_running and (not silent or not result["succeeded"]): + if ( + not is_test_running + and not only_monitor + and (not silent or not result["succeeded"]) + ): print_processing_footer(result) return result diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index cfa5165c85..f8bfdcba5d 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -18,6 +18,10 @@ from platformio import fs from platformio.project.helpers import compute_project_checksum, get_project_dir +KNOWN_CLEAN_TARGETS = ("clean",) +KNOWN_FULLCLEAN_TARGETS = ("cleanall", "fullclean") +KNOWN_ALLCLEAN_TARGETS = KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS + def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 053478acb8..3d427a1bac 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -15,6 +15,7 @@ from platformio.package.commands.install import install_project_env_dependencies from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError +from platformio.run.helpers import KNOWN_ALLCLEAN_TARGETS from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes @@ -36,7 +37,7 @@ def __init__( # pylint: disable=too-many-arguments self.cmd_ctx = cmd_ctx self.name = name self.config = config - self.targets = [str(t) for t in targets] + self.targets = targets self.upload_port = upload_port self.jobs = jobs self.program_args = program_args @@ -61,33 +62,29 @@ def get_build_variables(self): variables["upload_port"] = self.upload_port return variables - def get_build_targets(self): - return ( - self.targets - if self.targets - else self.config.get("env:" + self.name, "targets", []) - ) - def process(self): if "platform" not in self.options: raise UndefinedEnvPlatformError(self.name) build_vars = self.get_build_variables() - build_targets = list(self.get_build_targets()) + is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) + build_targets = list(set(self.targets) - set(KNOWN_ALLCLEAN_TARGETS)) - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - - if not set(["clean", "cleanall"]) & set(build_targets): - install_project_env_dependencies( - self.name, - { - "project_targets": build_targets, - "piotest_running_name": build_vars.get("piotest_running_name"), - }, - ) + # pre-clean + if is_clean: + result = PlatformFactory.new( + self.options["platform"], autoinstall=True + ).run(build_vars, self.targets, self.silent, self.verbose, self.jobs) + if not build_targets: + return result["returncode"] == 0 + install_project_env_dependencies( + self.name, + { + "project_targets": self.targets, + "piotest_running_name": build_vars.get("piotest_running_name"), + }, + ) result = PlatformFactory.new(self.options["platform"], autoinstall=True).run( build_vars, build_targets, self.silent, self.verbose, self.jobs ) From 6f0b1fbb91e4142015ce893afdba1f8bfdb6aff5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Apr 2023 22:32:18 +0300 Subject: [PATCH 77/98] Resolved an issue where multiple targets were not executed sequentially // Resolve #4604 --- HISTORY.rst | 1 + platformio/builder/main.py | 6 ++++++ platformio/platform/_run.py | 7 +++---- platformio/run/processor.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 786634ae70..6869230cf2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,7 @@ PlatformIO Core 6 * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) * Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) +* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e3932be5e5..8670278345 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -250,3 +250,9 @@ ) Default("sizedata") + +# issue #4604: process targets sequentially +for index, target in enumerate( + [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:] +): + env.Depends(target, COMMAND_LINE_TARGETS[index]) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 4fc41f9c67..dfd7c40675 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -93,6 +93,9 @@ def _run_scons(self, variables, targets, jobs): args.append("PIOVERBOSE=%d" % int(self.verbose)) # pylint: disable=protected-access args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): args.append("--clean") @@ -103,10 +106,6 @@ def _run_scons(self, variables, targets, jobs): elif targets: args.extend(targets) - # encode and append variables - for key, value in variables.items(): - args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - # force SCons output to Unicode os.environ["PYTHONIOENCODING"] = "utf-8" diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 3d427a1bac..329c4bf6d0 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -68,7 +68,7 @@ def process(self): build_vars = self.get_build_variables() is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) - build_targets = list(set(self.targets) - set(KNOWN_ALLCLEAN_TARGETS)) + build_targets = [t for t in self.targets if t not in KNOWN_ALLCLEAN_TARGETS] # pre-clean if is_clean: From 2312ca929d0abac35d670b9e992f0898594c38d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Apr 2023 14:19:31 +0300 Subject: [PATCH 78/98] Cleanup code --- platformio/telemetry.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 0798641955..e9cbcd583d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -105,18 +105,6 @@ def _prefill_sysargs(self): self["cd3"] = " ".join(args) def _prefill_custom_data(self): - def _filter_args(items): - result = [] - stop = False - for item in items: - item = str(item).lower() - result.append(item) - if stop: - break - if item == "account": - stop = True - return result - caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 From 2c94fb2aad804e10bae53704e113e107d04c2b89 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Apr 2023 14:24:34 +0300 Subject: [PATCH 79/98] Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs // Resolve #4425 , Resolve #4493 , Resolve #4607 --- HISTORY.rst | 1 + docs | 2 +- platformio/__init__.py | 4 +- platformio/package/manager/core.py | 139 +---------------------------- platformio/remote/cli.py | 10 ++- 5 files changed, 13 insertions(+), 143 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6869230cf2..0f9223e94c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ PlatformIO Core 6 * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) * Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) * Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) +* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 890d8e20fe..88d4514ccc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 890d8e20fed40f653ac84ff366ac16d805b4bf01 +Subproject commit 88d4514cccd0b99d7d5578e27c1396cb61b617e3 diff --git a/platformio/__init__.py b/platformio/__init__.py index 1f3229d9ba..8b19e12b29 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - VERSION = (6, 1, "7b2") __version__ = ".".join([str(s) for s in VERSION]) @@ -46,7 +44,7 @@ __core_packages__ = { "contrib-piohome": "~3.4.2", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), + "contrib-pioremote": "~1.0.0", "tool-scons": "~4.40502.0", "tool-cppcheck": "~1.270.0", "tool-clangtidy": "~1.150005.0", diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 28fab07b29..ca5c5e0136 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -12,19 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os -import shutil -import subprocess -import sys -from datetime import date -from platformio import __core_packages__, exception, fs, util -from platformio.exception import UserSideException +from platformio import __core_packages__, exception from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageItem, PackageSpec -from platformio.proc import get_pythonexe_path +from platformio.package.meta import PackageSpec def get_installed_core_packages(): @@ -98,131 +91,3 @@ def remove_unnecessary_core_packages(dry_run=False): pm.uninstall(pkg) return candidates - - -def inject_contrib_pysite(): - # pylint: disable=import-outside-toplevel - from site import addsitedir - - try: - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - except UnknownPackageError: - pm = ToolPackageManager() - contrib_pysite_dir = build_contrib_pysite_package( - os.path.join(pm.package_dir, "contrib-pysite") - ) - - if contrib_pysite_dir in sys.path: - return True - - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) - - try: - # pylint: disable=import-error,unused-import,unused-variable - from OpenSSL import SSL - - except: # pylint: disable=bare-except - build_contrib_pysite_package(contrib_pysite_dir) - - return True - - -def build_contrib_pysite_package(target_dir, with_metadata=True): - systype = util.get_systype() - if os.path.isdir(target_dir): - fs.rmtree(target_dir) - os.makedirs(target_dir) - - # issue 3865: There is no "rustup" in "Raspbian GNU/Linux 10 (buster)" - os.environ["CRYPTOGRAPHY_DONT_BUILD_RUST"] = "1" - - # build dependencies - args = [ - get_pythonexe_path(), - "-m", - "pip", - "install", - "--no-compile", - "-t", - target_dir, - ] - if "linux" in systype: - args.extend(["--no-binary", ":all:"]) - try: - subprocess.run(args + get_contrib_pysite_deps(), check=True, env=os.environ) - except subprocess.CalledProcessError as exc: - if "linux" in systype: - raise UserSideException( - "\n\nPlease ensure that the next packages are installed:\n\n" - "sudo apt install python3-dev libffi-dev libssl-dev\n" - ) from exc - raise exc - - # build manifests - with open( - os.path.join(target_dir, "package.json"), mode="w", encoding="utf8" - ) as fp: - json.dump( - dict( - name="contrib-pysite", - version="2.%d%d.%s" - % ( - sys.version_info.major, - sys.version_info.minor, - date.today().strftime("%y%m%d"), - ), - system=list( - set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) - ) - if systype.startswith("linux_arm") - else systype, - description="Extra Python package for PlatformIO Core", - keywords=["platformio", "platformio-core"], - homepage="https://docs.platformio.org/page/core/index.html", - repository={ - "type": "git", - "url": "https://github.com/platformio/platformio-core", - }, - ), - fp, - ) - - # generate package metadata - if with_metadata: - pm = ToolPackageManager() - pkg = PackageItem(target_dir) - pkg.metadata = pm.build_metadata( - target_dir, PackageSpec(owner="platformio", name="contrib-pysite") - ) - pkg.dump_meta() - - # remove unused files - for root, dirs, files in os.walk(target_dir): - for t in ("_test", "test", "tests"): - if t in dirs: - shutil.rmtree(os.path.join(root, t)) - for name in files: - if name.endswith((".chm", ".pyc")): - os.remove(os.path.join(root, name)) - - return target_dir - - -def get_contrib_pysite_deps(): - systype = util.get_systype() - twisted_version = "22.1.0" - if "linux_arm" in systype: - result = [ - # twisted[tls], see setup.py for %twisted_version% - "twisted == %s" % twisted_version, - # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", - "pyopenssl >= 16.0.0, <= 21.0.0", - "service_identity >= 18.1.0, <= 21.1.0", - ] - else: - result = ["twisted[tls] == %s" % twisted_version] - if "windows" in systype: - result.append("pywin32 != 226") - return result diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index f9462e0f31..f1775d498b 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -17,7 +17,9 @@ import os import subprocess +import sys import threading +from site import addsitedir from tempfile import mkdtemp from time import sleep @@ -29,7 +31,7 @@ device_monitor_cmd, get_project_options, ) -from platformio.package.manager.core import inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir from platformio.project.exception import NotPlatformIOProjectError from platformio.project.options import ProjectOptions from platformio.run.cli import cli as cmd_run @@ -41,7 +43,11 @@ @click.pass_context def cli(ctx, agent): ctx.obj = agent - inject_contrib_pysite() + # inject twisted dependencies + contrib_dir = get_core_package_dir("contrib-pioremote") + if contrib_dir not in sys.path: + addsitedir(contrib_dir) + sys.path.insert(0, contrib_dir) @cli.group("agent", short_help="Start a new agent or list active") From 21c4f091e254635a8993e0564b16ba07b8cdd9e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Apr 2023 15:12:01 +0300 Subject: [PATCH 80/98] Skip mbed legacy examples from PIO Core CI --- tests/test_examples.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_examples.py b/tests/test_examples.py index 852bb7aa6c..b49aa90ddd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -48,6 +48,8 @@ def pytest_generate_tests(metafunc): for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue + if "mbed-legacy-examples" in root: + continue group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] From 9dc6ed031fd2f13a44c282d560959d4186fbd0b6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Apr 2023 22:56:25 +0300 Subject: [PATCH 81/98] Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 // Resolve #4540 --- HISTORY.rst | 9 +-- platformio/commands/upgrade.py | 115 ++++++++++++--------------------- platformio/exception.py | 9 --- platformio/maintenance.py | 2 - 4 files changed, 48 insertions(+), 87 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0f9223e94c..0e8d5688fe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,22 +18,23 @@ PlatformIO Core 6 6.1.7 (2023-??-??) ~~~~~~~~~~~~~~~~~~ -* Implemented a new feature to store device monitor logs in the project's "logs" folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) -* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) -* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to +* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process * Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) +* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) * Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow +* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) * Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) * Resolved an issue where organization details could not be updated using the `pio org update `__ command * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) * Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) * Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) -* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) +* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 `_) 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 2766dbd5e1..da8a2b3e55 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -13,23 +13,28 @@ # limitations under the License. import json -import os import re -from zipfile import ZipFile +import subprocess import click from platformio import VERSION, __version__, app, exception -from platformio.compat import IS_WINDOWS from platformio.http import fetch_remote_content from platformio.package.manager.core import update_core_packages -from platformio.proc import exec_command, get_pythonexe_path -from platformio.project.helpers import get_project_cache_dir +from platformio.proc import get_pythonexe_path + +PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json" +DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip" +DEVELOP_INIT_SCRIPT_URL = ( + "https://raw.githubusercontent.com/platformio/platformio-core" + "/develop/platformio/__init__.py" +) @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") -def cli(dev): +@click.option("--verbose", "-v", is_flag=True) +def cli(dev, verbose): update_core_packages() if not dev and __version__ == get_latest_version(): return click.secho( @@ -38,29 +43,26 @@ def cli(dev): fg="green", ) - click.secho("Please wait while upgrading PlatformIO ...", fg="yellow") + click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow") + python_exe = get_pythonexe_path() to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") - cmds = ( - ["pip", "install", "--upgrade", download_dist_package(to_develop)], - ["platformio", "--version"], - ) + pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio" - cmd = None - r = {} try: - for cmd in cmds: - cmd = [get_pythonexe_path(), "-m"] + cmd - r = exec_command(cmd) - - # try pip with disabled cache - if r["returncode"] != 0 and cmd[2] == "pip": - cmd.insert(3, "--no-cache-dir") - r = exec_command(cmd) - - assert r["returncode"] == 0 - assert "version" in r["out"] - actual_version = r["out"].strip().split("version", 1)[1].strip() + subprocess.run( + [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], + check=True, + capture_output=not verbose, + ) + r = subprocess.run( + [python_exe, "-m", "platformio", "--version"], + check=True, + capture_output=True, + text=True, + ) + assert "version" in r.stdout + actual_version = r.stdout.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", @@ -71,52 +73,24 @@ def cli(dev): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) - except Exception as exc: - if not r: - raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc - permission_errors = ("permission denied", "not permitted") - if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: - click.secho( - """ ------------------ -Permission denied ------------------ -You need the `sudo` permission to install Python packages. Try - -> sudo pip install -U platformio - -WARNING! Don't use `sudo` for the rest PlatformIO commands. -""", - fg="yellow", - err=True, - ) - raise exception.ReturnErrorCode(1) - raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]])) + except (AssertionError, subprocess.CalledProcessError) as exc: + click.secho( + "\nWarning!!! Could not automatically upgrade the PlatformIO Core.", + fg="red", + ) + click.secho( + "Please upgrade it manually using the following command:\n", + fg="red", + ) + click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan") + raise exception.ReturnErrorCode(1) from exc return True -def download_dist_package(to_develop): - if not to_develop: - return "platformio" - dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" - cache_dir = get_project_cache_dir() - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") - try: - with open(pkg_name, "wb") as fp: - r = exec_command( - ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True - ) - assert r["returncode"] == 0 - # check ZIP structure - with ZipFile(pkg_name) as zp: - assert zp.testzip() is None - return pkg_name - except: # pylint: disable=bare-except - pass - return dl_url +def get_pkg_spec(to_develop): + if to_develop: + return def get_latest_version(): @@ -133,10 +107,7 @@ def get_latest_version(): def get_develop_latest_version(): version = None - content = fetch_remote_content( - "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py" - ) + content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL) for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): @@ -153,5 +124,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - content = fetch_remote_content("https://pypi.org/pypi/platformio/json") + content = fetch_remote_content(PYPI_JSON_URL) return json.loads(content)["info"]["version"] diff --git a/platformio/exception.py b/platformio/exception.py index c9afd751ea..bf2cec8c90 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -93,15 +93,6 @@ class CIBuildEnvsEmpty(UserSideException): ) -class UpgradeError(PlatformioException): - MESSAGE = """{0} - -* Upgrade using `pip install -U platformio` -* Try different installation/upgrading steps: - https://docs.platformio.org/page/installation.html -""" - - class HomeDirPermissionsError(UserSideException): MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 5518495364..0406f8515f 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -166,8 +166,6 @@ def after_upgrade(ctx): action="Upgrade", label="%s > %s" % (last_version, __version__), ) - else: - raise exception.UpgradeError("Auto upgrading...") # PlatformIO banner click.echo("*" * terminal_width) From 2908efd337cd7acfa1bf5c8bba53412ddafdc7e6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Apr 2023 22:56:39 +0300 Subject: [PATCH 82/98] Bump version to 6.1.7rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 8b19e12b29..30af3e611a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "7b2") +VERSION = (6, 1, "7rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1d0d89a4faeae20e746523239780cf6e887cf0dd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Apr 2023 16:48:14 +0300 Subject: [PATCH 83/98] Remove unused code --- platformio/home/rpc/handlers/os.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 333618b521..2add5afb4c 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -156,9 +156,5 @@ def _cmp(x, y): @staticmethod def get_logical_devices(): - items = [] - for item in list_logical_devices(): - if item["name"]: - item["name"] = item["name"] - items.append(item) - return items + return list_logical_devices() + From 939f8e0812bb05f53e6b3a1b4b7945279e9fabca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Apr 2023 16:48:33 +0300 Subject: [PATCH 84/98] Fetch framework data from PIO Registry --- scripts/docspregen.py | 56 +++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/scripts/docspregen.py b/scripts/docspregen.py index b0fe2adee5..09dd6777ec 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,7 @@ import click # noqa: E402 -from platformio import fs, util # noqa: E402 +from platformio import fs # noqa: E402 from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 from platformio.platform.factory import PlatformFactory # noqa: E402 @@ -41,6 +41,25 @@ SKIP_DEBUG_TOOLS = ["esp-bridge", "esp-builtin"] +STATIC_FRAMEWORK_DATA = { + "arduino": { + "title": "Arduino", + "description": ( + "Arduino Wiring-based Framework allows writing cross-platform software " + "to control devices attached to a wide range of Arduino boards to " + "create all kinds of creative coding, interactive objects, spaces " + "or physical experiences." + ), + }, + "freertos": { + "title": "FreeRTOS", + "description": ( + "FreeRTOS is a real-time operating system kernel for embedded devices " + "that has been ported to 40 microcontroller platforms." + ), + }, +} + DOCS_ROOT_DIR = os.path.realpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "docs") ) @@ -92,16 +111,29 @@ def install_platforms(): @functools.cache def get_frameworks(): items = {} - for pkg in PlatformPackageManager().get_installed(): - p = PlatformFactory.new(pkg) + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) for name, options in (p.frameworks or {}).items(): - if name in items or not set(options.keys()).issuperset( - set(["title", "description"]) - ): + if name in items: continue - items[name] = dict( - name=name, title=options["title"], description=options["description"] - ) + if name in STATIC_FRAMEWORK_DATA: + items[name] = dict( + name=name, + title=STATIC_FRAMEWORK_DATA[name]["title"], + description=STATIC_FRAMEWORK_DATA[name]["description"], + ) + continue + title = options.get("title") or name.title() + description = options.get("description") + if "package" in options: + regdata = REGCLIENT.get_package( + "tool", + p.packages[options["package"]].get("owner", "platformio"), + options["package"], + ) + title = regdata["title"] or title + description = regdata["description"] + items[name] = dict(name=name, title=title, description=description) return sorted(items.values(), key=lambda item: item["name"]) @@ -129,7 +161,7 @@ def generate_boards_table(boards, skip_columns=None): ) # add header - for (name, template) in columns: + for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " @@ -156,7 +188,7 @@ def generate_boards_table(boards, skip_columns=None): rom=fs.humanize_file_size(data["rom"]), ) - for (name, template) in columns: + for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " @@ -905,7 +937,7 @@ def update_embedded_board(rst_path, board): - On-board - Default""" ) - for (tool_name, tool_data) in sorted(board["debug"]["tools"].items()): + for tool_name, tool_data in sorted(board["debug"]["tools"].items()): lines.append( """ * - {tool} - {onboard} From 09f12694401db9d258c6f2f1a007e1aa79a714cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Apr 2023 16:48:46 +0300 Subject: [PATCH 85/98] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 88d4514ccc..54a797fb54 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 88d4514cccd0b99d7d5578e27c1396cb61b617e3 +Subproject commit 54a797fb5440a069d4790d495499552e5a7edb35 diff --git a/examples b/examples index f1e0ecff5b..3e23b5ac43 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f1e0ecff5b8dccc04ee6ecbca67e9ffaf0617eb2 +Subproject commit 3e23b5ac43ab7ec277ce7d68618458f3980f8089 From 4596acab81b582e8c63c907eeb8ad14ebb34bdd6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 May 2023 11:37:44 +0300 Subject: [PATCH 86/98] Add os.call_path_module_func RPC --- platformio/home/rpc/handlers/os.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 2add5afb4c..51b50ee353 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -89,6 +89,14 @@ def reveal_file(path): def open_file(path): return click.launch(path) + @staticmethod + def call_path_module_func(name, args, **kwargs): + return getattr(os.path, name)(*args, **kwargs) + + @staticmethod + def get_path_separator(): + return os.sep + @staticmethod def is_file(path): return os.path.isfile(path) @@ -157,4 +165,3 @@ def _cmp(x, y): @staticmethod def get_logical_devices(): return list_logical_devices() - From f6aa95a4fe5993a6684321727436b9c5b1d4ac48 Mon Sep 17 00:00:00 2001 From: Dawid Nowak Date: Mon, 1 May 2023 13:30:27 +0200 Subject: [PATCH 87/98] test/runners/unity.py: fixed warnings: `[-Wunused-parameter]` (#4610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They appear when the tests are run with the `Wextra` flag. I know of 3 possible solutions: * A.) `static_cast(unusedParameter);` – the preferred C++ way, but `unity` can also be used with `C` * B.) `(void) unusedParameter;` – "old" C–style casting – the one I used – compatible with both C and C++ * C.) remove `unusedParameter` var from the function arguments – but keeping the name better explains what it is ``` Library Manager: Installing throwtheswitch/Unity @ ^2.5.2 Unpacking [####################################] 100% Library Manager: Unity@2.5.2 has been installed! .pio/build/native/unity_config/unity_config.c:39:37: warning: unused parameter 'baudrate' [-Wunused-parameter] void unityOutputStart(unsigned long baudrate) { } ^ 1 warning generated. Testing... ``` --- platformio/test/runners/unity.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index 1c8ef795bf..25fea0ab62 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -114,7 +114,7 @@ class UnityTestRunner(TestRunnerBase): native=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -155,7 +155,7 @@ class UnityTestRunner(TestRunnerBase): espidf=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -165,7 +165,7 @@ class UnityTestRunner(TestRunnerBase): zephyr=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { printk("%c", c); } void unityOutputFlush(void) { } void unityOutputComplete(void) { } From 7876626f049741ebc9e43614238a058edebbded0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 May 2023 21:47:10 +0300 Subject: [PATCH 88/98] Add BaseRPCHandler --- platformio/home/rpc/handlers/account.py | 3 ++- platformio/home/rpc/handlers/app.py | 3 ++- platformio/home/rpc/handlers/base.py | 17 +++++++++++++++++ platformio/home/rpc/handlers/ide.py | 3 ++- platformio/home/rpc/handlers/misc.py | 3 ++- platformio/home/rpc/handlers/os.py | 3 ++- platformio/home/rpc/handlers/platform.py | 3 ++- platformio/home/rpc/handlers/project.py | 3 ++- platformio/home/rpc/handlers/registry.py | 3 ++- platformio/home/rpc/server.py | 1 + 10 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 platformio/home/rpc/handlers/base.py diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index 2987329eb5..f7e5dadee0 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -15,9 +15,10 @@ from ajsonrpc.core import JSONRPC20DispatchException from platformio.account.client import AccountClient +from platformio.home.rpc.handlers.base import BaseRPCHandler -class AccountRPC: +class AccountRPC(BaseRPCHandler): @staticmethod def call_client(method, *args, **kwargs): try: diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index dbeafbdbd6..9c79e3147c 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -16,11 +16,12 @@ from pathlib import Path from platformio import __version__, app, fs, util +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project -class AppRPC: +class AppRPC(BaseRPCHandler): IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", diff --git a/platformio/home/rpc/handlers/base.py b/platformio/home/rpc/handlers/base.py new file mode 100644 index 0000000000..7b3f5d8f46 --- /dev/null +++ b/platformio/home/rpc/handlers/base.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseRPCHandler: + factory = None diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index c2eb5e063c..643ee0c596 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -18,9 +18,10 @@ from ajsonrpc.core import JSONRPC20DispatchException from platformio.compat import aio_get_running_loop +from platformio.home.rpc.handlers.base import BaseRPCHandler -class IDERPC: +class IDERPC(BaseRPCHandler): COMMAND_TIMEOUT = 1.5 # in seconds def __init__(self): diff --git a/platformio/home/rpc/handlers/misc.py b/platformio/home/rpc/handlers/misc.py index c384fea99a..ab2508a4dc 100644 --- a/platformio/home/rpc/handlers/misc.py +++ b/platformio/home/rpc/handlers/misc.py @@ -17,10 +17,11 @@ from platformio.cache import ContentCache from platformio.compat import aio_create_task +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.os import OSRPC -class MiscRPC: +class MiscRPC(BaseRPCHandler): async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 51b50ee353..0d6b9f48ad 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -24,6 +24,7 @@ from platformio import fs from platformio.cache import ContentCache from platformio.device.list.util import list_logical_devices +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on @@ -35,7 +36,7 @@ async def request( # pylint: disable=signature-differs,invalid-overridden-metho return await run_in_threadpool(func, *args, **kwargs) -class OSRPC: +class OSRPC(BaseRPCHandler): @staticmethod async def fetch_content(url, data=None, headers=None, cache_valid=None): if not headers: diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py index 14383cd32e..2cc959df03 100644 --- a/platformio/home/rpc/handlers/platform.py +++ b/platformio/home/rpc/handlers/platform.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory -class PlatformRPC: +class PlatformRPC(BaseRPCHandler): @staticmethod def list_installed(options=None): result = [] diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 6c12bcae49..aa5ef83d1d 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -20,6 +20,7 @@ from platformio import exception, fs from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig @@ -29,7 +30,7 @@ from platformio.project.options import get_config_options_schema -class ProjectRPC: +class ProjectRPC(BaseRPCHandler): @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index 34d0974b21..1370f32898 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -15,10 +15,11 @@ from ajsonrpc.core import JSONRPC20DispatchException from starlette.concurrency import run_in_threadpool +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient -class RegistryRPC: +class RegistryRPC(BaseRPCHandler): @staticmethod async def call_client(method, *args, **kwargs): try: diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index 88f06704f9..942961643d 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -37,6 +37,7 @@ def __call__(self, *args, **kwargs): raise NotImplementedError def add_object_handler(self, handler, namespace): + handler.factory = self self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) def on_client_connect(self): From d728e0e873cc28775ed7b9435596b70e9c1c1a35 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 11:21:49 +0300 Subject: [PATCH 89/98] Introduced a new `--sample-code` option to the `pio project init` command --- HISTORY.rst | 5 +- docs | 2 +- platformio/commands/ci.py | 4 +- platformio/home/rpc/handlers/project.py | 93 ++------------ platformio/platform/base.py | 15 ++- platformio/project/commands/init.py | 134 +++++++++++++++----- platformio/project/integration/generator.py | 12 +- 7 files changed, 133 insertions(+), 132 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0e8d5688fe..56da9f0f0c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,10 +18,11 @@ PlatformIO Core 6 6.1.7 (2023-??-??) ~~~~~~~~~~~~~~~~~~ -* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) +* Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project +* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) * Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) * Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process -* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) * Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) * Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to * Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) diff --git a/docs b/docs index 54a797fb54..89233bd239 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 54a797fb5440a069d4790d495499552e5a7edb35 +Subproject commit 89233bd239fb56b45ea33b489bfb5dc213ddabb2 diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index de346afd1f..5be3128666 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -107,8 +107,8 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches ctx.invoke( project_init_cmd, project_dir=build_dir, - board=board, - project_option=project_option, + boards=board, + project_options=project_option, ) # process project diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index aa5ef83d1d..fb77f2582a 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -18,7 +18,7 @@ from ajsonrpc.core import JSONRPC20DispatchException -from platformio import exception, fs +from platformio import app, exception, fs from platformio.home.rpc.handlers.app import AppRPC from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.piocore import PIOCoreRPC @@ -185,83 +185,17 @@ def get_project_examples(): async def init(self, board, framework, project_dir): assert project_dir - state = AppRPC.load_state() if not os.path.isdir(project_dir): os.makedirs(project_dir) - args = ["init", "--board", board] + args = ["init", "--board", board, "--sample-code"] if framework: args.extend(["--project-option", "framework = %s" % framework]) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - return self._generate_project_main(project_dir, board, framework) - - @staticmethod - def _generate_project_main(project_dir, board, framework): - main_content = None - if framework == "arduino": - main_content = "\n".join( - [ - "#include ", - "", - "void setup() {", - " // put your setup code here, to run once:", - "}", - "", - "void loop() {", - " // put your main code here, to run repeatedly:", - "}", - "", - ] - ) - elif framework == "mbed": - main_content = "\n".join( - [ - "#include ", - "", - "int main() {", - "", - " // put your setup code here, to run once:", - "", - " while(1) {", - " // put your main code here, to run repeatedly:", - " }", - "}", - "", - ] - ) - if not main_content: - return project_dir - - is_cpp_project = True - pm = PlatformPackageManager() - try: - board = pm.board_config(board) - platforms = board.get("platforms", board.get("platform")) - if not isinstance(platforms, list): - platforms = [platforms] - c_based_platforms = ["intel_mcs51", "ststm8"] - is_cpp_project = not set(platforms) & set(c_based_platforms) - except exception.PlatformioException: - pass - - with fs.cd(project_dir): - config = ProjectConfig() - src_dir = config.get("platformio", "src_dir") - main_path = os.path.join( - src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") - ) - if os.path.isfile(main_path): - return project_dir - if not os.path.isdir(src_dir): - os.makedirs(src_dir) - with open(main_path, mode="w", encoding="utf8") as fp: - fp.write(main_content.strip()) return project_dir @staticmethod @@ -297,11 +231,9 @@ async def import_arduino(board, use_arduino_libs, arduino_project_dir): args.extend( ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] ) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) @@ -325,13 +257,10 @@ async def import_pio(project_dir): ) shutil.copytree(project_dir, new_project_dir, symlinks=True) - state = AppRPC.load_state() args = ["init"] - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 23a66929b7..3231bb4444 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -207,6 +207,15 @@ def configure_default_packages(self, options, targets): def configure_debug_session(self, debug_config): raise NotImplementedError + def generate_sample_code(self, project_config, environment): + raise NotImplementedError + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): @@ -227,9 +236,3 @@ def get_lib_storages(self): storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index d8cd5b8d2a..a0176f111f 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -24,12 +24,14 @@ from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project from platformio.project.integration.generator import ProjectGenerator +from platformio.project.options import ProjectOptions -def validate_boards(ctx, param, value): # pylint: disable=W0613 +def validate_boards(ctx, param, value): # pylint: disable=unused-argument pm = PlatformPackageManager() for id_ in value: try: @@ -49,21 +51,31 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) -@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) +@click.option( + "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards +) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-e", "--environment", help="Update existing environment") -@click.option("-O", "--project-option", multiple=True) -@click.option("--env-prefix", default="") +@click.option( + "-O", + "--project-option", + "project_options", + multiple=True, + help="A `name=value` pair", +) +@click.option("--sample-code", is_flag=True) @click.option("--no-install-dependencies", is_flag=True) +@click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) def project_init_cmd( project_dir, - board, + boards, ide, environment, - project_option, - env_prefix, + project_options, + sample_code, no_install_dependencies, + env_prefix, silent, ): is_new_project = not is_platformio_project(project_dir) @@ -72,23 +84,23 @@ def project_init_cmd( print_header(project_dir) init_base_project(project_dir) - if environment: - update_project_env(project_dir, environment, project_option) - elif board: - update_board_envs(project_dir, board, project_option, env_prefix) - with fs.cd(project_dir): + if environment: + update_project_env(environment, project_options) + elif boards: + update_board_envs(project_dir, boards, project_options, env_prefix) + generator = None config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) if ide: config.validate() # init generator and pick the best env if user didn't specify - generator = ProjectGenerator(config, environment, ide, board) + generator = ProjectGenerator(config, environment, ide, boards) if not environment: environment = generator.env_name # resolve project dependencies - if not no_install_dependencies and (environment or board): + if not no_install_dependencies and (environment or boards): install_project_dependencies( options=dict( project_dir=project_dir, @@ -97,6 +109,9 @@ def project_init_cmd( ) ) + if environment and sample_code: + init_sample_code(config, environment) + if generator: if not silent: click.echo( @@ -104,8 +119,8 @@ def project_init_cmd( ) generator.generate() - if is_new_project: - init_cvs_ignore(project_dir) + if is_new_project: + init_cvs_ignore() if not silent: print_footer(is_new_project) @@ -289,15 +304,15 @@ def init_test_readme(test_dir): ) -def init_cvs_ignore(project_dir): - conf_path = os.path.join(project_dir, ".gitignore") +def init_cvs_ignore(): + conf_path = ".gitignore" if os.path.isfile(conf_path): return with open(conf_path, mode="w", encoding="utf8") as fp: fp.write(".pio\n") -def update_board_envs(project_dir, board_ids, project_option, env_prefix): +def update_board_envs(project_dir, boards, extra_project_options, env_prefix): config = ProjectConfig( os.path.join(project_dir, "platformio.ini"), parse_extra=False ) @@ -309,7 +324,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): pm = PlatformPackageManager() modified = False - for id_ in board_ids: + for id_ in boards: board_config = pm.board_config(id_) if id_ in used_boards: continue @@ -322,7 +337,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): if frameworks: envopts["framework"] = frameworks[0] - for item in project_option: + for item in extra_project_options: if "=" not in item: continue _name, _value = item.split("=", 1) @@ -338,21 +353,74 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): config.save() -def update_project_env(project_dir, environment, project_option): - if not project_option: +def update_project_env(environment, extra_project_options=None): + if not extra_project_options: return + env_section = "env:%s" % environment + option_to_sections = {"platformio": [], env_section: []} + for item in extra_project_options: + assert "=" in item + name, value = item.split("=", 1) + name = name.strip() + destination = env_section + for option in ProjectOptions.values(): + if option.scope in option_to_sections and option.name == name: + destination = option.scope + break + option_to_sections[destination].append((name, value.strip())) + config = ProjectConfig( - os.path.join(project_dir, "platformio.ini"), parse_extra=False + "platformio.ini", parse_extra=False, expand_interpolations=False ) + for section, options in option_to_sections.items(): + if not config.has_section(section): + config.add_section(section) + for name, value in options: + config.set(section, name, value) - section = "env:%s" % environment - if not config.has_section(section): - config.add_section(section) + config.save() - for item in project_option: - if "=" not in item: - continue - _name, _value = item.split("=", 1) - config.set(section, _name.strip(), _value.strip()) - config.save() +def init_sample_code(config, environment): + platform_spec = config.get(f"env:{environment}", "platform", None) + if not platform_spec: + return None + p = PlatformFactory.new(platform_spec) + try: + return p.generate_sample_code(config, environment) + except NotImplementedError: + pass + + framework = config.get(f"env:{environment}", "framework", None) + if framework != ["arduino"]: + return None + main_content = """ +#include + +// put function declarations here: +int myFunction(int, int); + +void setup() { + // put your setup code here, to run once: + int result = myFunction(2, 3); +} + +void loop() { + // put your main code here, to run repeatedly: +} + +// put function definitions here: +int myFunction(int x, int y) { + return x + y; +} +""" + is_cpp_project = p.name not in ["intel_mcs51", "ststm8"] + src_dir = config.get("platformio", "src_dir") + main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")) + if os.path.isfile(main_path): + return None + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, mode="w", encoding="utf8") as fp: + fp.write(main_content.strip()) + return True diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 40b8e87a0f..654e5ac353 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -25,28 +25,28 @@ class ProjectGenerator: - def __init__(self, config, env_name, ide, board_ids=None): + def __init__(self, config, env_name, ide, boards=None): self.config = config self.project_dir = os.path.dirname(config.path) self.forced_env_name = env_name - self.env_name = str(env_name or self.get_best_envname(board_ids)) + self.env_name = str(env_name or self.get_best_envname(boards)) self.ide = str(ide) - def get_best_envname(self, board_ids=None): + def get_best_envname(self, boards=None): envname = None default_envs = self.config.default_envs() if default_envs: envname = default_envs[0] - if not board_ids: + if not boards: return envname for env in self.config.envs(): - if not board_ids: + if not boards: return env if not envname: envname = env items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in board_ids: + if "board" in items and items.get("board") in boards: return env return envname From f9cbf6cb974ddb2107aa704689a56e6f46a40f48 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 11:22:19 +0300 Subject: [PATCH 90/98] Set caller id when available --- platformio/maintenance.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 0406f8515f..3110b2c192 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -22,7 +22,6 @@ from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI -from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages @@ -67,15 +66,15 @@ def on_platformio_exception(e): def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") + if not caller: + if os.getenv("CODESPACES"): + caller = "codespaces" + elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): + caller = "vscode" + elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): + caller = "gitpod" if caller: - return app.set_session_var("caller_id", caller) - if os.getenv("CODESPACES"): - caller = "codespaces" - elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): - caller = "vscode" - elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): - caller = "gitpod" - return app.set_session_var("caller_id", caller) + app.set_session_var("caller_id", caller) class Upgrader: @@ -84,7 +83,6 @@ def __init__(self, from_version, to_version): self.to_version = pepver_to_semver(to_version) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] @@ -100,11 +98,6 @@ def run(self, ctx): return all(result) - @staticmethod - def _update_dev_platforms(ctx): - ctx.invoke(cmd_platform_update) - return True - @staticmethod def _update_pkg_metadata(_): pm = ToolPackageManager() From 475c5d2a3c29bf9a4a5e9e629a27d17b3b05a517 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 22:25:44 +0300 Subject: [PATCH 91/98] Implement native async RPC core.exec --- platformio/home/cli.py | 23 ++++++- platformio/home/helpers.py | 44 ------------- platformio/home/rpc/handlers/piocore.py | 87 +++++++++++++++++++++++-- platformio/home/run.py | 2 +- 4 files changed, 105 insertions(+), 51 deletions(-) delete mode 100644 platformio/home/helpers.py diff --git a/platformio/home/cli.py b/platformio/home/cli.py index 55d0959494..318c78facf 100644 --- a/platformio/home/cli.py +++ b/platformio/home/cli.py @@ -13,10 +13,11 @@ # limitations under the License. import mimetypes +import socket import click -from platformio.home.helpers import is_port_used +from platformio.compat import IS_WINDOWS from platformio.home.run import run_server from platformio.package.manager.core import get_core_package_dir @@ -95,3 +96,23 @@ def cli(port, host, no_open, shutdown_timeout, session_id): shutdown_timeout=shutdown_timeout, home_url=home_url, ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if IS_WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/home/helpers.py b/platformio/home/helpers.py deleted file mode 100644 index 0e88bde2c8..0000000000 --- a/platformio/home/helpers.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket - -from platformio import util -from platformio.compat import IS_WINDOWS -from platformio.proc import where_is_program - - -@util.memoized(expire="60s") -def get_core_fullpath(): - return where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) - - -def is_port_used(host, port): - socket.setdefaulttimeout(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if IS_WINDOWS: - try: - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass - else: - try: - s.connect((host, port)) - s.close() - except socket.error: - return False - - return True diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 0917a5f4b0..080ba96c7f 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import functools import io import json import os @@ -22,10 +24,45 @@ from ajsonrpc.core import JSONRPC20DispatchException from starlette.concurrency import run_in_threadpool -from platformio import __main__, __version__, fs, proc -from platformio.compat import get_locale_encoding, is_bytes +from platformio import __main__, __version__, app, fs, proc, util +from platformio.compat import ( + IS_WINDOWS, + aio_create_task, + aio_get_running_loop, + get_locale_encoding, + is_bytes, +) from platformio.exception import PlatformioException -from platformio.home import helpers +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class PIOCoreProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future, on_data_callback=None): + self.exit_future = exit_future + self.on_data_callback = on_data_callback + self.stdout = "" + self.stderr = "" + self._is_exited = False + self._encoding = get_locale_encoding() + + def pipe_data_received(self, fd, data): + data = data.decode(self._encoding, "replace") + pipe = ["stdin", "stdout", "stderr"][fd] + if pipe == "stdout": + self.stdout += data + if pipe == "stderr": + self.stderr += data + if self.on_data_callback: + self.on_data_callback(pipe=pipe, data=data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.exit_future.set_result(True) + self._is_exited = True class MultiThreadingStdStream: @@ -59,11 +96,51 @@ def get_value_and_reset(self): return result -class PIOCoreRPC: +@util.memoized(expire="60s") +def get_core_fullpath(): + return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) + + +class PIOCoreRPC(BaseRPCHandler): @staticmethod def version(): return __version__ + async def exec(self, args, options=None): + loop = aio_get_running_loop() + exit_future = loop.create_future() + data_callback = functools.partial( + self._on_exec_data_received, exec_options=options + ) + if args[0] != "--caller" and app.get_session_var("caller_id"): + args = ["--caller", app.get_session_var("caller_id")] + args + transport, protocol = await loop.subprocess_exec( + lambda: PIOCoreProtocol(exit_future, data_callback), + get_core_fullpath(), + *args, + stdin=None, + **options.get("spawn", {}), + ) + await exit_future + transport.close() + return { + "stdout": protocol.stdout, + "stderr": protocol.stderr, + "returncode": transport.get_returncode(), + } + + def _on_exec_data_received(self, exec_options, pipe, data): + notification_method = exec_options.get(f"{pipe}NotificationMethod") + if not notification_method: + return + aio_create_task( + self.factory.notify_clients( + method=notification_method, + params=[data], + actor="frontend", + ) + ) + @staticmethod def setup_multithreading_std_streams(): if isinstance(sys.stdout, MultiThreadingStdStream): @@ -102,7 +179,7 @@ async def call(args, options=None): async def _call_subprocess(args, options): result = await run_in_threadpool( proc.exec_command, - [helpers.get_core_fullpath()] + args, + [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), ) return (result["out"], result["err"], result["returncode"]) diff --git a/platformio/home/run.py b/platformio/home/run.py index 155e2926cf..df3fa47e1f 100644 --- a/platformio/home/run.py +++ b/platformio/home/run.py @@ -45,7 +45,7 @@ def __init__(self, app): self.app = app async def __call__(self, scope, receive, send): - if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""): await shutdown_server() await self.app(scope, receive, send) From f84057706626090e34b8694bb9ef52dde209ae0c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 22:26:12 +0300 Subject: [PATCH 92/98] Remove unnecessary logging --- platformio/project/commands/init.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index a0176f111f..09215e4095 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -127,15 +127,6 @@ def project_init_cmd( def print_header(project_dir): - if project_dir == os.getcwd(): - click.secho("\nThe current working directory ", fg="yellow", nl=False) - try: - click.secho(project_dir, fg="cyan", nl=False) - except UnicodeEncodeError: - click.secho(json.dumps(project_dir), fg="cyan", nl=False) - click.secho(" will be used for the project.", fg="yellow") - click.echo("") - click.echo("The next files/directories have been created in ", nl=False) try: click.secho(project_dir, fg="cyan") @@ -153,18 +144,9 @@ def print_header(project_dir): def print_footer(is_new_project): - if is_new_project: - return click.secho( - "\nProject has been successfully initialized! Useful commands:\n" - "`pio run` - process/build project from the current directory\n" - "`pio run --target upload` or `pio run -t upload` " - "- upload firmware to a target\n" - "`pio run --target clean` - clean project (remove compiled files)" - "\n`pio run --help` - additional information", - fg="green", - ) + action = "initialized" if is_new_project else "updated" return click.secho( - "Project has been successfully updated!", + f"Project has been successfully {action}!", fg="green", ) From c016d6827b04792be3682f2a1304a3cbfee846a9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 22:27:00 +0300 Subject: [PATCH 93/98] Implement RPC notification system --- platformio/home/rpc/server.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index 942961643d..2437e40ec0 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from urllib.parse import parse_qs + import click -from ajsonrpc.core import JSONRPC20Error +from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response from starlette.endpoints import WebSocketEndpoint @@ -32,6 +34,7 @@ def __init__(self, shutdown_timeout=0): self.manager = AsyncJSONRPCResponseManager( Dispatcher(), is_server_error_verbose=True ) + self._clients = {} def __call__(self, *args, **kwargs): raise NotImplementedError @@ -40,13 +43,16 @@ def add_object_handler(self, handler, namespace): handler.factory = self self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) - def on_client_connect(self): + def on_client_connect(self, connection, actor=None): + self._clients[connection] = {"actor": actor} self.connection_nums += 1 if self.shutdown_timer: self.shutdown_timer.cancel() self.shutdown_timer = None - def on_client_disconnect(self): + def on_client_disconnect(self, connection): + if connection in self._clients: + del self._clients[connection] self.connection_nums -= 1 if self.connection_nums < 1: self.connection_nums = 0 @@ -69,6 +75,14 @@ def _auto_shutdown_server(): self.shutdown_timeout, _auto_shutdown_server ) + async def notify_clients(self, method, params=None, actor=None): + for client, options in self._clients.items(): + if actor and options["actor"] != actor: + continue + request = JSONRPC20Request(method, params, is_notification=True) + await client.send_text(self.manager.serialize(request.body)) + return True + class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): def __call__(self, *args, **kwargs): @@ -83,13 +97,17 @@ class WebSocketJSONRPCServer(WebSocketEndpoint): async def on_connect(self, websocket): await websocket.accept() - self.factory.on_client_connect() # pylint: disable=no-member + qs = parse_qs(self.scope.get("query_string", b"")) + actors = qs.get(b"actor") + self.factory.on_client_connect( # pylint: disable=no-member + websocket, actor=actors[0].decode() if actors else None + ) async def on_receive(self, websocket, data): aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): - self.factory.on_client_disconnect() # pylint: disable=no-member + self.factory.on_client_disconnect(websocket) # pylint: disable=no-member async def _handle_rpc(self, websocket, data): # pylint: disable=no-member From 681b90e6b293470af36f7c00e7c8fbde88c514ea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 May 2023 22:28:16 +0300 Subject: [PATCH 94/98] Implement ProjectRPC.create_empty --- platformio/home/rpc/handlers/project.py | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index fb77f2582a..904d00e4d3 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -16,6 +16,7 @@ import shutil import time +import semantic_version from ajsonrpc.core import JSONRPC20DispatchException from platformio import app, exception, fs @@ -265,3 +266,42 @@ async def import_pio(project_dir): args, options={"cwd": new_project_dir, "force_subprocess": True} ) return new_project_dir + + async def create_empty(self, configuration, options=None): + project_dir = os.path.join(configuration["location"], configuration["name"]) + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + + project_options = [] + platform = configuration["platform"] + board = configuration.get("board", {}).get("id") + env_name = board or platform["name"] + if configuration.get("description"): + project_options.append(("description", configuration.get("description"))) + try: + v = semantic_version.Version(platform.get("version")) + assert not v.prerelease + project_options.append( + ("platform", "{name} @ ^{version}".format(**platform)) + ) + except (AssertionError, ValueError): + project_options.append( + ("platform", "{name} @ {version}".format(**platform)) + ) + if board: + project_options.append(("board", board)) + if configuration.get("framework"): + project_options.append(("framework", configuration["framework"]["name"])) + + args = ["project", "init", "-e", env_name, "--sample-code"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + for name, value in project_options: + args.extend(["-O", f"{name}={value}"]) + + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + return await self.factory.manager.dispatcher["core.exec"](args, options=options) From eb20f3410a197094dea3b0b8a4b91b22ed0fd3ca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 May 2023 23:13:10 +0300 Subject: [PATCH 95/98] Minor fixes --- docs | 2 +- platformio/project/commands/init.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs b/docs index 89233bd239..1bc87fa57c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 89233bd239fb56b45ea33b489bfb5dc213ddabb2 +Subproject commit 1bc87fa57cb6b108624c93d6396129d3e6d0fe0a diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index 09215e4095..098c31a57f 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -127,14 +127,14 @@ def project_init_cmd( def print_header(project_dir): - click.echo("The next files/directories have been created in ", nl=False) + click.echo("The following files/directories have been created in ", nl=False) try: click.secho(project_dir, fg="cyan") except UnicodeEncodeError: click.secho(json.dumps(project_dir), fg="cyan") click.echo("%s - Put project header files here" % click.style("include", fg="cyan")) click.echo( - "%s - Put here project specific (private) libraries" + "%s - Put project specific (private) libraries here" % click.style("lib", fg="cyan") ) click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) @@ -355,6 +355,8 @@ def update_project_env(environment, extra_project_options=None): "platformio.ini", parse_extra=False, expand_interpolations=False ) for section, options in option_to_sections.items(): + if not options: + continue if not config.has_section(section): config.add_section(section) for name, value in options: From f8f3e9863e27d0e00c72f58b43a8f193a60a1dae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 May 2023 23:14:21 +0300 Subject: [PATCH 96/98] Bump version to 6.1.7rc2 --- platformio/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 30af3e611a..c5d149a4fa 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "7rc1") +VERSION = (6, 1, "7rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/setup.py b/setup.py index 6e1a243f11..10e1a930e1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.21.*"), + "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From bba5bc9d0fda2f2bc6114ab298eb4e32667674ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 6 May 2023 14:15:40 +0300 Subject: [PATCH 97/98] Lock "urllib3<2" becuase urllib3 v2.0 only supports OpenSSL 1.1.1+ // Resolve #4614 --- docs | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs b/docs index 1bc87fa57c..27fe4be7e3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1bc87fa57cb6b108624c93d6396129d3e6d0fe0a +Subproject commit 27fe4be7e35f890398ebbcc062a33b2e178f52f3 diff --git a/setup.py b/setup.py index 10e1a930e1..42427a85c0 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ "pyelftools>=0.27,<1", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", + "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ "requests==%s" % ("2.27.1" if PY36 else "2.*"), "semantic_version==2.10.*", "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), From 41fb1ca8bd9fadcad74f13742c883fe793393549 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 May 2023 17:58:23 +0300 Subject: [PATCH 98/98] Bump version to 6.1.7 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 56da9f0f0c..3aa710ff90 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,7 +15,7 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** -6.1.7 (2023-??-??) +6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ * Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project diff --git a/docs b/docs index 27fe4be7e3..98609771ba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 27fe4be7e35f890398ebbcc062a33b2e178f52f3 +Subproject commit 98609771ba8f78505adad20e66c6505c262f2650 diff --git a/platformio/__init__.py b/platformio/__init__.py index c5d149a4fa..7d3c8f8abf 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "7rc2") +VERSION = (6, 1, 7) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"