diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2765b55d55..9a64d88295 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,7 +65,7 @@ jobs: mkdir ./${{ env.LATEST_DOCS_DIR }} tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} - name: Delete Artifact - uses: geekyeggo/delete-artifact@v1 + uses: geekyeggo/delete-artifact@v2 with: name: docs - name: Select Docs type diff --git a/.pylintrc b/.pylintrc index 0b9ff1f84b..172b43a191 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,4 +8,5 @@ disable= invalid-name, too-few-public-methods, consider-using-f-string, - cyclic-import + cyclic-import, + use-dict-literal diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..1958f6ced2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +See https://piolabs.com/legal/code-of-conduct.html diff --git a/HISTORY.rst b/HISTORY.rst index b963df382e..3aa710ff90 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ Release Notes .. |LDF| replace:: `LDF `__ .. |INTERPOLATION| replace:: `Interpolation of Values `__ .. |UNITTESTING| replace:: `Unit Testing `__ +.. |DEBUGGING| replace:: `Debugging `__ .. _release_notes_6: @@ -14,6 +15,28 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.7 (2023-05-08) +~~~~~~~~~~~~~~~~~~ + +* Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project +* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) +* Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process +* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) +* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) +* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to +* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) +* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) +* Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow +* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) +* Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) +* Resolved an issue where organization details could not be updated using the `pio org update `__ command +* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) +* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) +* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) +* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) +* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 `_) + 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..3d71c1ec2a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +## Supported Versions + +We are committed to ensuring the security and protection of PlatformIO Core. +To this end, we support only the following versions: + +| Version | Supported | +| ------- | ------------------ | +| 6.1.x | :white_check_mark: | +| < 6.1 | :x: | + +Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. +Therefore, it is important that all developers use only supported versions of the PlatformIO Core. + +## Reporting a Vulnerability + +We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately. + +To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including: + +- A description of the vulnerability and how it can be exploited +- Steps to reproduce the vulnerability +- Any additional information that can help us understand and reproduce the vulnerability + +Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed. + +If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion. + +If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively. + +Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy. + +Thank you for helping us keep our systems and data secure. diff --git a/docs b/docs index 95c339a711..98609771ba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 95c339a71162dc5bd28090f263e641a563229888 +Subproject commit 98609771ba8f78505adad20e66c6505c262f2650 diff --git a/examples b/examples index f98cb5a9be..3e23b5ac43 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f98cb5a9be74b9c87b6e5f3cd530d8d7f0548825 +Subproject commit 3e23b5ac43ab7ec277ce7d68618458f3980f8089 diff --git a/platformio/__init__.py b/platformio/__init__.py index 08d95a7985..7d3c8f8abf 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -VERSION = (6, 1, 6) +VERSION = (6, 1, 7) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -46,8 +44,8 @@ __core_packages__ = { "contrib-piohome": "~3.4.2", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-scons": "~4.40400.0", + "contrib-pioremote": "~1.0.0", + "tool-scons": "~4.40502.0", "tool-cppcheck": "~1.270.0", "tool-clangtidy": "~1.150005.0", "tool-pvs-studio": "~7.18.0", diff --git a/platformio/account/client.py b/platformio/account/client.py index 7aabb24dc0..a898de481d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -21,22 +21,18 @@ class AccountError(PlatformioException): - MESSAGE = "{0}" class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError): - MESSAGE = "You are already authorized with {0} account." class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods - SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): @@ -298,7 +294,7 @@ def remove_org_owner(self, orgname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/owners" % orgname, - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) @@ -351,6 +347,6 @@ def remove_team_member(self, orgname, teamname, username): return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) diff --git a/platformio/account/org/commands/update.py b/platformio/account/org/commands/update.py index 9da9564c2f..86e7cffa8b 100644 --- a/platformio/account/org/commands/update.py +++ b/platformio/account/org/commands/update.py @@ -22,29 +22,27 @@ @click.argument("cur_orgname") @click.option( "--orgname", - callback=lambda _, __, value: validate_orgname(value), + callback=lambda _, __, value: validate_orgname(value) if value else value, help="A new orgname", ) -@click.option("--email") +@click.option( + "--email", + callback=lambda _, __, value: validate_email(value) if value else value, +) @click.option("--displayname") def org_update_cmd(cur_orgname, **kwargs): client = AccountClient() org = client.get_org(cur_orgname) - del org["owners"] - new_org = org.copy() + new_org = { + key: value if value is not None else org[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in org: - new_org[field] = click.prompt( - field.replace("_", " ").capitalize(), default=org[field] - ) - if field == "email": - validate_email(new_org[field]) - if field == "orgname": - validate_orgname(new_org[field]) - else: - new_org.update( - {key.replace("new_", ""): value for key, value in kwargs.items() if value} - ) + for key in kwargs: + new_org[key] = click.prompt(key.capitalize(), default=org[key]) + if key == "email": + validate_email(new_org[key]) + if key == "orgname": + validate_orgname(new_org[key]) client.update_org(cur_orgname, new_org) return click.secho( "The organization `%s` has been successfully updated." % cur_orgname, diff --git a/platformio/account/team/commands/create.py b/platformio/account/team/commands/create.py index 891d33b43b..164ca6acad 100644 --- a/platformio/account/team/commands/create.py +++ b/platformio/account/team/commands/create.py @@ -22,9 +22,7 @@ @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", - callback=lambda _, __, value: validate_orgname_teamname( - value, teamname_validate=True - ), + callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( "--description", diff --git a/platformio/account/team/commands/update.py b/platformio/account/team/commands/update.py index 3ead0feda1..c4c2f8b4f0 100644 --- a/platformio/account/team/commands/update.py +++ b/platformio/account/team/commands/update.py @@ -26,7 +26,7 @@ ) @click.option( "--name", - callback=lambda _, __, value: validate_teamname(value), + callback=lambda _, __, value: validate_teamname(value) if value else value, help="A new team name", ) @click.option( @@ -36,18 +36,14 @@ def team_update_cmd(orgname_teamname, **kwargs): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() team = client.get_team(orgname, teamname) - del team["id"] - del team["members"] - new_team = team.copy() + new_team = { + key: value if value is not None else team[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in team: - new_team[field] = click.prompt( - field.replace("_", " ").capitalize(), default=team[field] - ) - if field == "name": - validate_teamname(new_team[field]) - else: - new_team.update({key: value for key, value in kwargs.items() if value}) + for key in kwargs: + new_team[key] = click.prompt(key.capitalize(), default=team[key]) + if key == "name": + validate_teamname(new_team[key]) client.update_team(orgname, teamname, new_team) return click.secho( "The team %s has been successfully updated." % teamname, diff --git a/platformio/account/validate.py b/platformio/account/validate.py index 7fb7722224..60d3bb2e65 100644 --- a/platformio/account/validate.py +++ b/platformio/account/validate.py @@ -18,8 +18,10 @@ def validate_username(value, field="username"): - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I + ): raise click.BadParameter( "Invalid %s format. " "%s must contain only alphanumeric characters " @@ -30,16 +32,22 @@ def validate_username(value, field="username"): return value +def validate_orgname(value): + return validate_username(value, "Organization name") + + def validate_email(value): - value = str(value).strip() - if not re.match(r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I + ): raise click.BadParameter("Invalid email address") return value def validate_password(value): - value = str(value).strip() - if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): + value = str(value).strip() if value else None + if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): raise click.BadParameter( "Invalid password format. " "Password must contain at least 8 characters" @@ -48,27 +56,11 @@ def validate_password(value): return value -def validate_orgname(value): - return validate_username(value, "Organization name") - - -def validate_orgname_teamname(value, teamname_validate=False): - if ":" not in value: - raise click.BadParameter( - "Please specify organization and team name in the next" - " format - orgname:teamname. For example, mycompany:DreamTeam" - ) - teamname = str(value.strip().split(":", 1)[1]) - if teamname_validate: - validate_teamname(teamname) - return value - - def validate_teamname(value): - if not value: - return value - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I + ): raise click.BadParameter( "Invalid team name format. " "Team name must only contain alphanumeric characters, " @@ -77,3 +69,16 @@ def validate_teamname(value): " not be longer than 20 characters." ) return value + + +def validate_orgname_teamname(value): + value = str(value).strip() if value else None + if not value or ":" not in value: + raise click.BadParameter( + "Please specify organization and team name using the following" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + orgname, teamname = value.split(":", 1) + validate_orgname(orgname) + validate_teamname(teamname) + return value diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl index 1575657537..c419d27ed7 100644 --- a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl +++ b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl @@ -16,7 +16,7 @@ % "request": "launch", % "name": "PIO Debug (skip Pre-Debug)", % "executable": _escape_path(prog_path), -% "projectEnvName": env_name, +% "projectEnvName": env_name if forced_env_name else default_debug_env_name, % "toolchainBinDir": _escape_path(os.path.dirname(gdb_path)), % "internalConsoleOptions": "openOnSessionStart", % } @@ -28,7 +28,7 @@ % debug["name"] = "PIO Debug" % debug["preLaunchTask"] = { % "type": "PlatformIO", -% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and original_env_name else "Pre-Debug", +% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug", % } % noloading = predebug.copy() % noloading["name"] = "PIO Debug (without uploading)" diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b56dc2a248..8670278345 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,9 +28,9 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import app, compat, fs +from platformio import app, fs from platformio.platform.base import PlatformBase -from platformio.proc import get_pythonexe_path +from platformio.proc import get_pythonexe_path, where_is_program from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) @@ -99,6 +99,7 @@ DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) +env.SConscriptChdir(False) # Load variables from CLI env.Replace( @@ -139,19 +140,6 @@ # pylint: disable=protected-access click._compat.isatty = lambda stream: True -if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"): - click.secho("!!! WARNING !!!\t\t" * 3, fg="red") - click.secho( - "Your project is located on a mapped network drive but the " - "current command-line shell does not support the UNC paths.", - fg="yellow", - ) - click.secho( - "Please move your project to a physical drive or check this workaround: " - "https://bit.ly/3kuU5mP\n", - fg="yellow", - ) - if env.subst("$BUILD_CACHE_DIR"): if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): os.makedirs(env.subst("$BUILD_CACHE_DIR")) @@ -170,18 +158,17 @@ env.LoadProjectOptions() env.LoadPioPlatform() -env.SConscriptChdir(0) env.SConsignFile( os.path.join( - "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]) + "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", + ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]), ) ) -for item in env.GetExtraScripts("pre"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("pre"), exports="env") if env.IsCleanTarget(): - env.CleanProject("cleanall" in COMMAND_LINE_TARGETS) + env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0))) env.Exit(0) env.SConscript("$BUILD_SCRIPT") @@ -191,8 +178,7 @@ if env.GetProjectOption("upload_command"): env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) -for item in env.GetExtraScripts("post"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("post"), exports="env") ############################################################################## @@ -208,6 +194,13 @@ Default("checkprogsize") if "compiledb" in COMMAND_LINE_TARGETS: + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")) env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) # Print configured protocols @@ -257,3 +250,9 @@ ) Default("sizedata") + +# issue #4604: process targets sequentially +for index, target in enumerate( + [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:] +): + env.Depends(target, COMMAND_LINE_TARGETS[index]) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py deleted file mode 100644 index 9fa11c8628..0000000000 --- a/platformio/builder/tools/compilation_db.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# Copyright 2020 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pylint: disable=unused-argument, protected-access, unused-variable, import-error -# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py - -import itertools -import json -import os - -import SCons - -from platformio.builder.tools.piobuild import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT -from platformio.proc import where_is_program - -# Implements the ability for SCons to emit a compilation database for the MongoDB project. See -# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation -# database is, and why you might want one. The only user visible entry point here is -# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that -# should hold the compilation database, otherwise, the file defaults to compile_commands.json, -# which is the name that most clang tools search for by default. - -# Is there a better way to do this than this global? Right now this exists so that the -# emitter we add can record all of the things it emits, so that the scanner for the top level -# compilation database can access the complete list, and also so that the writer has easy -# access to write all of the files. But it seems clunky. How can the emitter and the scanner -# communicate more gracefully? -__COMPILATION_DB_ENTRIES = [] - - -# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even -# integrate with the cache, but there doesn't seem to be much call for it. -class __CompilationDbNode(SCons.Node.Python.Value): - def __init__(self, value): - SCons.Node.Python.Value.__init__(self, value) - self.Decider(changed_since_last_build_node) - - -def changed_since_last_build_node(*args, **kwargs): - """Dummy decider to force always building""" - return True - - -def makeEmitCompilationDbEntry(comstr): - """ - Effectively this creates a lambda function to capture: - * command line - * source - * target - :param comstr: unevaluated command line - :return: an emitter which has captured the above - """ - user_action = SCons.Action.Action(comstr) - - def EmitCompilationDbEntry(target, source, env): - """ - This emitter will be added to each c/c++ object build to capture the info needed - for clang tools - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :return: target(s), source(s) - """ - - # Resolve absolute path of toolchain - for cmd in ("CC", "CXX", "AS"): - if cmd not in env: - continue - if os.path.isabs(env[cmd]): - continue - env[cmd] = where_is_program( - env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") - ) - - dbtarget = __CompilationDbNode(source) - - entry = env.__COMPILATIONDB_Entry( - target=dbtarget, - source=[], - __COMPILATIONDB_UTARGET=target, - __COMPILATIONDB_USOURCE=source, - __COMPILATIONDB_UACTION=user_action, - __COMPILATIONDB_ENV=env, - ) - - # Technically, these next two lines should not be required: it should be fine to - # cache the entries. However, they don't seem to update properly. Since they are quick - # to re-generate disable caching and sidestep this problem. - env.AlwaysBuild(entry) - env.NoCache(entry) - - __COMPILATION_DB_ENTRIES.append(dbtarget) - - return target, source - - return EmitCompilationDbEntry - - -def CompilationDbEntryAction(target, source, env, **kw): - """ - Create a dictionary with evaluated command line, target, source - and store that info as an attribute on the target - (Which has been stored in __COMPILATION_DB_ENTRIES array - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :param kw: - :return: None - """ - - command = env["__COMPILATIONDB_UACTION"].strfunction( - target=env["__COMPILATIONDB_UTARGET"], - source=env["__COMPILATIONDB_USOURCE"], - env=env["__COMPILATIONDB_ENV"], - ) - - entry = { - "directory": env.Dir("#").abspath, - "command": command, - "file": str(env["__COMPILATIONDB_USOURCE"][0]), - } - - target[0].write(entry) - - -def WriteCompilationDb(target, source, env): - entries = [] - - for s in __COMPILATION_DB_ENTRIES: - item = s.read() - item["file"] = os.path.abspath(item["file"]) - entries.append(item) - - with open(str(target[0]), mode="w", encoding="utf8") as target_file: - json.dump( - entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") - ) - - -def ScanCompilationDb(node, env, path): - return __COMPILATION_DB_ENTRIES - - -def generate(env, **kwargs): - static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - - env["COMPILATIONDB_COMSTR"] = kwargs.get( - "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" - ) - - components_by_suffix = itertools.chain( - itertools.product( - [".%s" % ext for ext in SRC_C_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_CXX_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_ASM_EXT], - [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")], - ), - ) - - for entry in components_by_suffix: - suffix = entry[0] - builder, base_emitter, command = entry[1] - - # Assumes a dictionary emitter - emitter = builder.emitter[suffix] - builder.emitter[suffix] = SCons.Builder.ListEmitter( - [emitter, makeEmitCompilationDbEntry(command)] - ) - - env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder( - action=SCons.Action.Action(CompilationDbEntryAction, None), - ) - - env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder( - action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), - target_scanner=SCons.Scanner.Scanner( - function=ScanCompilationDb, node_class=None - ), - ) - - def CompilationDatabase(env, target): - result = env.__COMPILATIONDB_Database(target=target, source=[]) - - env.AlwaysBuild(result) - env.NoCache(result) - - return result - - env.AddMethod(CompilationDatabase, "CompilationDatabase") - - -def exists(env): - return True diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 32a744e338..ba194eb076 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -239,7 +239,7 @@ def ProcessUnFlags(env, flags): for scope in unflag_scopes: for unflags in parsed.values(): for unflag in unflags: - for current in env.get(scope, []): + for current in list(env.get(scope, [])): conditions = [ unflag == current, not isinstance(unflag, (tuple, list)) diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index de1f8cf899..c44e72b74f 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -25,7 +25,6 @@ class InoToCPPConverter: - PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template @@ -103,7 +102,7 @@ def merge(self, nodes): return "\n".join(["#include "] + lines) if lines else None def process(self, contents): - out_file = self._main_ino + ".cpp" + out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp" assert self._gcc_preprocess(contents, out_file) contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8e1bec15fa..dee1d5bcef 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -29,7 +29,7 @@ from platformio import exception, fs from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -109,7 +109,6 @@ def get_used_frameworks(env, path): class LibBuilderBase: - CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() # Max depth of nested includes: @@ -298,11 +297,12 @@ def process_extra_options(self): with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: - self.env.SConscriptChdir(1) + self.env.SConscriptChdir(True) self.env.SConscript( os.path.abspath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) + self.env.SConscriptChdir(False) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): @@ -982,7 +982,11 @@ def _is_builtin(spec): try: lm.install(spec) did_install = True - except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc: + except ( + HTTPClientError, + UnknownPackageError, + InternetConnectionError, + ) as exc: click.secho("Warning! %s" % exc, fg="yellow") # reset cache @@ -1157,7 +1161,7 @@ def _print_deps_tree(root, level=0): click.echo("Path: %s" % lb.path, nl=False) click.echo(")", nl=False) click.echo("") - if lb.depbuilders: + if lb.verbose and lb.depbuilders: _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index f2dc774d44..32bb5ffeb6 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -16,7 +16,6 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from platformio import compat, fs @@ -29,10 +28,10 @@ def VerboseAction(_, act, actstr): def IsCleanTarget(env): - return env.GetOption("clean") or ("cleanall" in COMMAND_LINE_TARGETS) + return env.GetOption("clean") -def CleanProject(env, clean_all=False): +def CleanProject(env, fullclean=False): def _relpath(path): if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() @@ -56,7 +55,7 @@ def _clean_dir(path): else: print("Build environment is clean") - if clean_all and os.path.isdir(libdeps_dir): + if fullclean and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) print("Done cleaning") diff --git a/platformio/check/cli.py b/platformio/check/cli.py index e293e8e02d..d99da3137b 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -38,18 +38,15 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) -@click.option("--pattern", multiple=True) +@click.option("--pattern", multiple=True, hidden=True) +@click.option("-f", "--src-filters", multiple=True) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) @@ -67,6 +64,7 @@ def cli( environment, project_dir, project_conf, + src_filters, pattern, flags, severity, @@ -105,14 +103,24 @@ def cli( "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) ) - default_patterns = [ - config.get("platformio", "src_dir"), - config.get("platformio", "include_dir"), + default_src_filters = [ + "+<%s>" % os.path.basename(config.get("platformio", "src_dir")), + "+<%s>" % os.path.basename(config.get("platformio", "include_dir")), ] + + src_filters = ( + src_filters + or pattern + or env_options.get( + "check_src_filters", + env_options.get("check_patterns", default_src_filters), + ) + ) + tool_options = dict( verbose=verbose, silent=silent, - patterns=pattern or env_options.get("check_patterns", default_patterns), + src_filters=src_filters, flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent @@ -265,7 +273,7 @@ def print_defects_stats(results): tabular_data.append(total) headers = ["Component"] - headers.extend([l.upper() for l in severity_labels]) + headers.extend([label.upper() for label in severity_labels]) headers = [click.style(h, bold=True) for h in headers] click.echo(tabulate(tabular_data, headers=headers, numalign="center")) click.echo() diff --git a/platformio/check/defect.py b/platformio/check/defect.py index 15f9df70cb..5689a18478 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -16,6 +16,7 @@ import click +from platformio.exception import PlatformioException from platformio.project.helpers import get_project_dir # pylint: disable=too-many-instance-attributes, redefined-builtin @@ -23,7 +24,6 @@ class DefectItem: - SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 SEVERITY_LOW = 4 @@ -79,7 +79,7 @@ def severity_to_int(label): for key, value in DefectItem.SEVERITY_LABELS.items(): if label == value: return key - raise Exception("Unknown severity label -> %s" % label) + raise PlatformioException("Unknown severity label -> %s" % label) def as_dict(self): return { diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index d51b315880..77f0e55c16 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os import tempfile @@ -30,6 +29,7 @@ def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname self.options = options + self.project_dir = project_dir self.cc_flags = [] self.cxx_flags = [] self.cpp_includes = [] @@ -41,7 +41,7 @@ def __init__(self, project_dir, config, envname, options): self._defects = [] self._on_defect_callback = None self._bad_input = False - self._load_cpp_data(project_dir) + self._load_cpp_data() # detect all defects by default if not self.options.get("severity"): @@ -56,8 +56,8 @@ def __init__(self, project_dir, config, envname, options): for s in self.options["severity"] ] - def _load_cpp_data(self, project_dir): - data = load_build_metadata(project_dir, self.envname) + def _load_cpp_data(self): + data = load_build_metadata(self.project_dir, self.envname) if not data: return self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", "")) @@ -99,6 +99,13 @@ def _extract_defines(language, includes_file): includes_file, ) result = proc.exec_command(cmd, shell=True) + + if result["returncode"] != 0: + click.echo("Warning: Failed to extract toolchain defines!") + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": @@ -201,7 +208,7 @@ def execute_check_cmd(self, cmd): return result @staticmethod - def get_project_target_files(patterns): + def get_project_target_files(project_dir, src_filters): c_extension = (".c",) cpp_extensions = (".cc", ".cpp", ".cxx", ".ino") header_extensions = (".h", ".hh", ".hpp", ".hxx") @@ -216,13 +223,9 @@ def _add_file(path): elif path.endswith(cpp_extensions): result["c++"].append(os.path.abspath(path)) - for pattern in patterns: - for item in glob.glob(pattern, recursive=True): - if not os.path.isdir(item): - _add_file(item) - for root, _, files in os.walk(item, followlinks=True): - for f in files: - _add_file(os.path.join(root, f)) + src_filters = normalize_src_filters(src_filters) + for f in fs.match_src_files(project_dir, src_filters): + _add_file(f) return result @@ -243,3 +246,22 @@ def check(self, on_defect_callback=None): self.clean_up() return self._bad_input + + +# +# Helpers +# + + +def normalize_src_filters(src_filters): + def _normalize(src_filters): + return ( + src_filters + if src_filters.startswith(("+<", "-<")) + else "+<%s>" % src_filters + ) + + if isinstance(src_filters, (list, tuple)): + return " ".join([_normalize(f) for f in src_filters]) + + return _normalize(src_filters) diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 1dd4165eed..496acc7c89 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -64,7 +64,9 @@ def configure_command(self): ): cmd.append("--checks=*") - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files = [] for items in project_files.values(): diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index 0f8db40225..c3c595373c 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -96,7 +96,7 @@ def parse_defect(self, raw_line): # pylint: disable=arguments-differ ) click.echo() self._bad_input = True - self._buffer = "" + self._buffer = "" return None self._buffer = "" @@ -214,7 +214,9 @@ def clean_up(self): if not self.is_flag_set("--addon", self.get_flags("cppcheck")): return - for files in self.get_project_target_files(self.options["patterns"]).values(): + for files in self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ).values(): for f in files: dump_file = f + ".dump" if os.path.isfile(dump_file): @@ -243,7 +245,9 @@ def convert_language_standard(flag): def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files_scope = ("c", "c++") if not any(project_files[t] for t in src_files_scope): click.echo("Error: Nothing to check.") diff --git a/platformio/check/tools/pvsstudio.py b/platformio/check/tools/pvsstudio.py index ded65d1cb6..20979a0019 100644 --- a/platformio/check/tools/pvsstudio.py +++ b/platformio/check/tools/pvsstudio.py @@ -227,7 +227,7 @@ def is_check_successful(cmd_result): def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback for scope, files in self.get_project_target_files( - self.options["patterns"] + self.project_dir, self.options["src_filters"] ).items(): if scope not in ("c", "c++"): continue diff --git a/platformio/cli.py b/platformio/cli.py index bdb46a3d40..41aab522e1 100644 --- a/platformio/cli.py +++ b/platformio/cli.py @@ -19,7 +19,6 @@ class PlatformioCLI(click.MultiCommand): - leftover_args = [] def __init__(self, *args, **kwargs): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 8ca008f13a..ac8367da90 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -42,7 +42,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards[board["platform"]].append(board) terminal_width = shutil.get_terminal_size().columns - for (platform, boards) in sorted(grpboards.items()): + for platform, boards in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) click.secho(platform, bold=True) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index aebd9e7b09..5be3128666 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -51,15 +51,13 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option( "--build-dir", default=tempfile.mkdtemp, - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) @click.option("--keep-build-dir", is_flag=True) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("-O", "--project-option", multiple=True) @click.option("-e", "--environment", "environments", multiple=True) @@ -109,8 +107,8 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches ctx.invoke( project_init_cmd, project_dir=build_dir, - board=board, - project_option=project_option, + boards=board, + project_options=project_option, ) # process project diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ed3c9e8d58..1c150c0914 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -65,9 +65,7 @@ def invoke_command(ctx, cmd, **kwargs): "--storage-dir", multiple=True, default=None, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), help="Manage custom library storage", ) @click.option( diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 2766dbd5e1..da8a2b3e55 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -13,23 +13,28 @@ # limitations under the License. import json -import os import re -from zipfile import ZipFile +import subprocess import click from platformio import VERSION, __version__, app, exception -from platformio.compat import IS_WINDOWS from platformio.http import fetch_remote_content from platformio.package.manager.core import update_core_packages -from platformio.proc import exec_command, get_pythonexe_path -from platformio.project.helpers import get_project_cache_dir +from platformio.proc import get_pythonexe_path + +PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json" +DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip" +DEVELOP_INIT_SCRIPT_URL = ( + "https://raw.githubusercontent.com/platformio/platformio-core" + "/develop/platformio/__init__.py" +) @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") -def cli(dev): +@click.option("--verbose", "-v", is_flag=True) +def cli(dev, verbose): update_core_packages() if not dev and __version__ == get_latest_version(): return click.secho( @@ -38,29 +43,26 @@ def cli(dev): fg="green", ) - click.secho("Please wait while upgrading PlatformIO ...", fg="yellow") + click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow") + python_exe = get_pythonexe_path() to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") - cmds = ( - ["pip", "install", "--upgrade", download_dist_package(to_develop)], - ["platformio", "--version"], - ) + pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio" - cmd = None - r = {} try: - for cmd in cmds: - cmd = [get_pythonexe_path(), "-m"] + cmd - r = exec_command(cmd) - - # try pip with disabled cache - if r["returncode"] != 0 and cmd[2] == "pip": - cmd.insert(3, "--no-cache-dir") - r = exec_command(cmd) - - assert r["returncode"] == 0 - assert "version" in r["out"] - actual_version = r["out"].strip().split("version", 1)[1].strip() + subprocess.run( + [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], + check=True, + capture_output=not verbose, + ) + r = subprocess.run( + [python_exe, "-m", "platformio", "--version"], + check=True, + capture_output=True, + text=True, + ) + assert "version" in r.stdout + actual_version = r.stdout.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", @@ -71,52 +73,24 @@ def cli(dev): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) - except Exception as exc: - if not r: - raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc - permission_errors = ("permission denied", "not permitted") - if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: - click.secho( - """ ------------------ -Permission denied ------------------ -You need the `sudo` permission to install Python packages. Try - -> sudo pip install -U platformio - -WARNING! Don't use `sudo` for the rest PlatformIO commands. -""", - fg="yellow", - err=True, - ) - raise exception.ReturnErrorCode(1) - raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]])) + except (AssertionError, subprocess.CalledProcessError) as exc: + click.secho( + "\nWarning!!! Could not automatically upgrade the PlatformIO Core.", + fg="red", + ) + click.secho( + "Please upgrade it manually using the following command:\n", + fg="red", + ) + click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan") + raise exception.ReturnErrorCode(1) from exc return True -def download_dist_package(to_develop): - if not to_develop: - return "platformio" - dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" - cache_dir = get_project_cache_dir() - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") - try: - with open(pkg_name, "wb") as fp: - r = exec_command( - ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True - ) - assert r["returncode"] == 0 - # check ZIP structure - with ZipFile(pkg_name) as zp: - assert zp.testzip() is None - return pkg_name - except: # pylint: disable=bare-except - pass - return dl_url +def get_pkg_spec(to_develop): + if to_develop: + return def get_latest_version(): @@ -133,10 +107,7 @@ def get_latest_version(): def get_develop_latest_version(): version = None - content = fetch_remote_content( - "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py" - ) + content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL) for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): @@ -153,5 +124,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - content = fetch_remote_content("https://pypi.org/pypi/platformio/json") + content = fetch_remote_content(PYPI_JSON_URL) return json.loads(content)["info"]["version"] diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 64bd1b4399..12d6004af4 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -28,9 +28,9 @@ from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.gdb import GDBClientProcess +from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project from platformio.project.options import ProjectOptions @@ -44,16 +44,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--environment", "-e", metavar="") @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type) @@ -81,61 +77,57 @@ def cli( project_dir = os.getenv(name) with fs.cd(project_dir): - return _debug_in_project_dir( + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=[environment] if environment else None) + env_name = environment or helpers.get_default_debug_env(project_config) + + if not interface: + return helpers.predebug_project( + ctx, project_dir, project_config, env_name, False, verbose + ) + + configure_args = ( ctx, - project_dir, - project_conf, - environment, + project_config, + env_name, load_mode, verbose, - interface, __unprocessed, ) + if helpers.is_gdbmi_mode(): + os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true" + stream = helpers.GDBMIConsoleStream() + with proc.capture_std_streams(stream): + debug_config = _configure(*configure_args) + stream.close() + else: + debug_config = _configure(*configure_args) + _run(project_dir, debug_config, __unprocessed) -def _debug_in_project_dir( - ctx, - project_dir, - project_conf, - environment, - load_mode, - verbose, - interface, - __unprocessed, -): - project_config = ProjectConfig.get_instance(project_conf) - project_config.validate(envs=[environment] if environment else None) - env_name = environment or helpers.get_default_debug_env(project_config) - - if not interface: - return helpers.predebug_project( - ctx, project_dir, project_config, env_name, False, verbose - ) + return None - env_options = project_config.items(env=env_name, as_dict=True) - if "platform" not in env_options: - raise ProjectEnvsNotAvailableError() +def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed): + platform = PlatformFactory.new( + project_config.get(f"env:{env_name}", "platform"), autoinstall=True + ) debug_config = DebugConfigFactory.new( - PlatformFactory.new(env_options["platform"], autoinstall=True), + platform, project_config, env_name, ) - if "--version" in __unprocessed: - return subprocess.run( - [debug_config.client_executable_path, "--version"], check=True + raise ReturnErrorCode( + subprocess.run( + [debug_config.client_executable_path, "--version"], check=True + ).returncode ) try: fs.ensure_udev_rules() except exception.InvalidUdevRules as exc: - click.echo( - helpers.escape_gdbmi_stream("~", str(exc) + "\n") - if helpers.is_gdbmi_mode() - else str(exc) + "\n", - nl=False, - ) + click.echo(str(exc)) rebuild_prog = False preload = debug_config.load_cmds == ["preload"] @@ -157,25 +149,10 @@ def _debug_in_project_dir( debug_config.load_cmds = [] if rebuild_prog: - if helpers.is_gdbmi_mode(): - click.echo( - helpers.escape_gdbmi_stream( - "~", "Preparing firmware for debugging...\n" - ), - nl=False, - ) - stream = helpers.GDBMIConsoleStream() - with proc.capture_std_streams(stream): - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - stream.close() - else: - click.echo("Preparing firmware for debugging...") - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - + click.echo("Preparing firmware for debugging...") + helpers.predebug_project( + ctx, os.getcwd(), project_config, env_name, preload, verbose + ) # save SHA sum of newly created prog if load_mode == "modified": helpers.is_prog_obsolete(debug_config.program_path) @@ -183,6 +160,10 @@ def _debug_in_project_dir( if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") + return debug_config + + +def _run(project_dir, debug_config, __unprocessed): loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) @@ -199,5 +180,3 @@ def _debug_in_project_dir( finally: client.close() loop.close() - - return True diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 24e20516d8..cb49658174 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -146,9 +146,9 @@ def server_ready_pattern(self): def _load_build_data(self): data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True) - if data: - return data - raise DebugInvalidOptionsError("Could not load a build configuration") + if not data: + raise DebugInvalidOptionsError("Could not load a build configuration") + return data def _configure_server(self): # user disabled server in platformio.ini diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index 83d98d9c4c..370fdb7ebd 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -18,7 +18,6 @@ class BlackmagicDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target set language c diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py index 1f155ecbb9..faa5a3413e 100644 --- a/platformio/debug/config/generic.py +++ b/platformio/debug/config/generic.py @@ -16,7 +16,6 @@ class GenericDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset halt diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py index 03a1bd3ac3..a820e0aa7a 100644 --- a/platformio/debug/config/jlink.py +++ b/platformio/debug/config/jlink.py @@ -16,7 +16,6 @@ class JlinkDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 09266b3b9e..7f4d8e1abb 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -16,7 +16,6 @@ class MspdebugDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py index be15b5f46c..413a9c62df 100644 --- a/platformio/debug/config/native.py +++ b/platformio/debug/config/native.py @@ -17,7 +17,6 @@ class NativeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py index e9a57409aa..b998c78217 100644 --- a/platformio/debug/config/qemu.py +++ b/platformio/debug/config/qemu.py @@ -16,7 +16,6 @@ class QemuDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor system_reset diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py index 724be40726..f2f62d66fb 100644 --- a/platformio/debug/config/renode.py +++ b/platformio/debug/config/renode.py @@ -16,7 +16,6 @@ class RenodeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor machine Reset diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py index a1269b2fd4..7f4d0f4cf1 100644 --- a/platformio/debug/exception.py +++ b/platformio/debug/exception.py @@ -20,7 +20,6 @@ class DebugError(PlatformioException): class DebugSupportError(DebugError, UserSideException): - MESSAGE = ( "Currently, PlatformIO does not support debugging for `{0}`.\n" "Please request support at https://github.com/platformio/" diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index a2d89de328..8982c00f14 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -31,7 +31,6 @@ class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods - STDOUT = sys.stdout def write(self, text): @@ -91,7 +90,7 @@ def predebug_project( TestSuite(env_name, debug_testname), project_config, TestRunnerOptions( - verbose=verbose, + verbose=3 if verbose else 0, without_building=False, without_debugging=False, without_uploading=not preload, diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index 2c9280c26f..6ba8c70ec3 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -53,7 +53,6 @@ def process_exited(self): class DebugBaseProcess: - STDOUT_CHUNK_SIZE = 2048 LOG_FILE = None diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 4ce9aebe62..ce7e82c8bf 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -24,7 +24,6 @@ class GDBClientProcess(DebugClientProcess): - PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 89b4095ffc..693dfa445d 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -26,7 +26,6 @@ class DebugServerProcess(DebugBaseProcess): - STD_BUFFER_SIZE = 1024 def __init__(self, debug_config): diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 301de77296..3d3190fe48 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -104,7 +104,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", @@ -132,24 +132,24 @@ def device_monitor_cmd(**options): ensure_ready=True, ).find(initial_port=options["port"]) - if options["menu_char"] == options["exit_char"]: - raise exception.UserSideException( - "--exit-char can not be the same as --menu-char" - ) - - # check for unknown filters - if options["filters"]: - known_filters = set(get_available_filters()) - unknown_filters = set(options["filters"]) - known_filters - if unknown_filters: - options["filters"] = list(known_filters & set(options["filters"])) - click.secho( - ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") - % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), - fg="yellow", + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" ) - start_terminal(options) + # check for unknown filters + if options["filters"]: + known_filters = set(get_available_filters()) + unknown_filters = set(options["filters"]) - known_filters + if unknown_filters: + options["filters"] = list(known_filters & set(options["filters"])) + click.secho( + ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") + % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), + fg="yellow", + ) + + start_terminal(options) def get_project_options(environment=None): diff --git a/platformio/device/monitor/filters/log2file.py b/platformio/device/monitor/filters/log2file.py index bf97b5511f..8e00d29708 100644 --- a/platformio/device/monitor/filters/log2file.py +++ b/platformio/device/monitor/filters/log2file.py @@ -13,7 +13,7 @@ # limitations under the License. import io -import os.path +import os from datetime import datetime from platformio.device.monitor.filters.base import DeviceMonitorFilterBase @@ -27,8 +27,10 @@ def __init__(self, *args, **kwargs): self._log_fp = None def __call__(self): - log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( - "%y%m%d-%H%M%S" + if not os.path.isdir("logs"): + os.makedirs("logs") + log_file_name = os.path.join( + "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S") ) print("--- Logging an output to %s" % os.path.abspath(log_file_name)) # pylint: disable=consider-using-with diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index bec825e982..7b2a9b91b9 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -144,9 +144,8 @@ def new_serial_instance(options): # pylint: disable=too-many-branches except KeyboardInterrupt as exc: click.echo("", err=True) raise UserSideException("User aborted and port is not given") from exc - else: - if not port: - raise UserSideException("Port is not given") + if not port: + raise UserSideException("Port is not given") try: serial_instance = serial.serial_for_url( port, diff --git a/platformio/exception.py b/platformio/exception.py index a8287c047f..bf2cec8c90 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -14,7 +14,6 @@ class PlatformioException(Exception): - MESSAGE = None def __str__(self): # pragma: no cover @@ -26,7 +25,6 @@ def __str__(self): # pragma: no cover class ReturnErrorCode(PlatformioException): - MESSAGE = "{0}" @@ -35,7 +33,6 @@ class UserSideException(PlatformioException): class AbortedByUser(UserSideException): - MESSAGE = "Aborted by user" @@ -49,7 +46,6 @@ class InvalidUdevRules(UserSideException): class MissedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Please install `99-platformio-udev.rules`. \nMore details: " "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" @@ -57,7 +53,6 @@ class MissedUdevRules(InvalidUdevRules): class OutdatedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." "\nMore details: " @@ -71,32 +66,26 @@ class OutdatedUdevRules(InvalidUdevRules): class GetSerialPortsError(PlatformioException): - MESSAGE = "No implementation for your platform ('{0}') available" class GetLatestVersionError(PlatformioException): - MESSAGE = "Can not retrieve the latest PlatformIO version" class InvalidSettingName(UserSideException): - MESSAGE = "Invalid setting with the name '{0}'" class InvalidSettingValue(UserSideException): - MESSAGE = "Invalid value '{0}' for the setting '{1}'" class InvalidJSONFile(PlatformioException): - MESSAGE = "Could not load broken JSON: {0}" class CIBuildEnvsEmpty(UserSideException): - MESSAGE = ( "Can't find PlatformIO build environments.\n" "Please specify `--board` or path to `platformio.ini` with " @@ -104,18 +93,7 @@ class CIBuildEnvsEmpty(UserSideException): ) -class UpgradeError(PlatformioException): - - MESSAGE = """{0} - -* Upgrade using `pip install -U platformio` -* Try different installation/upgrading steps: - https://docs.platformio.org/page/installation.html -""" - - class HomeDirPermissionsError(UserSideException): - MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " "current user and PlatformIO can not store configuration data.\n" @@ -126,7 +104,6 @@ class HomeDirPermissionsError(UserSideException): class CygwinEnvDetected(PlatformioException): - MESSAGE = ( "PlatformIO does not work within Cygwin environment. " "Use native Terminal instead." diff --git a/platformio/fs.py b/platformio/fs.py index bd17950b2b..1f43a7a318 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -24,7 +24,7 @@ import click -from platformio import exception, proc +from platformio import exception from platformio.compat import IS_WINDOWS @@ -181,7 +181,7 @@ def _find_candidates(pattern): result = set() # correct fs directory separator src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) - for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): + for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter): candidates = _find_candidates(pattern) if action == "+": result |= candidates @@ -193,26 +193,7 @@ def _find_candidates(pattern): def to_unix_path(path): if not IS_WINDOWS or not path: return path - return re.sub(r"[\\]+", "/", path) - - -def normalize_path(path): - path = os.path.abspath(path) - if not IS_WINDOWS or not path.startswith("\\\\"): - return path - try: - result = proc.exec_command(["net", "use"]) - if result["returncode"] != 0: - return path - share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)") - for line in result["out"].split("\n"): - share = share_re.search(line) - if not share: - continue - path = path.replace(share.group(2), share.group(1)) - except OSError: - pass - return path + return path.replace("\\", "/") def expanduser(path): diff --git a/platformio/home/cli.py b/platformio/home/cli.py index 55d0959494..318c78facf 100644 --- a/platformio/home/cli.py +++ b/platformio/home/cli.py @@ -13,10 +13,11 @@ # limitations under the License. import mimetypes +import socket import click -from platformio.home.helpers import is_port_used +from platformio.compat import IS_WINDOWS from platformio.home.run import run_server from platformio.package.manager.core import get_core_package_dir @@ -95,3 +96,23 @@ def cli(port, host, no_open, shutdown_timeout, session_id): shutdown_timeout=shutdown_timeout, home_url=home_url, ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if IS_WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/home/helpers.py b/platformio/home/helpers.py deleted file mode 100644 index 0e88bde2c8..0000000000 --- a/platformio/home/helpers.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket - -from platformio import util -from platformio.compat import IS_WINDOWS -from platformio.proc import where_is_program - - -@util.memoized(expire="60s") -def get_core_fullpath(): - return where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) - - -def is_port_used(host, port): - socket.setdefaulttimeout(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if IS_WINDOWS: - try: - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass - else: - try: - s.connect((host, port)) - s.close() - except socket.error: - return False - - return True diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index d857d58703..f7e5dadee0 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -15,9 +15,10 @@ from ajsonrpc.core import JSONRPC20DispatchException from platformio.account.client import AccountClient +from platformio.home.rpc.handlers.base import BaseRPCHandler -class AccountRPC: +class AccountRPC(BaseRPCHandler): @staticmethod def call_client(method, *args, **kwargs): try: @@ -25,5 +26,5 @@ def call_client(method, *args, **kwargs): return getattr(client, method)(*args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Account Call Error", data=str(exc) + code=5000, message="PIO Account Call Error", data=str(exc) ) from exc diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 9d7fc59048..9c79e3147c 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -16,12 +16,12 @@ from pathlib import Path from platformio import __version__, app, fs, util +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project -class AppRPC: - +class AppRPC(BaseRPCHandler): IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", diff --git a/platformio/home/rpc/handlers/base.py b/platformio/home/rpc/handlers/base.py new file mode 100644 index 0000000000..7b3f5d8f46 --- /dev/null +++ b/platformio/home/rpc/handlers/base.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseRPCHandler: + factory = None diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index 5ff9fe3321..643ee0c596 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -18,10 +18,10 @@ from ajsonrpc.core import JSONRPC20DispatchException from platformio.compat import aio_get_running_loop +from platformio.home.rpc.handlers.base import BaseRPCHandler -class IDERPC: - +class IDERPC(BaseRPCHandler): COMMAND_TIMEOUT = 1.5 # in seconds def __init__(self): @@ -51,11 +51,12 @@ async def send_command(self, command, params=None): def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: - return + return False if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] + return True def _process_commands(self): for cmd_id in list(self._cmd_queue): diff --git a/platformio/home/rpc/handlers/misc.py b/platformio/home/rpc/handlers/misc.py index c384fea99a..ab2508a4dc 100644 --- a/platformio/home/rpc/handlers/misc.py +++ b/platformio/home/rpc/handlers/misc.py @@ -17,10 +17,11 @@ from platformio.cache import ContentCache from platformio.compat import aio_create_task +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.os import OSRPC -class MiscRPC: +class MiscRPC(BaseRPCHandler): async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 333618b521..0d6b9f48ad 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -24,6 +24,7 @@ from platformio import fs from platformio.cache import ContentCache from platformio.device.list.util import list_logical_devices +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on @@ -35,7 +36,7 @@ async def request( # pylint: disable=signature-differs,invalid-overridden-metho return await run_in_threadpool(func, *args, **kwargs) -class OSRPC: +class OSRPC(BaseRPCHandler): @staticmethod async def fetch_content(url, data=None, headers=None, cache_valid=None): if not headers: @@ -89,6 +90,14 @@ def reveal_file(path): def open_file(path): return click.launch(path) + @staticmethod + def call_path_module_func(name, args, **kwargs): + return getattr(os.path, name)(*args, **kwargs) + + @staticmethod + def get_path_separator(): + return os.sep + @staticmethod def is_file(path): return os.path.isfile(path) @@ -156,9 +165,4 @@ def _cmp(x, y): @staticmethod def get_logical_devices(): - items = [] - for item in list_logical_devices(): - if item["name"]: - item["name"] = item["name"] - items.append(item) - return items + return list_logical_devices() diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index df2521145a..080ba96c7f 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import functools import io import json import os @@ -22,9 +24,45 @@ from ajsonrpc.core import JSONRPC20DispatchException from starlette.concurrency import run_in_threadpool -from platformio import __main__, __version__, fs, proc -from platformio.compat import get_locale_encoding, is_bytes -from platformio.home import helpers +from platformio import __main__, __version__, app, fs, proc, util +from platformio.compat import ( + IS_WINDOWS, + aio_create_task, + aio_get_running_loop, + get_locale_encoding, + is_bytes, +) +from platformio.exception import PlatformioException +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class PIOCoreProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future, on_data_callback=None): + self.exit_future = exit_future + self.on_data_callback = on_data_callback + self.stdout = "" + self.stderr = "" + self._is_exited = False + self._encoding = get_locale_encoding() + + def pipe_data_received(self, fd, data): + data = data.decode(self._encoding, "replace") + pipe = ["stdin", "stdout", "stderr"][fd] + if pipe == "stdout": + self.stdout += data + if pipe == "stderr": + self.stderr += data + if self.on_data_callback: + self.on_data_callback(pipe=pipe, data=data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.exit_future.set_result(True) + self._is_exited = True class MultiThreadingStdStream: @@ -58,11 +96,51 @@ def get_value_and_reset(self): return result -class PIOCoreRPC: +@util.memoized(expire="60s") +def get_core_fullpath(): + return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) + + +class PIOCoreRPC(BaseRPCHandler): @staticmethod def version(): return __version__ + async def exec(self, args, options=None): + loop = aio_get_running_loop() + exit_future = loop.create_future() + data_callback = functools.partial( + self._on_exec_data_received, exec_options=options + ) + if args[0] != "--caller" and app.get_session_var("caller_id"): + args = ["--caller", app.get_session_var("caller_id")] + args + transport, protocol = await loop.subprocess_exec( + lambda: PIOCoreProtocol(exit_future, data_callback), + get_core_fullpath(), + *args, + stdin=None, + **options.get("spawn", {}), + ) + await exit_future + transport.close() + return { + "stdout": protocol.stdout, + "stderr": protocol.stderr, + "returncode": transport.get_returncode(), + } + + def _on_exec_data_received(self, exec_options, pipe, data): + notification_method = exec_options.get(f"{pipe}NotificationMethod") + if not notification_method: + return + aio_create_task( + self.factory.notify_clients( + method=notification_method, + params=[data], + actor="frontend", + ) + ) + @staticmethod def setup_multithreading_std_streams(): if isinstance(sys.stdout, MultiThreadingStdStream): @@ -94,14 +172,14 @@ async def call(args, options=None): return PIOCoreRPC._process_result(result, to_json) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Core Call Error", data=str(exc) + code=5000, message="PIO Core Call Error", data=str(exc) ) from exc @staticmethod async def _call_subprocess(args, options): result = await run_in_threadpool( proc.exec_command, - [helpers.get_core_fullpath()] + args, + [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), ) return (result["out"], result["err"], result["returncode"]) @@ -132,7 +210,7 @@ def _process_result(result, to_json=False): err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: - raise Exception(text) + raise PlatformioException(text) if not to_json: return text try: diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py new file mode 100644 index 0000000000..2cc959df03 --- /dev/null +++ b/platformio/home/rpc/handlers/platform.py @@ -0,0 +1,61 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory + + +class PlatformRPC(BaseRPCHandler): + @staticmethod + def list_installed(options=None): + result = [] + options = options or {} + + def _matchSearchQuery(p): + searchQuery = options.get("searchQuery") + if not searchQuery: + return True + content_blocks = [p.name, p.title, p.description] + if p.frameworks: + content_blocks.append(" ".join(p.frameworks.keys())) + for board in p.get_boards().values(): + board_data = board.get_brief_data() + for key in ("id", "mcu", "vendor"): + content_blocks.append(board_data.get(key)) + return searchQuery.strip() in " ".join(content_blocks) + + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + if not _matchSearchQuery(p): + continue + result.append( + dict( + __pkg_path=pkg.path, + __pkg_meta=pkg.metadata.as_dict(), + name=p.name, + title=p.title, + description=p.description, + ) + ) + return result + + @staticmethod + def get_boards(spec): + p = PlatformFactory.new(spec) + return sorted( + [b.get_brief_data() for b in p.get_boards().values()], + key=lambda item: item["name"], + ) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 6c12bcae49..904d00e4d3 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -16,10 +16,12 @@ import shutil import time +import semantic_version from ajsonrpc.core import JSONRPC20DispatchException -from platformio import exception, fs +from platformio import app, exception, fs from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig @@ -29,7 +31,7 @@ from platformio.project.options import get_config_options_schema -class ProjectRPC: +class ProjectRPC(BaseRPCHandler): @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) @@ -184,83 +186,17 @@ def get_project_examples(): async def init(self, board, framework, project_dir): assert project_dir - state = AppRPC.load_state() if not os.path.isdir(project_dir): os.makedirs(project_dir) - args = ["init", "--board", board] + args = ["init", "--board", board, "--sample-code"] if framework: args.extend(["--project-option", "framework = %s" % framework]) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - return self._generate_project_main(project_dir, board, framework) - - @staticmethod - def _generate_project_main(project_dir, board, framework): - main_content = None - if framework == "arduino": - main_content = "\n".join( - [ - "#include ", - "", - "void setup() {", - " // put your setup code here, to run once:", - "}", - "", - "void loop() {", - " // put your main code here, to run repeatedly:", - "}", - "", - ] - ) - elif framework == "mbed": - main_content = "\n".join( - [ - "#include ", - "", - "int main() {", - "", - " // put your setup code here, to run once:", - "", - " while(1) {", - " // put your main code here, to run repeatedly:", - " }", - "}", - "", - ] - ) - if not main_content: - return project_dir - - is_cpp_project = True - pm = PlatformPackageManager() - try: - board = pm.board_config(board) - platforms = board.get("platforms", board.get("platform")) - if not isinstance(platforms, list): - platforms = [platforms] - c_based_platforms = ["intel_mcs51", "ststm8"] - is_cpp_project = not set(platforms) & set(c_based_platforms) - except exception.PlatformioException: - pass - - with fs.cd(project_dir): - config = ProjectConfig() - src_dir = config.get("platformio", "src_dir") - main_path = os.path.join( - src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") - ) - if os.path.isfile(main_path): - return project_dir - if not os.path.isdir(src_dir): - os.makedirs(src_dir) - with open(main_path, mode="w", encoding="utf8") as fp: - fp.write(main_content.strip()) return project_dir @staticmethod @@ -296,11 +232,9 @@ async def import_arduino(board, use_arduino_libs, arduino_project_dir): args.extend( ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] ) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) @@ -324,14 +258,50 @@ async def import_pio(project_dir): ) shutil.copytree(project_dir, new_project_dir, symlinks=True) - state = AppRPC.load_state() args = ["init"] - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) return new_project_dir + + async def create_empty(self, configuration, options=None): + project_dir = os.path.join(configuration["location"], configuration["name"]) + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + + project_options = [] + platform = configuration["platform"] + board = configuration.get("board", {}).get("id") + env_name = board or platform["name"] + if configuration.get("description"): + project_options.append(("description", configuration.get("description"))) + try: + v = semantic_version.Version(platform.get("version")) + assert not v.prerelease + project_options.append( + ("platform", "{name} @ ^{version}".format(**platform)) + ) + except (AssertionError, ValueError): + project_options.append( + ("platform", "{name} @ {version}".format(**platform)) + ) + if board: + project_options.append(("board", board)) + if configuration.get("framework"): + project_options.append(("framework", configuration["framework"]["name"])) + + args = ["project", "init", "-e", env_name, "--sample-code"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + for name, value in project_options: + args.extend(["-O", f"{name}={value}"]) + + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + return await self.factory.manager.dispatcher["core.exec"](args, options=options) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index a6d5b8bff1..1370f32898 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -13,17 +13,19 @@ # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException +from starlette.concurrency import run_in_threadpool +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient -class RegistryRPC: +class RegistryRPC(BaseRPCHandler): @staticmethod - def call_client(method, *args, **kwargs): + async def call_client(method, *args, **kwargs): try: client = RegistryClient() - return getattr(client, method)(*args, **kwargs) + return await run_in_threadpool(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="Registry Call Error", data=str(exc) + code=5000, message="Registry Call Error", data=str(exc) ) from exc diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index eb5bd6a527..2437e40ec0 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from urllib.parse import parse_qs + import click +from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher -from ajsonrpc.manager import AsyncJSONRPCResponseManager +from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response from starlette.endpoints import WebSocketEndpoint from platformio.compat import aio_create_task, aio_get_running_loop +from platformio.http import InternetConnectionError from platformio.proc import force_exit class JSONRPCServerFactoryBase: - connection_nums = 0 shutdown_timer = None @@ -31,20 +34,25 @@ def __init__(self, shutdown_timeout=0): self.manager = AsyncJSONRPCResponseManager( Dispatcher(), is_server_error_verbose=True ) + self._clients = {} def __call__(self, *args, **kwargs): raise NotImplementedError def add_object_handler(self, handler, namespace): + handler.factory = self self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) - def on_client_connect(self): + def on_client_connect(self, connection, actor=None): + self._clients[connection] = {"actor": actor} self.connection_nums += 1 if self.shutdown_timer: self.shutdown_timer.cancel() self.shutdown_timer = None - def on_client_disconnect(self): + def on_client_disconnect(self, connection): + if connection in self._clients: + del self._clients[connection] self.connection_nums -= 1 if self.connection_nums < 1: self.connection_nums = 0 @@ -67,6 +75,14 @@ def _auto_shutdown_server(): self.shutdown_timeout, _auto_shutdown_server ) + async def notify_clients(self, method, params=None, actor=None): + for client, options in self._clients.items(): + if actor and options["actor"] != actor: + continue + request = JSONRPC20Request(method, params, is_notification=True) + await client.send_text(self.manager.serialize(request.body)) + return True + class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): def __call__(self, *args, **kwargs): @@ -81,17 +97,30 @@ class WebSocketJSONRPCServer(WebSocketEndpoint): async def on_connect(self, websocket): await websocket.accept() - self.factory.on_client_connect() # pylint: disable=no-member + qs = parse_qs(self.scope.get("query_string", b"")) + actors = qs.get(b"actor") + self.factory.on_client_connect( # pylint: disable=no-member + websocket, actor=actors[0].decode() if actors else None + ) async def on_receive(self, websocket, data): aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): - self.factory.on_client_disconnect() # pylint: disable=no-member + self.factory.on_client_disconnect(websocket) # pylint: disable=no-member async def _handle_rpc(self, websocket, data): # pylint: disable=no-member response = await self.factory.manager.get_response_for_payload(data) if response.error and response.error.data: click.secho("Error: %s" % response.error.data, fg="red", err=True) + if InternetConnectionError.MESSAGE in response.error.data: + response = JSONRPC20Response( + id=response.id, + error=JSONRPC20Error( + code=4008, + message="No Internet Connection", + data=response.error.data, + ), + ) await websocket.send_text(self.factory.manager.serialize(response.body)) diff --git a/platformio/home/run.py b/platformio/home/run.py index 25f1081ec3..df3fa47e1f 100644 --- a/platformio/home/run.py +++ b/platformio/home/run.py @@ -32,6 +32,7 @@ from platformio.home.rpc.handlers.misc import MiscRPC from platformio.home.rpc.handlers.os import OSRPC from platformio.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.home.rpc.handlers.platform import PlatformRPC from platformio.home.rpc.handlers.project import ProjectRPC from platformio.home.rpc.handlers.registry import RegistryRPC from platformio.home.rpc.server import WebSocketJSONRPCServerFactory @@ -44,7 +45,7 @@ def __init__(self, app): self.app = app async def __call__(self, scope, receive, send): - if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""): await shutdown_server() await self.app(scope, receive, send) @@ -73,6 +74,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") + ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform") ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry") path = urlparse(home_url).path diff --git a/platformio/http.py b/platformio/http.py index 9f4f44ddb2..48235f7422 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -37,8 +37,7 @@ def __str__(self): # pragma: no cover return self.message -class InternetIsOffline(UserSideException): - +class InternetConnectionError(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" @@ -204,7 +203,7 @@ def _internet_on(): def ensure_internet_on(raise_exception=False): result = _internet_on() if raise_exception and not result: - raise InternetIsOffline() + raise InternetConnectionError() return result diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7893a23db4..3110b2c192 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -22,9 +22,8 @@ from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI -from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.http import HTTPClientError, InternetIsOffline, ensure_internet_on +from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec @@ -51,7 +50,7 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_prune_system() except ( HTTPClientError, - InternetIsOffline, + InternetConnectionError, exception.GetLatestVersionError, ): click.secho( @@ -67,15 +66,15 @@ def on_platformio_exception(e): def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") + if not caller: + if os.getenv("CODESPACES"): + caller = "codespaces" + elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): + caller = "vscode" + elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): + caller = "gitpod" if caller: - return app.set_session_var("caller_id", caller) - if os.getenv("CODESPACES"): - caller = "codespaces" - elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): - caller = "vscode" - elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): - caller = "gitpod" - return app.set_session_var("caller_id", caller) + app.set_session_var("caller_id", caller) class Upgrader: @@ -84,7 +83,6 @@ def __init__(self, from_version, to_version): self.to_version = pepver_to_semver(to_version) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] @@ -100,11 +98,6 @@ def run(self, ctx): return all(result) - @staticmethod - def _update_dev_platforms(ctx): - ctx.invoke(cmd_platform_update) - return True - @staticmethod def _update_pkg_metadata(_): pm = ToolPackageManager() @@ -166,8 +159,6 @@ def after_upgrade(ctx): action="Upgrade", label="%s > %s" % (last_version, __version__), ) - else: - raise exception.UpgradeError("Auto upgrading...") # PlatformIO banner click.echo("*" * terminal_width) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 6041e3570f..e3eb05aa99 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -39,7 +39,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -55,7 +55,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index 9d538b7daa..c2426c9c6c 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -31,7 +31,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -41,7 +41,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("--only-platforms", is_flag=True, help="List only platform packages") @@ -137,7 +137,7 @@ def list_global_packages(options): only_packages = any( options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data ) - for (type_, pm) in data: + for type_, pm in data: skip_conds = [ only_packages and not options.get(type_) diff --git a/platformio/package/commands/outdated.py b/platformio/package/commands/outdated.py index 5a8c1ea406..18c3783aa1 100644 --- a/platformio/package/commands/outdated.py +++ b/platformio/package/commands/outdated.py @@ -58,7 +58,7 @@ def is_outdated(self): "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) def package_outdated_cmd(project_dir, environments): diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py index 038ffc157a..e9cd0f039b 100644 --- a/platformio/package/commands/pack.py +++ b/platformio/package/commands/pack.py @@ -26,7 +26,7 @@ "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 58230eefe2..228c1411ba 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -47,7 +47,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "--owner", diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py index e393d2a89e..2808d49107 100644 --- a/platformio/package/commands/uninstall.py +++ b/platformio/package/commands/uninstall.py @@ -33,7 +33,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py index 67fc5dd129..8788f7b596 100644 --- a/platformio/package/commands/update.py +++ b/platformio/package/commands/update.py @@ -33,7 +33,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/package/download.py b/platformio/package/download.py index 5045e8565a..17cc1f30eb 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -105,7 +105,7 @@ def start(self, with_progress=True, silent=False): label=label, update_min_steps=min( 256 * 1024, file_size / 100 - ), # every 256Kb or less, + ), # every 256Kb or less ) as pb: for chunk in pb: pb.update(len(chunk)) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 580137a00f..d38c0e602b 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -48,12 +48,10 @@ def __str__(self): class MissingPackageManifestError(ManifestException): - MESSAGE = "Could not find one of '{0}' manifest files in the package" class UnknownPackageError(UserSideException): - MESSAGE = ( "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() @@ -61,7 +59,6 @@ class UnknownPackageError(UserSideException): class NotGlobalLibDir(UserSideException): - MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 9d9c611819..bf844980ee 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -26,7 +26,6 @@ class PackageManagerDownloadMixin: - DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): @@ -62,7 +61,7 @@ def download(self, url, checksum=None): self.set_download_utime(dl_path) return dl_path - with_progress = not silent and not app.is_disabled_progressbar() + with_progress = not app.is_disabled_progressbar() tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): @@ -71,7 +70,7 @@ def download(self, url, checksum=None): fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) except IOError as exc: - raise_error = not with_progress + raise_error = not silent if with_progress: try: fd = FileDownloader(url) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index c89d7b861d..fc4d5ddbc6 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -27,7 +27,6 @@ class PackageManagerInstallMixin: - _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 28fab07b29..ca5c5e0136 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -12,19 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os -import shutil -import subprocess -import sys -from datetime import date -from platformio import __core_packages__, exception, fs, util -from platformio.exception import UserSideException +from platformio import __core_packages__, exception from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageItem, PackageSpec -from platformio.proc import get_pythonexe_path +from platformio.package.meta import PackageSpec def get_installed_core_packages(): @@ -98,131 +91,3 @@ def remove_unnecessary_core_packages(dry_run=False): pm.uninstall(pkg) return candidates - - -def inject_contrib_pysite(): - # pylint: disable=import-outside-toplevel - from site import addsitedir - - try: - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - except UnknownPackageError: - pm = ToolPackageManager() - contrib_pysite_dir = build_contrib_pysite_package( - os.path.join(pm.package_dir, "contrib-pysite") - ) - - if contrib_pysite_dir in sys.path: - return True - - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) - - try: - # pylint: disable=import-error,unused-import,unused-variable - from OpenSSL import SSL - - except: # pylint: disable=bare-except - build_contrib_pysite_package(contrib_pysite_dir) - - return True - - -def build_contrib_pysite_package(target_dir, with_metadata=True): - systype = util.get_systype() - if os.path.isdir(target_dir): - fs.rmtree(target_dir) - os.makedirs(target_dir) - - # issue 3865: There is no "rustup" in "Raspbian GNU/Linux 10 (buster)" - os.environ["CRYPTOGRAPHY_DONT_BUILD_RUST"] = "1" - - # build dependencies - args = [ - get_pythonexe_path(), - "-m", - "pip", - "install", - "--no-compile", - "-t", - target_dir, - ] - if "linux" in systype: - args.extend(["--no-binary", ":all:"]) - try: - subprocess.run(args + get_contrib_pysite_deps(), check=True, env=os.environ) - except subprocess.CalledProcessError as exc: - if "linux" in systype: - raise UserSideException( - "\n\nPlease ensure that the next packages are installed:\n\n" - "sudo apt install python3-dev libffi-dev libssl-dev\n" - ) from exc - raise exc - - # build manifests - with open( - os.path.join(target_dir, "package.json"), mode="w", encoding="utf8" - ) as fp: - json.dump( - dict( - name="contrib-pysite", - version="2.%d%d.%s" - % ( - sys.version_info.major, - sys.version_info.minor, - date.today().strftime("%y%m%d"), - ), - system=list( - set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) - ) - if systype.startswith("linux_arm") - else systype, - description="Extra Python package for PlatformIO Core", - keywords=["platformio", "platformio-core"], - homepage="https://docs.platformio.org/page/core/index.html", - repository={ - "type": "git", - "url": "https://github.com/platformio/platformio-core", - }, - ), - fp, - ) - - # generate package metadata - if with_metadata: - pm = ToolPackageManager() - pkg = PackageItem(target_dir) - pkg.metadata = pm.build_metadata( - target_dir, PackageSpec(owner="platformio", name="contrib-pysite") - ) - pkg.dump_meta() - - # remove unused files - for root, dirs, files in os.walk(target_dir): - for t in ("_test", "test", "tests"): - if t in dirs: - shutil.rmtree(os.path.join(root, t)) - for name in files: - if name.endswith((".chm", ".pyc")): - os.remove(os.path.join(root, name)) - - return target_dir - - -def get_contrib_pysite_deps(): - systype = util.get_systype() - twisted_version = "22.1.0" - if "linux_arm" in systype: - result = [ - # twisted[tls], see setup.py for %twisted_version% - "twisted == %s" % twisted_version, - # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", - "pyopenssl >= 16.0.0, <= 21.0.0", - "service_identity >= 18.1.0, <= 21.1.0", - ] - else: - result = ["twisted[tls] == %s" % twisted_version] - if "windows" in systype: - result.append("pywin32 != 226") - return result diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index e5e948babf..6dbd480c7f 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -15,7 +15,7 @@ import os from platformio import util -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.core import get_installed_core_packages @@ -128,7 +128,7 @@ def get_all_boards(self): key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (HTTPClientError, InternetIsOffline): + except (HTTPClientError, InternetConnectionError): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 937bbf91df..2075ac0381 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -276,7 +276,7 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.19" + version = "3.20" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version diff --git a/platformio/package/meta.py b/platformio/package/meta.py index fbd2b7343f..4db5310607 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -65,7 +65,6 @@ def from_archive(cls, path): class PackageCompatibility: - KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") @classmethod @@ -468,7 +467,6 @@ def load(path): class PackageItem: - METAFILE_NAME = ".piopm" def __init__(self, path, metadata=None): diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index e39222c29e..9d8919c32f 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -20,11 +20,11 @@ import click from platformio import fs +from platformio.compat import is_terminal from platformio.package.exception import PackageException class ExtractArchiveItemError(PackageException): - MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> https://bit.ly/faq-package-manager" @@ -159,18 +159,38 @@ def new_archiver(path): def unpack( self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False - ): + ): # pylint: disable=too-many-branches assert self._archiver + label = "Unpacking" + items = self._archiver.get_items() if not dest_dir: dest_dir = os.getcwd() + if not with_progress or silent: if not silent: - click.echo("Unpacking...") - for item in self._archiver.get_items(): + click.echo(f"{label}...") + for item in items: + self._archiver.extract_item(item, dest_dir) + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + unpacked_nums = 0 + for item in items: self._archiver.extract_item(item, dest_dir) + unpacked_nums += 1 + if (unpacked_nums / len(items) * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") else: - items = self._archiver.get_items() - with click.progressbar(items, label="Unpacking") as pb: + with click.progressbar( + items, + label=label, + update_min_steps=min(50, len(items) / 100), # every 50 files or less + ) as pb: for item in pb: self._archiver.extract_item(item, dest_dir) diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index b4dafc0319..058a4983f7 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -58,7 +58,6 @@ def new(src_dir, remote_url, silent=False): class VCSClientBase: - command = None def __init__(self, src_dir, remote_url=None, tag=None, silent=False): @@ -128,7 +127,6 @@ def get_cmd_output(self, args, **kwargs): class GitClient(VCSClientBase): - command = "git" _configured = False @@ -232,7 +230,6 @@ def get_latest_revision(self): class HgClient(VCSClientBase): - command = "hg" def export(self): @@ -256,7 +253,6 @@ def get_latest_revision(self): class SvnClient(VCSClientBase): - command = "svn" def export(self): diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 79afff9435..dfd7c40675 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -25,10 +25,10 @@ from platformio.compat import hashlib_encode_data from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound +from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS class PlatformRunMixin: - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) @staticmethod @@ -57,9 +57,6 @@ def run( # pylint: disable=too-many-arguments self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") - if "clean" in targets: - targets = ["-c", "."] - variables["platform_manifest"] = self.manifest_path if "build_script" not in variables: @@ -93,16 +90,22 @@ def _run_scons(self, variables, targets, jobs): "--sconstruct", os.path.join(fs.get_source_dir(), "builder", "main.py"), ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + args.append("PIOVERBOSE=%d" % int(self.verbose)) # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - + args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) # encode and append variables for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - proc.copy_pythonpath_to_osenv() + if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): + args.append("--clean") + args.append( + "FULLCLEAN=%d" + % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0) + ) + elif targets: + args.extend(targets) + # force SCons output to Unicode os.environ["PYTHONIOENCODING"] = "utf-8" diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 49db0df45c..3231bb4444 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -29,7 +29,6 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods PlatformPackagesMixin, PlatformRunMixin ): - CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} @@ -208,6 +207,15 @@ def configure_default_packages(self, options, targets): def configure_debug_session(self, debug_config): raise NotImplementedError + def generate_sample_code(self, project_config, environment): + raise NotImplementedError + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): @@ -228,9 +236,3 @@ def get_lib_storages(self): storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index 604c322876..f044d3b7a7 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -20,12 +20,10 @@ class PlatformException(PlatformioException): class UnknownPlatform(PlatformException): - MESSAGE = "Unknown development platform '{0}'" class IncompatiblePlatform(PlatformException): - MESSAGE = ( "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " "depends on PlatformIO Core {2}.\n" @@ -33,20 +31,16 @@ class IncompatiblePlatform(PlatformException): class UnknownBoard(PlatformException): - MESSAGE = "Unknown board ID '{0}'" class InvalidBoardManifest(PlatformException): - MESSAGE = "Invalid board JSON manifest '{0}'" class UnknownFramework(PlatformException): - MESSAGE = "Unknown framework '{0}'" class BuildScriptNotFound(PlatformException): - MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py index b59ff005d1..9f218e8024 100644 --- a/platformio/project/commands/config.py +++ b/platformio/project/commands/config.py @@ -28,7 +28,7 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("--json-output", is_flag=True) def project_config_cmd(project_dir, json_output): diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index ca57de34bd..098c31a57f 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -24,12 +24,14 @@ from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project from platformio.project.integration.generator import ProjectGenerator +from platformio.project.options import ProjectOptions -def validate_boards(ctx, param, value): # pylint: disable=W0613 +def validate_boards(ctx, param, value): # pylint: disable=unused-argument pm = PlatformPackageManager() for id_ in value: try: @@ -47,25 +49,33 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 "--project-dir", "-d", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option( + "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards ) -@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-e", "--environment", help="Update existing environment") -@click.option("-O", "--project-option", multiple=True) -@click.option("--env-prefix", default="") +@click.option( + "-O", + "--project-option", + "project_options", + multiple=True, + help="A `name=value` pair", +) +@click.option("--sample-code", is_flag=True) @click.option("--no-install-dependencies", is_flag=True) +@click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) def project_init_cmd( project_dir, - board, + boards, ide, environment, - project_option, - env_prefix, + project_options, + sample_code, no_install_dependencies, + env_prefix, silent, ): is_new_project = not is_platformio_project(project_dir) @@ -74,23 +84,23 @@ def project_init_cmd( print_header(project_dir) init_base_project(project_dir) - if environment: - update_project_env(project_dir, environment, project_option) - elif board: - update_board_envs(project_dir, board, project_option, env_prefix) - with fs.cd(project_dir): + if environment: + update_project_env(environment, project_options) + elif boards: + update_board_envs(project_dir, boards, project_options, env_prefix) + generator = None config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) if ide: config.validate() # init generator and pick the best env if user didn't specify - generator = ProjectGenerator(config, environment, ide, board) + generator = ProjectGenerator(config, environment, ide, boards) if not environment: environment = generator.env_name # resolve project dependencies - if not no_install_dependencies and (environment or board): + if not no_install_dependencies and (environment or boards): install_project_dependencies( options=dict( project_dir=project_dir, @@ -99,6 +109,9 @@ def project_init_cmd( ) ) + if environment and sample_code: + init_sample_code(config, environment) + if generator: if not silent: click.echo( @@ -106,31 +119,22 @@ def project_init_cmd( ) generator.generate() - if is_new_project: - init_cvs_ignore(project_dir) + if is_new_project: + init_cvs_ignore() if not silent: print_footer(is_new_project) def print_header(project_dir): - if project_dir == os.getcwd(): - click.secho("\nThe current working directory ", fg="yellow", nl=False) - try: - click.secho(project_dir, fg="cyan", nl=False) - except UnicodeEncodeError: - click.secho(json.dumps(project_dir), fg="cyan", nl=False) - click.secho(" will be used for the project.", fg="yellow") - click.echo("") - - click.echo("The next files/directories have been created in ", nl=False) + click.echo("The following files/directories have been created in ", nl=False) try: click.secho(project_dir, fg="cyan") except UnicodeEncodeError: click.secho(json.dumps(project_dir), fg="cyan") click.echo("%s - Put project header files here" % click.style("include", fg="cyan")) click.echo( - "%s - Put here project specific (private) libraries" + "%s - Put project specific (private) libraries here" % click.style("lib", fg="cyan") ) click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) @@ -140,18 +144,9 @@ def print_header(project_dir): def print_footer(is_new_project): - if is_new_project: - return click.secho( - "\nProject has been successfully initialized! Useful commands:\n" - "`pio run` - process/build project from the current directory\n" - "`pio run --target upload` or `pio run -t upload` " - "- upload firmware to a target\n" - "`pio run --target clean` - clean project (remove compiled files)" - "\n`pio run --help` - additional information", - fg="green", - ) + action = "initialized" if is_new_project else "updated" return click.secho( - "Project has been successfully updated!", + f"Project has been successfully {action}!", fg="green", ) @@ -166,7 +161,7 @@ def init_base_project(project_dir): (config.get("platformio", "lib_dir"), init_lib_readme), (config.get("platformio", "test_dir"), init_test_readme), ] - for (path, cb) in dir_to_readme: + for path, cb in dir_to_readme: if os.path.isdir(path): continue os.makedirs(path) @@ -291,15 +286,15 @@ def init_test_readme(test_dir): ) -def init_cvs_ignore(project_dir): - conf_path = os.path.join(project_dir, ".gitignore") +def init_cvs_ignore(): + conf_path = ".gitignore" if os.path.isfile(conf_path): return with open(conf_path, mode="w", encoding="utf8") as fp: fp.write(".pio\n") -def update_board_envs(project_dir, board_ids, project_option, env_prefix): +def update_board_envs(project_dir, boards, extra_project_options, env_prefix): config = ProjectConfig( os.path.join(project_dir, "platformio.ini"), parse_extra=False ) @@ -311,7 +306,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): pm = PlatformPackageManager() modified = False - for id_ in board_ids: + for id_ in boards: board_config = pm.board_config(id_) if id_ in used_boards: continue @@ -324,7 +319,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): if frameworks: envopts["framework"] = frameworks[0] - for item in project_option: + for item in extra_project_options: if "=" not in item: continue _name, _value = item.split("=", 1) @@ -340,21 +335,76 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): config.save() -def update_project_env(project_dir, environment, project_option): - if not project_option: +def update_project_env(environment, extra_project_options=None): + if not extra_project_options: return + env_section = "env:%s" % environment + option_to_sections = {"platformio": [], env_section: []} + for item in extra_project_options: + assert "=" in item + name, value = item.split("=", 1) + name = name.strip() + destination = env_section + for option in ProjectOptions.values(): + if option.scope in option_to_sections and option.name == name: + destination = option.scope + break + option_to_sections[destination].append((name, value.strip())) + config = ProjectConfig( - os.path.join(project_dir, "platformio.ini"), parse_extra=False + "platformio.ini", parse_extra=False, expand_interpolations=False ) - - section = "env:%s" % environment - if not config.has_section(section): - config.add_section(section) - - for item in project_option: - if "=" not in item: + for section, options in option_to_sections.items(): + if not options: continue - _name, _value = item.split("=", 1) - config.set(section, _name.strip(), _value.strip()) + if not config.has_section(section): + config.add_section(section) + for name, value in options: + config.set(section, name, value) config.save() + + +def init_sample_code(config, environment): + platform_spec = config.get(f"env:{environment}", "platform", None) + if not platform_spec: + return None + p = PlatformFactory.new(platform_spec) + try: + return p.generate_sample_code(config, environment) + except NotImplementedError: + pass + + framework = config.get(f"env:{environment}", "framework", None) + if framework != ["arduino"]: + return None + main_content = """ +#include + +// put function declarations here: +int myFunction(int, int); + +void setup() { + // put your setup code here, to run once: + int result = myFunction(2, 3); +} + +void loop() { + // put your main code here, to run repeatedly: +} + +// put function definitions here: +int myFunction(int x, int y) { + return x + y; +} +""" + is_cpp_project = p.name not in ["intel_mcs51", "ststm8"] + src_dir = config.get("platformio", "src_dir") + main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")) + if os.path.isfile(main_path): + return None + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, mode="w", encoding="utf8") as fp: + fp.write(main_content.strip()) + return True diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index 0645757449..fbf71b8bbd 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -31,11 +31,11 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("--json-output", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): with fs.cd(project_dir): config = ProjectConfig.get_instance() diff --git a/platformio/project/config.py b/platformio/project/config.py index ee380bb943..c2b072ef16 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -39,7 +39,7 @@ class ProjectConfigBase: - + ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -389,16 +389,38 @@ def get_default_env(self): def validate(self, envs=None, silent=False): if not os.path.isfile(self.path): raise exception.NotPlatformIOProjectError(os.path.dirname(self.path)) + + known_envs = set(self.envs()) + # check envs - known = set(self.envs()) - if not known: + if not known_envs: raise exception.ProjectEnvsNotAvailableError() - unknown = set(list(envs or []) + self.default_envs()) - known - if unknown: - raise exception.UnknownEnvNamesError(", ".join(unknown), ", ".join(known)) + unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs + if unknown_envs: + raise exception.UnknownEnvNamesError( + ", ".join(unknown_envs), ", ".join(known_envs) + ) + + for env in known_envs: + # check envs names + if not self.ENVNAME_RE.match(env): + raise exception.InvalidEnvNameError(env) + + # check simultaneous use of `monitor_raw` and `monitor_filters` + if self.get(f"env:{env}", "monitor_raw", False) and self.get( + f"env:{env}", "monitor_filters", None + ): + self.warnings.append( + "The `monitor_raw` and `monitor_filters` options cannot be " + f"used simultaneously for the `{env}` environment in the " + "`platformio.ini` file. The `monitor_filters` option will " + "be disabled to avoid conflicts." + ) + if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") + return True @@ -412,7 +434,6 @@ def get_optional_dir(self, name): class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): - _instances = {} @staticmethod diff --git a/platformio/project/exception.py b/platformio/project/exception.py index aa45eb07c4..95681bc0df 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -20,7 +20,6 @@ class ProjectError(PlatformioException): class NotPlatformIOProjectError(ProjectError, UserSideException): - MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " @@ -29,25 +28,27 @@ class NotPlatformIOProjectError(ProjectError, UserSideException): class InvalidProjectConfError(ProjectError, UserSideException): - MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" class UndefinedEnvPlatformError(ProjectError, UserSideException): - MESSAGE = "Please specify platform for '{0}' environment" class ProjectEnvsNotAvailableError(ProjectError, UserSideException): - MESSAGE = "Please setup environments in `platformio.ini` file" class UnknownEnvNamesError(ProjectError, UserSideException): - MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" -class ProjectOptionValueError(ProjectError, UserSideException): +class InvalidEnvNameError(ProjectError, UserSideException): + MESSAGE = ( + "Invalid environment name '{0}'. The name can contain " + "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" + ) + +class ProjectOptionValueError(ProjectError, UserSideException): MESSAGE = "{0} for option `{1}` in section [{2}]" diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 200321f26c..aec12dd98a 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import subprocess from hashlib import sha1 @@ -24,7 +25,7 @@ def get_project_dir(): - return fs.normalize_path(os.getcwd()) + return os.getcwd() def is_platformio_project(project_dir=None): @@ -94,7 +95,16 @@ def compute_project_checksum(config): checksum = sha1(hashlib_encode_data(__version__)) # configuration file state - checksum.update(hashlib_encode_data(config.to_json())) + config_data = config.to_json() + if IS_WINDOWS: + # issue #4600: fix drive letter + config_data = re.sub( + r"([A-Z]):\\", + lambda match: "%s:\\" % match.group(1).lower(), + config_data, + flags=re.I, + ) + checksum.update(hashlib_encode_data(config_data)) # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 4ea3bbf3f8..654e5ac353 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -19,33 +19,34 @@ import bottle from platformio import fs, util +from platformio.debug.helpers import get_default_debug_env from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata class ProjectGenerator: - def __init__(self, config, env_name, ide, board_ids=None): + def __init__(self, config, env_name, ide, boards=None): self.config = config self.project_dir = os.path.dirname(config.path) - self.original_env_name = env_name - self.env_name = str(env_name or self.get_best_envname(board_ids)) + self.forced_env_name = env_name + self.env_name = str(env_name or self.get_best_envname(boards)) self.ide = str(ide) - def get_best_envname(self, board_ids=None): + def get_best_envname(self, boards=None): envname = None default_envs = self.config.default_envs() if default_envs: envname = default_envs[0] - if not board_ids: + if not boards: return envname for env in self.config.envs(): - if not board_ids: + if not boards: return env if not envname: envname = env items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in board_ids: + if "board" in items and items.get("board") in boards: return env return envname @@ -86,7 +87,8 @@ def _load_tplvars(self): "platformio", "name", os.path.basename(self.project_dir) ), "project_dir": self.project_dir, - "original_env_name": self.original_env_name, + "forced_env_name": self.forced_env_name, + "default_debug_env_name": get_default_debug_env(self.config), "env_name": self.env_name, "user_home_dir": os.path.abspath(fs.expanduser("~")), "platformio_path": sys.argv[0] @@ -132,7 +134,7 @@ def get_src_files(self): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( - os.path.relpath(os.path.join(os.path.realpath(root), f)) + os.path.relpath(os.path.join(os.path.abspath(root), f)) ) return result diff --git a/platformio/project/options.py b/platformio/project/options.py index 5e200629f4..2e13815ac4 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -114,7 +114,7 @@ def validate_dir(path): path = fs.expanduser(path) if "$" in path: path = expand_dir_templates(path) - return fs.normalize_path(path) + return os.path.abspath(path) def get_default_core_dir(): @@ -649,10 +649,10 @@ def get_default_core_dir(): ), ConfigEnvOption( group="check", - name="check_patterns", + name="check_src_filters", + oldnames=["check_patterns"], description=( - "Configure a list of target files or directories for checking " - "(Unix shell-style wildcards)" + "Configure a list of target files or directories for checking" ), multiple=True, ), diff --git a/platformio/registry/client.py b/platformio/registry/client.py index a682db5720..1a89e345a2 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -142,12 +142,15 @@ def list_packages(self, query=None, qualifiers=None, page=None, sort=None): x_with_authorization=self.allowed_private_packages(), ) - def get_package(self, type_, owner, name, version=None): + def get_package(self, type_, owner, name, version=None, extra_path=None): try: return self.fetch_json_data( "get", - "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=name.lower() + "/v3/packages/{owner}/{type}/{name}{extra_path}".format( + type=type_, + owner=owner.lower(), + name=name.lower(), + extra_path=extra_path or "", ), params=dict(version=version) if version else None, x_cache_valid="1h", diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index 4235e88d4f..4b4508f662 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -22,7 +22,6 @@ class RegistryFileMirrorIterator: - HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index 8105062c70..0c631c64cc 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -17,7 +17,6 @@ class AsyncCommandBase: - MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb def __init__(self, options=None, on_end_callback=None): diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index bede41d160..f1775d498b 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -17,7 +17,9 @@ import os import subprocess +import sys import threading +from site import addsitedir from tempfile import mkdtemp from time import sleep @@ -29,7 +31,7 @@ device_monitor_cmd, get_project_options, ) -from platformio.package.manager.core import inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir from platformio.project.exception import NotPlatformIOProjectError from platformio.project.options import ProjectOptions from platformio.run.cli import cli as cmd_run @@ -41,7 +43,11 @@ @click.pass_context def cli(ctx, agent): ctx.obj = agent - inject_contrib_pysite() + # inject twisted dependencies + contrib_dir = get_core_package_dir("contrib-pioremote") + if contrib_dir not in sys.path: + addsitedir(contrib_dir) + sys.path.insert(0, contrib_dir) @cli.group("agent", short_help="Start a new agent or list active") @@ -56,7 +62,7 @@ def remote_agent(): "-d", "--working-dir", envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) def remote_agent_start(name, share, working_dir): from platformio.remote.client.agent_service import RemoteAgentService @@ -96,9 +102,7 @@ def remote_update(agents, only_check, dry_run): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @@ -118,7 +122,6 @@ def remote_run( silent, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( @@ -187,9 +190,7 @@ def remote_run( "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @@ -211,7 +212,6 @@ def remote_test( # pylint: disable=redefined-builtin without_uploading, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( @@ -336,7 +336,7 @@ def device_list(agents, json_output): "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", @@ -345,9 +345,7 @@ def device_list(agents, json_output): ) @click.option( "--sock", - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.pass_obj @click.pass_context diff --git a/platformio/remote/client/async_base.py b/platformio/remote/client/async_base.py index 488802a583..2a67264132 100644 --- a/platformio/remote/client/async_base.py +++ b/platformio/remote/client/async_base.py @@ -34,7 +34,7 @@ def agent_pool_ready(self): def cb_async_result(self, result): if self._acs_total == 0: self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) self.acread_data(*value) diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py index fe2c4fb4c9..cf5c240537 100644 --- a/platformio/remote/client/base.py +++ b/platformio/remote/client/base.py @@ -33,7 +33,6 @@ class RemoteClientBase( # pylint: disable=too-many-instance-attributes pb.Referenceable ): - PING_DELAY = 60 PING_MAX_FAILURES = 3 DEBUG = False diff --git a/platformio/remote/client/device_list.py b/platformio/remote/client/device_list.py index a22911b461..f05bbfd27c 100644 --- a/platformio/remote/client/device_list.py +++ b/platformio/remote/client/device_list.py @@ -32,7 +32,7 @@ def agent_pool_ready(self): def _cbResult(self, result): data = {} - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py index 46ca19e4ad..1499dd5866 100644 --- a/platformio/remote/client/device_monitor.py +++ b/platformio/remote/client/device_monitor.py @@ -71,7 +71,6 @@ def send_to_server(self, data): class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes RemoteClientBase ): - MAX_BUFFER_SIZE = 1024 * 1024 def __init__(self, agents, **kwargs): @@ -96,7 +95,7 @@ def agent_pool_ready(self): def _cb_device_list(self, result): devices = [] hwid_devindexes = [] - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/run_or_test.py b/platformio/remote/client/run_or_test.py index 71065640f7..d23d92c737 100644 --- a/platformio/remote/client/run_or_test.py +++ b/platformio/remote/client/run_or_test.py @@ -28,7 +28,6 @@ class RunOrTestClient(AsyncClientBase): - MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb @@ -147,7 +146,7 @@ def psync_init(self): def cb_psync_init_result(self, result): self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) agent_id, ac_id = value diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py index 820034ae26..1715145fde 100644 --- a/platformio/remote/projectsync.py +++ b/platformio/remote/projectsync.py @@ -47,13 +47,13 @@ def get_items(self): def rebuild_dbindex(self): self._db = {} - for (path, relpath, cb_filter) in self.items: + for path, relpath, cb_filter in self.items: if cb_filter and not cb_filter(path): continue self._insert_to_db(path, relpath) if not isdir(path): continue - for (root, _, files) in os.walk(path, followlinks=True): + for root, _, files in os.walk(path, followlinks=True): for name in files: self._insert_to_db( join(root, name), join(relpath, root[len(path) + 1 :], name) diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 6c64743156..dc3eb2ee51 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -26,7 +26,7 @@ from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import find_project_dir_above, load_build_metadata -from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps +from platformio.run.helpers import clean_build_dir from platformio.run.processor import EnvironmentProcessor from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING @@ -47,16 +47,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option( "-j", @@ -101,9 +97,12 @@ def cli( if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) + targets = list(target) if target else [] + del target + only_monitor = targets == ["monitor"] is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta + command_failed = False - results = [] with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) @@ -112,7 +111,7 @@ def cli( return print_target_list(list(environment) or config.envs()) # clean obsolete build dir - if not disable_auto_clean: + if not only_monitor and not disable_auto_clean: build_dir = config.get("platformio", "build_dir") try: clean_build_dir(build_dir, config) @@ -125,9 +124,8 @@ def cli( fg="yellow", ) - handle_legacy_libdeps(project_dir, config) - default_envs = config.default_envs() + results = [] for env in config.envs(): skipenv = any( [ @@ -148,8 +146,7 @@ def cli( ctx, env, config, - environment, - target, + targets, upload_port, monitor_port, jobs, @@ -159,17 +156,21 @@ def cli( verbose, ) ) - - command_failed = any(r.get("succeeded") is False for r in results) - - if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results, verbose) + command_failed = any(r.get("succeeded") is False for r in results) + if ( + not is_test_running + and not only_monitor + and (command_failed or not silent) + and len(results) > 1 + ): + print_processing_summary(results, verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) if command_failed: raise exception.ReturnErrorCode(1) + return True @@ -177,7 +178,6 @@ def process_env( ctx, name, config, - environments, targets, upload_port, monitor_port, @@ -190,34 +190,39 @@ def process_env( if not is_test_running and not silent: print_processing_header(name, config, verbose) - ep = EnvironmentProcessor( - ctx, - name, - config, - targets, - upload_port, - jobs, - program_args, - silent, - verbose, - ) - result = {"env": name, "duration": time(), "succeeded": ep.process()} + targets = targets or config.get(f"env:{name}", "targets", []) + only_monitor = targets == ["monitor"] + result = {"env": name, "duration": time(), "succeeded": True} + + if not only_monitor: + result["succeeded"] = EnvironmentProcessor( + ctx, + name, + config, + [t for t in targets if t != "monitor"], + upload_port, + jobs, + program_args, + silent, + verbose, + ).process() + + if "monitor" in targets and "nobuild" not in targets: + ctx.invoke( + device_monitor_cmd, + port=monitor_port, + environment=name, + ) + result["duration"] = time() - result["duration"] # print footer on error or when is not unit testing - if not is_test_running and (not silent or not result["succeeded"]): - print_processing_footer(result) - if ( - result["succeeded"] - and "monitor" in ep.get_build_targets() - and "nobuild" not in ep.get_build_targets() + not is_test_running + and not only_monitor + and (not silent or not result["succeeded"]) ): - ctx.invoke( - device_monitor_cmd, - port=monitor_port, - environment=environments[0] if environments else None, - ) + print_processing_footer(result) return result diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index 925b558484..f8bfdcba5d 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -15,31 +15,12 @@ from os import makedirs from os.path import isdir, isfile, join -import click - from platformio import fs from platformio.project.helpers import compute_project_checksum, get_project_dir - -def handle_legacy_libdeps(project_dir, config): - legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get( - "platformio", "libdeps_dir" - ): - return - if not config.has_section("env"): - config.add_section("env") - lib_extra_dirs = config.get("env", "lib_extra_dirs", []) - lib_extra_dirs.append(legacy_libdeps_dir) - config.set("env", "lib_extra_dirs", lib_extra_dirs) - click.secho( - "DEPRECATED! A legacy library storage `{0}` has been found in a " - "project. \nPlease declare project dependencies in `platformio.ini`" - " file using `lib_deps` option and remove `{0}` folder." - "\nMore details -> https://docs.platformio.org/page/projectconf/" - "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow", - ) +KNOWN_CLEAN_TARGETS = ("clean",) +KNOWN_FULLCLEAN_TARGETS = ("cleanall", "fullclean") +KNOWN_ALLCLEAN_TARGETS = KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS def clean_build_dir(build_dir, config): diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 053478acb8..329c4bf6d0 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -15,6 +15,7 @@ from platformio.package.commands.install import install_project_env_dependencies from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError +from platformio.run.helpers import KNOWN_ALLCLEAN_TARGETS from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes @@ -36,7 +37,7 @@ def __init__( # pylint: disable=too-many-arguments self.cmd_ctx = cmd_ctx self.name = name self.config = config - self.targets = [str(t) for t in targets] + self.targets = targets self.upload_port = upload_port self.jobs = jobs self.program_args = program_args @@ -61,33 +62,29 @@ def get_build_variables(self): variables["upload_port"] = self.upload_port return variables - def get_build_targets(self): - return ( - self.targets - if self.targets - else self.config.get("env:" + self.name, "targets", []) - ) - def process(self): if "platform" not in self.options: raise UndefinedEnvPlatformError(self.name) build_vars = self.get_build_variables() - build_targets = list(self.get_build_targets()) + is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) + build_targets = [t for t in self.targets if t not in KNOWN_ALLCLEAN_TARGETS] - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - - if not set(["clean", "cleanall"]) & set(build_targets): - install_project_env_dependencies( - self.name, - { - "project_targets": build_targets, - "piotest_running_name": build_vars.get("piotest_running_name"), - }, - ) + # pre-clean + if is_clean: + result = PlatformFactory.new( + self.options["platform"], autoinstall=True + ).run(build_vars, self.targets, self.silent, self.verbose, self.jobs) + if not build_targets: + return result["returncode"] == 0 + install_project_env_dependencies( + self.name, + { + "project_targets": self.targets, + "piotest_running_name": build_vars.get("piotest_running_name"), + }, + ) result = PlatformFactory.new(self.options["platform"], autoinstall=True).run( build_vars, build_targets, self.silent, self.verbose, self.jobs ) diff --git a/platformio/system/commands/completion.py b/platformio/system/commands/completion.py index aac68592cc..5ff73bb163 100644 --- a/platformio/system/commands/completion.py +++ b/platformio/system/commands/completion.py @@ -33,7 +33,7 @@ def system_completion_cmd(): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) @@ -54,7 +54,7 @@ def system_completion_install(shell, path): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 211c0beac6..e9cbcd583d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -54,7 +54,6 @@ def send(self, hittype): class MeasurementProtocol(TelemetryBase): - TID = "UA-1768265-9" PARAMS_MAP = { "screen_name": "cd", @@ -106,18 +105,6 @@ def _prefill_sysargs(self): self["cd3"] = " ".join(args) def _prefill_custom_data(self): - def _filter_args(items): - result = [] - stop = False - for item in items: - item = str(item).lower() - result.append(item) - if stop: - break - if item == "account": - stop = True - return result - caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 @@ -201,7 +188,6 @@ def send(self, hittype): @util.singleton class MPDataPusher: - MAX_WORKERS = 5 def __init__(self): diff --git a/platformio/test/cli.py b/platformio/test/cli.py index d0a6e53a4a..748580c8fb 100644 --- a/platformio/test/cli.py +++ b/platformio/test/cli.py @@ -48,16 +48,12 @@ "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -83,8 +79,8 @@ help="A program argument (multiple are allowed)", ) @click.option("--list-tests", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) -@click.option("--junit-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) +@click.option("--junit-output-path", type=click.Path()) @click.option( "--verbose", "-v", diff --git a/platformio/test/exception.py b/platformio/test/exception.py index 12b76ea491..14cbe5acf3 100644 --- a/platformio/test/exception.py +++ b/platformio/test/exception.py @@ -20,7 +20,6 @@ class UnitTestError(PlatformioException): class TestDirNotExistsError(UnitTestError, UserSideException): - MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " "directory in the project root and put a test suite.\n" diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index 01650a0b1d..25551ef664 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -40,7 +40,6 @@ def list_test_suites(project_config, environments, filters, ignores): test_names = list_test_names(project_config) for env_name in project_config.envs(): for test_name in test_names: - # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) for key, value in patterns.items(): diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 8f9136f126..845a8c3771 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -18,7 +18,7 @@ from platformio.platform.factory import PlatformFactory from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestStatus -from platformio.test.runners.readers.program import ProgramTestOutputReader +from platformio.test.runners.readers.native import NativeTestOutputReader from platformio.test.runners.readers.serial import SerialTestOutputReader CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" @@ -54,7 +54,6 @@ def __init__( # pylint: disable=too-many-arguments class TestRunnerBase: - NAME = None EXTRA_LIB_DEPS = None TESTCASE_PARSE_RE = None @@ -161,7 +160,7 @@ def stage_testing(self): return None click.secho("Testing...", bold=True) test_port = self.get_test_port() - program_conds = [ + native_conds = [ not self.platform.is_embedded() and (not test_port or "://" not in test_port), self.project_config.get( @@ -169,8 +168,8 @@ def stage_testing(self): ), ] reader = ( - ProgramTestOutputReader(self) - if any(program_conds) + NativeTestOutputReader(self) + if any(native_conds) else SerialTestOutputReader(self) ) return reader.begin() diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index 52a0916fa2..dd30bc925a 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -101,7 +101,6 @@ def _parse_assert(self, line): class DoctestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index b299002e3c..8a4d1b15ae 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -22,7 +22,6 @@ class GoogletestTestCaseParser: - # Examples: # [ RUN ] FooTest.Bar # ... @@ -89,7 +88,6 @@ def _parse_source_and_message(self, stdout): class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/native.py similarity index 91% rename from platformio/test/runners/readers/program.py rename to platformio/test/runners/readers/native.py index d80d170d26..981febc375 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/native.py @@ -24,6 +24,7 @@ get_filesystem_encoding, get_locale_encoding, ) +from platformio.project.helpers import load_build_metadata from platformio.test.exception import UnitTestError EXITING_TIMEOUT = 5 # seconds @@ -56,7 +57,7 @@ def _stop_testing(self): self._exit_timer.cancel() -class ProgramTestOutputReader: +class NativeTestOutputReader: def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( @@ -78,6 +79,13 @@ def get_testing_command(self): "program.exe" if IS_WINDOWS else "program", ) ] + # if user changed PROGNAME + if not os.path.exists(cmd[0]): + build_data = load_build_metadata( + os.getcwd(), self.test_runner.test_suite.env_name, cache=True + ) + if build_data: + cmd[0] = build_data["prog_path"] if self.test_runner.options.program_args: cmd.extend(self.test_runner.options.program_args) return cmd diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index ab35775b39..a9732d7fe6 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -22,7 +22,6 @@ class SerialTestOutputReader: - SERIAL_TIMEOUT = 600 def __init__(self, test_runner): @@ -47,7 +46,7 @@ def begin(self): ser.open() except serial.SerialException as exc: click.secho(str(exc), fg="red", err=True) - return None + return if not self.test_runner.options.no_reset: ser.flushInput() diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index b7d8026be5..25fea0ab62 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -26,7 +26,6 @@ class UnityTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.5.2"] # Examples: @@ -115,7 +114,7 @@ class UnityTestRunner(TestRunnerBase): native=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -156,7 +155,7 @@ class UnityTestRunner(TestRunnerBase): espidf=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -166,7 +165,7 @@ class UnityTestRunner(TestRunnerBase): zephyr=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { printk("%c", c); } void unityOutputFlush(void) { } void unityOutputComplete(void) { } diff --git a/platformio/util.py b/platformio/util.py index 63a16d6ed6..c981384ad8 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -96,7 +96,6 @@ class RetryStopException(RetryException): class retry: - RetryNextException = RetryNextException RetryStopException = RetryStopException diff --git a/scripts/docspregen.py b/scripts/docspregen.py index f895c25d84..09dd6777ec 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,7 @@ import click # noqa: E402 -from platformio import fs, util # noqa: E402 +from platformio import fs # noqa: E402 from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 from platformio.platform.factory import PlatformFactory # noqa: E402 @@ -41,6 +41,25 @@ SKIP_DEBUG_TOOLS = ["esp-bridge", "esp-builtin"] +STATIC_FRAMEWORK_DATA = { + "arduino": { + "title": "Arduino", + "description": ( + "Arduino Wiring-based Framework allows writing cross-platform software " + "to control devices attached to a wide range of Arduino boards to " + "create all kinds of creative coding, interactive objects, spaces " + "or physical experiences." + ), + }, + "freertos": { + "title": "FreeRTOS", + "description": ( + "FreeRTOS is a real-time operating system kernel for embedded devices " + "that has been ported to 40 microcontroller platforms." + ), + }, +} + DOCS_ROOT_DIR = os.path.realpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "docs") ) @@ -92,16 +111,29 @@ def install_platforms(): @functools.cache def get_frameworks(): items = {} - for pkg in PlatformPackageManager().get_installed(): - p = PlatformFactory.new(pkg) + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) for name, options in (p.frameworks or {}).items(): - if name in items or not set(options.keys()).issuperset( - set(["title", "description"]) - ): + if name in items: continue - items[name] = dict( - name=name, title=options["title"], description=options["description"] - ) + if name in STATIC_FRAMEWORK_DATA: + items[name] = dict( + name=name, + title=STATIC_FRAMEWORK_DATA[name]["title"], + description=STATIC_FRAMEWORK_DATA[name]["description"], + ) + continue + title = options.get("title") or name.title() + description = options.get("description") + if "package" in options: + regdata = REGCLIENT.get_package( + "tool", + p.packages[options["package"]].get("owner", "platformio"), + options["package"], + ) + title = regdata["title"] or title + description = regdata["description"] + items[name] = dict(name=name, title=title, description=description) return sorted(items.values(), key=lambda item: item["name"]) @@ -129,7 +161,7 @@ def generate_boards_table(boards, skip_columns=None): ) # add header - for (name, template) in columns: + for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " @@ -156,7 +188,7 @@ def generate_boards_table(boards, skip_columns=None): rom=fs.humanize_file_size(data["rom"]), ) - for (name, template) in columns: + for name, template in columns: if skip_columns and name in skip_columns: continue prefix = " * - " if name == "Name" else " - " @@ -473,11 +505,12 @@ def generate_platform(pkg, rst_dir): .. code-block:: ini - ; Latest stable version + ; Latest stable version, NOT recommended + ; Pin the version as shown below [env:latest_stable] platform = {name} {board} - ; Custom stable version + ; Specific version [env:custom_stable] platform = {name}@x.y.z {board} @@ -904,7 +937,7 @@ def update_embedded_board(rst_path, board): - On-board - Default""" ) - for (tool_name, tool_data) in sorted(board["debug"]["tools"].items()): + for tool_name, tool_data in sorted(board["debug"]["tools"].items()): lines.append( """ * - {tool} - {onboard} diff --git a/setup.py b/setup.py index 2e1fa86c0e..42427a85c0 100644 --- a/setup.py +++ b/setup.py @@ -36,16 +36,17 @@ "pyelftools>=0.27,<1", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", + "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ "requests==%s" % ("2.27.1" if PY36 else "2.*"), "semantic_version==2.10.*", "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), ] home_requirements = [ - "aiofiles==%s" % ("0.8.0" if PY36 else "22.1.*"), + "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.23.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.20.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), + "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] @@ -67,6 +68,8 @@ "assets/templates/ide-projects/*/*.tpl", "assets/templates/ide-projects/*/.*.tpl", # include hidden files "assets/templates/ide-projects/*/.*/*.tpl", # include hidden folders + "assets/templates/ide-projects/*/*/*.tpl", # NetBeans + "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans ] }, entry_points={ diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index be6042ca18..cb6f77772c 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -15,8 +15,8 @@ # pylint: disable=redefined-outer-name import json +import os import sys -from os.path import isfile, join import pytest @@ -83,12 +83,12 @@ def check_dir(tmpdir_factory): def count_defects(output): error, warning, style = 0, 0, 0 - for l in output.split("\n"): - if "[high:error]" in l: + for line in output.split("\n"): + if "[high:error]" in line: error += 1 - elif "[medium:warning]" in l: + elif "[medium:warning]" in line: warning += 1 - elif "[low:style]" in l: + elif "[low:style]" in line: style += 1 return error, warning, style @@ -240,9 +240,9 @@ def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) inc_count = 0 - for l in result.output.split("\n"): - if l.startswith("Includes:"): - inc_count = l.count("-I") + for line in result.output.split("\n"): + if line.startswith("Includes:"): + inc_count = line.count("-I") # at least 1 include path for default mode assert inc_count > 0 @@ -259,46 +259,6 @@ def test_check_silent_mode(clirunner, validate_cliresult, check_dir): assert style == 0 -def test_check_custom_pattern_absolute_path( - clirunner, validate_cliresult, tmpdir_factory -): - project_dir = tmpdir_factory.mktemp("project") - project_dir.join("platformio.ini").write(DEFAULT_CONFIG) - - check_dir = tmpdir_factory.mktemp("custom_src_dir") - check_dir.join("main.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors == EXPECTED_ERRORS - assert warnings == EXPECTED_WARNINGS - assert style == EXPECTED_STYLE - - -def test_check_custom_pattern_relative_path( - clirunner, validate_cliresult, tmpdir_factory -): - tmpdir = tmpdir_factory.mktemp("project") - tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - - tmpdir.mkdir("app").join("main.cpp").write(TEST_CODE) - tmpdir.mkdir("prj").join("test.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors + warnings + style == EXPECTED_DEFECTS * 2 - - def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -427,7 +387,7 @@ def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, check_dir): validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output - assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) + assert not os.path.isfile(os.path.join(str(check_dir), "src", "main.cpp.dump")) def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): @@ -607,10 +567,10 @@ def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in result.output.split("\n"): - if not l.startswith("Includes:"): + for line in result.output.split("\n"): + if not line.startswith("Includes:"): continue - for inc in l.split(" "): + for inc in line.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) @@ -656,3 +616,147 @@ def test_check_handles_spaces_in_paths(clirunner, validate_cliresult, tmpdir_fac default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(default_result) + + +# +# Files filtering functionality +# + + +@pytest.mark.parametrize( + "src_filter,number_of_checked_files", + [ + (["+"], 1), + (["+"], 1), + (["+", "-"], 2), + (["-<*> + + +"], 3), + ], + ids=["Single file", "Glob pattern", "Exclude pattern", "Filter as string"], +) +def test_check_src_filter( + clirunner, + validate_cliresult, + tmpdir_factory, + src_filter, + number_of_checked_files, +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.join("app.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + cmd_args = ["--project-dir", str(tmpdir)] + [ + "--src-filters=%s" % f for f in src_filter + ] + + result = clirunner.invoke(cmd_check, cmd_args) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * number_of_checked_files + + +def test_check_src_filter_from_config(clirunner, validate_cliresult, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_src_filters = + + + + + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output + + +def test_check_custom_pattern_absolute_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + check_dir = tmpdir_factory.mktemp("custom_src_dir") + check_dir.join("main.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] + ) + + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == EXPECTED_WARNINGS + assert style == EXPECTED_STYLE + + +def test_check_custom_pattern_relative_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, + ["--project-dir", str(tmpdir), "--pattern=src/uart", "--pattern=src/spi"], + ) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + + +def test_check_src_filter_from_config_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_patterns = + src/spi/*.c* + tests/test.cpp + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index ae63228001..f67bada4d9 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -112,7 +112,6 @@ def test_ci_keep_build_dir_single_src_dir( def test_ci_keep_build_dir_nested_src_dirs( clirunner, tmpdir_factory, validate_cliresult ): - build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) # Split default Arduino project in two parts diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 009f33c95f..cb9e3d859e 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -22,6 +22,7 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir): ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), + ("-Iextra_inc", "-Iextra_inc"), ] tmpdir.join("platformio.ini").write( @@ -58,8 +59,20 @@ def post_prog_action(source, target, env): """ ) + tmpdir.mkdir("extra_inc").join("foo.h").write( + """ +#define FOO + """ + ) + tmpdir.mkdir("src").join("main.cpp").write( """ +#include "foo.h" + +#ifndef FOO +#error "FOO" +#endif + #ifdef I_AM_ONLY_SRC_FLAG #include #else diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2608950049..29ff20ab7d 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -642,7 +642,7 @@ def test_googletest_framework(clirunner, tmp_path: Path): pio_test_cmd, [ "-d", - project_dir, + str(project_dir), "-e", "native", "--json-output-path", diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index 52a6cac4c7..349e9c7920 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -35,7 +35,7 @@ def test_ping_internet_ips(): def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() - with pytest.raises(http.InternetIsOffline): + with pytest.raises(http.InternetConnectionError): regclient.fetch_json_data("get", "/v2/stats") diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 828412f4f6..a3279e051d 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -901,5 +901,5 @@ def test_broken_schemas(): ) # invalid package name - with pytest.raises(ManifestValidationError, match=("are not allowed")): + with pytest.raises(ManifestValidationError, match="are not allowed"): ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) diff --git a/tests/project/test_config.py b/tests/project/test_config.py index 6076a7d6c7..f559502ed3 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -23,7 +23,11 @@ from platformio import fs from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError +from platformio.project.exception import ( + InvalidEnvNameError, + InvalidProjectConfError, + UnknownEnvNamesError, +) BASE_CONFIG = """ [platformio] @@ -662,9 +666,21 @@ def test_extends_order(tmp_path: Path): [c] upload_tool = three -[env:native] +[env:na_ti-ve13] extends = a, b, c """ ) config = ProjectConfig(str(project_conf)) - assert config.get("env:native", "upload_tool") == "three" + assert config.get("env:na_ti-ve13", "upload_tool") == "three" + + +def test_invalid_env_names(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app:1] + """ + ) + config = ProjectConfig(str(project_conf)) + with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): + config.validate() diff --git a/tests/test_examples.py b/tests/test_examples.py index 27164fd71d..b49aa90ddd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,6 +22,7 @@ from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError def pytest_generate_tests(metafunc): @@ -47,6 +48,8 @@ def pytest_generate_tests(metafunc): for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue + if "mbed-legacy-examples" in root: + continue group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] @@ -64,6 +67,13 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() + + # temporary fix for unreleased dev-platforms with broken env name + try: + config.validate() + except ProjectError as exc: + pytest.skip(str(exc)) + build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) diff --git a/tox.ini b/tox.ini index 0a51741a44..608328066a 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.1.1 + sphinx-rtd-theme==1.2.0 sphinx-notfound-page sphinx-copybutton restructuredtext-lint