From e3c6237430246c50a3ba989316b51d1b05fcbce0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Oct 2021 23:29:34 +0300 Subject: [PATCH 01/46] Remove unused files --- .coveragerc | 21 --------------------- .readthedocs.yml | 12 ------------ 2 files changed, 33 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .readthedocs.yml diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 01ec788a43..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[report] -# Regexes for lines to exclude from consideration -exclude_lines = - pragma: no cover - def __repr__ - raise AssertionError - raise NotImplementedError diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 42a9ba4feb..0000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,12 +0,0 @@ -# See https://docs.readthedocs.io/en/stable/config-file/index.html - -version: 2 - -sphinx: - configuration: docs/conf.py - -formats: - - pdf - -submodules: - include: all From 8d8b0807e2dc330eb2c02e1a42d27d5778b45e33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Oct 2021 19:13:24 +0300 Subject: [PATCH 02/46] Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name // Resolve #4086 --- HISTORY.rst | 5 +++++ platformio/debug/config/base.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1089d25dfd..9801729efd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,11 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.2.3 (2021-??-??) +~~~~~~~~~~~~~~~~~~ + +- Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name (`issue #4086 `_) + 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 65ceba6008..de726bc9a5 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -210,7 +210,7 @@ def reveal_patterns(self, source, recursive=True): patterns = { "PLATFORMIO_CORE_DIR": get_project_core_dir(), "PYTHONEXE": proc.get_pythonexe_path(), - "PROJECT_DIR": self.project_config.path, + "PROJECT_DIR": os.getcwd(), "PROG_PATH": program_path, "PROG_DIR": os.path.dirname(program_path), "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]), From f86ed97820b8f945c704f3180a8422d71814ba14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Oct 2021 19:14:17 +0300 Subject: [PATCH 03/46] Bump version to 5.2.3a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 56d4567293..d49eef05e7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, 2) +VERSION = (5, 2, "3a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 9914b7ea3834c47b5ad1e92b8591ee4c19b32d7b Mon Sep 17 00:00:00 2001 From: Phill Price Date: Sat, 23 Oct 2021 11:01:48 +0100 Subject: [PATCH 04/46] Typo (#4087) showed > shown --- platformio/package/manager/_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 740afe328c..80d428891b 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -180,7 +180,7 @@ def print_multi_package_issue(self, packages, spec): ) self.print_message( "Please specify detailed REQUIREMENTS using package owner and version " - "(showed above) to avoid name conflicts", + "(shown above) to avoid name conflicts", fg="yellow", ) From 4839fe37a3d1433737c80147ce9a205100dc4925 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 18:19:40 +0300 Subject: [PATCH 05/46] =?UTF-8?q?Improved=20PlatformIO=20directory=20inter?= =?UTF-8?q?polation=20(${platformio.***=5Fdir})=20in=20=E2=80=9Cplatformio?= =?UTF-8?q?.ini=E2=80=9D=20configuration=20file=20//=20Resolve=20#3934?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 2 + docs | 2 +- platformio/app.py | 19 +++-- platformio/builder/main.py | 28 +++---- platformio/builder/tools/pioide.py | 2 +- platformio/commands/check/command.py | 4 +- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 4 +- platformio/commands/check/tools/pvsstudio.py | 2 +- platformio/commands/home/rpc/handlers/app.py | 18 +++-- .../commands/home/rpc/handlers/project.py | 6 +- platformio/commands/lib/command.py | 4 +- platformio/commands/project.py | 8 +- .../commands/remote/client/agent_service.py | 25 +++--- .../commands/remote/client/run_or_test.py | 16 ++-- platformio/commands/run/command.py | 2 +- platformio/commands/run/helpers.py | 4 +- platformio/commands/system/command.py | 4 +- platformio/commands/test/helpers.py | 2 +- platformio/commands/test/native.py | 2 +- platformio/commands/test/processor.py | 2 +- platformio/debug/config/base.py | 4 +- platformio/ide/projectgenerator.py | 10 +-- .../ide/tpls/clion/CMakeListsPrivate.txt.tpl | 2 +- platformio/package/manager/platform.py | 2 +- platformio/package/manager/tool.py | 2 +- platformio/platform/base.py | 6 +- platformio/project/config.py | 75 ++---------------- platformio/project/helpers.py | 63 ++++++--------- platformio/project/options.py | 78 +++++++++++++++++-- tests/package/test_manager.py | 2 +- tests/test_examples.py | 2 +- tests/test_projectconf.py | 45 ++++++++++- 33 files changed, 243 insertions(+), 206 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9801729efd..bcb6d0085d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,8 +11,10 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Improved PlatformIO directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name (`issue #4086 `_) + 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 66f67cb335..2f0abf6aba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 66f67cb3357265ab52a0567b67337ca7fda6c3e4 +Subproject commit 2f0abf6aba38c9310b08fd5200ed36eeacbfdb5a diff --git a/platformio/app.py b/platformio/app.py index e630dd3ceb..845d7e81e8 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -21,17 +21,17 @@ import platform import socket import uuid -from os.path import dirname, isdir, isfile, join, realpath from platformio import __version__, exception, fs, proc from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile -from platformio.project.helpers import get_default_projects_dir, get_project_core_dir +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_default_projects_dir def projects_dir_validate(projects_dir): - assert isdir(projects_dir) - return realpath(projects_dir) + assert os.path.isdir(projects_dir) + return os.path.realpath(projects_dir) DEFAULT_SETTINGS = { @@ -91,7 +91,10 @@ def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: - self.path = join(get_project_core_dir(), "appstate.json") + core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") + if not os.path.isdir(core_dir): + os.makedirs(core_dir) + self.path = os.path.join(core_dir, "appstate.json") self._storage = {} self._lockfile = None self.modified = False @@ -99,7 +102,7 @@ def __init__(self, path=None, lock=False): def __enter__(self): try: self._lock_state_file() - if isfile(self.path): + if os.path.isfile(self.path): self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) except ( @@ -117,7 +120,7 @@ def __exit__(self, type_, value, traceback): with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) except IOError: - raise exception.HomeDirPermissionsError(get_project_core_dir()) + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) self._unlock_state_file() def _lock_state_file(self): @@ -127,7 +130,7 @@ def _lock_state_file(self): try: self._lockfile.acquire() except IOError: - raise exception.HomeDirPermissionsError(dirname(self.path)) + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) def _unlock_state_file(self): if hasattr(self, "_lockfile") and self._lockfile: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 7f719cca68..055eaf56a9 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -109,22 +109,22 @@ config = env.GetProjectConfig() env.Replace( PROJECT_DIR=get_project_dir(), - PROJECT_CORE_DIR=config.get_optional_dir("core"), - PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"), - PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"), - PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"), - PROJECT_INCLUDE_DIR=config.get_optional_dir("include"), - PROJECT_SRC_DIR=config.get_optional_dir("src"), - PROJECTSRC_DIR=config.get_optional_dir("src"), # legacy for dev/platform - PROJECT_TEST_DIR=config.get_optional_dir("test"), - PROJECT_DATA_DIR=config.get_optional_dir("data"), - PROJECTDATA_DIR=config.get_optional_dir("data"), # legacy for dev/platform - PROJECT_BUILD_DIR=config.get_optional_dir("build"), - BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), + PROJECT_CORE_DIR=config.get("platformio", "core_dir"), + PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"), + PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"), + PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), + PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), + PROJECT_SRC_DIR=config.get("platformio", "src_dir"), + PROJECTSRC_DIR=config.get("platformio", "src_dir"), # legacy for dev/platform + PROJECT_TEST_DIR=config.get("platformio", "test_dir"), + PROJECT_DATA_DIR=config.get("platformio", "data_dir"), + PROJECTDATA_DIR=config.get("platformio", "data_dir"), # legacy for dev/platform + PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), + BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), LIBSOURCE_DIRS=[ - config.get_optional_dir("lib"), + config.get("platformio", "lib_dir"), os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), - config.get_optional_dir("globallib"), + config.get("platformio", "globallib_dir"), ], ) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 9d278b20cd..3903f87984 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -63,7 +63,7 @@ def _dump_includes(env): # include Unity framework if there are tests in project includes["unity"] = [] auto_install_unity = False - test_dir = env.GetProjectConfig().get_optional_dir("test") + test_dir = env.GetProjectConfig().get("platformio", "test_dir") if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]: auto_install_unity = True unity_dir = get_core_package_dir( diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 87673f992c..82ab7566c2 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -106,8 +106,8 @@ def cli( ) default_patterns = [ - config.get_optional_dir("src"), - config.get_optional_dir("include"), + config.get("platformio", "src_dir"), + config.get("platformio", "include_dir"), ] tool_options = dict( verbose=verbose, diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 3206cdba18..d6f990586c 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -77,7 +77,7 @@ def configure_command(self): includes = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( - self.config.get_optional_dir("packages").lower() + self.config.get("platformio", "packages_dir").lower() ): continue includes.append(inc) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index bc06799ab4..99512d528c 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -84,7 +84,7 @@ def parse_defect(self, raw_line): if ( args.get("file", "") .lower() - .startswith(self.config.get_optional_dir("packages").lower()) + .startswith(self.config.get("platformio", "packages_dir").lower()) ): if args["id"] in breaking_defect_ids: if self.options.get("verbose"): @@ -201,7 +201,7 @@ def _generate_inc_file(self): result = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( - self.config.get_optional_dir("packages").lower() + self.config.get("platformio", "packages_dir").lower() ): continue result.append(inc) diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 36298d259d..a872c0af82 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs): with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp: fp.write( "exclude-path = " - + self.config.get_optional_dir("packages").replace("\\", "/") + + self.config.get("platformio", "packages_dir").replace("\\", "/") ) with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp: diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 3c0ce4657b..b567f06b9d 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -14,16 +14,15 @@ from __future__ import absolute_import -from os.path import join +import os from platformio import __version__, app, fs, util -from platformio.project.helpers import get_project_core_dir, is_platformio_project +from platformio.project.config import ProjectConfig +from platformio.project.helpers import is_platformio_project class AppRPC: - APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") - IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", @@ -34,9 +33,16 @@ class AppRPC: "projectsDir", ] + @staticmethod + def get_state_path(): + core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") + if not os.path.isdir(core_dir): + os.makedirs(core_dir) + return os.path.join(core_dir, "homestate.json") + @staticmethod def load_state(): - with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: + with app.State(AppRPC.get_state_path(), lock=True) as state: storage = state.get("storage", {}) # base data @@ -72,7 +78,7 @@ def get_state(): @staticmethod def save_state(state): - with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: + with app.State(AppRPC.get_state_path(), lock=True) as s: s.clear() s.update(state) storage = s.get("storage", {}) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 4c48a5d9f1..535bc9be98 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -81,7 +81,7 @@ def _get_project_data(): data["description"] = config.get("platformio", "description") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) - libdeps_dir = config.get_optional_dir("libdeps") + libdeps_dir = config.get("platformio", "libdeps_dir") for section in config.sections(): if not section.startswith("env:"): continue @@ -253,7 +253,7 @@ def _generate_project_main(project_dir, board, framework): with fs.cd(project_dir): config = ProjectConfig() - src_dir = config.get_optional_dir("src") + src_dir = config.get("platformio", "src_dir") main_path = os.path.join( src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") ) @@ -308,7 +308,7 @@ async def import_arduino(board, use_arduino_libs, arduino_project_dir): ) with fs.cd(project_dir): config = ProjectConfig() - src_dir = config.get_optional_dir("src") + src_dir = config.get("platformio", "src_dir") if os.path.isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir, symlinks=True) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index dac038164d..7b6c7bb79e 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -43,7 +43,7 @@ def get_project_global_lib_dir(): - return ProjectConfig.get_instance().get_optional_dir("globallib") + return ProjectConfig.get_instance().get("platformio", "globallib_dir") @click.group(short_help="Library manager") @@ -111,7 +111,7 @@ def cli(ctx, **options): os.path.join(storage_dir, "platformio.ini") ) config.validate(options["environment"], silent=in_silence) - libdeps_dir = config.get_optional_dir("libdeps") + libdeps_dir = config.get("platformio", "libdeps_dir") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue diff --git a/platformio/commands/project.py b/platformio/commands/project.py index d45d6dcd98..c79979fd57 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -226,10 +226,10 @@ def init_base_project(project_dir): config = ProjectConfig() config.save() dir_to_readme = [ - (config.get_optional_dir("src"), None), - (config.get_optional_dir("include"), init_include_readme), - (config.get_optional_dir("lib"), init_lib_readme), - (config.get_optional_dir("test"), init_test_readme), + (config.get("platformio", "src_dir"), None), + (config.get("platformio", "include_dir"), init_include_readme), + (config.get("platformio", "lib_dir"), init_lib_readme), + (config.get("platformio", "test_dir"), init_test_readme), ] for (path, cb) in dir_to_readme: if os.path.isdir(path): diff --git a/platformio/commands/remote/client/agent_service.py b/platformio/commands/remote/client/agent_service.py index 5918d20550..01af44da46 100644 --- a/platformio/commands/remote/client/agent_service.py +++ b/platformio/commands/remote/client/agent_service.py @@ -13,7 +13,6 @@ # limitations under the License. import os -from os.path import getatime, getmtime, isdir, isfile, join from twisted.logger import LogLevel # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error @@ -25,15 +24,16 @@ from platformio.commands.remote.client.base import RemoteClientBase from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.helpers import get_project_core_dir class RemoteAgentService(RemoteClientBase): def __init__(self, name, share, working_dir=None): RemoteClientBase.__init__(self) self.log_level = LogLevel.info - self.working_dir = working_dir or join(get_project_core_dir(), "remote") - if not isdir(self.working_dir): + self.working_dir = working_dir or os.path.join( + ProjectConfig.get_instance().get("platformio", "core_dir"), "remote" + ) + if not os.path.isdir(self.working_dir): os.makedirs(self.working_dir) if name: self.name = str(name)[:50] @@ -138,14 +138,14 @@ def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branch self, command, options ): assert options and "project_id" in options - project_dir = join(self.working_dir, "projects", options["project_id"]) - origin_pio_ini = join(project_dir, "platformio.ini") - back_pio_ini = join(project_dir, "platformio.ini.bak") + project_dir = os.path.join(self.working_dir, "projects", options["project_id"]) + origin_pio_ini = os.path.join(project_dir, "platformio.ini") + back_pio_ini = os.path.join(project_dir, "platformio.ini.bak") # remove insecure project options try: conf = ProjectConfig(origin_pio_ini) - if isfile(back_pio_ini): + if os.path.isfile(back_pio_ini): os.remove(back_pio_ini) os.rename(origin_pio_ini, back_pio_ini) # cleanup @@ -159,7 +159,10 @@ def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branch conf.save(origin_pio_ini) # restore A/M times - os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini))) + os.utime( + origin_pio_ini, + (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), + ) except NotPlatformIOProjectError as e: raise pb.Error(str(e)) @@ -194,8 +197,8 @@ def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branch paused_acs.append(ac) def _cb_on_end(): - if isfile(back_pio_ini): - if isfile(origin_pio_ini): + if os.path.isfile(back_pio_ini): + if os.path.isfile(origin_pio_ini): os.remove(origin_pio_ini) os.rename(back_pio_ini, origin_pio_ini) for ac in paused_acs: diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py index 10a9b008da..cdc1465dc8 100644 --- a/platformio/commands/remote/client/run_or_test.py +++ b/platformio/commands/remote/client/run_or_test.py @@ -69,8 +69,8 @@ def add_project_items(self, psync): os.path.join(self.options["project_dir"], "platformio.ini") ) psync.add_item(cfg.path, "platformio.ini") - psync.add_item(cfg.get_optional_dir("shared"), "shared") - psync.add_item(cfg.get_optional_dir("boards"), "boards") + psync.add_item(cfg.get("platformio", "shared_dir"), "shared") + psync.add_item(cfg.get("platformio", "boards_dir"), "boards") if self.options["force_remote"]: self._add_project_source_items(cfg, psync) @@ -78,26 +78,26 @@ def add_project_items(self, psync): self._add_project_binary_items(cfg, psync) if self.command == "test": - psync.add_item(cfg.get_optional_dir("test"), "test") + psync.add_item(cfg.get("platformio", "test_dir"), "test") def _add_project_source_items(self, cfg, psync): - psync.add_item(cfg.get_optional_dir("lib"), "lib") + psync.add_item(cfg.get("platformio", "lib_dir"), "lib") psync.add_item( - cfg.get_optional_dir("include"), + cfg.get("platformio", "include_dir"), "include", cb_filter=self._cb_tarfile_filter, ) psync.add_item( - cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter + cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter ) if set(["buildfs", "uploadfs", "uploadfsota"]) & set( self.options.get("target", []) ): - psync.add_item(cfg.get_optional_dir("data"), "data") + psync.add_item(cfg.get("platformio", "data_dir"), "data") @staticmethod def _add_project_binary_items(cfg, psync): - build_dir = cfg.get_optional_dir("build") + build_dir = cfg.get("platformio", "build_dir") for env_name in os.listdir(build_dir): env_dir = os.path.join(build_dir, env_name) if not os.path.isdir(env_dir): diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 5684f1192f..85fddd727a 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -101,7 +101,7 @@ def cli( # clean obsolete build dir if not disable_auto_clean: - build_dir = config.get_optional_dir("build") + build_dir = config.get("platformio", "build_dir") try: clean_build_dir(build_dir, config) except: # pylint: disable=bare-except diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index c976d9352a..925b558484 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -23,8 +23,8 @@ def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir( - "libdeps" + if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get( + "platformio", "libdeps_dir" ): return if not config.has_section("env"): diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 63d1727cd0..c207d0ae34 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -64,7 +64,7 @@ def system_info(json_output): } data["core_dir"] = { "title": "PlatformIO Core Directory", - "value": project_config.get_optional_dir("core"), + "value": project_config.get("platformio", "core_dir"), } data["platformio_exe"] = { "title": "PlatformIO Core Executable", @@ -88,7 +88,7 @@ def system_info(json_output): "title": "Tools & Toolchains", "value": len( ToolPackageManager( - project_config.get_optional_dir("packages") + project_config.get("platformio", "packages_dir") ).get_installed() ), } diff --git a/platformio/commands/test/helpers.py b/platformio/commands/test/helpers.py index ce72360fd3..e490ea7cd3 100644 --- a/platformio/commands/test/helpers.py +++ b/platformio/commands/test/helpers.py @@ -18,7 +18,7 @@ def get_test_names(config): - test_dir = config.get_optional_dir("test") + test_dir = config.get("platformio", "test_dir") if not os.path.isdir(test_dir): raise exception.TestDirNotExists(test_dir) names = [] diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index a73c03fe6b..3c30e97b82 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -31,7 +31,7 @@ def process(self): return self.run() def run(self): - build_dir = self.options["project_config"].get_optional_dir("build") + build_dir = self.options["project_config"].get("platformio", "build_dir") result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 3e9a519d4b..0e1e367fd6 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -124,7 +124,7 @@ def print_progress(self, text): def build_or_upload(self, target): if not self._output_file_generated: self.generate_output_file( - self.options["project_config"].get_optional_dir("test") + self.options["project_config"].get("platformio", "test_dir") ) self._output_file_generated = True diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index de726bc9a5..bcafa8fdfd 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -20,7 +20,7 @@ from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.helpers import reveal_debug_port from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_core_dir, load_project_ide_data +from platformio.project.helpers import load_project_ide_data from platformio.project.options import ProjectOptions @@ -208,7 +208,7 @@ def get_init_script(self, debugger): def reveal_patterns(self, source, recursive=True): program_path = self.program_path or "" patterns = { - "PLATFORMIO_CORE_DIR": get_project_core_dir(), + "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"), "PYTHONEXE": proc.get_pythonexe_path(), "PROJECT_DIR": os.getcwd(), "PROG_PATH": program_path, diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 56930b9a0c..f611219623 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -79,11 +79,11 @@ def _load_tplvars(self): tpl_vars.update( { "src_files": self.get_src_files(), - "project_src_dir": self.config.get_optional_dir("src"), - "project_lib_dir": self.config.get_optional_dir("lib"), - "project_test_dir": self.config.get_optional_dir("test"), + "project_src_dir": self.config.get("platformio", "src_dir"), + "project_lib_dir": self.config.get("platformio", "lib_dir"), + "project_test_dir": self.config.get("platformio", "test_dir"), "project_libdeps_dir": os.path.join( - self.config.get_optional_dir("libdeps"), self.env_name + self.config.get("platformio", "libdeps_dir"), self.env_name ), } ) @@ -103,7 +103,7 @@ def _load_tplvars(self): def get_src_files(self): result = [] with fs.cd(self.project_dir): - for root, _, files in os.walk(self.config.get_optional_dir("src")): + for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append(os.path.relpath(os.path.join(root, f))) return result diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 9df294ee20..c0edf6aaa2 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -40,7 +40,7 @@ % end % % def _get_lib_dirs(envname): -% env_libdeps_dir = os.path.join(config.get_optional_dir("libdeps"), envname) +% env_libdeps_dir = os.path.join(config.get("platformio", "libdeps_dir"), envname) % env_lib_extra_dirs = config.get("env:" + envname, "lib_extra_dirs", []) % return _fix_lib_dirs([env_libdeps_dir] + env_lib_extra_dirs) % end diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index efe8a361a5..ecce6a2250 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -31,7 +31,7 @@ def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super(PlatformPackageManager, self).__init__( PackageType.PLATFORM, - package_dir or self.config.get_optional_dir("platforms"), + package_dir or self.config.get("platformio", "platforms_dir"), ) @property diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index 60aededd45..70a76377e7 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -20,7 +20,7 @@ class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): if not package_dir: - package_dir = ProjectConfig.get_instance().get_optional_dir("packages") + package_dir = ProjectConfig.get_instance().get("platformio", "packages_dir") super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir) @property diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 1d7911b009..d5ce103dc0 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -45,7 +45,7 @@ def __init__(self, manifest_path): self._custom_packages = None self.config = ProjectConfig.get_instance() - self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) + self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir")) @property def name(self): @@ -145,8 +145,8 @@ def _append_board(board_id, manifest_path): self._BOARDS_CACHE[board_id] = config bdirs = [ - self.config.get_optional_dir("boards"), - os.path.join(self.config.get_optional_dir("core"), "boards"), + self.config.get("platformio", "boards_dir"), + os.path.join(self.config.get("platformio", "core_dir"), "boards"), os.path.join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 187cb99305..30c8a9aef2 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,7 +13,6 @@ # limitations under the License. import glob -import hashlib import json import os import re @@ -21,7 +20,7 @@ import click from platformio import fs -from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types +from platformio.compat import string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -311,8 +310,11 @@ def get(self, section, option, default=MISSING): if not option_meta: return value + if option_meta.validate: + value = option_meta.validate(value) if option_meta.multiple: value = self.parse_multi_values(value or []) + try: return self.cast_to(value, option_meta.type) except click.BadParameter as e: @@ -353,74 +355,7 @@ def validate(self, envs=None, silent=False): return True -class ProjectConfigDirsMixin(object): - def _get_core_dir(self, exists=False): - default = ProjectOptions["platformio.core_dir"].default - core_dir = self.get("platformio", "core_dir") - win_core_dir = None - if IS_WINDOWS and core_dir == default: - win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio" - if os.path.isdir(win_core_dir): - core_dir = win_core_dir - - if exists and not os.path.isdir(core_dir): - try: - os.makedirs(core_dir) - except OSError as e: - if win_core_dir: - os.makedirs(win_core_dir) - core_dir = win_core_dir - else: - raise e - - return core_dir - - def get_optional_dir(self, name, exists=False): - if not ProjectOptions.get("platformio.%s_dir" % name): - raise ValueError("Unknown optional directory -> " + name) - - if name == "core": - result = self._get_core_dir(exists) - else: - result = self.get("platformio", name + "_dir") - - if result is None: - return None - - project_dir = os.getcwd() - - # patterns - if "$PROJECT_HASH" in result: - result = result.replace( - "$PROJECT_HASH", - "%s-%s" - % ( - os.path.basename(project_dir), - hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], - ), - ) - - if "$PROJECT_DIR" in result: - result = result.replace("$PROJECT_DIR", project_dir) - if "$PROJECT_CORE_DIR" in result: - result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core")) - if "$PROJECT_WORKSPACE_DIR" in result: - result = result.replace( - "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace") - ) - - if result.startswith("~"): - result = fs.expanduser(result) - - result = os.path.realpath(result) - - if exists and not os.path.isdir(result): - os.makedirs(result) - - return result - - -class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): +class ProjectConfig(ProjectConfigBase): _instances = {} diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 655e968540..100c1a75f2 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,8 +15,6 @@ import json import os from hashlib import sha1 -from os import walk -from os.path import dirname, isdir, isfile, join from click.testing import CliRunner @@ -32,65 +30,50 @@ def get_project_dir(): def is_platformio_project(project_dir=None): if not project_dir: project_dir = get_project_dir() - return isfile(join(project_dir, "platformio.ini")) + return os.path.isfile(os.path.join(project_dir, "platformio.ini")) def find_project_dir_above(path): - if isfile(path): - path = dirname(path) + if os.path.isfile(path): + path = os.path.dirname(path) if is_platformio_project(path): return path - if isdir(dirname(path)): - return find_project_dir_above(dirname(path)) + if os.path.isdir(os.path.dirname(path)): + return find_project_dir_above(os.path.dirname(path)) return None -def get_project_core_dir(): - """Deprecated, use ProjectConfig.get_optional_dir("core") instead""" - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("core", exists=True) - - def get_project_cache_dir(): - """Deprecated, use ProjectConfig.get_optional_dir("cache") instead""" - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("cache") + """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead""" + return ProjectConfig.get_instance().get("platformio", "cache_dir") def get_project_global_lib_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("globallib") instead + Deprecated, use ProjectConfig.get("platformio", "globallib_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("globallib") + return ProjectConfig.get_instance().get("platformio", "globallib_dir") def get_project_lib_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("lib") instead + Deprecated, use ProjectConfig.get("platformio", "lib_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("lib") + return ProjectConfig.get_instance().get("platformio", "lib_dir") def get_project_libdeps_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead + Deprecated, use ProjectConfig.get("platformio", "libdeps_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("libdeps") + return ProjectConfig.get_instance().get("platformio", "libdeps_dir") def get_default_projects_dir(): - docs_dir = join(fs.expanduser("~"), "Documents") + docs_dir = os.path.join(fs.expanduser("~"), "Documents") try: assert IS_WINDOWS import ctypes.wintypes # pylint: disable=import-outside-toplevel @@ -100,7 +83,7 @@ def get_default_projects_dir(): docs_dir = buf.value except: # pylint: disable=bare-except pass - return join(docs_dir, "PlatformIO", "Projects") + return os.path.join(docs_dir, "PlatformIO", "Projects") def compute_project_checksum(config): @@ -113,16 +96,16 @@ def compute_project_checksum(config): # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") for d in ( - config.get_optional_dir("include"), - config.get_optional_dir("src"), - config.get_optional_dir("lib"), + config.get("platformio", "include_dir"), + config.get("platformio", "src_dir"), + config.get("platformio", "lib_dir"), ): - if not isdir(d): + if not os.path.isdir(d): continue chunks = [] - for root, _, files in walk(d): + for root, _, files in os.walk(d): for f in files: - path = join(root, f) + path = os.path.join(root, f) if path.endswith(check_suffixes): chunks.append(path) if not chunks: @@ -171,8 +154,8 @@ def _load_project_ide_data(project_dir, env_names): def _load_cached_project_ide_data(project_dir, env_names): build_dir = ProjectConfig.get_instance( - join(project_dir, "platformio.ini") - ).get_optional_dir("build") + os.path.join(project_dir, "platformio.ini") + ).get("platformio", "build_dir") result = {} for name in env_names: if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): diff --git a/platformio/project/options.py b/platformio/project/options.py index 3c9db96cc8..f94a74c6d2 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -14,12 +14,14 @@ # pylint: disable=redefined-builtin, too-many-arguments +import hashlib import os from collections import OrderedDict import click from platformio import fs +from platformio.compat import IS_WINDOWS, hashlib_encode_data class ConfigOption(object): # pylint: disable=too-many-instance-attributes @@ -35,6 +37,7 @@ def __init__( buildenvvar=None, oldnames=None, default=None, + validate=None, ): self.scope = scope self.group = group @@ -46,6 +49,7 @@ def __init__( self.buildenvvar = buildenvvar self.oldnames = oldnames self.default = default + self.validate = validate def as_dict(self): result = dict( @@ -60,13 +64,11 @@ def as_dict(self): ) if isinstance(self.type, click.ParamType): result["type"] = self.type.name - if isinstance(self.type, (click.IntRange, click.FloatRange)): result["min"] = self.type.min result["max"] = self.type.max if isinstance(self.type, click.Choice): result["choices"] = self.type.choices - return result @@ -78,6 +80,50 @@ def ConfigEnvOption(*args, **kwargs): return ConfigOption("env", *args, **kwargs) +def calculate_path_hash(path): + return "%s-%s" % ( + os.path.basename(path), + hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10], + ) + + +def expand_dir_templates(path): + project_dir = os.getcwd() + tpls = { + "$PROJECT_DIR": lambda: project_dir, + "$PROJECT_HASH": lambda: calculate_path_hash(project_dir), + } + done = False + while not done: + done = True + for tpl, cb in tpls.items(): + if tpl not in path: + continue + path = path.replace(tpl, cb()) + done = False + return path + + +def validate_dir(path): + if not path: + return path + if path.startswith("~"): + path = fs.expanduser(path) + if "$" in path: + path = expand_dir_templates(path) + return os.path.realpath(path) + + +def validate_core_dir(path): + default = ProjectOptions["platformio.core_dir"].default + win_core_dir = None + if IS_WINDOWS and path == default: + win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" + if os.path.isdir(win_core_dir): + path = win_core_dir + return validate_dir(path) + + ProjectOptions = OrderedDict( [ ("%s.%s" % (option.scope, option.name), option) @@ -121,6 +167,7 @@ def ConfigEnvOption(*args, **kwargs): oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", default=os.path.join(fs.expanduser("~"), ".platformio"), + validate=validate_core_dir, ), ConfigPlatformioOption( group="directory", @@ -130,7 +177,8 @@ def ConfigEnvOption(*args, **kwargs): "Finder (LDF) looks for global libraries" ), sysenvvar="PLATFORMIO_GLOBALLIB_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "lib"), + default=os.path.join("${platformio.core_dir}", "lib"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -140,7 +188,8 @@ def ConfigEnvOption(*args, **kwargs): "platforms" ), sysenvvar="PLATFORMIO_PLATFORMS_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "platforms"), + default=os.path.join("${platformio.core_dir}", "platforms"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -149,7 +198,8 @@ def ConfigEnvOption(*args, **kwargs): "A location where PlatformIO Core keeps installed packages" ), sysenvvar="PLATFORMIO_PACKAGES_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "packages"), + default=os.path.join("${platformio.core_dir}", "packages"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -160,7 +210,8 @@ def ConfigEnvOption(*args, **kwargs): "other service information)" ), sysenvvar="PLATFORMIO_CACHE_DIR", - default=os.path.join("$PROJECT_CORE_DIR", ".cache"), + default=os.path.join("${platformio.core_dir}", ".cache"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -171,6 +222,7 @@ def ConfigEnvOption(*args, **kwargs): "build environments" ), sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -182,6 +234,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_WORKSPACE_DIR", default=os.path.join("$PROJECT_DIR", ".pio"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -192,7 +245,8 @@ def ConfigEnvOption(*args, **kwargs): "and other cached information" ), sysenvvar="PLATFORMIO_BUILD_DIR", - default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), + default=os.path.join("${platformio.workspace_dir}", "build"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -202,7 +256,8 @@ def ConfigEnvOption(*args, **kwargs): "dependencies declared via `lib_deps` option" ), sysenvvar="PLATFORMIO_LIBDEPS_DIR", - default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), + default=os.path.join("${platformio.workspace_dir}", "libdeps"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -213,6 +268,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_INCLUDE_DIR", default=os.path.join("$PROJECT_DIR", "include"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -223,6 +279,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_SRC_DIR", default=os.path.join("$PROJECT_DIR", "src"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -230,6 +287,7 @@ def ConfigEnvOption(*args, **kwargs): description="A storage for the custom/private project libraries", sysenvvar="PLATFORMIO_LIB_DIR", default=os.path.join("$PROJECT_DIR", "lib"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -240,6 +298,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_DATA_DIR", default=os.path.join("$PROJECT_DIR", "data"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -250,6 +309,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_TEST_DIR", default=os.path.join("$PROJECT_DIR", "test"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -257,6 +317,7 @@ def ConfigEnvOption(*args, **kwargs): description="A global storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("$PROJECT_DIR", "boards"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -267,6 +328,7 @@ def ConfigEnvOption(*args, **kwargs): ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), + validate=validate_dir, ), # # [env] diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 19d4a98e97..5bd118e7cf 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -486,5 +486,5 @@ def test_update_without_metadata(isolated_pio_core, tmpdir_factory): # update lm = LibraryPackageManager(str(storage_dir)) new_pkg = lm.update(pkg, silent=True) - assert len(lm.get_installed()) == 3 + assert len(lm.get_installed()) == 4 assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/test_examples.py b/tests/test_examples.py index 9e2b7cc079..0be516eb4a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -67,7 +67,7 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() - build_dir = config.get_optional_dir("build") + build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 99c71ec09a..ca859218e3 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -18,12 +18,15 @@ import pytest +from platformio import fs from platformio.project.config import ConfigParser, ProjectConfig from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError +from platformio.project.options import calculate_path_hash BASE_CONFIG = """ [platformio] env_default = base, extra_2 +build_dir = ~/tmp/pio-$PROJECT_HASH extra_configs = extra_envs.ini extra_debug.ini @@ -83,17 +86,23 @@ build_flags = ${custom.debug_flags} ${custom.extra_flags} lib_ignore = ${env.lib_ignore}, Lib3 upload_port = /dev/extra_2/port +debug_server = ${custom.debug_server} """ EXTRA_DEBUG_CONFIG = """ # Override original "custom.debug_flags" [custom] debug_flags = -D DEBUG=1 +debug_server = + ${platformio.packages_dir}/tool-openocd/openocd + --help [env:extra_2] build_flags = -Og """ +DEFAULT_CORE_DIR = os.path.join(fs.expanduser("~"), ".platformio") + @pytest.fixture(scope="module") def config(tmpdir_factory): @@ -124,7 +133,7 @@ def test_warnings(config): def test_defaults(config): - assert config.get_optional_dir("core") == os.path.join( + assert config.get("platformio", "core_dir") == os.path.join( os.path.expanduser("~"), ".platformio" ) assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] @@ -272,6 +281,14 @@ def test_getraw_value(config): assert config.getraw("env", "monitor_speed") == "9600" assert config.getraw("env:test_extends", "monitor_speed") == "115200" + # dir options + packages_dir = os.path.join(DEFAULT_CORE_DIR, "packages") + assert config.get("platformio", "packages_dir") == packages_dir + assert ( + config.getraw("custom", "debug_server") + == f"\n{packages_dir}/tool-openocd/openocd\n--help" + ) + def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" @@ -293,6 +310,15 @@ def test_get_value(config): "-D CUSTOM_DEBUG_FLAG", ] + # dir options + assert config.get("platformio", "packages_dir") == os.path.join( + DEFAULT_CORE_DIR, "packages" + ) + assert config.get("env:extra_2", "debug_server") == [ + os.path.join(DEFAULT_CORE_DIR, "packages", "tool-openocd", "openocd"), + "--help", + ] + def test_items(config): assert config.items("custom") == [ @@ -300,6 +326,11 @@ def test_items(config): ("lib_flags", "-lc -lm"), ("extra_flags", ""), ("lib_ignore", "LibIgnoreCustom"), + ( + "debug_server", + "\n%s/tool-openocd/openocd\n--help" + % os.path.join(DEFAULT_CORE_DIR, "packages"), + ), ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), @@ -326,6 +357,13 @@ def test_items(config): ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), + ( + "debug_server", + [ + "%s/tool-openocd/openocd" % os.path.join(DEFAULT_CORE_DIR, "packages"), + "--help", + ], + ), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), @@ -426,6 +464,11 @@ def test_dump(tmpdir_factory): ( "platformio", [ + ( + "build_dir", + "%s-%s" + % (fs.expanduser("~/tmp/pio"), calculate_path_hash(os.getcwd())), + ), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ("default_envs", ["base", "extra_2"]), ], From e0e4a594e92f6ab1a5db14bda851db4ed6939dda Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 19:59:52 +0300 Subject: [PATCH 06/46] Fix conf tests on Windows --- tests/test_projectconf.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index ca859218e3..b1859c5d86 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -245,8 +245,9 @@ def test_sysenv_options(config): ] # sysenv - os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir" - assert config.get("platformio", "core_dir") == "/custom/core/dir" + custom_core_dir = os.path.join(os.getcwd(), "custom") + os.environ["PLATFORMIO_HOME_DIR"] = custom_core_dir + assert config.get("platformio", "core_dir") == os.path.realpath(custom_core_dir) # cleanup system environment variables del os.environ["PLATFORMIO_BUILD_FLAGS"] @@ -315,7 +316,7 @@ def test_get_value(config): DEFAULT_CORE_DIR, "packages" ) assert config.get("env:extra_2", "debug_server") == [ - os.path.join(DEFAULT_CORE_DIR, "packages", "tool-openocd", "openocd"), + os.path.join(DEFAULT_CORE_DIR, "packages/tool-openocd/openocd"), "--help", ] @@ -467,7 +468,10 @@ def test_dump(tmpdir_factory): ( "build_dir", "%s-%s" - % (fs.expanduser("~/tmp/pio"), calculate_path_hash(os.getcwd())), + % ( + os.path.realpath(fs.expanduser("~/tmp/pio")), + calculate_path_hash(os.getcwd()), + ), ), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ("default_envs", ["base", "extra_2"]), From 947e57b5b4e7f7bad2156bd3cd008ff622a87b42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 20:00:30 +0300 Subject: [PATCH 07/46] Bump version to 5.2.3a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index d49eef05e7..2276222181 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3a1") +VERSION = (5, 2, "3a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 78182fea0adb17079f2cd3d262b8c4782ac7513b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 21:27:25 +0300 Subject: [PATCH 08/46] Disabled resolving of SCons variables when preprocessing "Interpolation of Values" // Resolve #3933 --- HISTORY.rst | 9 +++++---- platformio/project/config.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bcb6d0085d..52b2e40735 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,14 +11,15 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ -- Improved PlatformIO directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) -- Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name (`issue #4086 `_) +- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) +- Disabled resolving of SCons variables (e.g., ``${SOURCE.get_abspath()}``) when preprocessing `Interpolation of Values `__ (`issue #3933 `_) +- Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the ``debug_server`` option (`issue #4086 `_) 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ -- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug `__ command +- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug `__ command - Added support for CLion IDE 2021.3 (`pull #4085 `_) - Removed debugging "legacy Click" message from CLI (`issue #4083 `_) - Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" `__ with the multi-line default value (`issue #4082 `_) @@ -32,7 +33,7 @@ PlatformIO Core 5 - Handle the "test" folder as a part of CLion project (`issue #4005 `_) - Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 `_) - Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 `_) -- Fixed a "FileExist" error when the `platformio ci `__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 `_) +- Fixed a "FileExist" error when the `platformio ci `__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 `_) - Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 `_) 5.2.0 (2021-09-13) diff --git a/platformio/project/config.py b/platformio/project/config.py index 30c8a9aef2..124667e053 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -48,7 +48,7 @@ class ProjectConfigBase(object): INLINE_COMMENT_RE = re.compile(r"\s+;.*$") - VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") + VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") expand_interpolations = True warnings = [] From 7d7480c120675d97b1fcd031645bf7ad50724d22 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 22:21:15 +0300 Subject: [PATCH 09/46] Show human-readable message when infinite recursion is detected while processing "Interpolation of Values" // Resolve #3883 --- HISTORY.rst | 5 +++-- platformio/project/config.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 52b2e40735..6aa1832cc4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,9 +11,10 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) -- Disabled resolving of SCons variables (e.g., ``${SOURCE.get_abspath()}``) when preprocessing `Interpolation of Values `__ (`issue #3933 `_) -- Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the ``debug_server`` option (`issue #4086 `_) +- Ignore resolving of SCons variables (e.g., ``${SOURCE.get_abspath()}``) when preprocessing interpolations (`issue #3933 `_) +- Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) 5.2.2 (2021-10-20) diff --git a/platformio/project/config.py b/platformio/project/config.py index 124667e053..1ed6b4d30c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -294,7 +294,12 @@ def _re_interpolation_handler(self, match): section, option = match.group(1), match.group(2) if section == "sysenv": return os.getenv(option) - value = self.getraw(section, option) + try: + value = self.getraw(section, option) + except RecursionError: + raise exception.ProjectOptionValueError( + "Infinite recursion has been detected", option, section + ) if isinstance(value, list): return "\n".join(value) return value From 4943504898c8744b693972658fe140239ba7a20b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Oct 2021 23:17:30 +0300 Subject: [PATCH 10/46] Bump version to 5.2.3a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2276222181..2e740a7fe9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3a2") +VERSION = (5, 2, "3a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f1638c9cd7cda0fd3238ee55db144ba708f6a00b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 13:24:36 +0300 Subject: [PATCH 11/46] Fixed an issue when PIO Remote device monitor crashes on the first keypress // Resolve #3832 --- HISTORY.rst | 1 + platformio/commands/remote/client/device_monitor.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6aa1832cc4..2607fb415e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO Core 5 - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${SOURCE.get_abspath()}``) when preprocessing interpolations (`issue #3933 `_) - Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) +- Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) 5.2.2 (2021-10-20) diff --git a/platformio/commands/remote/client/device_monitor.py b/platformio/commands/remote/client/device_monitor.py index 4c68f85f15..b880d27047 100644 --- a/platformio/commands/remote/client/device_monitor.py +++ b/platformio/commands/remote/client/device_monitor.py @@ -84,7 +84,7 @@ def __init__(self, agents, **kwargs): self._ac_id = None self._d_acread = None self._d_acwrite = None - self._acwrite_buffer = "" + self._acwrite_buffer = b"" def agent_pool_ready(self): d = task.deferLater( @@ -226,7 +226,7 @@ def acwrite_data(self, data, force=False): return data = self._acwrite_buffer - self._acwrite_buffer = "" + self._acwrite_buffer = b"" try: d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data) d.addCallback(self.cb_acwrite_result) @@ -237,4 +237,4 @@ def acwrite_data(self, data, force=False): def cb_acwrite_result(self, result): assert result > 0 if self._acwrite_buffer: - self.acwrite_data("") + self.acwrite_data(b"") From d01435f4f21006a2c068d2720f9f3440b9807b9a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 13:28:57 +0300 Subject: [PATCH 12/46] Bump version to 5.2.3b1 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2607fb415e..f05a4637e6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ PlatformIO Core 5 - Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) -- Ignore resolving of SCons variables (e.g., ``${SOURCE.get_abspath()}``) when preprocessing interpolations (`issue #3933 `_) +- Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) - Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) - Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2e740a7fe9..b2e0a694a4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3a3") +VERSION = (5, 2, "3b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3a670b55b688c9f2160d0de34c74d46a466aae24 Mon Sep 17 00:00:00 2001 From: Ilia Motornyi Date: Mon, 25 Oct 2021 14:56:12 +0300 Subject: [PATCH 13/46] Update CMakeLists.txt.tpl (#4089) --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 3153e616d2..ca3099eefc 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -26,7 +26,7 @@ add_custom_target( add_custom_target( Debug ALL - COMMAND platformio -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" + COMMAND platformio -c clion debug "$<$>:-e${CMAKE_BUILD_TYPE}>" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) From 852c25230213cc295446105e4d358e5a25cc5440 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 15:18:18 +0300 Subject: [PATCH 14/46] Added support for custom device monitor filters // Resolve #3924 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/device/command.py | 21 ++++++++++++++------- platformio/commands/device/helpers.py | 4 +--- platformio/project/options.py | 10 +++++++++- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f05a4637e6..d0dd63ad59 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Added support for custom `device monitor filters `__ (`issue #3924 `_) - Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) diff --git a/docs b/docs index 2f0abf6aba..7e751e6990 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2f0abf6aba38c9310b08fd5200ed36eeacbfdb5a +Subproject commit 7e751e6990958ca6b8120f37b2405a41f0b5b72d diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 1f386438e0..1cf0b2650c 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -23,6 +23,7 @@ from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -182,19 +183,25 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches ) project_options = {} + platform = None try: with fs.cd(kwargs["project_dir"]): project_options = device_helpers.get_project_options(kwargs["environment"]) - kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options) + device_helpers.register_filters( + ProjectConfig.get_instance().get("platformio", "monitor_dir"), + options=kwargs, + ) + if "platform" in project_options: + platform = PlatformFactory.new(project_options["platform"]) + device_helpers.register_filters( + os.path.join(platform.get_dir(), "monitor"), options=kwargs + ) + kwargs = device_helpers.apply_project_monitor_options( + kwargs, project_options + ) except NotPlatformIOProjectError: pass - platform = None - if "platform" in project_options: - with fs.cd(kwargs["project_dir"]): - platform = PlatformFactory.new(project_options["platform"]) - device_helpers.register_platform_filters(platform, options=kwargs) - if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py index a65b4895db..8b4b789c1f 100644 --- a/platformio/commands/device/helpers.py +++ b/platformio/commands/device/helpers.py @@ -92,11 +92,9 @@ def load_monitor_filter(path, options=None): return True -def register_platform_filters(platform, options=None): - monitor_dir = os.path.join(platform.get_dir(), "monitor") +def register_filters(monitor_dir, options=None): if not os.path.isdir(monitor_dir): return - for name in os.listdir(monitor_dir): if not name.startswith("filter_") or not name.endswith(".py"): continue diff --git a/platformio/project/options.py b/platformio/project/options.py index f94a74c6d2..c8a46da9ca 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -314,11 +314,19 @@ def validate_core_dir(path): ConfigPlatformioOption( group="directory", name="boards_dir", - description="A global storage for custom board manifests", + description="A storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("$PROJECT_DIR", "boards"), validate=validate_dir, ), + ConfigPlatformioOption( + group="directory", + name="monitor_dir", + description="A storage for custom monitor filters", + sysenvvar="PLATFORMIO_MONITOR_DIR", + default=os.path.join("$PROJECT_DIR", "monitor"), + validate=validate_dir, + ), ConfigPlatformioOption( group="directory", name="shared_dir", From 6a70ab74bc813d67655dc3060917ccac9b26ecf0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 15:24:24 +0300 Subject: [PATCH 15/46] Update history --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d0dd63ad59..4b172830c2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,8 +15,9 @@ PlatformIO Core 5 - Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) -- Fixed an issue when the "$PROJECT_DIR" variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) +- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) - Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) +- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) 5.2.2 (2021-10-20) From acb6cbffa043558622434cd069abdf11bc6e016f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 15:54:06 +0300 Subject: [PATCH 16/46] Add "arduplot" to the "Community Filters" // Resolve #4058 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7e751e6990..63ab8589a7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7e751e6990958ca6b8120f37b2405a41f0b5b72d +Subproject commit 63ab8589a7edd9aa82b71fecb7270e2ed53d7652 From 9aaa80a213ed1678d06e7aa888d1dd508dd1d9ae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 18:36:10 +0300 Subject: [PATCH 17/46] Cast Python warnings to errors when running "pytest" --- platformio/package/manifest/schema.py | 2 +- pytest.ini | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 pytest.ini diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index d17e6a208e..416dccfddc 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -89,7 +89,7 @@ def _deserialize( # pylint: disable=arguments-differ class AuthorSchema(StrictSchema): name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) - maintainer = fields.Bool(default=False) + maintainer = fields.Bool(dump_default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..03c8658036 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +filterwarnings = + error + # Marshmallow + ignore:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning \ No newline at end of file From 3fbb4cde36e3030a586dcab5d2e856eac32cf2d1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 18:45:04 +0300 Subject: [PATCH 18/46] Bump version to 5.2.3b2 --- platformio/__init__.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b2e0a694a4..27cc87e131 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3b1") +VERSION = (5, 2, "3b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/tox.ini b/tox.ini index 67323226f2..5d38aa332f 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ commands = [testenv:testcore] commands = {envpython} --version - py.test -v --basetemp="{envtmpdir}" -k-skip_ci tests --ignore tests/test_examples.py + py.test -v --basetemp="{envtmpdir}" -k "not skip_ci" tests --ignore tests/test_examples.py [testenv:testexamples] commands = From 67506511c3a142110f6f61dc53de823bd4418fa4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 19:45:47 +0300 Subject: [PATCH 19/46] Update token for docs/deploy --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1a80dbf60d..f11b000608 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -103,7 +103,7 @@ jobs: - name: Deploy to Github Pages uses: peaceiris/actions-gh-pages@v3 with: - personal_token: ${{ secrets.PERSONAL_TOKEN }} + personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }} external_repository: ${{ env.DOCS_REPO }} publish_dir: ./${{ env.DOCS_DIR }} commit_message: Sync Docs From 1dd0635e5e2785bfab5788c7e478f12ee221ca23 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 20:25:23 +0300 Subject: [PATCH 20/46] Use secured bitly --- docs | 2 +- platformio/app.py | 2 +- platformio/builder/tools/piolib.py | 2 +- platformio/commands/device/command.py | 2 +- platformio/debug/process/gdb.py | 4 ++-- platformio/package/manager/_download.py | 2 +- platformio/package/unpack.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs b/docs index 63ab8589a7..c07845a1c8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 63ab8589a7edd9aa82b71fecb7270e2ed53d7652 +Subproject commit c07845a1c810c318551ebe42af6395c3eed9d0bd diff --git a/platformio/app.py b/platformio/app.py index 845d7e81e8..d6930a4768 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -64,7 +64,7 @@ def projects_dir_validate(projects_dir): "value": True, }, "enable_telemetry": { - "description": ("Telemetry service (Yes/No)"), + "description": ("Telemetry service (Yes/No)"), "value": True, }, "force_verbose": { diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index bdebf6ab97..588238b86e 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1095,7 +1095,7 @@ def _print_deps_tree(root, level=0): project = ProjectAsLibBuilder(env, "$PROJECT_DIR") ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member - click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") + click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") click.echo( "LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode) diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 1cf0b2650c..c3941cb834 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -238,7 +238,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches "--- Available filters and text transformations: %s" % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) ) - click.echo("--- More details at http://bit.ly/pio-monitor-filters") + click.echo("--- More details at https://bit.ly/pio-monitor-filters") try: miniterm.main( default_port=kwargs["port"], diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 079b8b5304..ed8483bb3e 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -98,7 +98,7 @@ def generate_init_script(self, dst): ] banner = [ - "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", + "echo PlatformIO Unified Debugger -> https://bit.ly/pio-debug\\n", "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, "echo PlatformIO: Initializing remote target...\\n", ] @@ -157,7 +157,7 @@ def _auto_exec_continue(self): % self.debug_config.init_break ) self.console_log( - "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" + "PlatformIO: More configuration options -> https://bit.ly/pio-debug\n" ) if self.debug_config.platform.is_embedded(): self.transport.get_pipe_transport(0).write( diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 4039568b61..f48be79bb0 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -76,7 +76,7 @@ def download(self, url, checksum=None, silent=False): raise_error = True if raise_error: self.print_message( - "Error: Please read http://bit.ly/package-manager-ioerror", + "Error: Please read https://bit.ly/package-manager-ioerror", fg="red", err=True, ) diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index 6bbbef634e..f9e68ff8b8 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -27,7 +27,7 @@ class ExtractArchiveItemError(PackageException): MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " - "tool or check this solution -> http://bit.ly/faq-package-manager" + "tool or check this solution -> https://bit.ly/faq-package-manager" ) From d7b7d2de6e76680dd5bee6301113fcac499e7e6f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 21:12:29 +0300 Subject: [PATCH 21/46] Pass system STDIN stream to SCons subprocess --- platformio/platform/_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 731901677a..0964268998 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -137,12 +137,14 @@ def _write_and_flush(stream, data): line_callback=self._on_stderr_line, data_callback=lambda data: _write_and_flush(sys.stderr, data), ), + stdin=sys.stdin, ) return proc.exec_command( args, stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + stdin=sys.stdin, ) def _on_stdout_line(self, line): From c835ce780aea9ba361bd731a7ce82c02a2edae2a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Oct 2021 22:01:11 +0300 Subject: [PATCH 22/46] Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters // Resolve #3971 --- HISTORY.rst | 2 +- platformio/platform/_run.py | 2 ++ platformio/proc.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4b172830c2..07021307a7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,7 @@ PlatformIO Core 5 - Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) - Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) - Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) - +- Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 `_) 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 0964268998..00588c5a50 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -110,6 +110,8 @@ def _run_scons(self, variables, targets, jobs): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) proc.copy_pythonpath_to_osenv() + # force SCons output to Unicode + os.environ["PYTHONIOENCODING"] = "utf-8" if targets and "menuconfig" in targets: return proc.exec_command( diff --git a/platformio/proc.py b/platformio/proc.py index 0484cc42a0..f041d61cc6 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -30,7 +30,9 @@ class AsyncPipeBase(object): def __init__(self): self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace") + self._pipe_reader = os.fdopen( + self._fd_read, encoding="utf-8", errors="backslashreplace" + ) self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() From 256a9ee45d285ce0e0fa429b22bf8a39a998d3cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 13:54:49 +0300 Subject: [PATCH 23/46] Revert "Pass system STDIN stream to SCons subprocess" This reverts commit d7b7d2de6e76680dd5bee6301113fcac499e7e6f. --- platformio/platform/_run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 00588c5a50..bd9b687fd0 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -139,14 +139,12 @@ def _write_and_flush(stream, data): line_callback=self._on_stderr_line, data_callback=lambda data: _write_and_flush(sys.stderr, data), ), - stdin=sys.stdin, ) return proc.exec_command( args, stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - stdin=sys.stdin, ) def _on_stdout_line(self, line): From c0f2275b613b5e7bece2899028b9db61d2b58820 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 14:34:32 +0300 Subject: [PATCH 24/46] Restore ProjectConfig.get_optional_dir API, "platformio-node-helpers" depends on it --- platformio/project/config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 1ed6b4d30c..43b6c5644e 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -360,7 +360,16 @@ def validate(self, envs=None, silent=False): return True -class ProjectConfig(ProjectConfigBase): +class ProjectConfigDirsMixin(object): + def get_optional_dir(self, name): + """ + Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs + PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 + """ + return self.get("platformio", f"{name}_dir") + + +class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): _instances = {} From 6399de7a66a708abd1ccd9de03f4fec40473d8d0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 14:35:28 +0300 Subject: [PATCH 25/46] Removed deprecated project.helpers API --- platformio/project/helpers.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 100c1a75f2..9e995b7be1 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -48,30 +48,6 @@ def get_project_cache_dir(): return ProjectConfig.get_instance().get("platformio", "cache_dir") -def get_project_global_lib_dir(): - """ - Deprecated, use ProjectConfig.get("platformio", "globallib_dir") instead - "platformio-node-helpers" depends on it - """ - return ProjectConfig.get_instance().get("platformio", "globallib_dir") - - -def get_project_lib_dir(): - """ - Deprecated, use ProjectConfig.get("platformio", "lib_dir") instead - "platformio-node-helpers" depends on it - """ - return ProjectConfig.get_instance().get("platformio", "lib_dir") - - -def get_project_libdeps_dir(): - """ - Deprecated, use ProjectConfig.get("platformio", "libdeps_dir") instead - "platformio-node-helpers" depends on it - """ - return ProjectConfig.get_instance().get("platformio", "libdeps_dir") - - def get_default_projects_dir(): docs_dir = os.path.join(fs.expanduser("~"), "Documents") try: From 1174958e8b5cbe3a5d1ed6e3bc1ec16a7ae0dbb8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 14:36:18 +0300 Subject: [PATCH 26/46] Add project.helpers.get_project_all_lib_dirs API (used by platformio-node-helpers) --- platformio/package/manager/library.py | 6 +++-- platformio/project/helpers.py | 17 ++++++++++++++ platformio/project/options.py | 4 ++-- tests/test_projectconf.py | 34 +++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 3ce2a1fa4b..e67a511045 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -21,13 +21,15 @@ ) from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageItem, PackageSpec, PackageType -from platformio.project.helpers import get_project_global_lib_dir +from platformio.project.config import ProjectConfig class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): super(LibraryPackageManager, self).__init__( - PackageType.LIBRARY, package_dir or get_project_global_lib_dir() + PackageType.LIBRARY, + package_dir + or ProjectConfig.get_instance().get("platformio", "globallib_dir"), ) @property diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 9e995b7be1..4cf66470bc 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -43,6 +43,23 @@ def find_project_dir_above(path): return None +def get_project_all_lib_dirs(): + """Used by platformio-node-helpers.project.observer.fetchLibDirs""" + config = ProjectConfig.get_instance() + libdeps_dir = config.get("platformio", "libdeps_dir") + result = [ + config.get("platformio", "globallib_dir"), + config.get("platformio", "lib_dir"), + libdeps_dir, + ] + if not os.path.isdir(libdeps_dir): + return result + for d in os.listdir(libdeps_dir): + if os.path.isdir(os.path.join(libdeps_dir, d)): + result.append(os.path.join(libdeps_dir, d)) + return result + + def get_project_cache_dir(): """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead""" return ProjectConfig.get_instance().get("platformio", "cache_dir") diff --git a/platformio/project/options.py b/platformio/project/options.py index c8a46da9ca..5314da6eb3 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -115,9 +115,9 @@ def validate_dir(path): def validate_core_dir(path): - default = ProjectOptions["platformio.core_dir"].default + default_dir = ProjectOptions["platformio.core_dir"].default win_core_dir = None - if IS_WINDOWS and path == default: + if IS_WINDOWS and path == default_dir: win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" if os.path.isdir(win_core_dir): path = win_core_dir diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index b1859c5d86..2408463a47 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -15,6 +15,7 @@ # pylint: disable=redefined-outer-name import os +import sys import pytest @@ -521,3 +522,36 @@ def test_dump(tmpdir_factory): ], ), ] + + +@pytest.mark.skipif(sys.platform != "win32", reason="runs only on windows") +def test_win_core_root_dir(tmpdir_factory): + try: + win_core_root_dir = os.path.splitdrive(fs.expanduser("~"))[0] + "\\.platformio" + remove_dir_at_exit = False + if not os.path.isdir(win_core_root_dir): + remove_dir_at_exit = True + os.makedirs(win_core_root_dir) + + # Default config + config = ProjectConfig() + assert config.get("platformio", "core_dir") == win_core_root_dir + + # Override in config + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +core_dir = ~/.pio + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.get("platformio", "core_dir") != win_core_root_dir + assert config.get("platformio", "core_dir") == os.path.realpath( + fs.expanduser("~/.pio") + ) + + if remove_dir_at_exit: + fs.rmtree(win_core_root_dir) + except PermissionError: + pass From 8c8a94fc7112a38cf97d06e4f037c01e2cf43adc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 15:41:41 +0300 Subject: [PATCH 27/46] Run config option validation even in raw mode --- platformio/project/config.py | 6 +++--- platformio/project/options.py | 3 +++ tests/test_projectconf.py | 12 ++++-------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 43b6c5644e..d930dfa745 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -279,6 +279,9 @@ def getraw( # pylint: disable=too-many-branches if value == MISSING: return None + if option_meta.validate: + value = option_meta.validate(value) + return self._expand_interpolations(value) def _expand_interpolations(self, value): @@ -315,11 +318,8 @@ def get(self, section, option, default=MISSING): if not option_meta: return value - if option_meta.validate: - value = option_meta.validate(value) if option_meta.multiple: value = self.parse_multi_values(value or []) - try: return self.cast_to(value, option_meta.type) except click.BadParameter as e: diff --git a/platformio/project/options.py b/platformio/project/options.py index 5314da6eb3..d47cf46122 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -107,6 +107,9 @@ def expand_dir_templates(path): def validate_dir(path): if not path: return path + # if not all values expanded, ignore validation + if "${" in path and "}" in path: + return path if path.startswith("~"): path = fs.expanduser(path) if "$" in path: diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 2408463a47..275ac95b18 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -466,14 +466,7 @@ def test_dump(tmpdir_factory): ( "platformio", [ - ( - "build_dir", - "%s-%s" - % ( - os.path.realpath(fs.expanduser("~/tmp/pio")), - calculate_path_hash(os.getcwd()), - ), - ), + ("build_dir", "~/tmp/pio-$PROJECT_HASH"), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ("default_envs", ["base", "extra_2"]), ], @@ -536,6 +529,9 @@ def test_win_core_root_dir(tmpdir_factory): # Default config config = ProjectConfig() assert config.get("platformio", "core_dir") == win_core_root_dir + assert config.get("platformio", "packages_dir") == os.path.join( + win_core_root_dir, "packages" + ) # Override in config tmpdir = tmpdir_factory.mktemp("project") From 751c82fd299f3a3d90b6454656e97a23b8735ece Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 15:42:05 +0300 Subject: [PATCH 28/46] Bump version to 5.2.3b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 27cc87e131..622304942a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3b2") +VERSION = (5, 2, "3b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From de4b02eaf18c4e54f29f01b673ad9a460766b0d2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 15:52:16 +0300 Subject: [PATCH 29/46] Remove unused module --- tests/test_projectconf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 275ac95b18..da9b773289 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -22,7 +22,6 @@ from platformio import fs from platformio.project.config import ConfigParser, ProjectConfig from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError -from platformio.project.options import calculate_path_hash BASE_CONFIG = """ [platformio] From 78a67b754e954a6f290602955e6364d5e3971dbd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Oct 2021 16:01:50 +0300 Subject: [PATCH 30/46] Docs: Extend a project configuration example with the common "[env]" section --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c07845a1c8..17136d7591 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c07845a1c810c318551ebe42af6395c3eed9d0bd +Subproject commit 17136d75913bc70865753ac1630f95008d314186 From dbc73f5086beaa63eba8d7bb3b0436553a83a5ec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Oct 2021 14:30:30 +0300 Subject: [PATCH 31/46] Use Rust-less "cryptography" dependency for PIO Remote --- platformio/package/manager/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 270ce8e61b..e1776ae3f2 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -216,7 +216,13 @@ def get_contrib_pysite_deps(): # twisted[tls], see setup.py for %twisted_version% result.extend( - ["pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3"] + [ + # pyopenssl depends on it, use RUST-less version + "cryptography >= 3.3, < 35.0.0", + "pyopenssl >= 16.0.0", + "service_identity >= 18.1.0", + "idna >= 0.6, != 2.3", + ] ) if "windows" in sys_type: From a0e9f6a92df63d8aa9edba50b2fabf2394084742 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Nov 2021 15:57:17 +0200 Subject: [PATCH 32/46] Docs: Sync dev-platforms --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 17136d7591..0a5201b0c7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 17136d75913bc70865753ac1630f95008d314186 +Subproject commit 0a5201b0c72e5e8ca39d1c35bdecf0c777487b9c From 2a2f7825cc5d278c09d5828b8ea2f490432ef149 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Nov 2021 16:21:47 +0200 Subject: [PATCH 33/46] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0a5201b0c7..96396eaf75 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0a5201b0c72e5e8ca39d1c35bdecf0c777487b9c +Subproject commit 96396eaf75431af71e7cf7fcdb543e5994ca0463 From ceb3a19b8166700d24a717e45c557a5bd1aa0250 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Nov 2021 20:05:40 +0200 Subject: [PATCH 34/46] Automatically synchronize active projects between IDE and `PlatformIO Home` --- HISTORY.rst | 117 ++++++++++--------- platformio/__init__.py | 2 +- platformio/commands/home/rpc/handlers/ide.py | 42 ++++--- platformio/commands/home/rpc/server.py | 2 +- platformio/commands/home/run.py | 14 +-- 5 files changed, 91 insertions(+), 86 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 07021307a7..9a5a7b9643 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,33 +11,34 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ -- Added support for custom `device monitor filters `__ (`issue #3924 `_) -- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) -- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) +- Automatically synchronize active projects between IDE and `PlatformIO Home `__ +- Added support for custom `device monitor filters `__ (`issue #3924 `_) +- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) +- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) -- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) -- Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) -- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) +- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) +- Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) +- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) - Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 `_) 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ -- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug `__ command +- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug `__ command - Added support for CLion IDE 2021.3 (`pull #4085 `_) - Removed debugging "legacy Click" message from CLI (`issue #4083 `_) -- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" `__ with the multi-line default value (`issue #4082 `_) +- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" `__ with the multi-line default value (`issue #4082 `_) 5.2.1 (2021-10-11) ~~~~~~~~~~~~~~~~~~ - Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 `_) -- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json `__ manifest (`issue #3957 `_) -- Updated `Cppcheck `__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements +- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json `__ manifest (`issue #3957 `_) +- Updated `Cppcheck `__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements - Handle the "test" folder as a part of CLion project (`issue #4005 `_) - Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 `_) - Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 `_) -- Fixed a "FileExist" error when the `platformio ci `__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 `_) +- Fixed a "FileExist" error when the `platformio ci `__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 `_) - Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 `_) 5.2.0 (2021-09-13) @@ -45,11 +46,11 @@ PlatformIO Core 5 * **PlatformIO Debugging** - - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - - `Debug unit tests `__ created with `PlatformIO Unit Testing `__ solution (`issue #948 `_) + - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - `Debug unit tests `__ created with `PlatformIO Unit Testing `__ solution (`issue #948 `_) - Debug native (desktop) applications on a host machine (`issue #980 `_) - - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option + - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) + - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) * **Package Management** @@ -61,7 +62,7 @@ PlatformIO Core 5 * Check for duplicates and used version * Validate package manifest - - Added a new option ``--non-interactive`` to `pio package publish `__ command + - Added a new option ``--non-interactive`` to `pio package publish `__ command * **Build System** @@ -74,9 +75,9 @@ PlatformIO Core 5 - Updated analysis tools: - * `Clang-Tidy `__ v12.0.1 with new modules and extended checks list - * `Cppcheck `__ v2.5.0 with improved code analysis and MISRA improvements - * `PVS-Studio `__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics + * `Clang-Tidy `__ v12.0.1 with new modules and extended checks list + * `Cppcheck `__ v2.5.0 with improved code analysis and MISRA improvements + * `PVS-Studio `__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics * **Miscellaneous** @@ -88,7 +89,7 @@ PlatformIO Core 5 ~~~~~~~~~~~~~~~~~~ * Fixed a "The command line is too long" issue with a linking process on Windows (`issue #3827 `_) -* Fixed an issue with `device monitor `__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 `_) +* Fixed an issue with `device monitor `__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 `_) * Fixed an issue with silent mode when unwanted data is printed to stdout (`issue #3837 `_) * Fixed an issue when code inspection fails with "Bad JSON" (`issue #3790 `_) * Fixed an issue with overriding user-specified debugging configuration information in VSCode (`issue #3824 `_) @@ -98,8 +99,8 @@ PlatformIO Core 5 * **PlatformIO Home** - - Boosted `PlatformIO Home `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) + - Boosted `PlatformIO Home `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) * **Build System** @@ -109,28 +110,28 @@ PlatformIO Core 5 * **Package Management** - - New options for `pio system prune `__ command: + - New options for `pio system prune `__ command: + ``--dry-run`` option to show data that will be removed + ``--core-packages`` option to remove unnecessary core packages + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 `_) - - Added new `check_prune_system_threshold `__ setting + - Added new `check_prune_system_threshold `__ setting - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 `_) * **Debugging** - - Configure a custom debug adapter speed using a new `debug_speed `__ option (`issue #3799 `_) + - Configure a custom debug adapter speed using a new `debug_speed `__ option (`issue #3799 `_) - Handle debugging server's "ready_pattern" in "stderr" output * **Miscellaneous** - - Improved listing of `multicast DNS services `_ + - Improved listing of `multicast DNS services `_ - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) - - Fixed an issue when `pio device monitor --eol `__ and "send_on_enter" filter do not work properly (`issue #3787 `_) + - Fixed an issue when `pio device monitor --eol `__ and "send_on_enter" filter do not work properly (`issue #3787 `_) 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ @@ -139,8 +140,8 @@ PlatformIO Core 5 - Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations - Updated analysis tools: - * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules - * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism + * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules + * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism - Show a warning message about deprecated support for Python 2 and Python 3.5 - Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension @@ -150,12 +151,12 @@ PlatformIO Core 5 5.0.3 (2020-11-12) ~~~~~~~~~~~~~~~~~~ -- Added an error selector for `Sublime Text `__ build runner (`issue #3733 `_) +- Added an error selector for `Sublime Text `__ build runner (`issue #3733 `_) - Generate a working "projectEnvName" for PlatformIO IDE's debugger for VSCode - Force VSCode's intelliSenseMode to "gcc-x64" when GCC toolchain is used - Print ignored test suites and environments in the test summary report only in verbose mode (`issue #3726 `_) - Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 `_) -- Fixed an issue when `pio package pack `__ ignores some folders (`issue #3730 `_) +- Fixed an issue when `pio package pack `__ ignores some folders (`issue #3730 `_) 5.0.2 (2020-10-30) ~~~~~~~~~~~~~~~~~~ @@ -163,21 +164,21 @@ PlatformIO Core 5 - Initialize a new project or update the existing passing working environment name and its options (`issue #3686 `_) - Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) - Improved "core.call" RPC for PlatformIO Home (`issue #3671 `_) -- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps `__ option (`issue #3664 `_) +- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) - Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) -- Fixed an issue when `pio package publish `__ command removes original archive after submitting to the registry (`issue #3716 `_) -- Fixed an issue when multiple `pio lib install `__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 `_) -- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter `__ (`issue #3712 `_) +- Fixed an issue when `pio package publish `__ command removes original archive after submitting to the registry (`issue #3716 `_) +- Fixed an issue when multiple `pio lib install `__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 `_) +- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter `__ (`issue #3712 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ -- Added support for "owner" requirement when declaring ``dependencies`` using `library.json `__ -- Fixed an issue when using a custom git/ssh package with `platform_packages `__ option (`issue #3624 `_) -- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development `__ on RaspberryPi device (`issue #3652 `_) -- Fixed an issue when `pio package unpublish `__ command crashes (`issue #3660 `_) +- Added support for "owner" requirement when declaring ``dependencies`` using `library.json `__ +- Fixed an issue when using a custom git/ssh package with `platform_packages `__ option (`issue #3624 `_) +- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development `__ on RaspberryPi device (`issue #3652 `_) +- Fixed an issue when `pio package unpublish `__ command crashes (`issue #3660 `_) - Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 `_) - Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 `_) - Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 `_) @@ -186,7 +187,7 @@ PlatformIO Core 5 5.0.0 (2020-09-03) ~~~~~~~~~~~~~~~~~~ -Please check `Migration guide from 4.x to 5.0 `__. +Please check `Migration guide from 4.x to 5.0 `__. * Integration with the new **PlatformIO Trusted Registry** @@ -201,19 +202,19 @@ Please check `Migration guide from 4.x to 5.0 `__ – manage packages in the registry - * `pio access `__ – manage package access for users, teams, and maintainers + * `pio package `__ – manage packages in the registry + * `pio access `__ – manage package access for users, teams, and maintainers * Integration with the new **Account Management System** - - `Manage organizations `__ - - `Manage teams and team memberships `__ + - `Manage organizations `__ + - `Manage teams and team memberships `__ * New **Package Management System** - Integrated PlatformIO Core with the new PlatformIO Registry - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 `_) - - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) - Follow SemVer complaint version constraints when checking library updates `issue #1281 `_) - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) @@ -221,29 +222,29 @@ Please check `Migration guide from 4.x to 5.0 `__ - * `Configuration files are Python scripts `__ – use the power of a real programming language to solve build problems + * `Configuration files are Python scripts `__ – use the power of a real programming language to solve build problems * Built-in reliable and automatic dependency analysis * Improved support for parallel builds - * Ability to `share built files in a cache `__ to speed up multiple builds + * Ability to `share built files in a cache `__ to speed up multiple builds - - New `Custom Targets `__ + - New `Custom Targets `__ * Pre/Post processing based on dependent sources (another target, source file, etc.) * Command launcher with own arguments - * Launch command with custom options declared in `"platformio.ini" `__ + * Launch command with custom options declared in `"platformio.ini" `__ * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) + * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) + - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) * **Project Management** - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using the existing environment - - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using the existing environment + - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command - Do not generate ".travis.yml" for a new project, let the user have a choice * **Unit Testing** @@ -256,16 +257,16 @@ Please check `Migration guide from 4.x to 5.0 `__ v2.1 with a new "soundy" analysis option and improved code parser - * `PVS-Studio `__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics + * `Cppcheck `__ v2.1 with a new "soundy" analysis option and improved code parser + * `PVS-Studio `__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) * **Miscellaneous** - - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) + - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - Show ignored project environments only in the verbose mode (`issue #3641 `_) - Do not escape compiler arguments in VSCode template on Windows - Drop support for Python 2 and 3.5. diff --git a/platformio/__init__.py b/platformio/__init__.py index 622304942a..aa7301fbb3 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -47,7 +47,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.4", + "contrib-piohome": "~3.4.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~4.40200.0", diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index 59a52df595..44286dd85f 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -21,28 +21,32 @@ class IDERPC: def __init__(self): - self._queue = {} + self._cmd_queue = [] + self._result_queue = {} - def send_command(self, sid, command, params): - if not self._queue.get(sid): + async def listen_commands(self): + self._cmd_queue.append(aio_get_running_loop().create_future()) + return await self._cmd_queue[-1] + + async def send_command(self, command, params=None): + if not self._cmd_queue: raise JSONRPC20DispatchException( code=4005, message="PIO Home IDE agent is not started" ) - while self._queue[sid]: - self._queue[sid].pop().set_result( - {"id": time.time(), "method": command, "params": params} + cmd_id = None + while self._cmd_queue: + cmd_id = f"ide-{command}-{time.time()}" + self._cmd_queue.pop().set_result( + { + "id": cmd_id, + "method": command, + "params": params, + } ) + if not cmd_id: + return + self._result_queue[cmd_id] = aio_get_running_loop().create_future() + return await self._result_queue[cmd_id] - async def listen_commands(self, sid=0): - if sid not in self._queue: - self._queue[sid] = [] - self._queue[sid].append(aio_get_running_loop().create_future()) - return await self._queue[sid][-1] - - def open_project(self, sid, project_dir): - return self.send_command(sid, "open_project", project_dir) - - def open_text_document(self, sid, path, line=None, column=None): - return self.send_command( - sid, "open_text_document", dict(path=path, line=line, column=column) - ) + def on_command_result(self, cmd_id, value): + self._result_queue[cmd_id].set_result(value) diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 8e0dd44f5e..c10fa17697 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -35,7 +35,7 @@ def __init__(self, shutdown_timeout=0): def __call__(self, *args, **kwargs): raise NotImplementedError - def addObjectHandler(self, handler, namespace): + def add_object_handler(self, handler, namespace): self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) def on_client_connect(self): diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index b923cd998b..64d820eacb 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -65,13 +65,13 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): raise PlatformioException("Invalid path to PIO Home Contrib") ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) - ws_rpc_factory.addObjectHandler(AccountRPC(), namespace="account") - ws_rpc_factory.addObjectHandler(AppRPC(), namespace="app") - ws_rpc_factory.addObjectHandler(IDERPC(), namespace="ide") - ws_rpc_factory.addObjectHandler(MiscRPC(), namespace="misc") - ws_rpc_factory.addObjectHandler(OSRPC(), namespace="os") - ws_rpc_factory.addObjectHandler(PIOCoreRPC(), namespace="core") - ws_rpc_factory.addObjectHandler(ProjectRPC(), namespace="project") + ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account") + ws_rpc_factory.add_object_handler(AppRPC(), namespace="app") + ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide") + ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc") + ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") + ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") + ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") path = urlparse(home_url).path routes = [ From abcc4c0a129661131eb5a01e15f4829483760a1d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Nov 2021 20:06:08 +0200 Subject: [PATCH 35/46] Bump version to 5.2.3b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index aa7301fbb3..a0d2366bff 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3b3") +VERSION = (5, 2, "3b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 9040bbb75af5da72880be254ee1356bb77b9293b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 12:56:39 +0200 Subject: [PATCH 36/46] Update deps --- docs | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 96396eaf75..5ed82cb1b2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 96396eaf75431af71e7cf7fcdb543e5994ca0463 +Subproject commit 5ed82cb1b21c4bca98bc19033f256172f6466244 diff --git a/setup.py b/setup.py index 0c05743738..44a67e6826 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ home_requirements = [ "aiofiles==0.7.*", "ajsonrpc==1.*", - "starlette==0.16.*", + "starlette==0.17.*", "uvicorn==0.15.*", "wsproto==1.0.*", ] From 4a9a4782439500e56d4f571dae9aee6631d760e6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 12:57:09 +0200 Subject: [PATCH 37/46] Refactor PIO Home IDE RPC --- platformio/commands/home/rpc/handlers/ide.py | 76 ++++++++++++++------ 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index 44286dd85f..ad3cf0631e 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -20,33 +20,63 @@ class IDERPC: + + COMMAND_TIMEOUT = 1.5 # in seconds + def __init__(self): - self._cmd_queue = [] - self._result_queue = {} + self._ide_queue = [] + self._cmd_queue = {} async def listen_commands(self): - self._cmd_queue.append(aio_get_running_loop().create_future()) - return await self._cmd_queue[-1] + f = aio_get_running_loop().create_future() + self._ide_queue.append(f) + self._process_commands() + return await f async def send_command(self, command, params=None): - if not self._cmd_queue: - raise JSONRPC20DispatchException( - code=4005, message="PIO Home IDE agent is not started" - ) - cmd_id = None - while self._cmd_queue: - cmd_id = f"ide-{command}-{time.time()}" - self._cmd_queue.pop().set_result( - { - "id": cmd_id, - "method": command, - "params": params, - } - ) - if not cmd_id: - return - self._result_queue[cmd_id] = aio_get_running_loop().create_future() - return await self._result_queue[cmd_id] + cmd_id = f"ide-{command}-{time.time()}" + self._cmd_queue[cmd_id] = { + "method": command, + "params": params, + "time": time.time(), + "future": aio_get_running_loop().create_future(), + } + self._process_commands() + # in case if IDE agent has not been started + aio_get_running_loop().call_later( + self.COMMAND_TIMEOUT + 0.1, self._process_commands + ) + return await self._cmd_queue[cmd_id]["future"] def on_command_result(self, cmd_id, value): - self._result_queue[cmd_id].set_result(value) + if cmd_id not in self._cmd_queue: + return + self._cmd_queue[cmd_id]["future"].set_result(value) + del self._cmd_queue[cmd_id] + + def _process_commands(self): + for cmd_id in list(self._cmd_queue): + cmd_data = self._cmd_queue[cmd_id] + if cmd_data["future"].done(): + del self._cmd_queue[cmd_id] + continue + + if ( + not self._ide_queue + and (time.time() - cmd_data["time"]) > self.COMMAND_TIMEOUT + ): + cmd_data["future"].set_exception( + JSONRPC20DispatchException( + code=4005, message="PIO Home IDE agent is not started" + ) + ) + continue + + while self._ide_queue: + self._ide_queue.pop().set_result( + { + "id": cmd_id, + "method": cmd_data["method"], + "params": cmd_data["params"], + } + ) From 00d298935a9129e7e6fdc5cc79dab12fc731458e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 12:58:12 +0200 Subject: [PATCH 38/46] Bump version to 5.2.3b5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a0d2366bff..ecc0279e9b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3b4") +VERSION = (5, 2, "3b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1d80da2559ce8c8a3781c2f7cc9fa37ed0b2cdfc Mon Sep 17 00:00:00 2001 From: Maximilian Gerhardt Date: Fri, 5 Nov 2021 13:16:36 +0100 Subject: [PATCH 39/46] Add "inc" as sign that it's the root of the library (#4093) * Add "inc" as sign that it's the root of the library * Add "inc" and "Inc" Co-authored-by: Ivan Kravets --- platformio/package/manager/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index e67a511045..c3519a20aa 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -62,7 +62,7 @@ def find_pkg_root(self, path, spec): @staticmethod def find_library_root(path): - root_dir_signs = set(["include", "Include", "src", "Src"]) + root_dir_signs = set(["include", "Include", "inc", "Inc", "src", "Src"]) root_file_signs = set( [ "conanfile.py", # Conan-based library From 973f77012f4ca9cdc07dab2a250cd898a26efb77 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 14:46:57 +0200 Subject: [PATCH 40/46] Fixed an issue when VSCode's debugger does not honor default environment // Resolve #4098 --- HISTORY.rst | 2 ++ .../commands/home/rpc/handlers/project.py | 2 +- platformio/commands/project.py | 27 ++---------------- platformio/ide/__init__.py | 13 --------- .../generator.py} | 28 ++++++++++++++++--- .../tpls/atom/.clang_complete.tpl | 0 .../tpls/atom/.gcc-flags.json.tpl | 0 .../{ide => project}/tpls/atom/.gitignore.tpl | 0 .../tpls/clion/.gitignore.tpl | 0 .../tpls/clion/CMakeLists.txt.tpl | 0 .../tpls/clion/CMakeListsPrivate.txt.tpl | 0 .../tpls/codeblocks/platformio.cbp.tpl | 0 .../tpls/eclipse/.cproject.tpl | 0 .../tpls/eclipse/.project.tpl | 0 .../.settings/PlatformIO Debugger.launch.tpl | 0 .../.settings/language.settings.xml.tpl | 0 .../.settings/org.eclipse.cdt.core.prefs.tpl | 0 .../{ide => project}/tpls/emacs/.ccls.tpl | 0 .../tpls/emacs/.gitignore.tpl | 0 .../netbeans/nbproject/configurations.xml.tpl | 0 .../nbproject/private/configurations.xml.tpl | 0 .../nbproject/private/launcher.properties.tpl | 0 .../nbproject/private/private.xml.tpl | 0 .../tpls/netbeans/nbproject/project.xml.tpl | 0 .../tpls/qtcreator/platformio.pro.tpl | 0 .../tpls/sublimetext/.ccls.tpl | 0 .../platformio.sublime-project.tpl | 0 .../{ide => project}/tpls/vim/.ccls.tpl | 0 .../{ide => project}/tpls/vim/.gitignore.tpl | 0 .../platformio.vcxproj.filters.tpl | 0 .../tpls/visualstudio/platformio.vcxproj.tpl | 0 .../tpls/vscode/.gitignore.tpl | 0 .../vscode/.vscode/c_cpp_properties.json.tpl | 0 .../tpls/vscode/.vscode/extensions.json.tpl | 0 .../tpls/vscode/.vscode/launch.json.tpl | 2 +- setup.py | 8 +++--- 36 files changed, 34 insertions(+), 48 deletions(-) delete mode 100644 platformio/ide/__init__.py rename platformio/{ide/projectgenerator.py => project/generator.py} (84%) rename platformio/{ide => project}/tpls/atom/.clang_complete.tpl (100%) rename platformio/{ide => project}/tpls/atom/.gcc-flags.json.tpl (100%) rename platformio/{ide => project}/tpls/atom/.gitignore.tpl (100%) rename platformio/{ide => project}/tpls/clion/.gitignore.tpl (100%) rename platformio/{ide => project}/tpls/clion/CMakeLists.txt.tpl (100%) rename platformio/{ide => project}/tpls/clion/CMakeListsPrivate.txt.tpl (100%) rename platformio/{ide => project}/tpls/codeblocks/platformio.cbp.tpl (100%) rename platformio/{ide => project}/tpls/eclipse/.cproject.tpl (100%) rename platformio/{ide => project}/tpls/eclipse/.project.tpl (100%) rename platformio/{ide => project}/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl (100%) rename platformio/{ide => project}/tpls/eclipse/.settings/language.settings.xml.tpl (100%) rename platformio/{ide => project}/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl (100%) rename platformio/{ide => project}/tpls/emacs/.ccls.tpl (100%) rename platformio/{ide => project}/tpls/emacs/.gitignore.tpl (100%) rename platformio/{ide => project}/tpls/netbeans/nbproject/configurations.xml.tpl (100%) rename platformio/{ide => project}/tpls/netbeans/nbproject/private/configurations.xml.tpl (100%) rename platformio/{ide => project}/tpls/netbeans/nbproject/private/launcher.properties.tpl (100%) rename platformio/{ide => project}/tpls/netbeans/nbproject/private/private.xml.tpl (100%) rename platformio/{ide => project}/tpls/netbeans/nbproject/project.xml.tpl (100%) rename platformio/{ide => project}/tpls/qtcreator/platformio.pro.tpl (100%) rename platformio/{ide => project}/tpls/sublimetext/.ccls.tpl (100%) rename platformio/{ide => project}/tpls/sublimetext/platformio.sublime-project.tpl (100%) rename platformio/{ide => project}/tpls/vim/.ccls.tpl (100%) rename platformio/{ide => project}/tpls/vim/.gitignore.tpl (100%) rename platformio/{ide => project}/tpls/visualstudio/platformio.vcxproj.filters.tpl (100%) rename platformio/{ide => project}/tpls/visualstudio/platformio.vcxproj.tpl (100%) rename platformio/{ide => project}/tpls/vscode/.gitignore.tpl (100%) rename platformio/{ide => project}/tpls/vscode/.vscode/c_cpp_properties.json.tpl (100%) rename platformio/{ide => project}/tpls/vscode/.vscode/extensions.json.tpl (100%) rename platformio/{ide => project}/tpls/vscode/.vscode/launch.json.tpl (98%) diff --git a/HISTORY.rst b/HISTORY.rst index 9a5a7b9643..0115ce672a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,10 +16,12 @@ PlatformIO Core 5 - Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) +- Added "inc" as sign that it's the root of the library (`issue #4093 `_) - Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) - Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) - Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) - Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 `_) +- Fixed an issue when VSCode's debugger does not honor default environment (`issue #4098 `_) 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 535bc9be98..1c033f514e 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -23,10 +23,10 @@ from platformio import exception, fs from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC -from platformio.ide.projectgenerator import ProjectGenerator from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError +from platformio.project.generator import ProjectGenerator from platformio.project.helpers import get_project_dir, is_platformio_project from platformio.project.options import get_config_options_schema diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c79979fd57..22b06ee210 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -22,11 +22,11 @@ from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install -from platformio.ide.projectgenerator import ProjectGenerator from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.generator import ProjectGenerator from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -191,10 +191,7 @@ def project_init( os.path.join(project_dir, "platformio.ini") ) config.validate() - pg = ProjectGenerator( - config, environment or get_best_envname(config, board), ide - ) - pg.generate() + ProjectGenerator(config, environment, ide, board).generate() if is_new_project: init_cvs_ignore(project_dir) @@ -441,23 +438,3 @@ def update_project_env(project_dir, environment, project_option): config.set(section, _name.strip(), _value.strip()) config.save() - - -def get_best_envname(config, board_ids=None): - envname = None - default_envs = config.default_envs() - if default_envs: - envname = default_envs[0] - if not board_ids: - return envname - - for env in config.envs(): - if not board_ids: - return env - if not envname: - envname = env - items = config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in board_ids: - return env - - return envname diff --git a/platformio/ide/__init__.py b/platformio/ide/__init__.py deleted file mode 100644 index b051490361..0000000000 --- a/platformio/ide/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/ide/projectgenerator.py b/platformio/project/generator.py similarity index 84% rename from platformio/ide/projectgenerator.py rename to platformio/project/generator.py index f611219623..adf54e104b 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/project/generator.py @@ -24,15 +24,34 @@ class ProjectGenerator(object): - def __init__(self, config, env_name, ide): + def __init__(self, config, env_name, ide, board_ids=None): self.config = config self.project_dir = os.path.dirname(config.path) - self.env_name = str(env_name) + self.original_env_name = env_name + self.env_name = str(env_name or self.get_best_envname(board_ids)) self.ide = str(ide) + def get_best_envname(self, board_ids=None): + envname = None + default_envs = self.config.default_envs() + if default_envs: + envname = default_envs[0] + if not board_ids: + return envname + + for env in self.config.envs(): + if not board_ids: + return env + if not envname: + envname = env + items = self.config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in board_ids: + return env + return envname + @staticmethod def get_supported_ides(): - tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls") + tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls") return sorted( [ d @@ -61,6 +80,7 @@ def _load_tplvars(self): "systype": util.get_systype(), "project_name": os.path.basename(self.project_dir), "project_dir": self.project_dir, + "original_env_name": self.original_env_name, "env_name": self.env_name, "user_home_dir": os.path.realpath(fs.expanduser("~")), "platformio_path": sys.argv[0] @@ -110,7 +130,7 @@ def get_src_files(self): def get_tpls(self): tpls = [] - tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): diff --git a/platformio/ide/tpls/atom/.clang_complete.tpl b/platformio/project/tpls/atom/.clang_complete.tpl similarity index 100% rename from platformio/ide/tpls/atom/.clang_complete.tpl rename to platformio/project/tpls/atom/.clang_complete.tpl diff --git a/platformio/ide/tpls/atom/.gcc-flags.json.tpl b/platformio/project/tpls/atom/.gcc-flags.json.tpl similarity index 100% rename from platformio/ide/tpls/atom/.gcc-flags.json.tpl rename to platformio/project/tpls/atom/.gcc-flags.json.tpl diff --git a/platformio/ide/tpls/atom/.gitignore.tpl b/platformio/project/tpls/atom/.gitignore.tpl similarity index 100% rename from platformio/ide/tpls/atom/.gitignore.tpl rename to platformio/project/tpls/atom/.gitignore.tpl diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/project/tpls/clion/.gitignore.tpl similarity index 100% rename from platformio/ide/tpls/clion/.gitignore.tpl rename to platformio/project/tpls/clion/.gitignore.tpl diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/project/tpls/clion/CMakeLists.txt.tpl similarity index 100% rename from platformio/ide/tpls/clion/CMakeLists.txt.tpl rename to platformio/project/tpls/clion/CMakeLists.txt.tpl diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl similarity index 100% rename from platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl rename to platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl diff --git a/platformio/ide/tpls/codeblocks/platformio.cbp.tpl b/platformio/project/tpls/codeblocks/platformio.cbp.tpl similarity index 100% rename from platformio/ide/tpls/codeblocks/platformio.cbp.tpl rename to platformio/project/tpls/codeblocks/platformio.cbp.tpl diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/project/tpls/eclipse/.cproject.tpl similarity index 100% rename from platformio/ide/tpls/eclipse/.cproject.tpl rename to platformio/project/tpls/eclipse/.cproject.tpl diff --git a/platformio/ide/tpls/eclipse/.project.tpl b/platformio/project/tpls/eclipse/.project.tpl similarity index 100% rename from platformio/ide/tpls/eclipse/.project.tpl rename to platformio/project/tpls/eclipse/.project.tpl diff --git a/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl similarity index 100% rename from platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl rename to platformio/project/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl similarity index 100% rename from platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl rename to platformio/project/tpls/eclipse/.settings/language.settings.xml.tpl diff --git a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl similarity index 100% rename from platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl rename to platformio/project/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/project/tpls/emacs/.ccls.tpl similarity index 100% rename from platformio/ide/tpls/emacs/.ccls.tpl rename to platformio/project/tpls/emacs/.ccls.tpl diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/project/tpls/emacs/.gitignore.tpl similarity index 100% rename from platformio/ide/tpls/emacs/.gitignore.tpl rename to platformio/project/tpls/emacs/.gitignore.tpl diff --git a/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl similarity index 100% rename from platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl rename to platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl diff --git a/platformio/ide/tpls/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl similarity index 100% rename from platformio/ide/tpls/netbeans/nbproject/private/configurations.xml.tpl rename to platformio/project/tpls/netbeans/nbproject/private/configurations.xml.tpl diff --git a/platformio/ide/tpls/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl similarity index 100% rename from platformio/ide/tpls/netbeans/nbproject/private/launcher.properties.tpl rename to platformio/project/tpls/netbeans/nbproject/private/launcher.properties.tpl diff --git a/platformio/ide/tpls/netbeans/nbproject/private/private.xml.tpl b/platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl similarity index 100% rename from platformio/ide/tpls/netbeans/nbproject/private/private.xml.tpl rename to platformio/project/tpls/netbeans/nbproject/private/private.xml.tpl diff --git a/platformio/ide/tpls/netbeans/nbproject/project.xml.tpl b/platformio/project/tpls/netbeans/nbproject/project.xml.tpl similarity index 100% rename from platformio/ide/tpls/netbeans/nbproject/project.xml.tpl rename to platformio/project/tpls/netbeans/nbproject/project.xml.tpl diff --git a/platformio/ide/tpls/qtcreator/platformio.pro.tpl b/platformio/project/tpls/qtcreator/platformio.pro.tpl similarity index 100% rename from platformio/ide/tpls/qtcreator/platformio.pro.tpl rename to platformio/project/tpls/qtcreator/platformio.pro.tpl diff --git a/platformio/ide/tpls/sublimetext/.ccls.tpl b/platformio/project/tpls/sublimetext/.ccls.tpl similarity index 100% rename from platformio/ide/tpls/sublimetext/.ccls.tpl rename to platformio/project/tpls/sublimetext/.ccls.tpl diff --git a/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/project/tpls/sublimetext/platformio.sublime-project.tpl similarity index 100% rename from platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl rename to platformio/project/tpls/sublimetext/platformio.sublime-project.tpl diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/project/tpls/vim/.ccls.tpl similarity index 100% rename from platformio/ide/tpls/vim/.ccls.tpl rename to platformio/project/tpls/vim/.ccls.tpl diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/project/tpls/vim/.gitignore.tpl similarity index 100% rename from platformio/ide/tpls/vim/.gitignore.tpl rename to platformio/project/tpls/vim/.gitignore.tpl diff --git a/platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl similarity index 100% rename from platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl rename to platformio/project/tpls/visualstudio/platformio.vcxproj.filters.tpl diff --git a/platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl b/platformio/project/tpls/visualstudio/platformio.vcxproj.tpl similarity index 100% rename from platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl rename to platformio/project/tpls/visualstudio/platformio.vcxproj.tpl diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/project/tpls/vscode/.gitignore.tpl similarity index 100% rename from platformio/ide/tpls/vscode/.gitignore.tpl rename to platformio/project/tpls/vscode/.gitignore.tpl diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl similarity index 100% rename from platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl rename to platformio/project/tpls/vscode/.vscode/c_cpp_properties.json.tpl diff --git a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/tpls/vscode/.vscode/extensions.json.tpl similarity index 100% rename from platformio/ide/tpls/vscode/.vscode/extensions.json.tpl rename to platformio/project/tpls/vscode/.vscode/extensions.json.tpl diff --git a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl b/platformio/project/tpls/vscode/.vscode/launch.json.tpl similarity index 98% rename from platformio/ide/tpls/vscode/.vscode/launch.json.tpl rename to platformio/project/tpls/vscode/.vscode/launch.json.tpl index eb65480809..1575657537 100644 --- a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl +++ b/platformio/project/tpls/vscode/.vscode/launch.json.tpl @@ -28,7 +28,7 @@ % debug["name"] = "PIO Debug" % debug["preLaunchTask"] = { % "type": "PlatformIO", -% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 else "Pre-Debug", +% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and original_env_name else "Pre-Debug", % } % noloading = predebug.copy() % noloading["name"] = "PIO Debug (without uploading)" diff --git a/setup.py b/setup.py index 44a67e6826..79bc7092e8 100644 --- a/setup.py +++ b/setup.py @@ -62,10 +62,10 @@ packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ - "ide/tpls/*/.*.tpl", - "ide/tpls/*/*.tpl", - "ide/tpls/*/*/*.tpl", - "ide/tpls/*/.*/*.tpl", + "project/tpls/*/.*.tpl", + "project/tpls/*/*.tpl", + "project/tpls/*/*/*.tpl", + "project/tpls/*/.*/*.tpl", ], "scripts": ["99-platformio-udev.rules"], }, From 411764854b16ea4b286ad8fae6b7e7c87f9e8bed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 16:28:49 +0200 Subject: [PATCH 41/46] Add support for custom device monitor filters located in package folders // Issue #3924 --- docs | 2 +- platformio/commands/device/command.py | 24 +++++--------------- platformio/commands/device/helpers.py | 32 +++++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs b/docs index 5ed82cb1b2..fe44638e2d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5ed82cb1b21c4bca98bc19033f256172f6466244 +Subproject commit fe44638e2d482424bebd9f66e4eddb2c03180f22 diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index c3941cb834..fb7877e4fc 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -23,7 +23,6 @@ from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -173,35 +172,22 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches - # load default monitor filters - filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters") - for name in os.listdir(filters_dir): - if not name.endswith(".py"): - continue - device_helpers.load_monitor_filter( - os.path.join(filters_dir, name), options=kwargs - ) - project_options = {} platform = None try: with fs.cd(kwargs["project_dir"]): project_options = device_helpers.get_project_options(kwargs["environment"]) - device_helpers.register_filters( - ProjectConfig.get_instance().get("platformio", "monitor_dir"), - options=kwargs, - ) - if "platform" in project_options: - platform = PlatformFactory.new(project_options["platform"]) - device_helpers.register_filters( - os.path.join(platform.get_dir(), "monitor"), options=kwargs - ) kwargs = device_helpers.apply_project_monitor_options( kwargs, project_options ) + if "platform" in project_options: + platform = PlatformFactory.new(project_options["platform"]) except NotPlatformIOProjectError: pass + with fs.cd(kwargs["project_dir"]): + device_helpers.register_filters(platform=platform, options=kwargs) + if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py index 8b4b789c1f..558d6a2ea9 100644 --- a/platformio/commands/device/helpers.py +++ b/platformio/commands/device/helpers.py @@ -20,6 +20,7 @@ from platformio import fs from platformio.commands.device import DeviceMonitorFilter from platformio.compat import get_object_members, load_python_module +from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -92,13 +93,40 @@ def load_monitor_filter(path, options=None): return True -def register_filters(monitor_dir, options=None): +def load_monitor_filters(monitor_dir, prefix=None, options=None): if not os.path.isdir(monitor_dir): return for name in os.listdir(monitor_dir): - if not name.startswith("filter_") or not name.endswith(".py"): + if (prefix and not name.startswith(prefix)) or not name.endswith(".py"): continue path = os.path.join(monitor_dir, name) if not os.path.isfile(path): continue load_monitor_filter(path, options) + + +def register_filters(platform=None, options=None): + # project filters + load_monitor_filters( + ProjectConfig.get_instance().get("platformio", "monitor_dir"), + prefix="filter_", + options=options, + ) + # platform filters + if platform: + load_monitor_filters( + os.path.join(platform.get_dir(), "monitor"), + prefix="filter_", + options=options, + ) + # load package filters + pm = ToolPackageManager() + for pkg in pm.get_installed(): + load_monitor_filters( + os.path.join(pkg.path, "monitor"), prefix="filter_", options=options + ) + # default filters + load_monitor_filters( + os.path.join(fs.get_source_dir(), "commands", "device", "filters"), + options=options, + ) From d6ad6f96e84ebe4f1d37ba787122afb3144305eb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 16:29:18 +0200 Subject: [PATCH 42/46] Bump version to 5.2.3rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index ecc0279e9b..7fa5d8ca6a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3b5") +VERSION = (5, 2, "3rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From a24ec8b07a41d1585bbabfc5130611b5e4d520df Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 16:57:44 +0200 Subject: [PATCH 43/46] Grammar fixes --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0115ce672a..f2405ac213 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,12 +16,12 @@ PlatformIO Core 5 - Show human-readable message when infinite recursion is detected while processing `Interpolation of Values `__ (`issue #3883 `_) - Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 `_) -- Added "inc" as sign that it's the root of the library (`issue #4093 `_) +- Added "inc" as a sign that it's the root of the library (`issue #4093 `_) - Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server `__ option (`issue #4086 `_) - Fixed an issue when `PIO Remote `__ device monitor crashes on the first keypress (`issue #3832 `_) - Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE `__ (`pull #4089 `_) - Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 `_) -- Fixed an issue when VSCode's debugger does not honor default environment (`issue #4098 `_) +- Fixed an issue when VSCode's debugger does not the honor default environment (`issue #4098 `_) 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ From f88a2de8a91c41ac3cae5a3dbb1dc38ed49299ad Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 17:05:30 +0200 Subject: [PATCH 44/46] Filter duplicated recent projects on Windows --- platformio/commands/home/rpc/handlers/app.py | 10 +++++++--- platformio/commands/home/rpc/handlers/ide.py | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index b567f06b9d..d69cdc2f79 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -64,9 +64,13 @@ def load_state(): storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"] # skip non-existing recent projects - storage["recentProjects"] = [ - p for p in storage.get("recentProjects", []) if is_platformio_project(p) - ] + storage["recentProjects"] = list( + set( + os.path.normpath(p) + for p in storage.get("recentProjects", []) + if is_platformio_project(p) + ) + ) state["storage"] = storage state.modified = False # skip saving extra fields diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index ad3cf0631e..703f75a6d5 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import time from ajsonrpc.core import JSONRPC20DispatchException @@ -51,6 +52,8 @@ async def send_command(self, command, params=None): def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: return + if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": + value = [os.path.normpath(p) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] From 7c040ed99f457d2152fe24fbf8113da96274a628 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 17:21:15 +0200 Subject: [PATCH 45/46] Normalize Windows path with Python's pathlib --- platformio/commands/home/rpc/handlers/app.py | 3 ++- platformio/commands/home/rpc/handlers/ide.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index d69cdc2f79..5ef4afa853 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import os +from pathlib import Path from platformio import __version__, app, fs, util from platformio.project.config import ProjectConfig @@ -66,7 +67,7 @@ def load_state(): # skip non-existing recent projects storage["recentProjects"] = list( set( - os.path.normpath(p) + str(Path(p).resolve()) for p in storage.get("recentProjects", []) if is_platformio_project(p) ) diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index 703f75a6d5..5ff9fe3321 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import time +from pathlib import Path from ajsonrpc.core import JSONRPC20DispatchException @@ -53,7 +53,7 @@ def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: return if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": - value = [os.path.normpath(p) for p in value] + value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] From 133fa1495b7e6536bfb291658cf56aeaaa514e12 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 17:31:23 +0200 Subject: [PATCH 46/46] Bump version to 5.2.3 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f2405ac213..ff5da7e2a0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.2.3 (2021-??-??) +5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ - Automatically synchronize active projects between IDE and `PlatformIO Home `__ diff --git a/docs b/docs index fe44638e2d..fed771ae8d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fe44638e2d482424bebd9f66e4eddb2c03180f22 +Subproject commit fed771ae8d8cb6acfe3b19c2e3afd411537ca6f5 diff --git a/platformio/__init__.py b/platformio/__init__.py index 7fa5d8ca6a..61793cfd51 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "3rc1") +VERSION = (5, 2, 3) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"