From b54a8b40a4784047aae5925f911128762984cddf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Mar 2021 17:42:11 +0200 Subject: [PATCH 001/120] Refactor Unified Debugger to native Python Asynchronous I/O stack // Resolve #3793 , Resolve #3595 --- .../commands/{debug/command.py => debug.py} | 38 ++-- platformio/commands/debug/process/base.py | 93 --------- platformio/{commands => }/debug/__init__.py | 0 platformio/{commands => }/debug/exception.py | 0 platformio/{commands => }/debug/helpers.py | 2 +- platformio/{commands => }/debug/initcfgs.py | 0 .../{commands => }/debug/process/__init__.py | 0 platformio/debug/process/base.py | 189 ++++++++++++++++++ .../{commands => }/debug/process/client.py | 142 ++++++------- .../{commands => }/debug/process/server.py | 89 +++------ platformio/platform/board.py | 5 +- 11 files changed, 306 insertions(+), 252 deletions(-) rename platformio/commands/{debug/command.py => debug.py} (86%) delete mode 100644 platformio/commands/debug/process/base.py rename platformio/{commands => }/debug/__init__.py (100%) rename platformio/{commands => }/debug/exception.py (100%) rename platformio/{commands => }/debug/helpers.py (99%) rename platformio/{commands => }/debug/initcfgs.py (100%) rename platformio/{commands => }/debug/process/__init__.py (100%) create mode 100644 platformio/debug/process/base.py rename platformio/{commands => }/debug/process/client.py (68%) rename platformio/{commands => }/debug/process/server.py (64%) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug.py similarity index 86% rename from platformio/commands/debug/command.py rename to platformio/commands/debug.py index 2ff969329e..85e441162b 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug.py @@ -12,20 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-arguments, too-many-statements -# pylint: disable=too-many-locals, too-many-branches +# pylint: disable=too-many-arguments, too-many-locals +# pylint: disable=too-many-branches, too-many-statements +import asyncio import os -import signal -from os.path import isfile import click from platformio import app, exception, fs, proc -from platformio.commands.debug import helpers -from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install -from platformio.package.manager.core import inject_contrib_pysite +from platformio.compat import WINDOWS +from platformio.debug import helpers +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.process.client import DebugClientProcess from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -131,7 +131,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro ide_data["prog_path"] ) or not helpers.has_debug_symbols(ide_data["prog_path"]) else: - rebuild_prog = not isfile(ide_data["prog_path"]) + rebuild_prog = not os.path.isfile(ide_data["prog_path"]) if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server @@ -157,19 +157,17 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro if load_mode == "modified": helpers.is_prog_obsolete(ide_data["prog_path"]) - if not isfile(ide_data["prog_path"]): + if not os.path.isfile(ide_data["prog_path"]): raise DebugInvalidOptionsError("Program/firmware is missed") - # run debugging client - inject_contrib_pysite() - - # pylint: disable=import-outside-toplevel - from platformio.commands.debug.process.client import GDBClient, reactor - - client = GDBClient(project_dir, __unprocessed, debug_options, env_options) - client.spawn(ide_data["gdb_path"], ide_data["prog_path"]) - - signal.signal(signal.SIGINT, lambda *args, **kwargs: None) - reactor.run() + loop = asyncio.ProactorEventLoop() if WINDOWS else asyncio.get_event_loop() + asyncio.set_event_loop(loop) + client = DebugClientProcess(project_dir, __unprocessed, debug_options, env_options) + coro = client.run(ide_data["gdb_path"], ide_data["prog_path"]) + loop.run_until_complete(coro) + if WINDOWS: + # an issue with asyncio executor and STIDIN, it cannot be closed gracefully + os._exit(0) # pylint: disable=protected-access + loop.close() return True diff --git a/platformio/commands/debug/process/base.py b/platformio/commands/debug/process/base.py deleted file mode 100644 index 67557f3d7b..0000000000 --- a/platformio/commands/debug/process/base.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import signal -import time - -import click -from twisted.internet import protocol # pylint: disable=import-error - -from platformio import fs -from platformio.compat import string_types -from platformio.proc import get_pythonexe_path -from platformio.project.helpers import get_project_core_dir - - -class BaseProcess(protocol.ProcessProtocol, object): - - STDOUT_CHUNK_SIZE = 2048 - LOG_FILE = None - - COMMON_PATTERNS = { - "PLATFORMIO_HOME_DIR": get_project_core_dir(), - "PLATFORMIO_CORE_DIR": get_project_core_dir(), - "PYTHONEXE": get_pythonexe_path(), - } - - def __init__(self): - self._last_activity = 0 - - def apply_patterns(self, source, patterns=None): - _patterns = self.COMMON_PATTERNS.copy() - _patterns.update(patterns or {}) - - for key, value in _patterns.items(): - if key.endswith(("_DIR", "_PATH")): - _patterns[key] = fs.to_unix_path(value) - - def _replace(text): - for key, value in _patterns.items(): - pattern = "$%s" % key - text = text.replace(pattern, value or "") - return text - - if isinstance(source, string_types): - source = _replace(source) - elif isinstance(source, (list, dict)): - items = enumerate(source) if isinstance(source, list) else source.items() - for key, value in items: - if isinstance(value, string_types): - source[key] = _replace(value) - elif isinstance(value, (list, dict)): - source[key] = self.apply_patterns(value, patterns) - - return source - - def onStdInData(self, data): - self._last_activity = time.time() - if self.LOG_FILE: - with open(self.LOG_FILE, "ab") as fp: - fp.write(data) - - def outReceived(self, data): - self._last_activity = time.time() - if self.LOG_FILE: - with open(self.LOG_FILE, "ab") as fp: - fp.write(data) - while data: - chunk = data[: self.STDOUT_CHUNK_SIZE] - click.echo(chunk, nl=False) - data = data[self.STDOUT_CHUNK_SIZE :] - - def errReceived(self, data): - self._last_activity = time.time() - if self.LOG_FILE: - with open(self.LOG_FILE, "ab") as fp: - fp.write(data) - click.echo(data, nl=False, err=True) - - def processEnded(self, _): - self._last_activity = time.time() - # Allow terminating via SIGINT/CTRL+C - signal.signal(signal.SIGINT, signal.default_int_handler) diff --git a/platformio/commands/debug/__init__.py b/platformio/debug/__init__.py similarity index 100% rename from platformio/commands/debug/__init__.py rename to platformio/debug/__init__.py diff --git a/platformio/commands/debug/exception.py b/platformio/debug/exception.py similarity index 100% rename from platformio/commands/debug/exception.py rename to platformio/debug/exception.py diff --git a/platformio/commands/debug/helpers.py b/platformio/debug/helpers.py similarity index 99% rename from platformio/commands/debug/helpers.py rename to platformio/debug/helpers.py index e2935b5a1d..72c3ff4b85 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -22,9 +22,9 @@ from platformio import fs, util from platformio.commands import PlatformioCLI -from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes +from platformio.debug.exception import DebugInvalidOptionsError from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions diff --git a/platformio/commands/debug/initcfgs.py b/platformio/debug/initcfgs.py similarity index 100% rename from platformio/commands/debug/initcfgs.py rename to platformio/debug/initcfgs.py diff --git a/platformio/commands/debug/process/__init__.py b/platformio/debug/process/__init__.py similarity index 100% rename from platformio/commands/debug/process/__init__.py rename to platformio/debug/process/__init__.py diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py new file mode 100644 index 0000000000..10bdc86e97 --- /dev/null +++ b/platformio/debug/process/base.py @@ -0,0 +1,189 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import signal +import subprocess +import sys +import time + +from platformio import fs +from platformio.compat import ( + create_task, + get_locale_encoding, + get_running_loop, + string_types, +) +from platformio.proc import get_pythonexe_path +from platformio.project.helpers import get_project_core_dir + + +class DebugSubprocessProtocol(asyncio.SubprocessProtocol): + def __init__(self, factory): + self.factory = factory + self._is_exited = False + + def connection_made(self, transport): + self.factory.connection_made(transport) + + def pipe_data_received(self, fd, data): + pipe_to_cb = [ + self.factory.stdin_data_received, + self.factory.stdout_data_received, + self.factory.stderr_data_received, + ] + pipe_to_cb[fd](data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.factory.process_exited() + self._is_exited = True + + +class DebugBaseProcess: + + STDOUT_CHUNK_SIZE = 2048 + LOG_FILE = None + + COMMON_PATTERNS = { + "PLATFORMIO_HOME_DIR": get_project_core_dir(), + "PLATFORMIO_CORE_DIR": get_project_core_dir(), + "PYTHONEXE": get_pythonexe_path(), + } + + def __init__(self): + self.transport = None + self._is_running = False + self._last_activity = 0 + self._exit_future = None + self._stdin_read_task = None + self._std_encoding = get_locale_encoding() + + async def spawn(self, *args, **kwargs): + wait_until_exit = False + if "wait_until_exit" in kwargs: + wait_until_exit = kwargs["wait_until_exit"] + del kwargs["wait_until_exit"] + for pipe in ("stdin", "stdout", "stderr"): + if pipe not in kwargs: + kwargs[pipe] = subprocess.PIPE + loop = get_running_loop() + await loop.subprocess_exec( + lambda: DebugSubprocessProtocol(self), *args, **kwargs + ) + if wait_until_exit: + self._exit_future = loop.create_future() + await self._exit_future + + def is_running(self): + return self._is_running + + def connection_made(self, transport): + self._is_running = True + self.transport = transport + + def connect_stdin_pipe(self): + self._stdin_read_task = create_task(self._read_stdin_pipe()) + + async def _read_stdin_pipe(self): + loop = get_running_loop() + try: + loop.add_reader( + sys.stdin.fileno(), + lambda: self.stdin_data_received(sys.stdin.buffer.readline()), + ) + except NotImplementedError: + while True: + self.stdin_data_received( + await loop.run_in_executor(None, sys.stdin.buffer.readline) + ) + + def stdin_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + + def stdout_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + while data: + chunk = data[: self.STDOUT_CHUNK_SIZE] + print(chunk.decode(self._std_encoding, "replace"), end="", flush=True) + data = data[self.STDOUT_CHUNK_SIZE :] + + def stderr_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + print( + data.decode(self._std_encoding, "replace"), + end="", + file=sys.stderr, + flush=True, + ) + + def process_exited(self): + self._is_running = False + self._last_activity = time.time() + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) + if self._stdin_read_task: + self._stdin_read_task.cancel() + self._stdin_read_task = None + if self._exit_future: + self._exit_future.set_result(True) + self._exit_future = None + + def apply_patterns(self, source, patterns=None): + _patterns = self.COMMON_PATTERNS.copy() + _patterns.update(patterns or {}) + + for key, value in _patterns.items(): + if key.endswith(("_DIR", "_PATH")): + _patterns[key] = fs.to_unix_path(value) + + def _replace(text): + for key, value in _patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, list) else source.items() + for key, value in items: + if isinstance(value, string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)): + source[key] = self.apply_patterns(value, patterns) + + return source + + def terminate(self): + if not self.is_running() or not self.transport: + return + try: + self.transport.kill() + self.transport.close() + except: # pylint: disable=bare-except + pass diff --git a/platformio/commands/debug/process/client.py b/platformio/debug/process/client.py similarity index 68% rename from platformio/commands/debug/process/client.py rename to platformio/debug/process/client.py index 45374727c1..a5765b394e 100644 --- a/platformio/commands/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -12,80 +12,75 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib import os import re import signal +import tempfile import time -from hashlib import sha1 -from os.path import basename, dirname, isdir, join, realpath, splitext -from tempfile import mkdtemp - -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import protocol # pylint: disable=import-error -from twisted.internet import reactor # pylint: disable=import-error -from twisted.internet import stdio # pylint: disable=import-error -from twisted.internet import task # pylint: disable=import-error from platformio import fs, proc, telemetry, util from platformio.cache import ContentCache -from platformio.commands.debug import helpers -from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.commands.debug.initcfgs import get_gdb_init_config -from platformio.commands.debug.process.base import BaseProcess -from platformio.commands.debug.process.server import DebugServer -from platformio.compat import hashlib_encode_data, is_bytes +from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes +from platformio.debug import helpers +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.initcfgs import get_gdb_init_config +from platformio.debug.process.base import DebugBaseProcess +from platformio.debug.process.server import DebugServerProcess from platformio.project.helpers import get_project_cache_dir -class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes +class DebugClientProcess( + DebugBaseProcess +): # pylint: disable=too-many-instance-attributes PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" def __init__(self, project_dir, args, debug_options, env_options): - super(GDBClient, self).__init__() + super(DebugClientProcess, self).__init__() self.project_dir = project_dir self.args = list(args) self.debug_options = debug_options self.env_options = env_options - self._debug_server = DebugServer(debug_options, env_options) + self._server_process = DebugServerProcess(debug_options, env_options) self._session_id = None - if not isdir(get_project_cache_dir()): + if not os.path.isdir(get_project_cache_dir()): os.makedirs(get_project_cache_dir()) - self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-") + self._gdbsrc_dir = tempfile.mkdtemp( + dir=get_project_cache_dir(), prefix=".piodebug-" + ) - self._target_is_run = False - self._auto_continue_timer = None + self._target_is_running = False self._errors_buffer = b"" - @defer.inlineCallbacks - def spawn(self, gdb_path, prog_path): + async def run(self, gdb_path, prog_path): session_hash = gdb_path + prog_path - self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() + self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() patterns = { "PROJECT_DIR": self.project_dir, "PROG_PATH": prog_path, - "PROG_DIR": dirname(prog_path), - "PROG_NAME": basename(splitext(prog_path)[0]), + "PROG_DIR": os.path.dirname(prog_path), + "PROG_NAME": os.path.basename(os.path.splitext(prog_path)[0]), "DEBUG_PORT": self.debug_options["port"], "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"], "INIT_BREAK": self.debug_options["init_break"] or "", "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), } - yield self._debug_server.spawn(patterns) + await self._server_process.run(patterns) if not patterns["DEBUG_PORT"]: - patterns["DEBUG_PORT"] = self._debug_server.get_debug_port() + patterns["DEBUG_PORT"] = self._server_process.get_debug_port() self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client args = [ - "piogdb", + gdb_path, "-q", "--directory", self._gdbsrc_dir, @@ -101,18 +96,16 @@ def spawn(self, gdb_path, prog_path): if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) args.append(patterns["PROG_PATH"]) - - transport = reactor.spawnProcess( - self, gdb_path, args, path=self.project_dir, env=os.environ - ) - defer.returnValue(transport) + await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) @staticmethod def _get_data_dir(gdb_path): if "msp430" in gdb_path: return None - gdb_data_dir = realpath(join(dirname(gdb_path), "..", "share", "gdb")) - return gdb_data_dir if isdir(gdb_data_dir) else None + gdb_data_dir = os.path.realpath( + os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") + ) + return gdb_data_dir if os.path.isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): # default GDB init commands depending on debug tool @@ -153,72 +146,57 @@ def generate_pioinit(self, dst_dir, patterns): footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] commands = banner + commands + footer - with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp: + with open(os.path.join(dst_dir, self.PIO_SRC_NAME), "w") as fp: fp.write("\n".join(self.apply_patterns(commands, patterns))) - def connectionMade(self): - self._lock_session(self.transport.pid) + def connection_made(self, transport): + super(DebugClientProcess, self).connection_made(transport) + self._lock_session(transport.get_pid()) + # Disable SIGINT and allow GDB's Ctrl+C interrupt + signal.signal(signal.SIGINT, lambda *args, **kwargs: None) + self.connect_stdin_pipe() - p = protocol.Protocol() - p.dataReceived = self.onStdInData - stdio.StandardIO(p) - - def onStdInData(self, data): - super(GDBClient, self).onStdInData(data) + def stdin_data_received(self, data): + super(DebugClientProcess, self).stdin_data_received(data) if b"-exec-run" in data: - if self._target_is_run: + if self._target_is_running: token, _ = data.split(b"-", 1) - self.outReceived(token + b"^running\n") + self.stdout_data_received(token + b"^running\n") return data = data.replace(b"-exec-run", b"-exec-continue") if b"-exec-continue" in data: - self._target_is_run = True + self._target_is_running = True if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) - self.transport.write(b"pio_reset_run_target\n") - self.transport.write(data) - - def processEnded(self, reason): # pylint: disable=unused-argument - self._unlock_session() - if self._gdbsrc_dir and isdir(self._gdbsrc_dir): - fs.rmtree(self._gdbsrc_dir) - if self._debug_server: - self._debug_server.terminate() + self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n") + self.transport.get_pipe_transport(0).write(data) - reactor.stop() - - def outReceived(self, data): - super(GDBClient, self).outReceived(data) + def stdout_data_received(self, data): + super(DebugClientProcess, self).stdout_data_received(data) self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: telemetry.send_event( "Debug", "Started", telemetry.dump_run_environment(self.env_options) ) - self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue) - self._auto_continue_timer.start(0.1) - - def errReceived(self, data): - super(GDBClient, self).errReceived(data) - self._handle_error(data) + self._auto_exec_continue() def console_log(self, msg): if helpers.is_gdbmi_mode(): msg = helpers.escape_gdbmi_stream("~", msg) - self.outReceived(msg if is_bytes(msg) else msg.encode()) + self.stdout_data_received(msg if is_bytes(msg) else msg.encode()) def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds if self._last_activity > (time.time() - auto_exec_delay): + get_running_loop().call_later(0.1, self._auto_exec_continue) return - if self._auto_continue_timer: - self._auto_continue_timer.stop() - self._auto_continue_timer = None - if not self.debug_options["init_break"] or self._target_is_run: + if not self.debug_options["init_break"] or self._target_is_running: return + self.console_log( "PlatformIO: Resume the execution to `debug_init_break = %s`\n" % self.debug_options["init_break"] @@ -226,10 +204,14 @@ def _auto_exec_continue(self): self.console_log( "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" ) - self.transport.write( + self.transport.get_pipe_transport(0).write( b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" ) - self._target_is_run = True + self._target_is_running = True + + def stderr_data_received(self, data): + super(DebugClientProcess, self).stderr_data_received(data) + self._handle_error(data) def _handle_error(self, data): self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes @@ -248,7 +230,15 @@ def _handle_error(self, data): last_erros, ) telemetry.send_exception("DebugInitError: %s" % err) - self.transport.loseConnection() + self.transport.close() + + def process_exited(self): + self._unlock_session() + if self._gdbsrc_dir and os.path.isdir(self._gdbsrc_dir): + fs.rmtree(self._gdbsrc_dir) + if self._server_process: + self._server_process.terminate() + super(DebugClientProcess, self).process_exited() def _kill_previous_session(self): assert self._session_id diff --git a/platformio/commands/debug/process/server.py b/platformio/debug/process/server.py similarity index 64% rename from platformio/commands/debug/process/server.py rename to platformio/debug/process/server.py index 7a302c9be6..8d97d4c40f 100644 --- a/platformio/commands/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -12,53 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import os import time -from os.path import isdir, isfile, join - -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import reactor # pylint: disable=import-error from platformio import fs, util -from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode -from platformio.commands.debug.process.base import BaseProcess +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode +from platformio.debug.process.base import DebugBaseProcess from platformio.proc import where_is_program -class DebugServer(BaseProcess): +class DebugServerProcess(DebugBaseProcess): def __init__(self, debug_options, env_options): - super(DebugServer, self).__init__() + super(DebugServerProcess, self).__init__() self.debug_options = debug_options self.env_options = env_options self._debug_port = ":3333" - self._transport = None - self._process_ended = False self._ready = False - @defer.inlineCallbacks - def spawn(self, patterns): # pylint: disable=too-many-branches + async def run(self, patterns): # pylint: disable=too-many-branches systype = util.get_systype() server = self.debug_options.get("server") if not server: - defer.returnValue(None) + return None server = self.apply_patterns(server, patterns) server_executable = server["executable"] if not server_executable: - defer.returnValue(None) + return None if server["cwd"]: - server_executable = join(server["cwd"], server_executable) + server_executable = os.path.join(server["cwd"], server_executable) if ( "windows" in systype and not server_executable.endswith(".exe") - and isfile(server_executable + ".exe") + and os.path.isfile(server_executable + ".exe") ): server_executable = server_executable + ".exe" - if not isfile(server_executable): + if not os.path.isfile(server_executable): server_executable = where_is_program(server_executable) - if not isfile(server_executable): + if not os.path.isfile(server_executable): raise DebugInvalidOptionsError( "\nCould not launch Debug Server '%s'. Please check that it " "is installed and is included in a system PATH\n\n" @@ -70,6 +64,7 @@ def spawn(self, patterns): # pylint: disable=too-many-branches openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] ) + openocd_pipe_allowed = False if openocd_pipe_allowed: args = [] if server["cwd"]: @@ -83,34 +78,31 @@ def spawn(self, patterns): # pylint: disable=too-many-branches ) self._debug_port = '| "%s" %s' % (server_executable, str_args) self._debug_port = fs.to_unix_path(self._debug_port) - defer.returnValue(self._debug_port) + return self._debug_port env = os.environ.copy() # prepend server "lib" folder to LD path if ( "windows" not in systype and server["cwd"] - and isdir(join(server["cwd"], "lib")) + and os.path.isdir(os.path.join(server["cwd"], "lib")) ): ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" - env[ld_key] = join(server["cwd"], "lib") + env[ld_key] = os.path.join(server["cwd"], "lib") if os.environ.get(ld_key): env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) # prepend BIN to PATH - if server["cwd"] and isdir(join(server["cwd"], "bin")): + if server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "bin")): env["PATH"] = "%s%s%s" % ( - join(server["cwd"], "bin"), + os.path.join(server["cwd"], "bin"), os.pathsep, os.environ.get("PATH", os.environ.get("Path", "")), ) - self._transport = reactor.spawnProcess( - self, - server_executable, - [server_executable] + server["arguments"], - path=server["cwd"], - env=env, + await self.spawn( + *([server_executable] + server["arguments"]), cwd=server["cwd"], env=env ) + if "mspdebug" in server_executable.lower(): self._debug_port = ":2000" elif "jlink" in server_executable.lower(): @@ -118,19 +110,18 @@ def spawn(self, patterns): # pylint: disable=too-many-branches elif "qemu" in server_executable.lower(): self._debug_port = ":1234" - yield self._wait_until_ready() + await self._wait_until_ready() - defer.returnValue(self._debug_port) + return self._debug_port - @defer.inlineCallbacks - def _wait_until_ready(self): + async def _wait_until_ready(self): ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") timeout = 60 if ready_pattern else 10 elapsed = 0 delay = 0.5 auto_ready_delay = 0.5 - while not self._ready and not self._process_ended and elapsed < timeout: - yield self.async_sleep(delay) + while not self._ready and self.is_running() and elapsed < timeout: + await asyncio.sleep(delay) if not ready_pattern: self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay @@ -143,33 +134,15 @@ def _check_ready_by_pattern(self, data): self._ready = ready_pattern.encode() in data return self._ready - @staticmethod - def async_sleep(secs): - d = defer.Deferred() - reactor.callLater(secs, d.callback, None) - return d - def get_debug_port(self): return self._debug_port - def outReceived(self, data): - super(DebugServer, self).outReceived( + def stdout_data_received(self, data): + super(DebugServerProcess, self).stdout_data_received( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) self._check_ready_by_pattern(data) - def errReceived(self, data): - super(DebugServer, self).errReceived(data) + def stderr_data_received(self, data): + super(DebugServerProcess, self).stderr_data_received(data) self._check_ready_by_pattern(data) - - def processEnded(self, reason): - self._process_ended = True - super(DebugServer, self).processEnded(reason) - - def terminate(self): - if self._process_ended or not self._transport: - return - try: - self._transport.signalProcess("KILL") - except: # pylint: disable=bare-except - pass diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 900892cdf3..34d9dc3478 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -15,11 +15,8 @@ import os from platformio import fs, telemetry, util -from platformio.commands.debug.exception import ( - DebugInvalidOptionsError, - DebugSupportError, -) from platformio.compat import PY2 +from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError from platformio.exception import UserSideException from platformio.platform.exception import InvalidBoardManifest From 779e02a05e60496545f66d572f94ea762702c9df Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Mar 2021 20:06:52 +0200 Subject: [PATCH 002/120] Use "connect_read_pipe" on Unix --- platformio/debug/process/base.py | 14 ++++++++------ platformio/debug/process/server.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index 10bdc86e97..f7b38dffc6 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -20,6 +20,7 @@ from platformio import fs from platformio.compat import ( + WINDOWS, create_task, get_locale_encoding, get_running_loop, @@ -102,16 +103,17 @@ def connect_stdin_pipe(self): async def _read_stdin_pipe(self): loop = get_running_loop() - try: - loop.add_reader( - sys.stdin.fileno(), - lambda: self.stdin_data_received(sys.stdin.buffer.readline()), - ) - except NotImplementedError: + if WINDOWS: while True: self.stdin_data_received( await loop.run_in_executor(None, sys.stdin.buffer.readline) ) + else: + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await loop.connect_read_pipe(lambda: protocol, sys.stdin) + while True: + self.stdin_data_received(await reader.readline()) def stdin_data_received(self, data): self._last_activity = time.time() diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 8d97d4c40f..5371733868 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -64,7 +64,7 @@ async def run(self, patterns): # pylint: disable=too-many-branches openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] ) - openocd_pipe_allowed = False + # openocd_pipe_allowed = False if openocd_pipe_allowed: args = [] if server["cwd"]: From 064fa6027de46ebf2d88a82e537e1dc494551de5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Mar 2021 20:07:26 +0200 Subject: [PATCH 003/120] Bump version to 5.2.0a1 --- HISTORY.rst | 8 ++++++++ platformio/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7a93f6301c..05fca2d889 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,14 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.2.0 (2021-??-??) +~~~~~~~~~~~~~~~~~~ + +* **PlatformIO Debugging** + + - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - Support debugging on Windows using Windows CMD (CLI) (`issue #3793 `_) + 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 56d356cbc8..be6578ed4d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 1, 1) +VERSION = (5, 2, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c0357daf0158eac073e3b421e2c8bd2e331e3c7f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Mar 2021 21:08:06 +0200 Subject: [PATCH 004/120] Remove Python 2 code --- HISTORY.rst | 2 +- platformio/__main__.py | 17 +- platformio/app.py | 4 +- platformio/builder/main.py | 37 +++-- platformio/builder/tools/pioide.py | 9 +- platformio/builder/tools/piomisc.py | 9 +- platformio/builder/tools/piosize.py | 4 +- platformio/commands/boards.py | 3 +- platformio/commands/check/command.py | 4 +- platformio/commands/check/tools/base.py | 9 +- platformio/commands/ci.py | 66 ++++---- platformio/commands/debug.py | 2 +- platformio/commands/device/command.py | 6 +- platformio/commands/home/command.py | 4 +- platformio/commands/home/rpc/handlers/ide.py | 4 +- platformio/commands/home/rpc/handlers/misc.py | 4 +- platformio/commands/home/rpc/server.py | 6 +- platformio/commands/home/run.py | 4 +- platformio/commands/lib/command.py | 14 +- platformio/commands/platform.py | 16 +- platformio/compat.py | 152 +++++------------- platformio/debug/process/base.py | 10 +- platformio/debug/process/client.py | 4 +- platformio/fs.py | 7 +- platformio/maintenance.py | 3 - platformio/proc.py | 8 +- platformio/project/config.py | 15 +- tox.ini | 2 +- 28 files changed, 175 insertions(+), 250 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 05fca2d889..d3f412fdd9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,7 +14,7 @@ PlatformIO Core 5 * **PlatformIO Debugging** - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - - Support debugging on Windows using Windows CMD (CLI) (`issue #3793 `_) + - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__main__.py b/platformio/__main__.py index 537b1d0a42..dbf2215b00 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=import-outside-toplevel + import os import sys from traceback import format_exc import click -from platformio import __version__, exception, maintenance, util +from platformio import __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import CYGWIN +from platformio.compat import CYGWIN, PY2, ensure_python3 try: import click_completion # pylint: disable=import-error @@ -60,16 +62,19 @@ def cli(ctx, force, caller, no_ansi): except: # pylint: disable=bare-except pass + from platformio import maintenance + maintenance.on_platformio_start(ctx, force, caller) @cli.resultcallback() @click.pass_context def process_result(ctx, result, *_, **__): + from platformio import maintenance + maintenance.on_platformio_end(ctx, result) -@util.memoized() def configure(): if CYGWIN: raise exception.CygwinEnvDetected() @@ -105,6 +110,7 @@ def main(argv=None): assert isinstance(argv, list) sys.argv = argv try: + ensure_python3(raise_exception=True) configure() cli() # pylint: disable=no-value-for-parameter except SystemExit as e: @@ -112,7 +118,10 @@ def main(argv=None): exit_code = int(e.code) except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): - maintenance.on_platformio_exception(e) + if not PY2: + from platformio import maintenance + + maintenance.on_platformio_exception(e) error_str = "Error: " if isinstance(e, exception.PlatformioException): error_str += str(e) diff --git a/platformio/app.py b/platformio/app.py index 04d02c39c0..9d64994c39 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -24,7 +24,7 @@ from os.path import dirname, isdir, isfile, join, realpath from platformio import __version__, exception, fs, proc -from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import get_default_projects_dir, get_project_core_dir @@ -115,7 +115,7 @@ def __exit__(self, type_, value, traceback): if self.modified: try: with open(self.path, "w") as fp: - fp.write(dump_json_to_unicode(self._storage)) + fp.write(json.dumps(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6a060dd134..208cb0e2ae 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import sys -from os import environ, makedirs -from os.path import isdir, join from time import time import click @@ -29,7 +29,6 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import compat, fs -from platformio.compat import dump_json_to_unicode from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -65,18 +64,18 @@ "pioide", "piosize", ], - toolpath=[join(fs.get_source_dir(), "builder", "tools")], + toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], variables=clivars, # Propagating External Environment - ENV=environ, + ENV=os.environ, UNIX_TIME=int(time()), - BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"), - BUILD_SRC_DIR=join("$BUILD_DIR", "src"), - BUILD_TEST_DIR=join("$BUILD_DIR", "test"), - COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"), + BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), + BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), + COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", - PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), IDE_EXTRA_DATA={}, ) @@ -124,7 +123,7 @@ BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), LIBSOURCE_DIRS=[ config.get_optional_dir("lib"), - join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), + os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), config.get_optional_dir("globallib"), ], ) @@ -142,8 +141,8 @@ ) if env.subst("$BUILD_CACHE_DIR"): - if not isdir(env.subst("$BUILD_CACHE_DIR")): - makedirs(env.subst("$BUILD_CACHE_DIR")) + if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): + os.makedirs(env.subst("$BUILD_CACHE_DIR")) env.CacheDir("$BUILD_CACHE_DIR") if int(ARGUMENTS.get("ISATTY", 0)): @@ -160,15 +159,17 @@ if "compiledb" in COMMAND_LINE_TARGETS: env.Tool("compilation_db") -if not isdir(env.subst("$BUILD_DIR")): - makedirs(env.subst("$BUILD_DIR")) +if not os.path.isdir(env.subst("$BUILD_DIR")): + os.makedirs(env.subst("$BUILD_DIR")) env.LoadProjectOptions() env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])) + os.path.join( + "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]) + ) ) for item in env.GetExtraScripts("pre"): @@ -225,9 +226,7 @@ projenv = env click.echo( "\n%s\n" - % dump_json_to_unicode( - projenv.DumpIDEData(env) # pylint: disable=undefined-variable - ) + % json.dumps(projenv.DumpIDEData(env)) # pylint: disable=undefined-variable ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index de7e0cc8be..332ec6ed64 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -14,13 +14,12 @@ from __future__ import absolute_import +import glob import os -from glob import glob import SCons.Defaults # pylint: disable=import-error import SCons.Subst # pylint: disable=import-error -from platformio.compat import glob_escape from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program @@ -49,7 +48,7 @@ def _dump_includes(env): for pkg in p.get_installed_packages(): if p.get_package_type(pkg.metadata.name) != "toolchain": continue - toolchain_dir = glob_escape(pkg.path) + toolchain_dir = glob.escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), @@ -57,7 +56,9 @@ def _dump_includes(env): os.path.join(toolchain_dir, "*", "include*"), ] for g in toolchain_incglobs: - includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) + includes["toolchain"].extend( + [os.path.realpath(inc) for inc in glob.glob(g)] + ) # include Unity framework if there are tests in project includes["unity"] = [] diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 799b192fbc..dbc39012c2 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -15,16 +15,17 @@ from __future__ import absolute_import import atexit +import glob import io import os import re import sys -from tempfile import mkstemp +import tempfile import click from platformio import fs, util -from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape +from platformio.compat import get_filesystem_encoding, get_locale_encoding from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command @@ -116,7 +117,7 @@ def process(self, contents): return out_file def _gcc_preprocess(self, contents, out_file): - tmp_path = mkstemp()[1] + tmp_path = tempfile.mkstemp()[1] self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( @@ -229,7 +230,7 @@ def append_prototypes(self, contents): def ConvertInoToCpp(env): - src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) + src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( os.path.join(src_dir, "*.pde") ) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 83b3a3f5cc..eec4c87c6f 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -16,6 +16,7 @@ from __future__ import absolute_import +import json import sys from os import environ, makedirs, remove from os.path import isdir, join, splitdrive @@ -23,7 +24,6 @@ from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile -from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command from platformio.util import get_systype @@ -242,7 +242,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument data["memory"]["files"].append(file_data) with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: - fp.write(dump_json_to_unicode(data)) + fp.write(json.dumps(data)) def exists(_): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 4170b32ff7..f93607d436 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -18,7 +18,6 @@ from tabulate import tabulate from platformio import fs -from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager @@ -83,4 +82,4 @@ def _print_boards_json(query, installed=False): if query.lower() not in search_data.lower(): continue result.append(board) - click.echo(dump_json_to_unicode(result)) + click.echo(json.dumps(result)) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 8f9a6dcae2..082373f346 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -15,6 +15,7 @@ # pylint: disable=too-many-arguments,too-many-locals,too-many-branches # pylint: disable=redefined-builtin,too-many-statements +import json import os from collections import Counter from os.path import dirname, isfile @@ -26,7 +27,6 @@ from platformio import app, exception, fs, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools import CheckToolFactory -from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_dir @@ -163,7 +163,7 @@ def cli( print_processing_footer(result) if json_output: - click.echo(dump_json_to_unicode(results_to_json(results))) + click.echo(json.dumps(results_to_json(results))) elif not silent: print_check_summary(results) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 7eda69361c..da38c97e94 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import os -from tempfile import NamedTemporaryFile +import tempfile import click -from platformio import compat, fs, proc +from platformio import fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import load_project_ide_data @@ -104,7 +105,7 @@ def _extract_defines(language, includes_file): return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")} def _create_tmp_file(self, data): - with NamedTemporaryFile("w", delete=False) as fp: + with tempfile.NamedTemporaryFile("w", delete=False) as fp: fp.write(data) self._tmp_files.append(fp.name) return fp.name @@ -207,7 +208,7 @@ def _add_file(path): result["c++"].append(os.path.realpath(path)) for pattern in patterns: - for item in compat.glob_recursive(pattern): + for item in glob.glob(pattern, recursive=True): if not os.path.isdir(item): _add_file(item) for root, _, files in os.walk(item, followlinks=True): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index e72ddf76fb..5bc5b38b1f 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import getenv, makedirs, remove -from os.path import basename, isdir, isfile, join, realpath -from shutil import copyfile, copytree -from tempfile import mkdtemp +import glob +import os +import shutil +import tempfile import click -from platformio import app, compat, fs +from platformio import app, fs from platformio.commands.project import project_init as cmd_project_init from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run @@ -33,8 +33,8 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument for i, p in enumerate(value): if p.startswith("~"): value[i] = fs.expanduser(p) - value[i] = realpath(value[i]) - if not compat.glob_recursive(value[i]): + value[i] = os.path.realpath(value[i]) + if not glob.glob(value[i], recursive=True): invalid_path = p break try: @@ -51,7 +51,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option( "--build-dir", - default=mkdtemp, + default=tempfile.mkdtemp, type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), ) @click.option("--keep-build-dir", is_flag=True) @@ -78,28 +78,28 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches verbose, ): - if not src and getenv("PLATFORMIO_CI_SRC"): - src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":")) + if not src and os.getenv("PLATFORMIO_CI_SRC"): + src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":")) if not src: raise click.BadParameter("Missing argument 'src'") try: app.set_session_var("force_option", True) - if not keep_build_dir and isdir(build_dir): + if not keep_build_dir and os.path.isdir(build_dir): fs.rmtree(build_dir) - if not isdir(build_dir): - makedirs(build_dir) + if not os.path.isdir(build_dir): + os.makedirs(build_dir) for dir_name, patterns in dict(lib=lib, src=src).items(): if not patterns: continue contents = [] for p in patterns: - contents += compat.glob_recursive(p) - _copy_contents(join(build_dir, dir_name), contents) + contents += glob.glob(p, recursive=True) + _copy_contents(os.path.join(build_dir, dir_name), contents) - if project_conf and isfile(project_conf): + if project_conf and os.path.isfile(project_conf): _copy_project_conf(build_dir, project_conf) elif not board: raise CIBuildEnvsEmpty() @@ -126,48 +126,50 @@ def _copy_contents(dst_dir, contents): items = {"dirs": set(), "files": set()} for path in contents: - if isdir(path): + if os.path.isdir(path): items["dirs"].add(path) - elif isfile(path): + elif os.path.isfile(path): items["files"].add(path) - dst_dir_name = basename(dst_dir) + dst_dir_name = os.path.basename(dst_dir) if dst_dir_name == "src" and len(items["dirs"]) == 1: - copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) + shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) else: - if not isdir(dst_dir): - makedirs(dst_dir) + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir) for d in items["dirs"]: - copytree(d, join(dst_dir, basename(d)), symlinks=True) + shutil.copytree( + d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True + ) if not items["files"]: return if dst_dir_name == "lib": - dst_dir = join(dst_dir, mkdtemp(dir=dst_dir)) + dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir)) for f in items["files"]: - dst_file = join(dst_dir, basename(f)) + dst_file = os.path.join(dst_dir, os.path.basename(f)) if f == dst_file: continue - copyfile(f, dst_file) + shutil.copyfile(f, dst_file) def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p)) + contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True) for path in contents: - path = realpath(path) - if isdir(path): + path = os.path.realpath(path) + if os.path.isdir(path): fs.rmtree(path) - elif isfile(path): - remove(path) + elif os.path.isfile(path): + os.remove(path) def _copy_project_conf(build_dir, project_conf): config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") - config.save(join(build_dir, "platformio.ini")) + config.save(os.path.join(build_dir, "platformio.ini")) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 85e441162b..11e25248d5 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -167,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro loop.run_until_complete(coro) if WINDOWS: # an issue with asyncio executor and STIDIN, it cannot be closed gracefully - os._exit(0) # pylint: disable=protected-access + proc.force_exit() loop.close() return True diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index fd385a46fa..1f386438e0 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import sys from fnmatch import fnmatch @@ -21,7 +22,6 @@ from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers -from platformio.compat import dump_json_to_unicode from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError @@ -52,9 +52,7 @@ def device_list( # pylint: disable=too-many-branches single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: - return click.echo( - dump_json_to_unicode(data[single_key] if single_key else data) - ) + return click.echo(json.dumps(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 2973bdd25b..81764a54e3 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -17,6 +17,7 @@ import click from platformio.commands.home.helpers import is_port_used +from platformio.commands.home.run import run_server from platformio.compat import ensure_python3 @@ -87,9 +88,6 @@ def cli(port, host, no_open, shutdown_timeout, session_id): click.launch(home_url) return - # pylint: disable=import-outside-toplevel - from platformio.commands.home.run import run_server - run_server( host=host, port=port, diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index bacf8391a0..59a52df595 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -16,7 +16,7 @@ from ajsonrpc.core import JSONRPC20DispatchException -from platformio.compat import get_running_loop +from platformio.compat import aio_get_running_loop class IDERPC: @@ -36,7 +36,7 @@ def send_command(self, sid, command, params): async def listen_commands(self, sid=0): if sid not in self._queue: self._queue[sid] = [] - self._queue[sid].append(get_running_loop().create_future()) + self._queue[sid].append(aio_get_running_loop().create_future()) return await self._queue[sid][-1] def open_project(self, sid, project_dir): diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index c16a6cc9db..7626456a7c 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -17,7 +17,7 @@ from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC -from platformio.compat import create_task +from platformio.compat import aio_create_task class MiscRPC: @@ -30,7 +30,7 @@ async def load_latest_tweets(self, data_url): cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): - create_task( + aio_create_task( self._preload_latest_tweets(data_url, cache_key, cache_valid) ) return cache_data["result"] diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 6aef10e32e..8e0dd44f5e 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -17,7 +17,7 @@ from ajsonrpc.manager import AsyncJSONRPCResponseManager from starlette.endpoints import WebSocketEndpoint -from platformio.compat import create_task, get_running_loop +from platformio.compat import aio_create_task, aio_get_running_loop from platformio.proc import force_exit @@ -63,7 +63,7 @@ def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") force_exit() - self.shutdown_timer = get_running_loop().call_later( + self.shutdown_timer = aio_get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server ) @@ -84,7 +84,7 @@ async def on_connect(self, websocket): self.factory.on_client_connect() # pylint: disable=no-member async def on_receive(self, websocket, data): - create_task(self._handle_rpc(websocket, data)) + aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): self.factory.on_client_disconnect() # pylint: disable=no-member diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index 4e22572029..b923cd998b 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -32,7 +32,7 @@ from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.commands.home.rpc.handlers.project import ProjectRPC from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory -from platformio.compat import get_running_loop +from platformio.compat import aio_get_running_loop from platformio.exception import PlatformioException from platformio.package.manager.core import get_core_package_dir from platformio.proc import force_exit @@ -49,7 +49,7 @@ async def __call__(self, scope, receive, send): async def shutdown_server(_=None): - get_running_loop().call_later(0.5, force_exit) + aio_get_running_loop().call_later(0.5, force_exit) return PlainTextResponse("Server has been shutdown!") diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index d49e6cb6b1..dac038164d 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -14,6 +14,7 @@ # pylint: disable=too-many-branches, too-many-locals +import json import os import time @@ -23,7 +24,6 @@ from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps -from platformio.compat import dump_json_to_unicode from platformio.package.exception import NotGlobalLibDir, UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec @@ -286,7 +286,7 @@ def lib_update( # pylint: disable=too-many-arguments if json_output: return click.echo( - dump_json_to_unicode( + json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @@ -315,7 +315,7 @@ def lib_list(ctx, json_output): if json_output: return click.echo( - dump_json_to_unicode( + json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @@ -359,7 +359,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): ) if json_output: - click.echo(dump_json_to_unicode(result)) + click.echo(json.dumps(result)) return if result["total"] == 0: @@ -418,7 +418,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): def lib_builtin(storage, json_output): items = get_builtin_libs(storage) if json_output: - return click.echo(dump_json_to_unicode(items)) + return click.echo(json.dumps(items)) for storage_ in items: if not storage_["items"]: @@ -442,7 +442,7 @@ def lib_show(library, json_output): regclient = lm.get_registry_client_instance() lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: - return click.echo(dump_json_to_unicode(lib)) + return click.echo(json.dumps(lib)) title = "{ownername}/{name}".format(**lib) click.secho(title, fg="cyan") @@ -538,7 +538,7 @@ def lib_stats(json_output): result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: - return click.echo(dump_json_to_unicode(result)) + return click.echo(json.dumps(result)) for key in ("updated", "added"): tabular_data = [ diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 054a7a1217..f03f88337c 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import click from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards -from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.package.version import get_original_version @@ -172,7 +172,7 @@ def platform_search(query, json_output): for platform in _get_registry_platforms(): if query == "all": query = "" - search_data = dump_json_to_unicode(platform) + search_data = json.dumps(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( @@ -182,7 +182,7 @@ def platform_search(query, json_output): ) if json_output: - click.echo(dump_json_to_unicode(platforms)) + click.echo(json.dumps(platforms)) else: _print_platforms(platforms) @@ -198,7 +198,7 @@ def platform_frameworks(query, json_output): ): if query == "all": query = "" - search_data = dump_json_to_unicode(framework) + search_data = json.dumps(framework) if query and query.lower() not in search_data.lower(): continue framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"] @@ -211,7 +211,7 @@ def platform_frameworks(query, json_output): frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) if json_output: - click.echo(dump_json_to_unicode(frameworks)) + click.echo(json.dumps(frameworks)) else: _print_platforms(frameworks) @@ -228,7 +228,7 @@ def platform_list(json_output): platforms = sorted(platforms, key=lambda manifest: manifest["name"]) if json_output: - click.echo(dump_json_to_unicode(platforms)) + click.echo(json.dumps(platforms)) else: _print_platforms(platforms) @@ -241,7 +241,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if not data: raise UnknownPlatform(platform) if json_output: - return click.echo(dump_json_to_unicode(data)) + return click.echo(json.dumps(data)) dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] click.echo( @@ -401,7 +401,7 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments str(outdated.latest) if outdated.latest else None ) result.append(data) - return click.echo(dump_json_to_unicode(result)) + return click.echo(json.dumps(result)) # cleanup cached board and platform lists cleanup_content_cache("http") diff --git a/platformio/compat.py b/platformio/compat.py index 53c1507cfb..95bbd5dfa3 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -12,23 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-import, no-name-in-module, import-error, -# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg +# pylint: disable=unused-import -import glob import inspect -import json import locale -import os -import re import sys from platformio.exception import UserSideException +if sys.version_info >= (3,): + if sys.version_info >= (3, 7): + from asyncio import create_task as aio_create_task + from asyncio import get_running_loop as aio_get_running_loop + else: + from asyncio import ensure_future as aio_create_task + from asyncio import get_event_loop as aio_get_running_loop + + PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") WINDOWS = sys.platform.startswith("win") MACOS = sys.platform.startswith("darwin") +string_types = (str,) + + +def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + +def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() + + +def load_python_module(name, pathname): + import importlib.util # pylint: disable=import-outside-toplevel + + spec = importlib.util.spec_from_file_location(name, pathname) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def get_filesystem_encoding(): @@ -53,14 +87,6 @@ def get_object_members(obj, ignore_private=True): } -def ci_strings_are_equal(a, b): - if a == b: - return True - if not a or not b: - return False - return a.strip().lower() == b.strip().lower() - - def ensure_python3(raise_exception=True): compatible = sys.version_info >= (3, 6) if not raise_exception or compatible: @@ -71,101 +97,3 @@ def ensure_python3(raise_exception=True): "https://docs.platformio.org/en/latest/core/migration.html" "#drop-support-for-python-2-and-3-5" ) - - -if PY2: - import imp - - string_types = (str, unicode) - - def create_task(coro, name=None): - raise NotImplementedError - - def get_running_loop(): - raise NotImplementedError - - def is_bytes(x): - return isinstance(x, (buffer, bytearray)) - - def path_to_unicode(path): - if isinstance(path, unicode): - return path - return path.decode(get_filesystem_encoding()) - - def hashlib_encode_data(data): - if is_bytes(data): - return data - if isinstance(data, unicode): - data = data.encode(get_filesystem_encoding()) - elif not isinstance(data, string_types): - data = str(data) - return data - - def dump_json_to_unicode(obj): - if isinstance(obj, unicode): - return obj - return json.dumps( - obj, encoding=get_filesystem_encoding(), ensure_ascii=False - ).encode("utf8") - - _magic_check = re.compile("([*?[])") - _magic_check_bytes = re.compile(b"([*?[])") - - def glob_recursive(pathname): - return glob.glob(pathname) - - def glob_escape(pathname): - """Escape all special characters.""" - # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be - # escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = _magic_check_bytes.sub(br"[\1]", pathname) - else: - pathname = _magic_check.sub(r"[\1]", pathname) - return drive + pathname - - def load_python_module(name, pathname): - return imp.load_source(name, pathname) - - -else: - import importlib.util - from glob import escape as glob_escape - - if sys.version_info >= (3, 7): - from asyncio import create_task, get_running_loop - else: - from asyncio import ensure_future as create_task - from asyncio import get_event_loop as get_running_loop - - string_types = (str,) - - def is_bytes(x): - return isinstance(x, (bytes, memoryview, bytearray)) - - def path_to_unicode(path): - return path - - def hashlib_encode_data(data): - if is_bytes(data): - return data - if not isinstance(data, string_types): - data = str(data) - return data.encode() - - def dump_json_to_unicode(obj): - if isinstance(obj, string_types): - return obj - return json.dumps(obj) - - def glob_recursive(pathname): - return glob.glob(pathname, recursive=True) - - def load_python_module(name, pathname): - spec = importlib.util.spec_from_file_location(name, pathname) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index f7b38dffc6..a97d544641 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -21,9 +21,9 @@ from platformio import fs from platformio.compat import ( WINDOWS, - create_task, + aio_create_task, + aio_get_running_loop, get_locale_encoding, - get_running_loop, string_types, ) from platformio.proc import get_pythonexe_path @@ -83,7 +83,7 @@ async def spawn(self, *args, **kwargs): for pipe in ("stdin", "stdout", "stderr"): if pipe not in kwargs: kwargs[pipe] = subprocess.PIPE - loop = get_running_loop() + loop = aio_get_running_loop() await loop.subprocess_exec( lambda: DebugSubprocessProtocol(self), *args, **kwargs ) @@ -99,10 +99,10 @@ def connection_made(self, transport): self.transport = transport def connect_stdin_pipe(self): - self._stdin_read_task = create_task(self._read_stdin_pipe()) + self._stdin_read_task = aio_create_task(self._read_stdin_pipe()) async def _read_stdin_pipe(self): - loop = get_running_loop() + loop = aio_get_running_loop() if WINDOWS: while True: self.stdin_data_received( diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py index a5765b394e..e8b05715f2 100644 --- a/platformio/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -21,7 +21,7 @@ from platformio import fs, proc, telemetry, util from platformio.cache import ContentCache -from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes +from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes from platformio.debug import helpers from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.initcfgs import get_gdb_init_config @@ -191,7 +191,7 @@ def console_log(self, msg): def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds if self._last_activity > (time.time() - auto_exec_delay): - get_running_loop().call_later(0.1, self._auto_exec_continue) + aio_get_running_loop().call_later(0.1, self._auto_exec_continue) return if not self.debug_options["init_break"] or self._target_is_running: diff --git a/platformio/fs.py b/platformio/fs.py index da2101c5fb..3aefdb6e67 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import hashlib import io import json @@ -24,7 +25,7 @@ import click from platformio import exception -from platformio.compat import WINDOWS, glob_escape, glob_recursive +from platformio.compat import WINDOWS class cd(object): @@ -158,7 +159,9 @@ def _append_build_item(items, item, src_dir): src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): items = set() - for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)): + for item in glob.glob( + os.path.join(glob.escape(src_dir), pattern), recursive=True + ): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=followlinks): for f in files: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index e22f8407d1..3ef47015f3 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -28,7 +28,6 @@ from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.system.prune import calculate_unnecessary_system_data from platformio.commands.upgrade import get_latest_version -from platformio.compat import ensure_python3 from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager @@ -40,8 +39,6 @@ def on_platformio_start(ctx, force, caller): - ensure_python3(raise_exception=True) - app.set_session_var("command_ctx", ctx) app.set_session_var("force_option", force) set_caller(caller) diff --git a/platformio/proc.py b/platformio/proc.py index 24640c3818..b4ef4505cc 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -24,7 +24,6 @@ WINDOWS, get_filesystem_encoding, get_locale_encoding, - get_running_loop, string_types, ) @@ -221,9 +220,4 @@ def append_env_path(name, value): def force_exit(code=0): - try: - get_running_loop().stop() - except: # pylint: disable=bare-except - pass - finally: - sys.exit(code) + os._exit(code) # pylint: disable=protected-access diff --git a/platformio/project/config.py b/platformio/project/config.py index 786f080aab..9f78a67b81 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -12,21 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob +import hashlib import json import os import re -from hashlib import sha1 import click from platformio import fs -from platformio.compat import ( - PY2, - WINDOWS, - glob_recursive, - hashlib_encode_data, - string_types, -) +from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -122,7 +117,7 @@ def read(self, path, parse_extra=True): for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) - for item in glob_recursive(pattern): + for item in glob.glob(pattern, recursive=True): self.read(item) def _maintain_renaimed_options(self): @@ -402,7 +397,7 @@ def get_optional_dir(self, name, exists=False): "%s-%s" % ( os.path.basename(project_dir), - sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], + hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], ), ) diff --git a/tox.ini b/tox.ini index c8c18db985..be83255558 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope passenv = * usedevelop = True deps = - black + py36,py37,py38,py39: black isort pylint pytest From 2745dbd124528ea6bb6071ffac5b542bbe13c8bb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Mar 2021 23:14:22 +0200 Subject: [PATCH 005/120] PyLint fix --- platformio/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/compat.py b/platformio/compat.py index 95bbd5dfa3..97cfb8d382 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-import +# pylint: disable=unused-import,no-name-in-module import inspect import locale From dbb9998f696ab69ecc5db194efe6fd1e76697977 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Mar 2021 23:42:54 +0200 Subject: [PATCH 006/120] Refactor debugging configuration, add support for server_ready_pattern // Resolve #3401 --- docs | 2 +- platformio/commands/debug.py | 90 +++++----- platformio/debug/config/__init__.py | 13 ++ platformio/debug/config/base.py | 227 ++++++++++++++++++++++++++ platformio/debug/config/blackmagic.py | 49 ++++++ platformio/debug/config/factory.py | 41 +++++ platformio/debug/config/generic.py | 38 +++++ platformio/debug/config/jlink.py | 48 ++++++ platformio/debug/config/mspdebug.py | 36 ++++ platformio/debug/config/qemu.py | 37 +++++ platformio/debug/config/renode.py | 45 +++++ platformio/debug/helpers.py | 128 +-------------- platformio/debug/initcfgs.py | 161 ------------------ platformio/debug/process/base.py | 36 ---- platformio/debug/process/client.py | 68 +++----- platformio/debug/process/server.py | 53 +++--- platformio/platform/base.py | 2 +- 17 files changed, 622 insertions(+), 452 deletions(-) create mode 100644 platformio/debug/config/__init__.py create mode 100644 platformio/debug/config/base.py create mode 100644 platformio/debug/config/blackmagic.py create mode 100644 platformio/debug/config/factory.py create mode 100644 platformio/debug/config/generic.py create mode 100644 platformio/debug/config/jlink.py create mode 100644 platformio/debug/config/mspdebug.py create mode 100644 platformio/debug/config/qemu.py create mode 100644 platformio/debug/config/renode.py delete mode 100644 platformio/debug/initcfgs.py diff --git a/docs b/docs index 3293903cac..0487c24f93 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3293903cac7c050908b594a838bd5a220e47e2c6 +Subproject commit 0487c24f930e3b1f45d594b6564148dc5324f263 diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 11e25248d5..b0c1ed93d5 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -17,6 +17,7 @@ import asyncio import os +import subprocess import click @@ -24,13 +25,14 @@ from platformio.commands.platform import platform_install as cmd_platform_install from platformio.compat import WINDOWS from platformio.debug import helpers +from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.client import DebugClientProcess from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError -from platformio.project.helpers import is_platformio_project, load_project_ide_data +from platformio.project.helpers import is_platformio_project @click.command( @@ -62,46 +64,40 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro app.set_session_var("custom_project_conf", project_conf) # use env variables from Eclipse or CLion - for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): + for name in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): if is_platformio_project(project_dir): break - if os.getenv(sysenv): - project_dir = os.getenv(sysenv) + if os.getenv(name): + project_dir = os.getenv(name) with fs.cd(project_dir): - config = ProjectConfig.get_instance(project_conf) - config.validate(envs=[environment] if environment else None) - - env_name = environment or helpers.get_default_debug_env(config) - env_options = config.items(env=env_name, as_dict=True) - if not set(env_options.keys()) >= set(["platform", "board"]): - raise ProjectEnvsNotAvailableError() - - try: - platform = PlatformFactory.new(env_options["platform"]) - except UnknownPlatform: - ctx.invoke( - cmd_platform_install, - platforms=[env_options["platform"]], - skip_default_package=True, - ) - platform = PlatformFactory.new(env_options["platform"]) - - debug_options = helpers.configure_initial_debug_options(platform, env_options) - assert debug_options + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=[environment] if environment else None) + env_name = environment or helpers.get_default_debug_env(project_config) if not interface: return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) - ide_data = load_project_ide_data(project_dir, env_name) - if not ide_data: - raise DebugInvalidOptionsError("Could not load a build configuration") + env_options = project_config.items(env=env_name, as_dict=True) + if not set(env_options.keys()) >= set(["platform", "board"]): + raise ProjectEnvsNotAvailableError() + + try: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: + ctx.invoke( + cmd_platform_install, + platforms=[env_options["platform"]], + skip_default_package=True, + ) + platform = PlatformFactory.new(env_options["platform"]) + + debug_config = DebugConfigFactory.new(platform, project_config, env_name) if "--version" in __unprocessed: - result = proc.exec_command([ide_data["gdb_path"], "--version"]) - if result["returncode"] == 0: - return click.echo(result["out"]) - raise exception.PlatformioException("\n".join([result["out"], result["err"]])) + return subprocess.run( + [debug_config.client_executable_path, "--version"], check=True + ) try: fs.ensure_udev_rules() @@ -113,29 +109,23 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) - try: - debug_options = platform.configure_debug_options(debug_options, ide_data) - except NotImplementedError: - # legacy for ESP32 dev-platform <=2.0.0 - debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( - debug_options, ide_data - ) - rebuild_prog = False - preload = debug_options["load_cmds"] == ["preload"] - load_mode = debug_options["load_mode"] + preload = debug_config.load_cmds == ["preload"] + load_mode = debug_config.load_mode if load_mode == "always": - rebuild_prog = preload or not helpers.has_debug_symbols(ide_data["prog_path"]) + rebuild_prog = preload or not helpers.has_debug_symbols( + debug_config.program_path + ) elif load_mode == "modified": rebuild_prog = helpers.is_prog_obsolete( - ide_data["prog_path"] - ) or not helpers.has_debug_symbols(ide_data["prog_path"]) + debug_config.program_path + ) or not helpers.has_debug_symbols(debug_config.program_path) else: - rebuild_prog = not os.path.isfile(ide_data["prog_path"]) + rebuild_prog = not os.path.isfile(debug_config.program_path) if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server - debug_options["load_cmds"] = [] + debug_config.load_cmds = [] if rebuild_prog: if helpers.is_gdbmi_mode(): @@ -155,15 +145,15 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro # save SHA sum of newly created prog if load_mode == "modified": - helpers.is_prog_obsolete(ide_data["prog_path"]) + helpers.is_prog_obsolete(debug_config.program_path) - if not os.path.isfile(ide_data["prog_path"]): + if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") loop = asyncio.ProactorEventLoop() if WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) - client = DebugClientProcess(project_dir, __unprocessed, debug_options, env_options) - coro = client.run(ide_data["gdb_path"], ide_data["prog_path"]) + client = DebugClientProcess(project_dir, debug_config) + coro = client.run(__unprocessed) loop.run_until_complete(coro) if WINDOWS: # an issue with asyncio executor and STIDIN, it cannot be closed gracefully diff --git a/platformio/debug/config/__init__.py b/platformio/debug/config/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/debug/config/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py new file mode 100644 index 0000000000..25c295c2b2 --- /dev/null +++ b/platformio/debug/config/base.py @@ -0,0 +1,227 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, proc, util +from platformio.compat import string_types +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.helpers import reveal_debug_port +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_core_dir, load_project_ide_data +from platformio.project.options import ProjectOptions + + +class DebugConfigBase: # pylint: disable=too-many-instance-attributes + def __init__(self, platform, project_config, env_name): + self.platform = platform + self.project_config = project_config + self.env_name = env_name + self.env_options = project_config.items(env=env_name, as_dict=True) + self.board_config = platform.board_config(self.env_options["board"]) + self.build_data = self._load_build_data() + self.tool_name = self.board_config.get_debug_tool_name( + self.env_options.get("debug_tool") + ) + self.tool_settings = ( + self.board_config.get("debug", {}).get("tools", {}).get(self.tool_name, {}) + ) + + self._load_cmds = None + self._port = None + + self.server = self._configure_server() + + try: + platform.configure_debug_session(self) + except NotImplementedError: + pass + + @staticmethod + def cleanup_cmds(items): + items = ProjectConfig.parse_multi_values(items) + return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] + + @property + def program_path(self): + return self.build_data["prog_path"] + + @property + def client_executable_path(self): + return self.build_data["gdb_path"] + + @property + def load_cmds(self): + if self._load_cmds is not None: + return self._load_cmds + result = self.env_options.get("debug_load_cmds") + if not result: + result = self.tool_settings.get("load_cmds") + if not result: + # legacy + result = self.tool_settings.get("load_cmd") + if not result: + result = ProjectOptions["env.debug_load_cmds"].default + return self.cleanup_cmds(result) + + @load_cmds.setter + def load_cmds(self, cmds): + self._load_cmds = cmds + + @property + def load_mode(self): + result = self.env_options.get("debug_load_mode") + if not result: + result = self.tool_settings.get("load_mode") + return result or ProjectOptions["env.debug_load_mode"].default + + @property + def init_break(self): + result = self.env_options.get("debug_init_break") + if not result: + result = self.tool_settings.get("init_break") + return result or ProjectOptions["env.debug_init_break"].default + + @property + def init_cmds(self): + return self.env_options.get( + "debug_init_cmds", self.tool_settings.get("init_cmds") + ) + + @property + def extra_cmds(self): + return self.cleanup_cmds( + self.env_options.get("debug_extra_cmds") + ) + self.cleanup_cmds(self.tool_settings.get("extra_cmds")) + + @property + def port(self): + return reveal_debug_port( + self.env_options.get("debug_port", self.tool_settings.get("port")) + or self._port, + self.tool_name, + self.tool_settings, + ) + + @port.setter + def port(self, value): + self._port = value + + @property + def upload_protocol(self): + return self.env_options.get( + "upload_protocol", self.board_config.get("upload", {}).get("protocol") + ) + + @property + def speed(self): + return self.env_options.get("debug_speed", self.tool_settings.get("speed")) + + @property + def server_ready_pattern(self): + return (self.server or {}).get("ready_pattern") + + def _load_build_data(self): + data = load_project_ide_data(self.project_config.path, self.env_name) + if data: + return data + raise DebugInvalidOptionsError("Could not load a build configuration") + + def _configure_server(self): + result = None + # specific server per a system + if isinstance(self.tool_settings.get("server", {}), list): + for item in self.tool_settings["server"][:]: + self.tool_settings["server"] = item + if util.get_systype() in item.get("system", []): + break + + # user overwrites debug server + if self.env_options.get("debug_server"): + result = { + "cwd": None, + "executable": None, + "arguments": self.env_options.get("debug_server"), + } + result["executable"] = result["arguments"][0] + result["arguments"] = result["arguments"][1:] + elif "server" in self.tool_settings: + result = self.tool_settings["server"] + server_package = result.get("package") + server_package_dir = ( + self.platform.get_package_dir(server_package) + if server_package + else None + ) + if server_package and not server_package_dir: + self.platform.install_packages( + with_packages=[server_package], + skip_default_package=True, + silent=True, + ) + server_package_dir = self.platform.get_package_dir(server_package) + result.update( + dict( + cwd=server_package_dir if server_package else None, + executable=result.get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", server_package_dir) + if server_package_dir + else a + for a in result.get("arguments", []) + ], + ) + ) + return self.reveal_patterns(result) if result else None + + def get_init_script(self, debugger): + try: + return getattr(self, "%s_INIT_SCRIPT" % debugger.upper()) + except AttributeError: + raise NotImplementedError + + def reveal_patterns(self, source, recursive=True): + patterns = { + "PLATFORMIO_CORE_DIR": get_project_core_dir(), + "PYTHONEXE": proc.get_pythonexe_path(), + "PROJECT_DIR": self.project_config.path, + "PROG_PATH": self.program_path, + "PROG_DIR": os.path.dirname(self.program_path), + "PROG_NAME": os.path.basename(os.path.splitext(self.program_path)[0]), + "DEBUG_PORT": self.port, + "UPLOAD_PROTOCOL": self.upload_protocol, + "INIT_BREAK": self.init_break or "", + "LOAD_CMDS": "\n".join(self.load_cmds or []), + } + for key, value in patterns.items(): + if key.endswith(("_DIR", "_PATH")): + patterns[key] = fs.to_unix_path(value) + + def _replace(text): + for key, value in patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, list) else source.items() + for key, value in items: + if isinstance(value, string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)) and recursive: + source[key] = self.reveal_patterns(value, patterns) + + return source diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py new file mode 100644 index 0000000000..bfc1624690 --- /dev/null +++ b/platformio/debug/config/blackmagic.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class BlackmagicDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + set language c + set *0xE000ED0C = 0x05FA0004 + set $busy = (*0xE000ED0C & 0x4) + while ($busy) + set $busy = (*0xE000ED0C & 0x4) + end + set language auto +end + +define pio_reset_run_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +monitor swdp_scan +attach 1 +set mem inaccessible-by-default off +$LOAD_CMDS +$INIT_BREAK + +set language c +set *0xE000ED0C = 0x05FA0004 +set $busy = (*0xE000ED0C & 0x4) +while ($busy) + set $busy = (*0xE000ED0C & 0x4) +end +set language auto +""" diff --git a/platformio/debug/config/factory.py b/platformio/debug/config/factory.py new file mode 100644 index 0000000000..87019dfa75 --- /dev/null +++ b/platformio/debug/config/factory.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import re + +from platformio.debug.config.generic import GenericDebugConfig + + +class DebugConfigFactory(object): + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) + return "%s%sDebugConfig" % (name.upper()[0], name.lower()[1:]) + + @classmethod + def new(cls, platform, project_config, env_name): + board_config = platform.board_config( + project_config.get("env:" + env_name, "board") + ) + tool_name = board_config.get_debug_tool_name( + project_config.get("env:" + env_name, "debug_tool") + ) + config_cls = None + try: + mod = importlib.import_module("platformio.debug.config.%s" % tool_name) + config_cls = getattr(mod, cls.get_clsname(tool_name)) + except ModuleNotFoundError: + config_cls = GenericDebugConfig + return config_cls(platform, project_config, env_name) diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py new file mode 100644 index 0000000000..a8c6c41042 --- /dev/null +++ b/platformio/debug/config/generic.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class GenericDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor reset halt +end + +define pio_reset_run_target + monitor reset +end + +target extended-remote $DEBUG_PORT +monitor init +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + super(GenericDebugConfig, self).__init__(*args, **kwargs) + self.port = ":3333" diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py new file mode 100644 index 0000000000..020decd7d1 --- /dev/null +++ b/platformio/debug/config/jlink.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class JlinkDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor reset + monitor halt +end + +define pio_reset_run_target + monitor clrbp + monitor reset + monitor go +end + +target extended-remote $DEBUG_PORT +monitor clrbp +monitor speed auto +pio_reset_halt_target +$LOAD_CMDS +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + super(JlinkDebugConfig, self).__init__(*args, **kwargs) + self.port = ":2331" + + @property + def server_ready_pattern(self): + return super(JlinkDebugConfig, self).server_ready_pattern or ( + "Waiting for GDB connection" + ) diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py new file mode 100644 index 0000000000..6c9b9412ef --- /dev/null +++ b/platformio/debug/config/mspdebug.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class MspdebugDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target +end + +define pio_reset_run_target +end + +target extended-remote $DEBUG_PORT +monitor erase +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + super(MspdebugDebugConfig, self).__init__(*args, **kwargs) + self.port = ":2000" diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py new file mode 100644 index 0000000000..d32af5a27f --- /dev/null +++ b/platformio/debug/config/qemu.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class QemuDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor system_reset +end + +define pio_reset_run_target + monitor system_reset +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + super(QemuDebugConfig, self).__init__(*args, **kwargs) + self.port = ":1234" diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py new file mode 100644 index 0000000000..3aef5ef824 --- /dev/null +++ b/platformio/debug/config/renode.py @@ -0,0 +1,45 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class RenodeDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor machine Reset + $LOAD_CMDS + monitor start +end + +define pio_reset_run_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +$INIT_BREAK +monitor start +""" + + def __init__(self, *args, **kwargs): + super(RenodeDebugConfig, self).__init__(*args, **kwargs) + self.port = ":3333" + + @property + def server_ready_pattern(self): + return super(RenodeDebugConfig, self).server_ready_pattern or ( + "GDB server with all CPUs started on port" + ) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index 72c3ff4b85..6038f00014 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -20,13 +20,11 @@ from io import BytesIO from os.path import isfile -from platformio import fs, util +from platformio import util from platformio.commands import PlatformioCLI from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes from platformio.debug.exception import DebugInvalidOptionsError -from platformio.project.config import ProjectConfig -from platformio.project.options import ProjectOptions class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods @@ -86,130 +84,6 @@ def predebug_project(ctx, project_dir, env_name, preload, verbose): time.sleep(5) -def configure_initial_debug_options(platform, env_options): - def _cleanup_cmds(items): - items = ProjectConfig.parse_multi_values(items) - return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] - - board_config = platform.board_config(env_options["board"]) - tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) - tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {}) - server_options = None - - # specific server per a system - if isinstance(tool_settings.get("server", {}), list): - for item in tool_settings["server"][:]: - tool_settings["server"] = item - if util.get_systype() in item.get("system", []): - break - - # user overwrites debug server - if env_options.get("debug_server"): - server_options = { - "cwd": None, - "executable": None, - "arguments": env_options.get("debug_server"), - } - server_options["executable"] = server_options["arguments"][0] - server_options["arguments"] = server_options["arguments"][1:] - elif "server" in tool_settings: - server_options = tool_settings["server"] - server_package = server_options.get("package") - server_package_dir = ( - platform.get_package_dir(server_package) if server_package else None - ) - if server_package and not server_package_dir: - platform.install_packages( - with_packages=[server_package], skip_default_package=True, silent=True - ) - server_package_dir = platform.get_package_dir(server_package) - server_options.update( - dict( - cwd=server_package_dir if server_package else None, - executable=server_options.get("executable"), - arguments=[ - a.replace("$PACKAGE_DIR", server_package_dir) - if server_package_dir - else a - for a in server_options.get("arguments", []) - ], - ) - ) - - extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) - extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds"))) - result = dict( - tool=tool_name, - upload_protocol=env_options.get( - "upload_protocol", board_config.get("upload", {}).get("protocol") - ), - load_cmds=_cleanup_cmds( - env_options.get( - "debug_load_cmds", - tool_settings.get( - "load_cmds", - tool_settings.get( - "load_cmd", ProjectOptions["env.debug_load_cmds"].default - ), - ), - ) - ), - load_mode=env_options.get( - "debug_load_mode", - tool_settings.get( - "load_mode", ProjectOptions["env.debug_load_mode"].default - ), - ), - init_break=env_options.get( - "debug_init_break", - tool_settings.get( - "init_break", ProjectOptions["env.debug_init_break"].default - ), - ), - init_cmds=_cleanup_cmds( - env_options.get("debug_init_cmds", tool_settings.get("init_cmds")) - ), - extra_cmds=extra_cmds, - require_debug_port=tool_settings.get("require_debug_port", False), - port=reveal_debug_port( - env_options.get("debug_port", tool_settings.get("port")), - tool_name, - tool_settings, - ), - speed=env_options.get("debug_speed", tool_settings.get("speed")), - server=server_options, - ) - return result - - -def configure_esp32_load_cmds(debug_options, configuration): - """ - DEPRECATED: Moved to ESP32 dev-platform - See platform.py::configure_debug_options - """ - flash_images = configuration.get("extra", {}).get("flash_images") - ignore_conds = [ - debug_options["load_cmds"] != ["load"], - "xtensa-esp32" not in configuration.get("cc_path", ""), - not flash_images, - not all(isfile(item["path"]) for item in flash_images), - ] - if any(ignore_conds): - return debug_options["load_cmds"] - - mon_cmds = [ - 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( - path=fs.to_unix_path(item["path"]), offset=item["offset"] - ) - for item in flash_images - ] - mon_cmds.append( - 'monitor program_esp32 "{%s.bin}" 0x10000 verify' - % fs.to_unix_path(configuration["prog_path"][:-4]) - ) - return mon_cmds - - def has_debug_symbols(prog_path): if not isfile(prog_path): return False diff --git a/platformio/debug/initcfgs.py b/platformio/debug/initcfgs.py deleted file mode 100644 index 55e7c34f99..0000000000 --- a/platformio/debug/initcfgs.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -GDB_DEFAULT_INIT_CONFIG = """ -define pio_reset_halt_target - monitor reset halt -end - -define pio_reset_run_target - monitor reset -end - -target extended-remote $DEBUG_PORT -monitor init -$LOAD_CMDS -pio_reset_halt_target -$INIT_BREAK -""" - -GDB_STUTIL_INIT_CONFIG = """ -define pio_reset_halt_target - monitor reset - monitor halt -end - -define pio_reset_run_target - monitor reset -end - -target extended-remote $DEBUG_PORT -$LOAD_CMDS -pio_reset_halt_target -$INIT_BREAK -""" - -GDB_JLINK_INIT_CONFIG = """ -define pio_reset_halt_target - monitor reset - monitor halt -end - -define pio_reset_run_target - monitor clrbp - monitor reset - monitor go -end - -target extended-remote $DEBUG_PORT -monitor clrbp -monitor speed auto -pio_reset_halt_target -$LOAD_CMDS -$INIT_BREAK -""" - -GDB_BLACKMAGIC_INIT_CONFIG = """ -define pio_reset_halt_target - set language c - set *0xE000ED0C = 0x05FA0004 - set $busy = (*0xE000ED0C & 0x4) - while ($busy) - set $busy = (*0xE000ED0C & 0x4) - end - set language auto -end - -define pio_reset_run_target - pio_reset_halt_target -end - -target extended-remote $DEBUG_PORT -monitor swdp_scan -attach 1 -set mem inaccessible-by-default off -$LOAD_CMDS -$INIT_BREAK - -set language c -set *0xE000ED0C = 0x05FA0004 -set $busy = (*0xE000ED0C & 0x4) -while ($busy) - set $busy = (*0xE000ED0C & 0x4) -end -set language auto -""" - -GDB_MSPDEBUG_INIT_CONFIG = """ -define pio_reset_halt_target -end - -define pio_reset_run_target -end - -target extended-remote $DEBUG_PORT -monitor erase -$LOAD_CMDS -pio_reset_halt_target -$INIT_BREAK -""" - -GDB_QEMU_INIT_CONFIG = """ -define pio_reset_halt_target - monitor system_reset -end - -define pio_reset_run_target - monitor system_reset -end - -target extended-remote $DEBUG_PORT -$LOAD_CMDS -pio_reset_halt_target -$INIT_BREAK -""" - -GDB_RENODE_INIT_CONFIG = """ -define pio_reset_halt_target - monitor machine Reset - $LOAD_CMDS - monitor start -end - -define pio_reset_run_target - pio_reset_halt_target -end - -target extended-remote $DEBUG_PORT -$LOAD_CMDS -$INIT_BREAK -monitor start -""" - - -TOOL_TO_CONFIG = { - "jlink": GDB_JLINK_INIT_CONFIG, - "mspdebug": GDB_MSPDEBUG_INIT_CONFIG, - "qemu": GDB_QEMU_INIT_CONFIG, - "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG, - "renode": GDB_RENODE_INIT_CONFIG, -} - - -def get_gdb_init_config(debug_options): - tool = debug_options.get("tool") - if tool and tool in TOOL_TO_CONFIG: - return TOOL_TO_CONFIG[tool] - server_exe = (debug_options.get("server") or {}).get("executable", "").lower() - if "st-util" in server_exe: - return GDB_STUTIL_INIT_CONFIG - return GDB_DEFAULT_INIT_CONFIG diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index a97d544641..846fbf7704 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -18,16 +18,12 @@ import sys import time -from platformio import fs from platformio.compat import ( WINDOWS, aio_create_task, aio_get_running_loop, get_locale_encoding, - string_types, ) -from platformio.proc import get_pythonexe_path -from platformio.project.helpers import get_project_core_dir class DebugSubprocessProtocol(asyncio.SubprocessProtocol): @@ -61,12 +57,6 @@ class DebugBaseProcess: STDOUT_CHUNK_SIZE = 2048 LOG_FILE = None - COMMON_PATTERNS = { - "PLATFORMIO_HOME_DIR": get_project_core_dir(), - "PLATFORMIO_CORE_DIR": get_project_core_dir(), - "PYTHONEXE": get_pythonexe_path(), - } - def __init__(self): self.transport = None self._is_running = False @@ -155,32 +145,6 @@ def process_exited(self): self._exit_future.set_result(True) self._exit_future = None - def apply_patterns(self, source, patterns=None): - _patterns = self.COMMON_PATTERNS.copy() - _patterns.update(patterns or {}) - - for key, value in _patterns.items(): - if key.endswith(("_DIR", "_PATH")): - _patterns[key] = fs.to_unix_path(value) - - def _replace(text): - for key, value in _patterns.items(): - pattern = "$%s" % key - text = text.replace(pattern, value or "") - return text - - if isinstance(source, string_types): - source = _replace(source) - elif isinstance(source, (list, dict)): - items = enumerate(source) if isinstance(source, list) else source.items() - for key, value in items: - if isinstance(value, string_types): - source[key] = _replace(value) - elif isinstance(value, (list, dict)): - source[key] = self.apply_patterns(value, patterns) - - return source - def terminate(self): if not self.is_running() or not self.transport: return diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py index e8b05715f2..aad6c69d49 100644 --- a/platformio/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -23,8 +23,6 @@ from platformio.cache import ContentCache from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes from platformio.debug import helpers -from platformio.debug.exception import DebugInvalidOptionsError -from platformio.debug.initcfgs import get_gdb_init_config from platformio.debug.process.base import DebugBaseProcess from platformio.debug.process.server import DebugServerProcess from platformio.project.helpers import get_project_cache_dir @@ -37,14 +35,12 @@ class DebugClientProcess( PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" - def __init__(self, project_dir, args, debug_options, env_options): + def __init__(self, project_dir, debug_config): super(DebugClientProcess, self).__init__() self.project_dir = project_dir - self.args = list(args) - self.debug_options = debug_options - self.env_options = env_options + self.debug_config = debug_config - self._server_process = DebugServerProcess(debug_options, env_options) + self._server_process = DebugServerProcess(debug_config) self._session_id = None if not os.path.isdir(get_project_cache_dir()): @@ -56,27 +52,13 @@ def __init__(self, project_dir, args, debug_options, env_options): self._target_is_running = False self._errors_buffer = b"" - async def run(self, gdb_path, prog_path): - session_hash = gdb_path + prog_path + async def run(self, extra_args): + gdb_path = self.debug_config.client_executable_path + session_hash = gdb_path + self.debug_config.program_path self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() - - patterns = { - "PROJECT_DIR": self.project_dir, - "PROG_PATH": prog_path, - "PROG_DIR": os.path.dirname(prog_path), - "PROG_NAME": os.path.basename(os.path.splitext(prog_path)[0]), - "DEBUG_PORT": self.debug_options["port"], - "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"], - "INIT_BREAK": self.debug_options["init_break"] or "", - "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), - } - - await self._server_process.run(patterns) - if not patterns["DEBUG_PORT"]: - patterns["DEBUG_PORT"] = self._server_process.get_debug_port() - - self.generate_pioinit(self._gdbsrc_dir, patterns) + self.debug_config.port = await self._server_process.run() + self.generate_init_script(os.path.join(self._gdbsrc_dir, self.PIO_SRC_NAME)) # start GDB client args = [ @@ -89,13 +71,11 @@ async def run(self, gdb_path, prog_path): "-l", "10", ] - args.extend(self.args) - if not gdb_path: - raise DebugInvalidOptionsError("GDB client is not configured") + args.extend(list(extra_args or [])) gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) - args.append(patterns["PROG_PATH"]) + args.append(self.debug_config.program_path) await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) @staticmethod @@ -107,13 +87,13 @@ def _get_data_dir(gdb_path): ) return gdb_data_dir if os.path.isdir(gdb_data_dir) else None - def generate_pioinit(self, dst_dir, patterns): + def generate_init_script(self, dst): # default GDB init commands depending on debug tool - commands = get_gdb_init_config(self.debug_options).split("\n") + commands = self.debug_config.get_init_script("gdb").split("\n") - if self.debug_options["init_cmds"]: - commands = self.debug_options["init_cmds"] - commands.extend(self.debug_options["extra_cmds"]) + if self.debug_config.init_cmds: + commands = self.debug_config.init_cmds + commands.extend(self.debug_config.extra_cmds) if not any("define pio_reset_run_target" in cmd for cmd in commands): commands = [ @@ -134,20 +114,20 @@ def generate_pioinit(self, dst_dir, patterns): "define pio_restart_target", " pio_reset_halt_target", " $INIT_BREAK", - " %s" % ("continue" if patterns["INIT_BREAK"] else "next"), + " %s" % ("continue" if self.debug_config.init_break else "next"), "end", ] banner = [ "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", - "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"], + "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, "echo PlatformIO: Initializing remote target...\\n", ] footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] commands = banner + commands + footer - with open(os.path.join(dst_dir, self.PIO_SRC_NAME), "w") as fp: - fp.write("\n".join(self.apply_patterns(commands, patterns))) + with open(dst, "w") as fp: + fp.write("\n".join(self.debug_config.reveal_patterns(commands))) def connection_made(self, transport): super(DebugClientProcess, self).connection_made(transport) @@ -179,7 +159,9 @@ def stdout_data_received(self, data): # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: telemetry.send_event( - "Debug", "Started", telemetry.dump_run_environment(self.env_options) + "Debug", + "Started", + telemetry.dump_run_environment(self.debug_config.env_options), ) self._auto_exec_continue() @@ -194,12 +176,12 @@ def _auto_exec_continue(self): aio_get_running_loop().call_later(0.1, self._auto_exec_continue) return - if not self.debug_options["init_break"] or self._target_is_running: + if not self.debug_config.init_break or self._target_is_running: return self.console_log( "PlatformIO: Resume the execution to `debug_init_break = %s`\n" - % self.debug_options["init_break"] + % self.debug_config.init_break ) self.console_log( "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" @@ -226,7 +208,7 @@ def _handle_error(self, data): last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) err = "%s -> %s" % ( - telemetry.dump_run_environment(self.env_options), + telemetry.dump_run_environment(self.debug_config.env_options), last_erros, ) telemetry.send_exception("DebugInitError: %s" % err) diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 5371733868..24684037ec 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -13,10 +13,12 @@ # limitations under the License. import asyncio +import fnmatch import os import time -from platformio import fs, util +from platformio import fs +from platformio.compat import MACOS, WINDOWS from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode from platformio.debug.process.base import DebugBaseProcess @@ -24,27 +26,22 @@ class DebugServerProcess(DebugBaseProcess): - def __init__(self, debug_options, env_options): + def __init__(self, debug_config): super(DebugServerProcess, self).__init__() - self.debug_options = debug_options - self.env_options = env_options - - self._debug_port = ":3333" + self.debug_config = debug_config self._ready = False - async def run(self, patterns): # pylint: disable=too-many-branches - systype = util.get_systype() - server = self.debug_options.get("server") + async def run(self): # pylint: disable=too-many-branches + server = self.debug_config.server if not server: return None - server = self.apply_patterns(server, patterns) server_executable = server["executable"] if not server_executable: return None if server["cwd"]: server_executable = os.path.join(server["cwd"], server_executable) if ( - "windows" in systype + WINDOWS and not server_executable.endswith(".exe") and os.path.isfile(server_executable + ".exe") ): @@ -56,15 +53,18 @@ async def run(self, patterns): # pylint: disable=too-many-branches raise DebugInvalidOptionsError( "\nCould not launch Debug Server '%s'. Please check that it " "is installed and is included in a system PATH\n\n" - "See documentation or contact contact@platformio.org:\n" + "See documentation:\n" "https://docs.platformio.org/page/plus/debugging.html\n" % server_executable ) openocd_pipe_allowed = all( - [not self.debug_options["port"], "openocd" in server_executable] + [ + not self.debug_config.env_options.get("debug_port"), + "gdb" in self.debug_config.client_executable_path, + "openocd" in server_executable, + ] ) - # openocd_pipe_allowed = False if openocd_pipe_allowed: args = [] if server["cwd"]: @@ -76,18 +76,16 @@ async def run(self, patterns): # pylint: disable=too-many-branches str_args = " ".join( [arg if arg.startswith("-") else '"%s"' % arg for arg in args] ) - self._debug_port = '| "%s" %s' % (server_executable, str_args) - self._debug_port = fs.to_unix_path(self._debug_port) - return self._debug_port + return fs.to_unix_path('| "%s" %s' % (server_executable, str_args)) env = os.environ.copy() # prepend server "lib" folder to LD path if ( - "windows" not in systype + not WINDOWS and server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "lib")) ): - ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" + ld_key = "DYLD_LIBRARY_PATH" if MACOS else "LD_LIBRARY_PATH" env[ld_key] = os.path.join(server["cwd"], "lib") if os.environ.get(ld_key): env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) @@ -102,20 +100,12 @@ async def run(self, patterns): # pylint: disable=too-many-branches await self.spawn( *([server_executable] + server["arguments"]), cwd=server["cwd"], env=env ) - - if "mspdebug" in server_executable.lower(): - self._debug_port = ":2000" - elif "jlink" in server_executable.lower(): - self._debug_port = ":2331" - elif "qemu" in server_executable.lower(): - self._debug_port = ":1234" - await self._wait_until_ready() - return self._debug_port + return self.debug_config.port async def _wait_until_ready(self): - ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + ready_pattern = self.debug_config.server_ready_pattern timeout = 60 if ready_pattern else 10 elapsed = 0 delay = 0.5 @@ -129,14 +119,11 @@ async def _wait_until_ready(self): def _check_ready_by_pattern(self, data): if self._ready: return self._ready - ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + ready_pattern = self.debug_config.server_ready_pattern if ready_pattern: self._ready = ready_pattern.encode() in data return self._ready - def get_debug_port(self): - return self._debug_port - def stdout_data_received(self, data): super(DebugServerProcess, self).stdout_data_received( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 3cadbd7347..1d7911b009 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -203,7 +203,7 @@ def configure_default_packages(self, options, targets): elif "nobuild" in targets and opts.get("type") != "framework": self.packages[name]["optional"] = True - def configure_debug_options(self, initial_debug_options, ide_data): + def configure_debug_session(self, debug_config): raise NotImplementedError def get_lib_storages(self): From a78db177846eb58f088a69b228f21142345e8757 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 00:21:44 +0200 Subject: [PATCH 007/120] Drop support for Python 2 --- platformio/__main__.py | 4 ++-- platformio/app.py | 4 ++-- platformio/builder/main.py | 2 +- platformio/builder/tools/piolib.py | 4 ++-- platformio/builder/tools/piomaxlen.py | 6 +++--- platformio/builder/tools/pioplatform.py | 7 +++---- platformio/builder/tools/piosize.py | 4 ++-- platformio/builder/tools/piotarget.py | 2 +- platformio/builder/tools/pioupload.py | 4 ++-- platformio/builder/tools/platformio.py | 4 ++-- platformio/commands/check/tools/pvsstudio.py | 9 ++++----- platformio/commands/debug.py | 6 +++--- platformio/commands/home/helpers.py | 8 +++----- platformio/commands/system/command.py | 2 +- platformio/commands/upgrade.py | 4 ++-- platformio/compat.py | 6 +++--- platformio/debug/helpers.py | 8 ++------ platformio/debug/process/base.py | 4 ++-- platformio/debug/process/client.py | 11 ++++++++--- platformio/debug/process/server.py | 9 ++++----- platformio/fs.py | 6 +++--- platformio/package/manager/core.py | 3 +-- platformio/package/pack.py | 4 ++-- platformio/platform/_run.py | 9 ++------- platformio/platform/board.py | 10 ---------- platformio/proc.py | 16 +++++----------- platformio/project/config.py | 10 +++------- platformio/project/helpers.py | 6 +++--- platformio/util.py | 12 +++--------- tests/package/test_manifest.py | 4 ++-- tests/package/test_pack.py | 6 ++---- 31 files changed, 78 insertions(+), 116 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index dbf2215b00..ae6413d1c0 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -22,7 +22,7 @@ from platformio import __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import CYGWIN, PY2, ensure_python3 +from platformio.compat import IS_CYGWIN, PY2, ensure_python3 try: import click_completion # pylint: disable=import-error @@ -76,7 +76,7 @@ def process_result(ctx, result, *_, **__): def configure(): - if CYGWIN: + if IS_CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org diff --git a/platformio/app.py b/platformio/app.py index 9d64994c39..ae483f1f56 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -24,7 +24,7 @@ from os.path import dirname, isdir, isfile, join, realpath from platformio import __version__, exception, fs, proc -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import get_default_projects_dir, get_project_core_dir @@ -277,7 +277,7 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) - if WINDOWS or os.getuid() > 0: # pylint: disable=no-member + if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 208cb0e2ae..245a0f984f 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -129,7 +129,7 @@ ) if ( - compat.WINDOWS + compat.IS_WINDOWS and sys.version_info >= (3, 8) and env["PROJECT_DIR"].startswith("\\\\") ): diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f6b9824be6..3849a91bdf 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,7 +33,7 @@ from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.clients.http import InternetIsOffline -from platformio.compat import WINDOWS, hashlib_encode_data, string_types +from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.manifest.parser import ( @@ -142,7 +142,7 @@ def __repr__(self): def __contains__(self, path): p1 = self.path p2 = path - if WINDOWS: + if IS_WINDOWS: p1 = p1.lower() p2 = p2.lower() if p1 == p2: diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 0059dcf389..02622b9d0e 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -21,20 +21,20 @@ from SCons.Platform import TempFileMunge # pylint: disable=import-error from SCons.Subst import quote_spaces # pylint: disable=import-error -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import IS_WINDOWS, hashlib_encode_data # There are the next limits depending on a platform: # - Windows = 8192 # - Unix = 131072 # We need ~512 characters for compiler and temporary file paths -MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 512 +MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512 WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") def tempfile_arg_esc_func(arg): arg = quote_spaces(arg) - if not WINDOWS: + if not IS_WINDOWS: return arg # GCC requires double Windows slashes, let's use UNIX separator return WINPATHSEP_RE.sub(r"/\1", arg) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index dc7f67bf00..5f7182c4b5 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -21,7 +21,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import fs, util -from platformio.compat import WINDOWS +from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.package.meta import PackageItem from platformio.package.version import get_original_version from platformio.platform.exception import UnknownBoard @@ -71,7 +71,6 @@ def LoadPioPlatform(env): env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH - systype = util.get_systype() for pkg in p.get_installed_packages(): type_ = p.get_package_type(pkg.metadata.name) if type_ not in ("toolchain", "uploader", "debugger"): @@ -83,12 +82,12 @@ def LoadPioPlatform(env): else pkg.path, ) if ( - not WINDOWS + not IS_WINDOWS and os.path.isdir(os.path.join(pkg.path, "lib")) and type_ != "toolchain" ): env.PrependENVPath( - "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH", os.path.join(pkg.path, "lib"), ) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index eec4c87c6f..e194511a3f 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -24,8 +24,8 @@ from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile +from platformio.compat import IS_WINDOWS from platformio.proc import exec_command -from platformio.util import get_systype def _run_tool(cmd, env, tool_args): @@ -164,7 +164,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections): location = symbol_locations.get(hex(symbol["addr"])) if not location or "?" in location: continue - if "windows" in get_systype(): + if IS_WINDOWS: drive, tail = splitdrive(location) location = join(drive.upper(), tail) symbol["file"] = location diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 948776fc0b..70b7c41bb5 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -31,7 +31,7 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): def _relpath(path): - if compat.WINDOWS: + if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() if ( ":" not in prefix diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 5e1117b501..3281ada455 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -26,7 +26,7 @@ from serial import Serial, SerialException from platformio import exception, fs, util -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS from platformio.proc import exec_command # pylint: disable=unused-argument @@ -134,7 +134,7 @@ def _look_for_serial_port(): continue port = item["port"] if upload_protocol.startswith("blackmagic"): - if WINDOWS and port.startswith("COM") and len(port) > 4: + if IS_WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item["description"]: return port diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 4e1ca9ca88..c412012efd 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,7 @@ from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs -from platformio.compat import MACOS, string_types +from platformio.compat import IS_MACOS, string_types from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] @@ -69,7 +69,7 @@ def BuildProgram(env): if ( env.get("LIBS") and env.GetCompilerType() == "gcc" - and (env.PioPlatform().is_embedded() or not MACOS) + and (env.PioPlatform().is_embedded() or not IS_MACOS) ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index d4dde208e1..e95e202c2e 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -19,9 +19,10 @@ import click -from platformio import proc, util +from platformio import proc from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase +from platformio.compat import IS_WINDOWS from platformio.package.manager.core import get_core_package_dir @@ -34,7 +35,7 @@ def __init__(self, *args, **kwargs): self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd" self.tool_path = os.path.join( get_core_package_dir("tool-pvs-studio"), - "x64" if "windows" in util.get_systype() else "bin", + "x64" if IS_WINDOWS else "bin", "pvs-studio", ) super(PvsStudioCheckTool, self).__init__(*args, **kwargs) @@ -70,9 +71,7 @@ def _process_defects(self, defects): def _demangle_report(self, output_file): converter_tool = os.path.join( get_core_package_dir("tool-pvs-studio"), - "HtmlGenerator" - if "windows" in util.get_systype() - else os.path.join("bin", "plog-converter"), + "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"), ) cmd = ( diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index b0c1ed93d5..7abb197656 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -23,7 +23,7 @@ from platformio import app, exception, fs, proc from platformio.commands.platform import platform_install as cmd_platform_install -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS from platformio.debug import helpers from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError @@ -150,12 +150,12 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") - loop = asyncio.ProactorEventLoop() if WINDOWS else asyncio.get_event_loop() + loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) client = DebugClientProcess(project_dir, debug_config) coro = client.run(__unprocessed) loop.run_until_complete(coro) - if WINDOWS: + if IS_WINDOWS: # an issue with asyncio executor and STIDIN, it cannot be closed gracefully proc.force_exit() loop.close() diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 5c6e0c888f..e7407eb97b 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -18,7 +18,7 @@ from starlette.concurrency import run_in_threadpool from platformio import util -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS from platformio.proc import where_is_program @@ -37,15 +37,13 @@ def requests_session(): @util.memoized(expire="60s") def get_core_fullpath(): - return where_is_program( - "platformio" + (".exe" if "windows" in util.get_systype() else "") - ) + return where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) def is_port_used(host, port): socket.setdefaulttimeout(1) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if WINDOWS: + if IS_WINDOWS: try: s.bind((host, port)) s.close() diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index d0c87d844e..63d1727cd0 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -69,7 +69,7 @@ def system_info(json_output): data["platformio_exe"] = { "title": "PlatformIO Core Executable", "value": proc.where_is_program( - "platformio.exe" if proc.WINDOWS else "platformio" + "platformio.exe" if compat.IS_WINDOWS else "platformio" ), } data["python_exe"] = { diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 2411f49cf5..d8f6e78c5c 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -21,7 +21,7 @@ from platformio import VERSION, __version__, app, exception from platformio.clients.http import fetch_remote_content -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -73,7 +73,7 @@ def cli(dev): if not r: raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") - if any(m in r["err"].lower() for m in permission_errors) and not WINDOWS: + if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: click.secho( """ ----------------- diff --git a/platformio/compat.py b/platformio/compat.py index 97cfb8d382..5c20df3a5a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -30,9 +30,9 @@ PY2 = sys.version_info[0] == 2 -CYGWIN = sys.platform.startswith("cygwin") -WINDOWS = sys.platform.startswith("win") -MACOS = sys.platform.startswith("darwin") +IS_CYGWIN = sys.platform.startswith("cygwin") +IS_WINDOWS = WINDOWS = sys.platform.startswith("win") +IS_MACOS = sys.platform.startswith("darwin") string_types = (str,) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index 6038f00014..a412ad5ee8 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -23,7 +23,7 @@ from platformio import util from platformio.commands import PlatformioCLI from platformio.commands.run.command import cli as cmd_run -from platformio.compat import is_bytes +from platformio.compat import IS_WINDOWS, is_bytes from platformio.debug.exception import DebugInvalidOptionsError @@ -152,11 +152,7 @@ def _look_for_serial_port(hwids): continue port = item["port"] if tool_name.startswith("blackmagic"): - if ( - "windows" in util.get_systype() - and port.startswith("COM") - and len(port) > 4 - ): + if IS_WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item["description"]: return port diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index 846fbf7704..2c9280c26f 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -19,7 +19,7 @@ import time from platformio.compat import ( - WINDOWS, + IS_WINDOWS, aio_create_task, aio_get_running_loop, get_locale_encoding, @@ -93,7 +93,7 @@ def connect_stdin_pipe(self): async def _read_stdin_pipe(self): loop = aio_get_running_loop() - if WINDOWS: + if IS_WINDOWS: while True: self.stdin_data_received( await loop.run_in_executor(None, sys.stdin.buffer.readline) diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py index aad6c69d49..7192472234 100644 --- a/platformio/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -19,9 +19,14 @@ import tempfile import time -from platformio import fs, proc, telemetry, util +from platformio import fs, proc, telemetry from platformio.cache import ContentCache -from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes +from platformio.compat import ( + IS_WINDOWS, + aio_get_running_loop, + hashlib_encode_data, + is_bytes, +) from platformio.debug import helpers from platformio.debug.process.base import DebugBaseProcess from platformio.debug.process.server import DebugServerProcess @@ -230,7 +235,7 @@ def _kill_previous_session(self): cc.delete(self._session_id) if not pid: return - if "windows" in util.get_systype(): + if IS_WINDOWS: kill = ["Taskkill", "/PID", pid, "/F"] else: kill = ["kill", pid] diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 24684037ec..4e1645e282 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -13,12 +13,11 @@ # limitations under the License. import asyncio -import fnmatch import os import time from platformio import fs -from platformio.compat import MACOS, WINDOWS +from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode from platformio.debug.process.base import DebugBaseProcess @@ -41,7 +40,7 @@ async def run(self): # pylint: disable=too-many-branches if server["cwd"]: server_executable = os.path.join(server["cwd"], server_executable) if ( - WINDOWS + IS_WINDOWS and not server_executable.endswith(".exe") and os.path.isfile(server_executable + ".exe") ): @@ -81,11 +80,11 @@ async def run(self): # pylint: disable=too-many-branches env = os.environ.copy() # prepend server "lib" folder to LD path if ( - not WINDOWS + not IS_WINDOWS and server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "lib")) ): - ld_key = "DYLD_LIBRARY_PATH" if MACOS else "LD_LIBRARY_PATH" + ld_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH" env[ld_key] = os.path.join(server["cwd"], "lib") if os.environ.get(ld_key): env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) diff --git a/platformio/fs.py b/platformio/fs.py index 3aefdb6e67..655c2ee87c 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -25,7 +25,7 @@ import click from platformio import exception -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS class cd(object): @@ -176,7 +176,7 @@ def _append_build_item(items, item, src_dir): def to_unix_path(path): - if not WINDOWS or not path: + if not IS_WINDOWS or not path: return path return re.sub(r"[\\]+", "/", path) @@ -185,7 +185,7 @@ def expanduser(path): """ Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE """ - if not WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ: + if not IS_WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ: return os.path.expanduser(path) return os.environ["USERPROFILE"] + path[1:] diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index dd3f26639f..ac5cad7b89 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -20,7 +20,6 @@ from datetime import date from platformio import __core_packages__, exception, fs, util -from platformio.compat import PY2 from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageItem, PackageSpec @@ -207,7 +206,7 @@ def get_contrib_pysite_deps(): sys_type = util.get_systype() py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) - twisted_version = "19.10.0" if PY2 else "20.3.0" + twisted_version = "20.3.0" result = [ "twisted == %s" % twisted_version, ] diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 66dff1d5bf..fb0d222fc1 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -20,7 +20,7 @@ import tempfile from platformio import fs -from platformio.compat import WINDOWS, ensure_python3 +from platformio.compat import IS_WINDOWS, ensure_python3 from platformio.package.exception import PackageException, UserSideException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema @@ -117,7 +117,7 @@ def pack(self, dst=None): # if zip/tar.gz -> unpack to tmp dir if not os.path.isdir(src): - if WINDOWS: + if IS_WINDOWS: raise UserSideException( "Packaging from an archive does not work on Windows OS. Please " "extract data from `%s` manually and pack a folder instead" diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index cb5ec99568..731901677a 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -20,7 +20,7 @@ import click from platformio import app, fs, proc, telemetry -from platformio.compat import PY2, hashlib_encode_data, is_bytes +from platformio.compat import hashlib_encode_data, is_bytes from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound @@ -90,14 +90,9 @@ def _report_non_sensitive_data(self, options, targets): def _run_scons(self, variables, targets, jobs): scons_dir = get_core_package_dir("tool-scons") - script_path = ( - os.path.join(scons_dir, "script", "scons") - if PY2 - else os.path.join(scons_dir, "scons.py") - ) args = [ proc.get_pythonexe_path(), - script_path, + os.path.join(scons_dir, "scons.py"), "-Q", "--warn=no-no-parallel-support", "--jobs", diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 34d9dc3478..6594096247 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -15,7 +15,6 @@ import os from platformio import fs, telemetry, util -from platformio.compat import PY2 from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError from platformio.exception import UserSideException from platformio.platform.exception import InvalidBoardManifest @@ -40,15 +39,6 @@ def get(self, path, default=None): value = self._manifest for k in path.split("."): value = value[k] - # pylint: disable=undefined-variable - if PY2 and isinstance(value, unicode): - # cast to plain string from unicode for PY2, resolves issue in - # dev/platform when BoardConfig.get() is used in pair with - # os.path.join(file_encoding, unicode_encoding) - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass return value except KeyError: if default is not None: diff --git a/platformio/proc.py b/platformio/proc.py index b4ef4505cc..6e9981afac 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -20,8 +20,7 @@ from platformio import exception from platformio.compat import ( - PY2, - WINDOWS, + IS_WINDOWS, get_filesystem_encoding, get_locale_encoding, string_types, @@ -31,10 +30,7 @@ class AsyncPipeBase(object): def __init__(self): self._fd_read, self._fd_write = os.pipe() - if PY2: - self._pipe_reader = os.fdopen(self._fd_read) - else: - self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace") + self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace") self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() @@ -129,9 +125,7 @@ def exec_command(*args, **kwargs): result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): - if PY2 and isinstance(v, unicode): # pylint: disable=undefined-variable - result[k] = v.encode() - elif not PY2 and isinstance(result[k], bytes): + if isinstance(result[k], bytes): try: result[k] = result[k].decode( get_locale_encoding() or get_filesystem_encoding() @@ -178,7 +172,7 @@ def copy_pythonpath_to_osenv(): _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) for p in os.sys.path: conditions = [p not in _PYTHONPATH] - if not WINDOWS: + if not IS_WINDOWS: conditions.append( os.path.isdir(os.path.join(p, "click")) or os.path.isdir(os.path.join(p, "platformio")) @@ -195,7 +189,7 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: - result = exec_command(["where" if WINDOWS else "which", program], env=env) + result = exec_command(["where" if IS_WINDOWS else "which", program], env=env) if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): return result["out"].strip() except OSError: diff --git a/platformio/project/config.py b/platformio/project/config.py index 9f78a67b81..6c5aa7b192 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -21,7 +21,7 @@ import click from platformio import fs -from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types +from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -88,11 +88,7 @@ def __init__(self, path=None, parse_extra=True, expand_interpolations=True): self.expand_interpolations = expand_interpolations self.warnings = [] self._parsed = [] - self._parser = ( - ConfigParser.ConfigParser() - if PY2 - else ConfigParser.ConfigParser(inline_comment_prefixes=("#", ";")) - ) + self._parser = ConfigParser.ConfigParser(inline_comment_prefixes=("#", ";")) if path and os.path.isfile(path): self.read(path, parse_extra) @@ -359,7 +355,7 @@ def _get_core_dir(self, exists=False): default = ProjectOptions["platformio.core_dir"].default core_dir = self.get("platformio", "core_dir") win_core_dir = None - if WINDOWS and core_dir == default: + if IS_WINDOWS and core_dir == default: win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio" if os.path.isdir(win_core_dir): core_dir = win_core_dir diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 642e1a7e4d..ec8af4a2e0 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -21,7 +21,7 @@ from click.testing import CliRunner from platformio import __version__, exception, fs -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -92,7 +92,7 @@ def get_project_libdeps_dir(): def get_default_projects_dir(): docs_dir = join(fs.expanduser("~"), "Documents") try: - assert WINDOWS + assert IS_WINDOWS import ctypes.wintypes # pylint: disable=import-outside-toplevel buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) @@ -128,7 +128,7 @@ def compute_project_checksum(config): if not chunks: continue chunks_to_str = ",".join(sorted(chunks)) - if WINDOWS: # case insensitive OS + if IS_WINDOWS: # case insensitive OS chunks_to_str = chunks_to_str.lower() checksum.update(hashlib_encode_data(chunks_to_str)) diff --git a/platformio/util.py b/platformio/util.py index 6b1af88614..cba128ec58 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -26,7 +26,7 @@ import click from platformio import __version__, compat, exception, proc -from platformio.compat import PY2, WINDOWS +from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import @@ -106,12 +106,6 @@ def get_serial_ports(filter_hwid=False): for p, d, h in comports(): if not p: continue - if WINDOWS and PY2: - try: - # pylint: disable=undefined-variable - d = unicode(d, errors="ignore") - except TypeError: - pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -119,7 +113,7 @@ def get_serial_ports(filter_hwid=False): return result # fix for PySerial - if not result and "darwin" in get_systype(): + if not result and IS_MACOS: for p in glob("/dev/tty.*"): result.append({"port": p, "description": "n/a", "hwid": "n/a"}) return result @@ -131,7 +125,7 @@ def get_serial_ports(filter_hwid=False): def get_logical_devices(): items = [] - if WINDOWS: + if IS_WINDOWS: try: result = proc.exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"] diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 4a0afc76ae..2cc64ddb3d 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -19,7 +19,7 @@ import jsondiff import pytest -from platformio.compat import WINDOWS +from platformio.compat import IS_WINDOWS from platformio.package.manifest import parser from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError @@ -715,7 +715,7 @@ def test_examples_from_dir(tmpdir_factory): pio_dir.join(".vimrc").write("") pio_ini = pio_dir.join("platformio.ini") pio_ini.write("") - if not WINDOWS: + if not IS_WINDOWS: pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) pio_dir.mkdir("include").join("main.h").write("") pio_dir.mkdir("src").join("main.cpp").write("") diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 92db31c4c9..cc898f9d9a 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -19,12 +19,10 @@ import pytest from platformio import fs -from platformio.compat import PY2, WINDOWS +from platformio.compat import IS_WINDOWS from platformio.package.exception import UnknownManifestError from platformio.package.pack import PackagePacker -pytestmark = pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") - def test_base(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") @@ -99,7 +97,7 @@ def test_filters(tmpdir_factory): def test_symlinks(tmpdir_factory): # Windows does not support symbolic links - if WINDOWS: + if IS_WINDOWS: return pkg_dir = tmpdir_factory.mktemp("package") src_dir = pkg_dir.mkdir("src") From b0c3e22a525c340c86c0a2cbb1aa691c64339acb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 12:30:16 +0200 Subject: [PATCH 008/120] Configure a custom pattern to determine when debugging server is started with a new debug_server_ready_pattern option --- HISTORY.rst | 1 + docs | 2 +- platformio/__main__.py | 4 ++-- platformio/debug/config/base.py | 4 +++- platformio/debug/process/server.py | 9 ++++++++- platformio/project/options.py | 8 ++++++++ 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d3f412fdd9..59edd6de48 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO Core 5 - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) + - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 0487c24f93..b76e3d53bb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0487c24f930e3b1f45d594b6564148dc5324f263 +Subproject commit b76e3d53bbe10c945e1dbe302fe7f0b07005447a diff --git a/platformio/__main__.py b/platformio/__main__.py index ae6413d1c0..0cc7dca34e 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -22,7 +22,7 @@ from platformio import __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import IS_CYGWIN, PY2, ensure_python3 +from platformio.compat import IS_CYGWIN, ensure_python3 try: import click_completion # pylint: disable=import-error @@ -118,7 +118,7 @@ def main(argv=None): exit_code = int(e.code) except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): - if not PY2: + if sys.version_info.major != 2: from platformio import maintenance maintenance.on_platformio_exception(e) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 25c295c2b2..d3d07c96d8 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -130,7 +130,9 @@ def speed(self): @property def server_ready_pattern(self): - return (self.server or {}).get("ready_pattern") + return self.env_options.get( + "debug_server_ready_pattern", (self.server or {}).get("ready_pattern") + ) def _load_build_data(self): data = load_project_ide_data(self.project_config.path, self.env_name) diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 4e1645e282..d1fe5e7d3d 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -14,6 +14,7 @@ import asyncio import os +import re import time from platformio import fs @@ -120,7 +121,13 @@ def _check_ready_by_pattern(self, data): return self._ready ready_pattern = self.debug_config.server_ready_pattern if ready_pattern: - self._ready = ready_pattern.encode() in data + if ready_pattern.startswith("^"): + self._ready = re.match( + ready_pattern, + data.decode("utf-8", "ignore"), + ) + else: + self._ready = ready_pattern.encode() in data return self._ready def stdout_data_received(self, data): diff --git a/platformio/project/options.py b/platformio/project/options.py index 213a2104ef..480ab4ace6 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -695,6 +695,14 @@ def ConfigEnvOption(*args, **kwargs): ), type=click.Path(exists=True, file_okay=True, dir_okay=False), ), + ConfigEnvOption( + group="debug", + name="debug_server_ready_pattern", + description=( + "A pattern to determine when debugging server is ready " + "for an incoming connection" + ), + ), # Advanced ConfigEnvOption( group="advanced", From 9ede20a367d379019528f95ef80e44ac209da9e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 13:10:29 +0200 Subject: [PATCH 009/120] Disable checking for "__PLATFORMIO_BUILD_DEBUG__" that is not available in g2 mode --- platformio/debug/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index a412ad5ee8..fc6e0116f8 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -92,7 +92,7 @@ def has_debug_symbols(prog_path): b".debug_abbrev": False, b" -Og": False, b" -g": False, - b"__PLATFORMIO_BUILD_DEBUG__": False, + # b"__PLATFORMIO_BUILD_DEBUG__": False, } with open(prog_path, "rb") as fp: last_data = b"" From eebdf0435732d240a10b945a972e91839273c013 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 13:46:27 +0200 Subject: [PATCH 010/120] Load "idedata" configuration from a dumped file --- platformio/builder/main.py | 9 ++++--- platformio/project/config.py | 2 +- platformio/project/helpers.py | 49 +++++++++++++++++++++++------------ 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 245a0f984f..2fbd0b3775 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -224,10 +224,11 @@ Import("projenv") except: # pylint: disable=bare-except projenv = env - click.echo( - "\n%s\n" - % json.dumps(projenv.DumpIDEData(env)) # pylint: disable=undefined-variable - ) + data = projenv.DumpIDEData(env) + # dump to file for the further reading by project.helpers.load_project_ide_data + with open(projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), "w") as fp: + json.dump(data, fp) + click.echo("\n%s\n" % json.dumps(data)) # pylint: disable=undefined-variable env.Exit(0) if "sizedata" in COMMAND_LINE_TARGETS: diff --git a/platformio/project/config.py b/platformio/project/config.py index 6c5aa7b192..dd75f70197 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -384,7 +384,7 @@ def get_optional_dir(self, name, exists=False): if result is None: return None - project_dir = os.getcwd() + project_dir = os.path.dirname(self.path) # patterns if "$PROJECT_HASH" in result: diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index ec8af4a2e0..7519e34a51 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -135,17 +135,29 @@ def compute_project_checksum(config): return checksum.hexdigest() -def load_project_ide_data(project_dir, env_or_envs): +def load_project_ide_data(project_dir, env_or_envs, cache=False): + assert env_or_envs + env_names = env_or_envs + if not isinstance(env_names, list): + env_names = [env_names] + + result = _load_cached_project_ide_data(project_dir, env_names) if cache else {} + missed_env_names = set(env_names) - set(result.keys()) + if missed_env_names: + result.update(_load_project_ide_data(project_dir, missed_env_names)) + + if not isinstance(env_or_envs, list) and env_or_envs in result: + return result[env_or_envs] + return result or None + + +def _load_project_ide_data(project_dir, env_names): # pylint: disable=import-outside-toplevel from platformio.commands.run.command import cli as cmd_run - assert env_or_envs - envs = env_or_envs - if not isinstance(envs, list): - envs = [envs] args = ["--project-dir", project_dir, "--target", "idedata"] - for env in envs: - args.extend(["-e", env]) + for name in env_names: + args.extend(["-e", name]) result = CliRunner().invoke(cmd_run, args) if result.exit_code != 0 and not isinstance( result.exception, exception.ReturnErrorCode @@ -153,14 +165,17 @@ def load_project_ide_data(project_dir, env_or_envs): raise result.exception if '"includes":' not in result.output: raise exception.PlatformioException(result.output) + return _load_cached_project_ide_data(project_dir, env_names) - data = {} - for line in result.output.split("\n"): - line = line.strip() - if line.startswith('{"') and line.endswith("}") and "env_name" in line: - _data = json.loads(line) - if "env_name" in _data: - data[_data["env_name"]] = _data - if not isinstance(env_or_envs, list) and env_or_envs in data: - return data[env_or_envs] - return data or None + +def _load_cached_project_ide_data(project_dir, env_names): + build_dir = ProjectConfig.get_instance( + join(project_dir, "platformio.ini") + ).get_optional_dir("build") + result = {} + for name in env_names: + if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): + continue + with open(os.path.join(build_dir, name, "idedata.json")) as fp: + result[name] = json.load(fp) + return result From 972d183d851684bba0b81d3716a3034ae2527769 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 13:46:54 +0200 Subject: [PATCH 011/120] Use a cached build configuration --- platformio/debug/config/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index d3d07c96d8..4143117d74 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -135,7 +135,9 @@ def server_ready_pattern(self): ) def _load_build_data(self): - data = load_project_ide_data(self.project_config.path, self.env_name) + data = load_project_ide_data( + os.path.dirname(self.project_config.path), self.env_name, cache=True + ) if data: return data raise DebugInvalidOptionsError("Could not load a build configuration") From f5cee5674013da2785db5ceed348f3ad1f2ec1fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 14:09:43 +0200 Subject: [PATCH 012/120] Fix issue when disabling "debug_init_break" did not work --- platformio/commands/debug.py | 5 +++-- platformio/debug/config/base.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 7abb197656..44e3302aaf 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -120,8 +120,9 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro rebuild_prog = helpers.is_prog_obsolete( debug_config.program_path ) or not helpers.has_debug_symbols(debug_config.program_path) - else: - rebuild_prog = not os.path.isfile(debug_config.program_path) + + if not (debug_config.program_path and os.path.isfile(debug_config.program_path)): + rebuild_prog = True if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 4143117d74..13fd804f8e 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -88,7 +88,12 @@ def load_mode(self): @property def init_break(self): - result = self.env_options.get("debug_init_break") + missed = object() + result = self.env_options.get("debug_init_break", missed) + if result != missed: + return result + else: + result = None if not result: result = self.tool_settings.get("init_break") return result or ProjectOptions["env.debug_init_break"].default @@ -196,13 +201,14 @@ def get_init_script(self, debugger): raise NotImplementedError def reveal_patterns(self, source, recursive=True): + program_path = self.program_path or "" patterns = { "PLATFORMIO_CORE_DIR": get_project_core_dir(), "PYTHONEXE": proc.get_pythonexe_path(), "PROJECT_DIR": self.project_config.path, - "PROG_PATH": self.program_path, - "PROG_DIR": os.path.dirname(self.program_path), - "PROG_NAME": os.path.basename(os.path.splitext(self.program_path)[0]), + "PROG_PATH": program_path, + "PROG_DIR": os.path.dirname(program_path), + "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]), "DEBUG_PORT": self.port, "UPLOAD_PROTOCOL": self.upload_protocol, "INIT_BREAK": self.init_break or "", From 9cca8f3f558c63f489e3e57a2dd73d449b26ca13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 15:47:20 +0200 Subject: [PATCH 013/120] Split debugging client to base and GDB // Resolve #3757 --- platformio/commands/debug.py | 5 +- platformio/debug/config/base.py | 3 +- platformio/debug/process/client.py | 190 +++-------------------------- platformio/debug/process/gdb.py | 187 ++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 177 deletions(-) create mode 100644 platformio/debug/process/gdb.py diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 44e3302aaf..c2df11eec4 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -27,7 +27,7 @@ from platformio.debug import helpers from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError -from platformio.debug.process.client import DebugClientProcess +from platformio.debug.process.gdb import GDBClientProcess from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -153,9 +153,10 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) - client = DebugClientProcess(project_dir, debug_config) + client = GDBClientProcess(project_dir, debug_config) coro = client.run(__unprocessed) loop.run_until_complete(coro) + del client if IS_WINDOWS: # an issue with asyncio executor and STIDIN, it cannot be closed gracefully proc.force_exit() diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 13fd804f8e..f1a6ed1268 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -92,8 +92,7 @@ def init_break(self): result = self.env_options.get("debug_init_break", missed) if result != missed: return result - else: - result = None + result = None if not result: result = self.tool_settings.get("init_break") return result or ProjectOptions["env.debug_init_break"].default diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py index 7192472234..6a6f1b9e71 100644 --- a/platformio/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -14,125 +14,45 @@ import hashlib import os -import re import signal import tempfile -import time -from platformio import fs, proc, telemetry +from platformio import fs, proc from platformio.cache import ContentCache -from platformio.compat import ( - IS_WINDOWS, - aio_get_running_loop, - hashlib_encode_data, - is_bytes, -) -from platformio.debug import helpers +from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.debug.process.base import DebugBaseProcess from platformio.debug.process.server import DebugServerProcess from platformio.project.helpers import get_project_cache_dir -class DebugClientProcess( - DebugBaseProcess -): # pylint: disable=too-many-instance-attributes - - PIO_SRC_NAME = ".pioinit" - INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" - +class DebugClientProcess(DebugBaseProcess): def __init__(self, project_dir, debug_config): super(DebugClientProcess, self).__init__() self.project_dir = project_dir self.debug_config = debug_config - self._server_process = DebugServerProcess(debug_config) + self._server_process = None self._session_id = None if not os.path.isdir(get_project_cache_dir()): os.makedirs(get_project_cache_dir()) - self._gdbsrc_dir = tempfile.mkdtemp( + self.working_dir = tempfile.mkdtemp( dir=get_project_cache_dir(), prefix=".piodebug-" ) self._target_is_running = False self._errors_buffer = b"" - async def run(self, extra_args): - gdb_path = self.debug_config.client_executable_path - session_hash = gdb_path + self.debug_config.program_path + async def run(self): + session_hash = ( + self.debug_config.client_executable_path + self.debug_config.program_path + ) self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() - self.debug_config.port = await self._server_process.run() - self.generate_init_script(os.path.join(self._gdbsrc_dir, self.PIO_SRC_NAME)) - - # start GDB client - args = [ - gdb_path, - "-q", - "--directory", - self._gdbsrc_dir, - "--directory", - self.project_dir, - "-l", - "10", - ] - args.extend(list(extra_args or [])) - gdb_data_dir = self._get_data_dir(gdb_path) - if gdb_data_dir: - args.extend(["--data-directory", gdb_data_dir]) - args.append(self.debug_config.program_path) - await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) - - @staticmethod - def _get_data_dir(gdb_path): - if "msp430" in gdb_path: - return None - gdb_data_dir = os.path.realpath( - os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") - ) - return gdb_data_dir if os.path.isdir(gdb_data_dir) else None - - def generate_init_script(self, dst): - # default GDB init commands depending on debug tool - commands = self.debug_config.get_init_script("gdb").split("\n") - - if self.debug_config.init_cmds: - commands = self.debug_config.init_cmds - commands.extend(self.debug_config.extra_cmds) - if not any("define pio_reset_run_target" in cmd for cmd in commands): - commands = [ - "define pio_reset_run_target", - " echo Warning! Undefined pio_reset_run_target command\\n", - " monitor reset", - "end", - ] + commands - if not any("define pio_reset_halt_target" in cmd for cmd in commands): - commands = [ - "define pio_reset_halt_target", - " echo Warning! Undefined pio_reset_halt_target command\\n", - " monitor reset halt", - "end", - ] + commands - if not any("define pio_restart_target" in cmd for cmd in commands): - commands += [ - "define pio_restart_target", - " pio_reset_halt_target", - " $INIT_BREAK", - " %s" % ("continue" if self.debug_config.init_break else "next"), - "end", - ] - - banner = [ - "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", - "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, - "echo PlatformIO: Initializing remote target...\\n", - ] - footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] - commands = banner + commands + footer - - with open(dst, "w") as fp: - fp.write("\n".join(self.debug_config.reveal_patterns(commands))) + if self.debug_config.server: + self._server_process = DebugServerProcess(self.debug_config) + self.debug_config.port = await self._server_process.run() def connection_made(self, transport): super(DebugClientProcess, self).connection_made(transport) @@ -141,88 +61,7 @@ def connection_made(self, transport): signal.signal(signal.SIGINT, lambda *args, **kwargs: None) self.connect_stdin_pipe() - def stdin_data_received(self, data): - super(DebugClientProcess, self).stdin_data_received(data) - if b"-exec-run" in data: - if self._target_is_running: - token, _ = data.split(b"-", 1) - self.stdout_data_received(token + b"^running\n") - return - data = data.replace(b"-exec-run", b"-exec-continue") - - if b"-exec-continue" in data: - self._target_is_running = True - if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): - # Allow terminating via SIGINT/CTRL+C - signal.signal(signal.SIGINT, signal.default_int_handler) - self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n") - self.transport.get_pipe_transport(0).write(data) - - def stdout_data_received(self, data): - super(DebugClientProcess, self).stdout_data_received(data) - self._handle_error(data) - # go to init break automatically - if self.INIT_COMPLETED_BANNER.encode() in data: - telemetry.send_event( - "Debug", - "Started", - telemetry.dump_run_environment(self.debug_config.env_options), - ) - self._auto_exec_continue() - - def console_log(self, msg): - if helpers.is_gdbmi_mode(): - msg = helpers.escape_gdbmi_stream("~", msg) - self.stdout_data_received(msg if is_bytes(msg) else msg.encode()) - - def _auto_exec_continue(self): - auto_exec_delay = 0.5 # in seconds - if self._last_activity > (time.time() - auto_exec_delay): - aio_get_running_loop().call_later(0.1, self._auto_exec_continue) - return - - if not self.debug_config.init_break or self._target_is_running: - return - - self.console_log( - "PlatformIO: Resume the execution to `debug_init_break = %s`\n" - % self.debug_config.init_break - ) - self.console_log( - "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" - ) - self.transport.get_pipe_transport(0).write( - b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" - ) - self._target_is_running = True - - def stderr_data_received(self, data): - super(DebugClientProcess, self).stderr_data_received(data) - self._handle_error(data) - - def _handle_error(self, data): - self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes - if not ( - self.PIO_SRC_NAME.encode() in self._errors_buffer - and b"Error in sourced" in self._errors_buffer - ): - return - - last_erros = self._errors_buffer.decode() - last_erros = " ".join(reversed(last_erros.split("\n"))) - last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) - - err = "%s -> %s" % ( - telemetry.dump_run_environment(self.debug_config.env_options), - last_erros, - ) - telemetry.send_exception("DebugInitError: %s" % err) - self.transport.close() - def process_exited(self): - self._unlock_session() - if self._gdbsrc_dir and os.path.isdir(self._gdbsrc_dir): - fs.rmtree(self._gdbsrc_dir) if self._server_process: self._server_process.terminate() super(DebugClientProcess, self).process_exited() @@ -255,3 +94,8 @@ def _unlock_session(self): return with ContentCache() as cc: cc.delete(self._session_id) + + def __del__(self): + self._unlock_session() + if self.working_dir and os.path.isdir(self.working_dir): + fs.rmtree(self.working_dir) diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py new file mode 100644 index 0000000000..6e7e2d40a8 --- /dev/null +++ b/platformio/debug/process/gdb.py @@ -0,0 +1,187 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import signal +import time + +from platformio import telemetry +from platformio.compat import aio_get_running_loop, is_bytes +from platformio.debug import helpers +from platformio.debug.process.client import DebugClientProcess + + +class GDBClientProcess(DebugClientProcess): + + PIO_SRC_NAME = ".pioinit" + INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" + + def __init__(self, *args, **kwargs): + super(GDBClientProcess, self).__init__(*args, **kwargs) + self._target_is_running = False + self._errors_buffer = b"" + + async def run(self, extra_args): # pylint: disable=arguments-differ + await super(GDBClientProcess, self).run() + + self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME)) + gdb_path = self.debug_config.client_executable_path + # start GDB client + args = [ + gdb_path, + "-q", + "--directory", + self.working_dir, + "--directory", + self.project_dir, + "-l", + "10", + ] + args.extend(list(extra_args or [])) + gdb_data_dir = self._get_data_dir(gdb_path) + if gdb_data_dir: + args.extend(["--data-directory", gdb_data_dir]) + args.append(self.debug_config.program_path) + + await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) + + @staticmethod + def _get_data_dir(gdb_path): + if "msp430" in gdb_path: + return None + gdb_data_dir = os.path.realpath( + os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") + ) + return gdb_data_dir if os.path.isdir(gdb_data_dir) else None + + def generate_init_script(self, dst): + # default GDB init commands depending on debug tool + commands = self.debug_config.get_init_script("gdb").split("\n") + + if self.debug_config.init_cmds: + commands = self.debug_config.init_cmds + commands.extend(self.debug_config.extra_cmds) + + if not any("define pio_reset_run_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_run_target", + " echo Warning! Undefined pio_reset_run_target command\\n", + " monitor reset", + "end", + ] + commands + if not any("define pio_reset_halt_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_halt_target", + " echo Warning! Undefined pio_reset_halt_target command\\n", + " monitor reset halt", + "end", + ] + commands + if not any("define pio_restart_target" in cmd for cmd in commands): + commands += [ + "define pio_restart_target", + " pio_reset_halt_target", + " $INIT_BREAK", + " %s" % ("continue" if self.debug_config.init_break else "next"), + "end", + ] + + banner = [ + "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", + "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, + "echo PlatformIO: Initializing remote target...\\n", + ] + footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] + commands = banner + commands + footer + + with open(dst, "w") as fp: + fp.write("\n".join(self.debug_config.reveal_patterns(commands))) + + def stdin_data_received(self, data): + super(GDBClientProcess, self).stdin_data_received(data) + if b"-exec-run" in data: + if self._target_is_running: + token, _ = data.split(b"-", 1) + self.stdout_data_received(token + b"^running\n") + return + data = data.replace(b"-exec-run", b"-exec-continue") + + if b"-exec-continue" in data: + self._target_is_running = True + if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) + self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n") + self.transport.get_pipe_transport(0).write(data) + + def stdout_data_received(self, data): + super(GDBClientProcess, self).stdout_data_received(data) + self._handle_error(data) + # go to init break automatically + if self.INIT_COMPLETED_BANNER.encode() in data: + telemetry.send_event( + "Debug", + "Started", + telemetry.dump_run_environment(self.debug_config.env_options), + ) + self._auto_exec_continue() + + def console_log(self, msg): + if helpers.is_gdbmi_mode(): + msg = helpers.escape_gdbmi_stream("~", msg) + self.stdout_data_received(msg if is_bytes(msg) else msg.encode()) + + def _auto_exec_continue(self): + auto_exec_delay = 0.5 # in seconds + if self._last_activity > (time.time() - auto_exec_delay): + aio_get_running_loop().call_later(0.1, self._auto_exec_continue) + return + + if not self.debug_config.init_break or self._target_is_running: + return + + self.console_log( + "PlatformIO: Resume the execution to `debug_init_break = %s`\n" + % self.debug_config.init_break + ) + self.console_log( + "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" + ) + self.transport.get_pipe_transport(0).write( + b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" + ) + self._target_is_running = True + + def stderr_data_received(self, data): + super(GDBClientProcess, self).stderr_data_received(data) + self._handle_error(data) + + def _handle_error(self, data): + self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes + if not ( + self.PIO_SRC_NAME.encode() in self._errors_buffer + and b"Error in sourced" in self._errors_buffer + ): + return + + last_erros = self._errors_buffer.decode() + last_erros = " ".join(reversed(last_erros.split("\n"))) + last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) + + err = "%s -> %s" % ( + telemetry.dump_run_environment(self.debug_config.env_options), + last_erros, + ) + telemetry.send_exception("DebugInitError: %s" % err) + self.transport.close() From c14b298cb91ceb12ecb0aeac5a8264b6f0f3f7d6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 15:55:42 +0200 Subject: [PATCH 014/120] Fixed an issue with silent hanging when a custom debug server is not found // Resolve #3756 --- HISTORY.rst | 1 + platformio/commands/debug.py | 14 ++++++++------ platformio/debug/process/server.py | 7 +++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 59edd6de48..141fba3e42 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 5 - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option + - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index c2df11eec4..3f468de99a 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -155,11 +155,13 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro asyncio.set_event_loop(loop) client = GDBClientProcess(project_dir, debug_config) coro = client.run(__unprocessed) - loop.run_until_complete(coro) - del client - if IS_WINDOWS: - # an issue with asyncio executor and STIDIN, it cannot be closed gracefully - proc.force_exit() - loop.close() + try: + loop.run_until_complete(coro) + if IS_WINDOWS: + # an issue with asyncio executor and STIDIN, it cannot be closed gracefully + proc.force_exit() + finally: + del client + loop.close() return True diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index d1fe5e7d3d..1fb08f4d2b 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -51,10 +51,9 @@ async def run(self): # pylint: disable=too-many-branches server_executable = where_is_program(server_executable) if not os.path.isfile(server_executable): raise DebugInvalidOptionsError( - "\nCould not launch Debug Server '%s'. Please check that it " - "is installed and is included in a system PATH\n\n" - "See documentation:\n" - "https://docs.platformio.org/page/plus/debugging.html\n" + "Could not launch Debug Server '%s'. Please check that it " + "is installed and is included in a system PATH\n" + "See https://docs.platformio.org/page/plus/debugging.html" % server_executable ) From a326b718f2538df7f2dd77d8af6abd9dc90f1581 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 16:09:38 +0200 Subject: [PATCH 015/120] Handle legacy $LOAD_CMD "init_cmds" --- platformio/debug/config/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index f1a6ed1268..8a8e52477e 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -99,8 +99,8 @@ def init_break(self): @property def init_cmds(self): - return self.env_options.get( - "debug_init_cmds", self.tool_settings.get("init_cmds") + return self.cleanup_cmds( + self.env_options.get("debug_init_cmds", self.tool_settings.get("init_cmds")) ) @property From 887d46725b637b4212e08d19c921a0aefc71f96a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 17:02:11 +0200 Subject: [PATCH 016/120] Debug native (desktop) application on a host machine // Resolve #980 --- HISTORY.rst | 1 + platformio/commands/debug.py | 2 +- platformio/debug/config/base.py | 21 +++++++++++++------- platformio/debug/config/factory.py | 13 +++++++++--- platformio/debug/config/native.py | 32 ++++++++++++++++++++++++++++++ platformio/debug/process/gdb.py | 16 ++++++++++----- 6 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 platformio/debug/config/native.py diff --git a/HISTORY.rst b/HISTORY.rst index 141fba3e42..e626c5caa8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 * **PlatformIO Debugging** - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - Debug native (desktop) application on a host machine (`issue #980 `_) - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 3f468de99a..c30fdc096b 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -79,7 +79,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) env_options = project_config.items(env=env_name, as_dict=True) - if not set(env_options.keys()) >= set(["platform", "board"]): + if "platform" not in env_options: raise ProjectEnvsNotAvailableError() try: diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 8a8e52477e..53abadc98f 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -29,14 +29,21 @@ def __init__(self, platform, project_config, env_name): self.project_config = project_config self.env_name = env_name self.env_options = project_config.items(env=env_name, as_dict=True) - self.board_config = platform.board_config(self.env_options["board"]) self.build_data = self._load_build_data() - self.tool_name = self.board_config.get_debug_tool_name( - self.env_options.get("debug_tool") - ) - self.tool_settings = ( - self.board_config.get("debug", {}).get("tools", {}).get(self.tool_name, {}) - ) + + self.tool_name = None + self.board_config = {} + self.tool_settings = {} + if "board" in self.env_options: + self.board_config = platform.board_config(self.env_options["board"]) + self.tool_name = self.board_config.get_debug_tool_name( + self.env_options.get("debug_tool") + ) + self.tool_settings = ( + self.board_config.get("debug", {}) + .get("tools", {}) + .get(self.tool_name, {}) + ) self._load_cmds = None self._port = None diff --git a/platformio/debug/config/factory.py b/platformio/debug/config/factory.py index 87019dfa75..d74dad3822 100644 --- a/platformio/debug/config/factory.py +++ b/platformio/debug/config/factory.py @@ -16,6 +16,7 @@ import re from platformio.debug.config.generic import GenericDebugConfig +from platformio.debug.config.native import NativeDebugConfig class DebugConfigFactory(object): @@ -29,13 +30,19 @@ def new(cls, platform, project_config, env_name): board_config = platform.board_config( project_config.get("env:" + env_name, "board") ) - tool_name = board_config.get_debug_tool_name( - project_config.get("env:" + env_name, "debug_tool") + tool_name = ( + board_config.get_debug_tool_name( + project_config.get("env:" + env_name, "debug_tool") + ) + if board_config + else None ) config_cls = None try: mod = importlib.import_module("platformio.debug.config.%s" % tool_name) config_cls = getattr(mod, cls.get_clsname(tool_name)) except ModuleNotFoundError: - config_cls = GenericDebugConfig + config_cls = ( + GenericDebugConfig if platform.is_embedded() else NativeDebugConfig + ) return config_cls(platform, project_config, env_name) diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py new file mode 100644 index 0000000000..c406fd590e --- /dev/null +++ b/platformio/debug/config/native.py @@ -0,0 +1,32 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class NativeDebugConfig(DebugConfigBase): + + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target +end + +define pio_reset_run_target +end + +define pio_restart_target +end + +$INIT_BREAK +set startup-with-shell off +""" diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 6e7e2d40a8..836c637f39 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -37,7 +37,7 @@ async def run(self, extra_args): # pylint: disable=arguments-differ await super(GDBClientProcess, self).run() self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME)) - gdb_path = self.debug_config.client_executable_path + gdb_path = self.debug_config.client_executable_path or "gdb" # start GDB client args = [ gdb_path, @@ -115,7 +115,8 @@ def stdin_data_received(self, data): token, _ = data.split(b"-", 1) self.stdout_data_received(token + b"^running\n") return - data = data.replace(b"-exec-run", b"-exec-continue") + if self.debug_config.platform.is_embedded(): + data = data.replace(b"-exec-run", b"-exec-continue") if b"-exec-continue" in data: self._target_is_running = True @@ -158,9 +159,14 @@ def _auto_exec_continue(self): self.console_log( "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n" ) - self.transport.get_pipe_transport(0).write( - b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" - ) + if self.debug_config.platform.is_embedded(): + self.transport.get_pipe_transport(0).write( + b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" + ) + else: + self.transport.get_pipe_transport(0).write( + b"0-exec-run\n" if helpers.is_gdbmi_mode() else b"run\n" + ) self._target_is_running = True def stderr_data_received(self, data): From ebe5785a917d12613910dc2f572c1f854582b64a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 17:11:25 +0200 Subject: [PATCH 017/120] Allow overriding default debugging flags from dev-platform --- platformio/builder/tools/piomisc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index dbc39012c2..3551c62b23 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -334,7 +334,13 @@ def _cleanup_debug_flags(scope): for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): _cleanup_debug_flags(scope) - debug_flags = env.ParseFlags(env.GetProjectOption("debug_build_flags")) + debug_flags = env.ParseFlags( + env.get("PIODEBUGFLAGS") + if env.get("PIODEBUGFLAGS") + and not env.GetProjectOptions(as_dict=True).get("debug_build_flags") + else env.GetProjectOption("debug_build_flags") + ) + env.MergeFlags(debug_flags) optimization_flags = [ f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g")) From a366d1af2a26a9e91d52fbbcb2103420e2501653 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 18:26:09 +0200 Subject: [PATCH 018/120] Use "target remote" for mpsdebug --- platformio/debug/config/mspdebug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 6c9b9412ef..e71b09caf0 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -24,7 +24,7 @@ class MspdebugDebugConfig(DebugConfigBase): define pio_reset_run_target end -target extended-remote $DEBUG_PORT +target remote $DEBUG_PORT monitor erase $LOAD_CMDS pio_reset_halt_target From 34b4f8265ae2c54d6d6b258eb8a53dc40ba01cba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 20:25:30 +0200 Subject: [PATCH 019/120] Debug unit tests created with PlatformIO Unit Testing solution // Resolve #948 --- HISTORY.rst | 3 +- docs | 2 +- platformio/builder/main.py | 2 +- platformio/builder/tools/platformio.py | 2 +- platformio/commands/debug.py | 12 +++++-- platformio/commands/test/command.py | 41 ++++++++--------------- platformio/commands/test/helpers.py | 30 +++++++++++++++++ platformio/commands/test/processor.py | 4 +-- platformio/debug/helpers.py | 46 +++++++++++++++++++++----- platformio/project/options.py | 5 +++ 10 files changed, 102 insertions(+), 45 deletions(-) create mode 100644 platformio/commands/test/helpers.py diff --git a/HISTORY.rst b/HISTORY.rst index e626c5caa8..3ea37b5862 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,7 +14,8 @@ PlatformIO Core 5 * **PlatformIO Debugging** - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - - Debug native (desktop) application on a host machine (`issue #980 `_) + - `Debug unit tests `__ created with `PlatformIO Unit Testing `__ solution (`issue #948 `_) + - Debug native (desktop) applications on a host machine (`issue #980 `_) - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) diff --git a/docs b/docs index b76e3d53bb..b502c2a8dd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b76e3d53bbe10c945e1dbe302fe7f0b07005447a +Subproject commit b502c2a8dd6667e669f4c36a73007a89edf94af2 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 2fbd0b3775..dcbd480054 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -210,7 +210,7 @@ ), ) -AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS)) +AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c412012efd..544e1de8dd 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -50,7 +50,7 @@ def GetBuildType(env): return ( "debug" if ( - set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) + set(["__debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) or env.GetProjectOption("build_type") == "debug" ) else "release" diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index c30fdc096b..661ad7e1b4 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -76,7 +76,9 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro env_name = environment or helpers.get_default_debug_env(project_config) if not interface: - return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) + return helpers.predebug_project( + ctx, project_dir, project_config, env_name, False, verbose + ) env_options = project_config.items(env=env_name, as_dict=True) if "platform" not in env_options: @@ -138,11 +140,15 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro ) stream = helpers.GDBMIConsoleStream() with proc.capture_std_streams(stream): - helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) + helpers.predebug_project( + ctx, project_dir, project_config, env_name, preload, verbose + ) stream.close() else: click.echo("Preparing firmware for debugging...") - helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) + helpers.predebug_project( + ctx, project_dir, project_config, env_name, preload, verbose + ) # save SHA sum of newly created prog if load_mode == "modified": diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 07f95226e0..40780ab454 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -14,9 +14,8 @@ # pylint: disable=too-many-arguments, too-many-locals, too-many-branches -from fnmatch import fnmatch -from os import getcwd, listdir -from os.path import isdir, join +import fnmatch +import os from time import time import click @@ -24,6 +23,7 @@ from platformio import app, exception, fs, util from platformio.commands.test.embedded import EmbeddedTestProcessor +from platformio.commands.test.helpers import get_test_names from platformio.commands.test.native import NativeTestProcessor from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -50,7 +50,7 @@ @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True ), @@ -102,11 +102,7 @@ def cli( # pylint: disable=redefined-builtin with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(envs=environment) - - test_dir = config.get_optional_dir("test") - if not isdir(test_dir): - raise exception.TestDirNotExists(test_dir) - test_names = get_test_names(test_dir) + test_names = get_test_names(config) if not verbose: click.echo("Verbose mode can be enabled via `-v, --verbose` option") @@ -129,9 +125,11 @@ def cli( # pylint: disable=redefined-builtin not environment and default_envs and envname not in default_envs, testname != "*" and patterns["filter"] - and not any(fnmatch(testname, p) for p in patterns["filter"]), + and not any( + fnmatch.fnmatch(testname, p) for p in patterns["filter"] + ), testname != "*" - and any(fnmatch(testname, p) for p in patterns["ignore"]), + and any(fnmatch.fnmatch(testname, p) for p in patterns["ignore"]), ] if any(skip_conditions): results.append({"env": envname, "test": testname}) @@ -142,7 +140,10 @@ def cli( # pylint: disable=redefined-builtin cls = ( EmbeddedTestProcessor - if is_embedded_platform(config.get(section, "platform")) + if config.get(section, "platform") + and PlatformFactory.new( + config.get(section, "platform") + ).is_embedded() else NativeTestProcessor ) tp = cls( @@ -185,22 +186,6 @@ def cli( # pylint: disable=redefined-builtin raise exception.ReturnErrorCode(1) -def get_test_names(test_dir): - names = [] - for item in sorted(listdir(test_dir)): - if isdir(join(test_dir, item)): - names.append(item) - if not names: - names = ["*"] - return names - - -def is_embedded_platform(name): - if not name: - return False - return PlatformFactory.new(name).is_embedded() - - def print_processing_header(test, env): click.echo( "Processing %s in %s environment" diff --git a/platformio/commands/test/helpers.py b/platformio/commands/test/helpers.py new file mode 100644 index 0000000000..ce72360fd3 --- /dev/null +++ b/platformio/commands/test/helpers.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import exception + + +def get_test_names(config): + test_dir = config.get_optional_dir("test") + if not os.path.isdir(test_dir): + raise exception.TestDirNotExists(test_dir) + names = [] + for item in sorted(os.listdir(test_dir)): + if os.path.isdir(os.path.join(test_dir, item)): + names.append(item) + if not names: + names = ["*"] + return names diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index de09b5f9a4..ea975e6260 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -139,9 +139,9 @@ def build_or_upload(self, target): cmd_run, project_dir=self.options["project_dir"], project_conf=self.options["project_config"].path, - upload_port=self.options["upload_port"], + upload_port=self.options.get("upload_port"), verbose=self.options["verbose"], - silent=self.options["silent"], + silent=self.options.get("silent"), environment=[self.env_name], disable_auto_clean="nobuild" in target, target=target, diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index fc6e0116f8..d2d2716c57 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -23,6 +23,9 @@ from platformio import util from platformio.commands import PlatformioCLI from platformio.commands.run.command import cli as cmd_run +from platformio.commands.run.command import print_processing_header +from platformio.commands.test.helpers import get_test_names +from platformio.commands.test.processor import TestProcessorBase from platformio.compat import IS_WINDOWS, is_bytes from platformio.debug.exception import DebugInvalidOptionsError @@ -72,14 +75,41 @@ def get_default_debug_env(config): return default_envs[0] if default_envs else all_envs[0] -def predebug_project(ctx, project_dir, env_name, preload, verbose): - ctx.invoke( - cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["debug"] + (["upload"] if preload else []), - verbose=verbose, - ) +def predebug_project( + ctx, project_dir, project_config, env_name, preload, verbose +): # pylint: disable=too-many-arguments + debug_testname = project_config.get("env:" + env_name, "debug_test") + if debug_testname: + test_names = get_test_names(project_config) + if debug_testname not in test_names: + raise DebugInvalidOptionsError( + "Unknown test name `%s`. Valid names are `%s`" + % (debug_testname, ", ".join(test_names)) + ) + print_processing_header(env_name, project_config, verbose) + tp = TestProcessorBase( + ctx, + debug_testname, + env_name, + dict( + project_config=project_config, + project_dir=project_dir, + without_building=False, + without_uploading=True, + without_testing=True, + verbose=False, + ), + ) + tp.build_or_upload(["__debug", "__test"] + (["upload"] if preload else [])) + else: + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["__debug"] + (["upload"] if preload else []), + verbose=verbose, + ) + if preload: time.sleep(5) diff --git a/platformio/project/options.py b/platformio/project/options.py index 480ab4ace6..3c9db96cc8 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -703,6 +703,11 @@ def ConfigEnvOption(*args, **kwargs): "for an incoming connection" ), ), + ConfigEnvOption( + group="debug", + name="debug_test", + description=("A name of a unit test to be debugged"), + ), # Advanced ConfigEnvOption( group="advanced", From f543e003079bdba8fb61a17d0261beb56f51c2f3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Mar 2021 20:26:26 +0200 Subject: [PATCH 020/120] Bump version to 5.2.0a2 --- platformio/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index be6578ed4d..fcd0ef2567 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a1") +VERSION = (5, 2, "0a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -50,7 +50,7 @@ "contrib-piohome": "~3.3.4", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.2", + "tool-scons": "~4.40100.2", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", From 990071af5c536424c04b81ea9ded3741d8f55080 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 20 Mar 2021 10:31:55 +0200 Subject: [PATCH 021/120] Fix issue with missed compat.path_to_unicode // Resolve #3894 --- platformio/compat.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platformio/compat.py b/platformio/compat.py index 5c20df3a5a..f3f79ea689 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -97,3 +97,11 @@ def ensure_python3(raise_exception=True): "https://docs.platformio.org/en/latest/core/migration.html" "#drop-support-for-python-2-and-3-5" ) + + +def path_to_unicode(path): + """ + Deprecated: Compatibility with dev-platforms, + and custom device monitor filters + """ + return path From 1542b1cebb849ef22af6de8b446da6f0d2ff2d0e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 20 Mar 2021 10:32:14 +0200 Subject: [PATCH 022/120] Bump version to 5.2.0a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index fcd0ef2567..26440999fe 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a2") +VERSION = (5, 2, "0a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 86db237e5daef79ce468daf7c5a6d0e83026d6bd Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 23 Mar 2021 21:17:32 +0200 Subject: [PATCH 023/120] Update Cppcheck and PVS-Studio packages // Resolve #3898 --- HISTORY.rst | 7 +++++++ platformio/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3ea37b5862..3f602892b5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,13 @@ PlatformIO Core 5 - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) +* **Static Code Analysis** + + - Updated analysis tools: + + * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements + * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards + 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 26440999fe..da045cf5c8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -51,9 +51,9 @@ "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~4.40100.2", - "tool-cppcheck": "~1.230.0", + "tool-cppcheck": "~1.241.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.11.0", + "tool-pvs-studio": "~7.12.0", } __check_internet_hosts__ = [ From 0230374709d3b6925ea5fa07a038fa594591174f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 24 Mar 2021 13:04:20 +0200 Subject: [PATCH 024/120] Document new VSCode settings: activateProjectOnTextEditorChange & autoOpenPlatformIOIniFile --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b502c2a8dd..32df8a73fb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b502c2a8dd6667e669f4c36a73007a89edf94af2 +Subproject commit 32df8a73fb1a1ceb8142a43d68c7ff560f4bdf7d From 37e601e5b57286c2c67c25dc61ab61314cb52d1d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 24 Mar 2021 19:07:40 +0200 Subject: [PATCH 025/120] Ensure that a serial port is ready before running unit tests on a remote target // Resolve #3742 --- HISTORY.rst | 4 ++++ platformio/commands/test/embedded.py | 22 +++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3f602892b5..3088ddd97d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,10 @@ PlatformIO Core 5 * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards +* **Miscellaneous** + + - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) + 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 02bf675bf7..d0b533909b 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -117,13 +117,10 @@ def get_test_port(self): port = item["port"] for hwid in board_hwids: hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") - if hwid_str in item["hwid"]: + if hwid_str in item["hwid"] and self.is_serial_port_ready(port): return port - # check if port is already configured - try: - serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close() - except serial.SerialException: + if port and not self.is_serial_port_ready(port): port = None if not port: @@ -136,3 +133,18 @@ def get_test_port(self): "global `--test-port` option." ) return port + + @staticmethod + def is_serial_port_ready(port, timeout=3): + if not port: + return False + elapsed = 0 + while elapsed < timeout: + try: + serial.Serial(port, timeout=1).close() + return True + except: # pylint: disable=bare-except + pass + sleep(1) + elapsed += 1 + return False From 940b25f158a92b13501d24c220f5b3a443259863 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 Mar 2021 17:32:57 +0300 Subject: [PATCH 026/120] Sync docs & examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 32df8a73fb..76504f870c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 32df8a73fb1a1ceb8142a43d68c7ff560f4bdf7d +Subproject commit 76504f870c50ab4da0ac92c28496b539e341e8c8 diff --git a/examples b/examples index a0631a8b07..84bc1a967a 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit a0631a8b07f01de96eb5e0431798b41c7ab8f23e +Subproject commit 84bc1a967ab44dcfdeeb58c501739ca3a2bf87e3 From 35397248438ede2ba0032230be068652bbba2675 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 Mar 2021 17:33:26 +0300 Subject: [PATCH 027/120] Update "zeroconf" dependency to 0.29 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2434bcc8c0..af921b7666 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.28.*") + minimal_requirements.append("zeroconf==0.29.*") home_requirements = [ "aiofiles==0.6.*", From 32e1cbe2a35d32d36ac21255337dbd21fa2005c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 Mar 2021 18:28:06 +0300 Subject: [PATCH 028/120] Provide solution for issue #3417 --- platformio/builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index dcbd480054..cac5c8fec4 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -135,7 +135,7 @@ ): click.secho( "There is a known issue with Python 3.8+ and mapped network drives on " - "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n" + "Windows.\nSee a solution at:\n" "https://github.com/platformio/platformio-core/issues/3417", fg="yellow", ) From ee7ea77fc3e7d001c2b7b4e2d8a07436afa10d50 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 1 Apr 2021 21:15:14 +0300 Subject: [PATCH 029/120] Fixed an error "Unknown development platform" when running unit tests on a clean machine // Resolve #3901 --- HISTORY.rst | 1 + platformio/commands/debug.py | 18 +- platformio/commands/platform.py | 304 +++++++++++++++------------ platformio/commands/run/processor.py | 18 +- platformio/commands/test/command.py | 6 +- 5 files changed, 180 insertions(+), 167 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3088ddd97d..8f6a20cc12 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ PlatformIO Core 5 * **Miscellaneous** - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) + - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 661ad7e1b4..e176bab3e0 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -22,14 +22,12 @@ import click from platformio import app, exception, fs, proc -from platformio.commands.platform import platform_install as cmd_platform_install +from platformio.commands.platform import init_platform from platformio.compat import IS_WINDOWS from platformio.debug import helpers from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.gdb import GDBClientProcess -from platformio.platform.exception import UnknownPlatform -from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project @@ -84,17 +82,9 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro if "platform" not in env_options: raise ProjectEnvsNotAvailableError() - try: - platform = PlatformFactory.new(env_options["platform"]) - except UnknownPlatform: - ctx.invoke( - cmd_platform_install, - platforms=[env_options["platform"]], - skip_default_package=True, - ) - platform = PlatformFactory.new(env_options["platform"]) - - debug_config = DebugConfigFactory.new(platform, project_config, env_name) + debug_config = DebugConfigFactory.new( + init_platform(env_options["platform"]), project_config, env_name + ) if "--version" in __unprocessed: return subprocess.run( diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index f03f88337c..84945e391c 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -31,139 +31,6 @@ def cli(): pass -def _print_platforms(platforms): - for platform in platforms: - click.echo( - "{name} ~ {title}".format( - name=click.style(platform["name"], fg="cyan"), title=platform["title"] - ) - ) - click.echo("=" * (3 + len(platform["name"] + platform["title"]))) - click.echo(platform["description"]) - click.echo() - if "homepage" in platform: - click.echo("Home: %s" % platform["homepage"]) - if "frameworks" in platform and platform["frameworks"]: - click.echo("Frameworks: %s" % ", ".join(platform["frameworks"])) - if "packages" in platform: - click.echo("Packages: %s" % ", ".join(platform["packages"])) - if "version" in platform: - if "__src_url" in platform: - click.echo( - "Version: %s (%s)" % (platform["version"], platform["__src_url"]) - ) - else: - click.echo("Version: " + platform["version"]) - click.echo() - - -def _get_registry_platforms(): - regclient = PlatformPackageManager().get_registry_client_instance() - return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") - - -def _get_platform_data(*args, **kwargs): - try: - return _get_installed_platform_data(*args, **kwargs) - except UnknownPlatform: - return _get_registry_platform_data(*args, **kwargs) - - -def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): - p = PlatformFactory.new(platform) - data = dict( - name=p.name, - title=p.title, - description=p.description, - version=p.version, - homepage=p.homepage, - url=p.homepage, - repository=p.repository_url, - license=p.license, - forDesktop=not p.is_embedded(), - frameworks=sorted(list(p.frameworks) if p.frameworks else []), - packages=list(p.packages) if p.packages else [], - ) - - # if dump to API - # del data['version'] - # return data - - # overwrite VCS version and add extra fields - manifest = PlatformPackageManager().legacy_load_manifest( - os.path.dirname(p.manifest_path) - ) - assert manifest - for key in manifest: - if key == "version" or key.startswith("__"): - data[key] = manifest[key] - - if with_boards: - data["boards"] = [c.get_brief_data() for c in p.get_boards().values()] - - if not data["packages"] or not expose_packages: - return data - - data["packages"] = [] - installed_pkgs = { - pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() - } - for name, options in p.packages.items(): - item = dict( - name=name, - type=p.get_package_type(name), - requirements=options.get("version"), - optional=options.get("optional") is True, - ) - if name in installed_pkgs: - for key, value in installed_pkgs[name].items(): - if key not in ("url", "version", "description"): - continue - item[key] = value - if key == "version": - item["originalVersion"] = get_original_version(value) - data["packages"].append(item) - - return data - - -def _get_registry_platform_data( # pylint: disable=unused-argument - platform, with_boards=True, expose_packages=True -): - _data = None - for p in _get_registry_platforms(): - if p["name"] == platform: - _data = p - break - - if not _data: - return None - - data = dict( - ownername=_data.get("ownername"), - name=_data["name"], - title=_data["title"], - description=_data["description"], - homepage=_data["homepage"], - repository=_data["repository"], - url=_data["url"], - license=_data["license"], - forDesktop=_data["forDesktop"], - frameworks=_data["frameworks"], - packages=_data["packages"], - versions=_data.get("versions"), - ) - - if with_boards: - data["boards"] = [ - board - for board in PlatformPackageManager().get_registered_boards() - if board["platform"] == _data["name"] - ] - - return data - - @cli.command("search", short_help="Search for development platform") @click.argument("query", required=False) @click.option("--json-output", is_flag=True) @@ -319,13 +186,33 @@ def platform_install( # pylint: disable=too-many-arguments with_all_packages, silent, force, +): + return _platform_install( + platforms, + with_package, + without_package, + skip_default_package, + with_all_packages, + silent, + force, + ) + + +def _platform_install( # pylint: disable=too-many-arguments + platforms, + with_package=None, + without_package=None, + skip_default_package=False, + with_all_packages=False, + silent=False, + force=False, ): pm = PlatformPackageManager() for platform in platforms: pkg = pm.install( spec=platform, - with_packages=with_package, - without_packages=without_package, + with_packages=with_package or [], + without_packages=without_package or [], skip_default_package=skip_default_package, with_all_packages=with_all_packages, silent=silent, @@ -423,3 +310,150 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments click.echo() return True + + +# +# Helpers +# + + +def init_platform(name, skip_default_package=True, auto_install=True): + try: + return PlatformFactory.new(name) + except UnknownPlatform: + if auto_install: + _platform_install([name], skip_default_package=skip_default_package) + return PlatformFactory.new(name) + + +def _print_platforms(platforms): + for platform in platforms: + click.echo( + "{name} ~ {title}".format( + name=click.style(platform["name"], fg="cyan"), title=platform["title"] + ) + ) + click.echo("=" * (3 + len(platform["name"] + platform["title"]))) + click.echo(platform["description"]) + click.echo() + if "homepage" in platform: + click.echo("Home: %s" % platform["homepage"]) + if "frameworks" in platform and platform["frameworks"]: + click.echo("Frameworks: %s" % ", ".join(platform["frameworks"])) + if "packages" in platform: + click.echo("Packages: %s" % ", ".join(platform["packages"])) + if "version" in platform: + if "__src_url" in platform: + click.echo( + "Version: %s (%s)" % (platform["version"], platform["__src_url"]) + ) + else: + click.echo("Version: " + platform["version"]) + click.echo() + + +def _get_registry_platforms(): + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") + + +def _get_platform_data(*args, **kwargs): + try: + return _get_installed_platform_data(*args, **kwargs) + except UnknownPlatform: + return _get_registry_platform_data(*args, **kwargs) + + +def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): + p = PlatformFactory.new(platform) + data = dict( + name=p.name, + title=p.title, + description=p.description, + version=p.version, + homepage=p.homepage, + url=p.homepage, + repository=p.repository_url, + license=p.license, + forDesktop=not p.is_embedded(), + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else [], + ) + + # if dump to API + # del data['version'] + # return data + + # overwrite VCS version and add extra fields + manifest = PlatformPackageManager().legacy_load_manifest( + os.path.dirname(p.manifest_path) + ) + assert manifest + for key in manifest: + if key == "version" or key.startswith("__"): + data[key] = manifest[key] + + if with_boards: + data["boards"] = [c.get_brief_data() for c in p.get_boards().values()] + + if not data["packages"] or not expose_packages: + return data + + data["packages"] = [] + installed_pkgs = { + pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() + } + for name, options in p.packages.items(): + item = dict( + name=name, + type=p.get_package_type(name), + requirements=options.get("version"), + optional=options.get("optional") is True, + ) + if name in installed_pkgs: + for key, value in installed_pkgs[name].items(): + if key not in ("url", "version", "description"): + continue + item[key] = value + if key == "version": + item["originalVersion"] = get_original_version(value) + data["packages"].append(item) + + return data + + +def _get_registry_platform_data( # pylint: disable=unused-argument + platform, with_boards=True, expose_packages=True +): + _data = None + for p in _get_registry_platforms(): + if p["name"] == platform: + _data = p + break + + if not _data: + return None + + data = dict( + ownername=_data.get("ownername"), + name=_data["name"], + title=_data["title"], + description=_data["description"], + homepage=_data["homepage"], + repository=_data["repository"], + url=_data["url"], + license=_data["license"], + forDesktop=_data["forDesktop"], + frameworks=_data["frameworks"], + packages=_data["packages"], + versions=_data.get("versions"), + ) + + if with_boards: + data["boards"] = [ + board + for board in PlatformPackageManager().get_registered_boards() + if board["platform"] == _data["name"] + ] + + return data diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index d07c581ca4..191a071f00 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.commands.platform import platform_install as cmd_platform_install +from platformio.commands.platform import init_platform from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME -from platformio.platform.exception import UnknownPlatform -from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -66,15 +64,7 @@ def process(self): if "monitor" in build_targets: build_targets.remove("monitor") - try: - p = PlatformFactory.new(self.options["platform"]) - except UnknownPlatform: - self.cmd_ctx.invoke( - cmd_platform_install, - platforms=[self.options["platform"]], - skip_default_package=True, - ) - p = PlatformFactory.new(self.options["platform"]) - - result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) + result = init_platform(self.options["platform"]).run( + build_vars, build_targets, self.silent, self.verbose, self.jobs + ) return result["returncode"] == 0 diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 40780ab454..bc503093c3 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -22,10 +22,10 @@ from tabulate import tabulate from platformio import app, exception, fs, util +from platformio.commands.platform import init_platform from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.helpers import get_test_names from platformio.commands.test.native import NativeTestProcessor -from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -141,9 +141,7 @@ def cli( # pylint: disable=redefined-builtin cls = ( EmbeddedTestProcessor if config.get(section, "platform") - and PlatformFactory.new( - config.get(section, "platform") - ).is_embedded() + and init_platform(config.get(section, "platform")).is_embedded() else NativeTestProcessor ) tp = cls( From 73d4f10f4bf88dde79148583fbfcf6914810f083 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 1 Apr 2021 21:16:42 +0300 Subject: [PATCH 030/120] Bump version to 5.2.0a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index da045cf5c8..e089e96742 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a3") +VERSION = (5, 2, "0a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 66091bae24425c633d60dabfa1d1ee85869b20cb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 2 Apr 2021 14:44:38 +0300 Subject: [PATCH 031/120] Disable GDB "startup-with-shell" only on Unix platform --- platformio/debug/config/native.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py index c406fd590e..be15b5f46c 100644 --- a/platformio/debug/config/native.py +++ b/platformio/debug/config/native.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from platformio.compat import IS_WINDOWS from platformio.debug.config.base import DebugConfigBase @@ -28,5 +29,6 @@ class NativeDebugConfig(DebugConfigBase): end $INIT_BREAK -set startup-with-shell off -""" +""" + ( + "set startup-with-shell off" if not IS_WINDOWS else "" + ) From 80c24a1993efbbdefd55b67dedcee58fe0d322ee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 2 Apr 2021 15:19:18 +0300 Subject: [PATCH 032/120] Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms // Resolve #3872 --- HISTORY.rst | 1 + .../commands/home/rpc/handlers/project.py | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8f6a20cc12..46510eb382 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,6 +31,7 @@ PlatformIO Core 5 - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 `_) + - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index cb8eda5a7c..8bac4d1c52 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -200,10 +200,10 @@ async def init(self, board, framework, project_dir): await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - return self._generate_project_main(project_dir, framework) + return self._generate_project_main(project_dir, board, framework) @staticmethod - def _generate_project_main(project_dir, framework): + def _generate_project_main(project_dir, board, framework): main_content = None if framework == "arduino": main_content = "\n".join( @@ -238,10 +238,25 @@ def _generate_project_main(project_dir, framework): ) if not main_content: return project_dir + + is_cpp_project = True + pm = PlatformPackageManager() + try: + board = pm.board_config(board) + platforms = board.get("platforms", board.get("platform")) + if not isinstance(platforms, list): + platforms = [platforms] + c_based_platforms = ["intel_mcs51", "ststm8"] + is_cpp_project = not (set(platforms) & set(c_based_platforms)) + except exception.PlatformioException: + pass + with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get_optional_dir("src") - main_path = os.path.join(src_dir, "main.cpp") + main_path = os.path.join( + src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") + ) if os.path.isfile(main_path): return project_dir if not os.path.isdir(src_dir): From 7e9956963a7dfe11762ced858f1aef151970cc75 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 2 Apr 2021 15:23:34 +0300 Subject: [PATCH 033/120] Remove a note with using `pio ci` for uploading // Resolve #3903 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 76504f870c..81ac271bf1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 76504f870c50ab4da0ac92c28496b539e341e8c8 +Subproject commit 81ac271bf18a4fa117f48fb8b00f60ba0416b66f From 551bd3dbfea8c371d443daecadca33c9b60e53cb Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Fri, 2 Apr 2021 17:09:38 +0300 Subject: [PATCH 034/120] Explicitly specify PROGSUFFIX when compiling final binary (#3918) Resolves #3906 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 46510eb382..4acc072cf8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 5 - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) + - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) * **Static Code Analysis** diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 544e1de8dd..0415c79fd2 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -75,7 +75,7 @@ def BuildProgram(env): env.Append(_LIBFLAGS=" -Wl,--end-group") program = env.Program( - os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] + os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")), env["PIOBUILDFILES"] ) env.Replace(PIOMAINPROG=program) From 3823c22dad68fc7428aa90d6c14c56b87efd7597 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 7 Apr 2021 21:30:06 +0300 Subject: [PATCH 035/120] Update Release Notes --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4acc072cf8..10735dbb8f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,7 +19,6 @@ PlatformIO Core 5 - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) - - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) * **Static Code Analysis** @@ -33,6 +32,7 @@ PlatformIO Core 5 - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 `_) - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 `_) + - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ From eecc825c9006f3f52e9e79076501ca7d466c58cc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 11 Apr 2021 22:20:09 +0300 Subject: [PATCH 036/120] PyLint --- platformio/builder/tools/platformio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 0415c79fd2..c3acf5557b 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -75,7 +75,8 @@ def BuildProgram(env): env.Append(_LIBFLAGS=" -Wl,--end-group") program = env.Program( - os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")), env["PIOBUILDFILES"] + os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")), + env["PIOBUILDFILES"], ) env.Replace(PIOMAINPROG=program) From b35c5a22bb62dd11e9d350697e9d0dee761fe284 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 11 Apr 2021 22:21:01 +0300 Subject: [PATCH 037/120] Fix a broken support for custom configuration file for `pio debug` command // Resolve #3922 --- platformio/commands/debug.py | 7 ++++--- platformio/debug/config/base.py | 4 +--- platformio/debug/helpers.py | 1 + platformio/project/config.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index e176bab3e0..46d26d899d 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -82,9 +82,10 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro if "platform" not in env_options: raise ProjectEnvsNotAvailableError() - debug_config = DebugConfigFactory.new( - init_platform(env_options["platform"]), project_config, env_name - ) + with fs.cd(project_dir): + debug_config = DebugConfigFactory.new( + init_platform(env_options["platform"]), project_config, env_name + ) if "--version" in __unprocessed: return subprocess.run( diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 53abadc98f..019f9edbb1 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -146,9 +146,7 @@ def server_ready_pattern(self): ) def _load_build_data(self): - data = load_project_ide_data( - os.path.dirname(self.project_config.path), self.env_name, cache=True - ) + data = load_project_ide_data(os.getcwd(), self.env_name, cache=True) if data: return data raise DebugInvalidOptionsError("Could not load a build configuration") diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index d2d2716c57..6b4d427898 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -105,6 +105,7 @@ def predebug_project( ctx.invoke( cmd_run, project_dir=project_dir, + project_conf=project_config.path, environment=[env_name], target=["__debug"] + (["upload"] if preload else []), verbose=verbose, diff --git a/platformio/project/config.py b/platformio/project/config.py index dd75f70197..6c5aa7b192 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -384,7 +384,7 @@ def get_optional_dir(self, name, exists=False): if result is None: return None - project_dir = os.path.dirname(self.path) + project_dir = os.getcwd() # patterns if "$PROJECT_HASH" in result: From 5bfe70142e74b91e734ae7a6f4cfbe4bb31efbb8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 12 Apr 2021 22:38:21 +0300 Subject: [PATCH 038/120] Switch to project directory before starting debugging process --- platformio/commands/debug.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 46d26d899d..d0cbf2332a 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -150,15 +150,18 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) - client = GDBClientProcess(project_dir, debug_config) - coro = client.run(__unprocessed) - try: - loop.run_until_complete(coro) - if IS_WINDOWS: - # an issue with asyncio executor and STIDIN, it cannot be closed gracefully - proc.force_exit() - finally: - del client - loop.close() + + with fs.cd(project_dir): + client = GDBClientProcess(project_dir, debug_config) + coro = client.run(__unprocessed) + try: + loop.run_until_complete(coro) + if IS_WINDOWS: + # an issue with `asyncio` executor and STIDIN, + # it cannot be closed gracefully + proc.force_exit() + finally: + del client + loop.close() return True From 834c7b0defe39ff998cd9dc856657aeb837c86ca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 12 Apr 2021 22:38:56 +0300 Subject: [PATCH 039/120] Bump version to 5.2.0a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index e089e96742..26d8dda9aa 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a4") +VERSION = (5, 2, "0a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b8c275223715fb7ab89cf936ea4e7668bcaa63a1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Apr 2021 13:36:53 +0300 Subject: [PATCH 040/120] Dccs: Add information how to avoid extra script running when IDE fetches metadata --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 81ac271bf1..5f10c6a20d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 81ac271bf18a4fa117f48fb8b00f60ba0416b66f +Subproject commit 5f10c6a20d72252d973a3abb61f64a1152a1a0ce From dfdccac67df5d5d686fc4c8cce61305da0b88ba4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Apr 2021 20:28:49 +0300 Subject: [PATCH 041/120] Remove unnecessary "ensure_python3()" blocks --- platformio/commands/home/command.py | 3 --- platformio/commands/package.py | 3 --- platformio/commands/remote/command.py | 2 -- platformio/package/pack.py | 3 +-- platformio/util.py | 8 ++------ 5 files changed, 3 insertions(+), 16 deletions(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 81764a54e3..b656fc0799 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -18,7 +18,6 @@ from platformio.commands.home.helpers import is_port_used from platformio.commands.home.run import run_server -from platformio.compat import ensure_python3 @click.command("home", short_help="GUI to manage PlatformIO") @@ -49,8 +48,6 @@ ), ) def cli(port, host, no_open, shutdown_timeout, session_id): - ensure_python3() - # Ensure PIO Home mimetypes are known mimetypes.add_type("text/html", ".html") mimetypes.add_type("text/css", ".css") diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 0013d1581d..3ac3a2dcd0 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -20,7 +20,6 @@ from platformio import fs from platformio.clients.registry import RegistryClient -from platformio.compat import ensure_python3 from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker from platformio.package.unpack import FileUnpacker, TARArchiver @@ -81,8 +80,6 @@ def package_pack(package, output): help="Notify by email when package is processed", ) def package_publish(package, owner, released_at, private, notify): - assert ensure_python3() - # publish .tar.gz instantly without repacking if not os.path.isdir(package) and isinstance( FileUnpacker.new_archiver(package), TARArchiver diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index 1a2c8ee2f4..edf6b07776 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -28,7 +28,6 @@ from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test -from platformio.compat import ensure_python3 from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError @@ -37,7 +36,6 @@ @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): - assert ensure_python3() ctx.obj = agent inject_contrib_pysite(verify_openssl=True) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index fb0d222fc1..6644311cf0 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -20,7 +20,7 @@ import tempfile from platformio import fs -from platformio.compat import IS_WINDOWS, ensure_python3 +from platformio.compat import IS_WINDOWS from platformio.package.exception import PackageException, UserSideException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema @@ -94,7 +94,6 @@ class PackagePacker(object): ] def __init__(self, package, manifest_uri=None): - assert ensure_python3() self.package = package self.manifest_uri = manifest_uri diff --git a/platformio/util.py b/platformio/util.py index cba128ec58..b81c57d66e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -24,8 +24,9 @@ from glob import glob import click +import zeroconf -from platformio import __version__, compat, exception, proc +from platformio import __version__, exception, proc from platformio.compat import IS_MACOS, IS_WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import @@ -156,11 +157,6 @@ def get_logical_devices(): def get_mdns_services(): - compat.ensure_python3() - - # pylint: disable=import-outside-toplevel - import zeroconf - class mDNSListener(object): def __init__(self): self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) From ad28d1906c452679a9642a5347574e516621b825 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Apr 2021 20:51:54 +0300 Subject: [PATCH 042/120] Improve a package publishing process --- HISTORY.rst | 10 ++++ docs | 2 +- platformio/clients/account.py | 3 + platformio/clients/registry.py | 19 ++---- platformio/commands/account.py | 2 +- platformio/commands/package.py | 104 ++++++++++++++++++++++++++++----- 6 files changed, 107 insertions(+), 33 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 10735dbb8f..36d0c4fa4f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,16 @@ PlatformIO Core 5 * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards +* **Package Management** + + - Improved a package publishing process: + + * Show package details + * Check for conflicting names in the PlatformIO Trusted Registry + * Check for duplicates and used version + + - Added a new option ``--non-interactive`` to `pio package publish `__ command + * **Miscellaneous** - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) diff --git a/docs b/docs index 5f10c6a20d..99e329384c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5f10c6a20d72252d973a3abb61f64a1152a1a0ce +Subproject commit 99e329384cf411b744004da92734f7680038a2d2 diff --git a/platformio/clients/account.py b/platformio/clients/account.py index cae863ada7..c7c2c6fac7 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -207,6 +207,9 @@ def get_account_info(self, offline=False): app.set_state_item("account", account) return result + def get_logged_username(self): + return self.get_account_info(offline=True).get("profile").get("username") + def destroy_account(self): return self.send_auth_request("delete", "/v1/account") diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index b0387ce350..4bd2aaf481 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -15,7 +15,6 @@ from platformio import __registry_api__, fs from platformio.clients.account import AccountClient from platformio.clients.http import HTTPClient, HTTPClientError -from platformio.package.meta import PackageType # pylint: disable=too-many-arguments @@ -32,18 +31,13 @@ def send_auth_request(self, *args, **kwargs): kwargs["headers"] = headers return self.fetch_json_data(*args, **kwargs) - def publish_package( - self, archive_path, owner=None, released_at=None, private=False, notify=True + def publish_package( # pylint: disable=redefined-builtin + self, owner, type, archive_path, released_at=None, private=False, notify=True ): - account = AccountClient() - if not owner: - owner = ( - account.get_account_info(offline=True).get("profile").get("username") - ) with open(archive_path, "rb") as fp: return self.send_auth_request( "post", - "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), + "/v3/packages/%s/%s" % (owner, type), params={ "private": 1 if private else 0, "notify": 1 if notify else 0, @@ -59,13 +53,8 @@ def publish_package( ) def unpublish_package( # pylint: disable=redefined-builtin - self, type, name, owner=None, version=None, undo=False + self, owner, type, name, version=None, undo=False ): - account = AccountClient() - if not owner: - owner = ( - account.get_account_info(offline=True).get("profile").get("username") - ) path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path += "/" + version diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 41af292297..0282767ebf 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -184,7 +184,7 @@ def account_destroy(): click.confirm( "Are you sure you want to delete the %s user account?\n" "Warning! All linked data will be permanently removed and can not be restored." - % client.get_account_info().get("profile").get("username"), + % client.get_logged_username(), abort=True, ) client.destroy_account() diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 3ac3a2dcd0..861de42df9 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -17,9 +17,13 @@ from datetime import datetime import click +from tabulate import tabulate from platformio import fs +from platformio.clients.account import AccountClient from platformio.clients.registry import RegistryClient +from platformio.exception import UserSideException +from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker from platformio.package.unpack import FileUnpacker, TARArchiver @@ -79,26 +83,94 @@ def package_pack(package, output): default=True, help="Notify by email when package is processed", ) -def package_publish(package, owner, released_at, private, notify): - # publish .tar.gz instantly without repacking - if not os.path.isdir(package) and isinstance( +@click.option( + "--non-interactive", + is_flag=True, + help="Do not show interactive prompt", +) +def package_publish( # pylint: disable=too-many-arguments, too-many-locals + package, owner, released_at, private, notify, non_interactive +): + click.secho("Preparing a package...", fg="cyan") + owner = owner or AccountClient().get_logged_username() + do_not_pack = not os.path.isdir(package) and isinstance( FileUnpacker.new_archiver(package), TARArchiver - ): + ) + archive_path = None + with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member + # publish .tar.gz instantly without repacking + if do_not_pack: + archive_path = package + else: + with fs.cd(tmp_dir): + p = PackagePacker(package) + archive_path = p.pack() + + type_ = PackageType.from_archive(archive_path) + manifest = ManifestParserFactory.new_from_archive(archive_path).as_dict() + name = manifest.get("name") + version = manifest.get("version") + data = [ + ("Type:", type_), + ("Owner:", owner), + ("Name:", name), + ("Version:", version), + ] + click.echo(tabulate(data, tablefmt="plain")) + + # look for duplicates + _check_duplicates(owner, type_, name, version) + + if not non_interactive: + click.confirm( + "Are you sure you want to publish the %s %s to the registry?\n" + % ( + type_, + click.style( + "%s/%s@%s" % (owner, name, version), + fg="cyan", + ), + ), + abort=True, + ) + response = RegistryClient().publish_package( - package, owner, released_at, private, notify + owner, type_, archive_path, released_at, private, notify ) + if not do_not_pack: + os.remove(archive_path) click.secho(response.get("message"), fg="green") - return - with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member - with fs.cd(tmp_dir): - p = PackagePacker(package) - archive_path = p.pack() - response = RegistryClient().publish_package( - archive_path, owner, released_at, private, notify - ) - os.remove(archive_path) - click.secho(response.get("message"), fg="green") + +def _check_duplicates(owner, type, name, version): # pylint: disable=redefined-builtin + items = ( + RegistryClient() + .list_packages(filters=dict(types=[type], names=[name])) + .get("items") + ) + if not items: + return True + # duplicated version by owner + if any( + item["owner"]["username"] == owner and item["version"]["name"] == version + for item in items + ): + raise UserSideException( + "The package `%s/%s@%s` is already published in the registry" + % (owner, name, version) + ) + other_owners = [ + item["owner"]["username"] + for item in items + if item["owner"]["username"] != owner + ] + if other_owners: + click.secho( + "\nWarning! A package with the name `%s` is already published by the next " + "owners: `%s`\n" % (name, ", ".join(other_owners)), + fg="yellow", + ) + return True @cli.command("unpublish", short_help="Remove a pushed package from the registry") @@ -119,9 +191,9 @@ def package_publish(package, owner, released_at, private, notify): def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin spec = PackageSpec(package) response = RegistryClient().unpublish_package( + owner=spec.owner or AccountClient().get_logged_username(), type=type, name=spec.name, - owner=spec.owner, version=str(spec.requirements), undo=undo, ) From 286f4ef9614da5d0d692b66c0422f1aec6aaf157 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Apr 2021 20:52:27 +0300 Subject: [PATCH 043/120] Bump version to 5.2.0a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 26d8dda9aa..64672ec46b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a5") +VERSION = (5, 2, "0a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b5b57790be1e814f1cd8e9fb7a03fdb7872315ca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 23 Apr 2021 22:02:07 +0300 Subject: [PATCH 044/120] Validate package manifest when packing archive or publishing a package --- HISTORY.rst | 11 ++--- platformio/commands/package.py | 81 ++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 36d0c4fa4f..94bb9be792 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,7 @@ PlatformIO Core 5 * Show package details * Check for conflicting names in the PlatformIO Trusted Registry * Check for duplicates and used version + * Validate package manifest - Added a new option ``--non-interactive`` to `pio package publish `__ command @@ -235,24 +236,24 @@ Please check `Migration guide from 4.x to 5.0 `__. +See `PlatformIO Core 4.0 history `__. PlatformIO Core 3 ----------------- -See `PlatformIO Core 3.0 history `__. +See `PlatformIO Core 3.0 history `__. PlatformIO Core 2 ----------------- -See `PlatformIO Core 2.0 history `__. +See `PlatformIO Core 2.0 history `__. PlatformIO Core 1 ----------------- -See `PlatformIO Core 1.0 history `__. +See `PlatformIO Core 1.0 history `__. PlatformIO Core Preview ----------------------- -See `PlatformIO Core Preview history `__. +See `PlatformIO Core Preview history `__. diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 861de42df9..1cc0cfb6c8 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -24,6 +24,7 @@ from platformio.clients.registry import RegistryClient from platformio.exception import UserSideException from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker from platformio.package.unpack import FileUnpacker, TARArchiver @@ -39,6 +40,45 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument return value +def load_manifest_from_archive(path): + return ManifestSchema().load_manifest( + ManifestParserFactory.new_from_archive(path).as_dict() + ) + + +def check_package_duplicates( + owner, type, name, version +): # pylint: disable=redefined-builtin + items = ( + RegistryClient() + .list_packages(filters=dict(types=[type], names=[name])) + .get("items") + ) + if not items: + return True + # duplicated version by owner + if any( + item["owner"]["username"] == owner and item["version"]["name"] == version + for item in items + ): + raise UserSideException( + "The package `%s/%s@%s` is already published in the registry" + % (owner, name, version) + ) + other_owners = [ + item["owner"]["username"] + for item in items + if item["owner"]["username"] != owner + ] + if other_owners: + click.secho( + "\nWarning! A package with the name `%s` is already published by the next " + "owners: %s\n" % (name, ", ".join(other_owners)), + fg="yellow", + ) + return True + + @click.group("package", short_help="Package manager") def cli(): pass @@ -57,6 +97,12 @@ def cli(): def package_pack(package, output): p = PackagePacker(package) archive_path = p.pack(output) + # validate manifest + try: + load_manifest_from_archive(archive_path) + except ManifestValidationError as e: + os.remove(archive_path) + raise e click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") @@ -107,7 +153,7 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals archive_path = p.pack() type_ = PackageType.from_archive(archive_path) - manifest = ManifestParserFactory.new_from_archive(archive_path).as_dict() + manifest = load_manifest_from_archive(archive_path) name = manifest.get("name") version = manifest.get("version") data = [ @@ -119,7 +165,7 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals click.echo(tabulate(data, tablefmt="plain")) # look for duplicates - _check_duplicates(owner, type_, name, version) + check_package_duplicates(owner, type_, name, version) if not non_interactive: click.confirm( @@ -142,37 +188,6 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals click.secho(response.get("message"), fg="green") -def _check_duplicates(owner, type, name, version): # pylint: disable=redefined-builtin - items = ( - RegistryClient() - .list_packages(filters=dict(types=[type], names=[name])) - .get("items") - ) - if not items: - return True - # duplicated version by owner - if any( - item["owner"]["username"] == owner and item["version"]["name"] == version - for item in items - ): - raise UserSideException( - "The package `%s/%s@%s` is already published in the registry" - % (owner, name, version) - ) - other_owners = [ - item["owner"]["username"] - for item in items - if item["owner"]["username"] != owner - ] - if other_owners: - click.secho( - "\nWarning! A package with the name `%s` is already published by the next " - "owners: `%s`\n" % (name, ", ".join(other_owners)), - fg="yellow", - ) - return True - - @cli.command("unpublish", short_help="Remove a pushed package from the registry") @click.argument( "package", required=True, metavar="[/][@]" From eb2cd001b6793fb268f11724c0d5c4bc751414ca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 24 Apr 2021 18:01:35 +0300 Subject: [PATCH 045/120] Use private "_idedata" target when fetching data for debugging --- platformio/builder/main.py | 2 +- platformio/project/helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index cac5c8fec4..5b09ad772f 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -219,7 +219,7 @@ click.echo(env.Dump()) env.Exit(0) -if "idedata" in COMMAND_LINE_TARGETS: +if set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS): try: Import("projenv") except: # pylint: disable=bare-except diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 7519e34a51..38f79617df 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -155,7 +155,7 @@ def _load_project_ide_data(project_dir, env_names): # pylint: disable=import-outside-toplevel from platformio.commands.run.command import cli as cmd_run - args = ["--project-dir", project_dir, "--target", "idedata"] + args = ["--project-dir", project_dir, "--target", "_idedata"] for name in env_names: args.extend(["-e", name]) result = CliRunner().invoke(cmd_run, args) From 603d524aafc0b8a8ff7c563ea7a85174671bbbab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 13:10:19 +0300 Subject: [PATCH 046/120] Refactor docs to be deployed as a static content --- .github/workflows/docs.yml | 2 +- docs | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 39de401e6a..4bab6a0a52 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: with: submodules: "recursive" - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.7 - name: Install dependencies diff --git a/docs b/docs index 99e329384c..dda03f5421 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 99e329384cf411b744004da92734f7680038a2d2 +Subproject commit dda03f54215d810d4590eb4883f7b93e9e802fb1 diff --git a/tox.ini b/tox.ini index be83255558..9bd702691f 100644 --- a/tox.ini +++ b/tox.ini @@ -48,13 +48,13 @@ commands = py.test -v --basetemp="{envtmpdir}" tests/test_examples.py [testenv:docs] +; basepython = ~/.pyenv/versions/3.6.12/bin/python deps = sphinx sphinx_rtd_theme restructuredtext-lint commands = sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html - sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex [testenv:docslinkcheck] deps = From 6f33460afd4286e9eb3f90d5e28ff117df04480c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 13:17:22 +0300 Subject: [PATCH 047/120] Remove debugging code --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dda03f5421..495f824a0f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dda03f54215d810d4590eb4883f7b93e9e802fb1 +Subproject commit 495f824a0f73556ddec9187d7d46dae1cbeb6ac8 From cccabf53307a0e87ca5a3b4e7ab98665c9b2f55e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 13:19:49 +0300 Subject: [PATCH 048/120] Add missed "sphinx-notfound-page" package for docs --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9bd702691f..9f71ed25a8 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,7 @@ commands = deps = sphinx sphinx_rtd_theme + sphinx-notfound-page restructuredtext-lint commands = sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html From 61d6cd3c18d7301d80e0779ea251d5358bdd7caf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 19:58:50 +0300 Subject: [PATCH 049/120] Apply black formatter --- platformio/builder/tools/compilation_db.py | 2 +- platformio/builder/tools/pioide.py | 2 +- platformio/cache.py | 2 +- platformio/clients/http.py | 2 +- platformio/commands/device/filters/base.py | 4 ++-- platformio/package/manager/_registry.py | 2 +- platformio/project/helpers.py | 4 ++-- platformio/util.py | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 90b6517a8a..f91fa3ab35 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -58,7 +58,7 @@ def __init__(self, value): def changed_since_last_build_node(*args, **kwargs): - """ Dummy decider to force always building""" + """Dummy decider to force always building""" return True diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 332ec6ed64..9d278b20cd 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -157,7 +157,7 @@ def _subst_cmd(env, cmd): def DumpIDEData(env, globalenv): - """ env here is `projenv`""" + """env here is `projenv`""" data = { "env_name": env["PIOENV"], diff --git a/platformio/cache.py b/platformio/cache.py index bc817f61ef..8737725993 100644 --- a/platformio/cache.py +++ b/platformio/cache.py @@ -92,7 +92,7 @@ def set(self, key, data, valid): return self._unlock_dbindex() def delete(self, keys=None): - """ Keys=None, delete expired items """ + """Keys=None, delete expired items""" if not os.path.isfile(self._db_path): return None if not keys: diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 1e22ca975f..4b0afc7387 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -80,7 +80,7 @@ def __iter__(self): # pylint: disable=non-iterator-returned return self def next(self): - """ For Python 2 compatibility """ + """For Python 2 compatibility""" return self.__next__() def __next__(self): diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py index 6745a626f3..bf95352e0a 100644 --- a/platformio/commands/device/filters/base.py +++ b/platformio/commands/device/filters/base.py @@ -19,7 +19,7 @@ class DeviceMonitorFilter(miniterm.Transform): def __init__(self, options=None): - """ Called by PlatformIO to pass context """ + """Called by PlatformIO to pass context""" miniterm.Transform.__init__(self) self.options = options or {} @@ -35,7 +35,7 @@ def __init__(self, options=None): self.environment = self.config.envs()[0] def __call__(self): - """ Called by the miniterm library when the filter is actually used """ + """Called by the miniterm library when the filter is actually used""" return self @property diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 4dfd43b6a1..740afe328c 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -42,7 +42,7 @@ def __iter__(self): # pylint: disable=non-iterator-returned return self def next(self): - """ For Python 2 compatibility """ + """For Python 2 compatibility""" return self.__next__() def __next__(self): diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 38f79617df..3c1e1b3728 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -46,14 +46,14 @@ def find_project_dir_above(path): def get_project_core_dir(): - """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """ + """Deprecated, use ProjectConfig.get_optional_dir("core") instead""" return ProjectConfig.get_instance( join(get_project_dir(), "platformio.ini") ).get_optional_dir("core", exists=True) def get_project_cache_dir(): - """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """ + """Deprecated, use ProjectConfig.get_optional_dir("cache") instead""" return ProjectConfig.get_instance( join(get_project_dir(), "platformio.ini") ).get_optional_dir("cache") diff --git a/platformio/util.py b/platformio/util.py index b81c57d66e..1d7c865565 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -77,7 +77,7 @@ def wrapper(*args, **kwargs): def singleton(cls): - """ From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples """ + """From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples""" _instances = {} def get_instance(*args, **kwargs): @@ -231,7 +231,7 @@ def get_services(self): def pioversion_to_intstr(): - """ Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" + """Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch return [int(i) for i in vermatch.group(1).split(".")[:3]] From 310cc086c626b08f599d087bb0f0b4f6ec4a7a95 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 19:59:12 +0300 Subject: [PATCH 050/120] Docs: Minor fixes to "redirect" page generator --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 495f824a0f..34ddaf3b1b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 495f824a0f73556ddec9187d7d46dae1cbeb6ac8 +Subproject commit 34ddaf3b1b20e1306bddae8c851ad86c871fa7d1 From b5c1a195be6ac0a160c9d778dc26606e06224076 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 19:59:37 +0300 Subject: [PATCH 051/120] Fix PyLint issues: consider-using-with --- platformio/builder/tools/pioupload.py | 6 +++--- .../commands/device/filters/log2file.py | 1 + platformio/package/download.py | 2 +- platformio/package/lockfile.py | 2 +- platformio/proc.py | 20 +++++++++---------- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 3281ada455..23f252e511 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -236,9 +236,9 @@ def _calculate_size(output, pattern): def _format_availale_bytes(value, total): percent_raw = float(value) / float(total) blocks_per_progress = 10 - used_blocks = int(round(blocks_per_progress * percent_raw)) - if used_blocks > blocks_per_progress: - used_blocks = blocks_per_progress + used_blocks = min( + int(round(blocks_per_progress * percent_raw)), blocks_per_progress + ) return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format( "=" * used_blocks, blocks_per_progress, percent_raw, value, total ) diff --git a/platformio/commands/device/filters/log2file.py b/platformio/commands/device/filters/log2file.py index 69118510e0..d7199a19bc 100644 --- a/platformio/commands/device/filters/log2file.py +++ b/platformio/commands/device/filters/log2file.py @@ -31,6 +31,7 @@ def __call__(self): "%y%m%d-%H%M%S" ) print("--- Logging an output to %s" % os.path.abspath(log_file_name)) + # pylint: disable=consider-using-with self._log_fp = io.open(log_file_name, "w", encoding="utf-8") return self diff --git a/platformio/package/download.py b/platformio/package/download.py index bd425ac630..ffc57d50e2 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -73,7 +73,7 @@ def get_size(self): def start(self, with_progress=True, silent=False): label = "Downloading" itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) - fp = open(self._destination, "wb") + fp = open(self._destination, "wb") # pylint: disable=consider-using-with try: if not with_progress or self.get_size() == -1: if not silent: diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index db4b1d3ff9..6cfc46ef1c 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -62,7 +62,7 @@ def _lock(self): else: raise LockFileExists - self._fp = open(self._lock_path, "w") + self._fp = open(self._lock_path, "w") # pylint: disable=consider-using-with try: if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/platformio/proc.py b/platformio/proc.py index 6e9981afac..7e93907d50 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -109,16 +109,16 @@ def exec_command(*args, **kwargs): default.update(kwargs) kwargs = default - p = subprocess.Popen(*args, **kwargs) - try: - result["out"], result["err"] = p.communicate() - result["returncode"] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() - finally: - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipeBase): - kwargs[s].close() + with subprocess.Popen(*args, **kwargs) as p: + try: + result["out"], result["err"] = p.communicate() + result["returncode"] = p.returncode + except KeyboardInterrupt: + raise exception.AbortedByUser() + finally: + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipeBase): + kwargs[s].close() for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): From 5953480807d5dd629e323ce6e0b13fa7bb8d3a33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Apr 2021 20:16:01 +0300 Subject: [PATCH 052/120] Docs: Fix broken link for RTD page --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 34ddaf3b1b..8ec02e968a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 34ddaf3b1b20e1306bddae8c851ad86c871fa7d1 +Subproject commit 8ec02e968ac1f06fe9daa44f1c651cdfa8374822 From 1a152ed7fa3838ef5d7756458566bcb993bd16fa Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 28 Apr 2021 20:18:23 +0300 Subject: [PATCH 053/120] Add deploy step to CI configuration --- .github/workflows/docs.yml | 78 +++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4bab6a0a52..2b03d146ec 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,6 +4,7 @@ on: [push, pull_request] jobs: build: + name: Build Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -29,4 +30,79 @@ jobs: type: ${{ job.status }} job_name: '*Docs*' commit: true - url: ${{ secrets.SLACK_BUILD_WEBHOOK }} \ No newline at end of file + url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + + - name: Preserve Docs + if: ${{ github.event_name == 'push' }} + run: | + tar -czvf docs.tar.gz -C docs/_build html rtdpage + + - name: Save artifact + if: ${{ github.event_name == 'push' }} + uses: actions/upload-artifact@v2 + with: + name: docs + path: ./docs.tar.gz + + deploy: + name: Deploy Docs + needs: build + runs-on: ubuntu-latest + env: + DOCS_DIR: platformio-docs + LATEST_DOCS_DIR: latest-docs + RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }} + DOCS_REPO: platformio/platformio-docs + if: ${{ github.event_name == 'push' }} + steps: + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: docs + - name: Unpack artifact + run: | + mkdir ./LATEST_DOCS_DIR + tar -xzf ./docs.tar.gz -C ./LATEST_DOCS_DIR + - name: Delete Artifact + uses: geekyeggo/delete-artifact@v1 + with: + name: docs + - name: Select Docs type + id: get-destination-dir + run: | + if [[ ${{ env.RELEASE_BUILD }} == true ]]; then + echo "::set-output name=dst_dir::stable" + else + echo "::set-output name=dst_dir::latest" + fi + - name: Checkout latest Docs + continue-on-error: true + uses: actions/checkout@v2 + with: + repository: ${{ env.DOCS_REPO }} + path: ${{ env.DOCS_DIR }} + ref: gh-pages + - name: Synchronize Docs + run: | + rm -rf ${{ env.DOCS_DIR }}/.git + rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + cp -rf LATEST_DOCS_DIR/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + if [[ ${{ env.RELEASE_BUILD }} == false ]]; then + rm -rf ${{ env.DOCS_DIR }}/page + mkdir -p ${{ env.DOCS_DIR }}/page + cp -rf LATEST_DOCS_DIR/rtdpage/* ${{ env.DOCS_DIR }}/page + fi + - name: Validate Docs + run: | + if [ -z "$(ls -A ${{ env.DOCS_DIR }})" ]; then + echo "Docs folder is empty. Aborting!" + exit 1 + fi + - name: Deploy to Github Pages + uses: peaceiris/actions-gh-pages@v3 + with: + personal_token: ${{ secrets.PERSONAL_TOKEN }} + external_repository: ${{ env.DOCS_REPO }} + publish_dir: ./${{ env.DOCS_DIR }} + commit_message: Sync Docs From 2c3f4302031e8e6dbf70715eb90b759dfdb6735b Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 28 Apr 2021 20:59:01 +0300 Subject: [PATCH 054/120] Tidy up Docs CI --- .github/workflows/docs.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2b03d146ec..7efa20e90b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -49,10 +49,10 @@ jobs: needs: build runs-on: ubuntu-latest env: + DOCS_REPO: platformio/platformio-docs DOCS_DIR: platformio-docs LATEST_DOCS_DIR: latest-docs RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }} - DOCS_REPO: platformio/platformio-docs if: ${{ github.event_name == 'push' }} steps: - name: Download artifact @@ -61,8 +61,8 @@ jobs: name: docs - name: Unpack artifact run: | - mkdir ./LATEST_DOCS_DIR - tar -xzf ./docs.tar.gz -C ./LATEST_DOCS_DIR + mkdir ./${{ env.LATEST_DOCS_DIR }} + tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} - name: Delete Artifact uses: geekyeggo/delete-artifact@v1 with: @@ -87,11 +87,11 @@ jobs: rm -rf ${{ env.DOCS_DIR }}/.git rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} - cp -rf LATEST_DOCS_DIR/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + cp -rf ${{ env.LATEST_DOCS_DIR }}/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} if [[ ${{ env.RELEASE_BUILD }} == false ]]; then rm -rf ${{ env.DOCS_DIR }}/page mkdir -p ${{ env.DOCS_DIR }}/page - cp -rf LATEST_DOCS_DIR/rtdpage/* ${{ env.DOCS_DIR }}/page + cp -rf ${{ env.LATEST_DOCS_DIR }}/rtdpage/* ${{ env.DOCS_DIR }}/page fi - name: Validate Docs run: | From 915c8507602b6542a30f88632efe426e3feb928c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Apr 2021 12:47:57 +0300 Subject: [PATCH 055/120] Docs: Fix JS redirect URL --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8ec02e968a..84ef20dcf8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8ec02e968ac1f06fe9daa44f1c651cdfa8374822 +Subproject commit 84ef20dcf84398009cbf634ca3dd27cd767a78e6 From c9e10b1a3e45fe98595eed8aaa5e71fd6311ed49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Apr 2021 14:43:27 +0300 Subject: [PATCH 056/120] Fix issue with broken redirect --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 84ef20dcf8..a7f281bccd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 84ef20dcf84398009cbf634ca3dd27cd767a78e6 +Subproject commit a7f281bccd21ef4d106b94b5951d510164fa3e3e From 14dc9c6c43ddf35e81a6d73758c78fc9be7246a0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Apr 2021 18:38:44 +0300 Subject: [PATCH 057/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a7f281bccd..8bde2ac5b6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a7f281bccd21ef4d106b94b5951d510164fa3e3e +Subproject commit 8bde2ac5b6917ba716a2980c6e7e2462ba22e2ad From 4281225b0205a0c22bf5b128f3d061558e586433 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Apr 2021 19:24:44 +0300 Subject: [PATCH 058/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8bde2ac5b6..e59171c993 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8bde2ac5b6917ba716a2980c6e7e2462ba22e2ad +Subproject commit e59171c993cd48f6f29e28b273ff6d10e8bdf118 From 1c90bb383f7124bcc15752b5eefe3421dbcf9317 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Apr 2021 19:46:17 +0300 Subject: [PATCH 059/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e59171c993..e94dd36bc1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e59171c993cd48f6f29e28b273ff6d10e8bdf118 +Subproject commit e94dd36bc1cc88386817aa3fd7ca49f86c842ce6 From 553c398c8ef118513f7607ceb8dd8b3ba97b771e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Apr 2021 18:06:35 +0300 Subject: [PATCH 060/120] Show package "system" info before publishing --- platformio/commands/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 1cc0cfb6c8..cebc10517c 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -162,6 +162,8 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals ("Name:", name), ("Version:", version), ] + if manifest.get("system"): + data.insert(len(data) - 1, ("System", ", ".join(manifest.get("system")))) click.echo(tabulate(data, tablefmt="plain")) # look for duplicates From 60c83bae931b9a2fcd61611e553a3493fcc87ef6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 May 2021 13:44:28 +0300 Subject: [PATCH 061/120] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- platformio/commands/package.py | 2 +- scripts/docspregen.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index e94dd36bc1..9ec47a577b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e94dd36bc1cc88386817aa3fd7ca49f86c842ce6 +Subproject commit 9ec47a577bc43de62ccaeb4ed044aa583162c695 diff --git a/examples b/examples index 84bc1a967a..3b78e61048 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 84bc1a967ab44dcfdeeb58c501739ca3a2bf87e3 +Subproject commit 3b78e610481e6a5b3dda5598e231dbd7e88f45e3 diff --git a/platformio/commands/package.py b/platformio/commands/package.py index cebc10517c..1a9cd76753 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -163,7 +163,7 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals ("Version:", version), ] if manifest.get("system"): - data.insert(len(data) - 1, ("System", ", ".join(manifest.get("system")))) + data.insert(len(data) - 1, ("System:", ", ".join(manifest.get("system")))) click.echo(tabulate(data, tablefmt="plain")) # look for duplicates diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 881160ed20..237f9d36b3 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -789,7 +789,7 @@ def update_embedded_board(rst_path, board): """ Uploading --------- -%s supports the next uploading protocols: +%s supports the following uploading protocols: """ % board["name"] ) From 0fa9006e45a62b47c976694deb112391cae44ee9 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 3 May 2021 22:34:43 +0300 Subject: [PATCH 062/120] Sync docs: CircleCI updates --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 9ec47a577b..7fb8fc27d8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9ec47a577bc43de62ccaeb4ed044aa583162c695 +Subproject commit 7fb8fc27d86f82b5f156e2c2ff8ebce292db8c1e From 186ab70bf9ae823e9fd87e890b4f8f947b530e5a Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 10 May 2021 11:38:05 +0300 Subject: [PATCH 063/120] Add udev rule for Raspberry Pi Pico boards --- scripts/99-platformio-udev.rules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 86edd5b054..a4fef7b81a 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -170,3 +170,6 @@ ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID # Atmel AVR Dragon ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Raspberry Pi Pico +SUBSYSTEMS=="usb", ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666" From 2be7e0f7e6178eea18e3c2a3b300256f8d53aca3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 May 2021 15:28:09 +0300 Subject: [PATCH 064/120] Docs: Promote PlatformIO Labs blog posts --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7fb8fc27d8..01219b63ed 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7fb8fc27d86f82b5f156e2c2ff8ebce292db8c1e +Subproject commit 01219b63ed20f829f6dc92a60adaa88ee1fc32b8 From 27feb1ddd79d310d8be69706042a4166cf3f5f74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 19 May 2021 19:43:41 +0300 Subject: [PATCH 065/120] Added support for Click 8.0; updated other deps --- docs | 2 +- platformio/commands/boards.py | 3 ++- platformio/commands/check/command.py | 3 ++- platformio/commands/run/command.py | 3 ++- platformio/commands/test/command.py | 3 ++- platformio/maintenance.py | 30 ++++++++++++++-------------- platformio/telemetry.py | 4 ++-- platformio/util.py | 3 ++- setup.py | 6 +++--- 9 files changed, 31 insertions(+), 26 deletions(-) diff --git a/docs b/docs index 01219b63ed..459f01352a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 01219b63ed20f829f6dc92a60adaa88ee1fc32b8 +Subproject commit 459f01352ae3eea3b5adfab9ff389f45768fc4ce diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index f93607d436..b51103ca28 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import shutil import click from tabulate import tabulate @@ -40,7 +41,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards[board["platform"]] = [] grpboards[board["platform"]].append(board) - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() for (platform, boards) in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 082373f346..2d4bb1091a 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -17,6 +17,7 @@ import json import os +import shutil from collections import Counter from os.path import dirname, isfile from time import time @@ -193,7 +194,7 @@ def print_processing_header(tool, envname, envdump): "Checking %s > %s (%s)" % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) ) - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.secho("-" * terminal_width, bold=True) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index db4b412175..5684f1192f 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -14,6 +14,7 @@ import operator import os +import shutil from multiprocessing import cpu_count from time import time @@ -200,7 +201,7 @@ def print_processing_header(env, config, verbose=False): "Processing %s (%s)" % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)) ) - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.secho("-" * terminal_width, bold=True) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index bc503093c3..c4b9049923 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -16,6 +16,7 @@ import fnmatch import os +import shutil from time import time import click @@ -192,7 +193,7 @@ def print_processing_header(test, env): click.style(env, fg="cyan", bold=True), ) ) - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.secho("-" * terminal_width, bold=True) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3ef47015f3..ba370032e9 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import getenv -from os.path import join +import os +import shutil from time import time import click @@ -75,19 +75,19 @@ def on_platformio_exception(e): def set_caller(caller=None): - caller = caller or getenv("PLATFORMIO_CALLER") + caller = caller or os.getenv("PLATFORMIO_CALLER") if caller: return app.set_session_var("caller_id", caller) - if getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): + if os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): caller = "vscode" - elif getenv("GITPOD_INSTANCE_ID") or getenv("GITPOD_WORKSPACE_URL"): + elif os.getenv("GITPOD_INSTANCE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): caller = "gitpod" elif is_container(): - if getenv("C9_UID"): + if os.getenv("C9_UID"): caller = "C9" - elif getenv("USER") == "cabox": + elif os.getenv("USER") == "cabox": caller = "CA" - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): caller = "Che" return app.set_session_var("caller_id", caller) @@ -139,7 +139,7 @@ def _update_pkg_metadata(_): def after_upgrade(ctx): - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() last_version = app.get_state_item("last_version", "0.0.0") if last_version == __version__: return @@ -204,7 +204,7 @@ def after_upgrade(ctx): click.style("https://github.com/platformio/platformio", fg="cyan"), ) ) - if not getenv("PLATFORMIO_IDE"): + if not os.getenv("PLATFORMIO_IDE"): click.echo( "- %s PlatformIO IDE for embedded development > %s" % ( @@ -235,7 +235,7 @@ def check_platformio_upgrade(): if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.echo("") click.echo("*" * terminal_width) @@ -245,10 +245,10 @@ def check_platformio_upgrade(): fg="yellow", nl=False, ) - if getenv("PLATFORMIO_IDE"): + if os.getenv("PLATFORMIO_IDE"): click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False) click.secho("`.", fg="yellow") - elif join("Cellar", "platformio") in fs.get_source_dir(): + elif os.path.join("Cellar", "platformio") in fs.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) click.secho("` command.", fg="yellow") else: @@ -288,7 +288,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches if not outdated_items: return - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.echo("") click.echo("*" * terminal_width) @@ -350,7 +350,7 @@ def check_prune_system(): if (unnecessary_size / 1024) < threshold_mb: return - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() click.echo() click.echo("*" * terminal_width) click.secho( diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 6ca60b0f10..94c4222a63 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -17,13 +17,13 @@ import json import os import re +import shutil import sys import threading from collections import deque from time import sleep, time from traceback import format_exc -import click import requests from platformio import __version__, app, exception, util @@ -74,7 +74,7 @@ def __init__(self): self["cid"] = app.get_cid() try: - self["sr"] = "%dx%d" % click.get_terminal_size() + self["sr"] = "%dx%d" % shutil.get_terminal_size() except ValueError: pass diff --git a/platformio/util.py b/platformio/util.py index 1d7c865565..8ab144a489 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -19,6 +19,7 @@ import os import platform import re +import shutil import time from functools import wraps from glob import glob @@ -269,7 +270,7 @@ def merge_dicts(d1, d2, path=None): def print_labeled_bar(label, is_error=False, fg=None): - terminal_width, _ = click.get_terminal_size() + terminal_width, _ = shutil.get_terminal_size() width = len(click.unstyle(label)) half_line = "=" * int((terminal_width - width - 2) / 2) click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) diff --git a/setup.py b/setup.py index af921b7666..a520931cc6 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ minimal_requirements = [ "bottle==0.12.*", - "click>=5,<8%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""), + "click>=5,<9%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""), "colorama", "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), "pyelftools>=0.27,<1", @@ -39,10 +39,10 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.29.*") + minimal_requirements.append("zeroconf==0.31.*") home_requirements = [ - "aiofiles==0.6.*", + "aiofiles==0.7.*", "ajsonrpc==1.1.*", "starlette==0.14.*", "uvicorn==0.13.*", From 90fdaf80e4567cd49812a205bc0055bd9edf9a89 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 31 May 2021 18:25:54 +0300 Subject: [PATCH 066/120] Sync docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 459f01352a..9eeca1a3c9 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 459f01352ae3eea3b5adfab9ff389f45768fc4ce +Subproject commit 9eeca1a3c9dac4e3ccf2cd2d85ca6cc2b09c2894 diff --git a/examples b/examples index 3b78e61048..c6759e3f1b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 3b78e610481e6a5b3dda5598e231dbd7e88f45e3 +Subproject commit c6759e3f1b8aa56fea0a4aaa2fc3190b8672ac0f From bee35acfa6476fdb14cd686d5f64336838f72066 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Jun 2021 17:56:55 +0300 Subject: [PATCH 067/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 9eeca1a3c9..47f5f40761 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9eeca1a3c9dac4e3ccf2cd2d85ca6cc2b09c2894 +Subproject commit 47f5f4076165755a66da851a2d675aed04810213 From 710f82de0f6a2dfae14442da36ca538f9ee2cf97 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Jun 2021 17:59:18 +0300 Subject: [PATCH 068/120] Up uvicorn to 0.14 & click to 8.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a520931cc6..362d928ac3 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ "aiofiles==0.7.*", "ajsonrpc==1.1.*", "starlette==0.14.*", - "uvicorn==0.13.*", + "uvicorn==0.14.*", "wsproto==1.0.*", ] From e8f703648a5b0d3e325dbe1c95c0cd9d8be582b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 1 Jun 2021 18:24:17 +0300 Subject: [PATCH 069/120] Docs: Use Python 3 for CI integration --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 47f5f40761..5149b01a11 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 47f5f4076165755a66da851a2d675aed04810213 +Subproject commit 5149b01a114678b8bf46a8105548e17153083eb2 From 6a9b7fdb6def3b49e7905da880cf776e7b5e964c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Jun 2021 16:32:53 +0300 Subject: [PATCH 070/120] Update SPDX License List to 3.13 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index addc4c5f66..123875bbea 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -253,7 +253,7 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.12" + version = "3.13" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version From 0d6eff2a9ace65915378663035e7d61c7ff86893 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 22 Jun 2021 14:27:33 +0300 Subject: [PATCH 071/120] Syn docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5149b01a11..0245fa05c2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5149b01a114678b8bf46a8105548e17153083eb2 +Subproject commit 0245fa05c2645235d7b01f06733047f31d44985a From fbcae11cd000250ca43b645aeb6142a031cbcb0c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 22 Jun 2021 14:28:04 +0300 Subject: [PATCH 072/120] Fix project generator --- platformio/project/helpers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 3c1e1b3728..c41744a10a 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -141,10 +141,11 @@ def load_project_ide_data(project_dir, env_or_envs, cache=False): if not isinstance(env_names, list): env_names = [env_names] - result = _load_cached_project_ide_data(project_dir, env_names) if cache else {} - missed_env_names = set(env_names) - set(result.keys()) - if missed_env_names: - result.update(_load_project_ide_data(project_dir, missed_env_names)) + with fs.cd(project_dir): + result = _load_cached_project_ide_data(project_dir, env_names) if cache else {} + missed_env_names = set(env_names) - set(result.keys()) + if missed_env_names: + result.update(_load_project_ide_data(project_dir, missed_env_names)) if not isinstance(env_or_envs, list) and env_or_envs in result: return result[env_or_envs] From 2b94791387be3d6f9f49115f205a04bcaecd4eb5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 22 Jun 2021 14:28:40 +0300 Subject: [PATCH 073/120] Bump version to 5.2.0a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 64672ec46b..c5a6e59fea 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a6") +VERSION = (5, 2, "0a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 311e10f91e825348df97c0701f575f2b42260b57 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Jun 2021 16:00:13 +0300 Subject: [PATCH 074/120] Ensure all patterns are replaces in debug init script --- platformio/debug/config/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 019f9edbb1..65ceba6008 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os from platformio import fs, proc, util @@ -238,4 +239,8 @@ def _replace(text): elif isinstance(value, (list, dict)) and recursive: source[key] = self.reveal_patterns(value, patterns) + data = json.dumps(source) + if any(("$" + key) in data for key in patterns): + source = self.reveal_patterns(source, patterns) + return source From 05374d1145e704145d156601e4ee6abaac5b7d05 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Jun 2021 16:42:45 +0300 Subject: [PATCH 075/120] Match buffered data from debugging server --- platformio/debug/process/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 1fb08f4d2b..b2653511f3 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -26,10 +26,14 @@ class DebugServerProcess(DebugBaseProcess): + + STD_BUFFER_SIZE = 1024 + def __init__(self, debug_config): super(DebugServerProcess, self).__init__() self.debug_config = debug_config self._ready = False + self._std_buffer = {"out": b"", "err": b""} async def run(self): # pylint: disable=too-many-branches server = self.debug_config.server @@ -133,8 +137,12 @@ def stdout_data_received(self, data): super(DebugServerProcess, self).stdout_data_received( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) - self._check_ready_by_pattern(data) + self._std_buffer["out"] += data + self._check_ready_by_pattern(self._std_buffer["out"]) + self._std_buffer["out"] = self._std_buffer["out"][-1 * self.STD_BUFFER_SIZE :] def stderr_data_received(self, data): super(DebugServerProcess, self).stderr_data_received(data) - self._check_ready_by_pattern(data) + self._std_buffer["err"] += data + self._check_ready_by_pattern(self._std_buffer["err"]) + self._std_buffer["err"] = self._std_buffer["err"][-1 * self.STD_BUFFER_SIZE :] From 5ab776974528e14a2685fb3280eec687bc16e7fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 24 Jun 2021 16:43:00 +0300 Subject: [PATCH 076/120] Bump version to 5.2.0a8 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c5a6e59fea..4fdfaee083 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a7") +VERSION = (5, 2, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5c3b5be613d04f148b9dc56cc04534012cd97467 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Jun 2021 18:07:45 +0300 Subject: [PATCH 077/120] Fix TypeError: 'NoneType' object is not callable --- platformio/clients/http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 4b0afc7387..650d1a3056 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -101,7 +101,10 @@ def __init__(self, endpoints): def __del__(self): if not self._session: return - self._session.close() + try: + self._session.close() + except: + pass self._session = None def _next_session(self): From 17c7d90d52090410b81877006a3b727c8f79b60a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Jun 2021 18:11:08 +0300 Subject: [PATCH 078/120] Sync docs --- docs | 2 +- scripts/docspregen.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs b/docs index 0245fa05c2..ac2716d8bc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0245fa05c2645235d7b01f06733047f31d44985a +Subproject commit ac2716d8bc22ea634cb3519e17e92e32c5b94a00 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 237f9d36b3..c4291cc036 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -625,6 +625,7 @@ def update_framework_docs(): def update_boards(): + print("Updating boards...") lines = [] lines.append(RST_COPYRIGHT) From 5390b4ed42f2012cf74d0b256f8cb3b7da66154b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Jun 2021 18:24:47 +0300 Subject: [PATCH 079/120] Add Github token for Slack notification --- .github/workflows/core.yml | 1 + .github/workflows/docs.yml | 1 + .github/workflows/examples.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index afc59002f4..6f21021add 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -42,3 +42,4 @@ jobs: job_name: '*Core*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7efa20e90b..1a80dbf60d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,6 +31,7 @@ jobs: job_name: '*Docs*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} - name: Preserve Docs if: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 34d261d4f4..29c7610338 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -65,3 +65,4 @@ jobs: job_name: '*Examples*' commit: true url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} From 72cc23ef46d4ab83babe30b7806c39217cc8da67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Jun 2021 18:25:20 +0300 Subject: [PATCH 080/120] Fix PyLint warning with "No exception type(s) specified (bare-except)" --- platformio/clients/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 650d1a3056..0f3e386022 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -103,7 +103,7 @@ def __del__(self): return try: self._session.close() - except: + except: # pylint: disable=bare-except pass self._session = None From dde8898aae7a294ac5659836265b1ba3ec0d4ed3 Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Mon, 5 Jul 2021 11:57:30 +0200 Subject: [PATCH 081/120] Bump zeroconf to 0.32.* (#3991) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 362d928ac3..978b3620d2 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.31.*") + minimal_requirements.append("zeroconf==0.32.*") home_requirements = [ "aiofiles==0.7.*", From ff6d1698626aff801f5fc96bd29f2d6cd254b365 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jul 2021 13:30:37 +0300 Subject: [PATCH 082/120] Fix PyLint for v2.9.3 --- .pylintrc | 1 + platformio/package/manager/core.py | 1 + platformio/package/unpack.py | 8 ++++++-- platformio/proc.py | 12 ++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.pylintrc b/.pylintrc index bb68f8a095..4a7f6601ee 100644 --- a/.pylintrc +++ b/.pylintrc @@ -15,6 +15,7 @@ disable= useless-object-inheritance, useless-import-alias, bad-option-value, + consider-using-dict-items, ; PY2 Compat super-with-arguments, diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index ac5cad7b89..63251af319 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -78,6 +78,7 @@ def remove_unnecessary_core_packages(dry_run=False): pkg = pm.get_package(spec) if not pkg: continue + # pylint: disable=no-member best_pkg_versions[pkg.metadata.name] = pkg.metadata.version for pkg in pm.get_installed(): diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index 2913660d63..6bbbef634e 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -57,7 +57,9 @@ def close(self): class TARArchiver(BaseArchiver): def __init__(self, archpath): - super(TARArchiver, self).__init__(tarfile_open(archpath)) + super(TARArchiver, self).__init__( + tarfile_open(archpath) # pylint: disable=consider-using-with + ) def get_items(self): return self._afo.getmembers() @@ -99,7 +101,9 @@ def extract_item(self, item, dest_dir): class ZIPArchiver(BaseArchiver): def __init__(self, archpath): - super(ZIPArchiver, self).__init__(ZipFile(archpath)) + super(ZIPArchiver, self).__init__( + ZipFile(archpath) # pylint: disable=consider-using-with + ) @staticmethod def preserve_permissions(item, dest_dir): diff --git a/platformio/proc.py b/platformio/proc.py index 7e93907d50..20f5971085 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -124,16 +124,16 @@ def exec_command(*args, **kwargs): if isinstance(kwargs[s], AsyncPipeBase): result[s[3:]] = kwargs[s].get_buffer() - for k, v in result.items(): - if isinstance(result[k], bytes): + for key, value in result.items(): + if isinstance(value, bytes): try: - result[k] = result[k].decode( + result[key] = value.decode( get_locale_encoding() or get_filesystem_encoding() ) except UnicodeDecodeError: - result[k] = result[k].decode("latin-1") - if v and isinstance(v, string_types): - result[k] = result[k].strip() + result[key] = value.decode("latin-1") + if value and isinstance(value, string_types): + result[key] = value.strip() return result From 5036d25b6084470ef064bfd26bd2877f1b1a584a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jul 2021 13:31:23 +0300 Subject: [PATCH 083/120] Enable Python version auto-detection for Black formatter --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a73d9cba4b..117725d321 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,8 @@ isort: isort ./tests format: - black --target-version py27 ./platformio - black --target-version py27 ./tests + black ./platformio + black ./tests test: py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py From 71ae579bc07b2e11fec16acda482dea04bc3a359 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jul 2021 16:06:02 +0300 Subject: [PATCH 084/120] PyLint fix --- Makefile | 2 +- tests/conftest.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 117725d321..5ba6f78862 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ lint: - pylint -j 6 --rcfile=./.pylintrc ./platformio pylint -j 6 --rcfile=./.pylintrc ./tests + pylint -j 6 --rcfile=./.pylintrc ./platformio isort: isort ./platformio diff --git a/tests/conftest.py b/tests/conftest.py index d81f0e8ad4..b3b1bc8884 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,7 @@ def clirunner(request): "PLATFORMIO_WORKSPACE_DIR": {"new": None}, } for key, item in backup_env_vars.items(): + # pylint: disable=unnecessary-dict-index-lookup backup_env_vars[key]["old"] = os.environ.get(key) if item["new"] is not None: os.environ[key] = item["new"] From 7dc8463da989c7c611e99d18be3b45e631801cf6 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Wed, 7 Jul 2021 21:25:55 +0600 Subject: [PATCH 085/120] Fix charmap error (#3998) * Fix charmap error Fix charmap error on cyrilic in platformio.ini file #3493 * Update config.py Co-authored-by: Ivan Kravets --- platformio/project/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 6c5aa7b192..0d1005071c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -102,7 +102,7 @@ def read(self, path, parse_extra=True): return self._parsed.append(path) try: - self._parser.read(path) + self._parser.read(path, "utf-8") except ConfigParser.Error as e: raise exception.InvalidProjectConfError(path, str(e)) From ac8443136194bd8fa04c4119c92fdca5265e6564 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 12 Jul 2021 15:06:06 +0300 Subject: [PATCH 086/120] Take into account package's "system" when checking for duplicates --- platformio/commands/package.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 1a9cd76753..8ce79dd5e4 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -47,8 +47,9 @@ def load_manifest_from_archive(path): def check_package_duplicates( - owner, type, name, version + owner, type, name, version, system ): # pylint: disable=redefined-builtin + found = False items = ( RegistryClient() .list_packages(filters=dict(types=[type], names=[name])) @@ -56,11 +57,19 @@ def check_package_duplicates( ) if not items: return True - # duplicated version by owner - if any( - item["owner"]["username"] == owner and item["version"]["name"] == version - for item in items - ): + # duplicated version by owner / system + found = False + for item in items: + if item["owner"]["username"] != owner or item["version"]["name"] != version: + continue + if not system: + found = True + break + published_systems = [] + for f in item["version"]["files"]: + published_systems.extend(f.get("system", [])) + found = set(system).issubset(set(published_systems)) + if found: raise UserSideException( "The package `%s/%s@%s` is already published in the registry" % (owner, name, version) @@ -167,7 +176,7 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals click.echo(tabulate(data, tablefmt="plain")) # look for duplicates - check_package_duplicates(owner, type_, name, version) + check_package_duplicates(owner, type_, name, version, manifest.get("system")) if not non_interactive: click.confirm( From 51b790b767d75c59d5c665fda969054ce2d0212d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 12 Jul 2021 15:06:42 +0300 Subject: [PATCH 087/120] Bump version to 5.2.0a9 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 4fdfaee083..e9944f0ef3 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a8") +VERSION = (5, 2, "0a9") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e9bf2b361f0542dfde90692005d1efec21386d51 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 23 Jul 2021 15:05:01 +0300 Subject: [PATCH 088/120] Update deps and sync docs --- docs | 2 +- setup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index ac2716d8bc..0f8fa69c93 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ac2716d8bc22ea634cb3519e17e92e32c5b94a00 +Subproject commit 0f8fa69c9327b2fa4bc916bde307b821dca84f5b diff --git a/setup.py b/setup.py index 978b3620d2..e60b233d6f 100644 --- a/setup.py +++ b/setup.py @@ -39,12 +39,12 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.32.*") + minimal_requirements.append("zeroconf==0.33.*") home_requirements = [ "aiofiles==0.7.*", - "ajsonrpc==1.1.*", - "starlette==0.14.*", + "ajsonrpc==1.*", + "starlette==0.16.*", "uvicorn==0.14.*", "wsproto==1.0.*", ] From 70153758920ac3aea395f1ec4edb92aba38e8059 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 23 Jul 2021 15:32:02 +0300 Subject: [PATCH 089/120] Docs: Revert "html_favicon" path --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0f8fa69c93..6e6518d567 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0f8fa69c9327b2fa4bc916bde307b821dca84f5b +Subproject commit 6e6518d567566962bf79abf6833c42ecbefbdd8f From 2372d0659162c2dfe6bb2baeffe754a2eaffe068 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Jul 2021 19:26:33 +0300 Subject: [PATCH 090/120] Sync docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 6e6518d567..3bf1910e05 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6e6518d567566962bf79abf6833c42ecbefbdd8f +Subproject commit 3bf1910e0520b37ff0e74f0c5c022745cd163f6b diff --git a/examples b/examples index c6759e3f1b..b4be3d3fa4 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit c6759e3f1b8aa56fea0a4aaa2fc3190b8672ac0f +Subproject commit b4be3d3fa4e7789549765f405614ecf24ec53a24 From 19fa108f61130dd31dce08c90ae1be4716db1f2a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Jul 2021 17:32:22 +0300 Subject: [PATCH 091/120] Docs: Add "Copy" button to CODE blocks --- docs | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs b/docs index 3bf1910e05..ef7532bd35 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3bf1910e0520b37ff0e74f0c5c022745cd163f6b +Subproject commit ef7532bd35b4a1841c33207485c7e834572568d8 diff --git a/tox.ini b/tox.ini index 9f71ed25a8..67323226f2 100644 --- a/tox.ini +++ b/tox.ini @@ -53,6 +53,7 @@ deps = sphinx sphinx_rtd_theme sphinx-notfound-page + sphinx_copybutton restructuredtext-lint commands = sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html From 76b46f59e9d150812882506904fb2e25bc9b33f8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Jul 2021 20:13:53 +0300 Subject: [PATCH 092/120] Fix lib test --- docs | 2 +- tests/commands/test_lib_complex.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs b/docs index ef7532bd35..ed1ae77d1b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ef7532bd35b4a1841c33207485c7e834572568d8 +Subproject commit ed1ae77d1bb42c09d0ec6b4a2aff09e40280522e diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 5b19c30ece..d74bf2071f 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -232,9 +232,9 @@ def test_global_lib_update_check(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) - assert set(["Adafruit PN532", "ESPAsyncTCP", "NeoPixelBus"]) == set( - lib["name"] for lib in output - ) + assert set( + ["Adafruit PN532", "AsyncMqttClient", "ESPAsyncTCP", "NeoPixelBus"] + ) == set(lib["name"] for lib in output) def test_global_lib_update(clirunner, validate_cliresult): @@ -254,7 +254,7 @@ def test_global_lib_update(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) assert result.output.count("[Detached]") == 1 - assert result.output.count("[Up-to-date]") == 14 + assert result.output.count("[Up-to-date]") == 13 # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) From 45d3207dfea6d526cce6d2e439c755f2b84de514 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 31 Jul 2021 18:48:08 +0300 Subject: [PATCH 093/120] Docs: Sync dev-platforms --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ed1ae77d1b..f40f5decd7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ed1ae77d1bb42c09d0ec6b4a2aff09e40280522e +Subproject commit f40f5decd7ec15ac0b0c63c7ea48f4fd32feeee7 From 1282a65bcbc5f03f1e7d0912c1cb12f52fc16cff Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 2 Aug 2021 12:12:52 +0300 Subject: [PATCH 094/120] Update Arduino udev rule to include latest Portenta board Resolves #4014 --- scripts/99-platformio-udev.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index a4fef7b81a..043ce11a61 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -37,7 +37,7 @@ ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVIC ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino boards -ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][023]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" # Arduino SAM-BA From 0607b86818620eee61f170a8f9538947eeb1b18c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Aug 2021 13:10:37 +0300 Subject: [PATCH 095/120] Upgraded build engine to the SCons 4.2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index e9944f0ef3..91c2a4c37d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ "contrib-piohome": "~3.3.4", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~4.40100.2", + "tool-scons": "~4.40200.0", "tool-cppcheck": "~1.241.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.12.0", From 173dbeb24aad72db1d3016dda47a86985b1b612c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Aug 2021 13:11:23 +0300 Subject: [PATCH 096/120] Bump version to 5.2.0b1 --- HISTORY.rst | 20 ++++++++++++-------- platformio/__init__.py | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 94bb9be792..0dd505bf23 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,13 +20,6 @@ PlatformIO Core 5 - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern `__ option - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 `_) -* **Static Code Analysis** - - - Updated analysis tools: - - * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements - * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards - * **Package Management** - Improved a package publishing process: @@ -38,12 +31,23 @@ PlatformIO Core 5 - Added a new option ``--non-interactive`` to `pio package publish `__ command +* **Build System** + + - Upgraded build engine to the SCons 4.2 (`release notes `_) + - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) + +* **Static Code Analysis** + + - Updated analysis tools: + + * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements + * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards + * **Miscellaneous** - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 `_) - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 `_) - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 `_) - - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 91c2a4c37d..dc05d60199 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0a9") +VERSION = (5, 2, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f3489a3b01661735bdcdd89dbbc5cf4512f71272 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Aug 2021 13:52:06 +0300 Subject: [PATCH 097/120] Sync docs --- HISTORY.rst | 2 +- docs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0dd505bf23..d15869869f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,7 +33,7 @@ PlatformIO Core 5 * **Build System** - - Upgraded build engine to the SCons 4.2 (`release notes `_) + - Upgraded build engine to the SCons 4.2 (`release notes `__) - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) * **Static Code Analysis** diff --git a/docs b/docs index f40f5decd7..c3a7367a7e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f40f5decd7ec15ac0b0c63c7ea48f4fd32feeee7 +Subproject commit c3a7367a7e2cf10050a399066642d764aa54385e From 3be0f58c30ffb848dda24721872e8957ff91bf76 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 4 Aug 2021 14:58:54 +0300 Subject: [PATCH 098/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c3a7367a7e..39d9681e4e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c3a7367a7e2cf10050a399066642d764aa54385e +Subproject commit 39d9681e4e9c58163df3a461fe34193e96b02937 From 6cdaf05f98061f0f58ec3a2135bc7413e5e41944 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Aug 2021 18:13:00 +0300 Subject: [PATCH 099/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 39d9681e4e..79cc0abba8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 39d9681e4e9c58163df3a461fe34193e96b02937 +Subproject commit 79cc0abba8949ac676c39b3249967b5b9efbf701 From 097b6d509701f593426bd203b824be63419fab6c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Aug 2021 18:13:22 +0300 Subject: [PATCH 100/120] PyLint fixes --- platformio/proc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/proc.py b/platformio/proc.py index 20f5971085..93be151b8a 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -118,11 +118,11 @@ def exec_command(*args, **kwargs): finally: for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): - kwargs[s].close() + kwargs[s].close() # pylint: disable=no-member for s in ("stdout", "stderr"): if isinstance(kwargs[s], AsyncPipeBase): - result[s[3:]] = kwargs[s].get_buffer() + result[s[3:]] = kwargs[s].get_buffer() # pylint: disable=no-member for key, value in result.items(): if isinstance(value, bytes): From 3ee281aaf9dd83cb793b3a2fb83c2aac4c255282 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 9 Aug 2021 17:46:56 +0300 Subject: [PATCH 101/120] Update SPDX License List to 3.14 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 123875bbea..d17e6a208e 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -253,7 +253,7 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.13" + version = "3.14" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version From efefb02d86d80c0db734d4c90ebf31370bb6ead9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Aug 2021 12:53:30 +0300 Subject: [PATCH 102/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 79cc0abba8..d39427fe60 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 79cc0abba8949ac676c39b3249967b5b9efbf701 +Subproject commit d39427fe60c5a2b3cf8f883690f1cf48d5fdf522 From 8c4d9021c2ac86107857789aaa2e2b6f1cc4e774 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Aug 2021 12:53:49 +0300 Subject: [PATCH 103/120] Update deps --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e60b233d6f..f8e3de439e 100644 --- a/setup.py +++ b/setup.py @@ -39,13 +39,13 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.33.*") + minimal_requirements.append("zeroconf==0.35.*") home_requirements = [ "aiofiles==0.7.*", "ajsonrpc==1.*", "starlette==0.16.*", - "uvicorn==0.14.*", + "uvicorn==0.15.*", "wsproto==1.0.*", ] From 40220f92c11a9140dadebd49e29b468a0c8f4f7c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Aug 2021 15:25:25 +0300 Subject: [PATCH 104/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d39427fe60..c5e6c57e65 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d39427fe60c5a2b3cf8f883690f1cf48d5fdf522 +Subproject commit c5e6c57e658408c2f2922817a108bdccdc7494d0 From cc11402bc9b6ee4fcae200478eb47c5ed224a08a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Aug 2021 15:41:44 +0300 Subject: [PATCH 105/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c5e6c57e65..affd2e3ac5 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c5e6c57e658408c2f2922817a108bdccdc7494d0 +Subproject commit affd2e3ac5234b1d1b3b63b7edb4baf5c3bef9b3 From 554e378dd6a88c05e95e64e36107811f34b49d14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Aug 2021 12:30:38 +0300 Subject: [PATCH 106/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index affd2e3ac5..f2092500e0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit affd2e3ac5234b1d1b3b63b7edb4baf5c3bef9b3 +Subproject commit f2092500e0f3e847b1d97d46512ed0df1ef99686 From b9219a2b629547b1769a5d8ee5c26fa4871c43df Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Aug 2021 12:31:02 +0300 Subject: [PATCH 107/120] Update "zeroconf" deps to 0.36 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8e3de439e..fb592ed254 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ ] if not PY2: - minimal_requirements.append("zeroconf==0.35.*") + minimal_requirements.append("zeroconf==0.36.*") home_requirements = [ "aiofiles==0.7.*", From d819617d2b2edfe7326c99f40d901d06f7bd7df8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Aug 2021 13:10:07 +0300 Subject: [PATCH 108/120] Specify encoding for "open()" functions --- platformio/app.py | 2 +- platformio/builder/main.py | 6 +++++- platformio/builder/tools/compilation_db.py | 2 +- platformio/builder/tools/piolib.py | 8 +++++--- platformio/builder/tools/piomaxlen.py | 2 +- platformio/builder/tools/piosize.py | 6 ++++-- platformio/cache.py | 8 ++++---- platformio/commands/check/tools/pvsstudio.py | 4 ++-- platformio/commands/home/rpc/handlers/project.py | 2 +- platformio/commands/project.py | 8 ++++---- platformio/commands/remote/client/device_monitor.py | 6 +++++- platformio/commands/remote/command.py | 2 +- platformio/commands/run/helpers.py | 4 ++-- platformio/commands/system/completion.py | 4 ++-- platformio/commands/test/processor.py | 2 +- platformio/commands/upgrade.py | 2 +- platformio/debug/helpers.py | 4 ++-- platformio/debug/process/gdb.py | 2 +- platformio/fs.py | 4 ++-- platformio/package/lockfile.py | 4 +++- platformio/package/manager/core.py | 4 +++- platformio/package/manager/library.py | 4 +++- platformio/package/meta.py | 4 ++-- platformio/package/pack.py | 4 +++- platformio/proc.py | 2 +- platformio/project/config.py | 2 +- platformio/project/helpers.py | 2 +- tests/package/test_manager.py | 2 +- 28 files changed, 63 insertions(+), 43 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index ae483f1f56..e630dd3ceb 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -114,7 +114,7 @@ def __enter__(self): def __exit__(self, type_, value, traceback): if self.modified: try: - with open(self.path, "w") as fp: + with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 5b09ad772f..04d6e342ec 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -226,7 +226,11 @@ projenv = env data = projenv.DumpIDEData(env) # dump to file for the further reading by project.helpers.load_project_ide_data - with open(projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), "w") as fp: + with open( + projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), + mode="w", + encoding="utf8", + ) as fp: json.dump(data, fp) click.echo("\n%s\n" % json.dumps(data)) # pylint: disable=undefined-variable env.Exit(0) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index f91fa3ab35..6dff17464e 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -152,7 +152,7 @@ def WriteCompilationDb(target, source, env): item["file"] = os.path.abspath(item["file"]) entries.append(item) - with open(str(target[0]), "w") as target_file: + with open(str(target[0]), mode="w", encoding="utf8") as target_file: json.dump( entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") ) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3849a91bdf..aa4b2fe607 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -86,7 +86,9 @@ def get_used_frameworks(env, path): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - with io.open(os.path.join(root, fname), errors="ignore") as fp: + with io.open( + os.path.join(root, fname), encoding="utf8", errors="ignore" + ) as fp: content = fp.read() if not content: continue @@ -671,7 +673,7 @@ def _mbed_lib_conf_parse_macros(self, mbed_lib_path): def _mbed_conf_append_macros(self, mbed_config_path, macros): lines = [] - with open(mbed_config_path) as fp: + with open(mbed_config_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if line == "#endif": @@ -690,7 +692,7 @@ def _mbed_conf_append_macros(self, mbed_config_path, macros): if len(tokens) < 2 or tokens[1] not in macros: lines.append(line) lines.append("") - with open(mbed_config_path, "w") as fp: + with open(mbed_config_path, mode="w", encoding="utf8") as fp: fp.write("\n".join(lines)) diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 02622b9d0e..c7360418ce 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -65,7 +65,7 @@ def _file_long_data(env, data): ) if os.path.isfile(tmp_file): return tmp_file - with open(tmp_file, "w") as fp: + with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write(data) return tmp_file diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index e194511a3f..6ef8d94515 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -37,7 +37,7 @@ def _run_tool(cmd, env, tool_args): makedirs(build_dir) tmp_file = join(build_dir, "size-data-longcmd.txt") - with open(tmp_file, "w") as fp: + with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write("\n".join(tool_args)) cmd.append("@" + tmp_file) @@ -241,7 +241,9 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument file_data.update(v) data["memory"]["files"].append(file_data) - with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: + with open( + join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8" + ) as fp: fp.write(json.dumps(data)) diff --git a/platformio/cache.py b/platformio/cache.py index 8737725993..e8b7982da1 100644 --- a/platformio/cache.py +++ b/platformio/cache.py @@ -78,9 +78,9 @@ def set(self, key, data, valid): if not os.path.isdir(os.path.dirname(cache_path)): os.makedirs(os.path.dirname(cache_path)) try: - with codecs.open(cache_path, "wb", encoding="utf8") as fp: + with codecs.open(cache_path, mode="wb", encoding="utf8") as fp: fp.write(data) - with open(self._db_path, "a") as fp: + with open(self._db_path, mode="a", encoding="utf8") as fp: fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) except UnicodeError: if os.path.isfile(cache_path): @@ -102,7 +102,7 @@ def delete(self, keys=None): paths_for_delete = [self.get_cache_path(k) for k in keys] found = False newlines = [] - with open(self._db_path) as fp: + with open(self._db_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if "=" not in line: @@ -129,7 +129,7 @@ def delete(self, keys=None): pass if found and self._lock_dbindex(): - with open(self._db_path, "w") as fp: + with open(self._db_path, mode="w", encoding="utf8") as fp: fp.write("\n".join(newlines) + "\n") self._unlock_dbindex() diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index e95e202c2e..36298d259d 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -40,13 +40,13 @@ def __init__(self, *args, **kwargs): ) super(PvsStudioCheckTool, self).__init__(*args, **kwargs) - with open(self._tmp_cfg_file, "w") as fp: + with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp: fp.write( "exclude-path = " + self.config.get_optional_dir("packages").replace("\\", "/") ) - with open(self._tmp_cmd_file, "w") as fp: + with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp: fp.write( " ".join( ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes] diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 8bac4d1c52..2263beb19a 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -261,7 +261,7 @@ def _generate_project_main(project_dir, board, framework): return project_dir if not os.path.isdir(src_dir): os.makedirs(src_dir) - with open(main_path, "w") as fp: + with open(main_path, mode="w", encoding="utf8") as fp: fp.write(main_content.strip()) return project_dir diff --git a/platformio/commands/project.py b/platformio/commands/project.py index e7809e2578..d45d6dcd98 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -240,7 +240,7 @@ def init_base_project(project_dir): def init_include_readme(include_dir): - with open(os.path.join(include_dir, "README"), "w") as fp: + with open(os.path.join(include_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for project header files. @@ -286,7 +286,7 @@ def init_include_readme(include_dir): def init_lib_readme(lib_dir): - with open(os.path.join(lib_dir, "README"), "w") as fp: + with open(os.path.join(lib_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for project specific (private) libraries. @@ -339,7 +339,7 @@ def init_lib_readme(lib_dir): def init_test_readme(test_dir): - with open(os.path.join(test_dir, "README"), "w") as fp: + with open(os.path.join(test_dir, "README"), mode="w", encoding="utf8") as fp: fp.write( """ This directory is intended for PlatformIO Unit Testing and project tests. @@ -360,7 +360,7 @@ def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): return - with open(conf_path, "w") as fp: + with open(conf_path, mode="w", encoding="utf8") as fp: fp.write(".pio\n") diff --git a/platformio/commands/remote/client/device_monitor.py b/platformio/commands/remote/client/device_monitor.py index 990bb4337e..4c68f85f15 100644 --- a/platformio/commands/remote/client/device_monitor.py +++ b/platformio/commands/remote/client/device_monitor.py @@ -173,7 +173,11 @@ def cb_async_result(self, result): address = port.getHost() self.log.debug("Serial Bridge is started on {address!r}", address=address) if "sock" in self.cmd_options: - with open(os.path.join(self.cmd_options["sock"], "sock"), "w") as fp: + with open( + os.path.join(self.cmd_options["sock"], "sock"), + mode="w", + encoding="utf8", + ) as fp: fp.write("socket://localhost:%d" % address.port) def client_terminal_stopped(self): diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index edf6b07776..f9486d51eb 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -350,7 +350,7 @@ def _tx_target(sock_dir): sleep(0.1) if not t.is_alive(): return - with open(sock_file) as fp: + with open(sock_file, encoding="utf8") as fp: kwargs["port"] = fp.read() ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index ab362c2426..c976d9352a 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -54,11 +54,11 @@ def clean_build_dir(build_dir, config): if isdir(build_dir): # check project structure if isfile(checksum_file): - with open(checksum_file) as fp: + with open(checksum_file, encoding="utf8") as fp: if fp.read() == checksum: return fs.rmtree(build_dir) makedirs(build_dir) - with open(checksum_file, "w") as fp: + with open(checksum_file, mode="w", encoding="utf8") as fp: fp.write(checksum) diff --git a/platformio/commands/system/completion.py b/platformio/commands/system/completion.py index 1a96920387..012df5fce5 100644 --- a/platformio/commands/system/completion.py +++ b/platformio/commands/system/completion.py @@ -42,7 +42,7 @@ def is_completion_code_installed(shell, path): import click_completion # pylint: disable=import-error,import-outside-toplevel - with open(path) as fp: + with open(path, encoding="utf8") as fp: return click_completion.get_code(shell=shell) in fp.read() @@ -64,7 +64,7 @@ def uninstall_completion_code(shell, path): import click_completion # pylint: disable=import-error,import-outside-toplevel - with open(path, "r+") as fp: + with open(path, "r+", encoding="utf8") as fp: contents = fp.read() fp.seek(0) fp.truncate() diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index ea975e6260..3e9a519d4b 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -224,7 +224,7 @@ def delete_tmptest_files(test_dir): test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")), ) - with open(tmp_file, "w") as fp: + with open(tmp_file, mode="w", encoding="utf8") as fp: fp.write(data) atexit.register(delete_tmptest_files, test_dir) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index d8f6e78c5c..66eda80490 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -103,7 +103,7 @@ def get_pip_package(to_develop): os.makedirs(cache_dir) pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: - with open(pkg_name, "w") as fp: + with open(pkg_name, "w", encoding="utf8") as fp: r = exec_command( ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True ) diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index 6b4d427898..5bac5d61e9 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -154,11 +154,11 @@ def is_prog_obsolete(prog_path): new_digest = shasum.hexdigest() old_digest = None if isfile(prog_hash_path): - with open(prog_hash_path) as fp: + with open(prog_hash_path, encoding="utf8") as fp: old_digest = fp.read() if new_digest == old_digest: return False - with open(prog_hash_path, "w") as fp: + with open(prog_hash_path, mode="w", encoding="utf8") as fp: fp.write(new_digest) return True diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 836c637f39..079b8b5304 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -105,7 +105,7 @@ def generate_init_script(self, dst): footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] commands = banner + commands + footer - with open(dst, "w") as fp: + with open(dst, mode="w", encoding="utf8") as fp: fp.write("\n".join(self.debug_config.reveal_patterns(commands))) def stdin_data_received(self, data): diff --git a/platformio/fs.py b/platformio/fs.py index 655c2ee87c..258fe53078 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -52,7 +52,7 @@ def get_source_dir(): def load_json(file_path): try: - with open(file_path, "r") as f: + with open(file_path, mode="r", encoding="utf8") as f: return json.load(f) except ValueError: raise exception.InvalidJSONFile(file_path) @@ -102,7 +102,7 @@ def ensure_udev_rules(): def _rules_to_set(rules_path): result = set() - with open(rules_path) as fp: + with open(rules_path, encoding="utf8") as fp: for line in fp.readlines(): line = line.strip() if not line or line.startswith("#"): diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 6cfc46ef1c..8d6c837cc4 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -62,7 +62,9 @@ def _lock(self): else: raise LockFileExists - self._fp = open(self._lock_path, "w") # pylint: disable=consider-using-with + self._fp = open( + self._lock_path, mode="w", encoding="utf8" + ) # pylint: disable=consider-using-with try: if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 63251af319..270ce8e61b 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -156,7 +156,9 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): subprocess.check_call(args + [dep]) # build manifests - with open(os.path.join(target_dir, "package.json"), "w") as fp: + with open( + os.path.join(target_dir, "package.json"), mode="w", encoding="utf8" + ) as fp: json.dump( dict( name="contrib-pysite", diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 3f77846b7e..0396030698 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -44,7 +44,9 @@ def find_pkg_root(self, path, spec): root_dir = self.find_library_root(path) # automatically generate library manifest - with open(os.path.join(root_dir, "library.json"), "w") as fp: + with open( + os.path.join(root_dir, "library.json"), mode="w", encoding="utf8" + ) as fp: json.dump( dict( name=spec.name, diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 74af191643..309c5fd8b0 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -382,12 +382,12 @@ def as_dict(self): ) def dump(self, path): - with open(path, "w") as fp: + with open(path, mode="w", encoding="utf8") as fp: return json.dump(self.as_dict(), fp) @staticmethod def load(path): - with open(path) as fp: + with open(path, encoding="utf8") as fp: data = json.load(fp) if data["spec"]: data["spec"] = PackageSpec(**data["spec"]) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 6644311cf0..8688d1711b 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -181,7 +181,9 @@ def _create_tarball(self, src, dst, manifest): and os.path.isdir(os.path.join(src, include[0])) ): src = os.path.join(src, include[0]) - with open(os.path.join(src, "library.json"), "w") as fp: + with open( + os.path.join(src, "library.json"), mode="w", encoding="utf8" + ) as fp: manifest_updated = manifest.copy() del manifest_updated["export"]["include"] json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) diff --git a/platformio/proc.py b/platformio/proc.py index 93be151b8a..0484cc42a0 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -158,7 +158,7 @@ def is_container(): return True if not os.path.isfile("/proc/1/cgroup"): return False - with open("/proc/1/cgroup") as fp: + with open("/proc/1/cgroup", encoding="utf8") as fp: return ":/docker/" in fp.read() diff --git a/platformio/project/config.py b/platformio/project/config.py index 0d1005071c..13ff8e2b62 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -456,7 +456,7 @@ def save(self, path=None): path = path or self.path if path in self._instances: del self._instances[path] - with open(path or self.path, "w+") as fp: + with open(path or self.path, mode="w+", encoding="utf8") as fp: fp.write(CONFIG_HEADER.strip() + "\n\n") self._parser.write(fp) fp.seek(0) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index c41744a10a..655e968540 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -177,6 +177,6 @@ def _load_cached_project_ide_data(project_dir, env_names): for name in env_names: if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): continue - with open(os.path.join(build_dir, name, "idedata.json")) as fp: + with open(os.path.join(build_dir, name, "idedata.json"), encoding="utf8") as fp: result[name] = json.load(fp) return result diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 920aee049f..19d4a98e97 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -46,7 +46,7 @@ def test_download(isolated_pio_core): lm.cleanup_expired_downloads() assert not os.path.isfile(archive_path) # check that key is deleted from DB - with open(lm.get_download_usagedb_path()) as fp: + with open(lm.get_download_usagedb_path(), encoding="utf8") as fp: assert os.path.basename(archive_path) not in fp.read() From 131f4be4eadbbcf7428a3162e885388d16b4e4e5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Aug 2021 13:14:40 +0300 Subject: [PATCH 109/120] Fix PyLint's "use-dict-literal" and "use-list-literal" --- platformio/builder/tools/piolib.py | 6 +++--- platformio/builder/tools/piosize.py | 4 ++-- platformio/commands/check/command.py | 4 ++-- platformio/commands/check/tools/cppcheck.py | 2 +- platformio/package/lockfile.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index aa4b2fe607..755900da6f 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -128,9 +128,9 @@ def __init__(self, env, path, manifest=None, verbose=False): self._is_dependent = False self._is_built = False - self._depbuilders = list() - self._circular_deps = list() - self._processed_files = list() + self._depbuilders = [] + self._circular_deps = [] + self._processed_files = [] # reset source filter, could be overridden with extra script self.env["SRC_FILTER"] = "" diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 6ef8d94515..7eece12a14 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -220,7 +220,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument "sections": sections, } - files = dict() + files = {} for symbol in _collect_symbols_info(env, elffile, elf_path, sections): file_path = symbol.get("file") or "unknown" if not files.get(file_path, {}): @@ -235,7 +235,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument files[file_path]["symbols"].append(symbol) - data["memory"]["files"] = list() + data["memory"]["files"] = [] for k, v in files.items(): file_data = {"path": k} file_data.update(v) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 2d4bb1091a..87673f992c 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -215,7 +215,7 @@ def print_processing_footer(result): def collect_component_stats(result): - components = dict() + components = {} def _append_defect(component, defect): if not components.get(component): @@ -250,7 +250,7 @@ def print_defects_stats(results): severity_labels = list(DefectItem.SEVERITY_LABELS.values()) severity_labels.reverse() - tabular_data = list() + tabular_data = [] for k, v in component_stats.items(): tool_defect = [v.get(s, 0) for s in severity_labels] tabular_data.append([k] + tool_defect) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index c1fe3e3bb2..8dd6041a4b 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -64,7 +64,7 @@ def parse_defect(self, raw_line): if any(f not in self._buffer for f in self.defect_fields): return None - args = dict() + args = {} for field in self._buffer.split(self._field_delimiter): field = field.strip().replace('"', "") name, value = field.split("=", 1) diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 8d6c837cc4..b04cd42879 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -62,9 +62,9 @@ def _lock(self): else: raise LockFileExists - self._fp = open( + self._fp = open( # pylint: disable=consider-using-with self._lock_path, mode="w", encoding="utf8" - ) # pylint: disable=consider-using-with + ) try: if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) From 55b786d9f0c3aa2f7ff5d592e2871d39449926c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Aug 2021 13:21:46 +0300 Subject: [PATCH 110/120] Use byte-mode for writing binary file --- platformio/commands/upgrade.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 66eda80490..25e5bd01a5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -40,7 +40,7 @@ def cli(dev): to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") cmds = ( - ["pip", "install", "--upgrade", get_pip_package(to_develop)], + ["pip", "install", "--upgrade", download_dist_package(to_develop)], ["platformio", "--version"], ) @@ -94,7 +94,7 @@ def cli(dev): return True -def get_pip_package(to_develop): +def download_dist_package(to_develop): if not to_develop: return "platformio" dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" @@ -103,7 +103,7 @@ def get_pip_package(to_develop): os.makedirs(cache_dir) pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: - with open(pkg_name, "w", encoding="utf8") as fp: + with open(pkg_name, "wb") as fp: r = exec_command( ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True ) From 7c755d4e2d6b4d2ce02b25d69ba93792ef0680b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 31 Aug 2021 16:23:24 +0300 Subject: [PATCH 111/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f2092500e0..3892da7949 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f2092500e0f3e847b1d97d46512ed0df1ef99686 +Subproject commit 3892da7949db6222a14ca3372b503ba3a9ef7a38 From e1dc12c14dd344415cb74ed7793f7a776263043a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Sep 2021 12:47:17 +0300 Subject: [PATCH 112/120] Docs: Document "platformio-ide.pioHomeServerHttpHost" setting for VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3892da7949..3e4566c81a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3892da7949db6222a14ca3372b503ba3a9ef7a38 +Subproject commit 3e4566c81a25884efc5becdb6e21afcc8bb3b428 From d97ed52e9147054bde4d6c414b9990f29d7f486e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Sep 2021 15:17:59 +0300 Subject: [PATCH 113/120] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3e4566c81a..90aa599093 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3e4566c81a25884efc5becdb6e21afcc8bb3b428 +Subproject commit 90aa5990935d04f822c9271b48bdad2e484c07c0 From 63a2465bac2cc548ea058aa712c141957bfae321 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Sep 2021 15:32:30 +0300 Subject: [PATCH 114/120] Update check tools to the latest available // Resolve #4041 --- HISTORY.rst | 5 +++-- platformio/__init__.py | 6 +++--- tests/commands/test_check.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d15869869f..41f993aff6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -40,8 +40,9 @@ PlatformIO Core 5 - Updated analysis tools: - * `Cppcheck `__ v2.4.1 with new checks and MISRA improvements - * `PVS-Studio `__ v7.12 with new diagnostics and extended capabilities for security and safety standards + * `Clang-Tidy `__ v12.0.1 with new modules and extended checks list + * `Cppcheck `__ v2.5.0 with improved code analysis and MISRA improvements + * `PVS-Studio `__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics * **Miscellaneous** diff --git a/platformio/__init__.py b/platformio/__init__.py index dc05d60199..73d2468eea 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -51,9 +51,9 @@ "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~4.40200.0", - "tool-cppcheck": "~1.241.0", - "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.12.0", + "tool-cppcheck": "~1.250.0", + "tool-clangtidy": "~1.120001.0", + "tool-pvs-studio": "~7.14.0", } __check_internet_hosts__ = [ diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 17f8ed2758..98400c177b 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -67,9 +67,9 @@ // PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com """ -EXPECTED_ERRORS = 4 +EXPECTED_ERRORS = 5 EXPECTED_WARNINGS = 1 -EXPECTED_STYLE = 1 +EXPECTED_STYLE = 2 EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE From d10cbb28236bf520731a337fb3b661d352b84ba2 Mon Sep 17 00:00:00 2001 From: Dmitry Antyneskul Date: Mon, 13 Sep 2021 02:36:56 -0700 Subject: [PATCH 115/120] Fix link to clang-tidy (#4049) --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 41f993aff6..da37187d65 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -40,7 +40,7 @@ PlatformIO Core 5 - Updated analysis tools: - * `Clang-Tidy `__ v12.0.1 with new modules and extended checks list + * `Clang-Tidy `__ v12.0.1 with new modules and extended checks list * `Cppcheck `__ v2.5.0 with improved code analysis and MISRA improvements * `PVS-Studio `__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics From 775357dd946bc820c73da8fd626a7d98a2fde167 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Sep 2021 13:31:53 +0300 Subject: [PATCH 116/120] Better error handling if git is not installed // Resolve #4013 --- platformio/package/vcsclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index eb85ae3781..adbcd6f5e7 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -150,7 +150,7 @@ def configure(cls): if path: proc.append_env_path("PATH", path) return True - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): pass return False From 03a23876a76412277c2bab101c817fa5f3248223 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Sep 2021 14:04:33 +0300 Subject: [PATCH 117/120] Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files // Resolve #4019 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index da37187d65..b2c73dac4b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ PlatformIO Core 5 - Upgraded build engine to the SCons 4.2 (`release notes `__) - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) + - Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files (`issue #4019 `_) * **Static Code Analysis** diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c3acf5557b..304c5bae61 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -348,9 +348,10 @@ def BuildFrameworks(env, frameworks): def BuildLibrary(env, variant_dir, src_dir, src_filter=None): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) - return env.StaticLibrary( - env.subst(variant_dir), env.CollectBuildFiles(variant_dir, src_dir, src_filter) - ) + nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) + if nodes: + return env.StaticLibrary(env.subst(variant_dir), nodes) + return env.BuildSources(variant_dir, src_dir, src_filter) def BuildSources(env, variant_dir, src_dir, src_filter=None): From dce5a39b10d50e402813b17c578c05dbfe79e55b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Sep 2021 14:48:48 +0300 Subject: [PATCH 118/120] Process "precompiled" and "ldflags" properties of the "library.properties" manifest // Resolve #3994 --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b2c73dac4b..cc55413096 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ PlatformIO Core 5 * **Build System** + - Process "precompiled" and "ldflags" properties of the "library.properties" manifest (`issue #3994 `_) - Upgraded build engine to the SCons 4.2 (`release notes `__) - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 `_) - Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files (`issue #4019 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 755900da6f..425cbc017d 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -547,6 +547,21 @@ def is_frameworks_compatible(self, frameworks): def is_platforms_compatible(self, platforms): return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) + @property + def build_flags(self): + ldflags = [ + LibBuilderBase.build_flags.fget(self), # pylint: disable=no-member + self._manifest.get("ldflags"), + ] + if self._manifest.get("precompiled") in ("true", "full"): + # add to LDPATH {build.mcu} folder + board_config = self.env.BoardConfig() + self.env.PrependUnique( + LIBPATH=os.path.join(self.src_dir, board_config.get("build.cpu")) + ) + ldflags = [flag for flag in ldflags if flag] # remove empty + return " ".join(ldflags) if ldflags else None + class MbedLibBuilder(LibBuilderBase): def load_manifest(self): From 55408f6ccb9c90a7e0abaea548b688bd780639dc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Sep 2021 14:56:24 +0300 Subject: [PATCH 119/120] Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files // Resolve #4019 --- platformio/builder/tools/piolib.py | 18 ++++++++++++++---- platformio/builder/tools/platformio.py | 8 +++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 425cbc017d..186485d251 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -461,12 +461,22 @@ def build(self): for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"): self.env.PrependUnique(**{key: lb.env.get(key)}) - if self.lib_archive: - libs.append( - self.env.BuildLibrary(self.build_dir, self.src_dir, self.src_filter) + do_not_archive = not self.lib_archive + if not do_not_archive: + nodes = self.env.CollectBuildFiles( + self.build_dir, self.src_dir, self.src_filter ) - else: + if nodes: + libs.append( + self.env.BuildLibrary( + self.build_dir, self.src_dir, self.src_filter, nodes + ) + ) + else: + do_not_archive = True + if do_not_archive: self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter) + return libs diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 304c5bae61..a9355c4937 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -346,12 +346,10 @@ def BuildFrameworks(env, frameworks): env.Exit(1) -def BuildLibrary(env, variant_dir, src_dir, src_filter=None): +def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) - nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) - if nodes: - return env.StaticLibrary(env.subst(variant_dir), nodes) - return env.BuildSources(variant_dir, src_dir, src_filter) + nodes = nodes or env.CollectBuildFiles(variant_dir, src_dir, src_filter) + return env.StaticLibrary(env.subst(variant_dir), nodes) def BuildSources(env, variant_dir, src_dir, src_filter=None): From 9528083a665aabb404dea4f7255c567d66144e9d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 Sep 2021 18:59:53 +0300 Subject: [PATCH 120/120] Bump version to 5.2.0 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index cc55413096..fa81cec093 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.2.0 (2021-??-??) +5.2.0 (2021-09-13) ~~~~~~~~~~~~~~~~~~ * **PlatformIO Debugging** diff --git a/docs b/docs index 90aa599093..c9d2ef9abe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 90aa5990935d04f822c9271b48bdad2e484c07c0 +Subproject commit c9d2ef9abe4d349465609616fc5c2b69b4a0823d diff --git a/platformio/__init__.py b/platformio/__init__.py index 73d2468eea..0314286b78 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "0b1") +VERSION = (5, 2, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"