Sync release 0.35.1 into Develop (#559)

Co-authored-by: Artalus <[email protected]>
Co-authored-by: Sascha Bannier <[email protected]>

README.md

@@ -1099,6 +1099,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor:
 - **upload_only_when_tag**: Will try to upload only if the branch is a tag. Default [False]
 - **upload_only_recipe**: If defined, will try to upload **only** the recipes. The built packages will **not** be uploaded. Default [False]
 - **upload_dependencies**: Will try to upload dependencies to your remote. Default [False]
+- **upload_force**: Will try to force uploaded all packages. Default [True]
 - **build_types**: List containing specific build types. Default ["Release", "Debug"]
 - **cppstds**: List containing specific cpp standards. Default None
 - **skip_check_credentials**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the
@@ -1219,6 +1220,7 @@ This is especially useful for CI integration.
 - **CONAN_UPLOAD_ONLY_WHEN_TAG**: If defined, will try to upload the packages only when the current branch is a tag.
 - **CONAN_UPLOAD_ONLY_RECIPE**: If defined, will try to upload **only** the recipes. The built packages will **not** be uploaded.
 - **CONAN_UPLOAD_DEPENDENCIES**: If defined, will try to upload the listed package dependencies to your remote.
+- **CONAN_UPLOAD_FORCE**: If defined, will try to force upload all packages. Default is `True`.
 
 - **CONAN_SKIP_CHECK_CREDENTIALS**: Conan will skip checking the user credentials before building the packages. And if no user/remote is specified, will try to upload with the
   already stored credentiales in the local cache. Default [False]

cpt/__init__.py

@@ -1,5 +1,5 @@
 
-__version__ = '0.36.0-dev'
+__version__ = '0.35.1'
 
 
 def get_client_version():

cpt/packager.py

@@ -21,9 +21,10 @@
 from cpt.profiles import get_profiles, save_profile_to_tmp
 from cpt.remotes import RemotesManager
 from cpt.runner import CreateRunner, DockerCreateRunner
-from cpt.tools import get_bool_from_env
+from cpt.tools import get_bool_from_env, get_custom_bool_from_env
 from cpt.tools import split_colon_env
 from cpt.uploader import Uploader
+from cpt.config import ConfigManager
 
 
 def load_cf_class(path, conan_api):
@@ -52,12 +53,12 @@ def load_cf_class(path, conan_api):
             conan_api.create_app()
         remotes = conan_api.app.cache.registry.load_remotes()
         conan_api.app.python_requires.enable_remotes(remotes=remotes)
-        conan_api.app.pyreq_loader.enable_remotes(remotes=remotes)
         if client_version < Version("1.20.0"):
             return conan_api.app.loader.load_class(path)
         elif client_version < Version("1.21.0"):
             return conan_api.app.loader.load_basic(path)
         else:
+            conan_api.app.pyreq_loader.enable_remotes(remotes=remotes)
             return conan_api.app.loader.load_named(path, None, None, None, None)
 
 
@@ -114,6 +115,7 @@ def __init__(self, username=None, channel=None, runner=None,
                  upload_only_when_stable=None,
                  upload_only_when_tag=None,
                  upload_only_recipe=None,
+                 upload_force=None,
                  build_types=None,
                  cppstds=None,
                  skip_check_credentials=False,
@@ -184,10 +186,12 @@ def __init__(self, username=None, channel=None, runner=None,
             self.upload_only_when_tag = get_bool_from_env("CONAN_UPLOAD_ONLY_WHEN_TAG")
 
         self.upload_only_recipe = upload_only_recipe or get_bool_from_env("CONAN_UPLOAD_ONLY_RECIPE")
+        self.upload_force = upload_force if upload_force is not None \
+                            else get_custom_bool_from_env("CONAN_UPLOAD_FORCE", True)
 
         self.remotes_manager.add_remotes_to_conan()
         self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager,
-                                 self.printer, self.upload_retry)
+                                 self.printer, self.upload_retry, self.upload_force)
 
         self._builds = []
         self._named_builds = {}
@@ -648,6 +652,8 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                 self.printer.print_message("Using specified default "
                                            "base profile: %s" % base_profile_name)
                 self.printer.print_message("**************************************************")
+                if self.config_url:
+                    ConfigManager(self.conan_api, self.printer).install(url=self.config_url, args=self.config_args)
 
             profile_text, base_profile_text = get_profiles(self.client_cache, build,
                                                            base_profile_name)
@@ -687,6 +693,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                                        upload=self._upload_enabled(),
                                        upload_retry=self.upload_retry,
                                        upload_only_recipe=self.upload_only_recipe,
+                                       upload_force=self.upload_force,
                                        runner=self.runner,
                                        docker_shell=self.docker_shell,
                                        docker_conan_home=self.docker_conan_home,

cpt/run_in_docker.py

@@ -3,18 +3,26 @@
 from conans import tools
 from conans.client.conan_api import Conan
 from conans.model.ref import ConanFileReference
+from conans.model.version import Version
 
 from cpt.auth import AuthManager
 from cpt.printer import Printer
 from cpt.profiles import save_profile_to_tmp
 from cpt.remotes import RemotesManager
 from cpt.runner import CreateRunner, unscape_env
 from cpt.uploader import Uploader
+from cpt import get_client_version
 
 
 def run():
-    # Get all from environ
-    conan_api, client_cache, _ = Conan.factory()
+    conan_version = get_client_version()
+    if conan_version < Version("1.18.0"):
+        conan_api, client_cache, _ = Conan.factory()
+    else:
+        conan_api, _, _ = Conan.factory()
+        conan_api.create_app()
+        client_cache = conan_api.app.cache
+
     printer = Printer()
 
     remotes_manager = RemotesManager(conan_api, printer)
@@ -24,7 +32,8 @@ def run():
 
     upload_retry = os.getenv("CPT_UPLOAD_RETRY")
     upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE")
-    uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry)
+    upload_force = os.getenv("CPT_UPLOAD_FORCE")
+    uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry, upload_force)
     build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY"))
     test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER"))
     reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))

cpt/runner.py

@@ -2,6 +2,7 @@
 import sys
 import subprocess
 import re
+import time
 from collections import namedtuple
 
 from conans import tools
@@ -170,6 +171,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
                  docker_image_skip_pull=False,
                  always_update_conan_in_docker=False,
                  upload=False, upload_retry=None, upload_only_recipe=None,
+                 upload_force=None,
                  runner=None,
                  docker_shell="", docker_conan_home="",
                  docker_platform_param="", docker_run_options="",
@@ -192,6 +194,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
         self._upload = upload
         self._upload_retry = upload_retry
         self._upload_only_recipe = upload_only_recipe
+        self._upload_force = upload_force
         self._reference = reference
         self._conan_pip_package = conan_pip_package
         self._build_policy = build_policy
@@ -325,9 +328,15 @@ def run(self, pull_image=True, docker_entry_script=None):
 
     def pull_image(self):
         with self.printer.foldable_output("docker pull"):
-            ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image))
-            if ret != 0:
-                raise Exception("Error pulling the image: %s" % self._docker_image)
+            for retry in range(1, 4):
+                ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image))
+                if ret == 0:
+                    break
+                elif retry == 3:
+                    raise Exception("Error pulling the image: %s" % self._docker_image)
+                self.printer.print_message("Could not pull docker image '{}'. Retry ({})"
+                                           .format(self._docker_image, retry))
+                time.sleep(3)
 
     def get_env_vars(self):
         ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and
@@ -338,11 +347,12 @@ def get_env_vars(self):
         ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text)
         ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name)
 
-        ret["CONAN_USERNAME"] = escape_env(self._reference.user)
+        ret["CONAN_USERNAME"] = escape_env(self._reference.user or ret.get("CONAN_USERNAME"))
         ret["CONAN_TEMP_TEST_FOLDER"] = "1"  # test package folder to a temp one
         ret["CPT_UPLOAD_ENABLED"] = self._upload
         ret["CPT_UPLOAD_RETRY"] = self._upload_retry
         ret["CPT_UPLOAD_ONLY_RECIPE"] = self._upload_only_recipe
+        ret["CPT_UPLOAD_FORCE"] = self._upload_force
         ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)
         ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder)
         ret["CPT_CONFIG_URL"] = escape_env(self._config_url)
@@ -367,7 +377,7 @@ def unscape_env(text):
 def escape_env(text):
     if not text:
         return text
-    return text.replace("\n", "@@").replace('"', '||')
+    return text.replace("\r", "").replace("\n", "@@").replace('"', '||')
 
 
 class PrintRunner(object):

cpt/test/integration/docker_test.py

@@ -3,6 +3,7 @@
 import time
 import textwrap
 
+
 from conans import tools
 from conans.model.ref import ConanFileReference
 from conans.model.version import Version
@@ -34,13 +35,13 @@ def test_docker(self):
         client_version = get_client_version()
         ci_manager = MockCIManager()
         unique_ref = "zlib/%s" % str(time.time())
-        conanfile = """from conans import ConanFile
-import os
-
-class Pkg(ConanFile):
-    settings = "os", "compiler", "build_type", "arch"
+        conanfile = textwrap.dedent("""
+                from conans import ConanFile
+                import os
 
-"""
+                class Pkg(ConanFile):
+                    settings = "os", "compiler", "build_type", "arch"
+            """)
 
         self.save_conanfile(conanfile)
         with tools.environment_append({"CONAN_DOCKER_RUN_OPTIONS": "--network=host -v{}:/tmp/cpt".format(self.root_project_folder),
@@ -53,7 +54,7 @@ class Pkg(ConanFile):
                                        "CONAN_PASSWORD": "demo"}):
 
             self.packager = ConanMultiPackager(channel="mychannel",
-                                               gcc_versions=["6"],
+                                               gcc_versions=["8"],
                                                archs=["x86", "x86_64"],
                                                build_types=["Release"],
                                                reference=unique_ref,
@@ -96,7 +97,7 @@ class Pkg(ConanFile):
                                        "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE",
                                        "CONAN_UPLOAD_ONLY_WHEN_STABLE": "1"}):
             self.packager = ConanMultiPackager(channel="mychannel",
-                                               gcc_versions=["6"],
+                                               gcc_versions=["8"],
                                                archs=["x86", "x86_64"],
                                                build_types=["Release"],
                                                reference=unique_ref,
@@ -116,16 +117,17 @@ class Pkg(ConanFile):
 
     @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker")
     def test_docker_run_options(self):
-        conanfile = """from conans import ConanFile
-import os
+        conanfile = textwrap.dedent("""
+                from conans import ConanFile
+                import os
 
-class Pkg(ConanFile):
-    settings = "os", "compiler", "build_type", "arch"
-    requires = "zlib/1.2.11@conan/stable"
+                class Pkg(ConanFile):
+                    settings = "os", "compiler", "build_type", "arch"
+                    requires = "zlib/1.2.11@conan/stable"
 
-    def build(self):
-        pass
-"""
+                    def build(self):
+                        pass
+            """)
         self.save_conanfile(conanfile)
         # Validate by Environemnt Variable
         with tools.environment_append({"CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt",
@@ -204,16 +206,17 @@ def test_docker_run_android(self):
 
     @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker")
     def test_docker_custom_pip_command(self):
-        conanfile = """from conans import ConanFile
-        import os
+        conanfile = textwrap.dedent("""
+                from conans import ConanFile
+                import os
 
-        class Pkg(ConanFile):
-            settings = "os", "compiler", "build_type", "arch"
-            requires = "zlib/1.2.11@conan/stable"
+                class Pkg(ConanFile):
+                    settings = "os", "compiler", "build_type", "arch"
+                    requires = "zlib/1.2.11@conan/stable"
 
-            def build(self):
-                pass
-        """
+                    def build(self):
+                        pass
+            """)
         self.save_conanfile(conanfile)
         with tools.environment_append({"CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt",
                                        "CONAN_USERNAME": "bar",
@@ -236,6 +239,38 @@ def build(self):
                 self.assertIn("Error updating the image", str(raised.exception))
                 self.assertIn("foobar install conan_package_tools", str(raised.exception))
 
+    @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker")
+    def test_docker_base_profile(self):
+        conanfile = textwrap.dedent("""
+                from conans import ConanFile
+
+                class Pkg(ConanFile):
+
+                    def build(self):
+                        pass
+            """)
+
+        self.save_conanfile(conanfile)
+        with tools.environment_append({"CONAN_DOCKER_RUN_OPTIONS": "--network=host -v{}:/tmp/cpt".format(self.root_project_folder),
+                                       "CONAN_DOCKER_ENTRY_SCRIPT": "pip install -U /tmp/cpt",
+                                       "CONAN_DOCKER_IMAGE": "conanio/gcc8",
+                                       "CONAN_USE_DOCKER": "1",
+                                       "CONAN_REFERENCE": "foo/0.0.1@bar/testing",
+                                       "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE",
+                                       "CONAN_FORCE_SELINUX": "TRUE",
+                                       "CONAN_DOCKER_USE_SUDO": "FALSE",
+                                       "CONAN_DOCKER_SHELL": "/bin/bash -c",
+                                       }):
+            self.packager = ConanMultiPackager(gcc_versions=["8"],
+                                               archs=["x86_64"],
+                                               build_types=["Release"],
+                                               config_url="https://github.com/bincrafters/bincrafters-config.git",
+                                               out=self.output.write)
+            self.packager.add({})
+            self.packager.run(base_profile_name="linux-gcc8-amd64")
+            self.assertIn('Using specified default base profile: linux-gcc8-amd64', self.output)
+            self.assertIn('-e CPT_BASE_PROFILE_NAME="linux-gcc8-amd64"', self.output)
+
     @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker")
     def test_docker_hidden_password(self):
         conanfile = textwrap.dedent("""
@@ -267,3 +302,32 @@ def build(self):
             self.packager.run()
             self.assertIn('-e CONAN_LOGIN_USERNAME="xxxxxxxx"', self.output)
             self.assertIn('-e CONAN_PASSWORD="xxxxxxxx"', self.output)
+
+    @unittest.skipUnless(is_linux_and_have_docker(), "Requires Linux and Docker")
+    def test_docker_underscore_user_channel(self):
+        conanfile = textwrap.dedent("""
+                from conans import ConanFile
+
+                class Pkg(ConanFile):
+                    def build(self):
+                        pass
+            """)
+
+        self.save_conanfile(conanfile)
+        with tools.environment_append({"CONAN_USERNAME": "_",
+                                       "CONAN_CHANNEL": "_",
+                                       "CONAN_DOCKER_IMAGE": "conanio/gcc8",
+                                       "CONAN_REFERENCE": "foo/0.0.1",
+                                       "CONAN_DOCKER_IMAGE_SKIP_UPDATE": "TRUE",
+                                       "CONAN_FORCE_SELINUX": "TRUE",
+                                       "CONAN_DOCKER_USE_SUDO": "FALSE",
+                                       "CONAN_DOCKER_SHELL": "/bin/bash -c",
+                                       }):
+            self.packager = ConanMultiPackager(gcc_versions=["8"],
+                                               archs=["x86_64"],
+                                               build_types=["Release"],
+                                               out=self.output.write)
+            self.packager.add({})
+            self.packager.run()
+            self.assertIn('-e CONAN_USERNAME="_"', self.output)
+            self.assertIn('-e CONAN_CHANNEL="_"', self.output)