Backports of several patches from master to 0.x-maint

.github/workflows/unittests.yaml

@@ -55,10 +55,11 @@ jobs:
       - name: 'Install packages (OpenSUSE)'
         if: ${{ startsWith(matrix.container, 'opensuse/') }}
         run: |
-            zypper --non-interactive --gpg-auto-import-keys refresh
-            zypper --non-interactive dist-upgrade
-            zypper --non-interactive install git-lfs
-            zypper --non-interactive install diffstat diffutils python3 python3-M2Crypto python3-pip python3-rpm python3-setuptools
+            zypper -n modifyrepo --disable repo-openh264 || :
+            zypper -n --gpg-auto-import-keys refresh
+            zypper -n dist-upgrade
+            zypper -n install git-lfs
+            zypper -n install diffstat diffutils python3 python3-M2Crypto python3-pip python3-rpm python3-setuptools
 
       - name: 'Install packages (Fedora/CentOS)'
         if: ${{ startsWith(matrix.container, 'fedora:') || contains(matrix.container, 'centos:')  }}
@@ -80,6 +81,7 @@ jobs:
 
       - name: 'Run unit tests'
         run: |
+          pip3 config set global.break-system-packages 1
           pip3 install -e .
           python3 setup.py test
 
@@ -113,15 +115,17 @@ jobs:
       - name: 'Install packages (OpenSUSE)'
         if: ${{ startsWith(matrix.container, 'opensuse/') }}
         run: |
-            zypper --non-interactive --gpg-auto-import-keys refresh
-            zypper --non-interactive dist-upgrade
-            zypper --non-interactive install git-lfs
-            zypper --non-interactive install diffstat diffutils python python2-M2Crypto python2-pip python2-rpm python2-setuptools
+            zypper -n modifyrepo --disable repo-openh264 || :
+            zypper -n --gpg-auto-import-keys refresh
+            zypper -n dist-upgrade
+            zypper -n install git-lfs
+            zypper -n install diffstat diffutils python python2-M2Crypto python2-pip python2-rpm python2-setuptools
 
       - uses: actions/checkout@v3
 
       - name: 'Run unit tests'
         run: |
           mkdir -p /usr/local/lib/python2.7/site-packages/
+          pip2 config set global.break-system-packages 1
           pip2 install -e .
           python2 setup.py test

osc/build.py

@@ -145,7 +145,10 @@ def __init__(self, filename, apiurl, buildtype = 'spec', localpkgs = [], binaryt
         else:
             self.release = None
         if config['api_host_options'][apiurl]['downloadurl']:
-            self.enable_cpio = False
+            # Formerly, this was set to False, but we have to set it to True, because a large
+            # number of repos in OBS are misconfigured and don't actually have repos setup - they
+            # are API only.
+            self.enable_cpio = True
             self.downloadurl = config['api_host_options'][apiurl]['downloadurl'] + "/repositories"
             if config['http_debug']:
                 print("⚠️   setting dl_url to %s" % config['api_host_options'][apiurl]['downloadurl'])
@@ -1351,8 +1354,11 @@ def __str__(self):
                 print("Error: cannot get hdrmd5 for %s" % i.fullfilename)
                 sys.exit(1)
             if hdrmd5 != i.hdrmd5:
-                print("Error: hdrmd5 mismatch for %s: %s != %s" % (i.fullfilename, hdrmd5, i.hdrmd5))
-                sys.exit(1)
+                if conf.config["api_host_options"][apiurl]["disable_hdrmd5_check"]:
+                    print(f"Warning: Ignoring a hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                else:
+                    print(f"Error: hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                    sys.exit(1)
 
     print('Writing build configuration')
 

osc/commandline.py

@@ -2470,7 +2470,7 @@ def do_request(self, subcmd, opts, *args):
                 if subcmd == 'review':
                     # FIXME: do the review list for the user and for all groups he belong to
                     results = get_review_list(apiurl, project, package, who, opts.group, opts.project, opts.package, state_list,
-                                              opts.type)
+                                              opts.type, req_states=("new", "review", "declined"))
                 else:
                     if opts.involved_projects:
                         who = who or conf.get_apiurl_usr(apiurl)

osc/conf.py

@@ -158,6 +158,28 @@ def _identify_osccookiejar():
             'cookiejar': _identify_osccookiejar(),
             # fallback for osc build option --no-verify
             'no_verify': '0',
+
+            # Disable hdrmd5 checks of downloaded and cached packages in `osc build`
+            # Recommended value: 0
+            #
+            # OBS builds the noarch packages once per binary arch.
+            # Such noarch packages are supposed to be nearly identical across all build arches,
+            # any discrepancy in the payload and dependencies is considered a packaging bug.
+            # But to guarantee that the local builds work identically to builds in OBS,
+            # using the arch-specific copy of the noarch package is required.
+            # Unfortunatelly only one of the noarch packages gets distributed
+            # and can be downloaded from a local mirror.
+            # All other noarch packages are available through the OBS API only.
+            # Since there is currently no information about hdrmd5 checksums of published noarch packages,
+            # we download them, verify hdrmd5 and re-download the package from OBS API on mismatch.
+            #
+            # The same can also happen for architecture depend packages when someone is messing around
+            # with the source history or the release number handling in a way that it is not increasing.
+            #
+            # If you want to save some bandwidth and don't care about the exact rebuilds
+            # you can turn this option on to disable hdrmd5 checks completely.
+            'disable_hdrmd5_check': '0',
+
             # enable project tracking by default
             'do_package_tracking': '1',
             # default for osc build
@@ -221,13 +243,13 @@ def _identify_osccookiejar():
 
 boolean_opts = ['debug', 'do_package_tracking', 'http_debug', 'post_mortem', 'traceback', 'check_filelist', 'plaintext_passwd',
     'checkout_no_colon', 'checkout_rooted', 'check_for_request_on_action', 'linkcontrol', 'show_download_progress', 'request_show_interactive',
-    'request_show_source_buildstatus', 'review_inherit_group', 'use_keyring', 'gnome_keyring', 'no_verify', 'builtin_signature_check',
+    'request_show_source_buildstatus', 'review_inherit_group', 'use_keyring', 'gnome_keyring', 'no_verify', 'disable_hdrmd5_check', 'builtin_signature_check',
     'http_full_debug', 'include_request_from_project', 'local_service_run', 'buildlog_strip_time', 'no_preinstallimage',
     'status_mtime_heuristic', 'print_web_links', 'ccache', 'sccache', 'build-shell-after-fail']
 integer_opts = ['build-jobs']
 
 api_host_options = ['user', 'pass', 'passx', 'aliases', 'http_headers', 'realname', 'email', 'sslcertck', 'cafile', 'capath', 'trusted_prj',
-    'downloadurl', 'sshkey']
+    'downloadurl', 'sshkey', 'disable_hdrmd5_check']
 
 new_conf_template = """
 [general]
@@ -583,7 +605,12 @@ def __init__(self, user, sshkey):
 
         def list_ssh_agent_keys(self):
             cmd = ['ssh-add', '-l']
-            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            try:
+                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            except OSError:
+                # ssh-add is not available
+                return []
+
             stdout, _ = proc.communicate()
             if proc.returncode == 0 and stdout.strip():
                 return [self.get_fingerprint(line) for line in stdout.splitlines()]
@@ -677,7 +704,10 @@ def ssh_sign(self, data, namespace, keyfile=None):
                 keyfile = os.path.expanduser(keyfile)
 
             cmd = ['ssh-keygen', '-Y', 'sign', '-f', keyfile, '-n', namespace, '-q']
-            proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            try:
+                proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            except OSError:
+                return None
             stdout, _ = proc.communicate(data)
             if proc.returncode:
                 raise oscerr.OscIOError(None, 'ssh-keygen signature creation failed: %d' % proc.returncode)
@@ -693,6 +723,9 @@ def get_authorization(self, req, chal):
             now = int(time.time())
             sigdata = "(created): %d" % now
             signature = self.ssh_sign(sigdata, realm, self.sshkey)
+            if not signature:
+                # the signing step failed due to missing ssh-keygen
+                return None
             signature = decode_it(base64.b64encode(signature))
             return 'keyId="%s",algorithm="ssh",headers="(created)",created=%d,signature="%s"' \
                 % (self.user, now, signature)
@@ -1207,6 +1240,10 @@ def get_config(override_conffile=None,
         if api_host_options[apiurl]['sshkey'] is None:
             api_host_options[apiurl]['sshkey'] = config['sshkey']
 
+        api_host_options[apiurl]["disable_hdrmd5_check"] = config["disable_hdrmd5_check"]
+        if cp.has_option(url, "disable_hdrmd5_check"):
+            api_host_options[apiurl][key] = cp.getboolean(url, "disable_hdrmd5_check")
+
     # add the auth data we collected to the config dict
     config['api_host_options'] = api_host_options
     config['apiurl_aliases'] = aliases

osc/core.py

@@ -4450,7 +4450,7 @@ def change_request_state_template(req, newstate):
         return ''
 
 def get_review_list(apiurl, project='', package='', byuser='', bygroup='', byproject='', bypackage='', states=(),
-                    req_type=''):
+                    req_type='', req_states=("review",)):
     # this is so ugly...
     def build_by(xpath, val):
         if 'all' in states:
@@ -4467,9 +4467,14 @@ def build_by(xpath, val):
         return ''
 
     xpath = ''
+
+    # By default we're interested only in reviews of requests that are in state review.
+    for req_state in req_states:
+        xpath = xpath_join(xpath, "state/@name='%s'" % req_state, inner=True)
+
+    xpath = "(%s)" % xpath
+
     if states == ():
-        # default: requests which are still open and have reviews in "new" state
-        xpath = xpath_join('', 'state/@name=\'review\'', op='and')
         xpath = xpath_join(xpath, 'review/@state=\'new\'', op='and')
     if byuser:
         xpath = build_by(xpath, '@by_user=\'%s\'' % byuser)

osc/fetch.py

@@ -206,6 +206,7 @@ def _build_urllist(self, buildinfo, pac):
         return urllist
 
     def run(self, buildinfo):
+        apiurl = buildinfo.apiurl
         cached = 0
         all = len(buildinfo.deps)
         for i in buildinfo.deps:
@@ -222,12 +223,24 @@ def run(self, buildinfo):
                 cached += 1
                 if not i.name.startswith('container:') and i.pacsuffix != 'rpm':
                     continue
+
+                hdrmd5_is_valid = True
                 if i.hdrmd5:
                     if i.name.startswith('container:'):
                         hdrmd5 = dgst(i.fullfilename)
+                        if hdrmd5 != i.hdrmd5:
+                            hdrmd5_is_valid = False
                     else:
                         hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename)
-                    if not hdrmd5 or hdrmd5 != i.hdrmd5:
+                        if hdrmd5 != i.hdrmd5:
+                            if conf.config["api_host_options"][apiurl]["disable_hdrmd5_check"]:
+                                print(f"Warning: Ignoring a hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                                hdrmd5_is_valid = True
+                            else:
+                                print(f"The file will be redownloaded from the API due to a hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                                hdrmd5_is_valid = False
+
+                    if not hdrmd5_is_valid:
                         os.unlink(i.fullfilename)
                         cached -= 1
 
@@ -259,14 +272,14 @@ def run(self, buildinfo):
                         # mark it for downloading from the API
                         self.__add_cpio(i)
                     else:
-                        # if the checksum of the downloaded package doesn't match,
-                        # delete it and mark it for downloading from the API
                         hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename)
-                        if not hdrmd5 or hdrmd5 != i.hdrmd5:
-                            print('%s/%s: attempting download from api, since the hdrmd5 did not match - %s != %s'
-                                % (i.project, i.name, hdrmd5, i.hdrmd5))
-                            os.unlink(i.fullfilename)
-                            self.__add_cpio(i)
+                        if hdrmd5 != i.hdrmd5:
+                            if conf.config["api_host_options"][apiurl]["disable_hdrmd5_check"]:
+                                print(f"Warning: Ignoring a hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                            else:
+                                print(f"The file will be redownloaded from the API due to a hdrmd5 mismatch for {i.fullfilename}: {hdrmd5} (actual) != {i.hdrmd5} (expected)")
+                                os.unlink(i.fullfilename)
+                                self.__add_cpio(i)
 
                 except KeyboardInterrupt:
                     print('Cancelled by user (ctrl-c)')