From e22db124be61a23f308617cbbcc8a8870e40aed7 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Fri, 7 Sep 2018 19:15:08 -0400 Subject: [PATCH] Do not check source repos for comps or prestodelta. fixes #2572 Signed-off-by: Randy Barlow (cherry picked from commit 5a0e9fe462fa1c39eb50faeb3655ffd96458a295) --- bodhi/server/consumers/masher.py | 3 ++- bodhi/server/util.py | 25 ++++++++++++-------- bodhi/tests/server/base.py | 19 ++++++++------- bodhi/tests/server/consumers/test_masher.py | 9 ++++--- bodhi/tests/server/test_util.py | 26 +++++++++++++++++---- 5 files changed, 54 insertions(+), 28 deletions(-) diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py index 39a9b97914..2e25cf11a6 100644 --- a/bodhi/server/consumers/masher.py +++ b/bodhi/server/consumers/masher.py @@ -1148,10 +1148,11 @@ def _sanity_check_repo(self): if arch == 'source': repodata = os.path.join(self.path, 'compose', 'Everything', arch, 'tree', 'repodata') + sanity_check_repodata(repodata, source=True) else: repodata = os.path.join(self.path, 'compose', 'Everything', arch, 'os', 'repodata') - sanity_check_repodata(repodata) + sanity_check_repodata(repodata, source=False) except Exception: self.log.exception("Repodata sanity check failed, compose thrown out") self._toss_out_repo() diff --git a/bodhi/server/util.py b/bodhi/server/util.py index 1723acc3e1..792db1adb1 100644 --- a/bodhi/server/util.py +++ b/bodhi/server/util.py @@ -245,12 +245,13 @@ def get_critpath_components(collection='master', component_type='rpm', component return critpath_components -def sanity_check_repodata(myurl): +def sanity_check_repodata(myurl, source): """ Sanity check the repodata for a given repository. Args: myurl (basestring): A path to a repodata directory. + source (bool): True if we are checking a source RPM repository, False otherwise. Raises: Exception: If the repodata is not valid or does not exist. """ @@ -279,7 +280,9 @@ def sanity_check_repodata(myurl): hk_repo = hawkey.Repo(myurl) try: hk_repo.filelists_fn = repo_info['filelists'] - hk_repo.presto_fn = repo_info['prestodelta'] + # Source repos don't have DRPMs. + if not source: + hk_repo.presto_fn = repo_info['prestodelta'] hk_repo.primary_fn = repo_info['primary'] hk_repo.repomd_fn = repo_info['repomd'] hk_repo.updateinfo_fn = repo_info['updateinfo'] @@ -291,14 +294,16 @@ def sanity_check_repodata(myurl): load_presto=True, load_updateinfo=True) - # Test comps - comps = libcomps.Comps() - try: - ret = comps.fromxml_f(repo_info['group']) - except Exception: - raise RepodataException('Comps file unable to be parsed') - if len(comps.groups) < 1: - raise RepodataException('Comps file empty') + # Source repos don't have comps. + if not source: + # Test comps + comps = libcomps.Comps() + try: + ret = comps.fromxml_f(repo_info['group']) + except Exception: + raise RepodataException('Comps file unable to be parsed') + if len(comps.groups) < 1: + raise RepodataException('Comps file empty') # Test updateinfo ret = subprocess.call(['zgrep', '', repo_info['updateinfo']]) diff --git a/bodhi/tests/server/base.py b/bodhi/tests/server/base.py index 8c1767b35f..d6a191362e 100644 --- a/bodhi/tests/server/base.py +++ b/bodhi/tests/server/base.py @@ -362,7 +362,7 @@ def __call__(self): raise -def mkmetadatadir(path, updateinfo=None, comps=None): +def mkmetadatadir(path, updateinfo=None, comps=None, source=False): """ Generate package metadata for a given directory. @@ -374,6 +374,7 @@ def mkmetadatadir(path, updateinfo=None, comps=None): No updateinfo is inserted if False is passed. Passing True provides undefined behavior. comps (basestring or None): The comps to insert instead of example. + source (True): If True, do not insert comps or prestodelta. Defaults to False. """ compsfile = ''' @@ -390,7 +391,7 @@ def mkmetadatadir(path, updateinfo=None, comps=None): updateinfofile = '' if not os.path.isdir(path): os.makedirs(path) - if not comps: + if not comps and not source: comps = os.path.join(path, 'comps.xml') with open(comps, 'w') as f: f.write(compsfile) @@ -399,13 +400,13 @@ def mkmetadatadir(path, updateinfo=None, comps=None): with open(updateinfo, 'w') as f: f.write(updateinfofile) - subprocess.check_call(['createrepo_c', - '--groupfile', 'comps.xml', - '--deltas', - '--xz', - '--database', - '--quiet', - path]) + createrepo_command = ['createrepo_c', '--xz', '--database', '--quiet', path] + + if not source: + for arg in ('--deltas', 'comps.xml', '--groupfile'): + createrepo_command.insert(1, arg) + + subprocess.check_call(createrepo_command) if updateinfo is not False: metadata.insert_in_repo(createrepo_c.XZ, os.path.join(path, 'repodata'), 'updateinfo', 'xml', os.path.join(path, 'updateinfo.xml')) diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py index b437d4f3a9..48112d11dc 100644 --- a/bodhi/tests/server/consumers/test_masher.py +++ b/bodhi/tests/server/consumers/test_masher.py @@ -752,7 +752,8 @@ def test_sanity_check_valid(self, save_state): with open(os.path.join(repo, 'Packages', 'a', name), 'w') as tf: tf.write('foo') - base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree')) + base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'), + source=True) os.makedirs(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages', 'a')) with open(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages', 'a', @@ -826,7 +827,8 @@ def test_sanity_check_symlink(self, save_state): os.makedirs(os.path.join(repo, 'Packages', 'a')) os.symlink('/dev/null', os.path.join(repo, 'Packages', 'a', 'test.notrpm')) - base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree')) + base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'), + source=True) os.makedirs(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', 'Packages', 'a')) os.symlink('/dev/null', os.path.join(t.path, 'compose', 'Everything', 'source', 'tree', @@ -863,7 +865,8 @@ def test_sanity_check_directories_missing(self, save_state): base.mkmetadatadir(repo) shutil.rmtree(os.path.join(t.path, 'compose', 'Everything', arch, 'os', 'Packages')) - base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree')) + base.mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree'), + source=True) assert 'completed_repo' in t._checkpoints save_state.reset_mock() diff --git a/bodhi/tests/server/test_util.py b/bodhi/tests/server/test_util.py index b78d7e7ed2..b4db743f4f 100644 --- a/bodhi/tests/server/test_util.py +++ b/bodhi/tests/server/test_util.py @@ -540,7 +540,7 @@ def test_correct_repo(self): base.mkmetadatadir(self.tempdir) # No exception should be raised here. - util.sanity_check_repodata(self.tempdir) + util.sanity_check_repodata(self.tempdir, source=False) def test_updateinfo_empty_tags(self): """RepodataException should be raised if is found in updateinfo.""" @@ -550,7 +550,7 @@ def test_updateinfo_empty_tags(self): base.mkmetadatadir(self.tempdir, updateinfo=updateinfo) with self.assertRaises(util.RepodataException) as exc: - util.sanity_check_repodata(self.tempdir) + util.sanity_check_repodata(self.tempdir, source=False) self.assertEqual(str(exc.exception), 'updateinfo.xml.gz contains empty ID tags') @@ -562,7 +562,7 @@ def test_comps_invalid_notxml(self): base.mkmetadatadir(self.tempdir, comps=comps) with self.assertRaises(util.RepodataException) as exc: - util.sanity_check_repodata(self.tempdir) + util.sanity_check_repodata(self.tempdir, source=False) self.assertEqual(str(exc.exception), 'Comps file unable to be parsed') @@ -574,7 +574,7 @@ def test_comps_invalid_nonsense(self): base.mkmetadatadir(self.tempdir, comps=comps) with self.assertRaises(util.RepodataException) as exc: - util.sanity_check_repodata(self.tempdir) + util.sanity_check_repodata(self.tempdir, source=False) self.assertEqual(str(exc.exception), 'Comps file empty') @@ -592,10 +592,26 @@ def test_repomd_missing_updateinfo(self): repomd.write(repomd_path, encoding='UTF-8', xml_declaration=True) with self.assertRaises(util.RepodataException) as exc: - util.sanity_check_repodata(self.tempdir) + util.sanity_check_repodata(self.tempdir, source=False) self.assertEqual(str(exc.exception), 'Required part not in repomd.xml') + def test_source_true(self): + """It should not fail source repos for missing prestodelta or comps.""" + base.mkmetadatadir(self.tempdir) + repomd_path = os.path.join(self.tempdir, 'repodata', 'repomd.xml') + repomd = ElementTree.parse(repomd_path) + ElementTree.register_namespace('', 'http://linux.duke.edu/metadata/repo') + root = repomd.getroot() + for data in root.findall('{http://linux.duke.edu/metadata/repo}data'): + # Source repos don't have drpms or comps. + if data.attrib['type'] in ('group', 'prestodelta'): + root.remove(data) + repomd.write(repomd_path, encoding='UTF-8', xml_declaration=True) + + # No exception should be raised. + util.sanity_check_repodata(self.tempdir, source=True) + class TestType2Icon(unittest.TestCase): """Test the type2icon() function."""