Pass both raw NSVs and moduledefs to module variants template

Signed-off-by: Patrick Uiterwijk <[email protected]>

bodhi/server/consumers/masher.py

@@ -1291,10 +1291,15 @@ def _copy_additional_pungi_files(self, pungi_conf_dir, template_env):
         """
         template = template_env.get_template('variants.module.xml.j2')
 
-        module_list = self._generate_module_list()
+        # These are assigned to self to be testable
+        self._module_defs = self._generate_module_list()
+        # This is so as to not break existing Pungi configurations, but ideally they get updated.
+        self._module_list = ['%(name)s:%(stream)s:%(version)s' % mod for mod in self._module_defs]
 
         with open(os.path.join(pungi_conf_dir, 'module-variants.xml'), 'w') as variantsfile:
-            variantsfile.write(template.render(modules=module_list))
+            self._variants_file = template.render(modules=self._module_list,
+                                                  moduledefs=self._module_defs)
+            variantsfile.write(self._variants_file)
 
     def generate_testing_digest(self):
         """Temporarily disable testing digests for modules.
@@ -1325,12 +1330,44 @@ def _raise_on_get_build_multicall_error(self, result, build):
             self.log.error(err)
             raise Exception(err)
 
+    def _add_build_to_newest_builds(self, newest_builds, koji_build, override=False):
+        """
+        Add Koji build to newest_builds dict if it's newer than the one there or override is set.
+
+        Args:
+            newest_builds (dict): Dict with name:stream as a key and moduledef as value
+                (see _generate_module_list).
+            koji_build (dict): Koji build to add obtained by koji.getBuild(...) method.
+            override (bool): When False, the koji_build is added to newest_builds only
+                if it is newer than the one currently stored in newest_builds for given
+                name:stream. When True, koji_build is added to newest_build even it is
+                not newer than the one currently stored there.
+        """
+        # name:stream:version(.context) maps to Koji's name-version-release.
+        ns = "%s:%s" % (koji_build["name"], koji_build["version"])
+        version = koji_build["release"]
+        context = ''
+        if '.' in version:
+            version, context = version.split('.', 1)
+
+        moduledef = {'name': koji_build['name'],
+                     'stream': koji_build['version'],
+                     'version': version,
+                     'context': context}
+
+        if ns in newest_builds and not override:
+            curr_version = newest_builds[ns]['version']
+            if int(curr_version) < int(version):
+                newest_builds[ns] = moduledef
+        else:
+            newest_builds[ns] = moduledef
+
     def _generate_module_list(self):
         """
-        Generate a list of NSVs which should be used for pungi modular compose.
+        Generate a list of modules which should be used for pungi modular compose.
 
         Returns:
-            list: list of NSV string which should be composed.
+            list: list of moduledef dicts with name, stream, version and context
         """
         # For modules, both name and version can contain dashes. This makes it
         # impossible to distinguish between name and version from "nvr". We
@@ -1348,16 +1385,7 @@ def _generate_module_list(self):
         for result, build in zip(results, self.compose.release.builds):
             self._raise_on_get_build_multicall_error(result, build)
             koji_build = result[0]
-            # name:stream:version maps to Koji's name-version-release.
-            ns = "%s:%s" % (koji_build["name"], koji_build["version"])
-            version = koji_build["release"]
-
-            if ns in newest_builds:
-                curr_version = newest_builds[ns]
-                if int(curr_version) < int(version):
-                    newest_builds[ns] = version
-            else:
-                newest_builds[ns] = version
+            self._add_build_to_newest_builds(newest_builds, koji_build)
 
         # make sure that the modules we want to update get their correct versions
         for update in self.compose.updates:
@@ -1370,9 +1398,8 @@ def _generate_module_list(self):
             for result, build in zip(results, update.builds):
                 self._raise_on_get_build_multicall_error(result, build)
                 koji_build = result[0]
-                # name:stream:version maps to Koji's name-version-release.
-                ns = "%s:%s" % (koji_build["name"], koji_build["version"])
-                version = koji_build["release"]
-                newest_builds[ns] = version
+                self._add_build_to_newest_builds(newest_builds, koji_build, True)
 
-        return ["%s:%s" % (nstream, v) for nstream, v in six.iteritems(newest_builds)]
+        # The keys are just used for easy name-stream finding. The name and stream are already in
+        # the module definitions.
+        return newest_builds.values()

bodhi/tests/server/consumers/pungi.basepath/variants.module.xml.j2

@@ -1 +1,9 @@
-This file doesn't need to have a real Pungi config, it just needs to exist.
+Raw NSVs:
+[%- for module in modules %]
+[[ module ]]
+[%- endfor %]
+
+Calculated NSVCs:
+[%- for module in moduledefs %]
+[[ module.name ]]:[[ module.stream ]]:[[ module.version ]]:[[ module.context ]]
+[% endfor %]

bodhi/tests/server/consumers/test_masher.py

@@ -1112,16 +1112,16 @@ def test_mash_module(self, publish, *args):
             package = Package(name=u'testmodule',
                               type=ContentType.module)
             db.add(package)
-            build1 = ModuleBuild(nvr=u'testmodule-master-20171',
+            build1 = ModuleBuild(nvr=u'testmodule-master-20171.1',
                                  release=release, signed=True,
                                  package=package)
             db.add(build1)
-            build2 = ModuleBuild(nvr=u'testmodule-master-20172',
+            build2 = ModuleBuild(nvr=u'testmodule-master-20172.2',
                                  release=release, signed=True,
                                  package=package)
             db.add(build2)
             update = Update(
-                title=u'testmodule-master-20172',
+                title=u'testmodule-master-20172.2',
                 builds=[build2], user=user,
                 status=UpdateStatus.testing,
                 request=UpdateRequest.stable,
@@ -1154,6 +1154,20 @@ def test_mash_module(self, publish, *args):
                                 force=True,
                                 msg=mock.ANY)
 
+        self.assertEqual(t._module_defs, [{'context': '2',
+                                           'version': '20172',
+                                           'name': 'testmodule',
+                                           'stream': 'master'}])
+        self.assertEqual(t._module_list, ['testmodule:master:20172'])
+
+        EXPECTED_VARIANTS = '''Raw NSVs:
+testmodule:master:20172
+
+Calculated NSVCs:
+testmodule:master:20172:2
+'''
+        self.assertEqual(t._variants_file, EXPECTED_VARIANTS)
+
         self.assertEqual(
             Popen.mock_calls,
             [mock.call(
@@ -1182,8 +1196,8 @@ def test_mash_module_koji_multicall_result_empty_list(self):
         build = ModuleBuild(nvr=u'testmodule-master-20171',
                             release=release, signed=True,
                             package=package)
-        t = ModuleMasherThread({}, 'puiterwijk', log, self.db_factory,
-                               self.tempdir)
+        t = ModuleComposerThread(self.semmock, {}, 'puiterwijk', log, self.db_factory,
+                                 self.tempdir)
         with self.assertRaises(Exception) as exc:
             t._raise_on_get_build_multicall_error([], build)
 
@@ -1198,8 +1212,8 @@ def test_mash_module_koji_multicall_result_dict(self):
         build = ModuleBuild(nvr=u'testmodule-master-20171',
                             release=release, signed=True,
                             package=package)
-        t = ModuleMasherThread({}, 'puiterwijk', log, self.db_factory,
-                               self.tempdir)
+        t = ModuleComposerThread(self.semmock, {}, 'puiterwijk', log, self.db_factory,
+                                 self.tempdir)
         with self.assertRaises(Exception) as exc:
             t._raise_on_get_build_multicall_error({}, build)