From b5a7aec46aa80fd0de5b15829cc134f15eb36b16 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 19:35:49 +0000 Subject: [PATCH 1/9] generate-release-meta: Factor out build-to-stream function This avoids an indent; pass in the JSON data rather than having it under a `with` statement. Prep for further work. --- src/cmd-generate-release-meta | 206 +++++++++++++++++----------------- 1 file changed, 105 insertions(+), 101 deletions(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index 7be2117d59..c179d9c54d 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -68,112 +68,116 @@ if os.path.exists(args.output): out = json.load(w) print(f"Using existing release file {args.output}") -for f in archbuilds: - with open(f, 'r') as w: - input_ = json.load(w) - arch = input_.get("coreos-assembler.basearch") - - ensure_dup(input_, out, "buildid", "release") - ensure_dup(input_.get('coreos-assembler.container-config-git'), out, 'branch', 'stream') - - def artifact(i): - base_url = url_builder(out.get('stream'), out.get('release'), arch, i.get('path')) - return { - "location": base_url, - "signature": "{}.sig".format(base_url), - "sha256": i.get("sha256") - } - - print(f"{out['stream']} stream") - print(f" {arch} images:") - # build the architectures dict - arch_dict = {"media": {}} - ensure_dup(input_, arch_dict, "ostree-commit", "commit") - platforms = ["aliyun", "aws", "azure", "digitalocean", "exoscale", "gcp", "ibmcloud", "metal", "openstack", "qemu", "vmware", "vultr"] - for platform in platforms: - if input_.get("images", {}).get(platform, None) is not None: - print(f" - {platform}") - i = input_.get("images").get(platform) - ext = get_extension(i.get('path'), platform, arch) - arch_dict['media'][platform] = { - "artifacts": { - ext: { - "disk": artifact(i) - } +# Append the coreos-assembler build json `input_` to `out`, the target release stream. +def append_build(out, input_): + arch = input_.get("coreos-assembler.basearch") + + ensure_dup(input_, out, "buildid", "release") + ensure_dup(input_.get('coreos-assembler.container-config-git'), out, 'branch', 'stream') + + def artifact(i): + base_url = url_builder(out.get('stream'), out.get('release'), arch, i.get('path')) + return { + "location": base_url, + "signature": "{}.sig".format(base_url), + "sha256": i.get("sha256") + } + + print(f"{out['stream']} stream") + print(f" {arch} images:") + # build the architectures dict + arch_dict = {"media": {}} + ensure_dup(input_, arch_dict, "ostree-commit", "commit") + platforms = ["aliyun", "aws", "azure", "digitalocean", "exoscale", "gcp", "ibmcloud", "metal", "openstack", "qemu", "vmware", "vultr"] + for platform in platforms: + if input_.get("images", {}).get(platform, None) is not None: + print(f" - {platform}") + i = input_.get("images").get(platform) + ext = get_extension(i.get('path'), platform, arch) + arch_dict['media'][platform] = { + "artifacts": { + ext: { + "disk": artifact(i) } } - - # AMI specific additions - if input_.get("amis", None) is not None: - arch_dict["media"]["aws"] = arch_dict["media"].get("aws", {}) - arch_dict["media"]["aws"]["images"] = arch_dict["media"]["aws"].get("images", {}) - for ami_dict in input_.get("amis"): - arch_dict["media"]["aws"]["images"][ami_dict["name"]] = { - "image": ami_dict["hvm"] - } - - # GCP specific additions - if input_.get("gcp", None) is not None: - arch_dict["media"]["gcp"] = arch_dict["media"].get("gcp", {}) - arch_dict["media"]["gcp"]["image"] = arch_dict["media"]["gcp"].get("image", {}) - arch_dict["media"]["gcp"]["image"].update(input_.get("gcp", {})) - arch_dict["media"]["gcp"]["image"]["name"] = arch_dict["media"]["gcp"]["image"].pop("image") - # remove the url as we haven't decided to expose that information publicly yet - arch_dict["media"]["gcp"]["image"].pop("url") - - # metal specific additions - arch_dict["media"]["metal"] = arch_dict["media"].get("metal", {}) - arch_dict["media"]["metal"]["artifacts"] = arch_dict["media"]["metal"].get("artifacts", {}) - i = input_.get("images", {}).get("metal4k", None) - if i is not None: - # the 4k image is kinda weird; we want it at the same level as e.g. - # the regular 512b image, which normally is under `raw.xz` - ext = get_extension(i['path'], 'metal4k', arch) - arch_dict["media"]["metal"]["artifacts"][f"4k.{ext}"] = { - "disk": artifact(i) - } - i = input_.get("images", {}).get("iso", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"]["installer.iso"] = { - "disk": artifact(i) } - i = input_.get("images", {}).get("kernel", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"].setdefault("installer-pxe", {})["kernel"] = artifact(i) - i = input_.get("images", {}).get("initramfs", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"].setdefault("installer-pxe", {})["initramfs"] = artifact(i) - i = input_.get("images", {}).get("live-iso", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"]["iso"] = { - "disk": artifact(i) + + # AMI specific additions + if input_.get("amis", None) is not None: + arch_dict["media"]["aws"] = arch_dict["media"].get("aws", {}) + arch_dict["media"]["aws"]["images"] = arch_dict["media"]["aws"].get("images", {}) + for ami_dict in input_.get("amis"): + arch_dict["media"]["aws"]["images"][ami_dict["name"]] = { + "image": ami_dict["hvm"] } - i = input_.get("images", {}).get("live-kernel", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["kernel"] = artifact(i) - i = input_.get("images", {}).get("live-initramfs", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["initramfs"] = artifact(i) - i = input_.get("images", {}).get("live-rootfs", None) - if i is not None: - arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["rootfs"] = artifact(i) - - # if architectures as a whole or the individual arch is empty just push our changes - if out.get('architectures', None) is None or out['architectures'].get(arch, None) is None: - oa = out.get('architectures', {}) - oa[arch] = arch_dict - out['architectures'] = oa - # else check media warning if key present, appending if not - else: - out_arch = out['architectures'][arch] - for media_type, val in arch_dict.get('media').items(): - if media_type not in out_arch['media']: - out['architectures'][arch]['media'].update({media_type: val}) - elif val == out_arch['media'][media_type]: - continue - else: - raise Exception("differing content detected for media type '{}'".format(media_type)) + + # GCP specific additions + if input_.get("gcp", None) is not None: + arch_dict["media"]["gcp"] = arch_dict["media"].get("gcp", {}) + arch_dict["media"]["gcp"]["image"] = arch_dict["media"]["gcp"].get("image", {}) + arch_dict["media"]["gcp"]["image"].update(input_.get("gcp", {})) + arch_dict["media"]["gcp"]["image"]["name"] = arch_dict["media"]["gcp"]["image"].pop("image") + # remove the url as we haven't decided to expose that information publicly yet + arch_dict["media"]["gcp"]["image"].pop("url") + + # metal specific additions + arch_dict["media"]["metal"] = arch_dict["media"].get("metal", {}) + arch_dict["media"]["metal"]["artifacts"] = arch_dict["media"]["metal"].get("artifacts", {}) + i = input_.get("images", {}).get("metal4k", None) + if i is not None: + # the 4k image is kinda weird; we want it at the same level as e.g. + # the regular 512b image, which normally is under `raw.xz` + ext = get_extension(i['path'], 'metal4k', arch) + arch_dict["media"]["metal"]["artifacts"][f"4k.{ext}"] = { + "disk": artifact(i) + } + i = input_.get("images", {}).get("iso", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"]["installer.iso"] = { + "disk": artifact(i) + } + i = input_.get("images", {}).get("kernel", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"].setdefault("installer-pxe", {})["kernel"] = artifact(i) + i = input_.get("images", {}).get("initramfs", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"].setdefault("installer-pxe", {})["initramfs"] = artifact(i) + i = input_.get("images", {}).get("live-iso", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"]["iso"] = { + "disk": artifact(i) + } + i = input_.get("images", {}).get("live-kernel", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["kernel"] = artifact(i) + i = input_.get("images", {}).get("live-initramfs", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["initramfs"] = artifact(i) + i = input_.get("images", {}).get("live-rootfs", None) + if i is not None: + arch_dict["media"]["metal"]["artifacts"].setdefault("pxe", {})["rootfs"] = artifact(i) + + # if architectures as a whole or the individual arch is empty just push our changes + if out.get('architectures', None) is None or out['architectures'].get(arch, None) is None: + oa = out.get('architectures', {}) + oa[arch] = arch_dict + out['architectures'] = oa + # else check media warning if key present, appending if not + else: + out_arch = out['architectures'][arch] + for media_type, val in arch_dict.get('media').items(): + if media_type not in out_arch['media']: + out['architectures'][arch]['media'].update({media_type: val}) + elif val == out_arch['media'][media_type]: + continue + else: + raise Exception("differing content detected for media type '{}'".format(media_type)) + + +for path in archbuilds: + with open(path) as f: + append_build(out, json.load(f)) with open(args.output, 'w') as w: json.dump(out, w) From 17c085bfce0b31a02a1f2248480d2ca923d7080f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 19:13:32 +0000 Subject: [PATCH 2/9] generate-release-meta: Support URLs as input This is convenient. --- src/cmd-generate-release-meta | 39 +++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index c179d9c54d..54d9a6d1f5 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -7,6 +7,7 @@ from argparse import ArgumentParser import json import os +import requests from cosalib.builds import Builds @@ -35,13 +36,9 @@ parser = ArgumentParser() parser.add_argument("--workdir", help="cosa workdir") parser.add_argument("--build-id", help="build id") parser.add_argument("--output", help="Output to file; default is build directory") +parser.add_argument("url", metavar='URL', help="URL to a coreos-assembler meta.json", default=[], nargs='*') args = parser.parse_args() -# FIXME: Remove this once https://github.com/coreos/fedora-coreos-pipeline/ is ported -# not to pass --workdir (it always uses `.` anyways) -if args.workdir not in (None, '.'): - os.chdir(args.workdir) - def gather_buildmeta_from_workdir(): builds = Builds() @@ -53,15 +50,32 @@ def gather_buildmeta_from_workdir(): print(f"Creating release.json for build {buildid}") base_builddir = f"builds/{buildid}" arches = builds.get_build_arches(buildid) + parsed_builds = [] + for arch in arches: + with open(os.path.join(base_builddir, arch, "meta.json")) as f: + parsed_builds.append(json.load(f)) - return (base_builddir, [os.path.join(base_builddir, arch, "meta.json") for arch in arches]) + return (base_builddir, parsed_builds) -(builddir, archbuilds) = gather_buildmeta_from_workdir() out = {} -# Default to writing into the builddir for now -if args.output is None: - args.output = os.path.join(builddir, "release.json") +parsed_builds = [] + +if len(args.url) == 0: + # FIXME: Remove this once https://github.com/coreos/fedora-coreos-pipeline/ is ported + # not to pass --workdir (it always uses `.` anyways) + if args.workdir not in (None, '.'): + os.chdir(args.workdir) + + (builddir, parsed_builds) = gather_buildmeta_from_workdir() + # Default to writing into the builddir for now + if args.output is None: + args.output = os.path.join(builddir, "release.json") +else: + for url in args.url: + print(f"Downloading {url}...") + parsed_builds.append(requests.get(url).json()) + # If any existing data, inherit it if os.path.exists(args.output): with open(args.output, 'r') as w: @@ -175,9 +189,8 @@ def append_build(out, input_): raise Exception("differing content detected for media type '{}'".format(media_type)) -for path in archbuilds: - with open(path) as f: - append_build(out, json.load(f)) +for build in parsed_builds: + append_build(out, build) with open(args.output, 'w') as w: json.dump(out, w) From 29c425290ebe7ced5763baa34074262b4246c30f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 20:09:51 +0000 Subject: [PATCH 3/9] generate-release-meta: Ignore zero sized existing output files This makes it convenient to fork this and pass it a temporary file. --- src/cmd-generate-release-meta | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index 54d9a6d1f5..ba6a01df83 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -76,8 +76,8 @@ else: print(f"Downloading {url}...") parsed_builds.append(requests.get(url).json()) -# If any existing data, inherit it -if os.path.exists(args.output): +# If any existing data, inherit it (if it's non-empty) +if os.path.exists(args.output) and os.stat(args.output).st_size > 0: with open(args.output, 'r') as w: out = json.load(w) print(f"Using existing release file {args.output}") From a58a300aaa4cfa6d31b3d7a42385cf0301416c84 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 20:30:52 +0000 Subject: [PATCH 4/9] generate-release-meta: Support customizing the base URL Needed for RHCOS. --- src/cmd-generate-release-meta | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index ba6a01df83..740475a932 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -25,7 +25,7 @@ def ensure_dup(inp, out, inp_key, out_key): def url_builder(stream, version, arch, path): - return f"https://builds.coreos.fedoraproject.org/prod/streams/{stream}/builds/{version}/{arch}/{path}" + return f"{args.stream_baseurl}/{stream}/builds/{version}/{arch}/{path}" def get_extension(path, modifier, arch): @@ -35,6 +35,7 @@ def get_extension(path, modifier, arch): parser = ArgumentParser() parser.add_argument("--workdir", help="cosa workdir") parser.add_argument("--build-id", help="build id") +parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default="https://builds.coreos.fedoraproject.org/prod/streams") parser.add_argument("--output", help="Output to file; default is build directory") parser.add_argument("url", metavar='URL', help="URL to a coreos-assembler meta.json", default=[], nargs='*') args = parser.parse_args() From df8cf14d968d9be454f88166122af3d84d25f5d3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 16:34:17 +0000 Subject: [PATCH 5/9] mantle: bump coreos/stream-metadata-go Prep for further work on https://github.com/openshift/os/issues/477 --- mantle/go.mod | 2 +- mantle/go.sum | 3 + .../stream-metadata-go/release/release.go | 58 +++--- .../stream-metadata-go/release/translate.go | 183 ++++++++++++++++++ .../stream-metadata-go/stream/stream.go | 55 +++--- mantle/vendor/modules.txt | 2 +- 6 files changed, 248 insertions(+), 55 deletions(-) create mode 100644 mantle/vendor/github.com/coreos/stream-metadata-go/release/translate.go diff --git a/mantle/go.mod b/mantle/go.mod index 736fc1f550..6cffbf5916 100644 --- a/mantle/go.mod +++ b/mantle/go.mod @@ -19,7 +19,7 @@ require ( github.com/coreos/ignition/v2 v2.9.0 github.com/coreos/ioprogress v0.0.0-20151023204047-4637e494fd9b github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f - github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c + github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952 github.com/digitalocean/go-libvirt v0.0.0-20200810224808-b9c702499bf7 // indirect github.com/digitalocean/go-qemu v0.0.0-20200529005954-1b453d036a9c github.com/digitalocean/godo v1.33.0 diff --git a/mantle/go.sum b/mantle/go.sum index fc1031572f..5e51e04f58 100644 --- a/mantle/go.sum +++ b/mantle/go.sum @@ -107,6 +107,9 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c h1:7VO10dpKljeaYJUQtObhqjNxpuTCUDELTviJsGy9OeM= github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko= +github.com/coreos/stream-metadata-go v0.0.0-20210112152733-52b38c241a3d h1:a65dhEcT+kL9Bf5pDpdoOMdT5w0VjwUXE+XO2MoFuSg= +github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952 h1:t7IgMcyflINfXWPISnHTXwa/F+NxLgWGRGejoyfHUII= +github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068 h1:y2aHj7QqyAJ6YBBONTAr17YxHHiogDkYnTsJvFNhxwY= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20201120045928-b0e13dab675c h1:jA28WeORitsxGFVWhyWB06sAG2HbLHPQuHwDydhU2CQ= diff --git a/mantle/vendor/github.com/coreos/stream-metadata-go/release/release.go b/mantle/vendor/github.com/coreos/stream-metadata-go/release/release.go index e4c2dca252..ae7afc9977 100644 --- a/mantle/vendor/github.com/coreos/stream-metadata-go/release/release.go +++ b/mantle/vendor/github.com/coreos/stream-metadata-go/release/release.go @@ -1,10 +1,9 @@ +// Package release contains APIs for interacting with a +// particular "release". Avoid this unless you are sure +// you need it. It's expected that CoreOS users interact +// with streams instead. package release -// Metadata is common metadata that contains last-modified -type Metadata struct { - LastModified string `json:"last-modified"` -} - // Index models the release index: // https://github.com/coreos/fedora-coreos-tracker/tree/master/metadata/release-index type Index struct { @@ -27,28 +26,6 @@ type IndexReleaseCommit struct { Checksum string `json:"checksum"` } -// ImageFormat contains all artifacts for a single OS image -type ImageFormat struct { - Disk *Artifact `json:"disk,omitempty"` - Kernel *Artifact `json:"kernel,omitempty"` - Initramfs *Artifact `json:"initramfs,omitempty"` - Rootfs *Artifact `json:"rootfs,omitempty"` -} - -// Artifact represents one image file, plus its metadata -type Artifact struct { - Location string `json:"location"` - Signature string `json:"signature"` - Sha256 string `json:"sha256"` -} - -// GcpImage represents a GCP cloud image -type GcpImage struct { - Project string `json:"project,omitempty"` - Family string `json:"family,omitempty"` - Name string `json:"name,omitempty"` -} - // Release contains details from release.json type Release struct { Release string `json:"release"` @@ -57,6 +34,11 @@ type Release struct { Architectures map[string]Arch `json:"architectures"` } +// Metadata is common metadata that contains last-modified +type Metadata struct { + LastModified string `json:"last-modified"` +} + // Arch release details type Arch struct { Commit string `json:"commit"` @@ -96,7 +78,29 @@ type PlatformGcp struct { Image *GcpImage `json:"image"` } +// ImageFormat contains all artifacts for a single OS image +type ImageFormat struct { + Disk *Artifact `json:"disk,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Rootfs *Artifact `json:"rootfs,omitempty"` +} + +// Artifact represents one image file, plus its metadata +type Artifact struct { + Location string `json:"location"` + Signature string `json:"signature"` + Sha256 string `json:"sha256"` +} + // CloudImage generic image detail type CloudImage struct { Image string `json:"image"` } + +// GcpImage represents a GCP cloud image +type GcpImage struct { + Project string `json:"project,omitempty"` + Family string `json:"family,omitempty"` + Name string `json:"name,omitempty"` +} diff --git a/mantle/vendor/github.com/coreos/stream-metadata-go/release/translate.go b/mantle/vendor/github.com/coreos/stream-metadata-go/release/translate.go new file mode 100644 index 0000000000..adcf9336cb --- /dev/null +++ b/mantle/vendor/github.com/coreos/stream-metadata-go/release/translate.go @@ -0,0 +1,183 @@ +package release + +import ( + "github.com/coreos/stream-metadata-go/stream" +) + +func mapArtifact(ra *Artifact) *stream.Artifact { + if ra == nil { + return nil + } + return &stream.Artifact{ + Location: ra.Location, + Signature: ra.Signature, + Sha256: ra.Sha256, + } +} + +func mapFormats(m map[string]ImageFormat) map[string]stream.ImageFormat { + r := make(map[string]stream.ImageFormat) + for k, v := range m { + r[k] = stream.ImageFormat{ + Disk: mapArtifact(v.Disk), + Kernel: mapArtifact(v.Kernel), + Initramfs: mapArtifact(v.Initramfs), + Rootfs: mapArtifact(v.Rootfs), + } + } + return r +} + +// Convert a release architecture to a stream architecture +func (releaseArch *Arch) toStreamArch(rel *Release) stream.Arch { + artifacts := make(map[string]stream.PlatformArtifacts) + cloudImages := stream.Images{} + if releaseArch.Media.Aws != nil { + artifacts["aws"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aws.Artifacts), + } + awsAmis := stream.AwsImage{ + Regions: make(map[string]stream.AwsRegionImage), + } + if releaseArch.Media.Aws.Images != nil { + for region, ami := range releaseArch.Media.Aws.Images { + ri := stream.AwsRegionImage{Release: rel.Release, Image: ami.Image} + awsAmis.Regions[region] = ri + + } + cloudImages.Aws = &awsAmis + } + } + + if releaseArch.Media.Azure != nil { + artifacts["azure"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Azure.Artifacts), + } + + // Not enabled right now + // if az := releaseArch.Media.Azure.Images; az != nil && az.Global != nil && az.Global.Image != nil { + // azureImage := StreamCloudImage{} + // azureImage.Image = fmt.Sprintf("Fedora:CoreOS:%s:latest", rel.Stream) + // cloudImages.Azure = &azureImage + // } + } + + if releaseArch.Media.Aliyun != nil { + artifacts["aliyun"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aliyun.Artifacts), + } + } + + if releaseArch.Media.Exoscale != nil { + artifacts["exoscale"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Exoscale.Artifacts), + } + } + + if releaseArch.Media.Vultr != nil { + artifacts["vultr"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vultr.Artifacts), + } + } + + if releaseArch.Media.Gcp != nil { + artifacts["gcp"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Gcp.Artifacts), + } + + if releaseArch.Media.Gcp.Image != nil { + cloudImages.Gcp = &stream.GcpImage{ + Name: releaseArch.Media.Gcp.Image.Name, + Family: releaseArch.Media.Gcp.Image.Family, + Project: releaseArch.Media.Gcp.Image.Project, + } + } + } + + if releaseArch.Media.Digitalocean != nil { + artifacts["digitalocean"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Digitalocean.Artifacts), + } + + /* We're producing artifacts but they're not yet available + in DigitalOcean as distribution images. + digitalOceanImage := stream.CloudImage{Image: fmt.Sprintf("fedora-coreos-%s", Stream)} + cloudImages.Digitalocean = &digitalOceanImage + */ + } + + if releaseArch.Media.Ibmcloud != nil { + artifacts["ibmcloud"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Ibmcloud.Artifacts), + } + } + + // if releaseArch.Media.Packet != nil { + // packet := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Packet.Artifacts, + // } + // artifacts.Packet = &packet + + // packetImage := StreamCloudImage{Image: fmt.Sprintf("fedora_coreos_%s", rel.Stream)} + // cloudImages.Packet = &packetImage + // } + + if releaseArch.Media.Openstack != nil { + artifacts["openstack"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Openstack.Artifacts), + } + } + + if releaseArch.Media.Qemu != nil { + artifacts["qemu"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Qemu.Artifacts), + } + } + + // if releaseArch.Media.Virtualbox != nil { + // virtualbox := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Virtualbox.Artifacts, + // } + // artifacts.Virtualbox = &virtualbox + // } + + if releaseArch.Media.Vmware != nil { + artifacts["vmware"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vmware.Artifacts), + } + } + + if releaseArch.Media.Metal != nil { + artifacts["metal"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Metal.Artifacts), + } + } + + return stream.Arch{ + Artifacts: artifacts, + Images: cloudImages, + } +} + +// ToStreamArchitectures converts a release to a stream +func (rel *Release) ToStreamArchitectures() map[string]stream.Arch { + streamArch := make(map[string]stream.Arch) + for arch, releaseArch := range rel.Architectures { + streamArch[arch] = releaseArch.toStreamArch(rel) + } + return streamArch +} diff --git a/mantle/vendor/github.com/coreos/stream-metadata-go/stream/stream.go b/mantle/vendor/github.com/coreos/stream-metadata-go/stream/stream.go index 0c74117e1d..3571440012 100644 --- a/mantle/vendor/github.com/coreos/stream-metadata-go/stream/stream.go +++ b/mantle/vendor/github.com/coreos/stream-metadata-go/stream/stream.go @@ -1,10 +1,32 @@ +// Package stream models a CoreOS "stream", which is +// a description of the recommended set of binary images for CoreOS. Use +// this API to find cloud images, bare metal disk images, etc. package stream +// Stream contains artifacts available in a stream +type Stream struct { + Stream string `json:"stream"` + Metadata Metadata `json:"metadata"` + Architectures map[string]Arch `json:"architectures"` +} + // Metadata for a release or stream type Metadata struct { LastModified string `json:"last-modified"` } +// Arch contains release details for a particular hardware architecture +type Arch struct { + Artifacts map[string]PlatformArtifacts `json:"artifacts"` + Images Images `json:"images,omitempty"` +} + +// PlatformArtifacts contains images for a platform +type PlatformArtifacts struct { + Release string `json:"release"` + Formats map[string]ImageFormat `json:"formats"` +} + // ImageFormat contains all artifacts for a single OS image type ImageFormat struct { Disk *Artifact `json:"disk,omitempty"` @@ -20,32 +42,6 @@ type Artifact struct { Sha256 string `json:"sha256"` } -// GcpImage represents a GCP cloud image -type GcpImage struct { - Project string `json:"project,omitempty"` - Family string `json:"family,omitempty"` - Name string `json:"name,omitempty"` -} - -// Stream contains artifacts available in a stream -type Stream struct { - Stream string `json:"stream"` - Metadata Metadata `json:"metadata"` - Architectures map[string]Arch `json:"architectures"` -} - -// Architecture release details -type Arch struct { - Artifacts map[string]PlatformArtifacts `json:"artifacts"` - Images Images `json:"images,omitempty"` -} - -// PlatformArtifacts contains images for a platform -type PlatformArtifacts struct { - Release string `json:"release"` - Formats map[string]ImageFormat `json:"formats"` -} - // Images contains images available in cloud providers type Images struct { Aws *AwsImage `json:"aws,omitempty"` @@ -62,3 +58,10 @@ type AwsRegionImage struct { Release string `json:"release"` Image string `json:"image"` } + +// GcpImage represents a GCP cloud image +type GcpImage struct { + Project string `json:"project,omitempty"` + Family string `json:"family,omitempty"` + Name string `json:"name,omitempty"` +} diff --git a/mantle/vendor/modules.txt b/mantle/vendor/modules.txt index b80ec67d7f..7a36485013 100644 --- a/mantle/vendor/modules.txt +++ b/mantle/vendor/modules.txt @@ -167,7 +167,7 @@ github.com/coreos/ioprogress # github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/coreos/pkg/capnslog github.com/coreos/pkg/multierror -# github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c +# github.com/coreos/stream-metadata-go v0.0.0-20210115160721-ba77d4e64952 github.com/coreos/stream-metadata-go/fedoracoreos github.com/coreos/stream-metadata-go/fedoracoreos/internals github.com/coreos/stream-metadata-go/release From f042a752663d9bb73321315c67ef9a53c3c6ee64 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Jan 2021 21:04:53 +0000 Subject: [PATCH 6/9] generate-release-meta: Support overriding release name The command currently derives the cosa branch to determine the release stream ID; for RHCOS we have `HEAD` there. Support overriding it. --- src/cmd-generate-release-meta | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index 740475a932..8df826403b 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -35,6 +35,7 @@ def get_extension(path, modifier, arch): parser = ArgumentParser() parser.add_argument("--workdir", help="cosa workdir") parser.add_argument("--build-id", help="build id") +parser.add_argument("--stream-name", help="Override the stream ID (default is derived from coreos-assembler)") parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default="https://builds.coreos.fedoraproject.org/prod/streams") parser.add_argument("--output", help="Output to file; default is build directory") parser.add_argument("url", metavar='URL', help="URL to a coreos-assembler meta.json", default=[], nargs='*') @@ -89,7 +90,12 @@ def append_build(out, input_): arch = input_.get("coreos-assembler.basearch") ensure_dup(input_, out, "buildid", "release") - ensure_dup(input_.get('coreos-assembler.container-config-git'), out, 'branch', 'stream') + streamnamesrc = None + if args.stream_name: + streamnamesrc = {'branch': args.stream_name} + else: + streamnamesrc = input_.get('coreos-assembler.container-config-git') + ensure_dup(streamnamesrc, out, 'branch', 'stream') def artifact(i): base_url = url_builder(out.get('stream'), out.get('release'), arch, i.get('path')) From bd6ac31a70349755193e2fc1b888a99e143efd95 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 19 Jan 2021 15:46:52 +0000 Subject: [PATCH 7/9] generate-release-meta: Extract fcos endpoint to constant Cleanup from review. --- src/cmd-generate-release-meta | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index 8df826403b..b3e9d41dac 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -11,6 +11,8 @@ import requests from cosalib.builds import Builds +FCOS_STREAM_ENDPOINT = "https://builds.coreos.fedoraproject.org/prod/streams" + def ensure_dup(inp, out, inp_key, out_key): ''' @@ -36,7 +38,7 @@ parser = ArgumentParser() parser.add_argument("--workdir", help="cosa workdir") parser.add_argument("--build-id", help="build id") parser.add_argument("--stream-name", help="Override the stream ID (default is derived from coreos-assembler)") -parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default="https://builds.coreos.fedoraproject.org/prod/streams") +parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default=FCOS_STREAM_ENDPOINT) parser.add_argument("--output", help="Output to file; default is build directory") parser.add_argument("url", metavar='URL', help="URL to a coreos-assembler meta.json", default=[], nargs='*') args = parser.parse_args() From 6cd3635b8e51a82e9cf70c5012b7a4c40f25c766 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 9 Jan 2021 15:58:20 +0000 Subject: [PATCH 8/9] plume: add cosa2stream Part of implementing https://github.com/openshift/os/issues/477 I don't think for RHCOS (at least initially) we will go through the whole "release.json" middle ground, so let's add a command to directly turn 1 or more cosa build(s) into a stream JSON. Because the "cosa build" -> "release" bits are in Python, we semi-hackily fork it off as a subprocess. But, it works. --- mantle/cmd/plume/cosa2stream.go | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 mantle/cmd/plume/cosa2stream.go diff --git a/mantle/cmd/plume/cosa2stream.go b/mantle/cmd/plume/cosa2stream.go new file mode 100644 index 0000000000..da791c7fb4 --- /dev/null +++ b/mantle/cmd/plume/cosa2stream.go @@ -0,0 +1,88 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "time" + + "github.com/coreos/stream-metadata-go/release" + "github.com/coreos/stream-metadata-go/stream" + "github.com/spf13/cobra" +) + +var ( + cmdCosaBuildToStream = &cobra.Command{ + Use: "cosa2stream [options]", + Short: "Generate stream JSON from a coreos-assembler build", + RunE: runCosaBuildToStream, + + SilenceUsage: true, + } + + streamBaseURL string + streamName string +) + +func init() { + cmdCosaBuildToStream.Flags().StringVar(&streamBaseURL, "url", "", "Base URL for build") + cmdCosaBuildToStream.Flags().StringVar(&streamName, "name", "", "Stream name") + cmdCosaBuildToStream.MarkFlagRequired("name") + root.AddCommand(cmdCosaBuildToStream) +} + +func runCosaBuildToStream(cmd *cobra.Command, args []string) error { + releaseTmpf, err := ioutil.TempFile("", "release") + if err != nil { + return err + } + childArgs := []string{"generate-release-meta", "--stream-name=" + streamName, "--output=" + releaseTmpf.Name()} + if streamBaseURL != "" { + childArgs = append(childArgs, "--stream-baseurl="+streamBaseURL) + } + childArgs = append(childArgs, args...) + c := exec.Command("cosa", childArgs...) + c.Stderr = os.Stderr + if err := c.Run(); err != nil { + return err + } + + var rel release.Release + buf, err := ioutil.ReadAll(releaseTmpf) + if err != nil { + return err + } + if err := json.Unmarshal(buf, &rel); err != nil { + return err + } + + // Generate output stream from release + outStream := stream.Stream{ + Stream: streamName, + Metadata: stream.Metadata{LastModified: time.Now().UTC().Format(time.RFC3339)}, + Architectures: rel.ToStreamArchitectures(), + } + + // Serialize to JSON + encoder := json.NewEncoder(os.Stdout) + if err := encoder.Encode(&outStream); err != nil { + return fmt.Errorf("Error while encoding: %v", err) + } + return nil +} From 02010f7674624cca2da063fcd8a4478a1abb735b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 20 Jan 2021 17:51:55 +0000 Subject: [PATCH 9/9] Switch to --url argument for generating release metadata To clearly show that url and local workdir can both be used. --- mantle/cmd/plume/cosa2stream.go | 4 +++- src/cmd-generate-release-meta | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mantle/cmd/plume/cosa2stream.go b/mantle/cmd/plume/cosa2stream.go index da791c7fb4..ca6c9dfc95 100644 --- a/mantle/cmd/plume/cosa2stream.go +++ b/mantle/cmd/plume/cosa2stream.go @@ -56,7 +56,9 @@ func runCosaBuildToStream(cmd *cobra.Command, args []string) error { if streamBaseURL != "" { childArgs = append(childArgs, "--stream-baseurl="+streamBaseURL) } - childArgs = append(childArgs, args...) + for _, arg := range args { + childArgs = append(childArgs, fmt.Sprintf("--url="+arg)) + } c := exec.Command("cosa", childArgs...) c.Stderr = os.Stderr if err := c.Run(); err != nil { diff --git a/src/cmd-generate-release-meta b/src/cmd-generate-release-meta index b3e9d41dac..0fad939791 100755 --- a/src/cmd-generate-release-meta +++ b/src/cmd-generate-release-meta @@ -40,7 +40,7 @@ parser.add_argument("--build-id", help="build id") parser.add_argument("--stream-name", help="Override the stream ID (default is derived from coreos-assembler)") parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default=FCOS_STREAM_ENDPOINT) parser.add_argument("--output", help="Output to file; default is build directory") -parser.add_argument("url", metavar='URL', help="URL to a coreos-assembler meta.json", default=[], nargs='*') +parser.add_argument("--url", help="URL to a coreos-assembler meta.json", default=[], action='append') args = parser.parse_args()