diff --git a/Cargo.lock b/Cargo.lock index 96f7f82bd..2b20caacd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" +checksum = "5220b8ba44c68a9a7f7a7659e864dd73692e417ef0211bea133c7b74e031eeb9" dependencies = [ "bitflags", "libc", @@ -459,9 +459,9 @@ checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libgit2-sys" -version = "0.18.0+1.9.0" +version = "0.18.1+1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" +checksum = "e1dcb20f84ffcdd825c7a311ae347cce604a6f084a767dec4a4929829645290e" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 673aa5881..805dab66f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.dependencies] clap = { version = "4.5.27", features = ["derive"] } -git2 = "0.20.0" +git2 = "0.20.1" serde = { version = "1.0.217", features = ["derive"] } snafu = "0.8.5" tempfile = "3.16.0" diff --git a/druid/Dockerfile b/druid/Dockerfile index a45d6b02a..7acb8f3f1 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -32,6 +32,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/patchable.toml /stackable/src/druid/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default diff --git a/druid/stackable/patches/30.0.0/patchable.toml b/druid/stackable/patches/30.0.0/patchable.toml index 892b6fab4..53c505db4 100644 --- a/druid/stackable/patches/30.0.0/patchable.toml +++ b/druid/stackable/patches/30.0.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "09d36ee324747f1407705c27618b6d415c3fa8a9" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/30.0.1/patchable.toml b/druid/stackable/patches/30.0.1/patchable.toml index aad1cde81..2f31eb1e9 100644 --- a/druid/stackable/patches/30.0.1/patchable.toml +++ b/druid/stackable/patches/30.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "a30af7a91d528e5c3a90356a5592abc7119191c6" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/31.0.1/patchable.toml b/druid/stackable/patches/31.0.1/patchable.toml index 97ae47d66..63a03b2f9 100644 --- a/druid/stackable/patches/31.0.1/patchable.toml +++ b/druid/stackable/patches/31.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/druid.git" base = "520482cb9638e452b0553595b4f29bb397a63758" +mirror = "https://github.com/stackabletech/druid.git" diff --git a/druid/stackable/patches/patchable.toml b/druid/stackable/patches/patchable.toml new file mode 100644 index 000000000..7b7139a39 --- /dev/null +++ b/druid/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/druid.git" +default-mirror = "https://github.com/stackabletech/druid.git" diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index cdbc8caa8..990425c5f 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -51,6 +51,7 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j EOF WORKDIR /build +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/patchable.toml /build/src/hadoop/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches/${PRODUCT} /build/src/hadoop/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /build COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx diff --git a/hadoop/stackable/patches/3.3.4/patchable.toml b/hadoop/stackable/patches/3.3.4/patchable.toml index b35894cbc..a7fece048 100644 --- a/hadoop/stackable/patches/3.3.4/patchable.toml +++ b/hadoop/stackable/patches/3.3.4/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "a585a73c3e02ac62350c136643a5e7f6095a3dbb" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.3.6/patchable.toml b/hadoop/stackable/patches/3.3.6/patchable.toml index 26e9adf44..54002b2ca 100644 --- a/hadoop/stackable/patches/3.3.6/patchable.toml +++ b/hadoop/stackable/patches/3.3.6/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "1be78238728da9266a4f88195058f08fd012bf9c" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.0/patchable.toml b/hadoop/stackable/patches/3.4.0/patchable.toml index ef364542d..038c64315 100644 --- a/hadoop/stackable/patches/3.4.0/patchable.toml +++ b/hadoop/stackable/patches/3.4.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "bd8b77f398f626bb7791783192ee7a5dfaeec760" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/stackable/patches/3.4.1/patchable.toml index 6a697d142..fd6df6895 100644 --- a/hadoop/stackable/patches/3.4.1/patchable.toml +++ b/hadoop/stackable/patches/3.4.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hadoop.git" base = "4d7825309348956336b8f06a08322b78422849b1" +mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hadoop/stackable/patches/patchable.toml b/hadoop/stackable/patches/patchable.toml new file mode 100644 index 000000000..79e086ca7 --- /dev/null +++ b/hadoop/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hadoop.git" +default-mirror = "https://github.com/stackabletech/hadoop.git" diff --git a/hbase/Dockerfile b/hbase/Dockerfile index ae10f8054..4a0ba9dd5 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -26,6 +26,7 @@ COPY hbase/licenses /licenses USER ${STACKABLE_USER_UID} WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/patchable.toml /stackable/src/hbase/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/${PRODUCT} /stackable/src/hbase/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx @@ -142,6 +143,7 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase-operator-tools/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} /stackable/src/hbase-operator-tools/stackable/patches/${HBASE_OPERATOR_TOOLS} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ @@ -241,6 +243,7 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/patchable.toml /stackable/src/phoenix/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} USER ${STACKABLE_USER_UID} WORKDIR /stackable diff --git a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml index 312b144c1..a6296d2f7 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.2.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase-operator-tools.git" base = "478af00af79f82624264fd2bb447b97fecc8e790" +mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml index 165a2494e..e6f7c576d 100644 --- a/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml +++ b/hbase/hbase-operator-tools/stackable/patches/1.3.0-fd5a5fb/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase-operator-tools.git" base = "fd5a5fb90755949a90c502c76de8313130403fa3" +mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/hbase-operator-tools/stackable/patches/patchable.toml b/hbase/hbase-operator-tools/stackable/patches/patchable.toml new file mode 100644 index 000000000..ce4c62faf --- /dev/null +++ b/hbase/hbase-operator-tools/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hbase-operator-tools.git" +default-mirror = "https://github.com/stackabletech/hbase-operator-tools.git" diff --git a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml index fd13de0cb..a0b11000b 100644 --- a/hbase/phoenix/stackable/patches/5.2.1/patchable.toml +++ b/hbase/phoenix/stackable/patches/5.2.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/phoenix.git" base = "b738d66cb5863b759bb98eaa417b3b5731d41f95" +mirror = "https://github.com/stackabletech/phoenix.git" diff --git a/hbase/phoenix/stackable/patches/patchable.toml b/hbase/phoenix/stackable/patches/patchable.toml new file mode 100644 index 000000000..6dd31259f --- /dev/null +++ b/hbase/phoenix/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/phoenix.git" +default-mirror = "https://github.com/stackabletech/phoenix.git" diff --git a/hbase/stackable/patches/2.4.18/patchable.toml b/hbase/stackable/patches/2.4.18/patchable.toml index e8de3270a..20dfd9e95 100644 --- a/hbase/stackable/patches/2.4.18/patchable.toml +++ b/hbase/stackable/patches/2.4.18/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "a1767f4d76859c0068720a6c1e5cb78282ebfe1e" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.0/patchable.toml b/hbase/stackable/patches/2.6.0/patchable.toml index 0e7f1956f..c914eb82b 100644 --- a/hbase/stackable/patches/2.6.0/patchable.toml +++ b/hbase/stackable/patches/2.6.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "de99f8754135ea69adc39da48d2bc2b2710a5366" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/2.6.1/patchable.toml b/hbase/stackable/patches/2.6.1/patchable.toml index 4a7b15c36..c1c9eacbf 100644 --- a/hbase/stackable/patches/2.6.1/patchable.toml +++ b/hbase/stackable/patches/2.6.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hbase.git" base = "7ed50b4dd742269a78875fb32112215f831284ff" +mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hbase/stackable/patches/patchable.toml b/hbase/stackable/patches/patchable.toml new file mode 100644 index 000000000..2a84f6d70 --- /dev/null +++ b/hbase/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hbase.git" +default-mirror = "https://github.com/stackabletech/hbase.git" diff --git a/hive/Dockerfile b/hive/Dockerfile index 65c7209b5..e04987743 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -23,6 +23,7 @@ ARG STACKABLE_USER_UID ARG DELETE_CACHES="true" # Copy patches into the builder +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/patchable.toml /stackable/src/hive/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackable/src/hive/stackable/patches/${PRODUCT} # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here diff --git a/hive/stackable/patches/3.1.3/patchable.toml b/hive/stackable/patches/3.1.3/patchable.toml index bf3ab1b59..b44ca05a9 100644 --- a/hive/stackable/patches/3.1.3/patchable.toml +++ b/hive/stackable/patches/3.1.3/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "4df4d75bf1e16fe0af75aad0b4179c34c07fc975" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.0/patchable.toml b/hive/stackable/patches/4.0.0/patchable.toml index 7e70d2b90..f8bd6ce3f 100644 --- a/hive/stackable/patches/4.0.0/patchable.toml +++ b/hive/stackable/patches/4.0.0/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "183f8cb41d3dbed961ffd27999876468ff06690c" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/4.0.1/patchable.toml b/hive/stackable/patches/4.0.1/patchable.toml index 37091bbe1..807793caf 100644 --- a/hive/stackable/patches/4.0.1/patchable.toml +++ b/hive/stackable/patches/4.0.1/patchable.toml @@ -1,2 +1,2 @@ -upstream = "https://github.com/apache/hive.git" base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264" +mirror = "https://github.com/stackabletech/hive.git" diff --git a/hive/stackable/patches/patchable.toml b/hive/stackable/patches/patchable.toml new file mode 100644 index 000000000..b193b451c --- /dev/null +++ b/hive/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hive.git" +default-mirror = "https://github.com/stackabletech/hive.git" diff --git a/kafka/Dockerfile b/kafka/Dockerfile index ff8f7a5d4..df64d10cb 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -15,6 +15,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/patchable.toml /stackable/src/kafka/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} RUN </stackable/patches//patchable.toml`. -It currently recognizes the following keys: +Patchable uses a two-level configuration system: + +1. A product-level config file at `docker-images//stackable/patches/patchable.toml` +2. A version-level config file at `docker-images//stackable/patches//patchable.toml` + +The product-level config contains: - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) -- `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`) +- `default_mirror` - optional: default URL of a mirror repository (such as `https://github.com/stackabletech/druid.git`) + +The version-level config contains: + +- `base` - the commit hash of the upstream base commit +- `mirror` - optional: URL of the mirror repository for this version, if mirroring is enabled ### Template -Instead of creating this manually, run `patchable init`: +If you're adding a completely new product, you need to create the product-level config once: ```toml -cargo patchable init druid 28.0.0 --upstream=https://github.com/apache/druid.git --base=druid-28.0.0 +# docker-images/druid/stackable/patches/patchable.toml +upstream = "https://github.com/apache/druid.git" +mirror = "https://github.com/stackabletech/druid.git" ``` +If you just want to add a new version, initialize the version-level config with patchable: + +```sh +cargo patchable init druid 28.0.0 --base=druid-28.0.0 --mirror +``` + +This will initialize the version-level config with the base commit hash and the default mirror URL from the product-level config. +You can optionally provide the `--ssh` flag to use SSH instead of HTTPS for Git operations. + ## Glossary - Images repo/directory - The checkout of stackabletech/docker-images diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index c1adcb53d..65983daf9 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -13,6 +13,8 @@ use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_indicatif::IndicatifLayer; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; +use crate::utils::setup_git_credentials; + #[derive(clap::Parser)] struct ProductVersion { /// The product name slug (such as druid) @@ -24,9 +26,35 @@ struct ProductVersion { version: String, } +/// Configuration that applies to all versions of a product, located at $ROOT/$product/stackable/patches/patchable.toml +/// +/// Must be created by hand (for now). #[derive(Deserialize, Serialize)] -struct ProductVersionConfig { +#[serde(rename_all = "kebab-case")] +struct ProductConfig { + /// The upstream repository URL upstream: String, + + /// The repository that commits are mirrored to by `init --mirror`, typically `https://github.com/stackabletech/$product.git` + /// + /// This value is _not_ used by `checkout`, that uses [`ProductVersionConfig::mirror`] instead. + /// `init --mirror` copies this value into [`ProductVersionConfig::mirror`]. + default_mirror: Option, +} + +/// Configuration that applies to an individual version of a product, located at $ROOT/$product/stackable/patches/$version/patchable.toml +/// +/// Typically created by `patchable init`. +#[derive(Deserialize, Serialize)] +struct ProductVersionConfig { + /// The mirror repository that this version can be fetched from + /// + /// Copied from [`ProductConfig::default_mirror`] by `init --mirror`. + mirror: Option, + + /// The upstream base commit for this version + /// + /// Must be a commit ID, not a generic commitish (branch, tag, or anotherother ref), those are resolved by `init`. #[serde(with = "utils::oid_serde")] base: Oid, } @@ -37,16 +65,24 @@ struct ProductVersionContext { } impl ProductVersionContext { - fn load_config(&self) -> Result { - let path = &self.config_path(); + fn load_config Deserialize<'de>>(&self, path: &PathBuf) -> Result { tracing::info!( config.path = ?path, "loading config" ); - toml::from_str::( - &std::fs::read_to_string(path).context(LoadConfigSnafu { path })?, - ) - .context(ParseConfigSnafu { path }) + + toml::from_str::(&std::fs::read_to_string(path).context(LoadConfigSnafu { path })?) + .context(ParseConfigSnafu { path }) + } + + fn load_product_config(&self) -> Result { + let path = self.product_config_path(); + self.load_config(&path) + } + + fn load_version_config(&self) -> Result { + let path = self.version_config_path(); + self.load_config(&path) } /// The root directory for files related to the product (across all versions). @@ -62,10 +98,15 @@ impl ProductVersionContext { } /// The patchable configuration file for the product version. - fn config_path(&self) -> PathBuf { + fn version_config_path(&self) -> PathBuf { self.patch_dir().join("patchable.toml") } + /// The product-level patchable configuration file + fn product_config_path(&self) -> PathBuf { + self.product_dir().join("stackable/patches/patchable.toml") + } + /// The directory containing all ephemeral data used by patchable for the product (across all versions). /// /// Should be gitignored, and can safely be deleted as long as all relevant versions have been `patchable export`ed. @@ -125,6 +166,10 @@ enum Cmd { /// Check out the base commit, without applying patches #[clap(long)] base_only: bool, + + /// Use SSH for git operations + #[clap(long)] + ssh: bool, }, /// Export the patches from the source tree at docker-images//patchable-work/worktree/ @@ -140,15 +185,19 @@ enum Cmd { #[clap(flatten)] pv: ProductVersion, - /// The upstream URL (such as https://github.com/apache/druid.git) - #[clap(long)] - upstream: String, - /// The upstream commit-ish (such as druid-28.0.0) that the patch series applies to /// /// Refs (such as tags and branches) will be resolved to commit IDs. #[clap(long)] base: String, + + /// Mirror the product version to the default mirror repository + #[clap(long)] + mirror: bool, + + /// Use SSH for git operations + #[clap(long)] + ssh: bool, }, /// Shows the patch directory for a given product version @@ -197,6 +246,23 @@ pub enum Error { path: PathBuf, }, + #[snafu(display("failed to rewrite URL for SSH"))] + UrlRewrite { source: utils::UrlRewriteError }, + + #[snafu(display( + "mirroring requested, but default-mirror is not configured in product configuration" + ))] + InitMirrorNotConfigured, + #[snafu(display("failed to add temporary mirror remote for {url:?}"))] + AddMirrorRemote { source: git2::Error, url: String }, + #[snafu(display("failed to push commit {commit} (as {refspec}) to mirror {url:?}"))] + PushToMirror { + source: git2::Error, + url: String, + refspec: String, + commit: Oid, + }, + #[snafu(display("failed to find images repository"))] FindImagesRepo { source: repo::Error }, #[snafu(display("images repository has no work directory"))] @@ -274,20 +340,31 @@ fn main() -> Result<()> { } }; match opts.cmd { - Cmd::Checkout { pv, base_only } => { + Cmd::Checkout { pv, base_only, ssh } => { let ctx = ProductVersionContext { pv, images_repo_root, }; - let config = ctx.load_config()?; + let product_config = ctx.load_product_config()?; + let version_config = ctx.load_version_config()?; let product_repo_root = ctx.product_repo(); let product_repo = repo::ensure_bare_repo(&product_repo_root) .context(OpenProductRepoForCheckoutSnafu)?; + let mut upstream = version_config.mirror.unwrap_or_else(|| { + tracing::warn!("this product version is not mirrored, re-init it with --mirror before merging it"); + product_config.upstream + }); + + if ssh { + upstream = + utils::rewrite_git_https_url_to_ssh(&upstream).context(UrlRewriteSnafu)?; + } + let base_commit = repo::resolve_and_fetch_commitish( &product_repo, - &config.base.to_string(), - &config.upstream, + &version_config.base.to_string(), + &upstream, ) .context(FetchBaseCommitSnafu)?; let base_branch = ctx.base_branch(); @@ -348,7 +425,7 @@ fn main() -> Result<()> { pv, images_repo_root, }; - let config = ctx.load_config()?; + let config = ctx.load_version_config()?; let product_worktree_root = ctx.worktree_root(); tracing::info!( @@ -397,7 +474,12 @@ fn main() -> Result<()> { ); } - Cmd::Init { pv, upstream, base } => { + Cmd::Init { + pv, + base, + mirror, + ssh, + } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -411,6 +493,13 @@ fn main() -> Result<()> { .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) .context(OpenProductRepoForCheckoutSnafu)?; + let config = ctx.load_product_config()?; + let upstream = if ssh { + utils::rewrite_git_https_url_to_ssh(&config.upstream).context(UrlRewriteSnafu)? + } else { + config.upstream + }; + // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); @@ -418,12 +507,67 @@ fn main() -> Result<()> { .context(FetchBaseCommitSnafu)?; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); - tracing::info!("saving configuration"); + let mirror_url = if mirror { + let mut mirror_url = config + .default_mirror + .context(InitMirrorNotConfiguredSnafu)?; + if ssh { + mirror_url = + utils::rewrite_git_https_url_to_ssh(&mirror_url).context(UrlRewriteSnafu)? + }; + // Add mirror remote + let mut mirror_remote = + product_repo + .remote_anonymous(&mirror_url) + .context(AddMirrorRemoteSnafu { + url: mirror_url.clone(), + })?; + + // Push the base commit to the mirror + tracing::info!(commit = %base_commit, base = base, url = mirror_url, "pushing commit to mirror"); + let mut callbacks = setup_git_credentials(); + + // Add progress tracking for push operation + let (span_push, mut quant_push) = + utils::setup_progress_tracking(tracing::info_span!("pushing")); + let _ = span_push.enter(); + + callbacks.push_transfer_progress(move |current, total, _| { + if total > 0 { + quant_push.update_span_progress(current, total, &span_push); + } + }); + + let mut push_options = git2::PushOptions::new(); + push_options.remote_callbacks(callbacks); + + // Always push the commit as a Git tag named like the value of `base` + let refspec = format!("{base_commit}:refs/tags/{base}"); + tracing::info!(refspec, "constructed push refspec"); + + mirror_remote + .push(&[&refspec], Some(&mut push_options)) + .context(PushToMirrorSnafu { + url: &mirror_url, + refspec: &refspec, + commit: base_commit, + })?; + + tracing::info!("successfully pushed base ref to mirror"); + Some(mirror_url) + } else { + tracing::warn!( + "this version is not mirrored, re-run with --mirror before merging into main" + ); + None + }; + + tracing::info!("saving version-level configuration"); let config = ProductVersionConfig { - upstream, base: base_commit, + mirror: mirror_url, }; - let config_path = ctx.config_path(); + let config_path = ctx.version_config_path(); if let Some(config_dir) = config_path.parent() { std::fs::create_dir_all(config_dir) .context(CreatePatchDirSnafu { path: config_dir })?; diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 1b41f726c..962dc03d9 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -1,15 +1,14 @@ use std::path::{self, Path, PathBuf}; use git2::{ - FetchOptions, ObjectType, Oid, RemoteCallbacks, Repository, RepositoryInitOptions, + FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, WorktreeAddOptions, }; use snafu::{ResultExt, Snafu}; -use tracing_indicatif::span_ext::IndicatifSpanExt; use crate::{ error::{self, CommitRef}, - utils::{progress_bar_style, Quantizer}, + utils::{setup_git_credentials, setup_progress_tracking}, }; #[derive(Debug, Snafu)] @@ -149,15 +148,16 @@ pub fn resolve_and_fetch_commitish( error = &err as &dyn std::error::Error, "base commit not found locally, fetching from upstream" ); - let span_recv = tracing::info_span!("receiving"); - let span_index = tracing::info_span!("indexing"); - span_recv.pb_set_style(&progress_bar_style()); - span_index.pb_set_style(&progress_bar_style()); + + let (span_recv, mut quant_recv) = + setup_progress_tracking(tracing::info_span!("receiving")); + let (span_index, mut quant_index) = + setup_progress_tracking(tracing::info_span!("indexing")); + let _ = span_recv.enter(); let _ = span_index.enter(); - let mut callbacks = RemoteCallbacks::new(); - let mut quant_recv = Quantizer::percent(); - let mut quant_index = Quantizer::percent(); + + let mut callbacks = setup_git_credentials(); callbacks.transfer_progress(move |progress| { quant_recv.update_span_progress( progress.received_objects(), diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index 9d08666d1..0acff7456 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -1,14 +1,25 @@ use std::path::Path; use git2::Repository; +use snafu::{OptionExt as _, Snafu}; use tracing::Span; use tracing_indicatif::{span_ext::IndicatifSpanExt, style::ProgressStyle}; -pub fn progress_bar_style() -> ProgressStyle { - ProgressStyle::with_template( - "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%", - ) - .expect("hard-coded template should be valid") +#[derive(Debug, Snafu)] +pub enum UrlRewriteError { + #[snafu(display("URL does not have https schema: {}", url))] + NoHttpsSchema { url: String }, +} + +/// Rewrites a given URL to an SSH-style Git URL +/// For example, `https://github.com/user/repo.git` becomes `git@github.com:user/repo.git`. +pub fn rewrite_git_https_url_to_ssh(original_url: &str) -> Result { + let schemaless = original_url + .strip_prefix("https://") + .context(NoHttpsSchemaSnafu { + url: original_url.to_string(), + })?; + Ok(format!("ssh://git@{schemaless}")) } /// Runs a function whenever a `value` changes "enough". @@ -94,3 +105,30 @@ pub mod oid_serde { .and_then(|oid| Oid::from_str(&oid).map_err(::custom)) } } + +/// Sets up progress tracking for Git operations with a progress bar. +pub fn setup_progress_tracking(span: tracing::Span) -> (tracing::Span, Quantizer) { + span.pb_set_style(&ProgressStyle::with_template( + "{span_child_prefix}{spinner} {span_name}{{{span_fields}}} {wide_msg} {bar:40} {percent:>3}%", + ) + .expect("hard-coded template should be valid")); + let quantizer = Quantizer::percent(); + (span, quantizer) +} + +/// Basic configuration of credentials for Git operations. +pub fn setup_git_credentials<'a>() -> git2::RemoteCallbacks<'a> { + let mut callbacks = git2::RemoteCallbacks::new(); + callbacks.credentials(|url, username_from_url, allowed_types| { + if allowed_types.contains(git2::CredentialType::SSH_KEY) { + git2::Cred::ssh_key_from_agent(username_from_url.unwrap_or("git")) + } else { + git2::Cred::credential_helper( + &git2::Config::open_default().expect("failed to open default Git configuration"), + url, + username_from_url, + ) + } + }); + callbacks +} diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 4f20dc1bb..05e39c9a1 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -15,6 +15,7 @@ ARG STACKABLE_USER_UID WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/patchable.toml /stackable/src/spark-k8s/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/${PRODUCT} /stackable/src/spark-k8s/stackable/patches/${PRODUCT} RUN /stackable/patchable --images-repo-root=src checkout spark-k8s ${PRODUCT} @@ -42,6 +43,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ # Patch the hbase-connectors source code WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/patchable.toml /stackable/src/spark-k8s/hbase-connectors/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR} /stackable/src/spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR} RUN <