diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java index 6ebc6e8ec6a3b..4cc3edff3eb52 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java @@ -54,6 +54,7 @@ static List generateSteps( .getBootstrap() .getTemplates() .stream() + .map(cfg -> cfg.withOverride(opContext.getObjectMapper())) .filter(cfg -> cfg.isBlocking() == isBlocking) .map(cfg -> new BootstrapMCPStep(opContext, entityService, cfg)) .collect(Collectors.toList()); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java index 8fd3dd7c7d897..009d19e453b6a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java @@ -1,5 +1,7 @@ package com.linkedin.datahub.upgrade.system.bootstrapmcps.model; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -7,6 +9,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; @AllArgsConstructor @NoArgsConstructor @@ -23,6 +26,7 @@ public static class Bootstrap { private List templates; } + @Slf4j @AllArgsConstructor @NoArgsConstructor @Data @@ -36,5 +40,19 @@ public static class MCPTemplate { @Builder.Default private boolean optional = false; @Nonnull private String mcps_location; @Nullable private String values_env; + @Nullable private String revision_env; + + public MCPTemplate withOverride(ObjectMapper objectMapper) { + if (revision_env != null) { + String overrideJson = System.getenv().getOrDefault(revision_env, "{}"); + try { + return objectMapper.readerForUpdating(this).readValue(overrideJson); + } catch (IOException e) { + log.error("Error applying override {} to {}", overrideJson, this); + throw new RuntimeException(e); + } + } + return this; + } } } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java index b471e366e906f..f914b355fe780 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java @@ -4,6 +4,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.UrnUtils; @@ -17,6 +18,7 @@ import io.datahubproject.test.metadata.context.TestOperationContexts; import java.io.IOException; import java.util.List; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Listeners; import org.testng.annotations.Test; import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; @@ -28,10 +30,17 @@ public class BootstrapMCPUtilTest { static final OperationContext OP_CONTEXT = TestOperationContexts.systemContextNoSearchAuthorization(); private static final String DATAHUB_TEST_VALUES_ENV = "DATAHUB_TEST_VALUES_ENV"; + private static final String DATAHUB_TEST_REVISION_ENV = "DATAHUB_TEST_REVISION_ENV"; private static final AuditStamp TEST_AUDIT_STAMP = AuditStampUtils.createDefaultAuditStamp(); @SystemStub private EnvironmentVariables environmentVariables; + @BeforeMethod + private void resetEnvironment() { + environmentVariables.remove(DATAHUB_TEST_VALUES_ENV); + environmentVariables.remove(DATAHUB_TEST_REVISION_ENV); + } + @Test public void testResolveYamlConf() throws IOException { BootstrapMCPConfigFile initConfig = @@ -51,9 +60,28 @@ public void testResolveYamlConf() throws IOException { } @Test - public void testResolveMCPTemplateDefaults() throws IOException { - environmentVariables.remove(DATAHUB_TEST_VALUES_ENV); + public void testResolveYamlConfOverride() throws IOException { + environmentVariables.set(DATAHUB_TEST_REVISION_ENV, "{\"version\":\"2024110600\"}"); + + BootstrapMCPConfigFile initConfig = + BootstrapMCPUtil.resolveYamlConf( + OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class); + assertEquals(initConfig.getBootstrap().getTemplates().size(), 1); + + BootstrapMCPConfigFile.MCPTemplate template = + initConfig.getBootstrap().getTemplates().get(0).withOverride(new ObjectMapper()); + assertEquals(template.getName(), "datahub-test"); + assertEquals(template.getVersion(), "2024110600"); + assertFalse(template.isForce()); + assertFalse(template.isBlocking()); + assertTrue(template.isAsync()); + assertFalse(template.isOptional()); + assertEquals(template.getMcps_location(), "bootstrapmcp/datahub-test-mcp.yaml"); + assertEquals(template.getValues_env(), "DATAHUB_TEST_VALUES_ENV"); + } + @Test + public void testResolveMCPTemplateDefaults() throws IOException { BootstrapMCPConfigFile.MCPTemplate template = BootstrapMCPUtil.resolveYamlConf( OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class) @@ -186,8 +214,6 @@ public void testResolveMCPTemplateOverride() throws IOException { @Test public void testMCPBatch() throws IOException { - environmentVariables.remove(DATAHUB_TEST_VALUES_ENV); - BootstrapMCPConfigFile.MCPTemplate template = BootstrapMCPUtil.resolveYamlConf( OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class) diff --git a/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml b/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml index 649cc09632fc2..5718ea3ac0e04 100644 --- a/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml +++ b/datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml @@ -6,4 +6,5 @@ bootstrap: # blocking: false # async: true mcps_location: "bootstrapmcp/datahub-test-mcp.yaml" - values_env: "DATAHUB_TEST_VALUES_ENV" \ No newline at end of file + values_env: "DATAHUB_TEST_VALUES_ENV" + revision_env: "DATAHUB_TEST_REVISION_ENV" \ No newline at end of file diff --git a/docs/advanced/bootstrap-mcps.md b/docs/advanced/bootstrap-mcps.md index 0aa4b7608740f..c3ad7db2016db 100644 --- a/docs/advanced/bootstrap-mcps.md +++ b/docs/advanced/bootstrap-mcps.md @@ -149,6 +149,28 @@ to the required json structure and stored as a string. executorId: default ``` +## `bootstrap_mcps.yaml` Override + +Additionally, the `bootstrap_mcps.yaml` can be overridden. +This might be useful for applying changes to the version when using helm defined template values. + +```yaml +bootstrap: + templates: + - name: myMCPTemplate + version: v1 + mcps_location: + values_env: + revision_env: REVISION_ENV +``` + +In the above example, we've added a `revision_env` which allows overriding the MCP bootstrap definition itself (excluding `revision_env`). + +In this example we could configure `REVISION_ENV` to contain a timestamp or hash: `{"version":"2024060600"}` +This value can be changed/incremented each time the helm supplied template values change. This ensures the MCP is updated +with the latest values during deployment. + + ## Known Limitations * Supported change types: diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml index a33fad1058962..f9497258c384f 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml @@ -42,3 +42,4 @@ bootstrap: optional: false mcps_location: "bootstrap_mcps/ingestion-datahub-gc.yaml" values_env: "DATAHUB_GC_BOOTSTRAP_VALUES" + revision_env: "DATAHUB_GC_BOOTSTRAP_REVISION" diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml index e78b709ad6cea..395eb5db53424 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml @@ -12,7 +12,7 @@ timezone: '{{schedule.timezone}}{{^schedule.timezone}}UTC{{/schedule.timezone}}' interval: '{{schedule.interval}}{{^schedule.interval}}0 1 * * *{{/schedule.interval}}' config: - version: '{{&ingestion.version}}{{^ingestion.version}}0.14.1.6{{/ingestion.version}}' + version: '{{&ingestion.version}}{{^ingestion.version}}0.14.1.7rc2{{/ingestion.version}}' recipe: source: type: 'datahub-gc'