From 2c3393c1bb13ffe69cb3af6061c3400beb69735b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 10 Oct 2024 20:42:34 +0000 Subject: [PATCH 1/3] Controlling the Migration Assistant Control Plane Signed-off-by: Peter Nied --- .../docker/entrypoint.sh | 91 ++------ .../migrations/RfsMigrateDocuments.java | 2 + .../opensearch/migrations/MetadataArgs.java | 3 + .../migrations/MetadataMigration.java | 74 ++++++- .../src/main/docker/docker-compose.yml | 2 +- .../docker-compose-console-only.yml | 7 +- .../lib/console_link/README.md | 12 +- .../lib/console_link/console_link/cli.py | 4 +- .../console_link/console_link/environment.py | 3 +- .../console_link/models/metadata.py | 194 +----------------- .../console_link/models/snapshot.py | 1 - .../migrationConsole/lib/integ_test/README.md | 2 +- .../lib/integ_test/integ_test/conftest.py | 2 +- .../loadServicesFromParameterStore.sh | 2 +- .../build.gradle | 2 + coreUtilities/build.gradle | 1 + .../migrations/config/Backfill.java | 7 + .../migrations/config/BasicAuth.java | 7 + .../opensearch/migrations/config/Cluster.java | 10 + .../config/ElasticContainerService.java | 7 + .../migrations/config/FileSystem.java | 5 + .../opensearch/migrations/config/Kafka.java | 7 + .../migrations/config/MetadataMigration.java | 13 ++ .../migrations/config/MetricsSource.java | 5 + .../migrations/config/MigrationConfig.java | 28 +++ .../opensearch/migrations/config/NoAuth.java | 5 + .../config/ReindexFromSnapshot.java | 10 + .../opensearch/migrations/config/Replay.java | 7 + .../migrations/config/S3Bucket.java | 6 + .../migrations/config/Sigv4Auth.java | 6 + .../migrations/config/Snapshot.java | 7 + .../migrations/config/SnapshotCreator.java | 5 + .../config/MigrationConfigTest.java | 61 ++++++ docs/ControlPlaneControls.md | 78 +++++++ 34 files changed, 400 insertions(+), 276 deletions(-) create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Backfill.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/BasicAuth.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Cluster.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/ElasticContainerService.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/FileSystem.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Kafka.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/MetadataMigration.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/MetricsSource.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/MigrationConfig.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/NoAuth.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/ReindexFromSnapshot.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Replay.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/S3Bucket.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Sigv4Auth.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/Snapshot.java create mode 100644 coreUtilities/src/main/java/org/opensearch/migrations/config/SnapshotCreator.java create mode 100644 coreUtilities/src/test/java/org/opensearch/migrations/config/MigrationConfigTest.java create mode 100644 docs/ControlPlaneControls.md diff --git a/DocumentsFromSnapshotMigration/docker/entrypoint.sh b/DocumentsFromSnapshotMigration/docker/entrypoint.sh index 407277328..7a942796f 100755 --- a/DocumentsFromSnapshotMigration/docker/entrypoint.sh +++ b/DocumentsFromSnapshotMigration/docker/entrypoint.sh @@ -3,78 +3,22 @@ # Fail the script if any command fails set -e -# Print our ENV variables -if [[ $RFS_COMMAND != *"--target-password"* ]]; then - echo "RFS_COMMAND: $RFS_COMMAND" -else - echo "RFS Target Cluster password found in RFS_COMMAND; skipping logging of the value" -fi - -echo "RFS_TARGET_USER: $RFS_TARGET_USER" -echo "RFS_TARGET_PASSWORD: " -echo "RFS_TARGET_PASSWORD_ARN: $RFS_TARGET_PASSWORD_ARN" - -# Check if the RFS Command already contains a username; only do special work if it does not -if [[ $RFS_COMMAND != *"--target-username"* ]]; then - if [[ -n "$RFS_TARGET_USER" ]]; then - echo "Using username from ENV variable RFS_TARGET_USER. Updating RFS Command with username." - RFS_COMMAND="$RFS_COMMAND --target-username \"$RFS_TARGET_USER\"" - fi -fi - -# Check if the RFS Command already contains a password; only do special work if it does not -if [[ $RFS_COMMAND != *"--target-password"* ]]; then - PASSWORD_TO_USE="" - - # Check if the password is available in plaintext; if, use it. Otherwise, retrieve it from AWS Secrets Manager - if [[ -n "$RFS_TARGET_PASSWORD" ]]; then - echo "Using plaintext password from ENV variable RFS_TARGET_PASSWORD" - PASSWORD_TO_USE="$RFS_TARGET_PASSWORD" - elif [[ -n "$RFS_TARGET_PASSWORD_ARN" ]]; then - # Retrieve password from AWS Secrets Manager if ARN is provided - echo "Using password from AWS Secrets Manager ARN in ENV variable RFS_TARGET_PASSWORD_ARN" - PASSWORD_TO_USE=$(aws secretsmanager get-secret-value --secret-id "$RFS_TARGET_PASSWORD_ARN" --query SecretString --output text) - fi - - # Append the username/password to the RFS Command if have an updated password - if [[ -n "$PASSWORD_TO_USE" ]]; then - echo "Updating RFS Command with password." - RFS_COMMAND="$RFS_COMMAND --target-password \"$PASSWORD_TO_USE\"" - fi -fi - -# Extract the value passed after --s3-local-dir -S3_LOCAL_DIR=$(echo "$RFS_COMMAND" | sed -n 's/.*--s3-local-dir\s\+\("[^"]\+"\|[^ ]\+\).*/\1/p' | tr -d '"') -# Extract the value passed after --lucene-dir -LUCENE_DIR=$(echo "$RFS_COMMAND" | sed -n 's/.*--lucene-dir\s\+\("[^"]\+"\|[^ ]\+\).*/\1/p' | tr -d '"') -if [[ -n "$S3_LOCAL_DIR" ]]; then - echo "Will delete S3 local directory between runs: $S3_LOCAL_DIR" -else - echo "--s3-local-dir argument not found in RFS_COMMAND. Will not delete S3 local directory between runs." -fi - -if [[ -n "$LUCENE_DIR" ]]; then - echo "Will delete lucene local directory between runs: $LUCENE_DIR" -else - echo "--lucene-dir argument not found in RFS_COMMAND. This is required." - exit 1 -fi - -cleanup_directories() { - if [[ -n "$S3_LOCAL_DIR" ]]; then - echo "Cleaning up S3 local directory: $S3_LOCAL_DIR" - rm -rf "$S3_LOCAL_DIR" - echo "Directory $S3_LOCAL_DIR has been cleaned up." - fi - - if [[ -n "$LUCENE_DIR" ]]; then - echo "Cleaning up Lucene local directory: $LUCENE_DIR" - rm -rf "$LUCENE_DIR" - echo "Directory $LUCENE_DIR has been cleaned up." - fi -} - - +# Discussion needed: Container will have a minimum number of +# parameters after on start services.yaml is loaded. +# +# This script was parsing the parameters that were feed into the +# container, unless this script is pulling parameters from the +# services.yaml its going to be out of sync. +# +# Metadata & Snapshot will already need to know how to read the +# Secrets from ARNs so this seems like it aligns well to do this +# at the same time. + +# Discussion needed: Directly cleanup was being done by script based +# on arguments +# +# Alternative, java can clean these directories on start since this script +# won't be able to resolve the path(s) without parsing the services.yaml [ -z "$RFS_COMMAND" ] && \ { echo "Warning: RFS_COMMAND is empty! Exiting."; exit 1; } || \ @@ -82,6 +26,5 @@ until ! { echo "Running command $RFS_COMMAND" eval "$RFS_COMMAND" }; do - echo "Cleaning up directories before the next run." - cleanup_directories + echo "About to start the next run." done diff --git a/DocumentsFromSnapshotMigration/src/main/java/org/opensearch/migrations/RfsMigrateDocuments.java b/DocumentsFromSnapshotMigration/src/main/java/org/opensearch/migrations/RfsMigrateDocuments.java index 592c09dd1..ec31fae28 100644 --- a/DocumentsFromSnapshotMigration/src/main/java/org/opensearch/migrations/RfsMigrateDocuments.java +++ b/DocumentsFromSnapshotMigration/src/main/java/org/opensearch/migrations/RfsMigrateDocuments.java @@ -58,6 +58,8 @@ public Duration convert(String value) { } public static class Args { + + @Parameter(names = {"--help", "-h"}, help = true, description = "Displays information about how to use this tool") private boolean help; diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataArgs.java index 9b05d988e..5a3620ef1 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataArgs.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataArgs.java @@ -9,4 +9,7 @@ public class MetadataArgs { @Parameter(names = { "--otel-collector-endpoint" }, description = "Endpoint (host:port) for the OpenTelemetry Collector to which metrics logs should be" + "forwarded. If no value is provided, metrics will not be forwarded.") public String otelCollectorEndpoint; + + @Parameter(names = { "--config-file", "-c" }, description = "The path to a config file") + public String configFile; } diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java index 94ac6f7a7..708855773 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java @@ -14,6 +14,7 @@ import org.opensearch.migrations.tracing.CompositeContextTracker; import org.opensearch.migrations.tracing.RootOtelContext; import org.opensearch.migrations.utils.ProcessHelpers; +import org.opensearch.migrations.config.MigrationConfig; import com.beust.jcommander.JCommander; import lombok.extern.slf4j.Slf4j; @@ -22,15 +23,62 @@ public class MetadataMigration { public static void main(String[] args) throws Exception { + var metadataArgs = new MetadataArgs(); var migrateArgs = new MigrateArgs(); + // Note; need to handle these effectively duplicated parsed args var evaluateArgs = new EvaluateArgs(); - var jCommander = JCommander.newBuilder() + + // Load from the command line first + JCommander.newBuilder() .addObject(metadataArgs) .addCommand(migrateArgs) .addCommand(evaluateArgs) - .build(); - jCommander.parse(args); + .build() + .parse(args); + + // Then override with settings + var config = getConfig(args); + if (config != null) { + metadataArgs.otelCollectorEndpoint = config.metadata_migration.otel_endpoint; + + // Note; we've got some serious null ref risk in this block of code, will need to use a lot of optionals. + migrateArgs.dataFilterArgs.indexAllowlist = config.metadata_migration.index_allowlist; + migrateArgs.dataFilterArgs.indexTemplateAllowlist = config.metadata_migration.index_template_allowlist; + migrateArgs.dataFilterArgs.componentTemplateAllowlist = config.metadata_migration.component_template_allowlist; + + migrateArgs.fileSystemRepoPath = config.snapshot.fs.repo_path; + migrateArgs.snapshotName = config.snapshot.snapshot_name; + migrateArgs.s3LocalDirPath = config.metadata_migration.local_dir; + migrateArgs.s3Region = config.snapshot.s3.aws_region; + migrateArgs.s3RepoUri = config.snapshot.s3.repo_uri; + + migrateArgs.sourceArgs.host = config.source_cluster.endpoint; + migrateArgs.sourceArgs.username = config.source_cluster.basic_auth.username; + migrateArgs.sourceArgs.password = config.source_cluster.basic_auth.password; + migrateArgs.sourceArgs.awsRegion = config.source_cluster.sigv4.region; + migrateArgs.sourceArgs.awsServiceSigningName = config.source_cluster.sigv4.service; + migrateArgs.sourceArgs.insecure = config.source_cluster.allow_insecure; + + // Need to special case indirect values such as AWS Secrets + if (config.source_cluster.basic_auth.password_from_secret_arn != null) { + migrateArgs.sourceArgs.password = ""; // Load this from AWS and insert into this arg + log a message + } + + migrateArgs.targetArgs.host = config.target_cluster.endpoint; + migrateArgs.targetArgs.username = config.target_cluster.basic_auth.username; + migrateArgs.targetArgs.password = config.target_cluster.basic_auth.password; + migrateArgs.targetArgs.awsRegion = config.target_cluster.sigv4.region; + migrateArgs.targetArgs.awsServiceSigningName = config.target_cluster.sigv4.service; + migrateArgs.targetArgs.insecure = config.target_cluster.allow_insecure; + + // Need to special case indirect values such as AWS Secrets + if (config.target_cluster.basic_auth.password != null) { + migrateArgs.targetArgs.password = ""; // Load this from AWS and insert into this arg + log a message + } + + migrateArgs.minNumberOfReplicas = config.metadata_migration.min_replicas; + } var context = new RootMetadataMigrationContext( RootOtelContext.initializeOpenTelemetryWithCollectorOrAsNoop(metadataArgs.otelCollectorEndpoint, "metadata", @@ -115,4 +163,24 @@ private static void printCommandUsage(JCommander jCommander) { jCommander.getUsageFormatter().usage(jCommander.getParsedCommand(), sb); log.info(sb.toString()); } + + private static MigrationConfig getConfig(String[] args) { + var metadataArgs = new MetadataArgs(); + + JCommander.newBuilder() + .addObject(metadataArgs) + .acceptUnknownOptions(true) + .build() + .parse(args); + + if (metadataArgs.configFile != null) { + try { + return MigrationConfig.loadFrom(metadataArgs.configFile); + } catch (Exception e) { + log.warn("Unable to load from config file, falling back to command line arguments."); + } + } + return null; + + } } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 3a3212b83..5ec47cd88 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -97,7 +97,7 @@ services: - migrations volumes: - sharedLogsVolume:/shared-logs-output - - ./migrationConsole/lib/console_link/services.yaml:/etc/migration_services.yaml + - ./migrationConsole/lib/console_link/services.yaml:/shared-logs-output/migration_services.yaml # this is a convenience thing for testing -- it should be removed before this makes it to prod. # - ./migrationConsole/lib/console_link:/root/lib/console_link - ~/.aws:/root/.aws diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/docker-compose-console-only.yml b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/docker-compose-console-only.yml index a68b9ae22..24b3e32a3 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/docker-compose-console-only.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/docker-compose-console-only.yml @@ -11,7 +11,8 @@ services: networks: - migrations volumes: - - ./lib/console_link/services.yaml:/etc/migration_services.yaml + - sharedLogsVolume:/shared-logs-output + - ./lib/console_link/services.yaml:/shared-logs-output/migration_services.yaml # this is a convenience thing for testing -- it should be removed before this makes it to prod. - ./lib:/root/lib - ~/.aws:/root/.aws @@ -26,6 +27,10 @@ services: - "8000:8000" command: pipenv run python /root/console_api/manage.py runserver_plus 0.0.0.0:8000 +volumes: + sharedLogsVolume: + driver: local + networks: migrations: driver: bridge diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md index d91cefb4f..a66e5f528 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md @@ -26,7 +26,7 @@ The console link library is designed to provide a unified interface for the many ![Console_link Library Diagram](console_library_diagram.svg) -The user defines their migration services in a `migration_services.yaml` file, by default found at `/etc/migration_services.yaml`. +The user defines their migration services in a `migration_services.yaml` file, by default found at `/shared-logs-output/migration_services.yaml`. Currently, the supported services are: @@ -46,12 +46,14 @@ source_cluster: endpoint: "https://capture-proxy-es:9200" allow_insecure: true no_auth: + version: ES_7_10 target_cluster: endpoint: "https://opensearchtarget:9200" allow_insecure: true basic_auth: username: "admin" password: "myStrongPassword123!" + version: OS_2_15 metrics_source: prometheus: endpoint: "http://prometheus:9090" @@ -71,8 +73,8 @@ backfill: - "migration_deployment=1.0.6" replay: ecs: - cluster-name: "migrations-dev-cluster" - service-name: "migrations-dev-replayer-service" + cluster_name: "migrations-dev-cluster" + service_name: "migrations-dev-replayer-service" snapshot: snapshot_name: "snapshot_2023_01_01" s3: @@ -165,7 +167,7 @@ backfill: ecs: cluster_name: migration-aws-integ-ecs-cluster service_name: migration-aws-integ-reindex-from-snapshot - aws-region: us-east-1 + aws_region: us-east-1 ``` #### OpenSearch Ingestion @@ -262,7 +264,7 @@ The structure of cli commands is: The available global options are: -- `--config-file FILE` to specify the path to a config file (default is `/etc/migration_services.yaml`) +- `--config-file FILE` to specify the path to a config file (default is `/shared-logs-output/migration_services.yaml`) - `--json` to get output in JSON designed for machine consumption instead of printing to the console #### Objects diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py index d98412c06..bfbdc5459 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py @@ -37,7 +37,7 @@ def __init__(self, config_file) -> None: @click.group() @click.option( - "--config-file", default="/etc/migration_services.yaml", help="Path to config file" + "--config-file", default="/shared-logs-output/migration_services.yaml", help="Path to config file" ) @click.option("--json", is_flag=True) @click.option('-v', '--verbose', count=True, help="Verbosity level. Default is warn, -v is info, -vv is debug.") @@ -522,7 +522,7 @@ def describe_topic_records_cmd(ctx, topic_name): @cli.command() @click.option( - "--config-file", default="/etc/migration_services.yaml", help="Path to config file" + "--config-file", default="/shared-logs-output/migration_services.yaml", help="Path to config file" ) @click.option("--json", is_flag=True) @click.argument('shell', type=click.Choice(['bash', 'zsh', 'fish'])) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/environment.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/environment.py index a544c8a49..7b42a6520 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/environment.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/environment.py @@ -102,8 +102,7 @@ def __init__(self, config_file: str): logger.info("No snapshot provided") if 'metadata_migration' in self.config: self.metadata: Metadata = Metadata(self.config["metadata_migration"], - target_cluster=self.target_cluster, - snapshot=self.snapshot) + config_file) if 'kafka' in self.config: self.kafka: Kafka = get_kafka(self.config["kafka"]) logger.info(f"Kafka initialized: {self.kafka}") diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py index bf4993854..f76d5739e 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py @@ -13,47 +13,12 @@ logger = logging.getLogger(__name__) -FROM_SNAPSHOT_SCHEMA = { - "type": "dict", - # In the future, there should be a "from_snapshot" and a "from_live_cluster" option, but for now only snapshot is - # supported, so this is required. It _can_ be null, but that requires a snapshot to defined on its own in - # the services.yaml file. - "required": True, - "nullable": True, - "schema": { - "snapshot_name": {"type": "string", "required": True}, - "otel_endpoint": {"type": "string", "required": False}, - "local_dir": {"type": "string", "required": False}, - "s3": { - 'type': 'dict', - "required": False, - 'schema': { - 'repo_uri': {'type': 'string', 'required': True}, - 'aws_region': {'type': 'string', 'required': True}, - } - }, - "fs": { - 'type': 'dict', - "required": False, - 'schema': { - 'repo_path': {'type': 'string', 'required': True}, - } - } - }, - # We _should_ have the check below, but I need to figure out how to combine it with a potentially - # nullable block (like this one) - # 'check_with': contains_one_of({'s3', 'fs'}) - -} - SCHEMA = { - "from_snapshot": FROM_SNAPSHOT_SCHEMA, "otel_endpoint": {"type": "string", "required": False}, "min_replicas": {"type": "integer", "min": 0, "required": False}, "index_allowlist": list_schema(required=False), "index_template_allowlist": list_schema(required=False), - "component_template_allowlist": list_schema(required=False), - "source_cluster_version": {"type": "string", "required": False} + "component_template_allowlist": list_schema(required=False) } @@ -62,108 +27,23 @@ def generate_tmp_dir(name: str) -> str: class Metadata: - def __init__(self, config, target_cluster: Cluster, snapshot: Optional[Snapshot] = None): + def __init__(self, config, config_file: str): logger.debug(f"Initializing Metadata with config: {config}") v = Validator(SCHEMA) if not v.validate(config): logger.error(f"Invalid config: {v.errors}") raise ValueError(v.errors) self._config = config - self._target_cluster = target_cluster - self._snapshot = snapshot - - if (not snapshot) and (config["from_snapshot"] is None): - raise ValueError("No snapshot is specified or can be assumed " - "for the metadata migration to use.") - - self._min_replicas = config.get("min_replicas", 0) - self._index_allowlist = config.get("index_allowlist", None) - self._index_template_allowlist = config.get("index_template_allowlist", None) - self._component_template_allowlist = config.get("component_template_allowlist", None) - self._otel_endpoint = config.get("otel_endpoint", None) - self._source_cluster_version = config.get("source_cluster_version", None) - - logger.debug(f"Min replicas: {self._min_replicas}") - logger.debug(f"Index allowlist: {self._index_allowlist}") - logger.debug(f"Index template allowlist: {self._index_template_allowlist}") - logger.debug(f"Component template allowlist: {self._component_template_allowlist}") - logger.debug(f"Otel endpoint: {self._otel_endpoint}") - - # If `from_snapshot` is fully specified, use those values to define snapshot params - if config["from_snapshot"] is not None: - logger.debug("Using fully specified snapshot config") - self._init_from_config() - else: - logger.debug("Using independently specified snapshot") - if isinstance(snapshot, S3Snapshot): - self._init_from_s3_snapshot(snapshot) - elif isinstance(snapshot, FileSystemSnapshot): - self._init_from_fs_snapshot(snapshot) - - if config["from_snapshot"] is not None and "local_dir" in config["from_snapshot"]: - self._local_dir = config["from_snapshot"]["local_dir"] - else: - self._local_dir = generate_tmp_dir(self._snapshot_name) - - logger.debug(f"Snapshot name: {self._snapshot_name}") - if self._snapshot_location == 's3': - logger.debug(f"S3 URI: {self._s3_uri}") - logger.debug(f"AWS region: {self._aws_region}") - else: - logger.debug(f"Local dir: {self._local_dir}") + self._config_file = config_file logger.info("Metadata migration configuration defined") - def _init_from_config(self) -> None: - config = self._config - self._snapshot_location = 's3' if 's3' in config["from_snapshot"] else 'fs' - self._snapshot_name = config["from_snapshot"]["snapshot_name"] - - if self._snapshot_location == 'fs': - self._repo_path = config["from_snapshot"]["fs"]["repo_path"] - else: - self._s3_uri = config["from_snapshot"]["s3"]["repo_uri"] - self._aws_region = config["from_snapshot"]["s3"]["aws_region"] - - def _init_from_s3_snapshot(self, snapshot: S3Snapshot) -> None: - self._snapshot_name = snapshot.snapshot_name - self._snapshot_location = "s3" - self._s3_uri = snapshot.s3_repo_uri - self._aws_region = snapshot.s3_region - - def _init_from_fs_snapshot(self, snapshot: FileSystemSnapshot) -> None: - self._snapshot_name = snapshot.snapshot_name - self._snapshot_location = "fs" - self._repo_path = snapshot.repo_path - - def _append_args(self, commands: Dict[str, Any], args_to_add: List[str]) -> None: - if args_to_add is None: - return - - def is_command(arg: Optional[str]) -> bool: - if arg is None: - return False - return arg.startswith('--') or arg.startswith('-') - - def is_value(arg: Optional[str]) -> bool: - if arg is None: - return False - return not is_command(arg) - - i = 0 - while i < len(args_to_add): - arg = args_to_add[i] - next_arg = args_to_add[i + 1] if (i + 1 < len(args_to_add)) else None - - if is_command(arg) and is_value(next_arg): - commands[arg] = next_arg - i += 2 # Move past the command and value - elif is_command(arg): - commands[arg] = None - i += 1 # Move past the command, its a flag - else: - logger.warning(f"Ignoring extra value {arg}, there was no command name before it") - i += 1 + # Discussion Needed: Verification of the config should be done by + # console_link or by running metadata tool? + # + # Tools need to handle errors such as access denied or malformed + # parameters, the validation being done here could be pushed down + # to those tools that already need to handle those scenarios. def evaluate(self, extra_args=None) -> CommandResult: logger.info("Starting metadata migration") @@ -177,65 +57,11 @@ def migrate_or_evaluate(self, command: str, extra_args=None) -> CommandResult: command_base = "/root/metadataMigration/bin/MetadataMigration" command_args = {} - # Add any common metadata parameter before the command - if self._otel_endpoint: - command_args.update({"--otel-collector-endpoint": self._otel_endpoint}) - command_args.update({ command: None, - "--snapshot-name": self._snapshot_name, - "--target-host": self._target_cluster.endpoint, - "--min-replicas": self._min_replicas + "--config-file": self._config_file }) - if self._snapshot_location == 's3': - command_args.update({ - "--s3-local-dir": self._local_dir, - "--s3-repo-uri": self._s3_uri, - "--s3-region": self._aws_region, - }) - elif self._snapshot_location == 'fs': - command_args.update({ - "--file-system-repo-path": self._repo_path, - }) - - if self._target_cluster.auth_type == AuthMethod.BASIC_AUTH: - try: - command_args.update({ - "--target-username": self._target_cluster.auth_details.get("username"), - "--target-password": self._target_cluster.get_basic_auth_password() - }) - logger.info("Using basic auth for target cluster") - except KeyError as e: - raise ValueError(f"Missing required auth details for target cluster: {e}") - elif self._target_cluster.auth_type == AuthMethod.SIGV4: - signing_name, region = self._target_cluster._get_sigv4_details(force_region=True) - logger.info(f"Using sigv4 auth for target cluster with signing_name {signing_name} and region {region}") - command_args.update({ - "--target-aws-service-signing-name": signing_name, - "--target-aws-region": region - }) - - if self._target_cluster.allow_insecure: - command_args.update({"--target-insecure": FlagOnlyArgument}) - - if self._index_allowlist: - command_args.update({"--index-allowlist": ",".join(self._index_allowlist)}) - - if self._index_template_allowlist: - command_args.update({"--index-template-allowlist": ",".join(self._index_template_allowlist)}) - - if self._component_template_allowlist: - command_args.update({"--component-template-allowlist": ",".join(self._component_template_allowlist)}) - - if self._source_cluster_version: - command_args.update({"--source-version": self._source_cluster_version}) - - # Extra args might not be represented with dictionary, so convert args to list and append commands - self._append_args(command_args, extra_args) - - command_runner = CommandRunner(command_base, command_args, - sensitive_fields=["--target-password"]) logger.info(f"Migrating metadata with command: {' '.join(command_runner.sanitized_command())}") try: return command_runner.run() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py index d6c967dd9..3ad550df9 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py @@ -17,7 +17,6 @@ 'type': 'dict', 'schema': { 'snapshot_name': {'type': 'string', 'required': True}, - 'otel_endpoint': {'type': 'string', 'required': False}, 's3': { 'type': 'dict', 'schema': { diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/README.md b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/README.md index e6b2c8b37..35ef747e0 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/README.md +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/README.md @@ -34,7 +34,7 @@ This script accepts various parameters to customize its behavior. Below is a lis - `--unique_id`: The unique identifier to apply to created indices/documents. - Default: Generated uuid - `--config_file_path`: The services yaml config file path for the console library. - - Default: `/etc/migration_services.yaml` + - Default: `/shared-logs-output/migration_services.yaml` #### Clean Up diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 3efb7fbe2..946d480b2 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -16,7 +16,7 @@ def pytest_configure(config): def pytest_addoption(parser): parser.addoption("--unique_id", action="store", default=uuid.uuid4().hex) - parser.addoption("--config_file_path", action="store", default="/etc/migration_services.yaml", + parser.addoption("--config_file_path", action="store", default="/shared-logs-output/migration_services.yaml", help="Path to config file for console library") diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/loadServicesFromParameterStore.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/loadServicesFromParameterStore.sh index ae3ec5a31..46d6943ac 100755 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/loadServicesFromParameterStore.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/loadServicesFromParameterStore.sh @@ -21,7 +21,7 @@ if [ $? -ne 0 ]; then fi # Define the output file path -OUTPUT_FILE="/etc/migration_services.yaml" +OUTPUT_FILE="/shared-logs-output/migration_services.yaml" # Write the parameter value to the file echo "$PARAMETER_VALUE" > "$OUTPUT_FILE" diff --git a/commonDependencyVersionConstraints/build.gradle b/commonDependencyVersionConstraints/build.gradle index f9e586f74..9f47a7b4c 100644 --- a/commonDependencyVersionConstraints/build.gradle +++ b/commonDependencyVersionConstraints/build.gradle @@ -119,6 +119,8 @@ dependencies { api group: 'org.semver4j', name: 'semver4j', version: '5.3.0' + api group: 'org.yaml', name: 'snakeyaml', version: '2.0' + // ************************************************************ // The following constraints are for mitigating transitive CVEs // ************************************************************ diff --git a/coreUtilities/build.gradle b/coreUtilities/build.gradle index 8494c3f30..714cb2809 100644 --- a/coreUtilities/build.gradle +++ b/coreUtilities/build.gradle @@ -41,6 +41,7 @@ dependencies { // JCommander compileOnly group: 'org.jcommander', name: 'jcommander' + implementation group: 'org.yaml', name: 'snakeyaml' // OpenTelemetry core api group: 'io.opentelemetry', name: 'opentelemetry-api' diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Backfill.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Backfill.java new file mode 100644 index 000000000..449a1ec3e --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Backfill.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class Backfill { + // Ignoring since OSI config isn't checked by java based tools + public Object opensearch_ingestion; + public ReindexFromSnapshot reindex_from_snapshot; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/BasicAuth.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/BasicAuth.java new file mode 100644 index 000000000..937b48e4c --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/BasicAuth.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class BasicAuth { + public String username; + public String password; + public String password_from_secret_arn; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Cluster.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Cluster.java new file mode 100644 index 000000000..c52798035 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Cluster.java @@ -0,0 +1,10 @@ +package org.opensearch.migrations.config; + +public class Cluster { + public String endpoint; + public boolean allow_insecure; + public String version; + public NoAuth no_auth; + public BasicAuth basic_auth; + public Sigv4Auth sigv4; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/ElasticContainerService.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/ElasticContainerService.java new file mode 100644 index 000000000..615c3badc --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/ElasticContainerService.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class ElasticContainerService { + public String cluster_name; + public String service_name; + public String aws_region; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/FileSystem.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/FileSystem.java new file mode 100644 index 000000000..31823db28 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/FileSystem.java @@ -0,0 +1,5 @@ +package org.opensearch.migrations.config; + +public class FileSystem { + public String repo_path; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Kafka.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Kafka.java new file mode 100644 index 000000000..d1297249e --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Kafka.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class Kafka { + public String broker_endpoints; + public Object msk; + public Object standard; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/MetadataMigration.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/MetadataMigration.java new file mode 100644 index 000000000..cbbc2d89f --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/MetadataMigration.java @@ -0,0 +1,13 @@ +package org.opensearch.migrations.config; + +import java.util.List; + +public class MetadataMigration { + public String otel_endpoint; + public String local_dir; + public int min_replicas; + + public List index_allowlist; + public List index_template_allowlist; + public List component_template_allowlist; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/MetricsSource.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/MetricsSource.java new file mode 100644 index 000000000..8b1b7c3d9 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/MetricsSource.java @@ -0,0 +1,5 @@ +package org.opensearch.migrations.config; + +public class MetricsSource { + +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/MigrationConfig.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/MigrationConfig.java new file mode 100644 index 000000000..37600b4d2 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/MigrationConfig.java @@ -0,0 +1,28 @@ +package org.opensearch.migrations.config; + +import java.io.FileInputStream; +import java.io.IOException; + +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; + +public class MigrationConfig { + public Cluster source_cluster; + public Cluster target_cluster; + public Object metrics_source; + public Backfill backfill; + public Replay replay; + public Snapshot snapshot; + public SnapshotCreator snapshotCreator; + public MetadataMigration metadata_migration; + public Kafka kafka; + public Object client_options; + + public static MigrationConfig loadFrom(String path) throws IOException { + var yaml = new Yaml(new Constructor(MigrationConfig.class, new LoaderOptions())); + try (var inputStream = new FileInputStream(path)) { + return yaml.load(inputStream); + } + } +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/NoAuth.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/NoAuth.java new file mode 100644 index 000000000..288bf3f9e --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/NoAuth.java @@ -0,0 +1,5 @@ +package org.opensearch.migrations.config; + +public class NoAuth { + +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/ReindexFromSnapshot.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/ReindexFromSnapshot.java new file mode 100644 index 000000000..4fd38365c --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/ReindexFromSnapshot.java @@ -0,0 +1,10 @@ +package org.opensearch.migrations.config; + +public class ReindexFromSnapshot { + public Object docker; + public ElasticContainerService ecs; + public String snapshot_name; + public String snapshot_repo; + public String local_dir; + public int scale; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Replay.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Replay.java new file mode 100644 index 000000000..968b0b9cd --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Replay.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class Replay { + public Object docker; + public ElasticContainerService ecs; + public int scale; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/S3Bucket.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/S3Bucket.java new file mode 100644 index 000000000..c226364d7 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/S3Bucket.java @@ -0,0 +1,6 @@ +package org.opensearch.migrations.config; + +public class S3Bucket { + public String repo_uri; + public String aws_region; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Sigv4Auth.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Sigv4Auth.java new file mode 100644 index 000000000..77f3d5565 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Sigv4Auth.java @@ -0,0 +1,6 @@ +package org.opensearch.migrations.config; + +public class Sigv4Auth { + public String region; + public String service; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/Snapshot.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/Snapshot.java new file mode 100644 index 000000000..72e0adefc --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/Snapshot.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.config; + +public class Snapshot { + public String snapshot_name; + public S3Bucket s3; + public FileSystem fs; +} diff --git a/coreUtilities/src/main/java/org/opensearch/migrations/config/SnapshotCreator.java b/coreUtilities/src/main/java/org/opensearch/migrations/config/SnapshotCreator.java new file mode 100644 index 000000000..b3311b648 --- /dev/null +++ b/coreUtilities/src/main/java/org/opensearch/migrations/config/SnapshotCreator.java @@ -0,0 +1,5 @@ +package org.opensearch.migrations.config; + +public class SnapshotCreator { + public String otel_endpoint; +} diff --git a/coreUtilities/src/test/java/org/opensearch/migrations/config/MigrationConfigTest.java b/coreUtilities/src/test/java/org/opensearch/migrations/config/MigrationConfigTest.java new file mode 100644 index 000000000..0f635246b --- /dev/null +++ b/coreUtilities/src/test/java/org/opensearch/migrations/config/MigrationConfigTest.java @@ -0,0 +1,61 @@ +package org.opensearch.migrations.config; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class MigrationConfigTest { + + @Test + void testLoad(@TempDir Path dir) throws IOException { + var file = dir.resolve("testFile.yml"); + + var contents = "source_cluster:\r\n" + // + " endpoint: \"https://capture-proxy-es:9200\"\r\n" + // + " allow_insecure: true\r\n" + // + " no_auth:\r\n" + // + "target_cluster:\r\n" + // + " endpoint: \"https://opensearchtarget:9200\"\r\n" + // + " allow_insecure: true\r\n" + // + " basic_auth:\r\n" + // + " username: \"admin\"\r\n" + // + " password: \"myStrongPassword123!\"\r\n" + // + "metrics_source:\r\n" + // + " prometheus:\r\n" + // + " endpoint: \"http://prometheus:9090\"\r\n" + // + "backfill:\r\n" + // + " reindex_from_snapshot:\r\n" + // + " snapshot_repo: \"abc\"\r\n" + // + " snapshot_name: \"def\"\r\n" + // + " scale: 3\r\n" + // + " ecs:\r\n" + // + " cluster_name: migration-aws-integ-ecs-cluster\r\n" + // + " service_name: migration-aws-integ-reindex-from-snapshot\r\n" + // + " aws_region: us-east-1\r\n" + // + "replay:\r\n" + // + " ecs:\r\n" + // + " cluster_name: \"migrations-dev-cluster\"\r\n" + // + " service_name: \"migrations-dev-replayer-service\"\r\n" + // + "snapshot:\r\n" + // + " snapshot_name: \"snapshot_2023_01_01\"\r\n" + // + " s3:\r\n" + // + " repo_uri: \"s3://my-snapshot-bucket\"\r\n" + // + " aws_region: \"us-east-2\"\r\n" + // + "metadata_migration:\r\n" + // + " min_replicas: 0\r\n" + // + "kafka:\r\n" + // + " broker_endpoints: \"kafka:9092\"\r\n" + // + " standard:\r\n" + // + "client_options:\r\n" + // + " user_agent_extra: \"test-user-agent-v1.0\""; + + Files.writeString(file, contents); + + var config = MigrationConfig.loadFrom(file.toString()); + + System.out.println(config); + } +} diff --git a/docs/ControlPlaneControls.md b/docs/ControlPlaneControls.md new file mode 100644 index 000000000..a06c87098 --- /dev/null +++ b/docs/ControlPlaneControls.md @@ -0,0 +1,78 @@ +# Controlling the Migration Assistant Control Plane + +Migration Assistant (MA) has several distinct workflows that affect how the different parts of MA work together. Ideally by editing a value once, it will be consistent in all tools as users would expect. + +## Customer Scenarios + +* Customer needs to swap out credentials for target cluster +* Customer refines the list of indices they want to migrate +* Customer needs to switch to a different target cluster +* Customer switches to a self managed Kafka instance. + +### Why are these scenario difficult today? + +* ECS task based tools are only modifiable by CDK deployment +* MA Console based tools allow overrides, but these are not savable +* It is not obvious what settings need to be treated differently by the CX +* Deployment takes time, pushing customer out of iterative workflow + +### Recommendation: All tools read from a single file +Within the MA Console the services.yaml contains nearly all of the details that are of interest to the different tools. + +1. Having all tools adopt `check services.yaml on startup` will ensure changes are propagated and reduce the amount of extra system consideration that is needed. +2. Move CloudFormation focus to deployment infrastructure, remove all parameters that are in `services.yaml`. +3. Customers need a way to make updates to `services.yaml` that with a degree of guardrails, if they are updating the target cluster and backfill/replayer is running they should be able to be warned. + +#### Desired State +```mermaid +graph TD + cx[Customer] --> cfn[Cloud Formation] + + subgraph bootstrap[MA EC2 Bootstrap] + ccj[cdk.context.json] + end + + subgraph efs[File Storage] + sy[Service.yaml] + end + + cx --> | config edit cluster source/target| cli + ccj --> |Cdk Deploy|cfn + + cfn --> | Create on deploy | sy + + subgraph console[MA Console] + cli[CLI] + snap[snapshot creation] + meta[metadata migration] + end + + cx --> | config edit --index-allow-list | cli + cx --> | config edit --index-template-allow-list | cli + cli --> | update config | sy + + subgraph esc[Elastic Container Service] + replayer[Replayer] + rfs[Rfs Backfill] + end + + rfs --> | load service.yaml | sy + replayer --> | load service.yaml | sy + console --> | load service.yaml | sy +``` + +### Read from AWS Parameter store +Parameter store is a centralized system within AWS and it build for parameter management. It is not recommended as it creates vendor lock-in on AWS. + +The same is true of any proprietary system. + +## Appendix + +#### Scenarios out of scope. + +* Customer wants to fully redeploy MA from the Console + - _Parts of MA might be updated/modified by the console, but there will be a set of core infrastructure that cannot be modified by the console_ +* Customer wants to change the VPC of MA + - _Requires a redeployment of the MA stack(s)_ +* Customer wants to make a permission change + - _IAM roles can be updated outside of MA in AWS Console/CFN_ From c2920ffe2de74baac8cf09fa7bb3181066a3f638 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 15 Oct 2024 20:13:22 +0000 Subject: [PATCH 2/3] Add graph showing disconnect in extra cli args scenario Signed-off-by: Peter Nied --- docs/ControlPlaneControls.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/ControlPlaneControls.md b/docs/ControlPlaneControls.md index a06c87098..b294b5492 100644 --- a/docs/ControlPlaneControls.md +++ b/docs/ControlPlaneControls.md @@ -16,6 +16,28 @@ Migration Assistant (MA) has several distinct workflows that affect how the diff * It is not obvious what settings need to be treated differently by the CX * Deployment takes time, pushing customer out of iterative workflow + +#### Update parameter in cdk.context.json, then deploy +```mermaid +graph TD + cx[Customer] --> | A1. Edit Parameters | cfn[Cloud Formation] + cfn --> | A2. Update Env Vars/CLI from parameters | ecs[Elastic Compute Service] + cfn --> | A3. Update SSM Document| ssm[System Manager Parameters] + cfn --> | A4. Redeploy Console| ma + ma[Migration Console] --> | A5. Refresh services.yml | ssm +``` + +#### In console pass extra command line args +```mermaid +graph TD + cx[Customer] --> | B1. Pass extra args | ma[Migration Console] + ma --> | B2. Directly called tool accepts extra args | sc[Snapshot Create] + ma --x | BX. Cannot update | cfn[Cloud Formation] + ma --x | BX. Cannot update | ecs[Elastic Compute Service] + ma --x | BX. Cannot update | ssm[System Manager Parameters] +``` + + ### Recommendation: All tools read from a single file Within the MA Console the services.yaml contains nearly all of the details that are of interest to the different tools. From 8e06740caf1ea57e52c6bbbad2eed7d3447616a0 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 16 Oct 2024 21:07:15 +0000 Subject: [PATCH 3/3] Small notes around when parameters are read Signed-off-by: Peter Nied --- docs/ControlPlaneControls.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/ControlPlaneControls.md b/docs/ControlPlaneControls.md index b294b5492..34bbec901 100644 --- a/docs/ControlPlaneControls.md +++ b/docs/ControlPlaneControls.md @@ -83,6 +83,16 @@ graph TD console --> | load service.yaml | sy ``` +#### Service Upload Flow + +For the services that are short lived, such as create snapshot or curl. The configuration would be loaded during application startup and never re-checked. + +> [!Note] +> There are services that are long running, this creates some user experience questions. +> * For RFS config changes can alter the shape of the migration which is defined once, should that run to completion or be stopped? +> * For Replayer the transformation would be passed via this mechanism, should they be reloaded as they come in? + + ### Read from AWS Parameter store Parameter store is a centralized system within AWS and it build for parameter management. It is not recommended as it creates vendor lock-in on AWS.