-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat!: added merge command and changed previous root command to impor… (#49) * feat!: added merge command and changed previous root command to import sub-command * docs: updated merge command * fix: use structured, configurably-named checkpoint file * feat: limit individual bundle size * test: more logging and fixed tests * fix: added SleepAfterImport to possibly address MERGE conflicts * chore(release): 2.0.0 ## [2.0.0](v1.3.2...v2.0.0) (2024-05-28) ### ⚠ BREAKING CHANGES * added merge command and changed previous root command to impor… (#49) ### Features * added merge command and changed previous root command to impor… ([#49](#49)) ([3c8fcbc](3c8fcbc)) * limit individual bundle size ([4ea3f83](4ea3f83)) ### Bug Fixes * added SleepAfterImport to possibly address MERGE conflicts ([e16fd4a](e16fd4a)) * use structured, configurably-named checkpoint file ([650868f](650868f)) ### Miscellaneous Chores * **deps:** update all non-major dependencies ([#46](#46)) ([4e3b2e0](4e3b2e0)) * **deps:** update mcr.microsoft.com/dotnet/sdk:8.0.204-jammy docker digest to 803a3c5 ([#39](#39)) ([bb6c947](bb6c947)) * **deps:** update miracum/.github action to v1.8.3 ([#47](#47)) ([7557d5d](7557d5d)) --------- Co-authored-by: semantic-release-bot <[email protected]>
- Loading branch information
1 parent
7557d5d
commit 269fa8b
Showing
13 changed files
with
50,481 additions
and
138 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"CreatedAt":"2024-05-27T20:29:16.4983552+00:00","LastImportedObjectUrl":"s3://fhir/staging-with-checkpoint/Patient/bundle-1708690114016.ndjson"} |
Large diffs are not rendered by default.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| using System.Reflection; | ||
| using System.Text.RegularExpressions; | ||
| using DotMake.CommandLine; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Logging.Console; | ||
| using Polly; | ||
| using Polly.Retry; | ||
|
|
||
| namespace PathlingS3Import; | ||
|
|
||
| public abstract partial class CommandBase | ||
| { | ||
| [GeneratedRegex(".*bundle-(?<timestamp>\\d*)\\.ndjson$")] | ||
| protected static partial Regex BundleObjectNameRegex(); | ||
|
|
||
| [CliOption(Description = "The S3 endpoint URI", Name = "--s3-endpoint")] | ||
| public Uri? S3Endpoint { get; set; } | ||
|
|
||
| [CliOption(Description = "The S3 access key", Name = "--s3-access-key")] | ||
| public string? S3AccessKey { get; set; } | ||
|
|
||
| [CliOption(Description = "The S3 secret key", Name = "--s3-secret-key")] | ||
| public string? S3SecretKey { get; set; } | ||
|
|
||
| [CliOption( | ||
| Description = "The name of the bucket containing the resources to import", | ||
| Name = "--s3-bucket-name" | ||
| )] | ||
| public string? S3BucketName { get; set; } = "fhir"; | ||
|
|
||
| [CliOption( | ||
| Description = "The S3 object name prefix. Corresponds to kafka-fhir-to-server's `S3_OBJECT_NAME_PREFIX`", | ||
| Name = "--s3-object-name-prefix" | ||
| )] | ||
| public string? S3ObjectNamePrefix { get; set; } = ""; | ||
|
|
||
| [CliOption( | ||
| Description = "If enabled, list and read all objects but don't invoke the import operation or store the progress.", | ||
| Name = "--dry-run" | ||
| )] | ||
| public bool IsDryRun { get; set; } = false; | ||
|
|
||
| [CliOption( | ||
| Description = "If enabled, push metrics about the import to the specified Prometheus PushGateway.", | ||
| Name = "--enable-metrics" | ||
| )] | ||
| public bool IsMetricsEnabled { get; set; } = false; | ||
|
|
||
| [CliOption( | ||
| Description = "Endpoint URL for the Prometheus PushGateway.", | ||
| Name = "--pushgateway-endpoint", | ||
| Required = false | ||
| )] | ||
| public Uri? PushGatewayEndpoint { get; set; } | ||
|
|
||
| [CliOption( | ||
| Description = "Prometheus PushGateway job name.", | ||
| Name = "--pushgateway-job-name", | ||
| Required = false | ||
| )] | ||
| public string PushGatewayJobName { get; set; } = | ||
| Assembly.GetExecutingAssembly().GetName().Name!; | ||
|
|
||
| [CliOption( | ||
| Description = "Prometheus PushGateway job instance.", | ||
| Name = "--pushgateway-job-instance", | ||
| Required = false | ||
| )] | ||
| public string? PushGatewayJobInstance { get; set; } | ||
|
|
||
| [CliOption( | ||
| Description = "Value for the `Authorization` header", | ||
| Name = "--pushgateway-auth-header", | ||
| Required = false | ||
| )] | ||
| public string? PushGatewayAuthHeader { get; set; } | ||
|
|
||
| public ILoggerFactory LogFactory { get; set; } | ||
|
|
||
| public ResiliencePipeline RetryPipeline { get; set; } | ||
|
|
||
| protected CommandBase() | ||
| { | ||
| LogFactory = LoggerFactory.Create(builder => | ||
| builder.AddSimpleConsole(options => | ||
| { | ||
| options.IncludeScopes = true; | ||
| options.SingleLine = true; | ||
| options.ColorBehavior = LoggerColorBehavior.Disabled; | ||
| options.TimestampFormat = "yyyy-MM-dd HH:mm:ss "; | ||
| }) | ||
| ); | ||
|
|
||
| var log = LogFactory.CreateLogger<CommandBase>(); | ||
|
|
||
| var retryOptions = new RetryStrategyOptions | ||
| { | ||
| ShouldHandle = new PredicateBuilder().Handle<Exception>(), | ||
| BackoffType = DelayBackoffType.Exponential, | ||
| UseJitter = true, // Adds a random factor to the delay | ||
| MaxRetryAttempts = 10, | ||
| Delay = TimeSpan.FromSeconds(10), | ||
| OnRetry = args => | ||
| { | ||
| log.LogWarning( | ||
| args.Outcome.Exception, | ||
| "Retrying. Attempt: {AttemptNumber}. Duration: {Duration}.", | ||
| args.AttemptNumber, | ||
| args.Duration | ||
| ); | ||
| // Event handlers can be asynchronous; here, we return an empty ValueTask. | ||
| return default; | ||
| } | ||
| }; | ||
|
|
||
| RetryPipeline = new ResiliencePipelineBuilder() | ||
| .AddRetry(retryOptions) // Add retry using the default options | ||
| .Build(); // Builds the resilience pipeline | ||
| } | ||
| } |
Oops, something went wrong.