Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP Azure Backups #855

Closed
wants to merge 9 commits into from
Closed

WIP Azure Backups #855

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c9c06f9
Rework backup spec gen based on cloud provider
ianstanton Jun 26, 2024
93fd539
Add azure credentials to crd
ianstanton Jun 26, 2024
39222e2
Add azure credentials to backup spec gen
ianstanton Jun 26, 2024
f552c10
Use azure credentials if cloud provider is azure
ianstanton Jun 27, 2024
6b8afd7
Set values none in azure credentials
ianstanton Jun 27, 2024
2448a72
Use secret key value ref for azure creds
ianstanton Jun 27, 2024
85b1e62
Add azure credentials to restore
ianstanton Jun 27, 2024
24425bb
Merge branch 'main' of github.com:CoreDB-io/coredb into azure-backup
ianstanton Jun 27, 2024
e4d0be9
Add debugging
ianstanton Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions charts/tembo-operator/templates/crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1930,6 +1930,7 @@ spec:
retentionPolicy: '30'
schedule: 0 0 * * *
endpointURL: null
azureCredentials: null
s3Credentials:
inheritFromIAMRole: true
volumeSnapshot:
Expand All @@ -1939,6 +1940,71 @@ spec:

**Default**: disabled
properties:
azureCredentials:
description: The Azure Blob Storage credentials to use for backups
nullable: true
properties:
connectionString:
description: The connection string to be used
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
inheritFromAzureAD:
description: Use the Azure AD based authentication without providing explicitly the keys.
nullable: true
type: boolean
storageAccount:
description: The storage account where to upload data
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
storageKey:
description: The storage account key to be used in conjunction with the storage account name
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
storageSasToken:
description: A shared-access-signature to be used in conjunction with the storage account name
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
type: object
destinationPath:
default: s3://
description: The S3 bucket path to store backups in
Expand Down Expand Up @@ -2336,6 +2402,71 @@ spec:
**Default**: disabled
nullable: true
properties:
azureCredentials:
description: The Azure Blob Storage credentials to use for restores
nullable: true
properties:
connectionString:
description: The connection string to be used
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
inheritFromAzureAD:
description: Use the Azure AD based authentication without providing explicitly the keys.
nullable: true
type: boolean
storageAccount:
description: The storage account where to upload data
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
storageKey:
description: The storage account key to be used in conjunction with the storage account name
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
storageSasToken:
description: A shared-access-signature to be used in conjunction with the storage account name
nullable: true
properties:
key:
description: The key to select
type: string
name:
description: Name of the referent.
type: string
required:
- key
- name
type: object
type: object
backupsPath:
description: |-
The object storage path and bucket name of the instance you wish to restore from. This maps to the `Backup` `destinationPath` field for the original instance.
Expand Down
2 changes: 1 addition & 1 deletion conductor/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

94 changes: 63 additions & 31 deletions conductor/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ async fn run(metrics: CustomMetrics) -> Result<(), ConductorError> {
.unwrap_or_else(|_| "us-east-1".to_owned())
.parse()
.expect("error parsing AWS_REGION");
let cloud_provider: String = env::var("CLOUD_PROVIDER")
.unwrap_or_else(|_| "aws".to_owned())
.parse()
.expect("error parsing CLOUD_PROVIDER");
let azure_storage_account: String = env::var("AZURE_STORAGE_ACCOUNT")
.unwrap_or_else(|_| "".to_owned())
.parse()
.expect("error parsing AZURE_STORAGE_ACCOUNT");

// Connect to pgmq
let queue = PGMQueueExt::new(pg_conn_url.clone(), 5).await?;
Expand Down Expand Up @@ -230,6 +238,8 @@ async fn run(metrics: CustomMetrics) -> Result<(), ConductorError> {
&read_msg,
&mut coredb_spec,
is_cloud_formation,
cloud_provider.clone(),
azure_storage_account.clone(),
&client,
)
.await
Expand Down Expand Up @@ -670,36 +680,63 @@ async fn init_cloud_perms(
read_msg: &Message<CRUDevent>,
coredb_spec: &mut CoreDBSpec,
is_cloud_formation: bool,
cloud_provider: String,
azure_storage_account: String,
_client: &Client,
) -> Result<(), ConductorError> {
if !is_cloud_formation {
if !is_cloud_formation && cloud_provider != "azure" {
return Ok(());
}

create_cloudformation(
aws_region.clone(),
backup_archive_bucket.clone(),
read_msg.message.namespace.clone(),
read_msg.message.backups_read_path.clone(),
read_msg.message.backups_write_path.clone(),
cf_template_bucket,
)
.await?;

// Lookup the CloudFormation stack's role ARN
let role_arn = lookup_role_arn(aws_region, &read_msg.message.namespace).await?;

info!("{}: Adding backup configuration to spec", read_msg.msg_id);
// Format ServiceAccountTemplate spec in CoreDBSpec
use std::collections::BTreeMap;
let mut annotations: BTreeMap<String, String> = BTreeMap::new();
annotations.insert("eks.amazonaws.com/role-arn".to_string(), role_arn.clone());
let service_account_template = ServiceAccountTemplate {
metadata: Some(ObjectMeta {
annotations: Some(annotations),
..ObjectMeta::default()
}),
};
// These vary based on cloud provider
let mut service_account_template = ServiceAccountTemplate::default();
let mut destination_path = "".to_string();

if cloud_provider == "aws" {
let write_path = read_msg
.message
.backups_write_path
.clone()
.unwrap_or(format!("v2/{}", read_msg.message.namespace));

create_cloudformation(
aws_region.clone(),
backup_archive_bucket.clone(),
read_msg.message.namespace.clone(),
read_msg.message.backups_read_path.clone(),
read_msg.message.backups_write_path.clone(),
cf_template_bucket,
)
.await?;

// Lookup the CloudFormation stack's role ARN
let role_arn = lookup_role_arn(aws_region, &read_msg.message.namespace).await?;

info!("{}: Adding backup configuration to spec", read_msg.msg_id);
// Format ServiceAccountTemplate spec in CoreDBSpec
use std::collections::BTreeMap;
let mut annotations: BTreeMap<String, String> = BTreeMap::new();
annotations.insert("eks.amazonaws.com/role-arn".to_string(), role_arn.clone());
service_account_template = ServiceAccountTemplate {
metadata: Some(ObjectMeta {
annotations: Some(annotations),
..ObjectMeta::default()
}),
};
destination_path = format!("s3://{}/{}", backup_archive_bucket, write_path);
}

if cloud_provider == "azure" {
let write_path = read_msg
.message
.backups_write_path
.clone()
.unwrap_or(format!("{}", read_msg.message.namespace));
destination_path = format!(
"https://{}.blob.core.windows.net/{}/{}",
azure_storage_account, backup_archive_bucket, write_path
);
}
Comment on lines +736 to +746
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this may need a step to provision in Azure the permissions for the workload required for this instance to access the bucket where its backups go

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 Yep we will need that


// TODO: disable volumesnapshots for now until we can make them work with CNPG
// Enable VolumeSnapshots for all instances being created
Expand All @@ -708,13 +745,8 @@ async fn init_cloud_perms(
snapshot_class: None,
});

let write_path = read_msg
.message
.backups_write_path
.clone()
.unwrap_or(format!("v2/{}", read_msg.message.namespace));
let backup = Backup {
destinationPath: Some(format!("s3://{}/{}", backup_archive_bucket, write_path)),
destinationPath: Some(destination_path),
encryption: Some(String::from("AES256")),
retentionPolicy: Some(String::from("30")),
schedule: Some(generate_cron_expression(&read_msg.message.namespace)),
Expand Down
86 changes: 86 additions & 0 deletions tembo-operator/src/apis/coredb_types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,84 @@ pub struct ServiceAccountTemplate {
pub metadata: Option<ObjectMeta>,
}

/// AzureCredentials is the type for the credentials to be used to upload files to Azure Blob Storage.
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
pub struct AzureCredentials {
Comment on lines +92 to +94
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are these types already present in the cnpg file, or could be kopium imported from the CNPG crd

Copy link
Contributor Author

@ianstanton ianstanton Jun 26, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes they are already present. The S3Credentials values (right below this section) are also present there, but it looks like we copied them in and renamed slightly. I followed the same pattern here. Happy to go with either approach

/// The connection string to be used
#[serde(
default,
skip_serializing_if = "Option::is_none",
rename = "connectionString"
)]
pub connection_string: Option<AzureCredentialsConnectionString>,
/// Use the Azure AD based authentication without providing explicitly the keys.
#[serde(
default,
skip_serializing_if = "Option::is_none",
rename = "inheritFromAzureAD"
)]
pub inherit_from_azure_ad: Option<bool>,
/// The storage account where to upload data
#[serde(
default,
skip_serializing_if = "Option::is_none",
rename = "storageAccount"
)]
pub storage_account: Option<AzureCredentialsStorageAccount>,
/// The storage account key to be used in conjunction with the storage account name
#[serde(
default,
skip_serializing_if = "Option::is_none",
rename = "storageKey"
)]
pub storage_key: Option<AzureCredentialsStorageKey>,
/// A shared-access-signature to be used in conjunction with the storage account name
#[serde(
default,
skip_serializing_if = "Option::is_none",
rename = "storageSasToken"
)]
pub storage_sas_token: Option<AzureCredentialsStorageSasToken>,
}

/// The connection string to be used for Azure Blob Storage backups
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
pub struct AzureCredentialsConnectionString {
/// The key to select
pub key: String,
/// Name of the referent.
pub name: String,
}

/// The storage account for Azure Blob Storage backups
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
pub struct AzureCredentialsStorageAccount {
/// The key to select
pub key: String,
/// Name of the referent.
pub name: String,
}

/// The storage account key to be used in conjunction with the storage account name for Azure Blob
/// Storage backups
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
pub struct AzureCredentialsStorageKey {
/// The key to select
pub key: String,
/// Name of the referent.
pub name: String,
}

/// A shared-access-signature to be used in conjunction with the storage account name for Azure Blob
/// Storage backups
#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)]
pub struct AzureCredentialsStorageSasToken {
/// The key to select
pub key: String,
/// Name of the referent.
pub name: String,
}

/// S3Credentials is the type for the credentials to be used to upload files to S3.
/// It can be provided in two alternative ways:
/// * explicitly passing accessKeyId and secretAccessKey
Expand Down Expand Up @@ -226,6 +304,10 @@ pub struct Backup {
#[serde(default, rename = "endpointURL")]
pub endpoint_url: Option<String>,

/// The Azure Blob Storage credentials to use for backups
#[serde(default, rename = "azureCredentials")]
pub azure_credentials: Option<AzureCredentials>,

/// The S3 credentials to use for backups (if not using IAM Role)
#[serde(default = "defaults::default_s3_credentials", rename = "s3Credentials")]
pub s3_credentials: Option<S3Credentials>,
Expand Down Expand Up @@ -286,6 +368,10 @@ pub struct Restore {
#[serde(default, rename = "endpointURL")]
pub endpoint_url: Option<String>,

/// The Azure Blob Storage credentials to use for restores
#[serde(default, rename = "azureCredentials")]
pub azure_credentials: Option<AzureCredentials>,

/// s3Credentials is the S3 credentials to use for backups.
#[serde(rename = "s3Credentials")]
pub s3_credentials: Option<S3Credentials>,
Expand Down
Loading