Skip to content

Commit

Permalink
add gc_compaction tenant config
Browse files Browse the repository at this point in the history 
Signed-off-by: Alex Chi Z <[email protected]>
  • Loading branch information
@skyzh
skyzh committed Dec 20, 2024
1 parent a993f63 commit c254500
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 2 deletions.
15 changes: 15 additions & 0 deletions control_plane/src/pageserver.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,21 @@ impl PageServerNode {
.map(serde_json::from_str)
.transpose()
.context("parse `wal_receiver_protocol_override` from json")?,
gc_compaction_enabled: settings
.remove("gc_compaction_enabled")
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'gc_compaction_enabled' as bool")?,
gc_compaction_initial_threshold_mb: settings
.remove("gc_compaction_initial_threshold_mb")
.map(|x| x.parse::<u64>())
.transpose()
.context("Failed to parse 'gc_compaction_initial_threshold_mb' as integer")?,
gc_compaction_ratio_percent: settings
.remove("gc_compaction_ratio_percent")
.map(|x| x.parse::<u64>())
.transpose()
.context("Failed to parse 'gc_compaction_ratio_percent' as integer")?,
};
if !settings.is_empty() {
bail!("Unrecognized tenant settings: {settings:?}")
Expand Down
11 changes: 11 additions & 0 deletions libs/pageserver_api/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,11 @@ pub struct TenantConfigToml {
pub timeline_offloading: bool,

pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,

/// gc-compaction related configs
pub gc_compaction_enabled: bool,
pub gc_compaction_initial_threshold_mb: u64,
pub gc_compaction_ratio_percent: u64,
}

pub mod defaults {
Expand Down Expand Up @@ -494,6 +499,9 @@ pub mod tenant_conf_defaults {
// By default ingest enough WAL for two new L0 layers before checking if new image
// image layers should be created.
pub const DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD: u8 = 2;
pub const DEFAULT_GC_COMPACTION_ENABLED: bool = false;
pub const DEFAULT_GC_COMPACTION_INITIAL_THRESHOLD_MB: u64 = 10240;
pub const DEFAULT_GC_COMPACTION_RATIO_PERCENT: u64 = 100;
}

impl Default for TenantConfigToml {
Expand Down Expand Up @@ -538,6 +546,9 @@ impl Default for TenantConfigToml {
lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS,
timeline_offloading: false,
wal_receiver_protocol_override: None,
gc_compaction_enabled: DEFAULT_GC_COMPACTION_ENABLED,
gc_compaction_initial_threshold_mb: DEFAULT_GC_COMPACTION_INITIAL_THRESHOLD_MB,
gc_compaction_ratio_percent: DEFAULT_GC_COMPACTION_RATIO_PERCENT,
}
}
}
24 changes: 24 additions & 0 deletions libs/pageserver_api/src/models.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,12 @@ pub struct TenantConfigPatch {
pub timeline_offloading: FieldPatch<bool>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub wal_receiver_protocol_override: FieldPatch<PostgresClientProtocol>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub gc_compaction_enabled: FieldPatch<bool>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub gc_compaction_initial_threshold_mb: FieldPatch<u64>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub gc_compaction_ratio_percent: FieldPatch<u64>,
}

/// An alternative representation of `pageserver::tenant::TenantConf` with
Expand Down Expand Up @@ -527,6 +533,9 @@ pub struct TenantConfig {
pub lsn_lease_length_for_ts: Option<String>,
pub timeline_offloading: Option<bool>,
pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
pub gc_compaction_enabled: Option<bool>,
pub gc_compaction_initial_threshold_mb: Option<u64>,
pub gc_compaction_ratio_percent: Option<u64>,
}

impl TenantConfig {
Expand Down Expand Up @@ -556,6 +565,9 @@ impl TenantConfig {
mut lsn_lease_length_for_ts,
mut timeline_offloading,
mut wal_receiver_protocol_override,
mut gc_compaction_enabled,
mut gc_compaction_initial_threshold_mb,
mut gc_compaction_ratio_percent,
} = self;

patch.checkpoint_distance.apply(&mut checkpoint_distance);
Expand Down Expand Up @@ -600,6 +612,15 @@ impl TenantConfig {
patch
.wal_receiver_protocol_override
.apply(&mut wal_receiver_protocol_override);
patch
.gc_compaction_enabled
.apply(&mut gc_compaction_enabled);
patch
.gc_compaction_initial_threshold_mb
.apply(&mut gc_compaction_initial_threshold_mb);
patch
.gc_compaction_ratio_percent
.apply(&mut gc_compaction_ratio_percent);

Self {
checkpoint_distance,
Expand All @@ -626,6 +647,9 @@ impl TenantConfig {
lsn_lease_length_for_ts,
timeline_offloading,
wal_receiver_protocol_override,
gc_compaction_enabled,
gc_compaction_initial_threshold_mb,
gc_compaction_ratio_percent,
}
}
}
Expand Down
36 changes: 36 additions & 0 deletions pageserver/src/tenant/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,15 @@ pub struct TenantConfOpt {

#[serde(skip_serializing_if = "Option::is_none")]
pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,

#[serde(skip_serializing_if = "Option::is_none")]
pub gc_compaction_enabled: Option<bool>,

#[serde(skip_serializing_if = "Option::is_none")]
pub gc_compaction_initial_threshold_mb: Option<u64>,

#[serde(skip_serializing_if = "Option::is_none")]
pub gc_compaction_ratio_percent: Option<u64>,
}

impl TenantConfOpt {
Expand Down Expand Up @@ -425,6 +434,15 @@ impl TenantConfOpt {
wal_receiver_protocol_override: self
.wal_receiver_protocol_override
.or(global_conf.wal_receiver_protocol_override),
gc_compaction_enabled: self
.gc_compaction_enabled
.unwrap_or(global_conf.gc_compaction_enabled),
gc_compaction_initial_threshold_mb: self
.gc_compaction_initial_threshold_mb
.unwrap_or(global_conf.gc_compaction_initial_threshold_mb),
gc_compaction_ratio_percent: self
.gc_compaction_ratio_percent
.unwrap_or(global_conf.gc_compaction_ratio_percent),
}
}

Expand Down Expand Up @@ -454,6 +472,9 @@ impl TenantConfOpt {
mut lsn_lease_length_for_ts,
mut timeline_offloading,
mut wal_receiver_protocol_override,
mut gc_compaction_enabled,
mut gc_compaction_initial_threshold_mb,
mut gc_compaction_ratio_percent,
} = self;

patch.checkpoint_distance.apply(&mut checkpoint_distance);
Expand Down Expand Up @@ -522,6 +543,15 @@ impl TenantConfOpt {
patch
.wal_receiver_protocol_override
.apply(&mut wal_receiver_protocol_override);
patch
.gc_compaction_enabled
.apply(&mut gc_compaction_enabled);
patch
.gc_compaction_initial_threshold_mb
.apply(&mut gc_compaction_initial_threshold_mb);
patch
.gc_compaction_ratio_percent
.apply(&mut gc_compaction_ratio_percent);

Ok(Self {
checkpoint_distance,
Expand All @@ -548,6 +578,9 @@ impl TenantConfOpt {
lsn_lease_length_for_ts,
timeline_offloading,
wal_receiver_protocol_override,
gc_compaction_enabled,
gc_compaction_initial_threshold_mb,
gc_compaction_ratio_percent,
})
}
}
Expand Down Expand Up @@ -603,6 +636,9 @@ impl From<TenantConfOpt> for models::TenantConfig {
lsn_lease_length_for_ts: value.lsn_lease_length_for_ts.map(humantime),
timeline_offloading: value.timeline_offloading,
wal_receiver_protocol_override: value.wal_receiver_protocol_override,
gc_compaction_enabled: value.gc_compaction_enabled,
gc_compaction_initial_threshold_mb: value.gc_compaction_initial_threshold_mb,
gc_compaction_ratio_percent: value.gc_compaction_ratio_percent,
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions pageserver/src/tenant/remote_timeline_client/index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -985,7 +985,7 @@ mod tests {
}
}
},
"l2_lsn": "0/1686070",
"l2_lsn": "0/1686070"
}"#;

let expected = IndexPart {
Expand Down Expand Up @@ -1025,7 +1025,7 @@ mod tests {
finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
}))),
l2_lsn: Some("0/16860E8".parse::<Lsn>().unwrap()),
l2_lsn: Some("0/1686070".parse::<Lsn>().unwrap()),
};

let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
Expand Down

0 comments on commit c254500

Please sign in to comment.