From 8e70b9e982975eee066d2299b6b87e5d420cc4ff Mon Sep 17 00:00:00 2001 From: JeremyHi Date: Fri, 15 Sep 2023 10:11:21 +0800 Subject: [PATCH] feat: remove deprecated metadata keys (#2398) * feat: remove deprecated metadata keys * feat: this time, weny indeed said [removes it] --- Cargo.lock | 4 + src/cmd/Cargo.toml | 4 + src/cmd/src/cli/upgrade.rs | 147 +++++++++++++++++++++++++- src/common/catalog/src/error.rs | 21 +--- src/common/meta/src/helper.rs | 178 -------------------------------- src/common/meta/src/lib.rs | 3 - 6 files changed, 155 insertions(+), 202 deletions(-) delete mode 100644 src/common/meta/src/helper.rs diff --git a/Cargo.lock b/Cargo.lock index df924fe0097d..af851888376c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1588,6 +1588,7 @@ dependencies = [ "clap 3.2.25", "client", "common-base", + "common-catalog", "common-config", "common-error", "common-meta", @@ -1604,6 +1605,7 @@ dependencies = [ "etcd-client", "frontend", "futures", + "lazy_static", "meta-client", "meta-srv", "metrics", @@ -1612,9 +1614,11 @@ dependencies = [ "prost", "query", "rand", + "regex", "rexpect", "rustyline 10.1.1", "serde", + "serde_json", "servers", "session", "snafu", diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 2997eb0f3054..b07ca011b1e5 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -23,6 +23,7 @@ chrono.workspace = true clap = { version = "3.1", features = ["derive"] } client = { workspace = true } common-base = { workspace = true } +common-catalog = { workspace = true } common-config = { workspace = true } common-error = { workspace = true } common-meta = { workspace = true } @@ -39,6 +40,7 @@ either = "1.8" etcd-client.workspace = true frontend = { workspace = true } futures.workspace = true +lazy_static.workspace = true meta-client = { workspace = true } meta-srv = { workspace = true } metrics.workspace = true @@ -47,8 +49,10 @@ partition = { workspace = true } prost.workspace = true query = { workspace = true } rand.workspace = true +regex.workspace = true rustyline = "10.1" serde.workspace = true +serde_json.workspace = true servers = { workspace = true } session = { workspace = true } snafu.workspace = true diff --git a/src/cmd/src/cli/upgrade.rs b/src/cmd/src/cli/upgrade.rs index cee58afb0c5f..32d43065ebe0 100644 --- a/src/cmd/src/cli/upgrade.rs +++ b/src/cmd/src/cli/upgrade.rs @@ -18,7 +18,6 @@ use async_trait::async_trait; use clap::Parser; use client::api::v1::meta::TableRouteValue; use common_meta::error as MetaError; -use common_meta::helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue}; use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue}; use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue}; use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue}; @@ -39,6 +38,7 @@ use meta_srv::service::store::etcd::EtcdStore; use meta_srv::service::store::kv::{KvBackendAdapter, KvStoreRef}; use prost::Message; use snafu::ResultExt; +use v1_helper::{CatalogKey as v1CatalogKey, SchemaKey as v1SchemaKey, TableGlobalValue}; use crate::cli::{Instance, Tool}; use crate::error::{self, ConnectEtcdSnafu, Result}; @@ -413,3 +413,148 @@ impl MigrateTableMetadata { } } } + +#[deprecated(since = "0.4.0", note = "Used for migrate old version(v0.3) metadata")] +mod v1_helper { + use std::collections::HashMap; + use std::fmt::{Display, Formatter}; + + use err::{DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu}; + use lazy_static::lazy_static; + use regex::Regex; + use serde::{Deserialize, Serialize}; + use snafu::{ensure, OptionExt, ResultExt}; + use table::metadata::{RawTableInfo, TableId}; + + pub const CATALOG_KEY_PREFIX: &str = "__c"; + pub const SCHEMA_KEY_PREFIX: &str = "__s"; + + /// The pattern of a valid catalog, schema or table name. + const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*"; + + lazy_static! { + static ref CATALOG_KEY_PATTERN: Regex = + Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap(); + } + + lazy_static! { + static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!( + "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$" + )) + .unwrap(); + } + + /// Table global info contains necessary info for a datanode to create table regions, including + /// table id, table meta(schema...), region id allocation across datanodes. + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + pub struct TableGlobalValue { + /// Id of datanode that created the global table info kv. only for debugging. + pub node_id: u64, + /// Allocation of region ids across all datanodes. + pub regions_id_map: HashMap>, + pub table_info: RawTableInfo, + } + + impl TableGlobalValue { + pub fn table_id(&self) -> TableId { + self.table_info.ident.table_id + } + } + + pub struct CatalogKey { + pub catalog_name: String, + } + + impl Display for CatalogKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(CATALOG_KEY_PREFIX)?; + f.write_str("-")?; + f.write_str(&self.catalog_name) + } + } + + impl CatalogKey { + pub fn parse(s: impl AsRef) -> Result { + let key = s.as_ref(); + let captures = CATALOG_KEY_PATTERN + .captures(key) + .context(InvalidCatalogSnafu { key })?; + ensure!(captures.len() == 2, InvalidCatalogSnafu { key }); + Ok(Self { + catalog_name: captures[1].to_string(), + }) + } + } + + #[derive(Debug, Serialize, Deserialize)] + pub struct CatalogValue; + + pub struct SchemaKey { + pub catalog_name: String, + pub schema_name: String, + } + + impl Display for SchemaKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(SCHEMA_KEY_PREFIX)?; + f.write_str("-")?; + f.write_str(&self.catalog_name)?; + f.write_str("-")?; + f.write_str(&self.schema_name) + } + } + + impl SchemaKey { + pub fn parse(s: impl AsRef) -> Result { + let key = s.as_ref(); + let captures = SCHEMA_KEY_PATTERN + .captures(key) + .context(InvalidCatalogSnafu { key })?; + ensure!(captures.len() == 3, InvalidCatalogSnafu { key }); + Ok(Self { + catalog_name: captures[1].to_string(), + schema_name: captures[2].to_string(), + }) + } + } + + #[derive(Debug, Serialize, Deserialize)] + pub struct SchemaValue; + + macro_rules! define_catalog_value { + ( $($val_ty: ty), *) => { + $( + impl $val_ty { + pub fn parse(s: impl AsRef) -> Result { + serde_json::from_str(s.as_ref()) + .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() }) + } + + pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result { + Self::parse(&String::from_utf8_lossy(bytes.as_ref())) + } + } + )* + } + } + + define_catalog_value!(TableGlobalValue); + + mod err { + use snafu::{Location, Snafu}; + + #[derive(Debug, Snafu)] + #[snafu(visibility(pub))] + pub enum Error { + #[snafu(display("Invalid catalog info: {}", key))] + InvalidCatalog { key: String, location: Location }, + + #[snafu(display("Failed to deserialize catalog entry value: {}", raw))] + DeserializeCatalogEntryValue { + raw: String, + location: Location, + source: serde_json::error::Error, + }, + } + } +} diff --git a/src/common/catalog/src/error.rs b/src/common/catalog/src/error.rs index 04c8a9c6d785..2ded9e47a576 100644 --- a/src/common/catalog/src/error.rs +++ b/src/common/catalog/src/error.rs @@ -21,36 +21,17 @@ use snafu::{Location, Snafu}; #[derive(Debug, Snafu)] #[snafu(visibility(pub))] pub enum Error { - #[snafu(display("Invalid catalog info: {}", key))] - InvalidCatalog { key: String, location: Location }, - #[snafu(display("Invalid full table name: {}", table_name))] InvalidFullTableName { table_name: String, location: Location, }, - - #[snafu(display("Failed to deserialize catalog entry value: {}", raw))] - DeserializeCatalogEntryValue { - raw: String, - location: Location, - source: serde_json::error::Error, - }, - - #[snafu(display("Failed to serialize catalog entry value"))] - SerializeCatalogEntryValue { - location: Location, - source: serde_json::error::Error, - }, } impl ErrorExt for Error { fn status_code(&self) -> StatusCode { match self { - Error::InvalidCatalog { .. } - | Error::DeserializeCatalogEntryValue { .. } - | Error::SerializeCatalogEntryValue { .. } - | Error::InvalidFullTableName { .. } => StatusCode::Unexpected, + Error::InvalidFullTableName { .. } => StatusCode::Unexpected, } } diff --git a/src/common/meta/src/helper.rs b/src/common/meta/src/helper.rs deleted file mode 100644 index 52b8a3b1a9cb..000000000000 --- a/src/common/meta/src/helper.rs +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::fmt::{Display, Formatter}; - -use common_catalog::error::{ - DeserializeCatalogEntryValueSnafu, Error, InvalidCatalogSnafu, SerializeCatalogEntryValueSnafu, -}; -use lazy_static::lazy_static; -use regex::Regex; -use serde::{Deserialize, Serialize}; -use snafu::{ensure, OptionExt, ResultExt}; -use table::metadata::{RawTableInfo, TableId}; - -pub const CATALOG_KEY_PREFIX: &str = "__c"; -pub const SCHEMA_KEY_PREFIX: &str = "__s"; - -/// The pattern of a valid catalog, schema or table name. -const NAME_PATTERN: &str = "[a-zA-Z_:][a-zA-Z0-9_:]*"; - -lazy_static! { - static ref CATALOG_KEY_PATTERN: Regex = - Regex::new(&format!("^{CATALOG_KEY_PREFIX}-({NAME_PATTERN})$")).unwrap(); -} - -lazy_static! { - static ref SCHEMA_KEY_PATTERN: Regex = Regex::new(&format!( - "^{SCHEMA_KEY_PREFIX}-({NAME_PATTERN})-({NAME_PATTERN})$" - )) - .unwrap(); -} - -pub fn build_catalog_prefix() -> String { - format!("{CATALOG_KEY_PREFIX}-") -} - -pub fn build_schema_prefix(catalog_name: impl AsRef) -> String { - format!("{SCHEMA_KEY_PREFIX}-{}-", catalog_name.as_ref()) -} - -/// Table global info contains necessary info for a datanode to create table regions, including -/// table id, table meta(schema...), region id allocation across datanodes. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct TableGlobalValue { - /// Id of datanode that created the global table info kv. only for debugging. - pub node_id: u64, - /// Allocation of region ids across all datanodes. - pub regions_id_map: HashMap>, - pub table_info: RawTableInfo, -} - -impl TableGlobalValue { - pub fn table_id(&self) -> TableId { - self.table_info.ident.table_id - } -} - -#[deprecated(since = "0.4.0", note = "Please use the CatalogNameKey instead")] -pub struct CatalogKey { - pub catalog_name: String, -} - -impl Display for CatalogKey { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(CATALOG_KEY_PREFIX)?; - f.write_str("-")?; - f.write_str(&self.catalog_name) - } -} - -impl CatalogKey { - pub fn parse(s: impl AsRef) -> Result { - let key = s.as_ref(); - let captures = CATALOG_KEY_PATTERN - .captures(key) - .context(InvalidCatalogSnafu { key })?; - ensure!(captures.len() == 2, InvalidCatalogSnafu { key }); - Ok(Self { - catalog_name: captures[1].to_string(), - }) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct CatalogValue; - -#[deprecated(since = "0.4.0", note = "Please use the SchemaNameKey instead")] -pub struct SchemaKey { - pub catalog_name: String, - pub schema_name: String, -} - -impl Display for SchemaKey { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(SCHEMA_KEY_PREFIX)?; - f.write_str("-")?; - f.write_str(&self.catalog_name)?; - f.write_str("-")?; - f.write_str(&self.schema_name) - } -} - -impl SchemaKey { - pub fn parse(s: impl AsRef) -> Result { - let key = s.as_ref(); - let captures = SCHEMA_KEY_PATTERN - .captures(key) - .context(InvalidCatalogSnafu { key })?; - ensure!(captures.len() == 3, InvalidCatalogSnafu { key }); - Ok(Self { - catalog_name: captures[1].to_string(), - schema_name: captures[2].to_string(), - }) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct SchemaValue; - -macro_rules! define_catalog_value { - ( $($val_ty: ty), *) => { - $( - impl $val_ty { - pub fn parse(s: impl AsRef) -> Result { - serde_json::from_str(s.as_ref()) - .context(DeserializeCatalogEntryValueSnafu { raw: s.as_ref() }) - } - - pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result { - Self::parse(&String::from_utf8_lossy(bytes.as_ref())) - } - - pub fn as_bytes(&self) -> Result, Error> { - Ok(serde_json::to_string(self) - .context(SerializeCatalogEntryValueSnafu)? - .into_bytes()) - } - } - )* - } -} - -define_catalog_value!(TableGlobalValue, CatalogValue, SchemaValue); - -#[cfg(test)] -mod tests { - - use super::*; - - #[test] - fn test_parse_catalog_key() { - let key = "__c-C"; - let catalog_key = CatalogKey::parse(key).unwrap(); - assert_eq!("C", catalog_key.catalog_name); - assert_eq!(key, catalog_key.to_string()); - } - - #[test] - fn test_parse_schema_key() { - let key = "__s-C-S"; - let schema_key = SchemaKey::parse(key).unwrap(); - assert_eq!("C", schema_key.catalog_name); - assert_eq!("S", schema_key.schema_name); - assert_eq!(key, schema_key.to_string()); - } -} diff --git a/src/common/meta/src/lib.rs b/src/common/meta/src/lib.rs index 4387ceaeaf82..8ed426adeef5 100644 --- a/src/common/meta/src/lib.rs +++ b/src/common/meta/src/lib.rs @@ -21,9 +21,6 @@ pub mod ddl; pub mod ddl_manager; pub mod error; pub mod heartbeat; -// TODO(weny): Removes it -#[allow(deprecated)] -pub mod helper; pub mod ident; pub mod instruction; pub mod key;