-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
22 changed files
with
744 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| fields: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
50 changes: 50 additions & 0 deletions
50
kaldb/src/main/java/com/slack/kaldb/metadata/schema/KaldbSchemaUtil.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package com.slack.kaldb.metadata.schema; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; | ||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import com.google.protobuf.util.JsonFormat; | ||
| import com.slack.kaldb.proto.schema.Schema; | ||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import org.apache.commons.text.StringSubstitutor; | ||
| import org.apache.commons.text.lookup.StringLookup; | ||
|
|
||
| public class KaldbSchemaUtil { | ||
|
|
||
| public static Schema.KaldbSchema parseSchema(Path schemaPath) throws IOException { | ||
| String filename = schemaPath.getFileName().toString(); | ||
| if (filename.endsWith(".yaml")) { | ||
| return parseSchemaYaml(Files.readString(schemaPath), System::getenv); | ||
| } else if (filename.endsWith(".json")) { | ||
| return parseJsonSchema(Files.readString(schemaPath)); | ||
| } else { | ||
| throw new RuntimeException( | ||
| "Invalid config file format provided - must be either .json or .yaml"); | ||
| } | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public static Schema.KaldbSchema parseSchemaYaml(String yamlStr, StringLookup variableResolver) | ||
| throws JsonProcessingException, InvalidProtocolBufferException { | ||
| StringSubstitutor substitute = new StringSubstitutor(variableResolver); | ||
| ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory()); | ||
| ObjectMapper jsonWriter = new ObjectMapper(); | ||
|
|
||
| Object obj = yamlReader.readValue(substitute.replace(yamlStr), Object.class); | ||
| return parseJsonSchema(jsonWriter.writeValueAsString(obj)); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public static Schema.KaldbSchema parseJsonSchema(String jsonStr) | ||
| throws InvalidProtocolBufferException { | ||
| Schema.KaldbSchema.Builder kaldbSchemaBuilder = Schema.KaldbSchema.newBuilder(); | ||
| JsonFormat.parser().merge(jsonStr, kaldbSchemaBuilder); | ||
| Schema.KaldbSchema kaldbSchema = kaldbSchemaBuilder.build(); | ||
| // TODO: validate schema | ||
| return kaldbSchema; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package slack.proto.kaldb.schema; | ||
|
|
||
| option java_package = "com.slack.kaldb.proto.schema"; | ||
|
|
||
| message KaldbSchema { | ||
| // convert to a map | ||
| map<string, SchemaField> fields = 1; | ||
| } | ||
|
|
||
| message SchemaField { | ||
| SchemaFieldType type = 2; | ||
| // other field definitions in the future | ||
| } | ||
|
|
||
| // https://opensearch.org/docs/2.4/opensearch/supported-field-types/index/ | ||
| // Add the remaining types as needed | ||
| enum SchemaFieldType { | ||
| KEYWORD = 0; | ||
| TEXT = 1; | ||
| IP = 2; | ||
| // expose format in the future | ||
| DATE = 3; | ||
| BOOLEAN = 4; | ||
| DOUBLE = 5; | ||
| FLOAT = 6; | ||
| HALF_FLOAT = 7; | ||
| INTEGER = 8; | ||
| LONG = 9; | ||
| SCALED_LONG = 10; | ||
| SHORT = 11; | ||
| BYTE = 12; | ||
| BINARY = 13; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.