Skip to content

Commit

Permalink
Blob in list and map
Browse files Browse the repository at this point in the history 
Signed-off-by: Hongxin Liang <[email protected]>
  • Loading branch information
@honnix
honnix committed Oct 14, 2023
1 parent b246564 commit 65d643c
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 28 deletions.
13 changes: 9 additions & 4 deletions ...on/src/main/java/org/flyte/flytekit/jackson/deserializers/SdkBindingDataDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,13 +141,13 @@ private <T> SdkBindingData<List<T>> transformCollection(JsonNode tree) {
case SIMPLE_TYPE:
case MAP_VALUE_TYPE:
case COLLECTION_TYPE:
case BLOB_TYPE:
List<T> collection =
(List<T>)
streamOf(elements).map(this::transform).map(SdkBindingData::get).collect(toList());
return SdkBindingDataFactory.of(literalType, collection);

case SCHEMA_TYPE:
case BLOB_TYPE:
default:
throw new UnsupportedOperationException(
"Type contains a collection of an supported literal type: " + literalType);
Expand All @@ -166,14 +166,14 @@ private <T> SdkBindingData<Map<String, T>> transformMap(JsonNode tree) {
case SIMPLE_TYPE:
case MAP_VALUE_TYPE:
case COLLECTION_TYPE:
case BLOB_TYPE:
Map<String, T> bindingDataMap =
entries.stream()
.map(entry -> Map.entry(entry.getKey(), (T) transform(entry.getValue()).get()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
return SdkBindingDataFactory.of(literalType, bindingDataMap);

case SCHEMA_TYPE:
case BLOB_TYPE:
default:
throw new UnsupportedOperationException(
"Type contains a map of an supported literal type: " + literalType);
Expand Down Expand Up @@ -207,9 +207,14 @@ private SdkLiteralType<?> readLiteralType(JsonNode typeNode) {
return SdkLiteralTypes.maps(readLiteralType(typeNode.get(VALUE).get(TYPE)));
case COLLECTION_TYPE:
return SdkLiteralTypes.collections(readLiteralType(typeNode.get(VALUE).get(TYPE)));

case SCHEMA_TYPE:
case BLOB_TYPE:
return SdkLiteralTypes.blobs(
BlobType.builder()
.format(typeNode.get(VALUE).get("format").asText())
.dimensionality(
BlobDimensionality.valueOf(typeNode.get(VALUE).get("dimensionality").asText()))
.build());
case SCHEMA_TYPE:
default:
throw new UnsupportedOperationException(
"Type contains a collection/map of an supported literal type: " + kind);
Expand Down
5 changes: 4 additions & 1 deletion ...t-jackson/src/main/java/org/flyte/flytekit/jackson/serializers/LiteralTypeSerializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,11 @@ static void serialize(LiteralType literalType, JsonGenerator gen) throws IOExcep
serialize(literalType.mapValueType(), gen);
gen.writeEndObject();
break;
case SCHEMA_TYPE:
case BLOB_TYPE:
// {type: {kind: blob, value: {format: string, dimensionality: string}}}}
gen.writeObject(literalType.blobType());
break;
case SCHEMA_TYPE:
throw new IllegalArgumentException(
String.format("Unsupported LiteralType.Kind: [%s]", literalType.getKind()));
}
Expand Down
56 changes: 33 additions & 23 deletions flytekit-jackson/src/test/java/org/flyte/flytekit/jackson/JacksonSdkTypeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,15 @@

public class JacksonSdkTypeTest {

@SuppressWarnings("UnusedVariable")
static final BlobType BLOB_TYPE =
private static final BlobType BLOB_TYPE =
BlobType.builder().format("").dimensionality(BlobType.BlobDimensionality.SINGLE).build();

private static final Blob BLOB =
Blob.builder()
.metadata(BlobMetadata.builder().type(BLOB_TYPE).build())
.uri("file://test")
.build();

public static AutoValueInput createAutoValueInput(
long i,
double f,
Expand All @@ -68,7 +73,9 @@ public static AutoValueInput createAutoValueInput(
Duration d,
Blob blob,
List<String> l,
List<Blob> lb,
Map<String, String> m,
Map<String, Blob> mb,
List<List<String>> ll,
List<Map<String, String>> lm,
Map<String, List<String>> ml,
Expand All @@ -82,7 +89,9 @@ public static AutoValueInput createAutoValueInput(
SdkBindingDataFactory.of(d),
SdkBindingDataFactory.of(blob),
SdkBindingDataFactory.ofStringCollection(l),
SdkBindingDataFactory.of(SdkLiteralTypes.blobs(BLOB_TYPE), lb),
SdkBindingDataFactory.ofStringMap(m),
SdkBindingDataFactory.of(SdkLiteralTypes.blobs(BLOB_TYPE), mb),
SdkBindingDataFactory.of(SdkLiteralTypes.collections(SdkLiteralTypes.strings()), ll),
SdkBindingDataFactory.of(SdkLiteralTypes.maps(SdkLiteralTypes.strings()), lm),
SdkBindingDataFactory.of(SdkLiteralTypes.collections(SdkLiteralTypes.strings()), ml),
Expand Down Expand Up @@ -122,21 +131,18 @@ public void testVariableMap() {
void testFromLiteralMap() {
Instant datetime = Instant.ofEpochSecond(12, 34);
Duration duration = Duration.ofSeconds(56, 78);
Blob blob =
Blob.builder()
.metadata(BlobMetadata.builder().type(BLOB_TYPE).build())
.uri("file://test")
.build();
Map<String, Literal> literalMap = new HashMap<>();
literalMap.put("i", literalOf(Primitive.ofIntegerValue(123L)));
literalMap.put("f", literalOf(Primitive.ofFloatValue(123.0)));
literalMap.put("s", literalOf(Primitive.ofStringValue("123")));
literalMap.put("b", literalOf(Primitive.ofBooleanValue(true)));
literalMap.put("t", literalOf(Primitive.ofDatetime(datetime)));
literalMap.put("d", literalOf(Primitive.ofDuration(duration)));
literalMap.put("blob", literalOf(blob));
literalMap.put("blob", literalOf(BLOB));
literalMap.put("l", Literal.ofCollection(List.of(literalOf(Primitive.ofStringValue("123")))));
literalMap.put("lb", Literal.ofCollection(List.of(literalOf(BLOB))));
literalMap.put("m", Literal.ofMap(Map.of("marco", literalOf(Primitive.ofStringValue("polo")))));
literalMap.put("mb", Literal.ofMap(Map.of("blob", literalOf(BLOB))));
literalMap.put(
"ll",
Literal.ofCollection(
Expand Down Expand Up @@ -178,9 +184,11 @@ void testFromLiteralMap() {
/* b= */ true,
/* t= */ datetime,
/* d= */ duration,
/* blob= */ blob,
/* blob= */ BLOB,
/* l= */ List.of("123"),
/* lb= */ List.of(BLOB),
/* m= */ Map.of("marco", "polo"),
/* mb= */ Map.of("blob", BLOB),
/* ll= */ List.of(List.of("foo", "bar"), List.of("a", "b", "c")),
/* lm= */ List.of(Map.of("A", "a", "B", "b"), Map.of("a", "A", "b", "B")),
/* ml= */ Map.of("frodo", List.of("baggins", "bolson")),
Expand All @@ -197,11 +205,6 @@ private static Literal stringLiteralOf(String string) {

@Test
void testToLiteralMap() {
Blob blob =
Blob.builder()
.metadata(BlobMetadata.builder().type(BLOB_TYPE).build())
.uri("file://test")
.build();
Map<String, Literal> literalMap =
JacksonSdkType.of(AutoValueInput.class)
.toLiteralMap(
Expand All @@ -212,9 +215,11 @@ void testToLiteralMap() {
/* b= */ false,
/* t= */ Instant.ofEpochSecond(42, 1),
/* d= */ Duration.ofSeconds(1, 42),
/* blob= */ blob,
/* blob= */ BLOB,
/* l= */ List.of("foo"),
/* lb= */ List.of(BLOB),
/* m= */ Map.of("marco", "polo"),
/* mb= */ Map.of("blob", BLOB),
/* ll= */ List.of(List.of("foo", "bar"), List.of("a", "b", "c")),
/* lm= */ List.of(Map.of("A", "a", "B", "b"), Map.of("a", "A", "b", "B")),
/* ml= */ Map.of("frodo", List.of("baggins", "bolson")),
Expand Down Expand Up @@ -275,16 +280,11 @@ void testToLiteralMap() {
"pi", stringLiteralOf("3.14"), "e", stringLiteralOf("2.72"))),
"pokemon",
Literal.ofMap(Map.of("ash", stringLiteralOf("pikachu")))))),
hasEntry("blob", literalOf(blob)))));
hasEntry("blob", literalOf(BLOB)))));
}

@Test
public void testToSdkBindingDataMap() {
Blob blob =
Blob.builder()
.metadata(BlobMetadata.builder().type(BLOB_TYPE).build())
.uri("file://test")
.build();
AutoValueInput input =
createAutoValueInput(
/* i= */ 42L,
Expand All @@ -293,9 +293,11 @@ public void testToSdkBindingDataMap() {
/* b= */ false,
/* t= */ Instant.ofEpochSecond(42, 1),
/* d= */ Duration.ofSeconds(1, 42),
/* blob= */ blob,
/* blob= */ BLOB,
/* l= */ List.of("foo"),
/* lb= */ List.of(BLOB),
/* m= */ Map.of("marco", "polo"),
/* mb= */ Map.of("blob", BLOB),
/* ll= */ List.of(List.of("foo", "bar"), List.of("a", "b", "c")),
/* lm= */ List.of(Map.of("A", "a", "B", "b"), Map.of("a", "A", "b", "B")),
/* ml= */ Map.of("frodo", List.of("baggins", "bolson")),
Expand All @@ -314,7 +316,9 @@ public void testToSdkBindingDataMap() {
expected.put("d", input.d());
expected.put("blob", input.blob());
expected.put("l", input.l());
expected.put("lb", input.lb());
expected.put("m", input.m());
expected.put("mb", input.mb());
expected.put("ll", input.ll());
expected.put("lm", input.lm());
expected.put("ml", input.ml());
Expand Down Expand Up @@ -548,8 +552,12 @@ public abstract static class AutoValueInput {

public abstract SdkBindingData<List<String>> l();

public abstract SdkBindingData<List<Blob>> lb();

public abstract SdkBindingData<Map<String, String>> m();

public abstract SdkBindingData<Map<String, Blob>> mb();

public abstract SdkBindingData<List<List<String>>> ll();

public abstract SdkBindingData<List<Map<String, String>>> lm();
Expand All @@ -567,13 +575,15 @@ public static AutoValueInput create(
SdkBindingData<Duration> d,
SdkBindingData<Blob> blob,
SdkBindingData<List<String>> l,
SdkBindingData<List<Blob>> lb,
SdkBindingData<Map<String, String>> m,
SdkBindingData<Map<String, Blob>> mb,
SdkBindingData<List<List<String>>> ll,
SdkBindingData<List<Map<String, String>>> lm,
SdkBindingData<Map<String, List<String>>> ml,
SdkBindingData<Map<String, Map<String, String>>> mm) {
return new AutoValue_JacksonSdkTypeTest_AutoValueInput(
i, f, s, b, t, d, blob, l, m, ll, lm, ml, mm);
i, f, s, b, t, d, blob, l, lb, m, mb, ll, lm, ml, mm);
}
}

Expand Down

0 comments on commit 65d643c

Please sign in to comment.