From d716c37712a5ce50c3d02d5b374f58199a80089d Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Wed, 18 Sep 2024 13:31:26 -0700 Subject: [PATCH] Adds changes for sequence data type --- code-gen-projects/schema/sequence.isl | 5 + src/bin/ion/commands/generate/generator.rs | 136 +++++++++++++++++- src/bin/ion/commands/generate/model.rs | 61 ++++++-- src/bin/ion/commands/generate/result.rs | 9 ++ .../generate/templates/java/nested_type.templ | 2 +- .../generate/templates/java/sequence.templ | 68 ++++----- src/bin/ion/commands/generate/utils.rs | 4 +- 7 files changed, 234 insertions(+), 51 deletions(-) create mode 100644 code-gen-projects/schema/sequence.isl diff --git a/code-gen-projects/schema/sequence.isl b/code-gen-projects/schema/sequence.isl new file mode 100644 index 0000000..8331aff --- /dev/null +++ b/code-gen-projects/schema/sequence.isl @@ -0,0 +1,5 @@ +type::{ + name: sequence, + type: list, + element: string +} diff --git a/src/bin/ion/commands/generate/generator.rs b/src/bin/ion/commands/generate/generator.rs index ad14a97..e95a3bd 100644 --- a/src/bin/ion/commands/generate/generator.rs +++ b/src/bin/ion/commands/generate/generator.rs @@ -1,7 +1,7 @@ -use crate::commands::generate::context::CodeGenContext; +use crate::commands::generate::context::{CodeGenContext, SequenceType}; use crate::commands::generate::model::{ AbstractDataType, DataModelNode, FieldPresence, FieldReference, FullyQualifiedTypeReference, - ScalarBuilder, StructureBuilder, WrappedScalarBuilder, + ScalarBuilder, SequenceBuilder, StructureBuilder, WrappedScalarBuilder, WrappedSequenceBuilder, }; use crate::commands::generate::result::{ invalid_abstract_data_type_error, invalid_abstract_data_type_raw_error, CodeGenResult, @@ -351,6 +351,19 @@ impl<'a, L: Language + 'static> CodeGenerator<'a, L> { .any(|it| matches!(it.constraint(), IslConstraintValue::Fields(_, _))) { self.build_structure_from_constraints(constraints, code_gen_context, isl_type)? + } else if constraints + .iter() + .any(|it| matches!(it.constraint(), IslConstraintValue::Element(_, _))) + { + if is_nested_type { + self.build_sequence_from_constraints(constraints, code_gen_context, isl_type)? + } else { + self.build_wrapped_sequence_from_constraints( + constraints, + code_gen_context, + isl_type, + )? + } } else if Self::contains_scalar_constraints(constraints) { if is_nested_type { self.build_scalar_from_constraints(constraints, code_gen_context, isl_type)? @@ -358,7 +371,7 @@ impl<'a, L: Language + 'static> CodeGenerator<'a, L> { self.build_wrapped_scalar_from_constraints(constraints, code_gen_context, isl_type)? } } else { - todo!("Support for sequences, maps, scalars, and tuples not implemented yet.") + todo!("Support for maps and tuples not implemented yet.") }; let data_model_node = DataModelNode { @@ -587,7 +600,6 @@ impl<'a, L: Language + 'static> CodeGenerator<'a, L> { isl_type )))?; - // by default fields aren't closed wrapped_scalar_builder.base_type(type_name); found_base_type = true; } @@ -656,6 +668,122 @@ impl<'a, L: Language + 'static> CodeGenerator<'a, L> { Ok(AbstractDataType::Scalar(scalar_builder.build()?)) } + + /// Builds `AbstractDataType::WrappedSequence` from the given constraints. + /// ``` + /// type::{ + /// name: foo, + /// type: list, + /// element: string, + /// } + /// ``` + /// This method builds `AbstractDataType`as following: + /// ``` + /// AbstractDataType::WrappedSequence( + /// WrappedSequence { + /// name: vec!["org", "example", "Foo"] // assuming the namespace here is `org.example` + /// element_type: FullyQualifiedTypeReference { type_name: vec!["String"], parameters: vec![] } // Represents the element type for the list + /// sequence_type: SequenceType::List, // Represents list type for the given sequence + /// doc_comment: None // There is no doc comment defined in above ISL type def + /// source: IslType { .. } // Represents the `IslType` that is getting converted to `AbstractDataType` + /// } + /// ) + /// ``` + fn build_wrapped_sequence_from_constraints( + &mut self, + constraints: &[IslConstraint], + code_gen_context: &mut CodeGenContext, + parent_isl_type: &IslType, + ) -> CodeGenResult { + let mut wrapped_sequence_builder = WrappedSequenceBuilder::default(); + wrapped_sequence_builder + .name(self.current_type_fully_qualified_name.to_owned()) + .source(parent_isl_type.to_owned()); + for constraint in constraints { + match constraint.constraint() { + IslConstraintValue::Element(isl_type_ref, _) => { + let type_name = self + .fully_qualified_type_ref_name(isl_type_ref, code_gen_context)? + .ok_or(invalid_abstract_data_type_raw_error(format!( + "Could not determine `FullQualifiedTypeReference` for type {:?}", + isl_type_ref + )))?; + + wrapped_sequence_builder.element_type(type_name); + } + IslConstraintValue::Type(isl_type_ref) => { + if isl_type_ref.name() == "sexp" { + wrapped_sequence_builder.sequence_type(SequenceType::SExp); + } else if isl_type_ref.name() == "list" { + wrapped_sequence_builder.sequence_type(SequenceType::List); + } + } + _ => { + return invalid_abstract_data_type_error( + "Could not determine the abstract data type due to conflicting constraints", + ); + } + } + } + Ok(AbstractDataType::WrappedSequence( + wrapped_sequence_builder.build()?, + )) + } + + /// Builds `AbstractDataType::Sequence` from the given constraints. + /// ``` + /// { + /// type: list, + /// element: string, + /// } + /// ``` + /// This method builds `AbstractDataType`as following: + /// ``` + /// AbstractDataType::Sequence( + /// Sequence { + /// element_type: FullyQualifiedTypeReference { type_name: vec!["String"], parameters: vec![] } // Represents the element type for the list + /// sequence_type: SequenceType::List, // Represents list type for the given sequence + /// doc_comment: None // There is no doc comment defined in above ISL type def + /// source: IslType { .. } // Represents the `IslType` that is getting converted to `AbstractDataType` + /// } + /// ) + /// ``` + fn build_sequence_from_constraints( + &mut self, + constraints: &[IslConstraint], + code_gen_context: &mut CodeGenContext, + parent_isl_type: &IslType, + ) -> CodeGenResult { + let mut sequence_builder = SequenceBuilder::default(); + sequence_builder.source(parent_isl_type.to_owned()); + for constraint in constraints { + match constraint.constraint() { + IslConstraintValue::Element(isl_type_ref, _) => { + let type_name = self + .fully_qualified_type_ref_name(isl_type_ref, code_gen_context)? + .ok_or(invalid_abstract_data_type_raw_error(format!( + "Could not determine `FullQualifiedTypeReference` for type {:?}", + isl_type_ref + )))?; + + sequence_builder.element_type(type_name); + } + IslConstraintValue::Type(isl_type_ref) => { + if isl_type_ref.name() == "sexp" { + sequence_builder.sequence_type(SequenceType::SExp); + } else if isl_type_ref.name() == "list" { + sequence_builder.sequence_type(SequenceType::List); + } + } + _ => { + return invalid_abstract_data_type_error( + "Could not determine the abstract data type due to conflicting constraints", + ); + } + } + } + Ok(AbstractDataType::Sequence(sequence_builder.build()?)) + } } #[cfg(test)] diff --git a/src/bin/ion/commands/generate/model.rs b/src/bin/ion/commands/generate/model.rs index e001ccc..3a4fef2 100644 --- a/src/bin/ion/commands/generate/model.rs +++ b/src/bin/ion/commands/generate/model.rs @@ -170,14 +170,13 @@ impl FullyQualifiedTypeReference { #[derive(Debug, Clone, PartialEq, Serialize)] pub enum AbstractDataType { // Represents a scalar type which also has a name attached to it and is nominally distinct from its base type. - #[allow(dead_code)] WrappedScalar(WrappedScalar), // Represents a scalar value (e.g. a string or integer or user defined type) - #[allow(dead_code)] Scalar(Scalar), // A series of zero or more values whose type is described by the nested `element_type` - #[allow(dead_code)] Sequence(Sequence), + // Represents a sequence type which also has anme attached to it and is nominally distinct from its element type. + WrappedSequence(WrappedSequence), // A collection of field name/value pairs (e.g. a map) Structure(Structure), } @@ -192,7 +191,12 @@ impl AbstractDataType { AbstractDataType::Scalar(Scalar { doc_comment, .. }) => { doc_comment.as_ref().map(|s| s.as_str()) } - AbstractDataType::Sequence(Sequence { doc_comment, .. }) => Some(doc_comment.as_str()), + AbstractDataType::Sequence(Sequence { doc_comment, .. }) => { + doc_comment.as_ref().map(|s| s.as_str()) + } + AbstractDataType::WrappedSequence(WrappedSequence { doc_comment, .. }) => { + doc_comment.as_ref().map(|s| s.as_str()) + } AbstractDataType::Structure(Structure { doc_comment, .. }) => { doc_comment.as_ref().map(|s| s.as_str()) } @@ -206,6 +210,7 @@ impl AbstractDataType { } AbstractDataType::Scalar(s) => Some(s.base_type.to_owned()), AbstractDataType::Sequence(seq) => Some(seq.element_type.to_owned()), + AbstractDataType::WrappedSequence(seq) => Some(seq.element_type.to_owned()), AbstractDataType::Structure(structure) => Some(structure.name.to_owned().into()), } } @@ -289,8 +294,6 @@ impl WrappedScalar { /// Represents series of zero or more values whose type is described by the nested `element_type` /// and sequence type is described by nested `sequence_type` (e.g. List or SExp). -/// If there is no `element` constraint present in schema type then `element_type` will be None. -/// If there is no `type` constraint present in schema type then `sequence_type` will be None. /// e.g. Given below ISL, /// ``` /// type::{ @@ -308,11 +311,12 @@ impl WrappedScalar { #[allow(dead_code)] #[derive(Debug, Clone, Builder, PartialEq, Serialize)] #[builder(setter(into))] -pub struct Sequence { +pub struct WrappedSequence { // Represents the fully qualified name for this data model name: FullyQualifiedTypeName, // Represents doc comment for the generated code - doc_comment: String, + #[builder(default)] + doc_comment: Option, // Represents the fully qualified name with namespace where each element of vector stores a module name or class/struct name. // _Note: that a hashmap with (FullQualifiedTypeReference, DataModel) pairs will be stored in code generator to get information on the element_type name used here._ element_type: FullyQualifiedTypeReference, @@ -326,6 +330,41 @@ pub struct Sequence { source: IslType, } +/// Represents series of zero or more values whose type is described by the nested `element_type` +/// and sequence type is described by nested `sequence_type` (e.g. List or SExp). +/// e.g. Given below ISL, +/// ``` +/// type::{ +/// name: sequence_type, +/// element: int, +/// type: list +/// } +/// ``` +/// Corresponding generated code in Rust would look like following: +/// ``` +/// struct SequenceType { +/// value: Vec +/// } +/// ``` +#[derive(Debug, Clone, Builder, PartialEq, Serialize)] +#[builder(setter(into))] +pub struct Sequence { + // Represents doc comment for the generated code + #[builder(default)] + pub(crate) doc_comment: Option, + // Represents the fully qualified name with namespace where each element of vector stores a module name or class/struct name. + // _Note: that a hashmap with (FullQualifiedTypeReference, DataModel) pairs will be stored in code generator to get information on the element_type name used here._ + pub(crate) element_type: FullyQualifiedTypeReference, + // Represents the type of the sequence which is either `sexp` or `list`. + pub(crate) sequence_type: SequenceType, + // Represents the source ISL type which can be used to get other constraints useful for this type. + // For example, getting the length of this sequence from `container_length` constraint or getting a `regex` value for string type. + // This will also be useful for `text` type to verify if this is a `string` or `symbol`. + // TODO: `IslType` does not implement `Serialize`, define a custom implementation or define methods on this field that returns values which could be serialized. + #[serde(skip_serializing)] + pub(crate) source: IslType, +} + /// Represents a collection of field name/value pairs (e.g. a map) /// e.g. Given below ISL, /// ``` @@ -448,8 +487,7 @@ mod model_tests { #[test] fn sequence_builder_test() { let expected_seq = Sequence { - name: vec![], - doc_comment: "This is sequence type of strings".to_string(), + doc_comment: Some("This is sequence type of strings".to_string()), element_type: FullyQualifiedTypeReference { type_name: vec!["String".to_string()], parameters: vec![], @@ -465,8 +503,7 @@ mod model_tests { // sets all the information about the sequence except the `element_type` seq_builder - .name(vec![]) - .doc_comment("This is sequence type of strings") + .doc_comment(Some("This is sequence type of strings".to_string())) .sequence_type(SequenceType::List) .source(anonymous_type(vec![ type_constraint(named_type_ref("list")), diff --git a/src/bin/ion/commands/generate/result.rs b/src/bin/ion/commands/generate/result.rs index d18dd19..8c6dcd6 100644 --- a/src/bin/ion/commands/generate/result.rs +++ b/src/bin/ion/commands/generate/result.rs @@ -1,5 +1,6 @@ use crate::commands::generate::model::{ ScalarBuilderError, SequenceBuilderError, StructureBuilderError, WrappedScalarBuilderError, + WrappedSequenceBuilderError, }; use ion_schema::result::IonSchemaError; use thiserror::Error; @@ -71,6 +72,14 @@ impl From for CodeGenError { } } +impl From for CodeGenError { + fn from(value: WrappedSequenceBuilderError) -> Self { + CodeGenError::DataModelBuilderError { + description: value.to_string(), + } + } +} + impl From for CodeGenError { fn from(value: StructureBuilderError) -> Self { CodeGenError::DataModelBuilderError { diff --git a/src/bin/ion/commands/generate/templates/java/nested_type.templ b/src/bin/ion/commands/generate/templates/java/nested_type.templ index 299a608..d59c015 100644 --- a/src/bin/ion/commands/generate/templates/java/nested_type.templ +++ b/src/bin/ion/commands/generate/templates/java/nested_type.templ @@ -49,7 +49,7 @@ {{ sequence_info["element_type"] | fully_qualified_type_name }} value {% elif inline_type.code_gen_type is containing("Scalar") %} {% set scalar_info = model.code_gen_type["WrappedScalar"] %} - {% set base_type = scalar_info["name"]["parameters"][0] | fully_qualified_type_name %} + {% set base_type = scalar_info["base_type"] | fully_qualified_type_name %} {{ base_type }} value {% endif %} {% endmacro %} \ No newline at end of file diff --git a/src/bin/ion/commands/generate/templates/java/sequence.templ b/src/bin/ion/commands/generate/templates/java/sequence.templ index 0f4c8a7..1537830 100644 --- a/src/bin/ion/commands/generate/templates/java/sequence.templ +++ b/src/bin/ion/commands/generate/templates/java/sequence.templ @@ -1,81 +1,83 @@ -package {{ namespace }}; -import java.util.ArrayList; +{% macro sequence(model) %} + +{% if is_nested == false %} +{% set full_namespace = namespace | join(sep=".") %} + +package {{ full_namespace }}; import com.amazon.ion.IonReader; import com.amazon.ion.IonException; import com.amazon.ion.IonWriter; import com.amazon.ion.IonType; import java.io.IOException; +{% endif %} + +{# Verify that the abstract data type is a sequence type and store information for this sequence value #} +{% set sequence_info = model.code_gen_type["WrappedSequence"] %} -public class {{ target_kind_name }} { - private {{ fields[0].value_type }} value; +class {{ model.name }} { + private java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value; - public {{ target_kind_name }}() {} + public {{ model.name }}() {} - public {{ fields[0].value_type }} getValue() { + public java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> getValue() { return this.value; } - public void setValue({{ fields[0].value_type }} value) { + public void setValue(java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value) { this.value = value; return; } /** - * Reads a {{ target_kind_name }} from an {@link IonReader}. + * Reads a {{ model.name }} from an {@link IonReader}. * * This method does not advance the reader at the current level. * The caller is responsible for positioning the reader on the value to read. */ - public static {{ target_kind_name }} readFrom(IonReader reader) { + public static {{ model.name }} readFrom(IonReader reader) { {# Initializes all the fields of this class #} - {{ fields[0].value_type }} value = - {% if fields[0].value_type == "boolean" %} - false - {% elif fields[0].value_type == "int" or fields[0].value_type == "double" %} - 0 - {% else %} - null - {% endif %}; + java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}> value = new java.util.ArrayList<{{ sequence_info["element_type"] | fully_qualified_type_name }}>(); {# Reads `Sequence` class with a single field `value` that is an `ArrayList` #} - if(reader.getType() != IonType.{{ abstract_data_type["Sequence"].sequence_type | upper }}) { - throw new IonException("Expected {{ abstract_data_type["Sequence"].sequence_type }}, found " + reader.getType() + " while reading {{ fields[0].name | camel }}."); + if(reader.getType() != IonType.{{ sequence_info["sequence_type"] | upper }}) { + throw new IonException("Expected {{ sequence_info["sequence_type"] }}, found " + reader.getType() + " while reading value."); } reader.stepIn(); - value = new {{ fields[0].value_type }}(); - {# Iterate through the `ArrayList` and read each element in it based on the data type provided in `abstract_data_type[Sequence]` #} + {# Iterate through the `ArrayList` and read each element in it based on the data type provided in `sequence_info["sequence_type"]` #} while (reader.hasNext()) { reader.next(); - {% if abstract_data_type["Sequence"].element_type | is_built_in_type == false %} - value.add({{ abstract_data_type["Sequence"].element_type }}.readFrom(reader)); - {% elif abstract_data_type["Sequence"].element_type == "bytes[]" %} + {% if sequence_info["element_type"] |fully_qualified_type_name | is_built_in_type == false %} + value.add({{ sequence_info["element_type"] | fully_qualified_type_name }}.readFrom(reader)); + {% elif sequence_info["element_type"] | fully_qualified_type_name == "bytes[]" %} value.add(reader.newBytes()); {% else %} - value.add(reader.{{ abstract_data_type["Sequence"].element_type | camel }}Value()); + value.add(reader.{{ sequence_info["element_type"] | fully_qualified_type_name | camel }}Value()); {% endif %} } reader.stepOut(); - {{ target_kind_name }} {{ target_kind_name | camel }} = new {{ target_kind_name }}(); - {{ target_kind_name | camel }}.value = value; + {{ model.name }} {{ model.name | camel }} = new {{ model.name }}(); + {{ model.name | camel }}.value = value; - return {{ target_kind_name | camel }}; + return {{ model.name | camel }}; } /** - * Writes a {{ target_kind_name }} as Ion from an {@link IonWriter}. + * Writes a {{ model.name }} as Ion from an {@link IonWriter}. * * This method does not close the writer after writing is complete. * The caller is responsible for closing the stream associated with the writer. */ public void writeTo(IonWriter writer) throws IOException { {# Writes `Sequence` class with a single field `value` that is an `ArrayList` as an Ion sequence #} - writer.stepIn(IonType.{{ abstract_data_type["Sequence"].sequence_type | upper }}); - for ({{ abstract_data_type["Sequence"].element_type }} value: this.value) { - {% if abstract_data_type["Sequence"].element_type | is_built_in_type == false %} + writer.stepIn(IonType.{{ sequence_info["sequence_type"] | upper }}); + for ({{ sequence_info["element_type"] | fully_qualified_type_name }} value: this.value) { + {% if sequence_info["element_type"] | fully_qualified_type_name | is_built_in_type == false %} value.writeTo(writer); {% else %} - writer.write{{ abstract_data_type["Sequence"].element_type | upper_camel }}(value); + writer.write{{ sequence_info["element_type"] | fully_qualified_type_name | replace(from="double", to="float") | replace(from="boolean", to="bool") | upper_camel }}(value); {% endif %} } writer.stepOut(); } } +{% endmacro %} +{{ self::sequence(model=model) }} \ No newline at end of file diff --git a/src/bin/ion/commands/generate/utils.rs b/src/bin/ion/commands/generate/utils.rs index dca7bf4..9364768 100644 --- a/src/bin/ion/commands/generate/utils.rs +++ b/src/bin/ion/commands/generate/utils.rs @@ -245,7 +245,9 @@ impl TryFrom<&DataModelNode> for Template { AbstractDataType::Scalar(_) | AbstractDataType::WrappedScalar(_) => { Ok(Template::Scalar) } - AbstractDataType::Sequence(_) => Ok(Template::Sequence), + AbstractDataType::Sequence(_) | AbstractDataType::WrappedSequence(_) => { + Ok(Template::Sequence) + } AbstractDataType::Structure(_) => Ok(Template::Struct), } } else {