Adding preliminary support for converting json schema to apache parquet schema.

.gitignore

@@ -97,3 +97,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# vscode
+.vscode/

aptos/__main__.py

@@ -7,7 +7,8 @@
 from .parser import SchemaParser
 from .primitive import Object
 from .visitor import ValidationVisitor
-from .schema.visitor import AvroSchemaVisitor
+from .schema.visitor import AvroSchemaVisitor, ParquetSchemaVisitor
+from .jinja_helper import JinjaHelper
 
 
 def validate(arguments):
@@ -30,8 +31,18 @@ def convert(arguments):
         sys.exit(colored('error', 'red') + ' cannot convert schema {!r} into {!r} format, schema must be of type "object"'.format(arguments.schema, arguments.format))  # noqa: E501
     Visitor = {
         'avro': AvroSchemaVisitor,
+        'parquet': ParquetSchemaVisitor
     }[arguments.format]
-    print(json.dumps(component.accept(Visitor()), indent=2))
+    converted_schema = component.accept(Visitor())
+    print_schema(arguments.format, converted_schema)
+
+
+def print_schema(schema_arg, converted_schema):
+    if schema_arg == 'avro':
+        print(json.dumps(converted_schema, indent=2))
+    elif schema_arg == 'parquet':
+        parquet_schema = JinjaHelper.get_template('inner_schema.jinja2')
+        print(parquet_schema.render(body=converted_schema))
 
 
 def main():
@@ -57,7 +68,7 @@ def main():
         'convert', help='''
         Convert a JSON Schema into a different data-interchange format''')
     conversion.add_argument(
-        '-format', type=str, choices=['avro'], help='data-interchange format')
+        '-format', type=str, choices=['avro', 'parquet'], help='data-interchange format')
     conversion.set_defaults(func=convert)
 
     parser.add_argument(

aptos/jinja_helper.py

@@ -0,0 +1,16 @@
+from jinja2 import Environment, FileSystemLoader
+
+class JinjaHelper:
+
+    jinja_env = None
+
+    @classmethod
+    def create_jinja_env(cls):
+        file_loader = FileSystemLoader('templates')
+        JinjaHelper.jinja_env = Environment(loader=file_loader)
+
+    @staticmethod
+    def get_template(template_name):
+        if JinjaHelper.jinja_env is None:
+            JinjaHelper.create_jinja_env()
+        return JinjaHelper.jinja_env.get_template(template_name)

aptos/schema/visitor.py

@@ -1,4 +1,5 @@
 from ..primitive import Array, Object, Reference, Enumeration
+from ..jinja_helper import JinjaHelper
 
 
 class AvroSchemaVisitor:
@@ -54,3 +55,33 @@ def visit_reference(self, reference, *args):
 
     def visit_union(self, union, *args):
         return {'type': union.type}
+
+##-------------------------------------##
+
+class ParquetSchemaVisitor:
+
+    def determine_repetition(self, name, required_fields):
+        if name in required_fields:
+            return 'required'
+        else:
+            return 'optional'
+
+    def visit_string(self, string, *args):
+        return '{repetition} binary {name} (UTF8);\n'
+
+    def visit_integer(self, integer, *args):
+        return '{repetition} int32 {name};\n'
+
+    def visit_object(self, obj, *args):
+        fields = ''
+        for name, member in obj.properties.items():
+            placeholder_field = member.accept(self, *args)
+            if isinstance(member, (Array, Object, Reference, Enumeration)):
+                fields += placeholder_field
+                fields += '\n'
+            else:
+                field = placeholder_field.format(repetition=self.determine_repetition(name, obj.required), name=name)
+                fields += field
+        obj_exp = '{repetition} group {name}'
+        obj_name = obj_exp.format(repetition=self.determine_repetition(obj.title, obj.required), name=obj.title)
+        return JinjaHelper.get_template('object.jinja2').render(obj_name=obj_name, body=fields)

setup.py

@@ -4,6 +4,7 @@
     install_requires=[
         'colorama',
         'termcolor',
+        'jinja2'
     ],
     name='aptos',
     version='1.0.2',

templates/inner_schema.jinja2

@@ -0,0 +1,5 @@
+{% extends "schema.jinja2" -%}
+{% block schema_content -%}
+{% macro fix_indent() %}{{ body }}{% endmacro -%}
+{{ fix_indent()|indent(width=2,first=True) -}}
+{% endblock -%}

templates/macros.jinja2

@@ -0,0 +1 @@
+{% macro fix_indent(body) %}{{ body }}{% endmacro -%}

templates/object.jinja2

@@ -0,0 +1,5 @@
+{% import "macros.jinja2" as macros -%} 
+{{ obj_name }} {
+{% block object_content -%}
+{{ macros.fix_indent(body)|indent(width=2,first=True) -}}
+{% endblock -%}}

templates/schema.jinja2

@@ -0,0 +1,4 @@
+message ParquetSchema {
+{% block schema_content %}
+{% endblock %}
+}

tests/schema/person

@@ -0,0 +1,28 @@
+{
+    "title": "Person",
+    "type": "object",
+    "properties": {
+        "firstName": {
+            "type": "string"
+        },
+        "lastName": {
+            "type": "string"
+        },
+        "age": {
+            "type" : "integer"
+        },
+        "address": {
+            "type" : "object",
+            "title": "Address",
+            "properties": {
+                "street_name": {
+                    "type": "string"
+                },
+                "zipcode": {
+                    "type": "integer"
+                }
+            }
+        }
+    },
+    "required": [ "firstName", "age" ]
+}