Support for writing Ion Binary

build.gradle

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

config/checkstyle/checkstyle.xml

@@ -5,7 +5,7 @@
   ~ You may not use this file except in compliance with the License.
   ~ A copy of the License is located at:
   ~
-  ~     http://aws.amazon.com/apache2.0/
+  ~      http://aws.amazon.com/apache2.0/
   ~
   ~ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
   ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
@@ -41,6 +41,11 @@
     <property name="eachLine" value="true"/>
   </module>
 
+  <module name="Header">
+    <property name="headerFile" value="config/checkstyle/copyright-header" />
+    <property name="fileExtensions" value="java,kt" />
+  </module>
+
   <module name="TreeWalker">
     <module name="SuppressionCommentFilter" /> <!-- allows for CHECKSTYLE:OFF comments -->
     <module name="OuterTypeFilename"/>

config/checkstyle/copyright-header

@@ -0,0 +1,13 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at:
+ *
+ *      http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+ * language governing permissions and limitations under the License.
+ *
+ */

config/intellij/codestyle.xml

@@ -4,7 +4,7 @@
   ~ You may not use this file except in compliance with the License.
   ~ A copy of the License is located at:
   ~
-  ~     http://aws.amazon.com/apache2.0/
+  ~      http://aws.amazon.com/apache2.0/
   ~
   ~ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
   ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/build.gradle

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/docker/conf/beeline-log4j2.properties

@@ -3,13 +3,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License").
 # You may not use this file except in compliance with the License.
 # A copy of the License is located at:
-#  
-#     http://aws.amazon.com/apache2.0/
-#  
+#
+#      http://aws.amazon.com/apache2.0/
+#
 # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-#  
+#
 #
 
 status = INFO

integration-test/docker/conf/hive-exec-log4j2.properties

@@ -3,13 +3,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License").
 # You may not use this file except in compliance with the License.
 # A copy of the License is located at:
-#  
-#     http://aws.amazon.com/apache2.0/
-#  
+#
+#      http://aws.amazon.com/apache2.0/
+#
 # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-#  
+#
 #
 
 status = INFO

integration-test/docker/conf/hive-log4j2.properties

@@ -3,13 +3,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License").
 # You may not use this file except in compliance with the License.
 # A copy of the License is located at:
-#  
-#     http://aws.amazon.com/apache2.0/
-#  
+#
+#      http://aws.amazon.com/apache2.0/
+#
 # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-#  
+#
 #
 
 status = INFO

integration-test/docker/conf/hive-site.xml

@@ -7,7 +7,7 @@
   ~ You may not use this file except in compliance with the License.
   ~ A copy of the License is located at:
   ~
-  ~     http://aws.amazon.com/apache2.0/
+  ~      http://aws.amazon.com/apache2.0/
   ~
   ~ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
   ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/docker/conf/ivysettings.xml

@@ -4,7 +4,7 @@
   ~ You may not use this file except in compliance with the License.
   ~ A copy of the License is located at:
   ~
-  ~     http://aws.amazon.com/apache2.0/
+  ~      http://aws.amazon.com/apache2.0/
   ~
   ~ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
   ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/docker/conf/llap-daemon-log4j2.properties

@@ -3,13 +3,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License").
 # You may not use this file except in compliance with the License.
 # A copy of the License is located at:
-#  
-#     http://aws.amazon.com/apache2.0/
-#  
+#
+#      http://aws.amazon.com/apache2.0/
+#
 # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-#  
+#
 #
 
 status = INFO

integration-test/src/test/java/software/amazon/ionhiveserde/integrationtest/AllTestsSuite.java

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/src/test/java/software/amazon/ionhiveserde/integrationtest/Base.java

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
@@ -62,7 +62,7 @@ public static void afterClass() {
 
     /**
      * Drops all hive tables to ensure a clean metastore. This will not clean up HDFS, it only removes the Hive tables
-     * which are views on the HDFS test data
+     * which are views on the HDFS test data.
      */
     @Before
     public void before() {

integration-test/src/test/java/software/amazon/ionhiveserde/integrationtest/Lifecycle.java

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/src/test/java/software/amazon/ionhiveserde/integrationtest/TestLifecycle.java

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/src/test/java/software/amazon/ionhiveserde/integrationtest/TestMode.java

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/FileSystemUtil.kt

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
@@ -13,15 +13,35 @@
  */
 
 /**
- * Utility functions for dealing with the local file system
+ * Utility functions for dealing with the local file system.
  */
 @file:JvmName("FileSystemUtil")
 
 package software.amazon.ionhiveserde.integrationtest
 
 import java.io.File
 
+/**
+ * Creates the directory including parent directories if necessary.
+ *
+ * @param path path to create the directory.
+ * @return true if the directory was created, false otherwise.
+ */
 fun mkdir(path: String) = File(path).mkdirs()
+
+/**
+ * Removes path recursively.
+ *
+ * @param path path to be removed.
+ * @return true if the path was removed, false otherwise.
+ */
 fun rm(path: String) = File(path).deleteRecursively()
 
-fun String.sanitize() = this.replace(Regex("[<>:,()]"), "_")
+/**
+ * Sanitizes a String to be a valid path by replacing invalid sequences with the replacement string.
+ *
+ * @param replacement String to be used, defaults to `"_"`.
+ *
+ * @return sanitized String.
+ */
+fun String.sanitize(replacement: String = "_") = this.replace(Regex("[<>:,()]"), replacement)

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/IonUtil.kt

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
@@ -13,7 +13,7 @@
  */
 
 /**
- * Ion related utility methods to be used in the test scope
+ * Ion related utility methods to be used in the test scope.
  */
 @file:JvmName("IonUtil")
 
@@ -25,23 +25,23 @@ import software.amazon.ion.IonValue
 import software.amazon.ion.IonWriter
 import software.amazon.ion.system.IonSystemBuilder
 import java.io.File
-import java.io.FileInputStream
 import java.io.FileOutputStream
 
+/** Standard Ion system with no catalog. */
 val ION: IonSystem = IonSystemBuilder.standard().build()
 
 // Extensions
 
 /**
- * [IonSystem.singleValue] from a local path
+ * [IonSystem.singleValue] from a local path.
  */
 fun IonSystem.singleValueFromPath(path: String): IonValue = this.singleValue(File(path).readBytes())
 
-/** [IonSystem.newTextWriter] from the local path */
+/** [IonSystem.newTextWriter] from the local path. */
 fun IonSystem.newTextWriterFromPath(path: String): IonWriter = ION.newTextWriter(FileOutputStream(File(path)))
 
-/** [IonSystem.newBinaryWriterFromPath] from the local path */
+/** [IonSystem.newBinaryWriterFromPath] from the local path. */
 fun IonSystem.newBinaryWriterFromPath(path: String): IonWriter = ION.newBinaryWriter(FileOutputStream(File(path)))
 
-/** Struct keys as a sequence */
+/** Struct keys as a sequence. */
 fun IonStruct.keys(): Sequence<String> = this.asSequence().map { it.fieldName }

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/docker/DockerUtil.kt

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/docker/HDFS.kt

@@ -4,7 +4,7 @@
  * You may not use this file except in compliance with the License.
  * A copy of the License is located at:
  *
- *     http://aws.amazon.com/apache2.0/
+ *      http://aws.amazon.com/apache2.0/
  *
  * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
@@ -23,12 +23,23 @@ package software.amazon.ionhiveserde.integrationtest.docker
  * See https://github.com/amzn/ion-hive-serde/issues/37 for more info
  */
 object HDFS {
+
+    /**
+     * Puts all data in path to HDFS /data directory
+     *
+     * @param path path relative to the shared directory
+     */
     @JvmStatic
     fun put(path: String) {
         runInDocker("hadoop fs -mkdir -p /data")
         runInDocker("hadoop fs -put -f /$SHARED_DIR/$path /data")
     }
 
+    /**
+     * Removes data from HDFS
+     *
+     * @param path HDFS absolute path
+     */
     @JvmStatic
     fun rm(path: String) = runInDocker("hadoop fs -rm -r -f $path")
 }