pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>
    <groupId>com.bigdatapassion</groupId>
    <artifactId>big-data-training</artifactId>
    <version>1.0-SNAPSHOT</version>
    <packaging>pom</packaging>
    <name>Big Data Training</name>

    <properties>

        <!--HDP 3.1.0 & HDF 3.3.1-->
        <!--https://docs.hortonworks.com/HDPDocuments/HDP3/HDP-3.1.0/release-notes/content/comp_versions.html-->
        <!--https://docs.hortonworks.com/HDPDocuments/HDF3/HDF-3.1.1/bk_release-notes/content/ch_hdf_relnotes.html-->
        <hadoop.version>[3.1.1]</hadoop.version>
        <spark.version>[2.4.0]</spark.version>
        <kafka.version>[3.3.1]</kafka.version>
        <hive.version>[3.1.0]</hive.version>
        <pig.version>[0.16.0]</pig.version>
        <hbase.version>[2.4.12]</hbase.version>
        <oozie.version>[4.3.1]</oozie.version>
        <nifi.version>[1.5.0]</nifi.version>

        <!-- Confluent Platform-->
        <confluent.version>[7.2.2]</confluent.version>

        <!--Other-->
        <phoenix.version>5.1.2</phoenix.version>
        <datanucleus.version>5.1.0-release</datanucleus.version>
        <mrunit.version>1.1.0</mrunit.version>
        <mahout.version>0.9</mahout.version>
        <avro.version>1.11.1</avro.version>
        <cascading.version>3.0.0-wip-60</cascading.version>

        <!--Other-->
        <junit.version>4.10</junit.version>
        <assertj.version>3.15.0</assertj.version>
        <log4j.version>2.11.0</log4j.version>
        <slf4j.version>1.7.25</slf4j.version>
        <jackson.version>2.10.0.pr1</jackson.version>

        <!-- Java -->
        <java.version>11</java.version>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>

        <!--Scala-->
        <scala.version>[2.12.7]</scala.version>
        <scala.binary.version>2.12</scala.binary.version>

        <!--Groovy-->
        <groovy.version>2.4.13</groovy.version>

        <!-- UTF-8 -->
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

        <!--Maven-->
        <maven-assembly-plugin.version>3.1.0</maven-assembly-plugin.version>
        <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
        <versions-maven-plugin.version>2.5</versions-maven-plugin.version>

    </properties>

    <dependencyManagement>
        <dependencies>

            <!-- Scala -->
            <dependency>
                <groupId>org.scala-lang</groupId>
                <artifactId>scala-library</artifactId>
                <version>${scala.version}</version>
            </dependency>

            <!--Groovy-->
            <dependency>
                <groupId>org.codehaus.groovy</groupId>
                <artifactId>groovy-all</artifactId>
                <version>${groovy.version}</version>
            </dependency>

            <!-- Hadoop -->
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-auth</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>${hadoop.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>${hadoop.version}</version>
            </dependency>

            <!-- Hive-->
            <dependency>
                <groupId>org.apache.hive</groupId>
                <artifactId>hive-exec</artifactId>
                <version>${hive.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hive</groupId>
                <artifactId>hive-jdbc</artifactId>
                <version>${hive.version}</version>
            </dependency>

            <!-- Pig -->
            <dependency>
                <groupId>org.apache.pig</groupId>
                <artifactId>pig</artifactId>
                <version>${pig.version}</version>
                <classifier>h2</classifier>
            </dependency>

            <!--Oozie-->
            <dependency>
                <groupId>org.apache.oozie</groupId>
                <artifactId>oozie-client</artifactId>
                <version>${oozie.version}</version>
            </dependency>

            <!-- Hbase -->
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-client</artifactId>
                <version>${hbase.version}</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.apache.hadoop</groupId>
                        <artifactId>hadoop-common</artifactId>
                    </exclusion>
                    <exclusion>
                        <groupId>org.apache.hadoop</groupId>
                        <artifactId>hadoop-client</artifactId>
                    </exclusion>
                    <exclusion>
                        <groupId>org.apache.hadoop</groupId>
                        <artifactId>hadoop-mapreduce-client-core</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-common</artifactId>
                <version>${hbase.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-mapreduce</artifactId>
                <version>${hbase.version}</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.glassfish</groupId>
                        <artifactId>javax.el</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>
            <dependency>
                <groupId>org.apache.phoenix</groupId>
                <artifactId>phoenix-core</artifactId>
                <version>${phoenix.version}</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.glassfish</groupId>
                        <artifactId>javax.el</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>

            <!--Kundera for HBase-->
            <dependency>
                <groupId>com.impetus.kundera.client</groupId>
                <artifactId>kundera-hbase-v2</artifactId>
                <version>3.8</version>
            </dependency>

            <!--JDO for HBase-->
            <dependency>
                <groupId>org.datanucleus</groupId>
                <artifactId>datanucleus-core</artifactId>
                <version>${datanucleus.version}</version>
            </dependency>
            <dependency>
                <groupId>org.datanucleus</groupId>
                <artifactId>datanucleus-hbase</artifactId>
                <version>${datanucleus.version}</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.apache.hbase</groupId>
                        <artifactId>hbase</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>
            <dependency>
                <groupId>org.datanucleus</groupId>
                <artifactId>datanucleus-api-jdo</artifactId>
                <version>${datanucleus.version}</version>
            </dependency>

            <!-- Javax Validation -->
            <dependency>
                <groupId>javax.validation</groupId>
                <artifactId>validation-api</artifactId>
                <version>2.0.1.Final</version>
            </dependency>

            <!--JDO-->
            <dependency>
                <groupId>javax.jdo</groupId>
                <artifactId>jdo-api</artifactId>
                <version>3.1</version>
            </dependency>

            <!-- Cascading -->
            <dependency>
                <groupId>cascading</groupId>
                <artifactId>cascading-local</artifactId>
                <version>${cascading.version}</version>
            </dependency>
            <dependency>
                <groupId>cascading</groupId>
                <artifactId>cascading-hadoop2-mr1</artifactId>
                <version>${cascading.version}</version>
            </dependency>

            <!--Mahout-->
            <dependency>
                <groupId>org.apache.mahout</groupId>
                <artifactId>mahout-core</artifactId>
                <version>${mahout.version}</version>
            </dependency>

            <!--Spark-->
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-core_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-sql_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-streaming_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-mllib_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-hive_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-sql-kafka-0-10_${scala.binary.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>

            <!--Kafka-->
            <dependency>
                <groupId>org.apache.kafka</groupId>
                <artifactId>kafka_${scala.binary.version}</artifactId>
                <version>${kafka.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.kafka</groupId>
                <artifactId>kafka-clients</artifactId>
                <version>${kafka.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.kafka</groupId>
                <artifactId>kafka-streams</artifactId>
                <version>${kafka.version}</version>
            </dependency>

            <!-- Confluent -->
            <dependency>
                <groupId>io.confluent</groupId>
                <artifactId>kafka-avro-serializer</artifactId>
                <version>${confluent.version}</version>
            </dependency>
            <dependency>
                <groupId>io.confluent</groupId>
                <artifactId>kafka-json-serializer</artifactId>
                <version>${confluent.version}</version>
            </dependency>
            <dependency>
                <groupId>io.confluent</groupId>
                <artifactId>kafka-json-schema-serializer</artifactId>
                <version>${confluent.version}</version>
            </dependency>
            <dependency>
                <groupId>io.confluent</groupId>
                <artifactId>kafka-json-schema-provider</artifactId>
                <version>${confluent.version}</version>
            </dependency>
            <dependency>
                <groupId>io.confluent</groupId>
                <artifactId>kafka-streams-avro-serde</artifactId>
                <version>${confluent.version}</version>
            </dependency>

            <!--NiFi-->
            <dependency>
                <groupId>org.apache.nifi</groupId>
                <artifactId>nifi-api</artifactId>
                <version>${nifi.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.nifi</groupId>
                <artifactId>nifi-utils</artifactId>
                <version>${nifi.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.nifi</groupId>
                <artifactId>nifi-processor-utils</artifactId>
                <version>${nifi.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.nifi</groupId>
                <artifactId>nifi-framework-core</artifactId>
                <version>${nifi.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.nifi</groupId>
                <artifactId>nifi-mock</artifactId>
                <version>${nifi.version}</version>
                <scope>test</scope>
            </dependency>

            <!--Vavr for Java-->
            <dependency>
                <groupId>io.vavr</groupId>
                <artifactId>vavr</artifactId>
                <version>0.9.2</version>
            </dependency>

            <!-- JSON -->
            <dependency>
                <groupId>org.json</groupId>
                <artifactId>json</artifactId>
                <version>20220924</version>
            </dependency>

            <!-- Avro -->
            <dependency>
                <groupId>org.apache.avro</groupId>
                <artifactId>avro</artifactId>
                <version>${avro.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.avro</groupId>
                <artifactId>avro-mapred</artifactId>
                <version>${avro.version}</version>
            </dependency>

            <!-- Apache Commons-->
            <dependency>
                <groupId>commons-io</groupId>
                <artifactId>commons-io</artifactId>
                <version>2.6</version>
            </dependency>
            <dependency>
                <groupId>org.apache.commons</groupId>
                <artifactId>commons-math3</artifactId>
                <version>3.6.1</version>
            </dependency>
            <dependency>
                <groupId>org.apache.commons</groupId>
                <artifactId>commons-lang3</artifactId>
                <version>3.9</version>
            </dependency>

            <!-- Google Guava -->
            <dependency>
                <groupId>com.google.guava</groupId>
                <artifactId>guava</artifactId>
                <version>29.0-jre</version>
            </dependency>

            <!--Jackson-->
            <dependency>
                <groupId>com.fasterxml.jackson.core</groupId>
                <artifactId>jackson-core</artifactId>
                <version>${jackson.version}</version>
            </dependency>
            <dependency>
                <groupId>com.fasterxml.jackson.core</groupId>
                <artifactId>jackson-databind</artifactId>
                <version>${jackson.version}</version>
            </dependency>
            <dependency>
                <groupId>com.fasterxml.jackson.core</groupId>
                <artifactId>jackson-annotations</artifactId>
                <version>${jackson.version}</version>
            </dependency>
            <dependency>
                <groupId>com.fasterxml.jackson.dataformat</groupId>
                <artifactId>jackson-dataformat-avro</artifactId>
                <version>2.13.0</version>
            </dependency>
            <dependency>
                <groupId>com.fasterxml.jackson.module</groupId>
                <artifactId>jackson-module-jsonSchema</artifactId>
                <version>2.13.0</version>
            </dependency>
            <dependency>
                <groupId>com.kjetland</groupId>
                <artifactId>mbknor-jackson-jsonschema_${scala.binary.version}</artifactId>
                <version>1.0.39</version>
            </dependency>


            <!-- Logging -->
            <dependency>
                <groupId>org.apache.logging.log4j</groupId>
                <artifactId>log4j-core</artifactId>
                <version>${log4j.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.logging.log4j</groupId>
                <artifactId>log4j-api</artifactId>
                <version>${log4j.version}</version>
            </dependency>
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-api</artifactId>
                <version>${slf4j.version}</version>
            </dependency>
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
                <version>${slf4j.version}</version>
            </dependency>

            <!-- Tests -->
            <dependency>
                <groupId>org.apache.mrunit</groupId>
                <artifactId>mrunit</artifactId>
                <version>${mrunit.version}</version>
                <classifier>hadoop2</classifier>
                <scope>test</scope>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-minicluster</artifactId>
                <version>${hadoop.version}</version>
                <scope>test</scope>
            </dependency>
            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>${junit.version}</version>
                <scope>test</scope>
            </dependency>
            <dependency>
                <groupId>org.assertj</groupId>
                <artifactId>assertj-core</artifactId>
                <version>${assertj.version}</version>
                <scope>test</scope>
            </dependency>
            <dependency>
                <groupId>org.apache.pig</groupId>
                <artifactId>pigunit</artifactId>
                <version>${pig.version}</version>
                <scope>test</scope>
            </dependency>

        </dependencies>
    </dependencyManagement>

    <distributionManagement>
        <repository>
            <id>radek-maven-repo-release</id>
            <name>AWS S3 Release Repository</name>
            <url>s3://radek-maven-repo/release</url>
        </repository>
        <snapshotRepository>
            <id>radek-maven-repo-snapshot</id>
            <name>AWS S3 Snapshot Repository</name>
            <url>s3://radek-maven-repo/snapshot</url>
        </snapshotRepository>
    </distributionManagement>

    <build>

        <resources>
            <resource>
                <directory>src/main/resources</directory>
                <filtering>true</filtering>
            </resource>
            <resource>
                <directory>src/test/resources</directory>
                <filtering>true</filtering>
            </resource>
        </resources>

        <extensions>
            <extension>
                <groupId>org.springframework.build</groupId>
                <artifactId>aws-maven</artifactId>
                <version>5.0.0.RELEASE</version>
            </extension>
        </extensions>

        <pluginManagement>
            <plugins>

                <plugin>
                    <groupId>org.apache.avro</groupId>
                    <artifactId>avro-maven-plugin</artifactId>
                    <version>${avro.version}</version>
                    <executions>
                        <execution>
                            <phase>generate-sources</phase>
                            <goals>
                                <goal>schema</goal>
                                <goal>protocol</goal>
                                <goal>idl-protocol</goal>
                            </goals>
                            <configuration>
                                <sourceDirectory>${project.basedir}/src/main/resources/avro</sourceDirectory>
                                <stringType>String</stringType>
                                <createSetters>true</createSetters>
                                <enableDecimalLogicalType>true</enableDecimalLogicalType>
                                <fieldVisibility>private</fieldVisibility>
                            </configuration>
                        </execution>
                    </executions>
                </plugin>

                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-jar-plugin</artifactId>
                    <version>3.0.2</version>
                </plugin>

                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-compiler-plugin</artifactId>
                    <version>3.7.0</version>
                    <configuration>
                        <source>${maven.compiler.source}</source>
                        <target>${maven.compiler.target}</target>
                    </configuration>
                </plugin>

                <!--Scala-->
                <plugin>
                    <groupId>net.alchim31.maven</groupId>
                    <artifactId>scala-maven-plugin</artifactId>
                    <version>${scala-maven-plugin.version}</version>
                    <configuration>
                        <source>${maven.compiler.source}</source>
                        <target>${maven.compiler.target}</target>
                    </configuration>
                    <executions>
                        <execution>
                            <id>scala-compile-first</id>
                            <phase>process-resources</phase>
                            <goals>
                                <goal>add-source</goal>
                                <goal>compile</goal>
                            </goals>
                        </execution>
                        <execution>
                            <id>scala-test-compile</id>
                            <phase>process-test-resources</phase>
                            <goals>
                                <goal>testCompile</goal>
                            </goals>
                        </execution>
                    </executions>
                </plugin>

                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-assembly-plugin</artifactId>
                    <version>${maven-assembly-plugin.version}</version>
                    <configuration>
                        <descriptorRefs>
                            <descriptorRef>jar-with-dependencies</descriptorRef>
                        </descriptorRefs>
                    </configuration>
                    <executions>
                        <execution>
                            <id>make-assembly</id>
                            <phase>package</phase>
                            <goals>
                                <goal>single</goal>
                            </goals>
                        </execution>
                    </executions>
                </plugin>

                <!--Maven versions plugin-->
                <plugin>
                    <groupId>org.codehaus.mojo</groupId>
                    <artifactId>versions-maven-plugin</artifactId>
                    <version>${versions-maven-plugin.version}</version>
                </plugin>

            </plugins>
        </pluginManagement>

        <plugins>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>versions-maven-plugin</artifactId>
                <version>${versions-maven-plugin.version}</version>
            </plugin>
        </plugins>

    </build>

    <profiles>

        <profile>
            <id>sandbox</id>
            <properties>
                <fs.defaultFS>hdfs://sandbox.hortonworks.com:8020</fs.defaultFS>
                <hbase.zookeeper.quorum>sandbox.hortonworks.com</hbase.zookeeper.quorum>
                <zookeeper.znode.parent>/hbase-unsecure</zookeeper.znode.parent>
                <phoenix.connection>jdbc:phoenix:localhost:/hbase-unsecure</phoenix.connection>
            </properties>
            <activation>
                <activeByDefault>true</activeByDefault>
            </activation>
        </profile>

        <profile>
            <id>hdp</id>
            <properties>
                <fs.defaultFS>hdfs://hdp1:8020</fs.defaultFS>
                <hbase.zookeeper.quorum>hdp1.hortonworks.com,hdp2.hortonworks.com,hdp3.hortonworks.com
                </hbase.zookeeper.quorum>
                <zookeeper.znode.parent>/hbase-unsecure</zookeeper.znode.parent>
                <phoenix.connection>jdbc:phoenix:hdp3:/hbase-unsecure</phoenix.connection>
            </properties>
            <activation>
                <activeByDefault>false</activeByDefault>
            </activation>
        </profile>

        <profile>
            <id>hdpoc</id>
            <properties>
                <fs.defaultFS>hdfs://hdpoc1:8020</fs.defaultFS>
                <hbase.zookeeper.quorum>hdpoc1.hortonworks.com,hdpoc2.hortonworks.com,hdpoc3.hortonworks.com
                </hbase.zookeeper.quorum>
                <zookeeper.znode.parent>/hbase-unsecure</zookeeper.znode.parent>
                <phoenix.connection>jdbc:phoenix:hdpoc3:/hbase-unsecure</phoenix.connection>
            </properties>
            <activation>
                <activeByDefault>false</activeByDefault>
            </activation>
        </profile>

        <profile>
            <id>aws</id>
            <properties>
                <fs.defaultFS>hdfs://aws1:8020</fs.defaultFS>
                <hbase.zookeeper.quorum>ec2-34-200-219-175.compute-1.amazonaws.com</hbase.zookeeper.quorum>
                <zookeeper.znode.parent>/hbase</zookeeper.znode.parent>
                <phoenix.connection>jdbc:phoenix:aws3:/hbase-unsecure</phoenix.connection>
            </properties>
            <activation>
                <activeByDefault>false</activeByDefault>
            </activation>
        </profile>

    </profiles>

    <repositories>

        <repository>
            <id>central</id>
            <url>https://repo1.maven.org/maven2/</url>
        </repository>

        <repository>
            <id>apache</id>
            <name>Apache Releases Repository</name>
            <url>https://repository.apache.org/content/repositories/releases/</url>
        </repository>

        <!--        <repository>-->
        <!--            <id>hortonworks-releases</id>-->
        <!--            <name>Hortonworks Releases Repository</name>-->
        <!--            <url>http://repo.hortonworks.com/content/repositories/releases/</url>-->
        <!--        </repository>-->

        <!--        <repository>-->
        <!--            <id>hortonworks-snapshots</id>-->
        <!--            <name>Hortonworks Snapshots Repository</name>-->
        <!--            <url>http://repo.hortonworks.com/content/repositories/snapshots/</url>-->
        <!--        </repository>-->

        <repository>
            <id>confluent</id>
            <url>https://packages.confluent.io/maven/</url>
        </repository>

        <!--Cascading repository-->
        <repository>
            <id>conjars.org</id>
            <url>http://conjars.org/repo</url>
        </repository>

    </repositories>

</project>