reverted

Jenkinsfile

@@ -0,0 +1,62 @@
+pipeline {
+  agent any
+  stages {
+    stage('RUN JFROG CLI') {
+      steps {
+        sh '''rm -rf ~/.m2/repository/dk/ignalina/lab/
+
+
+'''
+      }
+    }
+
+    stage('Spark232 insert only') {
+      parallel {
+        stage('Spark232') {
+          steps {
+            sh '''export M2_HOME=/usr/share/java/maven-3
+
+
+
+cd spark232job-mod
+mvn clean install -U
+ls -l target
+cd ..
+
+
+#ssh [email protected] \'/home/hadoop/welder_load.sh\'
+
+'''
+          }
+        }
+
+        stage('Spark3.2.0') {
+          steps {
+            sh '''export M2_HOME=/usr/share/java/maven-3
+export JAVA_HOME=/usr/lib/jvm/default-jvm/
+
+
+
+cd spark320job-mod
+mvn clean install -U
+ls -l target
+
+
+scp target/welder-delta-job-1.0-SNAPSHOT.jar [email protected]:.
+ssh [email protected] \'/home/spark/welder_load_spark320.sh\'
+
+cd ..
+
+'''
+          }
+        }
+
+      }
+    }
+
+  }
+  environment {
+    JFROG_CLI_BUILD_NAME = 'JOB_NAME'
+    JFROG_CLI_BUILD_NUMBER = 'BUILD_NUMBER'
+  }
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 [email protected]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,29 @@
+# Welder
+Trying out different streaming jobs to read avro from kafka and output to targets like :
+*  Append only: hive/parquet  
+*  full sync: Iceberg/parquet.
+*  full sync: Delta
+*  full sync: hudi
+
+Goals with this is to learn howto read from kafka with multiple partitions/offsets and spread out the work to multiple workers.
+The name Welder is the opposite from the Shredder , since the welder it makes the data "whole again" (Hopefully)
+
+
+
+# Screenshots
+An producer called the Shredder is started  , reading fixed column sized datafiles (30 columns in this example , 2 gig per file).  
+
+Time spend in total     : 2.471591492s  parsing  148804290  lines from  2620609413  bytes  
+Troughput bytes/s total : 1011.17MB /s  
+Troughput lines/s total : 57.42M  Lines/s  
+Troughput lines/s toAvro: 4.27M  Lines/s  
+Time spent toReadChunks : 0.7911964744166666 s  
+Time spent toAvro       : 33.271536043333334 s  
+Time spent toKafka      : 19.226326717666666 s  
+Time spent DoneKafka      : 8.036e-06 s  
+
+![Screenshot](screenshots/spark_232_streaming_1.png)  
+![Screenshot](screenshots/spark_232_streaming_2.png)  
+![Screenshot](screenshots/spark_232_streaming_3.png)  
+![Screenshot](screenshots/spark_232_streaming_4.png)  
+

settings.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 http://maven.apache.org/xsd/settings-1.2.0.xsd" xmlns="http://maven.apache.org/SETTINGS/1.2.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+    <profiles>
+        <profile>
+            <id>myprofile</id>
+
+            <repositories>
+
+                <repository>
+                    <id>my-repo0</id>
+                    <name>Googles fast mirror</name>
+                    <url>https://maven-central-eu.storage-download.googleapis.com/</url>
+                </repository>
+
+                <repository>
+                    <id>my-repo1</id>
+                    <name>your custom repo</name>
+                    <url>https://repo1.maven.org/maven2</url>
+                </repository>
+
+                <repository>
+                    <id>my-repo2</id>
+                    <name>your custom repo</name>
+                    <url>https://maven.wso2.org/nexus/content/groups/wso2-public</url>
+                </repository>
+
+                <repository>
+                    <id>my-repo3</id>
+                    <name>your custom repo</name>
+                    <url>https://packages.confluent.io/maven/</url>
+                </repository>
+                <repository>
+                    <id>my-repo4</id>
+                    <name>your custom repo</name>
+                    <url>https://mvnrepository.com/artifact</url>                 
+               </repository>
+
+                  <repository>
+                    <id>my-repo5</id>
+                    <name>your custom repo</name>
+                    <url>https://jcenter.bintray.com</url>
+                </repository>
+
+                <repository>
+                    <id>my-repo6</id>
+                    <name>your custom repo</name>
+                    <url>https://repo.maven.apache.org/maven2</url>
+                </repository>
+
+
+
+
+
+
+            </repositories>
+
+
+
+        </profile>
+    </profiles>
+
+    <activeProfiles>
+        <activeProfile>myprofile</activeProfile>
+    </activeProfiles>
+
+</settings>