Lots of new features and bug fixes

Added computed fields
fixed issue in start workflow
Added multi thread file part uploader
Added buffered writer for bin file

.gitignore

@@ -1,6 +1,7 @@
 ## build output
 target/
 local-proj-repo/
+SalesEdgeEltWorkflow/
 
 ## eclipse files
 .project

README.md

@@ -10,15 +10,15 @@ Just download the latest jar from release section (see link at the top) and foll
 
 Input Parameter
 
---action  :“load” OR “augment” OR “downloadxmd” OR “uploadxmd” OR "detectEncoding". Use load for loading csv, augment for augmenting existing datasets, downloadxmd to download existing xmd files, uploadxmd for uploading user.xmd.json, “extract” for extracting data from salesforce, "detectEncoding" to detect the encoding of the inputFile.
+--action  :"load"  OR "augment"  OR "downloadxmd"  OR "uploadxmd"  OR "detectEncoding". Use load for loading csv, augment for augmenting existing datasets, downloadxmd to download existing xmd files, uploadxmd for uploading user.xmd.json, "extract"  for extracting data from salesforce, "detectEncoding" to detect the encoding of the inputFile.
 
 --u       : Salesforce.com login
 
 --p       : (Optional) Salesforce.com password,if omitted you will be prompted
 
 --token   : (Optional) Salesforce.com token
 
---endpoint: (Optional) The salesforce soap api endpoint (test/prod) Default: https://login.salesforce.com/services/Soap/u/31.0
+--endpoint: (Optional) The Salesforce soap api endpoint (test/prod) Default: https://login.salesforce.com/services/Soap/u/31.0
 
 --dataset : (Optional) the dataset alias. required if action=load
 
@@ -30,12 +30,15 @@ Input Parameter
 
 --rowLimit: (Optional) the number of rows to extract, -1=all, default=1000
 
---sessionId : (Optional) the salesforce sessionId. if specified,specify endpoint
+--sessionId : (Optional) the Salesforce sessionId. if specified,specify endpoint
 
 --fileEncoding : (Optional) the encoding of the inputFile default UTF-8
 
 --CodingErrorAction:(optional) What to do in case input characters are not UTF8: IGNORE|REPORT|REPLACE. Default REPORT. If you change this option you risk importing garbage characters
 
+--uploadFormat : (Optional) the whether to upload as binary or csv. default binary");
+
+
 
 ## Usage Example 1: Only Generate the schema file from CSV
     java -jar datasetutils-32.0.0.jar --action load --inputFile Opportunity.csv

pom.xml

@@ -101,6 +101,11 @@
 			<version>4.11</version>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+                <groupId>com.google.code.externalsortinginjava</groupId>
+                <artifactId>externalsortinginjava</artifactId>
+                <version>0.1.9</version>
+        </dependency>
 	</dependencies>
 	<build>
 		<defaultGoal>clean install</defaultGoal>
@@ -206,6 +211,7 @@
 					<header>license.txt</header>
 					<excludes>
 						<exclude>src/test/resources/codegeneration/*.java</exclude>
+						<exclude>src/main/java/com/foundations/**/*.java</exclude>
 						<exclude>**/*.html</exclude>
 						<exclude>**/*.txt</exclude>
 						<exclude>**/*.xml</exclude>

src/main/java/com/foundations/comparator/attributes/DateTimeSortAttributes.java

@@ -0,0 +1,29 @@
+package com.foundations.comparator.attributes;
+
+public final class DateTimeSortAttributes extends SortAttributes {
+
+	public static final String DEFAULT_PATTERN = "yyyy-MM-dd HH:mm:ss";
+
+	private String _pattern;
+
+	public DateTimeSortAttributes() {
+		_pattern = DEFAULT_PATTERN;
+	}
+
+	/**
+	 * The date and time pattern used for the column to be sorted. 
+	 *  
+	 * Pattern syntax is based on java.text.SimpleDateFormat
+	 * class documentation
+	 */
+	public void setPattern(String value) {
+		_pattern = value;
+	}
+
+	/**
+	 * Returns the date and time pattern for the column to be sorted.
+	 */
+	public String getPattern() {
+		return _pattern;
+	}
+}

src/main/java/com/foundations/comparator/attributes/DecimalSortAttributes.java

@@ -0,0 +1,33 @@
+package com.foundations.comparator.attributes;
+
+import java.math.RoundingMode;
+
+public final class DecimalSortAttributes extends SortAttributes {
+
+	public static final int DEFAULT_SCALE = 2;
+	public static final RoundingMode DEFAULT_ROUNDING_MODE = RoundingMode.HALF_EVEN;
+
+	private int _scale;
+	private RoundingMode _roundingMode;
+
+	public DecimalSortAttributes() {
+		_scale = DEFAULT_SCALE;
+		_roundingMode = DEFAULT_ROUNDING_MODE;
+	}
+
+	public void setScale(int value) {
+		_scale = value;
+	}
+
+	public int getScale() {
+		return _scale;
+	}
+
+	public void setRoundingMode(RoundingMode value) {
+		_roundingMode = value;
+	}
+
+	public RoundingMode getRoundingMode() {
+		return _roundingMode;
+	}
+ }

src/main/java/com/foundations/comparator/attributes/SortAttributes.java

@@ -0,0 +1,42 @@
+package com.foundations.comparator.attributes;
+
+public class SortAttributes {
+
+	public static final boolean DEFAULT_ASCENDING_ORDER = true;
+	public static final boolean DEFAULT_NULL_LOW_SORT_ORDER = true;
+	public static final boolean DEFAULT_TRIM = false;
+
+	private boolean _ascendingOrder;
+	private boolean _trim;	
+	private boolean _nullLowSortOrder;
+
+	public SortAttributes() {
+		_ascendingOrder = DEFAULT_ASCENDING_ORDER;
+		_trim = DEFAULT_TRIM;
+		_nullLowSortOrder = DEFAULT_NULL_LOW_SORT_ORDER;
+	}
+
+	public void setAscendingOrder(boolean value) {
+		_ascendingOrder = value;
+	}
+
+	public boolean isAscendingOrder() {
+		return _ascendingOrder;
+	}
+
+	public void setTrim(boolean value) {
+		_trim = value;
+	}
+
+	public boolean isTrim() {
+		return _trim;
+	}
+
+	public void setNullLowSortOrder(boolean value) {
+		_nullLowSortOrder = value;
+	}
+
+	public boolean isNullLowSortOrder() {
+		return _nullLowSortOrder;
+	}
+}

src/main/java/com/foundations/comparator/attributes/StringSortAttributes.java

@@ -0,0 +1,31 @@
+package com.foundations.comparator.attributes;
+
+public final class StringSortAttributes extends SortAttributes {
+
+	public static final boolean DEFAULT_CASE_SENSITIVE = true;
+	public static final boolean DEFAULT_STRIP_ACCENTS = false;
+
+	private boolean _caseSensitive;
+	private boolean _stripAccents;
+
+	public StringSortAttributes() {
+		_caseSensitive = DEFAULT_CASE_SENSITIVE;
+		_stripAccents = DEFAULT_STRIP_ACCENTS;
+	}
+
+	public void setCaseSensitive(boolean value) {
+		_caseSensitive = value;
+	}
+
+	public boolean isCaseSensitive() {
+		return _caseSensitive;
+	}
+
+	public void setStripAccents(boolean value) {
+		_stripAccents = value;		
+	}
+
+	public boolean isStripAccents() {
+		return _stripAccents;
+	}
+}

src/main/java/com/foundations/comparator/column/AbstractComparator.java

@@ -0,0 +1,97 @@
+package com.foundations.comparator.column;
+
+import com.foundations.comparator.attributes.SortAttributes;
+
+public abstract class AbstractComparator implements IColumnComparator {
+	private String _name;	
+	private int _sortOrder;
+	private SortAttributes _sortAttributes;
+
+	public AbstractComparator(String name, int sortOrder, SortAttributes sortAttributes) {
+		_name = name;
+		_sortOrder = sortOrder;
+		_sortAttributes = sortAttributes;
+	}
+
+	public String getName() {
+		return _name;
+	}
+
+	public int getSortOrder() {
+		return _sortOrder;
+	}
+
+	public SortAttributes getSortAttributes() {
+		return _sortAttributes;
+	}
+
+	/**
+	 * Returns a result of zero, greater than one, or less than one
+	 * depending on the comparison of the two supplied Strings<p>
+	 * 
+	 * A return value of zero indicates the two Strings are equal
+	 * A return value greater than one indicates String a is bigger than String b
+	 * A return value less than one indicates String a is less than String b
+	 * 
+	 * The first step in comparing the Strings involves swapping them if they are not
+	 * already in ascending order. 
+	 * 
+	 * Next, any required trimming is performed if the Trim attribute has been set.
+	 * The Strings are then normalized, ensuring zero-length Strings are treated as 
+	 * nulls.
+	 * 
+	 * If both Strings turn out to be null after normalization, zero is returned.
+	 * If one of the two Strings is null, the compare will consider the NullLowSortOrder 
+	 * attribute to determine the final result.
+	 * 
+	 * If both Strings are not null, sub-classes must determine the final result
+	 * of the compare by returning the value from a call to abstract method 
+	 * extendedCompare. 
+	 */
+	public int compare(String a, String b) {
+		int result = 0;
+		String stringA = normalizeString((_sortAttributes.isAscendingOrder() ? a : b));
+		String stringB = normalizeString((_sortAttributes.isAscendingOrder() ? b : a));
+
+		if( stringA != null && stringB != null ) {
+			result = extendedCompare(stringA, stringB);
+		}
+		else if( stringA == null && stringB == null ) {
+			result = 0;
+		}
+		else if ( stringA == null ) {
+			result = _sortAttributes.isNullLowSortOrder() ? -1 : 1;
+		}
+		else {    
+			result = _sortAttributes.isNullLowSortOrder() ? 1 : -1;			
+		}
+		return result;
+	}
+
+	/**
+	 * Normalize the String for sorting<p>
+	 * 
+	 * Normalizing involves transforming the original value so
+	 * that zero length Strings are treated as nulls. It also
+	 * involves stripping trailing and leading spaces from the 
+	 * original, provided the isTrim attribute is set.
+	 * 
+	 * @param original the String to be normalized
+	 * @return the normalized text
+	 */
+	private String normalizeString(String original) {
+		String result = null;
+
+		if( original != null ) {
+			if( _sortAttributes.isTrim() ) {
+				original = original.trim();
+			}
+			if( original.length() > 0 ) {
+				result = original;
+			}
+		}		
+		return result;
+	}
+
+	protected abstract int extendedCompare(String a, String b);
+}

src/main/java/com/foundations/comparator/column/BooleanComparator.java

@@ -0,0 +1,32 @@
+package com.foundations.comparator.column;
+
+import com.foundations.comparator.attributes.SortAttributes;
+
+public class BooleanComparator extends AbstractComparator {
+
+	public BooleanComparator(String name, int sortOrder, SortAttributes attributes) {
+		super(name, sortOrder, attributes);
+	}
+
+	protected int extendedCompare(String a, String b) {
+	    Boolean aValue = new Boolean(parse(a));
+		Boolean bValue = new Boolean(parse(b));
+
+		return aValue.compareTo(bValue);
+	}
+
+	private boolean parse(String value) {
+		boolean result = false;
+
+		if ( value.toLowerCase().equals("true") || value.equals("1") ) {
+			result = true;
+		}
+		else if ( value.toLowerCase().equals("false") || value.equals("0") ) {
+			result = false;
+		}
+		else {
+			throw new RuntimeException( "Boolean Parse Exception: " + value);
+		}
+		return result;
+	}
+}

src/main/java/com/foundations/comparator/column/DateTimeComparator.java

@@ -0,0 +1,55 @@
+package com.foundations.comparator.column;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import com.foundations.comparator.attributes.DateTimeSortAttributes;
+
+public final class DateTimeComparator extends AbstractComparator {
+
+	private SimpleDateFormat _formatter;
+
+	public DateTimeComparator(String name, int sortOrder, DateTimeSortAttributes sortAttributes) {
+		super(name, sortOrder, sortAttributes);
+		_formatter = new SimpleDateFormat(sortAttributes.getPattern());
+	}
+
+	protected int extendedCompare(String a, String b) {
+		int result;
+
+		try {
+			Date aValue = _formatter.parse(a);
+			Date bValue = _formatter.parse(b);	
+			result = aValue.compareTo(bValue);
+		} 
+		catch (ParseException e) {
+			throw new RuntimeException("Parse Exception: " + e.getMessage());
+		} 
+
+		return result;
+	}
+}
+
+
+//////////////////////////  USE FOLLOWING CODE FOR JAVA 8 ///////
+// import java.time.LocalDateTime;
+// import java.time.format.DateTimeFormatter;
+// 
+// public final class DateTimeComparator extends AbstractComparator {
+// 
+// 	private DateTimeFormatter _formatter;
+// 	
+// 	public DateTimeComparator(String name, int sortOrder, DateTimeSortAttributes sortAttributes) {
+// 		super(name, sortOrder, sortAttributes);
+// 		_formatter = DateTimeFormatter.ofPattern(sortAttributes.getPattern());
+// 	}
+// 
+// 	protected int extendedCompare(String a, String b) {
+// 		LocalDateTime aValue = LocalDateTime.parse(a, _formatter);  
+// 		LocalDateTime bValue = LocalDateTime.parse(b, _formatter);  
+// 		
+// 		return aValue.compareTo(bValue);
+// 	}
+// }
+//////////////////////////USE FOLLOWING CODE FOR JAVA 8 ///////