Mapping main synced 24 feb

common/client/src/main/java/zingg/common/client/Arguments.java

@@ -104,12 +104,12 @@
 		this.threshold = threshold;
 	}
 
 	public static final Log LOG = LogFactory.getLog(Arguments.class);
 
 	/**
 	 * default constructor Use setters for setting individual fields
 	 */
 	public Arguments() {
 	}
 
 	@Override
@@ -165,7 +165,7 @@
 	 */
 	@Override
 	public List<? extends FieldDefinition> getFieldDefinition() {
-		return fieldDefinition;
+		return this.fieldDefinition;
 	}
 
 	/**

common/client/src/main/java/zingg/common/client/FieldDefUtil.java

@@ -15,13 +15,13 @@ public class FieldDefUtil implements Serializable{
 
 	public List<? extends FieldDefinition> getFieldDefinitionDontUse(List<? extends FieldDefinition> fieldDefinition) {
 		return fieldDefinition.stream()
-			    .filter(x->x.matchType.contains(MatchType.DONT_USE))
+			    .filter(x->x.matchType.contains(MatchTypes.DONT_USE))
 			    .collect(Collectors.toList());
 	}
 
 	public List<? extends FieldDefinition> getFieldDefinitionToUse(List<? extends FieldDefinition> fieldDefinition) {
 		return fieldDefinition.stream()
-			    .filter(x->!x.matchType.contains(MatchType.DONT_USE))
+			    .filter(x->!x.matchType.contains(MatchTypes.DONT_USE))
 			    .collect(Collectors.toList());
 	}
 

common/client/src/main/java/zingg/common/client/FieldDefinition.java

@@ -31,16 +31,15 @@
  * @author sgoyal
  *
  */
-public class FieldDefinition implements Named,
-		Serializable {
+public class FieldDefinition implements Named, Serializable {
 
 	private static final long serialVersionUID = 1L;
 
 	public static final Log LOG = LogFactory.getLog(FieldDefinition.class);
 
 	@JsonDeserialize(using = MatchTypeDeserializer.class)
 	@JsonSerialize(using = MatchTypeSerializer.class)
-	public List<MatchType> matchType;
+	public List<? extends IMatchType> matchType;
 
 	//@JsonSerialize(using = DataTypeSerializer.class)
 	public String dataType;
@@ -52,17 +51,21 @@ public class FieldDefinition implements Named,
 	public FieldDefinition() {
 	}
 
-	public String getFields() { return fields; }
+	public String getFields() { 
+		return fields; 
+	}
 
-	public void setFields(String fields) { this.fields = fields;}	
+	public void setFields(String fields) {
+		this.fields = fields;
+	}	
 
 	/**
 	 * Get the field type of the class
 	 * 
 	 * @return the type
 	 */
-	public List<MatchType> getMatchType() {
-		return matchType;
+	public List<? extends IMatchType> getMatchType() {
+		return this.matchType;
 	}
 
 	/**
@@ -73,12 +76,12 @@ public List<MatchType> getMatchType() {
 	 *            the type to set
 	*/
 	@JsonDeserialize(using = MatchTypeDeserializer.class)	
-	public void setMatchType(List<MatchType> type) {
+	public void setMatchType(List<? extends IMatchType> type) {
 		this.matchType = type; //MatchTypeDeserializer.getMatchTypeFromString(type);
 	}
 
 
-	public void setMatchTypeInternal(MatchType... type) {
+	public void setMatchTypeInternal(IMatchType... type) {
 		this.matchType = Arrays.asList(type);
 	}
 
@@ -113,7 +116,7 @@ public void setAbbreviations(String abbreviations) {
 	}
 
 	public String getFieldName() {
-		return fieldName;
+		return this.fieldName;
 	}
 
 	public void setFieldName(String fieldName) {
@@ -122,7 +125,7 @@ public void setFieldName(String fieldName) {
 
 	@JsonIgnore
 	public boolean isDontUse() {
-        return (matchType != null && matchType.contains(MatchType.DONT_USE));
+        return (matchType != null && matchType.contains(MatchTypes.DONT_USE));
     }
 
 	@Override
@@ -185,17 +188,17 @@ public void serialize(DataType dType, JsonGenerator jsonGenerator,
 		}
 	}*/
 
-	public static class MatchTypeSerializer extends StdSerializer<List<MatchType>> {
+	public static class MatchTypeSerializer extends StdSerializer<List<IMatchType>> {
 		public MatchTypeSerializer() {
 			this(null);
 		}
 
-		public MatchTypeSerializer(Class<List<MatchType>> t) {
+		public MatchTypeSerializer(Class<List<IMatchType>> t) {
 			super(t);
 		}
 
 		@Override
-		public void serialize(List<MatchType> matchType, JsonGenerator jsonGen, SerializerProvider provider)
+		public void serialize(List<IMatchType> matchType, JsonGenerator jsonGen, SerializerProvider provider)
 				throws IOException, JsonProcessingException {
 			try {
 				jsonGen.writeObject(getStringFromMatchType(matchType));
@@ -205,14 +208,14 @@ public void serialize(List<MatchType> matchType, JsonGenerator jsonGen, Serializ
 			}
 		}
 
-		public static String getStringFromMatchType(List<MatchType> matchType) throws ZinggClientException {
+		public static String getStringFromMatchType(List<IMatchType> matchType) throws ZinggClientException {
 			return String.join(",", matchType.stream()
-					.map(p -> p.value())
+					.map(p -> p.getName())
 					.collect(Collectors.toList()));
 		}
 	}
 
-	public static class MatchTypeDeserializer extends StdDeserializer<List<MatchType>> {
+	public static class MatchTypeDeserializer extends StdDeserializer<List<IMatchType>> {
 		private static final long serialVersionUID = 1L;
 
 		public MatchTypeDeserializer() { 
@@ -222,24 +225,24 @@ public MatchTypeDeserializer(Class<String> t) {
 		   super(t); 
 		} 
 		@Override 
-		public List<MatchType> deserialize(JsonParser parser, DeserializationContext context) 
+		public List<IMatchType> deserialize(JsonParser parser, DeserializationContext context) 
 		   throws IOException, JsonProcessingException { 
 			ObjectMapper mapper = new ObjectMapper();
 			try{
 				mapper.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY);
 				LOG.debug("Deserializing custom type");
 				return getMatchTypeFromString(mapper.readValue(parser, String.class)); 
 			}
-			catch(ZinggClientException e) {
+			catch(Exception | ZinggClientException e) {
 				throw new IOException(e);
 			}
 		}   
 
-		public static List<MatchType> getMatchTypeFromString(String m) throws ZinggClientException{
-			List<MatchType> matchTypes = new ArrayList<MatchType>();
+		public static List<IMatchType> getMatchTypeFromString(String m) throws ZinggClientException, Exception{
+			List<IMatchType> matchTypes = new ArrayList<IMatchType>();
 		    String[] matchTypeFromConfig = m.split(","); 
 			for (String s: matchTypeFromConfig) { 
-				MatchType mt = MatchType.getMatchType(s);
+				IMatchType mt = MatchTypes.getByName(s);
 				matchTypes.add(mt);
 			}     
 		   return matchTypes; 

common/client/src/main/java/zingg/common/client/IMatchType.java

@@ -0,0 +1,8 @@
+package zingg.common.client;
+
+public interface IMatchType extends Named { 
+
+    public String toString();
+
+} 
+

common/client/src/main/java/zingg/common/client/MatchType.java

@@ -1,86 +1,68 @@
 package zingg.common.client;
 
 import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
 
 /**
  * Field types used in defining the types of fields for matching. See the field
  * definitions and the user guide for more details
  */
 
-public enum MatchType implements Serializable {
-	/**
-	 * Short words like first names and organizations with focus on first
-	 * characters matching
-	 */
-	FUZZY("FUZZY"),
-
-	/**
-	 * Fields needing exact matches
-	 */
-	EXACT("EXACT"),
+public class MatchType implements IMatchType, Serializable{
 
-
-	/**
-	 * Many times pin code is xxxxx-xxxx and has to be matched with xxxxx.
-	 */
-	PINCODE("PINCODE"),
+	private static final long serialVersionUID = 1L;
+	protected String name;
 
-	/**
-	 * an email type which is supposed to look at only the first part of the email and ignore the domain.
-	 */
-	EMAIL("EMAIL"),
-
-	/**
-	 * Long descriptive text, usually more than a couple of words for example
-	 * product descriptions
-	 */
-	TEXT("TEXT"),
+	public MatchType(){
+
+	}
 
-	/**
-	 * Strings containing numbers which need to be same. Example in addresses,
-	 * we dont want 4th street to match 5th street
-	 * Matching numbers with deviations
-	 */
-	NUMERIC("NUMERIC"),
-	/*eg P301d, P00231*/
-	NUMERIC_WITH_UNITS("NUMBER_WITH_UNITS"),
-	NULL_OR_BLANK("NULL_OR_BLANK"),
-	ONLY_ALPHABETS_EXACT("ONLY_ALPHABETS_EXACT"),
-	ONLY_ALPHABETS_FUZZY("ONLY_ALPHABETS_FUZZY"),
-	DONT_USE("DONT_USE");
+	public MatchType(String n){
+		this.name = n;
+		MatchTypes.put(this);
+	}
 
-	private String value;
-	private static Map<String, MatchType> types;
+	@Override
+	public String getName() {
+		return this.name;
+	}
 
-	MatchType(String type) {
-		this.value = type;
+	@Override
+	public void setName(String name) {
+		this.name = name;
 	}
 
-	private static void init() {
-		types = new HashMap<String, MatchType>();
-		for (MatchType f : MatchType.values()) {
-			types.put(f.value, f);
-		}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((name == null) ? 0 : name.hashCode());
+		return result;
 	}
 
-	@JsonCreator
-	public static MatchType getMatchType(String t) throws ZinggClientException{
-		if (types == null) {
-			init();
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		MatchType other = (MatchType) obj;
+		if (name == null) {
+			if (other.name != null){
+				return false;
+			}
+		} 
+		else if (!name.equalsIgnoreCase(other.name)){
+			return false;
 		}
-		MatchType type = types.get(t.trim().toUpperCase());
-		if (type == null) throw new ZinggClientException("Unsupported Match Type: " + t);
-		return type;
+		return true;
 	}
 
-	@JsonValue
-	public String value() {
-		return value;
+	@Override
+	public String toString() {
+		return name;
 	}
 
 }

common/client/src/main/java/zingg/common/client/MatchTypes.java

@@ -0,0 +1,54 @@
+package zingg.common.client;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MatchTypes {
+
+    public final static IMatchType FUZZY = new MatchType("FUZZY");
+    public final static IMatchType EXACT = new MatchType("EXACT");
+    public final static IMatchType PINCODE = new MatchType("PINCODE");
+    public final static IMatchType EMAIL = new MatchType("EMAIL");
+    public final static IMatchType TEXT = new MatchType("TEXT");
+    public final static IMatchType NUMERIC = new MatchType("NUMERIC");
+    public final static IMatchType NUMERIC_WITH_UNITS = new MatchType("NUMERIC_WITH_UNITS");
+    public final static IMatchType NULL_OR_BLANK = new MatchType("NULL_OR_BLANK");
+    public final static IMatchType ONLY_ALPHABETS_EXACT = new MatchType("ONLY_ALPHABETS_EXACT");
+    public final static IMatchType ONLY_ALPHABETS_FUZZY = new MatchType("ONLY_ALPHABETS_FUZZY");
+    public final static IMatchType DONT_USE = new MatchType("DONT_USE");
+
+    public static Map<String, IMatchType> allMatchTypes;// = new HashMap<String, IMatchType>(); 
+
+    protected MatchTypes(){
+
+    }
+
+    public static final void put(IMatchType o) { 
+
+        if (allMatchTypes == null) { 
+            allMatchTypes = new HashMap<String, IMatchType>(); 
+        } 
+
+        allMatchTypes.put(o.getName().toUpperCase(), o);  
+    } 
+
+    public static String[] getAllMatchTypes() { 
+        IMatchType[] zo = allMatchTypes.values().toArray(new IMatchType[allMatchTypes.size()]); 
+        int i = 0; 
+        String[] s = new String[zo.length]; 
+        for (IMatchType z: zo) { 
+            s[i++] = z.getName(); 
+        } 
+        return s; 
+    } 
+
+    public static IMatchType getByName(String name) throws Exception{ 
+        for (IMatchType zo: MatchTypes.allMatchTypes.values()) { 
+            if (zo.getName().equalsIgnoreCase(name)) {
+                return zo; 
+            }
+        } 
+        return null; 
+    } 
+
+}

common/client/src/main/java/zingg/common/client/ZFrame.java

@@ -7,6 +7,7 @@ public interface ZFrame<D, R, C> {
 
 	public static final String RIGHT_JOIN = "right";
 	public static final String LEFT_JOIN = "left";
+    public static final String INNER_JOIN = "inner";
 
 	public static final String COL_COUNT = "count";
 	public static final String COL_VALUE = "VALUE";
@@ -75,6 +76,7 @@ public interface ZFrame<D, R, C> {
     public ZFrame<D, R, C> unionByName(ZFrame<D, R, C> other, boolean flag);
 
     public <A> ZFrame<D, R, C> withColumn(String s, A c);
+    public ZFrame<D, R, C> withColumns(String[] columns, C[] columnValues);
 
 
     public ZFrame<D, R, C> repartition(int num);

common/client/src/main/java/zingg/common/client/util/ColName.java

@@ -29,10 +29,9 @@ public interface ColName {
 	public static final String MODEL_ID_COL = COL_PREFIX + "modelId";
 	public static final String RAW_PREDICTION="rawPrediction";
 	public static final String COL_COUNT = COL_PREFIX + "count";
-	public static final String COL_WORD = COL_PREFIX + "word";
+	public static final String COL_WORD = COL_PREFIX + "stopword";
 	public static final String COL_SPLIT = COL_PREFIX + "split";
 	public static final String HASH_COUNTS_COL = ColName.HASH_COL + "_count";
 	public static final String BLOCK_SAMPLES = "blockSamples/";
-	public static final String STOPWORD_COL = COL_PREFIX + "word";
 
 }