Trained model for febrl5M #1068

examples/febrl5M/config.json

@@ -0,0 +1,115 @@
+{	
+	"fieldDefinition":[
+		{
+			"fieldName" : "id",
+			"matchType" : "dont_use",
+			"fields" : "id",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "fname",
+			"matchType" : "fuzzy",
+			"fields" : "fname",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "lname",
+			"matchType" : "fuzzy",
+			"fields" : "lname",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "stNo",
+			"matchType": "exact",
+			"fields" : "stNo",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "add1",
+			"matchType": "fuzzy",
+			"fields" : "add1",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "add2",
+			"matchType": "fuzzy",
+			"fields" : "add2",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "city",
+			"matchType": "fuzzy",
+			"fields" : "city",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "areacode",
+			"matchType": "exact",
+			"fields" : "areacode",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "state",
+			"matchType": "fuzzy",
+			"fields" : "state",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "dob",
+			"matchType": "fuzzy",
+			"fields" : "dob",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "ssn",
+			"matchType": "fuzzy",
+			"fields" : "ssn",
+			"dataType": "string" 
+		}
+		],
+		"output" : [{
+			"name":"output", 
+			"format":"csv", 
+			"props": {
+				"location": "/tmp/zinggOutput",
+				"delimiter": ",",
+				"header":true
+			}
+		}],
+		"data" : [{
+			"name":"test1", 
+			"format":"csv", 
+			"props": {
+				"location": "examples/febrl5M/febrl_data_5M_part1.csv.gz",
+				"delimiter": ",",
+				"header":false					
+			},
+			"schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn  string"
+		},
+			{
+			"name":"test2", 
+			"format":"csv", 
+			"props": {
+				"location": "examples/febrl5M/febrl_data_5M_part2.csv.gz",
+				"delimiter": ",",
+				"header":false					
+			},
+			"schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn  string"
+		},
+			{
+			"name":"test3", 
+			"format":"csv", 
+			"props": {
+				"location": "examples/febrl5M/febrl_data_5M_part3.csv.gz",
+				"delimiter": ",",
+				"header":false					
+			},
+			"schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn  string"
+		}
+		],
+		"labelDataSampleSize" : 0.01,
+		"numPartitions":4000,
+		"modelId": 107,
+		"zinggDir": "models"
+
+}

models/107/model/classifier/best.model/bestModel/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.PipelineModel","timestamp":1740539676401,"sparkVersion":"3.5.3","uid":"pipeline_05380dc4ec58","paramMap":{"stageUids":["vecAssembler_e6cf68a26da4","poly_50382857c20c","logreg_6baef02c8fb1"]},"defaultParamMap":{}}

models/107/model/classifier/best.model/bestModel/stages/0_vecAssembler_e6cf68a26da4/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1740539676433,"sparkVersion":"3.5.3","uid":"vecAssembler_e6cf68a26da4","paramMap":{"outputCol":"z_featurevector","inputCols":["z_sim0","z_sim1","z_sim2","z_sim3","z_sim4","z_sim5","z_sim6","z_sim7","z_sim8","z_sim9","z_sim10","z_sim11","z_sim12","z_sim13","z_sim14","z_sim15","z_sim16","z_sim17"]},"defaultParamMap":{"handleInvalid":"error","outputCol":"vecAssembler_e6cf68a26da4__output"}}

models/107/model/classifier/best.model/bestModel/stages/1_poly_50382857c20c/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.PolynomialExpansion","timestamp":1740539676471,"sparkVersion":"3.5.3","uid":"poly_50382857c20c","paramMap":{"degree":3,"outputCol":"z_feature","inputCol":"z_featurevector"},"defaultParamMap":{"degree":2,"outputCol":"poly_50382857c20c__output"}}

models/107/model/classifier/best.model/bestModel/stages/2_logreg_6baef02c8fb1/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegressionModel","timestamp":1740539676510,"sparkVersion":"3.5.3","uid":"logreg_6baef02c8fb1","paramMap":{"threshold":0.4,"featuresCol":"z_feature","fitIntercept":true,"maxIter":100,"predictionCol":"z_prediction","probabilityCol":"z_probability","labelCol":"z_isMatch","regParam":0.1},"defaultParamMap":{"family":"auto","featuresCol":"features","threshold":0.5,"fitIntercept":true,"maxIter":100,"standardization":true,"maxBlockSizeInMB":0.0,"rawPredictionCol":"rawPrediction","tol":1.0E-6,"aggregationDepth":2,"predictionCol":"prediction","labelCol":"label","probabilityCol":"probability","elasticNetParam":0.0,"regParam":0.0}}

models/107/model/classifier/best.model/estimator/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.Pipeline","timestamp":1740539676205,"sparkVersion":"3.5.3","uid":"pipeline_05380dc4ec58","paramMap":{"stageUids":["vecAssembler_e6cf68a26da4","poly_50382857c20c","logreg_6baef02c8fb1"]},"defaultParamMap":{}}

models/107/model/classifier/best.model/estimator/stages/0_vecAssembler_e6cf68a26da4/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1740539676277,"sparkVersion":"3.5.3","uid":"vecAssembler_e6cf68a26da4","paramMap":{"outputCol":"z_featurevector","inputCols":["z_sim0","z_sim1","z_sim2","z_sim3","z_sim4","z_sim5","z_sim6","z_sim7","z_sim8","z_sim9","z_sim10","z_sim11","z_sim12","z_sim13","z_sim14","z_sim15","z_sim16","z_sim17"]},"defaultParamMap":{"handleInvalid":"error","outputCol":"vecAssembler_e6cf68a26da4__output"}}

models/107/model/classifier/best.model/estimator/stages/1_poly_50382857c20c/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.PolynomialExpansion","timestamp":1740539676323,"sparkVersion":"3.5.3","uid":"poly_50382857c20c","paramMap":{"degree":3,"outputCol":"z_feature","inputCol":"z_featurevector"},"defaultParamMap":{"degree":2,"outputCol":"poly_50382857c20c__output"}}

models/107/model/classifier/best.model/estimator/stages/2_logreg_6baef02c8fb1/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegression","timestamp":1740539676362,"sparkVersion":"3.5.3","uid":"logreg_6baef02c8fb1","paramMap":{"featuresCol":"z_feature","fitIntercept":true,"maxIter":100,"predictionCol":"z_prediction","probabilityCol":"z_probability","labelCol":"z_isMatch"},"defaultParamMap":{"family":"auto","featuresCol":"features","threshold":0.5,"fitIntercept":true,"maxIter":100,"standardization":true,"maxBlockSizeInMB":0.0,"rawPredictionCol":"rawPrediction","tol":1.0E-6,"aggregationDepth":2,"predictionCol":"prediction","labelCol":"label","probabilityCol":"probability","elasticNetParam":0.0,"regParam":0.0}}

models/107/model/classifier/best.model/evaluator/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.evaluation.BinaryClassificationEvaluator","timestamp":1740539676119,"sparkVersion":"3.5.3","uid":"binEval_38aa9c386e8b","paramMap":{"labelCol":"z_isMatch"},"defaultParamMap":{"rawPredictionCol":"rawPrediction","numBins":1000,"labelCol":"label","metricName":"areaUnderROC"}}

models/107/model/classifier/best.model/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.tuning.CrossValidatorModel","timestamp":1740539676004,"sparkVersion":"3.5.3","uid":"cv_a87d953e1bff","paramMap":{"seed":-1191137437,"foldCol":"","numFolds":2,"estimatorParamMaps":[[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_6baef02c8fb1","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_6baef02c8fb1","name":"regParam","value":"1.0","isJson":"true"}]]},"defaultParamMap":{"seed":-1191137437,"foldCol":"","numFolds":3},"avgMetrics":[0.8829329521533823,0.8829329521533823,0.8829329521533823,0.8829329521533823,0.897388099505035,0.897388099505035,0.897388099505035,0.897388099505035,0.9150816763383968,0.9150816763383968,0.9150816763383968,0.9150816763383968,0.9195806309381579,0.9195806309381579,0.9195806309381579,0.9195806309381579,0.90873727029641,0.90873727029641,0.90873727029641,0.90873727029641],"persistSubModels":false}