From 95b6c766365085e76270e91771528cf41b21f0c6 Mon Sep 17 00:00:00 2001 From: Sonal Goyal Date: Thu, 7 Mar 2024 13:21:55 +0530 Subject: [PATCH] Revert "Merge pull request #788 from gnanaprakash-ravi/EntepriseIssue784" This reverts commit b8316e6e27f44c88ada181e08ad383d8c1c0e9bd, reversing changes made to 5fd1b385f07dd8bdead04e7b84babca95cef492c. --- .../java/zingg/common/client/Arguments.java | 2 +- .../zingg/common/client/FieldDefinition.java | 1 - .../java/zingg/common/client/pipe/Pipe.java | 2 +- .../common/py/annotations/PythonClass.java | 8 +- .../common/py/annotations/PythonMethod.java | 2 + .../py/processors/PythonClassProcessor.java | 123 ++++++++---------- .../py/processors/PythonMethodProcessor.java | 52 +++++++- examples/febrl/GeneratedFebrlExample.py | 7 +- python/MANIFEST.in | 1 - python/setup.py | 10 +- .../client.py => zingg/ArgumentsGenerated.py} | 37 ------ python/zingg/FieldDefinitionGenerated.py | 37 ++++++ .../pipes.py => zingg/PipeGenerated.py} | 0 python/{zinggOld => zingg}/client.py | 0 python/zingg/otherThanGeneratedArguments.py | 2 +- .../otherThanGeneratedFieldDefinition.py | 2 +- python/zingg/otherThanGeneratedPipe.py | 2 +- python/{zinggOld => zingg}/pipes.py | 0 18 files changed, 156 insertions(+), 132 deletions(-) rename python/{zinggGenerated/client.py => zingg/ArgumentsGenerated.py} (80%) create mode 100644 python/zingg/FieldDefinitionGenerated.py rename python/{zinggGenerated/pipes.py => zingg/PipeGenerated.py} (100%) rename python/{zinggOld => zingg}/client.py (100%) rename python/{zinggOld => zingg}/pipes.py (100%) diff --git a/common/client/src/main/java/zingg/common/client/Arguments.java b/common/client/src/main/java/zingg/common/client/Arguments.java index 47474c310..5116a5cd9 100644 --- a/common/client/src/main/java/zingg/common/client/Arguments.java +++ b/common/client/src/main/java/zingg/common/client/Arguments.java @@ -81,7 +81,7 @@ * } * */ -@PythonClass(module = "client", outputDirectory = "python/zinggGenerated") +@PythonClass @JsonInclude(Include.NON_NULL) public class Arguments implements Serializable, IArguments { diff --git a/common/client/src/main/java/zingg/common/client/FieldDefinition.java b/common/client/src/main/java/zingg/common/client/FieldDefinition.java index 0adbd9e1a..f162a9107 100644 --- a/common/client/src/main/java/zingg/common/client/FieldDefinition.java +++ b/common/client/src/main/java/zingg/common/client/FieldDefinition.java @@ -34,7 +34,6 @@ * @author sgoyal * */ -@PythonClass(module = "client", outputDirectory = "python/zinggGenerated") public class FieldDefinition implements Named, Serializable { diff --git a/common/client/src/main/java/zingg/common/client/pipe/Pipe.java b/common/client/src/main/java/zingg/common/client/pipe/Pipe.java index e726160d3..aab0878b1 100644 --- a/common/client/src/main/java/zingg/common/client/pipe/Pipe.java +++ b/common/client/src/main/java/zingg/common/client/pipe/Pipe.java @@ -20,7 +20,7 @@ * @author sgoyal * */ -@PythonClass(module = "pipes", outputDirectory = "python/zinggGenerated") +@PythonClass @JsonInclude(Include.NON_NULL) public class Pipe implements Serializable{ // St:StructType, Sv:SaveMode diff --git a/common/py/src/main/java/zingg/common/py/annotations/PythonClass.java b/common/py/src/main/java/zingg/common/py/annotations/PythonClass.java index e557f9a3c..0d3bf21a5 100644 --- a/common/py/src/main/java/zingg/common/py/annotations/PythonClass.java +++ b/common/py/src/main/java/zingg/common/py/annotations/PythonClass.java @@ -1,11 +1,9 @@ package zingg.common.py.annotations; +import javax.annotation.processing.*; + import java.lang.annotation.Target; import java.lang.annotation.ElementType; @Target({ElementType.TYPE}) -public @interface PythonClass { - String module(); - String parent() default ""; - String outputDirectory(); -} \ No newline at end of file +public @interface PythonClass {} \ No newline at end of file diff --git a/common/py/src/main/java/zingg/common/py/annotations/PythonMethod.java b/common/py/src/main/java/zingg/common/py/annotations/PythonMethod.java index a37807d90..f59a9c038 100644 --- a/common/py/src/main/java/zingg/common/py/annotations/PythonMethod.java +++ b/common/py/src/main/java/zingg/common/py/annotations/PythonMethod.java @@ -1,5 +1,7 @@ package zingg.common.py.annotations; +import javax.annotation.processing.*; + import java.lang.annotation.Target; import java.lang.annotation.ElementType; diff --git a/common/py/src/main/java/zingg/common/py/processors/PythonClassProcessor.java b/common/py/src/main/java/zingg/common/py/processors/PythonClassProcessor.java index 3f41ac899..17bd7bdba 100644 --- a/common/py/src/main/java/zingg/common/py/processors/PythonClassProcessor.java +++ b/common/py/src/main/java/zingg/common/py/processors/PythonClassProcessor.java @@ -3,8 +3,10 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.annotation.processing.*; import java.util.Set; @@ -18,88 +20,66 @@ @SupportedAnnotationTypes("zingg.common.py.annotations.PythonClass") public class PythonClassProcessor extends AbstractProcessor { - private Set processedElements = new HashSet<>(); - private Set folders = new HashSet<>(); + private Map> classMethodsMap = new HashMap<>(); @Override public synchronized void init(ProcessingEnvironment processingEnv) { - System.out.println("ProcessingEnv " + processingEnv); super.init(processingEnv); - - // Clear the output directory on initialization - folders.add("python/zinggGenerated"); - folders.add("common/python"); - folders.add("snowflake/python"); - folders.add("spark/python"); - - for (String folder : folders) { - File directory = new File(folder); - if (directory.exists()) { - for (File file : directory.listFiles()) { - file.delete(); - System.out.println(file + "deeellleeeeteeed"); - System.out.println(file + "geeneerateedddd"); - } - } - } } @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - // Process each PythonClass annotated element + + // process Services annotation for (Element element : roundEnv.getElementsAnnotatedWith(PythonClass.class)) { - if (element.getKind() == ElementKind.CLASS && !processedElements.contains(element)) { - processClass((TypeElement) element, roundEnv); + if (element.getKind() == ElementKind.CLASS) { + TypeElement classElement = (TypeElement) element; + PackageElement packageElement = (PackageElement) classElement.getEnclosingElement(); + String packageName = packageElement.getQualifiedName().toString(); + List methodNames = new ArrayList<>(); + + String outputDirectory = determineOutputDirectory(packageName); + + try (FileWriter fileWriter = new FileWriter(outputDirectory + File.separator + element.getSimpleName() + "Generated.py")) { + generateImportsAndDeclarations(element, fileWriter); + + fileWriter.write("class " + element.getSimpleName() + ":\n"); + + // __init__ method + fileWriter.write(" def __init__(self" + generateConstructorParameters(classElement, element) + "):\n"); + generateClassInitializationCode(classElement, element, fileWriter); + + for (ExecutableElement methodElement : ElementFilter.methodsIn(classElement.getEnclosedElements())) { + if (methodElement.getAnnotation(PythonMethod.class) != null) { + methodNames.add(methodElement.getSimpleName().toString()); + } + } + classMethodsMap.put(element.getSimpleName().toString(), methodNames); + } catch (IOException e) { + e.printStackTrace(); + } } } + ProcessorContext processorContext = ProcessorContext.getInstance(); + processorContext.getClassMethodsMap().putAll(classMethodsMap); + return false; } + Map> getClassMethodsMap() { + return classMethodsMap; + } - private void processClass(TypeElement classElement, RoundEnvironment roundEnv) { - - // Mark the class as processed - processedElements.add(classElement); - - PythonClass pythonClassAnnotation = classElement.getAnnotation(PythonClass.class); - - String outputDirectory = pythonClassAnnotation.outputDirectory(); - String moduleName = pythonClassAnnotation.module(); - String outputFile = outputDirectory + File.separator + moduleName + ".py"; - String parentClassName = pythonClassAnnotation.parent(); - - try (FileWriter fileWriter = new FileWriter(outputFile, true)) { - generateImportsAndDeclarations(classElement, fileWriter); - - if (!parentClassName.isEmpty()) { - fileWriter.write("class " + classElement.getSimpleName() + "(" + parentClassName + "):\n"); - } else { - fileWriter.write("class " + classElement.getSimpleName() + ":\n"); - } - // System.out.println(classElement.getSimpleName() + "ccccccccccccccccccccccccc"); - - // __init__ method - fileWriter.write(" def __init__(self" + generateConstructorParameters(classElement, classElement) + "):\n"); - generateClassInitializationCode(classElement, classElement, fileWriter); - - for (ExecutableElement methodElement : ElementFilter.methodsIn(classElement.getEnclosedElements())) { - if (methodElement.getAnnotation(PythonMethod.class) != null) { - String javadoc = processingEnv.getElementUtils().getDocComment(methodElement); - if (javadoc != null) { - fileWriter.write(" '''\n"); - fileWriter.write(javadoc.trim()); - fileWriter.write("\n '''\n"); - } - - fileWriter.write(" def " + methodElement.getSimpleName() + "(self" + PythonMethodProcessor.generateMethodSignature(methodElement) + "):\n"); - PythonMethodProcessor.generateMethodReturn(methodElement, fileWriter); - PythonMethodProcessor.generateFieldAssignment(methodElement, fileWriter); - fileWriter.write("\n"); - } - } - } catch (IOException e) { - e.printStackTrace(); + private String determineOutputDirectory(String packageName) { + if (packageName.contains("enterprise") && packageName.contains("common")) { + return "common/python"; + } else if (packageName.contains("enterprise") && packageName.contains("snowflake")) { + return "snowflake/python"; + } else if (packageName.contains("enterprise") && packageName.contains("spark")) { + return "spark/python"; + } else { + return "python/zingg"; } } @@ -152,6 +132,15 @@ else if (element.getSimpleName().contentEquals("FieldDefinition")) { fileWriter.write("\n"); } + // private void generateFieldInitializationCode(VariableElement field, Element element) { + // String fieldName = field.getSimpleName().toString(); + // String fieldAssignment = "self." + element.getSimpleName().toString().toLowerCase() + "." + fieldName + " = " + fieldName; + + // if (!fieldName.startsWith("FORMAT_")) { + // System.out.println(" " + fieldAssignment); + // } + // } + private String generateConstructorParameters(TypeElement classElement, Element element) { StringBuilder parameters = new StringBuilder(); diff --git a/common/py/src/main/java/zingg/common/py/processors/PythonMethodProcessor.java b/common/py/src/main/java/zingg/common/py/processors/PythonMethodProcessor.java index 7781edb6d..1971adcb2 100644 --- a/common/py/src/main/java/zingg/common/py/processors/PythonMethodProcessor.java +++ b/common/py/src/main/java/zingg/common/py/processors/PythonMethodProcessor.java @@ -1,8 +1,10 @@ package zingg.common.py.processors; +import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.List; +import java.util.Map; import javax.annotation.processing.*; import javax.lang.model.type.TypeMirror; @@ -10,22 +12,59 @@ import java.util.Set; import javax.lang.model.element.*; +import zingg.common.py.annotations.*; @SupportedAnnotationTypes("zingg.common.py.annotations.PythonMethod") public class PythonMethodProcessor extends AbstractProcessor { + + private Map> classMethodsMap; @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { + + ProcessorContext processorContext = ProcessorContext.getInstance(); + classMethodsMap = processorContext.getClassMethodsMap(); + + for (Element element : roundEnv.getElementsAnnotatedWith(PythonMethod.class)) { + + if (element.getKind() == ElementKind.METHOD) { + ExecutableElement methodElement = (ExecutableElement) element; + String className = methodElement.getEnclosingElement().getSimpleName().toString(); + + if (classMethodsMap.containsKey(className)) { + List methodNames = classMethodsMap.get(className); + + if (methodNames.contains(methodElement.getSimpleName().toString())) { + try (FileWriter fileWriter = new FileWriter("python/zingg" + File.separator + className + "Generated.py", true)) { + + String javadoc = processingEnv.getElementUtils().getDocComment(methodElement); + if (javadoc != null) { + fileWriter.write(" '''\n"); + fileWriter.write(javadoc.trim()); + fileWriter.write("\n '''\n"); + } + + fileWriter.write(" def " + methodElement.getSimpleName() + "(self" + generateMethodSignature(methodElement) + "):\n"); + generateMethodReturn(methodElement, fileWriter); + generateFieldAssignment(methodElement, fileWriter); + fileWriter.write("\n"); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + } return false; } - public static String generateMethodSignature(ExecutableElement methodElement) { + private String generateMethodSignature(ExecutableElement methodElement) { StringBuilder signature = new StringBuilder(); signature.append(generateMethodParameters(methodElement)); return signature.toString(); } - public static String generateMethodParameters(ExecutableElement methodElement) { + private String generateMethodParameters(ExecutableElement methodElement) { StringBuilder parameters = new StringBuilder(); for (VariableElement parameter : methodElement.getParameters()) { parameters.append(", "); @@ -34,18 +73,23 @@ public static String generateMethodParameters(ExecutableElement methodElement) { return parameters.toString(); } - public static void generateMethodReturn(ExecutableElement methodElement, FileWriter fileWriter) throws IOException { + private void generateMethodReturn(ExecutableElement methodElement, FileWriter fileWriter) throws IOException { TypeMirror returnType = methodElement.getReturnType(); if (returnType.getKind() == TypeKind.VOID) { return; } else { + String returnTypeString = resolveType(returnType); String methodName = methodElement.getSimpleName().toString(); String className = methodElement.getEnclosingElement().getSimpleName().toString(); fileWriter.write(" return self." + className.toLowerCase() + "." + methodName + "()\n"); } } - public static void generateFieldAssignment(ExecutableElement methodElement, FileWriter fileWriter) throws IOException { + private String resolveType(TypeMirror typeMirror) { + return typeMirror.toString(); + } + + private void generateFieldAssignment(ExecutableElement methodElement, FileWriter fileWriter) throws IOException { List parameters = methodElement.getParameters(); if (!parameters.isEmpty()) { diff --git a/examples/febrl/GeneratedFebrlExample.py b/examples/febrl/GeneratedFebrlExample.py index e667c2429..54c64e77e 100644 --- a/examples/febrl/GeneratedFebrlExample.py +++ b/examples/febrl/GeneratedFebrlExample.py @@ -1,5 +1,6 @@ -from zingg.zinggGenerated.client import * -from zingg.zinggGenerated.pipes import * +from zingg.ArgumentsGenerated import * +from zingg.FieldDefinitionGenerated import * +from zingg.PipeGenerated import * from zingg.otherThanGenerated import * from zingg.otherThanGeneratedPipe import * from zingg.otherThanGeneratedArguments import * @@ -38,7 +39,7 @@ args.setOutput(outputPipe) -options = ClientOptions([ClientOptions.PHASE,"match"]) +options = ClientOptions([ClientOptions.PHASE,"findTrainingData"]) #Zingg execution for the given phase zingg = Zingg(args, options) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 4caa178c5..b9582aea0 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -11,5 +11,4 @@ recursive-include zingg/examples/amazon-google * recursive-include zingg/examples/febrl * recursive-include zingg/models * recursive-include zingg/phases *.py -recursive-include zingg/zinggGenerated *.py recursive-include zingg/config * diff --git a/python/setup.py b/python/setup.py index 3be6a72ff..514c2e180 100644 --- a/python/setup.py +++ b/python/setup.py @@ -56,7 +56,6 @@ DATA_PATH = os.path.join(ZINGG_HOME, "models") CONF_PATH = os.path.join(ZINGG_HOME, "config") PHASES_PATH = os.path.join(ZINGG_HOME, "python/phases") -GENERATEDCODE_PATH = os.path.join(ZINGG_HOME, "python/zinggGenerated") SCRIPTS_TARGET = os.path.join("zingg", "scripts") JARS_TARGET = os.path.join("zingg", "jars") @@ -64,7 +63,6 @@ DATA_TARGET = os.path.join("zingg", "models") CONF_TARGET = os.path.join("zingg", "config") PHASES_TARGET = os.path.join("zingg", "phases") -GENERATEDCODE_TARGET = os.path.join("zingg", "zinggGenerated") # Check and see if we are under the Zingg path in which case we need to build the symlink farm. # This is important because we only want to build the symlink farm while under Zingg otherwise we @@ -114,7 +112,6 @@ def run(self): os.symlink(DATA_PATH, DATA_TARGET) os.symlink(CONF_PATH, CONF_TARGET) os.symlink(PHASES_PATH, PHASES_TARGET) - os.symlink(GENERATEDCODE_PATH, GENERATEDCODE_TARGET) else: # For windows fall back to the slower copytree copytree(JARS_PATH, JARS_TARGET) @@ -123,7 +120,6 @@ def run(self): copytree(DATA_PATH, DATA_TARGET) copytree(CONF_PATH, CONF_TARGET) copytree(PHASES_PATH, PHASES_TARGET) - copytree(GENERATEDCODE_PATH, GENERATEDCODE_TARGET) else: # If we are not inside of ZINGG_HOME verify we have the required symlink farm if not os.path.exists(JARS_TARGET): @@ -162,8 +158,7 @@ def run(self): 'zingg.data': 'zingg/models', 'zingg.examples': 'zingg/examples', 'zingg.conf': 'zingg/config', - 'zingg.phases': 'zingg/phases', - 'zingg.zinggGenerated': 'zingg/zinggGenerated' + 'zingg.phases': 'zingg/phases' }, package_data={ 'zingg.jars': ['*.jar'], @@ -172,7 +167,6 @@ def run(self): 'zingg.examples': ['*.py', '*/examples/*.py'], 'zingg.conf': ['*'], 'zingg.phases': ['*'], - 'zingg.zinggGenerated': ['*'], '':['*.py'], '':['LICENSE'] }, @@ -204,7 +198,6 @@ def run(self): os.remove(os.path.join("zingg", "examples")) os.remove(os.path.join("zingg", "phases")) os.remove(os.path.join("zingg", "config")) - os.remove(os.path.join("zingg", "zinggGenerated")) else: rmtree(os.path.join("zingg", "jars")) rmtree(os.path.join("zingg", "scripts")) @@ -212,4 +205,3 @@ def run(self): rmtree(os.path.join("zingg", "examples")) rmtree(os.path.join("zingg", "phases")) rmtree(os.path.join("zingg", "config")) - rmtree(os.path.join("zingg", "zinggGenerated")) diff --git a/python/zinggGenerated/client.py b/python/zingg/ArgumentsGenerated.py similarity index 80% rename from python/zinggGenerated/client.py rename to python/zingg/ArgumentsGenerated.py index 5045a8ca2..bafb8d96d 100644 --- a/python/zinggGenerated/client.py +++ b/python/zingg/ArgumentsGenerated.py @@ -1,40 +1,3 @@ -from zingg.otherThanGenerated import * -''' -This class defines each field that we use in matching We can use this to - configure the properties of each field we use for matching in Zingg. - - @author sgoyal -''' -class FieldDefinition: - def __init__(self, name, dataType, *matchType): - self.fielddefinition = getJVM().zingg.common.client.FieldDefinition() - self.fielddefinition.setFieldName(name) - self.fielddefinition.setDataType(self.stringify(dataType)) - self.fielddefinition.setMatchType(matchType) - self.fielddefinition.setFields(name) - - def getFieldDefinition(self): - return self.fielddefinition - - def setFields(self, fields): - self.fielddefinition.setFields(fields) - - ''' -Set the field type which defines the kind of matching we want to do - - @see MatchType - @param type - the type to set - ''' - def setMatchType(self, type): - self.fielddefinition.setMatchType(type) - - def setStopWords(self, stopWords): - self.fielddefinition.setStopWords(stopWords) - - def setFieldName(self, fieldName): - self.fielddefinition.setFieldName(fieldName) - from zingg.otherThanGenerated import * ''' This class helps supply match arguments to Zingg. There are 3 basic steps diff --git a/python/zingg/FieldDefinitionGenerated.py b/python/zingg/FieldDefinitionGenerated.py new file mode 100644 index 000000000..b08d75984 --- /dev/null +++ b/python/zingg/FieldDefinitionGenerated.py @@ -0,0 +1,37 @@ +from zingg.otherThanGenerated import * +''' +This class defines each field that we use in matching We can use this to + configure the properties of each field we use for matching in Zingg. + + @author sgoyal +''' +class FieldDefinition: + def __init__(self, name, dataType, *matchType): + self.fielddefinition = getJVM().zingg.common.client.FieldDefinition() + self.fielddefinition.setFieldName(name) + self.fielddefinition.setDataType(self.stringify(dataType)) + self.fielddefinition.setMatchType(matchType) + self.fielddefinition.setFields(name) + + def getFieldDefinition(self): + return self.fielddefinition + + def setFields(self, fields): + self.fielddefinition.setFields(fields) + + ''' +Set the field type which defines the kind of matching we want to do + + @see MatchType + @param type + the type to set + ''' + def setMatchType(self, type): + self.fielddefinition.setMatchType(type) + + def setStopWords(self, stopWords): + self.fielddefinition.setStopWords(stopWords) + + def setFieldName(self, fieldName): + self.fielddefinition.setFieldName(fieldName) + diff --git a/python/zinggGenerated/pipes.py b/python/zingg/PipeGenerated.py similarity index 100% rename from python/zinggGenerated/pipes.py rename to python/zingg/PipeGenerated.py diff --git a/python/zinggOld/client.py b/python/zingg/client.py similarity index 100% rename from python/zinggOld/client.py rename to python/zingg/client.py diff --git a/python/zingg/otherThanGeneratedArguments.py b/python/zingg/otherThanGeneratedArguments.py index 5abe1c7f5..113d08ead 100644 --- a/python/zingg/otherThanGeneratedArguments.py +++ b/python/zingg/otherThanGeneratedArguments.py @@ -1,4 +1,4 @@ -from zingg.zinggGenerated.client import * +from zingg.ArgumentsGenerated import * from zingg.otherThanGeneratedFieldDefinition import * class ExtendedArgumentsGenerated(Arguments): diff --git a/python/zingg/otherThanGeneratedFieldDefinition.py b/python/zingg/otherThanGeneratedFieldDefinition.py index 195499d07..43f3d229e 100644 --- a/python/zingg/otherThanGeneratedFieldDefinition.py +++ b/python/zingg/otherThanGeneratedFieldDefinition.py @@ -1,4 +1,4 @@ -from zingg.zinggGenerated.client import * +from zingg.FieldDefinitionGenerated import * class ExtendedFieldDefinitionGenerated(FieldDefinition): def __init__(self, name, dataType, *matchType): diff --git a/python/zingg/otherThanGeneratedPipe.py b/python/zingg/otherThanGeneratedPipe.py index e405b3386..a46df2794 100644 --- a/python/zingg/otherThanGeneratedPipe.py +++ b/python/zingg/otherThanGeneratedPipe.py @@ -1,4 +1,4 @@ -from zingg.zinggGenerated.pipes import * +from zingg.PipeGenerated import * class ExtendedPipeGenerated(Pipe): def __init__(self, name, format): diff --git a/python/zinggOld/pipes.py b/python/zingg/pipes.py similarity index 100% rename from python/zinggOld/pipes.py rename to python/zingg/pipes.py