Revert "Merge pull request #788 from gnanaprakash-ravi/EntepriseIssue…

…784"

This reverts commit b8316e6, reversing
changes made to 5fd1b38.

common/client/src/main/java/zingg/common/client/Arguments.java

@@ -81,7 +81,7 @@
  * }
  * </pre>
  */
-@PythonClass(module = "client", outputDirectory = "python/zinggGenerated")
+@PythonClass
 @JsonInclude(Include.NON_NULL)
 public class Arguments implements Serializable, IArguments {
 

common/client/src/main/java/zingg/common/client/FieldDefinition.java

@@ -34,7 +34,6 @@
  * @author sgoyal
  *
  */
-@PythonClass(module = "client", outputDirectory = "python/zinggGenerated")
 public class FieldDefinition implements Named,
 		Serializable {
 

common/client/src/main/java/zingg/common/client/pipe/Pipe.java

@@ -20,7 +20,7 @@
  * @author sgoyal
  *
  */
-@PythonClass(module = "pipes", outputDirectory = "python/zinggGenerated")
+@PythonClass
 @JsonInclude(Include.NON_NULL)
 public class Pipe<D,R,C> implements Serializable{ // St:StructType, Sv:SaveMode
 

common/py/src/main/java/zingg/common/py/annotations/PythonClass.java

@@ -1,11 +1,9 @@
 package zingg.common.py.annotations;
 
+import javax.annotation.processing.*;
+
 import java.lang.annotation.Target;
 import java.lang.annotation.ElementType;
 
 @Target({ElementType.TYPE})
-public @interface PythonClass {
-    String module();
-    String parent() default "";
-    String outputDirectory();
-}
+public @interface PythonClass {}

common/py/src/main/java/zingg/common/py/annotations/PythonMethod.java

@@ -1,5 +1,7 @@
 package zingg.common.py.annotations;
 
+import javax.annotation.processing.*;
+
 import java.lang.annotation.Target;
 import java.lang.annotation.ElementType;
 

common/py/src/main/java/zingg/common/py/processors/PythonClassProcessor.java

@@ -3,8 +3,10 @@
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.annotation.processing.*;
 import java.util.Set;
@@ -18,88 +20,66 @@
 @SupportedAnnotationTypes("zingg.common.py.annotations.PythonClass")
 public class PythonClassProcessor extends AbstractProcessor {
 
-    private Set<TypeElement> processedElements = new HashSet<>();
-    private Set<String> folders = new HashSet<>();
+    private Map<String, List<String>> classMethodsMap = new HashMap<>();
 
     @Override
     public synchronized void init(ProcessingEnvironment processingEnv) {
-        System.out.println("ProcessingEnv " + processingEnv);
         super.init(processingEnv);
-
-        // Clear the output directory on initialization
-        folders.add("python/zinggGenerated");
-        folders.add("common/python");
-        folders.add("snowflake/python");
-        folders.add("spark/python");
-
-        for (String folder : folders) {
-            File directory = new File(folder);
-            if (directory.exists()) {
-                for (File file : directory.listFiles()) {
-                    file.delete();
-                    System.out.println(file + "deeellleeeeteeed");
-                    System.out.println(file + "geeneerateedddd");
-                }
-            }
-        }
     }
 
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
 
-        // Process each PythonClass annotated element
+
+        // process Services annotation
         for (Element element : roundEnv.getElementsAnnotatedWith(PythonClass.class)) {
-            if (element.getKind() == ElementKind.CLASS && !processedElements.contains(element)) {
-                processClass((TypeElement) element, roundEnv);
+            if (element.getKind() == ElementKind.CLASS) {
+                TypeElement classElement = (TypeElement) element;
+                PackageElement packageElement = (PackageElement) classElement.getEnclosingElement();
+                String packageName = packageElement.getQualifiedName().toString();
+                List<String> methodNames = new ArrayList<>();
+
+                String outputDirectory = determineOutputDirectory(packageName);
+
+                try (FileWriter fileWriter = new FileWriter(outputDirectory + File.separator + element.getSimpleName() + "Generated.py")) {
+                    generateImportsAndDeclarations(element, fileWriter);
+
+                    fileWriter.write("class " + element.getSimpleName() + ":\n");
+
+                    // __init__ method
+                    fileWriter.write("    def __init__(self" + generateConstructorParameters(classElement, element) + "):\n");
+                    generateClassInitializationCode(classElement, element, fileWriter);
+
+                    for (ExecutableElement methodElement : ElementFilter.methodsIn(classElement.getEnclosedElements())) {
+                        if (methodElement.getAnnotation(PythonMethod.class) != null) {
+                            methodNames.add(methodElement.getSimpleName().toString());
+                        }
+                    }
+                    classMethodsMap.put(element.getSimpleName().toString(), methodNames);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
             }
         }
+        ProcessorContext processorContext = ProcessorContext.getInstance();
+        processorContext.getClassMethodsMap().putAll(classMethodsMap);
+
         return false;
     }
 
+    Map<String, List<String>> getClassMethodsMap() {
+        return classMethodsMap;
+    }
 
-    private void processClass(TypeElement classElement, RoundEnvironment roundEnv) {
-
-        // Mark the class as processed
-        processedElements.add(classElement);
-
-        PythonClass pythonClassAnnotation = classElement.getAnnotation(PythonClass.class);
-
-        String outputDirectory = pythonClassAnnotation.outputDirectory();
-        String moduleName = pythonClassAnnotation.module();
-        String outputFile = outputDirectory + File.separator + moduleName + ".py";
-        String parentClassName = pythonClassAnnotation.parent();
-
-        try (FileWriter fileWriter = new FileWriter(outputFile, true)) {
-            generateImportsAndDeclarations(classElement, fileWriter);
-
-            if (!parentClassName.isEmpty()) {
-                fileWriter.write("class " + classElement.getSimpleName() + "(" + parentClassName + "):\n");
-            } else {
-                fileWriter.write("class " + classElement.getSimpleName() + ":\n");
-            }
-            // System.out.println(classElement.getSimpleName() + "ccccccccccccccccccccccccc");
-
-            // __init__ method
-            fileWriter.write("    def __init__(self" + generateConstructorParameters(classElement, classElement) + "):\n");
-            generateClassInitializationCode(classElement, classElement, fileWriter);
-
-            for (ExecutableElement methodElement : ElementFilter.methodsIn(classElement.getEnclosedElements())) {
-                if (methodElement.getAnnotation(PythonMethod.class) != null) {
-                    String javadoc = processingEnv.getElementUtils().getDocComment(methodElement);
-                    if (javadoc != null) {
-                        fileWriter.write("    '''\n");
-                        fileWriter.write(javadoc.trim());
-                        fileWriter.write("\n    '''\n");
-                    }
-
-                    fileWriter.write("    def " + methodElement.getSimpleName() + "(self" + PythonMethodProcessor.generateMethodSignature(methodElement) + "):\n");
-                    PythonMethodProcessor.generateMethodReturn(methodElement, fileWriter);
-                    PythonMethodProcessor.generateFieldAssignment(methodElement, fileWriter);
-                    fileWriter.write("\n");
-                }
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
+    private String determineOutputDirectory(String packageName) {
+        if (packageName.contains("enterprise") && packageName.contains("common")) {
+            return "common/python";
+        } else if (packageName.contains("enterprise") && packageName.contains("snowflake")) {
+            return "snowflake/python";
+        } else if (packageName.contains("enterprise") && packageName.contains("spark")) {
+            return "spark/python";
+        } else {
+            return "python/zingg";
         }
     }    
 
@@ -152,6 +132,15 @@ else if (element.getSimpleName().contentEquals("FieldDefinition")) {
         fileWriter.write("\n");
     }
 
+    // private void generateFieldInitializationCode(VariableElement field, Element element) {
+    //     String fieldName = field.getSimpleName().toString();
+    //     String fieldAssignment = "self." + element.getSimpleName().toString().toLowerCase() + "." + fieldName + " = " + fieldName;
+
+    //     if (!fieldName.startsWith("FORMAT_")) {
+    //         System.out.println("        " + fieldAssignment);
+    //     }
+    // }
+
     private String generateConstructorParameters(TypeElement classElement, Element element) {
 
         StringBuilder parameters = new StringBuilder();

common/py/src/main/java/zingg/common/py/processors/PythonMethodProcessor.java

@@ -1,31 +1,70 @@
 package zingg.common.py.processors;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import javax.annotation.processing.*;
 import javax.lang.model.type.TypeMirror;
 import javax.lang.model.type.TypeKind;
 import java.util.Set;
 
 import javax.lang.model.element.*;
+import zingg.common.py.annotations.*;
 
 @SupportedAnnotationTypes("zingg.common.py.annotations.PythonMethod")
 public class PythonMethodProcessor extends AbstractProcessor {
+
+    private Map<String, List<String>> classMethodsMap;
 
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+
+        ProcessorContext processorContext = ProcessorContext.getInstance();
+        classMethodsMap = processorContext.getClassMethodsMap();
+
+        for (Element element : roundEnv.getElementsAnnotatedWith(PythonMethod.class)) {
+
+            if (element.getKind() == ElementKind.METHOD) {
+                ExecutableElement methodElement = (ExecutableElement) element;
+                String className = methodElement.getEnclosingElement().getSimpleName().toString();
+
+                if (classMethodsMap.containsKey(className)) {
+                    List<String> methodNames = classMethodsMap.get(className);
+
+                    if (methodNames.contains(methodElement.getSimpleName().toString())) {
+                        try (FileWriter fileWriter = new FileWriter("python/zingg" + File.separator + className + "Generated.py", true)) {
+
+                            String javadoc = processingEnv.getElementUtils().getDocComment(methodElement);
+                            if (javadoc != null) {
+                                fileWriter.write("    '''\n");
+                                fileWriter.write(javadoc.trim());
+                                fileWriter.write("\n    '''\n");
+                            }
+
+                            fileWriter.write("    def " + methodElement.getSimpleName() + "(self" + generateMethodSignature(methodElement) + "):\n");
+                            generateMethodReturn(methodElement, fileWriter);
+                            generateFieldAssignment(methodElement, fileWriter);
+                            fileWriter.write("\n");
+                        } catch (IOException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                }
+            }
+        }
         return false;
     }
 
-    public static String generateMethodSignature(ExecutableElement methodElement) {
+    private String generateMethodSignature(ExecutableElement methodElement) {
         StringBuilder signature = new StringBuilder();
         signature.append(generateMethodParameters(methodElement));
         return signature.toString();
     }
 
-    public static String generateMethodParameters(ExecutableElement methodElement) {
+    private String generateMethodParameters(ExecutableElement methodElement) {
         StringBuilder parameters = new StringBuilder();
         for (VariableElement parameter : methodElement.getParameters()) {
             parameters.append(", ");
@@ -34,18 +73,23 @@ public static String generateMethodParameters(ExecutableElement methodElement) {
         return parameters.toString();
     }
 
-    public static void generateMethodReturn(ExecutableElement methodElement, FileWriter fileWriter) throws IOException {
+    private void generateMethodReturn(ExecutableElement methodElement, FileWriter fileWriter) throws IOException {
         TypeMirror returnType = methodElement.getReturnType();
         if (returnType.getKind() == TypeKind.VOID) {
             return;
         } else {
+            String returnTypeString = resolveType(returnType);
             String methodName = methodElement.getSimpleName().toString();
             String className = methodElement.getEnclosingElement().getSimpleName().toString();
             fileWriter.write("        return self." + className.toLowerCase() + "." + methodName + "()\n");
         }
     }
 
-    public static void generateFieldAssignment(ExecutableElement methodElement, FileWriter fileWriter) throws IOException {
+    private String resolveType(TypeMirror typeMirror) {
+        return typeMirror.toString();
+    }
+
+    private void generateFieldAssignment(ExecutableElement methodElement, FileWriter fileWriter) throws IOException {
         List<? extends VariableElement> parameters = methodElement.getParameters();
 
         if (!parameters.isEmpty()) {

examples/febrl/GeneratedFebrlExample.py

@@ -1,5 +1,6 @@
-from zingg.zinggGenerated.client import *
-from zingg.zinggGenerated.pipes import *
+from zingg.ArgumentsGenerated import *
+from zingg.FieldDefinitionGenerated import *
+from zingg.PipeGenerated import *
 from zingg.otherThanGenerated import *
 from zingg.otherThanGeneratedPipe import *
 from zingg.otherThanGeneratedArguments import *
@@ -38,7 +39,7 @@
 
 args.setOutput(outputPipe)
 
-options = ClientOptions([ClientOptions.PHASE,"match"])
+options = ClientOptions([ClientOptions.PHASE,"findTrainingData"])
 
 #Zingg execution for the given phase
 zingg = Zingg(args, options)

python/MANIFEST.in

@@ -11,5 +11,4 @@ recursive-include zingg/examples/amazon-google *
 recursive-include zingg/examples/febrl *
 recursive-include zingg/models *
 recursive-include zingg/phases *.py
-recursive-include zingg/zinggGenerated *.py
 recursive-include zingg/config *

python/setup.py

@@ -56,15 +56,13 @@
 DATA_PATH = os.path.join(ZINGG_HOME, "models")
 CONF_PATH = os.path.join(ZINGG_HOME, "config")
 PHASES_PATH = os.path.join(ZINGG_HOME, "python/phases")
-GENERATEDCODE_PATH = os.path.join(ZINGG_HOME, "python/zinggGenerated")
 
 SCRIPTS_TARGET = os.path.join("zingg", "scripts")
 JARS_TARGET = os.path.join("zingg", "jars")
 EXAMPLES_TARGET = os.path.join("zingg", "examples")
 DATA_TARGET = os.path.join("zingg", "models")
 CONF_TARGET = os.path.join("zingg", "config")
 PHASES_TARGET = os.path.join("zingg", "phases")
-GENERATEDCODE_TARGET = os.path.join("zingg", "zinggGenerated")
 
 # Check and see if we are under the Zingg path in which case we need to build the symlink farm.
 # This is important because we only want to build the symlink farm while under Zingg otherwise we
@@ -114,7 +112,6 @@ def run(self):
             os.symlink(DATA_PATH, DATA_TARGET)
             os.symlink(CONF_PATH, CONF_TARGET)
             os.symlink(PHASES_PATH, PHASES_TARGET)
-            os.symlink(GENERATEDCODE_PATH, GENERATEDCODE_TARGET)
         else:
             # For windows fall back to the slower copytree
             copytree(JARS_PATH, JARS_TARGET)
@@ -123,7 +120,6 @@ def run(self):
             copytree(DATA_PATH, DATA_TARGET)
             copytree(CONF_PATH, CONF_TARGET)
             copytree(PHASES_PATH, PHASES_TARGET)
-            copytree(GENERATEDCODE_PATH, GENERATEDCODE_TARGET)
     else:
         # If we are not inside of ZINGG_HOME verify we have the required symlink farm
         if not os.path.exists(JARS_TARGET):
@@ -162,8 +158,7 @@ def run(self):
             'zingg.data': 'zingg/models',
             'zingg.examples': 'zingg/examples',
             'zingg.conf': 'zingg/config',
-            'zingg.phases': 'zingg/phases',
-            'zingg.zinggGenerated': 'zingg/zinggGenerated'
+            'zingg.phases': 'zingg/phases'
         },
         package_data={
             'zingg.jars': ['*.jar'],
@@ -172,7 +167,6 @@ def run(self):
             'zingg.examples': ['*.py', '*/examples/*.py'],
             'zingg.conf': ['*'],
             'zingg.phases': ['*'],
-            'zingg.zinggGenerated': ['*'],
             '':['*.py'],
             '':['LICENSE']
             },
@@ -204,12 +198,10 @@ def run(self):
             os.remove(os.path.join("zingg", "examples"))
             os.remove(os.path.join("zingg", "phases"))
             os.remove(os.path.join("zingg", "config"))
-            os.remove(os.path.join("zingg", "zinggGenerated"))
         else:
             rmtree(os.path.join("zingg", "jars"))
             rmtree(os.path.join("zingg", "scripts"))
             rmtree(os.path.join("zingg", "models"))
             rmtree(os.path.join("zingg", "examples"))
             rmtree(os.path.join("zingg", "phases"))
             rmtree(os.path.join("zingg", "config"))
-            rmtree(os.path.join("zingg", "zinggGenerated"))