Merge pull request #299 from warsaw/isfat-ontop

This is the isfat branch, rebased once more on top of the unrevert branch
linkedin · Apr 19, 2019 · 7bdc4b8 · 7bdc4b8
2 parents b5bb338 + 748880d
commit 7bdc4b8
Show file tree

Hide file tree

Showing 11 changed files with 183 additions and 68 deletions.
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -1,5 +1,9 @@
 # pygradle news #
 
+* 2019-04-08
+  - You must now change any usages of either the `python.pex.FatPex` or
+    `python.pex.isFat` flag to `python.zipapp.isFat`.
+
 * 2018-11-09
   - Added a default `mypy` task which you can enable by setting
     `python.mypy.run = true` in your `build.gradle` file.

diff --git a/...e-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy b/...e-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy
@@ -150,4 +150,61 @@ class PexIntegrationTest extends Specification {
         then:
         out.toString() == "Hello World${System.getProperty("line.separator")}".toString()
     }
+
+    def "can build fat pex with isFat"() {
+        given:
+        testProjectDir.buildFile << """\
+        | plugins {
+        |     id 'com.linkedin.python-pex'
+        | }
+        | version = '1.0.0'
+        | python {
+        |   pex {
+        |     isFat = true
+        |   }
+        | }
+        | ${PyGradleTestBuilder.createRepoClosure()}
+        """.stripMargin().stripIndent()
+
+        when:
+        def result = GradleRunner.create()
+            .withProjectDir(testProjectDir.root)
+            .withArguments('build', '--stacktrace')
+            .withPluginClasspath()
+            .withDebug(true)
+            .build()
+        println result.output
+
+        then:
+
+        result.output.contains("BUILD SUCCESS")
+        result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:installTestRequirements').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:createVirtualEnvironment').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:check').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:build').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:buildPex').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:assembleContainers').outcome == TaskOutcome.SUCCESS
+
+        Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin"))
+        def pexFile = deployablePath.resolve(PexFileUtil.createFatPexFilename('hello_world'))
+
+        pexFile.toFile().exists()
+
+        when: "we have a pex file"
+        def line = new String(pexFile.bytes, "UTF-8").substring(0, 100)
+
+        then: "its shebang line is not pointing to a virtualenv"
+        line.startsWith("#!") && !line.contains("venv")
+
+        when:
+        def out = ExecUtils.run(pexFile)
+        println out
+
+        then:
+        out.toString() == "Hello World${System.getProperty("line.separator")}".toString()
+    }
 }
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java
@@ -19,7 +19,6 @@
 import com.linkedin.gradle.python.tasks.BuildPexTask;
 import com.linkedin.gradle.python.util.ApplicationContainer;
 import com.linkedin.gradle.python.util.ExtensionUtils;
-import com.linkedin.gradle.python.util.OperatingSystem;
 import com.linkedin.gradle.python.util.StandardTextValues;
 import org.gradle.api.Project;
 
@@ -39,12 +38,12 @@ public class PexExtension implements ApplicationContainer {
     public static final String TASK_BUILD_NOOP_PEX = "buildPex";
 
     private File cache;
-    // Default to fat zipapps on Windows, since our wrappers are fairly POSIX specific.
-    private boolean isFat = OperatingSystem.current().isWindows();
     private boolean pythonWrapper = true;
+    private Project project;
 
     public PexExtension(Project project) {
         this.cache = new File(project.getBuildDir(), "pex-cache");
+        this.project = project;
     }
 
     public File getPexCache() {
@@ -55,40 +54,6 @@ public void setPexCache(File pexCache) {
         cache = pexCache;
     }
 
-    // These are kept for API backward compatibility.
-
-    /**
-     * @return when <code>true</code>, then skinny pex's will be used.
-     */
-    @Deprecated
-    public boolean isFatPex() {
-        return isFat();
-    }
-
-    /**
-     * @param fatPex when <code>true</code>, wrappers will be made all pointing to a single pex file.
-     */
-    @Deprecated
-    public void setFatPex(boolean fatPex) {
-        isFat = fatPex;
-     }
-
-    // Use these properties instead.
-
-    /**
-     * @return when <code>true</code>, then skinny pex's will be used.
-     */
-    public boolean isFat() {
-        return isFat;
-    }
-
-    /**
-     * @param fat when <code>true</code>, wrappers will be made all pointing to a single pex file.
-     */
-    public void setIsFat(boolean isFat) {
-        this.isFat = isFat;
-    }
-
     /**
      * TODO: Revisit if this is needed.
      *
@@ -122,6 +87,27 @@ public void addDependencies(Project project) {
     }
 
     public void makeTasks(Project project) {
-        project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class);
+        project.getTasks().maybeCreate(TASK_BUILD_PEX, BuildPexTask.class);
+    }
+
+    // For backward compatibility in build.gradle flies.
+    @Deprecated
+    public boolean isFatPex() {
+        return ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).isFat();
+    }
+
+    @Deprecated
+    public void setFatPex(boolean fatPex) {
+        ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).setIsFat(fatPex);
+     }
+
+    @Deprecated
+    public boolean isFat() {
+        return ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).isFat();
     }
+
+    @Deprecated
+    public void setIsFat(boolean isFat) {
+        ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).setIsFat(isFat);
+     }
 }
diff --git a/...plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappContainerExtension.java b/...plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappContainerExtension.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.extension;
+
+import com.linkedin.gradle.python.util.OperatingSystem;
+
+
+/**
+ * This class provides a stub in PythonExtension so that build.gradle files
+ * can use the following construct:
+ *
+ * python {
+ *     zipapp.isFat = true
+ * }
+ *
+ * This is the replacement for `python.pex.fatPex`, `python.pex.isFat`, and
+ * `python.shiv.isFat`.  The reason we need this is that we don't know which
+ * container format the user wants until *after* build.gradle is evaluated,
+ * but of course they want to set the isFat flag *in* their build.gradle.
+ */
+public class ZipappContainerExtension {
+    private boolean isFat = OperatingSystem.current().isWindows();
+
+    public boolean isFat() {
+        return isFat;
+    }
+
+    public void setIsFat(boolean isFat) {
+        this.isFat = isFat;
+    }
+}
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java
@@ -18,8 +18,10 @@
 import com.linkedin.gradle.python.PythonExtension;
 import com.linkedin.gradle.python.extension.DeployableExtension;
 import com.linkedin.gradle.python.extension.PexExtension;
+import com.linkedin.gradle.python.extension.ZipappContainerExtension;
 import com.linkedin.gradle.python.tasks.BuildWheelsTask;
 import com.linkedin.gradle.python.tasks.NoopBuildPexTask;
+import com.linkedin.gradle.python.tasks.NoopTask;
 import com.linkedin.gradle.python.tasks.PythonContainerTask;
 import com.linkedin.gradle.python.util.ApplicationContainer;
 import com.linkedin.gradle.python.util.ExtensionUtils;
@@ -42,6 +44,8 @@ public void applyTo(final Project project) {
         final DeployableExtension deployableExtension = ExtensionUtils.maybeCreateDeployableExtension(project);
         final ApplicationContainer applicationContainer = pythonExtension.getApplicationContainer();
 
+        ExtensionUtils.maybeCreate(project, "zipapp", ZipappContainerExtension.class);
+
         applicationContainer.addExtensions(project);
 
         /*
@@ -53,7 +57,6 @@ public void applyTo(final Project project) {
          */
         TaskContainer tasks = project.getTasks();
 
-
         // Add this no-op task for backward compatibility.  See PexExtension for details.
         Task noop = tasks.findByName(PexExtension.TASK_BUILD_NOOP_PEX);
         if (noop == null) {
@@ -112,19 +115,21 @@ public void applyTo(final Project project) {
             postContainer.addDependencies(project);
             postContainer.makeTasks(project);
 
-            NoopBuildPexTask noopTask = (NoopBuildPexTask) tasks.findByName(PexExtension.TASK_BUILD_NOOP_PEX);
-            noopTask.suppressWarning = true;
-
             Task assemble = tasks.getByName(ApplicationContainer.TASK_ASSEMBLE_CONTAINERS);
             Task parent = tasks.getByName(ApplicationContainer.TASK_BUILD_PROJECT_WHEEL);
 
             for (Task task : tasks.withType(PythonContainerTask.class)) {
+                if (task instanceof NoopTask) {
+                    ((NoopTask) task).setSuppressWarning(true);
+                }
+
                 assemble.dependsOn(task);
                 task.dependsOn(parent);
-            }
 
-            // Turn the warning back on.
-            noopTask.suppressWarning = false;
+                if (task instanceof NoopTask) {
+                    ((NoopTask) task).setSuppressWarning(false);
+                }
+            }
         });
     }
 }
diff --git a/...-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java b/...-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java
@@ -15,9 +15,10 @@
  */
 package com.linkedin.gradle.python.plugin;
 
-import com.linkedin.gradle.python.util.ApplicationContainer;
 import com.linkedin.gradle.python.extension.DeployableExtension;
+import com.linkedin.gradle.python.extension.PexExtension;
 import com.linkedin.gradle.python.tasks.BuildWebAppTask;
+import com.linkedin.gradle.python.util.ApplicationContainer;
 import com.linkedin.gradle.python.util.ExtensionUtils;
 import org.gradle.api.Project;
 
@@ -33,11 +34,15 @@ public class PythonWebApplicationPlugin extends PythonBasePlugin {
 
     @Override
     public void applyTo(final Project project) {
-
         project.getPlugins().apply(PythonContainerPlugin.class);
 
         final DeployableExtension deployableExtension = ExtensionUtils.maybeCreateDeployableExtension(project);
 
+        // 2019-04-11(warsaw): FIXME: For now, we're still hard coding pex
+        // for the gunicorn file.  Make sure the `pex` dependency is
+        // installed.
+        new PexExtension(project).addDependencies(project);
+
         /*
          * Build a gunicorn pex file.
          *

diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java
@@ -17,6 +17,7 @@
 
 import com.linkedin.gradle.python.extension.DeployableExtension;
 import com.linkedin.gradle.python.extension.PexExtension;
+import com.linkedin.gradle.python.extension.ZipappContainerExtension;
 import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider;
 import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer;
 import com.linkedin.gradle.python.util.ExtensionUtils;
@@ -79,6 +80,8 @@ public void buildPex() throws Exception {
 
         DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class);
         PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);
+        ZipappContainerExtension zipappExtension = ExtensionUtils.getPythonComponentExtension(
+            project, ZipappContainerExtension.class);
 
         // Recreate the pex cache if it exists so that we don't mistakenly use an old build's version of the local project.
         if (pexExtension.getPexCache().exists()) {
@@ -88,7 +91,7 @@ public void buildPex() throws Exception {
 
         deployableExtension.getDeployableBuildDir().mkdirs();
 
-        if (pexExtension.isFat()) {
+        if (zipappExtension.isFat()) {
             new FatPexGenerator(project, pexOptions).buildEntryPoints();
         } else {
             new ThinPexGenerator(project, pexOptions, templateProvider, additionalProperties).buildEntryPoints();

diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java
@@ -16,7 +16,7 @@
 package com.linkedin.gradle.python.tasks;
 
 import com.linkedin.gradle.python.PythonExtension;
-import com.linkedin.gradle.python.extension.PexExtension;
+import com.linkedin.gradle.python.extension.ZipappContainerExtension;
 import com.linkedin.gradle.python.util.ExtensionUtils;
 import com.linkedin.gradle.python.util.PexFileUtil;
 import com.linkedin.gradle.python.util.entrypoint.EntryPointWriter;
@@ -68,10 +68,17 @@ public String getEntryPoint() {
     @TaskAction
     public void buildWebapp() throws IOException, ClassNotFoundException {
         Project project = getProject();
-        PexExtension extension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);
         PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project);
-
-        if (extension.isFat()) {
+        ZipappContainerExtension zipappExtension = ExtensionUtils.getPythonComponentExtension(
+            project, ZipappContainerExtension.class);
+
+        // Regardless of whether fat or thin zipapps are used, the container
+        // plugin will build the right container (i.e. .pex or .pyz).
+        // However, for thin zipapps, we need additional wrapper scripts
+        // generated (e.g. the gunicorn wrapper).
+        if (zipappExtension.isFat()) {
+            // 2019-04-11(warsaw): FIXME: For now, we're still hard coding pex
+            // for the gunicorn file.
             new FatPexGenerator(project, pexOptions).buildEntryPoint(
                 PexFileUtil.createFatPexFilename(executable.getName()), entryPoint, null);
         } else {
@@ -104,5 +111,4 @@ public EntryPointTemplateProvider getTemplateProvider() {
     public void setTemplateProvider(EntryPointTemplateProvider templateProvider) {
         this.templateProvider = templateProvider;
     }
-
 }
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java
@@ -22,7 +22,7 @@
 import org.gradle.api.tasks.TaskAction;
 
 
-public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask {
+public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask, NoopTask {
     private static final Logger LOG = Logging.getLogger(NoopBuildPexTask.class);
     private static final String DISABLED_MESSAGE =
           "######################### WARNING ##########################\n"
@@ -32,7 +32,7 @@ public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask
 
     // This is used to suppress the warning when PythonContainerPlugin plumbs
     // this task into the task hierarchy, which isn't user code.
-    public boolean suppressWarning = false;
+    private boolean suppressWarning = false;
 
     @TaskAction
     public void noOp() { }
@@ -43,4 +43,12 @@ public Task dependsOn(Object... paths) {
         }
         return super.dependsOn(paths);
     }
+
+    public boolean suppressWarning() {
+        return suppressWarning;
+    }
+
+    public void setSuppressWarning(boolean suppressWarning) {
+        this.suppressWarning = suppressWarning;
+    }
 }