diff --git a/docs/changelog.md b/docs/changelog.md index ebc47b17..5b357baa 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # pygradle news # +* 2019-04-08 + - You must now change any usages of either the `python.pex.FatPex` or + `python.pex.isFat` flag to `python.zipapp.isFat`. + * 2018-11-09 - Added a default `mypy` task which you can enable by setting `python.mypy.run = true` in your `build.gradle` file. diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy index 876d6c34..4722e7a8 100644 --- a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy +++ b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PexIntegrationTest.groovy @@ -150,4 +150,61 @@ class PexIntegrationTest extends Specification { then: out.toString() == "Hello World${System.getProperty("line.separator")}".toString() } + + def "can build fat pex with isFat"() { + given: + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | version = '1.0.0' + | python { + | pex { + | isFat = true + | } + | } + | ${PyGradleTestBuilder.createRepoClosure()} + """.stripMargin().stripIndent() + + when: + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('build', '--stacktrace') + .withPluginClasspath() + .withDebug(true) + .build() + println result.output + + then: + + result.output.contains("BUILD SUCCESS") + result.task(':foo:flake8').outcome == TaskOutcome.SUCCESS + result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS + result.task(':foo:installTestRequirements').outcome == TaskOutcome.SUCCESS + result.task(':foo:createVirtualEnvironment').outcome == TaskOutcome.SUCCESS + result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS + result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS + result.task(':foo:check').outcome == TaskOutcome.SUCCESS + result.task(':foo:build').outcome == TaskOutcome.SUCCESS + result.task(':foo:buildPex').outcome == TaskOutcome.SUCCESS + result.task(':foo:assembleContainers').outcome == TaskOutcome.SUCCESS + + Path deployablePath = testProjectDir.root.toPath().resolve(Paths.get('foo', 'build', 'deployable', "bin")) + def pexFile = deployablePath.resolve(PexFileUtil.createFatPexFilename('hello_world')) + + pexFile.toFile().exists() + + when: "we have a pex file" + def line = new String(pexFile.bytes, "UTF-8").substring(0, 100) + + then: "its shebang line is not pointing to a virtualenv" + line.startsWith("#!") && !line.contains("venv") + + when: + def out = ExecUtils.run(pexFile) + println out + + then: + out.toString() == "Hello World${System.getProperty("line.separator")}".toString() + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java index 13373f9f..e09d5733 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java @@ -19,7 +19,6 @@ import com.linkedin.gradle.python.tasks.BuildPexTask; import com.linkedin.gradle.python.util.ApplicationContainer; import com.linkedin.gradle.python.util.ExtensionUtils; -import com.linkedin.gradle.python.util.OperatingSystem; import com.linkedin.gradle.python.util.StandardTextValues; import org.gradle.api.Project; @@ -39,12 +38,12 @@ public class PexExtension implements ApplicationContainer { public static final String TASK_BUILD_NOOP_PEX = "buildPex"; private File cache; - // Default to fat zipapps on Windows, since our wrappers are fairly POSIX specific. - private boolean isFat = OperatingSystem.current().isWindows(); private boolean pythonWrapper = true; + private Project project; public PexExtension(Project project) { this.cache = new File(project.getBuildDir(), "pex-cache"); + this.project = project; } public File getPexCache() { @@ -55,40 +54,6 @@ public void setPexCache(File pexCache) { cache = pexCache; } - // These are kept for API backward compatibility. - - /** - * @return when true, then skinny pex's will be used. - */ - @Deprecated - public boolean isFatPex() { - return isFat(); - } - - /** - * @param fatPex when true, wrappers will be made all pointing to a single pex file. - */ - @Deprecated - public void setFatPex(boolean fatPex) { - isFat = fatPex; - } - - // Use these properties instead. - - /** - * @return when true, then skinny pex's will be used. - */ - public boolean isFat() { - return isFat; - } - - /** - * @param fat when true, wrappers will be made all pointing to a single pex file. - */ - public void setIsFat(boolean isFat) { - this.isFat = isFat; - } - /** * TODO: Revisit if this is needed. * @@ -122,6 +87,27 @@ public void addDependencies(Project project) { } public void makeTasks(Project project) { - project.getTasks().create(TASK_BUILD_PEX, BuildPexTask.class); + project.getTasks().maybeCreate(TASK_BUILD_PEX, BuildPexTask.class); + } + + // For backward compatibility in build.gradle flies. + @Deprecated + public boolean isFatPex() { + return ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).isFat(); + } + + @Deprecated + public void setFatPex(boolean fatPex) { + ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).setIsFat(fatPex); + } + + @Deprecated + public boolean isFat() { + return ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).isFat(); } + + @Deprecated + public void setIsFat(boolean isFat) { + ExtensionUtils.getPythonComponentExtension(project, ZipappContainerExtension.class).setIsFat(isFat); + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappContainerExtension.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappContainerExtension.java new file mode 100644 index 00000000..762418c7 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappContainerExtension.java @@ -0,0 +1,44 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.extension; + +import com.linkedin.gradle.python.util.OperatingSystem; + + +/** + * This class provides a stub in PythonExtension so that build.gradle files + * can use the following construct: + * + * python { + * zipapp.isFat = true + * } + * + * This is the replacement for `python.pex.fatPex`, `python.pex.isFat`, and + * `python.shiv.isFat`. The reason we need this is that we don't know which + * container format the user wants until *after* build.gradle is evaluated, + * but of course they want to set the isFat flag *in* their build.gradle. + */ +public class ZipappContainerExtension { + private boolean isFat = OperatingSystem.current().isWindows(); + + public boolean isFat() { + return isFat; + } + + public void setIsFat(boolean isFat) { + this.isFat = isFat; + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java index 2a2057a2..97662386 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonContainerPlugin.java @@ -18,8 +18,10 @@ import com.linkedin.gradle.python.PythonExtension; import com.linkedin.gradle.python.extension.DeployableExtension; import com.linkedin.gradle.python.extension.PexExtension; +import com.linkedin.gradle.python.extension.ZipappContainerExtension; import com.linkedin.gradle.python.tasks.BuildWheelsTask; import com.linkedin.gradle.python.tasks.NoopBuildPexTask; +import com.linkedin.gradle.python.tasks.NoopTask; import com.linkedin.gradle.python.tasks.PythonContainerTask; import com.linkedin.gradle.python.util.ApplicationContainer; import com.linkedin.gradle.python.util.ExtensionUtils; @@ -42,6 +44,8 @@ public void applyTo(final Project project) { final DeployableExtension deployableExtension = ExtensionUtils.maybeCreateDeployableExtension(project); final ApplicationContainer applicationContainer = pythonExtension.getApplicationContainer(); + ExtensionUtils.maybeCreate(project, "zipapp", ZipappContainerExtension.class); + applicationContainer.addExtensions(project); /* @@ -53,7 +57,6 @@ public void applyTo(final Project project) { */ TaskContainer tasks = project.getTasks(); - // Add this no-op task for backward compatibility. See PexExtension for details. Task noop = tasks.findByName(PexExtension.TASK_BUILD_NOOP_PEX); if (noop == null) { @@ -112,19 +115,21 @@ public void applyTo(final Project project) { postContainer.addDependencies(project); postContainer.makeTasks(project); - NoopBuildPexTask noopTask = (NoopBuildPexTask) tasks.findByName(PexExtension.TASK_BUILD_NOOP_PEX); - noopTask.suppressWarning = true; - Task assemble = tasks.getByName(ApplicationContainer.TASK_ASSEMBLE_CONTAINERS); Task parent = tasks.getByName(ApplicationContainer.TASK_BUILD_PROJECT_WHEEL); for (Task task : tasks.withType(PythonContainerTask.class)) { + if (task instanceof NoopTask) { + ((NoopTask) task).setSuppressWarning(true); + } + assemble.dependsOn(task); task.dependsOn(parent); - } - // Turn the warning back on. - noopTask.suppressWarning = false; + if (task instanceof NoopTask) { + ((NoopTask) task).setSuppressWarning(false); + } + } }); } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java index fb09c9f3..8cb47938 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonWebApplicationPlugin.java @@ -15,9 +15,10 @@ */ package com.linkedin.gradle.python.plugin; -import com.linkedin.gradle.python.util.ApplicationContainer; import com.linkedin.gradle.python.extension.DeployableExtension; +import com.linkedin.gradle.python.extension.PexExtension; import com.linkedin.gradle.python.tasks.BuildWebAppTask; +import com.linkedin.gradle.python.util.ApplicationContainer; import com.linkedin.gradle.python.util.ExtensionUtils; import org.gradle.api.Project; @@ -33,11 +34,15 @@ public class PythonWebApplicationPlugin extends PythonBasePlugin { @Override public void applyTo(final Project project) { - project.getPlugins().apply(PythonContainerPlugin.class); final DeployableExtension deployableExtension = ExtensionUtils.maybeCreateDeployableExtension(project); + // 2019-04-11(warsaw): FIXME: For now, we're still hard coding pex + // for the gunicorn file. Make sure the `pex` dependency is + // installed. + new PexExtension(project).addDependencies(project); + /* * Build a gunicorn pex file. * diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java index a195a9b9..a72b65af 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java @@ -17,6 +17,7 @@ import com.linkedin.gradle.python.extension.DeployableExtension; import com.linkedin.gradle.python.extension.PexExtension; +import com.linkedin.gradle.python.extension.ZipappContainerExtension; import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider; import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer; import com.linkedin.gradle.python.util.ExtensionUtils; @@ -79,6 +80,8 @@ public void buildPex() throws Exception { DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(project, DeployableExtension.class); PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class); + ZipappContainerExtension zipappExtension = ExtensionUtils.getPythonComponentExtension( + project, ZipappContainerExtension.class); // Recreate the pex cache if it exists so that we don't mistakenly use an old build's version of the local project. if (pexExtension.getPexCache().exists()) { @@ -88,7 +91,7 @@ public void buildPex() throws Exception { deployableExtension.getDeployableBuildDir().mkdirs(); - if (pexExtension.isFat()) { + if (zipappExtension.isFat()) { new FatPexGenerator(project, pexOptions).buildEntryPoints(); } else { new ThinPexGenerator(project, pexOptions, templateProvider, additionalProperties).buildEntryPoints(); diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java index 243cc545..b396b4d9 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java @@ -16,7 +16,7 @@ package com.linkedin.gradle.python.tasks; import com.linkedin.gradle.python.PythonExtension; -import com.linkedin.gradle.python.extension.PexExtension; +import com.linkedin.gradle.python.extension.ZipappContainerExtension; import com.linkedin.gradle.python.util.ExtensionUtils; import com.linkedin.gradle.python.util.PexFileUtil; import com.linkedin.gradle.python.util.entrypoint.EntryPointWriter; @@ -68,10 +68,17 @@ public String getEntryPoint() { @TaskAction public void buildWebapp() throws IOException, ClassNotFoundException { Project project = getProject(); - PexExtension extension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class); PythonExtension pythonExtension = ExtensionUtils.getPythonExtension(project); - - if (extension.isFat()) { + ZipappContainerExtension zipappExtension = ExtensionUtils.getPythonComponentExtension( + project, ZipappContainerExtension.class); + + // Regardless of whether fat or thin zipapps are used, the container + // plugin will build the right container (i.e. .pex or .pyz). + // However, for thin zipapps, we need additional wrapper scripts + // generated (e.g. the gunicorn wrapper). + if (zipappExtension.isFat()) { + // 2019-04-11(warsaw): FIXME: For now, we're still hard coding pex + // for the gunicorn file. new FatPexGenerator(project, pexOptions).buildEntryPoint( PexFileUtil.createFatPexFilename(executable.getName()), entryPoint, null); } else { @@ -104,5 +111,4 @@ public EntryPointTemplateProvider getTemplateProvider() { public void setTemplateProvider(EntryPointTemplateProvider templateProvider) { this.templateProvider = templateProvider; } - } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java index 58599acf..9f847f97 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopBuildPexTask.java @@ -22,7 +22,7 @@ import org.gradle.api.tasks.TaskAction; -public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask { +public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask, NoopTask { private static final Logger LOG = Logging.getLogger(NoopBuildPexTask.class); private static final String DISABLED_MESSAGE = "######################### WARNING ##########################\n" @@ -32,7 +32,7 @@ public class NoopBuildPexTask extends DefaultTask implements PythonContainerTask // This is used to suppress the warning when PythonContainerPlugin plumbs // this task into the task hierarchy, which isn't user code. - public boolean suppressWarning = false; + private boolean suppressWarning = false; @TaskAction public void noOp() { } @@ -43,4 +43,12 @@ public Task dependsOn(Object... paths) { } return super.dependsOn(paths); } + + public boolean suppressWarning() { + return suppressWarning; + } + + public void setSuppressWarning(boolean suppressWarning) { + this.suppressWarning = suppressWarning; + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/ZipappContainer.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopTask.java similarity index 61% rename from pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/ZipappContainer.java rename to pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopTask.java index 22e3a9f8..b5876354 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/ZipappContainer.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/NoopTask.java @@ -13,16 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.linkedin.gradle.python.util; -public interface ZipappContainer extends ApplicationContainer { - /** - * @return when true, then skinny pex's will be used. - */ - public boolean isFat(); +package com.linkedin.gradle.python.tasks; - /** - * @param fat when true, wrappers will be made all pointing to a single pex file. - */ - public void setIsFat(boolean isFat); + +public interface NoopTask { + public boolean suppressWarning(); + + public void setSuppressWarning(boolean warning); } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/entrypoint/EntryPointWriter.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/entrypoint/EntryPointWriter.java index 195a6bc3..01d97b45 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/entrypoint/EntryPointWriter.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/entrypoint/EntryPointWriter.java @@ -15,17 +15,18 @@ */ package com.linkedin.gradle.python.util.entrypoint; +import com.linkedin.gradle.python.extension.CliExtension; +import com.linkedin.gradle.python.extension.ZipappContainerExtension; import com.linkedin.gradle.python.util.ExtensionUtils; import groovy.text.SimpleTemplateEngine; import org.apache.commons.io.FileUtils; import org.gradle.api.Project; -import com.linkedin.gradle.python.extension.CliExtension; -import com.linkedin.gradle.python.util.ZipappContainer; import java.io.File; import java.io.IOException; import java.util.Map; + public class EntryPointWriter { private final String template; @@ -36,7 +37,7 @@ public EntryPointWriter(Project project, String template) { this.template = template; this.isCliTool = ExtensionUtils.findPythonComponentExtension(project, CliExtension.class) != null; - this.isZipapp = ExtensionUtils.findPythonComponentExtension(project, ZipappContainer.class) != null; + this.isZipapp = ExtensionUtils.findPythonComponentExtension(project, ZipappContainerExtension.class) != null; } public void writeEntryPoint(File location, Map properties) throws IOException, ClassNotFoundException {