From d1f50ab3b557816e1769e03fb86e1df01dfdd157 Mon Sep 17 00:00:00 2001 From: Ethan Hall Date: Tue, 24 Jul 2018 17:32:58 -0700 Subject: [PATCH] Refactored IntegTest to become unit test. (#237) * Refactored IntegTest to become unit test. PackageSettingsIntegrationTest was flakey and really just wanted to ensure that specific arguments were passed to pip. So now of running a build, we will validate that the arguments would have been passed to pip. This makes the tests fast and not rely on the env. It also means that we don't need to do any strange regex's and rely on side affects of the build processes to validate things. The previous test was asserting that Gradle was passing args to the commandline, and that's the same as asserting that our code was calling Gradle with the correct parameters. This change splits the integ test into two tests, because it was testing two different things. `pip install` and `pip wheel`, so now there is one test for each. To do this, two actions were extracted. PipInstall and PipWheel. They both use the same base class so we can re-use as much code as possible. * Addressing Zvezdan's comments - Deleted unused class - Fixed some package declerations - Using PipWheelAction in BuildWheelTask * Updated tests to assert rebuild of wheels. Other misc changes to codebase based on feedback * Moving code to Java Noticed that this was all java, so removing the Groovy plugin and the classes are moved into the Java sourceSet * Added more test coverage Tests now verity skipping behavior * Test location matches package definition * Fixed FailureReasonProvider for BuildWheels This change adds an integ test a few tasks that also use FailureReasonProvider. AbstractPythonMainSourceDefaultTask was updated to always add details to the result call. --- buildSrc/build.gradle | 1 - .../linkedin/gradle/build/package-info.java | 0 .../gradle/build/version/Version.java | 0 .../gradle/build/version/VersionFile.java | 0 .../gradle/build/version/VersionPlugin.java | 0 gradle/wrapper/gradle-wrapper.properties | 2 +- ...ailureReasonProviderIntegrationTest.groovy | 128 ++++ .../PackageSettingsIntegrationTest.groovy | 594 ------------------ .../exception/PipExecutionException.java | 53 ++ .../python/extension/VirtualEnvironment.java | 12 +- .../AbstractPythonMainSourceDefaultTask.java | 42 +- .../python/tasks/BuildWheelsTask.groovy | 120 +--- .../gradle/python/tasks/PipInstallTask.groovy | 152 +---- .../tasks/action/pip/AbstractPipAction.java | 117 ++++ .../tasks/action/pip/PipInstallAction.java | 147 +++++ .../tasks/action/pip/PipWheelAction.java | 133 ++++ .../python/tasks/exec/ExternalExec.java | 24 + .../tasks/exec/ProjectExternalExec.java | 35 ++ .../gradle/python/util/PackageInfo.groovy | 8 + .../extension/PythonDetailsTestDouble.java | 38 ++ .../tasks/action/pip/PipActionHelpers.groovy | 116 ++++ .../action/pip/PipInstallActionTest.groovy | 186 ++++++ .../action/pip/PipWheelActionTest.groovy | 185 ++++++ .../tasks/exec/ExternalExecTestDouble.groovy | 35 ++ .../gradle/python/util/PackageInfoTest.groovy | 2 +- 25 files changed, 1288 insertions(+), 842 deletions(-) rename buildSrc/src/main/{groovy => java}/com/linkedin/gradle/build/package-info.java (100%) rename buildSrc/src/main/{groovy => java}/com/linkedin/gradle/build/version/Version.java (100%) rename buildSrc/src/main/{groovy => java}/com/linkedin/gradle/build/version/VersionFile.java (100%) rename buildSrc/src/main/{groovy => java}/com/linkedin/gradle/build/version/VersionPlugin.java (100%) create mode 100644 pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/FailureReasonProviderIntegrationTest.groovy delete mode 100644 pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/exception/PipExecutionException.java create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/AbstractPipAction.java create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelAction.java create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExec.java create mode 100644 pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ProjectExternalExec.java create mode 100644 pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/extension/PythonDetailsTestDouble.java create mode 100644 pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy create mode 100644 pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy create mode 100644 pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelActionTest.groovy create mode 100644 pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecTestDouble.groovy diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 48f9a0c4..4873d569 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -1,6 +1,5 @@ plugins { id "java-gradle-plugin" - id 'groovy' id 'idea' } diff --git a/buildSrc/src/main/groovy/com/linkedin/gradle/build/package-info.java b/buildSrc/src/main/java/com/linkedin/gradle/build/package-info.java similarity index 100% rename from buildSrc/src/main/groovy/com/linkedin/gradle/build/package-info.java rename to buildSrc/src/main/java/com/linkedin/gradle/build/package-info.java diff --git a/buildSrc/src/main/groovy/com/linkedin/gradle/build/version/Version.java b/buildSrc/src/main/java/com/linkedin/gradle/build/version/Version.java similarity index 100% rename from buildSrc/src/main/groovy/com/linkedin/gradle/build/version/Version.java rename to buildSrc/src/main/java/com/linkedin/gradle/build/version/Version.java diff --git a/buildSrc/src/main/groovy/com/linkedin/gradle/build/version/VersionFile.java b/buildSrc/src/main/java/com/linkedin/gradle/build/version/VersionFile.java similarity index 100% rename from buildSrc/src/main/groovy/com/linkedin/gradle/build/version/VersionFile.java rename to buildSrc/src/main/java/com/linkedin/gradle/build/version/VersionFile.java diff --git a/buildSrc/src/main/groovy/com/linkedin/gradle/build/version/VersionPlugin.java b/buildSrc/src/main/java/com/linkedin/gradle/build/version/VersionPlugin.java similarity index 100% rename from buildSrc/src/main/groovy/com/linkedin/gradle/build/version/VersionPlugin.java rename to buildSrc/src/main/java/com/linkedin/gradle/build/version/VersionPlugin.java diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3d3b2ff8..503b6717 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-all.zip diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/FailureReasonProviderIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/FailureReasonProviderIntegrationTest.groovy new file mode 100644 index 00000000..51e25351 --- /dev/null +++ b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/FailureReasonProviderIntegrationTest.groovy @@ -0,0 +1,128 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.plugin + + +import com.linkedin.gradle.python.plugin.testutils.DefaultProjectLayoutRule +import com.linkedin.gradle.python.plugin.testutils.PyGradleTestBuilder +import org.gradle.testkit.runner.GradleRunner +import org.junit.Rule +import spock.lang.Specification + +class FailureReasonProviderIntegrationTest extends Specification { + + @Rule + final DefaultProjectLayoutRule testProjectDir = new DefaultProjectLayoutRule() + + @SuppressWarnings("GStringExpressionWithinString") + def 'will report errors nicely'() { + given: + testProjectDir.buildFile << """\ + | plugins { + | id 'com.linkedin.python-pex' + | } + | + | configurations { + | testProj + | } + | + | dependencies { + | testProj 'pypi:pyflakes:+' + | } + | + | task testWheelTask(type: com.linkedin.gradle.python.tasks.BuildWheelsTask) { + | dependsOn 'installProject' + | packageSettings = new DefaultTestPackageSettings(project.projectDir) + | installFileCollection = configurations.testProj + | } + | + | import com.linkedin.gradle.python.util.DefaultPackageSettings + | import com.linkedin.gradle.python.util.PackageInfo + | import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider + | + | class DefaultTestPackageSettings extends DefaultPackageSettings { + | DefaultTestPackageSettings(File projectDir) { super(projectDir) } + | + | @Override + | List getSupportedLanguageVersions(PackageInfo packageInfo) { + | return ['2.8'] + | } + | } + | + | class MyTaskListener implements TaskExecutionListener { + | + | @Override + | void beforeExecute(Task task) { + | } + | + | @Override + | void afterExecute(Task task, TaskState taskState) { + | if (task instanceof FailureReasonProvider && taskState.failure != null) { + | println(task.reason.readLines().collect { "#\${task.name}>> \$it" }.join("\\n")) + | } + | } + | } + | + | project.gradle.addListener(new MyTaskListener()) + | + | version = '1.0.0' + | ${ PyGradleTestBuilder.createRepoClosure() } + """.stripMargin().stripIndent() + + testProjectDir.newFile('foo/test/foo.py').text = "import os #something" + + when: + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('testWheelTask', "pytest", "flake8", '--continue') + .withPluginClasspath() + .withDebug(true) + .buildAndFail() + println result.output + + def outputLines = result.output.readLines() + .findAll { it.startsWith("#") && it.contains(">>") } + .collect { it.replace(testProjectDir.root.absolutePath, '/root') } + + then: + // Output structure is "#${taskName}>> ${message} + // Map is task name to lines it should have. Each line is a "contains" operation, not equals. + def expectedErrors = [ + 'testWheelTask': [ + 'Package pyflakes works only with Python versions: [2.8]' + ], + 'pytest' : [ + "Traceback:", + "test/test_a.py:1: in ", + " from foo.hello import generate_welcome", + "E ImportError: No module named hello", + ], + 'flake8' : [ + "/foo/test/foo.py:1:1: F401 'os' imported but unused", + "/foo/test/foo.py:1:10: E261 at least two spaces before inline comment", + "/foo/test/foo.py:1:11: E262 inline comment should start with '# '" + ] + ] + expectedErrors.each { task, lines -> + def taskLines = outputLines + .findAll { it.startsWith("#$task>> ") } + .collect { it.replace("#$task>> ", '') } + lines.each { line -> + assert taskLines.any { it.contains(line) } + } + } + } +} diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy deleted file mode 100644 index 3dd9f08a..00000000 --- a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PackageSettingsIntegrationTest.groovy +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.linkedin.gradle.python.plugin - -import com.linkedin.gradle.python.plugin.testutils.DefaultProjectLayoutRule -import com.linkedin.gradle.python.plugin.testutils.PyGradleTestBuilder -import org.gradle.testkit.runner.GradleRunner -import org.gradle.testkit.runner.TaskOutcome -import org.gradle.testkit.runner.UnexpectedBuildFailure -import org.junit.Rule -import spock.lang.Specification - - -class PackageSettingsIntegrationTest extends Specification { - - @Rule - final DefaultProjectLayoutRule testProjectDir = new DefaultProjectLayoutRule() - - /////////////////////////////////// - // PipInstallTask specific tests // - /////////////////////////////////// - - def "pip install uses environment"() { - given: "package settings for 'foo' have custom environment" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-sdist' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | Map getEnvironment(PackageInfo packageInfo) { - | return (packageInfo.name != 'foo') ? [:] : [ - | 'CPPFLAGS': '-I/some/custom/path/include', - | 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib', - | 'DUMMY_MAP': '{\\n}', - | ] - | } - | } - | - | import com.linkedin.gradle.python.tasks.PipInstallTask - | - | project.tasks.withType(PipInstallTask) { PipInstallTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project with debug enabled" - def result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('-d', 'build') - .withPluginClasspath() - .withDebug(true) - .build() - println result.output - - then: "we can observe the environment for 'foo' provided a few lines after its install is logged" - def match = result.output.find(/Installing foo[\s\S]+?Environment for[^\n]+: \{[\s\S]+?\}\s*\n/) - match != null - match.findAll('Installing ').size() == 1 - match.contains('CPPFLAGS=-I/some/custom/path/include') - match.contains('LDFLAGS=-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib') - result.output.contains('BUILD SUCCESS') - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - } - - /* - * We are using setuptools for option testing because it's the first - * package installed and it will fail very early. This shortens the - * integration tests as much as possible. - */ - - def "pip install uses global options"() { - given: "package settings for 'setuptools' have global options" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-sdist' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getGlobalOptions(PackageInfo packageInfo) { - | return (packageInfo.name == 'setuptools') ? ['--global-option', '--dummy-global-option'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.PipInstallTask - | - | project.tasks.withType(PipInstallTask) { PipInstallTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe global options being passed to 'setuptools' and failing because it does not expect them" - // the global option for setup.py is passed *before* install command - result.output.find(/setup.py[^\n]+ --dummy-global-option install/) - result.output.contains('Running setup.py install for setuptools') - result.output.contains('error: option --dummy-global-option not recognized') - result.output.contains('BUILD FAILED') - result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED - } - - def "pip install uses install options"() { - given: "package settings for 'setuptools' have install options" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-sdist' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getInstallOptions(PackageInfo packageInfo) { - | return (packageInfo.name == 'setuptools') ? ['--install-option', '--ignore=E123,E234'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.PipInstallTask - | - | project.tasks.withType(PipInstallTask) { PipInstallTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe install options being passed to 'setuptools' and failing because it does not expect them" - // the install option is passed *after* install command - result.output.find(/setup.py[^\n]+ install [^\n]+ --ignore=E123,E234/) - result.output.contains('Running setup.py install for setuptools') - result.output.contains('error: option --ignore not recognized') - result.output.contains('BUILD FAILED') - result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED - } - - def "pip install uses supported language versions"() { - given: "package settings for 'setuptools' have supported language versions" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-sdist' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getSupportedLanguageVersions(PackageInfo packageInfo) { - | return (packageInfo.name == 'setuptools') ? ['2.8'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.PipInstallTask - | - | project.tasks.withType(PipInstallTask) { PipInstallTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe supported language version being checked and failing for non-existent version" - result.output.contains('Package setuptools works only with Python versions: [2.8]') - result.output.contains('BUILD FAILED') - result.task(':foo:installSetupRequirements').outcome == TaskOutcome.FAILED - } - - /* - * We are using a dependency on pyflakes because it's also a transitive - * build dependency and will be re-installed as a runtime dependency. - */ - - def "pip install requires source rebuild"() { - given: "package settings for 'pyflakes' require a source rebuild" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-sdist' - | } - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | boolean requiresSourceBuild(PackageInfo packageInfo) { - | return (packageInfo.name == 'pyflakes') ? true : super.requiresSourceBuild(packageInfo) - | } - | } - | - | import com.linkedin.gradle.python.tasks.PipInstallTask - | - | project.tasks.withType(PipInstallTask) { PipInstallTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project with info enabled" - def result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('-i', 'build') - .withPluginClasspath() - .withDebug(true) - .build() - println result.output - - then: "we can observe that required source rebuild happens" - // pyflakes should be installed in build requirements, and then again in runtime requirements - result.output.findAll(/Installing pyflakes[\s\S]+? --ignore-installed [^\n]+pyflakes/).size() == 2 - result.output.contains('BUILD SUCCESS') - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - } - - //////////////////////////////////// - // BuildWheelsTask specific tests // - //////////////////////////////////// - - // We are using a dependency on pyflakes, just to have a wheel to build. - def "wheel uses environment"() { - given: "package settings for 'foo' have custom environment" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-pex' - | } - | - | version = '1.0.0' - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | Map getEnvironment(PackageInfo packageInfo) { - | return (packageInfo.name != 'foo') ? [:] : [ - | 'CPPFLAGS': '-I/some/custom/path/include', - | 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib', - | 'DUMMY_MAP': '{\\n}', - | ] - | } - | } - | - | import com.linkedin.gradle.python.tasks.BuildWheelsTask - | - | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project with debug enabled" - def result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('-d', 'build') - .withPluginClasspath() - .withDebug(true) - .build() - println result.output - - then: "we can observe the environment for 'foo' provided a few lines after its build is logged" - def match = result.output.find(/Installing foo\S* wheel[\s\S]+?Environment for[^\n]+: \{[\s\S]+?\}\s*\n/) - match != null - match.findAll('Installing ').size() == 1 - match.contains('CPPFLAGS=-I/some/custom/path/include') - match.contains('LDFLAGS=-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib') - result.output.contains('BUILD SUCCESS') - result.task(':foo:buildWheels').outcome == TaskOutcome.SUCCESS - result.task(':foo:buildProjectWheel').outcome == TaskOutcome.SUCCESS - } - - def "wheel uses global options"() { - given: "package settings for 'pyflakes' have global options" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-pex' - | } - | - | version = '1.0.0' - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getGlobalOptions(PackageInfo packageInfo) { - | return (packageInfo.name == 'pyflakes') ? ['--global-option', '--dummy-global-option'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.BuildWheelsTask - | - | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe global options being passed to 'pyflakes' and failing because it does not expect them" - // the global option for setup.py is passed *before* bdist_wheel command - result.output.find(/setup.py[^\n]+ --dummy-global-option bdist_wheel/) - result.output.contains('Failed building wheel for pyflakes') - result.output.contains('error: option --dummy-global-option not recognized') - result.output.contains('BUILD FAILED') - // the build of everything in the virtualenv succeeded - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - // but the wheel build failed - result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED - } - - def "wheel uses build options"() { - given: "package settings for 'pyflakes' have build options" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-pex' - | } - | - | version = '1.0.0' - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getBuildOptions(PackageInfo packageInfo) { - | return (packageInfo.name == 'pyflakes') ? ['--build-option', '--disable-something'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.BuildWheelsTask - | - | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe build options being passed to 'pyflakes' and failing because it does not expect them" - // the build option is passed *after* bdist_wheel command - result.output.find(/setup.py[^\n]+ bdist_wheel [^\n]+ --disable-something/) - result.output.contains('Failed building wheel for pyflakes') - result.output.contains('error: option --disable-something not recognized') - result.output.contains('BUILD FAILED') - // the build of everything in the virtualenv succeeded - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - // but the wheel build failed - result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED - } - - def "wheel uses supported language versions"() { - given: "package settings for 'pyflakes' have supported language versions" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-pex' - | } - | - | version = '1.0.0' - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | List getSupportedLanguageVersions(PackageInfo packageInfo) { - | return (packageInfo.name == 'pyflakes') ? ['2.8'] : [] - | } - | } - | - | import com.linkedin.gradle.python.tasks.BuildWheelsTask - | - | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project" - def result - try { - result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('build') - .withPluginClasspath() - .withDebug(true) - .build() - } catch (UnexpectedBuildFailure buildFailure) { - // expected to fail - result = buildFailure.buildResult - } - println result.output - - then: "we can observe supported language version being checked and failing for non-existent version" - result.output.contains('Package pyflakes works only with Python versions: [2.8]') - result.output.contains('BUILD FAILED') - // the build of everything in the virtualenv succeeded - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - // but the wheel build failed - result.task(':foo:buildWheels').outcome == TaskOutcome.FAILED - } - - def "wheel requires source rebuild"() { - given: "package settings for 'pyflakes' require source rebuild" - testProjectDir.buildFile << """\ - | plugins { - | id 'com.linkedin.python-pex' - | } - | - | version = '1.0.0' - | - | dependencies { - | python 'pypi:pyflakes:+' - | } - | - | import com.linkedin.gradle.python.util.DefaultPackageSettings - | import com.linkedin.gradle.python.util.PackageInfo - | - | class DefaultTestPackageSettings extends DefaultPackageSettings { - | DefaultTestPackageSettings(File projectDir) { super(projectDir) } - | - | @Override - | boolean requiresSourceBuild(PackageInfo packageInfo) { - | return (packageInfo.name == 'pyflakes') ? true : super.requiresSourceBuild(packageInfo) - | } - | } - | - | import com.linkedin.gradle.python.tasks.BuildWheelsTask - | - | project.tasks.withType(BuildWheelsTask) { BuildWheelsTask task -> - | task.packageSettings = new DefaultTestPackageSettings(project.projectDir) - | } - | - |${PyGradleTestBuilder.createRepoClosure()} - """.stripMargin().stripIndent() - - when: "we build a project with info enabled" - def result = GradleRunner.create() - .withProjectDir(testProjectDir.root) - .withArguments('-i', 'build') - .withPluginClasspath() - .withDebug(true) - .build() - println result.output - - then: "we can observe that required source rebuild happens" - // pyflakes wheel should be re-installed after install in build and runtime requirements of virtualenv. - result.output.findAll(/Installing pyflakes[\s\S]+?pip install [^\n]+pyflakes/).size() == 2 - result.output.find(/Installing pyflakes\S+ wheel[\s\S]+?pip wheel [^\n]+pyflakes/) - result.output.contains('BUILD SUCCESS') - result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS - result.task(':foo:buildWheels').outcome == TaskOutcome.SUCCESS - } - -} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/exception/PipExecutionException.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/exception/PipExecutionException.java new file mode 100644 index 00000000..b8c72bb0 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/exception/PipExecutionException.java @@ -0,0 +1,53 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.exception; + +import com.linkedin.gradle.python.util.PackageInfo; +import org.gradle.api.GradleException; + +import java.util.List; + +public class PipExecutionException extends GradleException { + private final String pipText; + + private PipExecutionException(String message, String pipText) { + super(message); + this.pipText = pipText; + } + + public String getPipText() { + return pipText; + } + + public static PipExecutionException failedInstall(PackageInfo packageInfo, String pipText) { + String body = String.format("Failed to install %s. " + + "Please see above output for reason, or re-run your build using " + + "``gradle -i build`` for additional logging.", packageInfo.toShortHand()); + return new PipExecutionException(body, pipText); + } + + public static PipExecutionException failedWheel(PackageInfo packageInfo, String pipText) { + String body = String.format("Failed to build wheel for %s. " + + "Please see above output for reason, or re-run your build using " + + "``gradle -i build`` for additional logging.", packageInfo.toShortHand()); + return new PipExecutionException(body, pipText); + } + + public static PipExecutionException unsupportedPythonVersion(PackageInfo packageInfo, List supportedVersions) { + String message = String.format("Package %s works only with Python versions: %s", packageInfo.getName(), supportedVersions); + return new PipExecutionException(message, message); + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/VirtualEnvironment.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/VirtualEnvironment.java index b30d533d..b932cedd 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/VirtualEnvironment.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/VirtualEnvironment.java @@ -51,16 +51,20 @@ public static String getPythonApplicationDirectory() { return OperatingSystem.current().isWindows() ? "Scripts" : "bin"; } - public File getExecutable(String path) { - return validateFileExists(prefixBuilder().resolve(OperatingSystem.current().getExecutableName(path))); + public File getExecutable(String execName) { + return validateFileExists(findExecutable(prefixBuilder(), execName)); } public File getScript(String path) { return validateFileExists(prefixBuilder().resolve(OperatingSystem.current().getScriptName(path))); } - public File findExecutable(String path) { - return prefixBuilder().resolve(OperatingSystem.current().getExecutableName(path)).toFile(); + public File findExecutable(String name) { + return findExecutable(prefixBuilder(), name).toFile(); + } + + public static Path findExecutable(Path path, String name) { + return path.resolve(OperatingSystem.current().getExecutableName(name)); } private Path prefixBuilder() { diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/AbstractPythonMainSourceDefaultTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/AbstractPythonMainSourceDefaultTask.java index 8543ec06..af5c0dd2 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/AbstractPythonMainSourceDefaultTask.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/AbstractPythonMainSourceDefaultTask.java @@ -17,9 +17,10 @@ import com.linkedin.gradle.python.PythonExtension; import com.linkedin.gradle.python.extension.PythonDetails; +import com.linkedin.gradle.python.tasks.exec.ExternalExec; +import com.linkedin.gradle.python.tasks.exec.ProjectExternalExec; import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider; import com.linkedin.gradle.python.tasks.execution.TeeOutputContainer; -import org.gradle.api.Action; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileTree; import org.gradle.api.file.FileCollection; @@ -59,6 +60,8 @@ abstract public class AbstractPythonMainSourceDefaultTask extends DefaultTask im @Input public List additionalArguments = new ArrayList<>(); + ExternalExec externalExec = new ProjectExternalExec(getProject()); + @InputFiles public FileCollection getSourceFiles() { ConfigurableFileTree componentFiles = getProject().fileTree(getComponent().srcDir); @@ -129,28 +132,29 @@ public void executePythonProcess() { final TeeOutputContainer container = new TeeOutputContainer(stdOut, errOut); - ExecResult result = getProject().exec(new Action() { - @Override - public void execute(ExecSpec execSpec) { - execSpec.environment(getComponent().pythonEnvironment); - execSpec.environment(getComponent().pythonEnvironmentDistgradle); - execSpec.commandLine(getPythonDetails().getVirtualEnvInterpreter()); - // arguments are passed to the python interpreter - execSpec.args(arguments); - // subArguments are arguments for previous arguments. eg: arguments to py.test like -k - execSpec.args(subArguments); - // additionalArguments are same as subArguments, but are expected from user's build script - execSpec.args(additionalArguments); - execSpec.setIgnoreExitValue(ignoreExitValue); - - container.setOutputs(execSpec); - - configureExecution(execSpec); - } + ExecResult result = externalExec.exec(execSpec -> { + execSpec.environment(getComponent().pythonEnvironment); + execSpec.environment(getComponent().pythonEnvironmentDistgradle); + execSpec.commandLine(getPythonDetails().getVirtualEnvInterpreter()); + // arguments are passed to the python interpreter + execSpec.args(arguments); + // subArguments are arguments for previous arguments. eg: arguments to py.test like -k + execSpec.args(subArguments); + // additionalArguments are same as subArguments, but are expected from user's build script + execSpec.args(additionalArguments); + execSpec.setIgnoreExitValue(true); + + container.setOutputs(execSpec); + + configureExecution(execSpec); }); output = container.getCommandOutput(); + if (!ignoreExitValue) { + result.assertNormalExitValue(); + } + processResults(result); } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy index 7f24bbdd..8caf9b14 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWheelsTask.groovy @@ -16,9 +16,14 @@ package com.linkedin.gradle.python.tasks import com.linkedin.gradle.python.PythonExtension +import com.linkedin.gradle.python.exception.PipExecutionException import com.linkedin.gradle.python.extension.PythonDetails import com.linkedin.gradle.python.extension.WheelExtension import com.linkedin.gradle.python.plugin.PythonHelpers +import com.linkedin.gradle.python.tasks.action.pip.PipWheelAction +import com.linkedin.gradle.python.tasks.exec.ExternalExec +import com.linkedin.gradle.python.tasks.exec.ProjectExternalExec +import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider import com.linkedin.gradle.python.tasks.supports.SupportsPackageInfoSettings import com.linkedin.gradle.python.tasks.supports.SupportsWheelCache import com.linkedin.gradle.python.util.DefaultEnvironmentMerger @@ -30,14 +35,10 @@ import com.linkedin.gradle.python.util.PackageSettings import com.linkedin.gradle.python.util.internal.TaskTimer import com.linkedin.gradle.python.wheel.EmptyWheelCache import com.linkedin.gradle.python.wheel.WheelCache -import org.apache.commons.io.FileUtils import org.gradle.api.DefaultTask -import org.gradle.api.GradleException import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.file.FileCollection -import org.gradle.api.logging.Logger -import org.gradle.api.logging.Logging import org.gradle.api.specs.Spec import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles @@ -45,12 +46,8 @@ import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.internal.logging.progress.ProgressLogger import org.gradle.internal.logging.progress.ProgressLoggerFactory -import org.gradle.process.ExecResult -import org.gradle.process.ExecSpec -class BuildWheelsTask extends DefaultTask implements SupportsWheelCache, SupportsPackageInfoSettings { - - private static final Logger LOGGER = Logging.getLogger(BuildWheelsTask) +class BuildWheelsTask extends DefaultTask implements SupportsWheelCache, SupportsPackageInfoSettings, FailureReasonProvider { @Input WheelCache wheelCache = new EmptyWheelCache() @@ -71,6 +68,8 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache, Support PackageSettings packageSettings EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger() + ExternalExec externalExec = new ProjectExternalExec(getProject()) + String lastInstallMessage = null public BuildWheelsTask() { getOutputs().doNotCacheIf('When package packageExcludeFilter is set', new Spec() { @@ -147,104 +146,32 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache, Support WheelExtension wheelExtension = ExtensionUtils.getPythonComponentExtension(project, WheelExtension) def pythonExtension = ExtensionUtils.getPythonExtension(project) + def baseEnvironment = environmentMerger.mergeEnvironments([pythonExtension.pythonEnvironment, environment]) + def wheelAction = new PipWheelAction(packageSettings, project, externalExec, baseEnvironment, + pythonDetails, wheelCache, environmentMerger, wheelExtension) + def taskTimer = new TaskTimer() int counter = 0 def numberOfInstallables = installables.size() installables.each { File installable -> - def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor def packageInfo = PackageInfo.fromPath(installable) - def shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name + def shortHand = packageInfo.toShortHand() def clock = taskTimer.start(shortHand) - progressLogger.progress("Preparing wheel $shortHand (${ ++counter } of $numberOfInstallables)") + progressLogger.progress("Preparing wheel $shortHand (${++counter} of $numberOfInstallables)") if (packageExcludeFilter != null && packageExcludeFilter.isSatisfiedBy(packageInfo)) { if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Skipping {} wheel - Excluded", shortHand) - } - return - } - - // If supported versions are empty, there are no restrictions. - def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo) - if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) { - throw new GradleException( - "Package ${ packageInfo.name } works only with Python versions: ${ supportedVersions }") - } - - /* - * Check if a wheel exists for this product already and only build it - * if it is missing. We don't care about the wheel details because we - * always build these locally. - */ - if (!packageSettings.requiresSourceBuild(packageInfo)) { - def wheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonExtension.details) - if (wheel.isPresent()) { - File wheelFile = wheel.get() - FileUtils.copyFile(wheelFile, new File(wheelExtension.wheelCache, wheelFile.name)) - if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Skipping {}, in wheel cache {}", shortHand, wheelFile) - } - return + logger.lifecycle("Skipping {} - Excluded", packageInfo.toShortHand()) } - - def tree = project.fileTree( - dir: wheelExtension.wheelCache, - include: "**/${ packageInfo.name.replace('-', '_') }-${ (packageInfo.version ?: 'unspecified').replace('-', '_') }-*.whl") - - if (tree.files.size() >= 1) { - LOGGER.lifecycle("Skipping {} wheel - Installed", shortHand) - return - } - } - - if (PythonHelpers.isPlainOrVerbose(project)) { - LOGGER.lifecycle("Installing {} wheel", shortHand) - } - - def stream = new ByteArrayOutputStream() - - def mergedEnv = environmentMerger.mergeEnvironments( - [pythonExtension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)]) - - def commandLine = [ - pythonDetails.getVirtualEnvInterpreter().toString(), - pythonDetails.getVirtualEnvironment().getPip().toString(), - 'wheel', - '--disable-pip-version-check', - '--wheel-dir', wheelExtension.wheelCache.toString(), - '--no-deps', - ] - - commandLine.addAll(args) - - def globalOptions = packageSettings.getGlobalOptions(packageInfo) - if (globalOptions != null) { - commandLine.addAll(globalOptions) - } - - def buildOptions = packageSettings.getBuildOptions(packageInfo) - if (buildOptions != null) { - commandLine.addAll(buildOptions) - } - - commandLine.add(installable.toString()) - - ExecResult installResult = project.exec { ExecSpec execSpec -> - execSpec.environment mergedEnv - execSpec.commandLine(commandLine) - execSpec.standardOutput = stream - execSpec.errorOutput = stream - execSpec.ignoreExitValue = true - } - - if (installResult.exitValue != 0) { - LOGGER.error("Error installing package using `{}`", commandLine) - LOGGER.error(stream.toString().trim()) - throw new GradleException("Failed to build wheel for ${ shortHand }. Please see above output for reason, or re-run your build using ``--info`` for additional logging.") } else { - LOGGER.info(stream.toString().trim()) + try { + wheelAction.buildWheel(packageInfo, args) + } catch (PipExecutionException e) { + lastInstallMessage = e.pipText + throw e + } } clock.stop() @@ -254,4 +181,9 @@ class BuildWheelsTask extends DefaultTask implements SupportsWheelCache, Support new File(project.buildDir, getName() + "-task-runtime-report.txt").text = taskTimer.buildReport() } + + @Override + String getReason() { + return lastInstallMessage + } } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy index 493e00fe..71f00577 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy @@ -15,9 +15,12 @@ */ package com.linkedin.gradle.python.tasks -import com.linkedin.gradle.python.PythonExtension +import com.linkedin.gradle.python.exception.PipExecutionException import com.linkedin.gradle.python.extension.PythonDetails import com.linkedin.gradle.python.plugin.PythonHelpers +import com.linkedin.gradle.python.tasks.action.pip.PipInstallAction +import com.linkedin.gradle.python.tasks.exec.ExternalExec +import com.linkedin.gradle.python.tasks.exec.ProjectExternalExec import com.linkedin.gradle.python.tasks.execution.FailureReasonProvider import com.linkedin.gradle.python.tasks.supports.SupportsPackageInfoSettings import com.linkedin.gradle.python.tasks.supports.SupportsWheelCache @@ -25,7 +28,6 @@ import com.linkedin.gradle.python.util.DefaultEnvironmentMerger import com.linkedin.gradle.python.util.DependencyOrder import com.linkedin.gradle.python.util.EnvironmentMerger import com.linkedin.gradle.python.util.ExtensionUtils -import com.linkedin.gradle.python.util.OperatingSystem import com.linkedin.gradle.python.util.PackageInfo import com.linkedin.gradle.python.util.PackageSettings import com.linkedin.gradle.python.util.internal.TaskTimer @@ -33,7 +35,6 @@ import com.linkedin.gradle.python.wheel.EmptyWheelCache import com.linkedin.gradle.python.wheel.WheelCache import groovy.transform.CompileStatic import org.gradle.api.DefaultTask -import org.gradle.api.GradleException import org.gradle.api.Task import org.gradle.api.file.FileCollection import org.gradle.api.specs.Spec @@ -43,11 +44,6 @@ import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.internal.logging.progress.ProgressLogger import org.gradle.internal.logging.progress.ProgressLoggerFactory -import org.gradle.process.ExecResult -import org.gradle.process.ExecSpec - -import java.nio.file.Path -import java.nio.file.Paths /** * Execute pip install @@ -78,8 +74,8 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo boolean sorted = true PackageSettings packageSettings - EnvironmentMerger environmentMerger = new DefaultEnvironmentMerger() + ExternalExec externalExec = new ProjectExternalExec(getProject()) public PipInstallTask() { getOutputs().doNotCacheIf('When package packageExcludeFilter is set', new Spec() { @@ -129,9 +125,7 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo */ @TaskAction void pipInstall() { - def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor def extension = ExtensionUtils.getPythonExtension(project) - def sitePackages = findSitePackages() ProgressLoggerFactory progressLoggerFactory = getServices().get(ProgressLoggerFactory) ProgressLogger progressLogger = progressLoggerFactory.newOperation(PipInstallTask) @@ -139,17 +133,31 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo progressLogger.started() TaskTimer taskTimer = new TaskTimer() + def baseEnvironment = environmentMerger.mergeEnvironments([extension.pythonEnvironment, environment]) + def pipInstallAction = new PipInstallAction(packageSettings, project, externalExec, + baseEnvironment, pythonDetails, wheelCache, environmentMerger) int counter = 0 def installableFiles = DependencyOrder.getConfigurationFiles(installFileCollection, sorted) for (File installable : installableFiles) { if (isReadyForInstall(installable)) { def packageInfo = PackageInfo.fromPath(installable) - String shortHand = packageInfo.version ? "${ packageInfo.name }-${ packageInfo.version }" : packageInfo.name - def timer = taskTimer.start(shortHand) - progressLogger.progress("Installing $shortHand (${ ++counter } of ${ installableFiles.size() })") - doInstall(shortHand, packageInfo, sitePackages, pyVersion, extension, installable) + def timer = taskTimer.start(packageInfo.toShortHand()) + progressLogger.progress("Installing ${packageInfo.toShortHand()} (${++counter} of ${installableFiles.size()})") + + if (packageExcludeFilter != null && packageExcludeFilter.isSatisfiedBy(packageInfo)) { + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Skipping {} - Excluded", packageInfo.toShortHand()) + } + } else { + try { + pipInstallAction.installPackage(packageInfo, args) + } catch (PipExecutionException e) { + lastInstallMessage = e.pipText + throw e + } + } timer.stop() } } @@ -159,121 +167,9 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo new File(project.buildDir, getName() + "-task-runtime-report.txt").text = taskTimer.buildReport() } - @SuppressWarnings("ParameterCount") - private void doInstall(String shortHand, PackageInfo packageInfo, Path sitePackages, - String pyVersion, PythonExtension extension, File installable) { - if (packageExcludeFilter != null && packageExcludeFilter.isSatisfiedBy(packageInfo)) { - if (PythonHelpers.isPlainOrVerbose(project)) { - logger.lifecycle("Skipping {} - Excluded", shortHand) - } - return - } - - // If supported versions are empty, there are no restrictions. - def supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo) - if (supportedVersions != null && !supportedVersions.empty && !supportedVersions.contains(pyVersion)) { - throw new PipInstallException( - "Package ${ packageInfo.name } works only with Python versions: ${ supportedVersions }") - } - - String sanitizedName = packageInfo.name.replace('-', '_') - - // See: https://www.python.org/dev/peps/pep-0376/ - File egg = sitePackages.resolve("${ sanitizedName }-${ packageInfo.version }-py${ pyVersion }.egg-info").toFile() - File dist = sitePackages.resolve("${ sanitizedName }-${ packageInfo.version }.dist-info").toFile() - - if (!packageSettings.requiresSourceBuild(packageInfo) && - (project.file(egg).exists() || project.file(dist).exists())) { - if (PythonHelpers.isPlainOrVerbose(project)) { - logger.lifecycle("Skipping {} - Installed", shortHand) - } - return - } - - def mergedEnv = environmentMerger.mergeEnvironments( - [extension.pythonEnvironment, environment, packageSettings.getEnvironment(packageInfo)]) - - - def commandLine = [ - pythonDetails.getVirtualEnvInterpreter().toString(), - pythonDetails.getVirtualEnvironment().getPip().toString(), - 'install', - '--disable-pip-version-check', - '--no-deps', - ] - - commandLine.addAll(args) - - def globalOptions = packageSettings.getGlobalOptions(packageInfo) - if (globalOptions != null) { - commandLine.addAll(globalOptions) - } - - def installOptions = packageSettings.getInstallOptions(packageInfo) - if (installOptions != null) { - commandLine.addAll(installOptions) - } - - def cachedWheel = wheelCache.findWheel(packageInfo.name, packageInfo.version, pythonDetails) - if (!packageSettings.requiresSourceBuild(packageInfo) && cachedWheel.isPresent()) { - if (PythonHelpers.isPlainOrVerbose(project)) { - logger.lifecycle("{} from wheel: {}", shortHand, cachedWheel.get().getAbsolutePath()) - } - commandLine.add(cachedWheel.get().getAbsolutePath()) - } else { - commandLine.add(installable.getAbsolutePath()) - } - - if (PythonHelpers.isPlainOrVerbose(project)) { - logger.lifecycle("Installing {}", shortHand) - } - - def stream = new ByteArrayOutputStream() - ExecResult installResult = project.exec { ExecSpec execSpec -> - execSpec.environment mergedEnv - execSpec.commandLine(commandLine) - execSpec.standardOutput = stream - execSpec.errorOutput = stream - execSpec.ignoreExitValue = true - } - - def message = stream.toString().trim() - if (installResult.exitValue != 0) { - /* - * TODO: maintain a list of packages that failed to install, and report a failure - * report at the end. We can leverage our domain expertise here to provide very - * meaningful errors. E.g., we see lxml failed to install, do you have libxml2 - * installed? E.g., we see pyOpenSSL>0.15 failed to install, do you have libffi - * installed? - */ - logger.error("Error installing package using `{}`", commandLine) - logger.error(message) - lastInstallMessage = message - - throw new PipInstallException( - "Failed to install ${ shortHand }. Please see above output for reason, or re-run your build using ``gradle -i build`` for additional logging.") - } else { - logger.info(message) - } - } - - static class PipInstallException extends GradleException { - PipInstallException(String message) { - super(message) - } - } - - private Path findSitePackages() { - def pyVersion = pythonDetails.getPythonVersion().pythonMajorMinor - if (OperatingSystem.current().isUnix()) { - return pythonDetails.virtualEnv.toPath().resolve(Paths.get("lib", "python${ pyVersion }", "site-packages")) - } else { - return pythonDetails.virtualEnv.toPath().resolve(Paths.get("Lib", "site-packages")) - } - } - @Override String getReason() { return lastInstallMessage } + } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/AbstractPipAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/AbstractPipAction.java new file mode 100644 index 00000000..54fedcee --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/AbstractPipAction.java @@ -0,0 +1,117 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip; + +import com.linkedin.gradle.python.extension.PythonDetails; +import com.linkedin.gradle.python.extension.PythonVersion; +import com.linkedin.gradle.python.exception.PipExecutionException; +import com.linkedin.gradle.python.tasks.exec.ExternalExec; +import com.linkedin.gradle.python.util.EnvironmentMerger; +import com.linkedin.gradle.python.util.PackageInfo; +import com.linkedin.gradle.python.util.PackageSettings; +import com.linkedin.gradle.python.wheel.WheelCache; +import org.gradle.api.Project; +import org.gradle.process.ExecResult; + +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +abstract class AbstractPipAction { + + final PackageSettings packageSettings; + final Project project; + final ExternalExec externalExec; + final Map baseEnvironment; + final PythonDetails pythonDetails; + final WheelCache wheelCache; + final EnvironmentMerger environmentMerger; + final PythonVersion pythonVersion; + + AbstractPipAction(PackageSettings packageSettings, + Project project, + ExternalExec externalExec, + Map baseEnvironment, + PythonDetails pythonDetails, + WheelCache wheelCache, + EnvironmentMerger environmentMerger) { + this.packageSettings = packageSettings; + this.project = project; + this.externalExec = externalExec; + this.baseEnvironment = baseEnvironment; + this.pythonDetails = pythonDetails; + this.wheelCache = wheelCache; + this.environmentMerger = environmentMerger; + this.pythonVersion = pythonDetails.getPythonVersion(); + + } + + void throwIfPythonVersionIsNotSupported(PackageInfo packageInfo) { + // If supported versions are empty, there are no restrictions. + List supportedVersions = packageSettings.getSupportedLanguageVersions(packageInfo); + if (supportedVersions != null && !supportedVersions.isEmpty() + && !supportedVersions.contains(pythonVersion.getPythonMajorMinor())) { + throw PipExecutionException.unsupportedPythonVersion(packageInfo, supportedVersions); + } + } + + ExecResult execCommand(Map mergedEnv, List commandLine, OutputStream stream) { + return externalExec.exec(execSpec -> { + execSpec.environment(mergedEnv); + execSpec.commandLine(commandLine); + execSpec.setStandardOutput(stream); + execSpec.setErrorOutput(stream); + execSpec.setIgnoreExitValue(true); + }); + } + + /** + * @return Always returns a list, never null. + */ + List getGlobalOptions(PackageInfo packageInfo) { + List globalOptions = packageSettings.getGlobalOptions(packageInfo); + if (globalOptions == null) { + return Collections.emptyList(); + } else { + return globalOptions; + } + } + + /** + * @return Always returns a list, never null. + */ + List getInstallOptions(PackageInfo packageInfo) { + List installOptions = packageSettings.getInstallOptions(packageInfo); + if (installOptions == null) { + return Collections.emptyList(); + } else { + return installOptions; + } + } + + /** + * @return Always returns a list, never null. + */ + List getBuildOptions(PackageInfo packageInfo) { + List buildOptions = packageSettings.getBuildOptions(packageInfo); + if (buildOptions == null) { + return Collections.emptyList(); + } else { + return buildOptions; + } + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java new file mode 100644 index 00000000..75149f19 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java @@ -0,0 +1,147 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip; + +import com.linkedin.gradle.python.extension.PythonDetails; +import com.linkedin.gradle.python.plugin.PythonHelpers; +import com.linkedin.gradle.python.exception.PipExecutionException; +import com.linkedin.gradle.python.tasks.exec.ExternalExec; +import com.linkedin.gradle.python.util.EnvironmentMerger; +import com.linkedin.gradle.python.util.OperatingSystem; +import com.linkedin.gradle.python.util.PackageInfo; +import com.linkedin.gradle.python.util.PackageSettings; +import com.linkedin.gradle.python.wheel.WheelCache; +import org.gradle.api.Project; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class PipInstallAction extends AbstractPipAction { + + private static Logger logger = Logging.getLogger(PipInstallAction.class); + + private final Path sitePackagesPath; + + public PipInstallAction(PackageSettings packageSettings, + Project project, + ExternalExec externalExec, Map baseEnvironment, + PythonDetails pythonDetails, + WheelCache wheelCache, EnvironmentMerger environmentMerger) { + super(packageSettings, project, externalExec, baseEnvironment, pythonDetails, wheelCache, environmentMerger); + this.sitePackagesPath = findSitePackages(pythonDetails); + } + + private static Path findSitePackages(PythonDetails pythonDetails) { + String pyVersion = pythonDetails.getPythonVersion().getPythonMajorMinor(); + if (OperatingSystem.current().isUnix()) { + return pythonDetails.getVirtualEnv().toPath().resolve(Paths.get("lib", "python" + pyVersion, "site-packages")); + } else { + return pythonDetails.getVirtualEnv().toPath().resolve(Paths.get("Lib", "site-packages")); + } + } + + public void installPackage(PackageInfo packageInfo, List extraArgs) { + throwIfPythonVersionIsNotSupported(packageInfo); + + String pyVersion = pythonDetails.getPythonVersion().getPythonMajorMinor(); + String sanitizedName = packageInfo.getName().replace('-', '_'); + + // See: https://www.python.org/dev/peps/pep-0376/ + File egg = sitePackagesPath.resolve(sanitizedName + "-" + packageInfo.getVersion() + "-py" + pyVersion + ".egg-info").toFile(); + File dist = sitePackagesPath.resolve(sanitizedName + "-" + packageInfo.getVersion() + ".dist-info").toFile(); + + if (!packageSettings.requiresSourceBuild(packageInfo) + && (project.file(egg).exists() || project.file(dist).exists())) { + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Skipping {} - Installed", packageInfo.toShortHand()); + } + return; + } + + Map mergedEnv = environmentMerger.mergeEnvironments( + Arrays.asList(baseEnvironment, packageSettings.getEnvironment(packageInfo))); + + + List commandLine = makeCommandLine(packageInfo, extraArgs); + + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Installing {}", packageInfo.toShortHand()); + } + + OutputStream stream = new ByteArrayOutputStream(); + ExecResult installResult = execCommand(mergedEnv, commandLine, stream); + + String message = stream.toString().trim(); + if (installResult.getExitValue() != 0) { + /* + * TODO: maintain a list of packages that failed to install, and report a failure + * report at the end. We can leverage our domain expertise here to provide very + * meaningful errors. E.g., we see lxml failed to install, do you have libxml2 + * installed? E.g., we see pyOpenSSL>0.15 failed to install, do you have libffi + * installed? + */ + logger.error("Error installing package using `{}`", commandLine); + logger.error(message); + throw PipExecutionException.failedInstall(packageInfo, message); + } else { + logger.info(message); + } + } + + private List makeCommandLine(PackageInfo packageInfo, List extraArgs) { + List commandLine = new ArrayList<>(); + commandLine.addAll(baseInstallArguments()); + commandLine.addAll(extraArgs); + commandLine.addAll(getGlobalOptions(packageInfo)); + commandLine.addAll(getInstallOptions(packageInfo)); + + Optional cachedWheel = Optional.empty(); + + if (!packageSettings.requiresSourceBuild(packageInfo)) { + cachedWheel = wheelCache.findWheel(packageInfo.getName(), packageInfo.getVersion(), pythonDetails); + } + + if (cachedWheel.isPresent()) { + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("{} from wheel: {}", packageInfo.toShortHand(), cachedWheel.get().getAbsolutePath()); + } + commandLine.add(cachedWheel.get().getAbsolutePath()); + } else { + commandLine.add(packageInfo.getPackageFile().getAbsolutePath()); + } + return commandLine; + } + + private List baseInstallArguments() { + return Arrays.asList( + pythonDetails.getVirtualEnvInterpreter().toString(), + pythonDetails.getVirtualEnvironment().getPip().toString(), + "install", + "--disable-pip-version-check", + "--no-deps"); + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelAction.java new file mode 100644 index 00000000..0dea5f70 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelAction.java @@ -0,0 +1,133 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip; + +import com.linkedin.gradle.python.extension.PythonDetails; +import com.linkedin.gradle.python.extension.WheelExtension; +import com.linkedin.gradle.python.plugin.PythonHelpers; +import com.linkedin.gradle.python.exception.PipExecutionException; +import com.linkedin.gradle.python.tasks.exec.ExternalExec; +import com.linkedin.gradle.python.util.EnvironmentMerger; +import com.linkedin.gradle.python.util.PackageInfo; +import com.linkedin.gradle.python.util.PackageSettings; +import com.linkedin.gradle.python.wheel.WheelCache; +import org.apache.commons.io.FileUtils; +import org.gradle.api.Project; +import org.gradle.api.file.ConfigurableFileTree; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class PipWheelAction extends AbstractPipAction { + + private static Logger logger = Logging.getLogger(PipWheelAction.class); + private final WheelExtension wheelExtension; + + public PipWheelAction(PackageSettings packageSettings, + Project project, + ExternalExec externalExec, + Map baseEnvironment, + PythonDetails pythonDetails, + WheelCache wheelCache, + EnvironmentMerger environmentMerger, + WheelExtension wheelExtension) { + super(packageSettings, project, externalExec, baseEnvironment, pythonDetails, wheelCache, environmentMerger); + this.wheelExtension = wheelExtension; + } + + public void buildWheel(PackageInfo packageInfo, List extraArgs) throws IOException { + throwIfPythonVersionIsNotSupported(packageInfo); + + /* + * Check if a wheel exists for this product already and only build it + * if it is missing. We don't care about the wheel details because we + * always build these locally. + */ + if (!packageSettings.requiresSourceBuild(packageInfo)) { + Optional wheel = wheelCache.findWheel(packageInfo.getName(), packageInfo.getVersion(), pythonDetails); + if (wheel.isPresent()) { + File wheelFile = wheel.get(); + FileUtils.copyFile(wheelFile, new File(wheelExtension.getWheelCache(), wheelFile.getName())); + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Skipping {}, in wheel cache {}", packageInfo.toShortHand(), wheelFile); + } + return; + } + + ConfigurableFileTree tree = project.fileTree(wheelExtension.getWheelCache(), action -> { + String sanitizedName = packageInfo.getName().replace('-', '_'); + String sanitizedVersion = (packageInfo.getVersion() == null ? "unspecified" : packageInfo.getVersion()).replace('-', '_'); + action.include("**/" + sanitizedName + "-" + sanitizedVersion + "-*.whl"); + }); + + if (tree.getFiles().size() >= 1) { + logger.lifecycle("Skipping {} wheel - Installed", packageInfo.toShortHand()); + return; + } + } + + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Installing {} wheel", packageInfo.toShortHand()); + } + + Map mergedEnv = environmentMerger.mergeEnvironments( + Arrays.asList(baseEnvironment, packageSettings.getEnvironment(packageInfo))); + + List commandLine = makeCommandLine(packageInfo, extraArgs); + + OutputStream stream = new ByteArrayOutputStream(); + + ExecResult installResult = execCommand(mergedEnv, commandLine, stream); + + if (installResult.getExitValue() != 0) { + logger.error("Error installing package using `{}`", commandLine); + logger.error(stream.toString().trim()); + throw PipExecutionException.failedWheel(packageInfo, stream.toString().trim()); + } else { + logger.info(stream.toString().trim()); + } + + } + + private List makeCommandLine(PackageInfo packageInfo, List extraArgs) { + List commandLine = new ArrayList<>(); + commandLine.addAll(Arrays.asList( + pythonDetails.getVirtualEnvInterpreter().toString(), + pythonDetails.getVirtualEnvironment().getPip().toString(), + "wheel", + "--disable-pip-version-check", + "--wheel-dir", wheelExtension.getWheelCache().toString(), + "--no-deps")); + + commandLine.addAll(extraArgs); + commandLine.addAll(getGlobalOptions(packageInfo)); + commandLine.addAll(getBuildOptions(packageInfo)); + + commandLine.add(packageInfo.getPackageFile().toString()); + + return commandLine; + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExec.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExec.java new file mode 100644 index 00000000..680f06f3 --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExec.java @@ -0,0 +1,24 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.exec; + +import org.gradle.api.Action; +import org.gradle.process.ExecResult; +import org.gradle.process.ExecSpec; + +public interface ExternalExec { + ExecResult exec(Action action); +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ProjectExternalExec.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ProjectExternalExec.java new file mode 100644 index 00000000..c62c0dff --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/exec/ProjectExternalExec.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.exec; + +import org.gradle.api.Action; +import org.gradle.api.Project; +import org.gradle.process.ExecResult; +import org.gradle.process.ExecSpec; + +public class ProjectExternalExec implements ExternalExec { + + private final Project project; + + public ProjectExternalExec(Project project) { + this.project = project; + } + + @Override + public ExecResult exec(Action action) { + return project.exec(action); + } +} diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageInfo.groovy b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageInfo.groovy index 97128492..e62f3039 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageInfo.groovy +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/PackageInfo.groovy @@ -19,6 +19,7 @@ import com.linkedin.gradle.python.wheel.PythonWheelDetails import org.apache.commons.io.FilenameUtils import org.gradle.api.GradleException +import java.nio.file.Path import java.util.regex.Matcher @@ -96,9 +97,16 @@ class PackageInfo { } } + public static PackageInfo fromPath(Path packagePath) { + return fromPath(packagePath.toFile()) + } @Override public String toString() { return name + "-" + version } + + public String toShortHand() { + return version ? "${name}-${version}" : name + } } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/extension/PythonDetailsTestDouble.java b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/extension/PythonDetailsTestDouble.java new file mode 100644 index 00000000..14b4d793 --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/extension/PythonDetailsTestDouble.java @@ -0,0 +1,38 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.extension; + +import org.gradle.api.Project; + +import java.io.File; + +public class PythonDetailsTestDouble extends PythonDetails { + + private PythonVersion version = new PythonVersion("3.6"); + + public PythonDetailsTestDouble(Project project, File venvDir) { + super(project, venvDir); + } + + @Override + public PythonVersion getPythonVersion() { + return version; + } + + public void setPythonVersion(PythonVersion pythonVersion) { + this.version = pythonVersion; + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy new file mode 100644 index 00000000..1c67cbb8 --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy @@ -0,0 +1,116 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip + +import com.linkedin.gradle.python.util.DefaultPackageSettings +import com.linkedin.gradle.python.util.PackageInfo +import org.junit.rules.TemporaryFolder + +import java.nio.file.Paths + +class PipActionHelpers { + private PipActionHelpers() { + //NOOP + } + + static class RequiresRebuildOverridePackageSettings extends DefaultPackageSettings { + private final List override + + RequiresRebuildOverridePackageSettings(TemporaryFolder temporaryFolder, List override) { + super(temporaryFolder.root) + this.override = override + } + + @Override + boolean requiresSourceBuild(PackageInfo packageInfo) { + return override.contains(packageInfo.name) + } + } + + static class SupportedLanguageVersionOverridePackageSettings extends DefaultPackageSettings { + private final Map> override + + SupportedLanguageVersionOverridePackageSettings(TemporaryFolder temporaryFolder, Map> override) { + super(temporaryFolder.root) + this.override = override + } + + @Override + List getSupportedLanguageVersions(PackageInfo packageInfo) { + return override.getOrDefault(packageInfo.name, []) + } + } + + static class InstallOptionOverridePackageSettings extends DefaultPackageSettings { + private final Map> override + + InstallOptionOverridePackageSettings(TemporaryFolder temporaryFolder, Map> override) { + super(temporaryFolder.root) + this.override = override + } + + @Override + List getInstallOptions(PackageInfo packageInfo) { + return override.getOrDefault(packageInfo.name, []) + } + } + + static class GlobalOptionOverridePackageSettings extends DefaultPackageSettings { + private final Map> override + + GlobalOptionOverridePackageSettings(TemporaryFolder temporaryFolder, override) { + super(temporaryFolder.root) + this.override = override + } + + @Override + List getGlobalOptions(PackageInfo packageInfo) { + return override.getOrDefault(packageInfo.name, []) + } + } + + static class EnvOverridePackageSettings extends DefaultPackageSettings { + private final Map> envOverride + + EnvOverridePackageSettings(TemporaryFolder temporaryFolder, Map> envOverride) { + super(temporaryFolder.root) + this.envOverride = envOverride + } + + @Override + Map getEnvironment(PackageInfo packageInfo) { + envOverride.getOrDefault(packageInfo.name, [:]) + } + } + + static class BuildOptionOverridePackageSetting extends DefaultPackageSettings { + private final Map> envOverride + + BuildOptionOverridePackageSetting(TemporaryFolder temporaryFolder, Map> envOverride) { + super(temporaryFolder.root) + this.envOverride = envOverride + } + + @Override + List getBuildOptions(PackageInfo packageInfo) { + return envOverride.getOrDefault(packageInfo.name, []) + } + } + + static PackageInfo packageInGradleCache(String name) { + return PackageInfo.fromPath(Paths.get("foo", ".gradle", "caches", name)) + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy new file mode 100644 index 00000000..1b210e2b --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy @@ -0,0 +1,186 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip + +import com.linkedin.gradle.python.exception.PipExecutionException +import com.linkedin.gradle.python.extension.PythonDetailsTestDouble +import com.linkedin.gradle.python.extension.VirtualEnvironment +import com.linkedin.gradle.python.tasks.exec.ExternalExecTestDouble +import com.linkedin.gradle.python.util.DefaultEnvironmentMerger +import com.linkedin.gradle.python.util.PackageSettings +import com.linkedin.gradle.python.wheel.EmptyWheelCache +import com.linkedin.gradle.python.wheel.WheelCache +import org.gradle.process.ExecSpec +import org.gradle.testfixtures.ProjectBuilder +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import spock.lang.Specification + +import static com.linkedin.gradle.python.tasks.action.pip.PipActionHelpers.packageInGradleCache + +class PipInstallActionTest extends Specification { + + @Rule + TemporaryFolder temporaryFolder + + def "uses environment"() { + setup: + def override = [foo: ['CPPFLAGS': '-I/some/custom/path/include', + 'LDFLAGS' : '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib'] + ] + def settings = new PipActionHelpers.EnvOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def pipInstallAction = createPipInstallAction(settings, execSpec) + + when: + pipInstallAction.installPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.environment(['CPPFLAGS': '-I/some/custom/path/include', 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib']) + } + + def 'uses global options'() { + Map> override = ['setuptools': ['--global-option', '--dummy-global-option']] + def settings = new PipActionHelpers.GlobalOptionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def pipInstallAction = createPipInstallAction(settings, execSpec) + + when: + pipInstallAction.installPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--global-option') + assert idx != -1 + assert it[2] == 'install' + assert it[idx + 1] == '--dummy-global-option' + } + } + + def 'uses install options'() { + Map> override = ['setuptools': ['--install-option', '--ignore=E123,E234']] + def settings = new PipActionHelpers.InstallOptionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def pipInstallAction = createPipInstallAction(settings, execSpec) + + when: + pipInstallAction.installPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--install-option') + assert idx != -1 + assert it[2] == 'install' + assert it[idx + 1] == '--ignore=E123,E234' + } + } + + def "uses supported language versions"() { + Map> override = ['setuptools': ['2.8']] + def settings = new PipActionHelpers.SupportedLanguageVersionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def pipInstallAction = createPipInstallAction(settings, execSpec) + + when: + pipInstallAction.installPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + def e = thrown(PipExecutionException) + e.message == 'Package setuptools works only with Python versions: [2.8]' + } + + def 'requires source rebuild'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def mockWheelCache = Mock(WheelCache) + def pipInstallAction = createPipInstallAction(settings, execSpec, mockWheelCache) + + def eggFile = pipInstallAction.sitePackagesPath.resolve("pyflakes-1.6.0-py3.6.egg-info").toFile() + eggFile.parentFile.mkdirs() + eggFile.createNewFile() + + def distInfo = pipInstallAction.sitePackagesPath.resolve("pyflakes-1.6.0.dist-info").toFile() + distInfo.parentFile.mkdirs() + distInfo.createNewFile() + + when: + pipInstallAction.installPackage(packageInGradleCache("pyflakes-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--no-deps') + assert idx != -1 + assert it[2] == 'install' + assert it[idx + 1] == '--ignore-installed' + } + + 0 * mockWheelCache._ + } + + def 'will skip if package (egg) is installed'() { + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, []) + def execSpec = Mock(ExecSpec) + def action = createPipInstallAction(settings, execSpec) + + def eggFile = action.sitePackagesPath.resolve("pyflakes-1.6.0-py3.6.egg-info").toFile() + eggFile.parentFile.mkdirs() + eggFile.createNewFile() + + when: + action.installPackage(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: + 0 * execSpec._ + } + + def 'will skip if package (dist-info) is installed'() { + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, []) + def execSpec = Mock(ExecSpec) + def action = createPipInstallAction(settings, execSpec) + + def eggFile = action.sitePackagesPath.resolve("pyflakes-1.6.0.dist-info").toFile() + eggFile.parentFile.mkdirs() + eggFile.createNewFile() + + when: + action.installPackage(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: + 0 * execSpec._ + } + + private PipInstallAction createPipInstallAction(PackageSettings settings, ExecSpec execSpec) { + return createPipInstallAction(settings, execSpec, new EmptyWheelCache()) + } + + private PipInstallAction createPipInstallAction(PackageSettings settings, ExecSpec execSpec, WheelCache wheelCache) { + def project = new ProjectBuilder().withProjectDir(temporaryFolder.root).build() + def binDir = temporaryFolder.newFolder('build', 'venv', VirtualEnvironment.getPythonApplicationDirectory()) + VirtualEnvironment.findExecutable(binDir.toPath(), "pip").toFile().createNewFile() + VirtualEnvironment.findExecutable(binDir.toPath(), "python").toFile().createNewFile() + def details = new PythonDetailsTestDouble(project, binDir.parentFile) + return new PipInstallAction(settings, project, new ExternalExecTestDouble(execSpec), + ['CPPFLAGS': 'bogus', 'LDFLAGS': 'bogus'], + details, wheelCache, new DefaultEnvironmentMerger()) + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelActionTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelActionTest.groovy new file mode 100644 index 00000000..56f267a0 --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipWheelActionTest.groovy @@ -0,0 +1,185 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip + +import com.linkedin.gradle.python.exception.PipExecutionException +import com.linkedin.gradle.python.extension.PythonDetailsTestDouble +import com.linkedin.gradle.python.extension.VirtualEnvironment +import com.linkedin.gradle.python.extension.WheelExtension +import com.linkedin.gradle.python.tasks.exec.ExternalExecTestDouble +import com.linkedin.gradle.python.util.DefaultEnvironmentMerger +import com.linkedin.gradle.python.util.PackageSettings +import com.linkedin.gradle.python.wheel.EmptyWheelCache +import com.linkedin.gradle.python.wheel.WheelCache +import org.gradle.process.ExecSpec +import org.gradle.testfixtures.ProjectBuilder +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import spock.lang.Specification + +import static com.linkedin.gradle.python.tasks.action.pip.PipActionHelpers.packageInGradleCache + + +class PipWheelActionTest extends Specification { + + @Rule + TemporaryFolder temporaryFolder + + def "uses environment"() { + setup: + def override = [foo: ['CPPFLAGS': '-I/some/custom/path/include', + 'LDFLAGS' : '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib'] + ] + def settings = new PipActionHelpers.EnvOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def action = createPipWheelAction(settings, execSpec) + + when: + action.buildWheel(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.environment(['CPPFLAGS': '-I/some/custom/path/include', 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib']) + } + + def 'uses global options'() { + Map> override = ['setuptools': ['--global-option', '--dummy-global-option']] + def settings = new PipActionHelpers.GlobalOptionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def action = createPipWheelAction(settings, execSpec) + + when: + action.buildWheel(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--global-option') + assert idx != -1 + assert it[2] == 'wheel' + assert it[idx + 1] == '--dummy-global-option' + } + } + + def 'uses build options'() { + Map> override = ['setuptools': ['--build-option', '--disable-something']] + def settings = new PipActionHelpers.BuildOptionOverridePackageSetting(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def action = createPipWheelAction(settings, execSpec) + + when: + action.buildWheel(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--build-option') + assert idx != -1 + assert it[2] == 'wheel' + assert it[idx + 1] == '--disable-something' + } + } + + def "uses supported language versions"() { + Map> override = ['setuptools': ['2.8']] + def settings = new PipActionHelpers.SupportedLanguageVersionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def action = createPipWheelAction(settings, execSpec) + + when: + action.buildWheel(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: + def e = thrown(PipExecutionException) + e.message == 'Package setuptools works only with Python versions: [2.8]' + } + + def 'requires source rebuild'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def wheelCacheMock = Mock(WheelCache) + def action = createPipWheelAction(settings, execSpec, wheelCacheMock) + + def wheelCache = temporaryFolder.newFolder('build', 'wheel-cache') + new File(wheelCache, "pyflakes-1.6.0-py2.py3-none-any.whl").createNewFile() + + when: + action.buildWheel(packageInGradleCache("pyflakes-1.0.0.tar.gz"), []) + + then: + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--no-deps') + assert idx != -1 + assert it[2] == 'wheel' + assert !it.any { entry -> entry == '--ignore-installed' } + } + 0 * wheelCacheMock._ + } + + def 'will skip if package is installed in project wheel cache'() { + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, []) + def execSpec = Mock(ExecSpec) + def action = createPipWheelAction(settings, execSpec) + + def wheelCache = temporaryFolder.newFolder('build', 'wheel-cache') + new File(wheelCache, "pyflakes-1.6.0-py2.py3-none-any.whl").createNewFile() + + when: + action.buildWheel(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: + 0 * execSpec._ + } + + def 'will use pre-built wheel when available in global cache'() { + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, []) + def execSpec = Mock(ExecSpec) + def wheelCacheMock = Mock(WheelCache) + def action = createPipWheelAction(settings, execSpec, wheelCacheMock) + + def fakeWheel = temporaryFolder.newFile('pyflakes-1.6.0-py2.py3-none-any.whl') + + assert !new File(temporaryFolder.root, "build/wheel-cache/pyflakes-1.6.0-py2.py3-none-any.whl").exists() + + when: + action.buildWheel(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: + 0 * execSpec._ + 1 * wheelCacheMock.findWheel(_, _, _) >> Optional.of(fakeWheel) + new File(temporaryFolder.root, "build/wheel-cache/pyflakes-1.6.0-py2.py3-none-any.whl").exists() + } + + private PipWheelAction createPipWheelAction(PackageSettings settings, ExecSpec execSpec) { + return createPipWheelAction(settings, execSpec, new EmptyWheelCache()) + } + + private PipWheelAction createPipWheelAction(PackageSettings settings, ExecSpec execSpec, WheelCache wheelCache) { + def project = new ProjectBuilder().withProjectDir(temporaryFolder.root).build() + def binDir = temporaryFolder.newFolder('build', 'venv', VirtualEnvironment.getPythonApplicationDirectory()) + VirtualEnvironment.findExecutable(binDir.toPath(), "pip").toFile().createNewFile() + VirtualEnvironment.findExecutable(binDir.toPath(), "python").toFile().createNewFile() + def details = new PythonDetailsTestDouble(project, binDir.parentFile) + return new PipWheelAction(settings, project, new ExternalExecTestDouble(execSpec), + ['CPPFLAGS': 'bogus', 'LDFLAGS': 'bogus'], + details, wheelCache, new DefaultEnvironmentMerger(), + new WheelExtension(project)) + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecTestDouble.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecTestDouble.groovy new file mode 100644 index 00000000..b0fd6c45 --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecTestDouble.groovy @@ -0,0 +1,35 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.exec + +import org.gradle.api.Action +import org.gradle.process.ExecResult +import org.gradle.process.ExecSpec + +class ExternalExecTestDouble implements ExternalExec { + + private final ExecSpec spec + + ExternalExecTestDouble(ExecSpec spec) { + this.spec = spec + } + + @Override + ExecResult exec(Action action) { + action.execute(spec) + return ['getExitValue': { -> 0 }] as ExecResult + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageInfoTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageInfoTest.groovy index aa3a2052..d34d0f8f 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageInfoTest.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/util/PackageInfoTest.groovy @@ -62,7 +62,7 @@ class PackageInfoTest extends Specification { def 'can parse a windows path'() { when: - def packageInfo = packageInGradleCache('Z:\\pygradle\\build\\ivy-repo\\pypi\\setuptools\\19.1.1\\setuptools-19.1.1') + def packageInfo = PackageInfo.fromPath(new File('Z:\\pygradle\\.gradle\\caches\\build\\ivy-repo\\pypi\\setuptools\\19.1.1\\setuptools-19.1.1.tar.gz')) then: assert packageInfo.name == 'setuptools' assert packageInfo.version == '19.1.1'