diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/internal/DefaultVirtualEnvironment.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/internal/DefaultVirtualEnvironment.java index 11855b16..cb7569c5 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/internal/DefaultVirtualEnvironment.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/internal/DefaultVirtualEnvironment.java @@ -64,7 +64,7 @@ public File findExecutable(String name) { return findExecutable(prefixBuilder(), name); } - static File findExecutable(Path path, String name) { + public static File findExecutable(Path path, String name) { return path.resolve(OperatingSystem.current().getExecutableName(name)).toFile(); } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java index 6c67a9da..b1cbd5be 100644 --- a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallAction.java @@ -39,13 +39,14 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; + public class PipInstallAction extends AbstractPipAction { private static Logger logger = Logging.getLogger(PipInstallAction.class); private final Path sitePackagesPath; + private final WheelBuilder wheelBuilder; public PipInstallAction(PackageSettings packageSettings, Project project, @@ -54,8 +55,14 @@ public PipInstallAction(PackageSettings packageSettings, WheelCache wheelCache, EnvironmentMerger environmentMerger, Spec packageExcludeFilter) { super(packageSettings, project, externalExec, baseEnvironment, pythonDetails, wheelCache, - environmentMerger, packageExcludeFilter); + environmentMerger, packageExcludeFilter); this.sitePackagesPath = findSitePackages(pythonDetails); + this.wheelBuilder = new WheelBuilder(packageSettings, project, externalExec, baseEnvironment, pythonDetails, + wheelCache, environmentMerger, packageExcludeFilter); + } + + public Path getSitePackagesPath() { + return sitePackagesPath; } private static Path findSitePackages(PythonDetails pythonDetails) { @@ -121,51 +128,13 @@ void doPipOperation(PackageInfo packageInfo, List extraArgs) { } } - private List prepareCommandLine(PackageInfo packageInfo, List extraArgs) { + private List makeCommandLine(PackageInfo packageInfo, List extraArgs) { List commandLine = new ArrayList<>(); commandLine.addAll(baseInstallArguments()); commandLine.addAll(extraArgs); commandLine.addAll(getGlobalOptions(packageInfo)); commandLine.addAll(getInstallOptions(packageInfo)); - - return commandLine; - } - - private boolean appendCachedWheel(PackageInfo packageInfo, Optional cachedWheel, List commandLine) { - if (!packageSettings.requiresSourceBuild(packageInfo)) { - // TODO: Check whether project layer cache exists. - - if (!cachedWheel.isPresent() && !packageSettings.isCustomized(packageInfo)) { - cachedWheel = wheelCache.findWheel(packageInfo.getName(), packageInfo.getVersion(), pythonDetails); - } - } - - if (cachedWheel.isPresent()) { - if (PythonHelpers.isPlainOrVerbose(project)) { - logger.lifecycle("{} from wheel: {}", packageInfo.toShortHand(), cachedWheel.get().getAbsolutePath()); - } - commandLine.add(cachedWheel.get().getAbsolutePath()); - return true; - } - - return false; - } - - private List makeCommandLine(PackageInfo packageInfo, List extraArgs) { - List commandLine = prepareCommandLine(packageInfo, extraArgs); - Optional cachedWheel = Optional.empty(); - boolean allowBuildingFromSdist = false; - - while (!appendCachedWheel(packageInfo, cachedWheel, commandLine)) { - if (allowBuildingFromSdist) { - commandLine.add(packageInfo.getPackageFile().getAbsolutePath()); - break; - } else { - // TODO: Make wheel from sdist, store to local cache and global cache if needed. - } - - allowBuildingFromSdist = true; - } + commandLine.add(wheelBuilder.getPackage(packageInfo, extraArgs).toString()); return commandLine; } diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilder.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilder.java new file mode 100644 index 00000000..e591916a --- /dev/null +++ b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilder.java @@ -0,0 +1,282 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip; + +import com.linkedin.gradle.python.exception.PipExecutionException; +import com.linkedin.gradle.python.extension.PythonDetails; +import com.linkedin.gradle.python.plugin.PythonHelpers; +import com.linkedin.gradle.python.tasks.exec.ExternalExec; +import com.linkedin.gradle.python.util.EnvironmentMerger; +import com.linkedin.gradle.python.util.PackageInfo; +import com.linkedin.gradle.python.util.PackageSettings; +import com.linkedin.gradle.python.wheel.WheelCache; +import com.linkedin.gradle.python.wheel.WheelCacheLayer; +import org.gradle.api.Project; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.specs.Spec; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * Action class that handles wheel building and fetching from cache. + * + * This class is used from PipInstallAction and does not perform the + * checks already done in that class. It does nicely separate the concern + * of wheel building and finding from PipInstallAction and ensures + * it returns back the package file in some form while leaving + * the wheel in at least one cache layer. + */ +public class WheelBuilder extends AbstractPipAction { + // Options for "pip install" that do not work with "pip wheel" command. + private static final List NOT_WHEEL_OPTIONS = Arrays.asList("--upgrade", "--ignore-installed"); + + // Environment variables used for a specific package only and customizing its build. + private static final Map> CUSTOM_ENVIRONMENT = Collections.unmodifiableMap(Stream.of( + new AbstractMap.SimpleEntry<>("numpy", Arrays.asList("BLAS", "OPENBLAS", "ATLAS")), + new AbstractMap.SimpleEntry<>("pycurl", Collections.singletonList("PYCURL_SSL_LIBRARY")) + ).collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue))); + + private static final Logger logger = Logging.getLogger(WheelBuilder.class); + + private boolean customBuild = true; + + // package-private + WheelBuilder(PackageSettings packageSettings, + Project project, + ExternalExec externalExec, + Map baseEnvironment, + PythonDetails pythonDetails, + WheelCache wheelCache, + EnvironmentMerger environmentMerger, + Spec packageExcludeFilter) { + super(packageSettings, project, externalExec, baseEnvironment, + pythonDetails, wheelCache, environmentMerger, packageExcludeFilter); + } + + @Override + Logger getLogger() { + return logger; + } + + /* + * Since WheelBuilder class is always called from PipInstallAction, + * and that class already performed all the filtering checks, + * we're overriding the original execute method from the abstract class + * to avoid duplicate checks. It becomes just a call to doPipOperation, + * but we have to implement that abstract method. + */ + @Override + public void execute(PackageInfo packageInfo, List extraArgs) { + doPipOperation(packageInfo, extraArgs); + } + + /* + * Similar to execute method above, we're skipping some checks already + * done in PipInstallAction where WheelBuilder is called from. + * For example, the check for supported versions. + * Some other checks, such as source build requirement, are moved to + * the main method of the wheel builder -- getPackage. + */ + @Override + void doPipOperation(PackageInfo packageInfo, List extraArgs) { + List commandLine = makeCommandLine(packageInfo, extraArgs); + if (commandLine.isEmpty()) { + return; + } + + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("Building wheel for {}", packageInfo.toShortHand()); + } + + Map mergedEnv; + // The flag is set in getPackage method. + if (customBuild) { + mergedEnv = environmentMerger.mergeEnvironments( + Arrays.asList(baseEnvironment, packageSettings.getEnvironment(packageInfo))); + } else { + // Have to use this for customized environments that are explicitly marked non-customized + mergedEnv = packageSettings.getEnvironment(packageInfo); + } + + OutputStream stream = new ByteArrayOutputStream(); + + ExecResult installResult = execCommand(mergedEnv, commandLine, stream); + + if (installResult.getExitValue() == 0) { + logger.info(stream.toString().trim()); + } else { + logger.error("Error building package wheel using `{}`", commandLine); + logger.error(stream.toString().trim()); + throw PipExecutionException.failedWheel(packageInfo, stream.toString().trim()); + } + + } + + // package-private + File getPackage(PackageInfo packageInfo, List extraArgs) { + File packageFile = packageInfo.getPackageFile(); + + // Cut it short if there's no target directory in the cache. + if (!wheelCache.getTargetDirectory().isPresent()) { + return packageFile; + } + + String name = packageInfo.getName(); + String version = packageInfo.getVersion(); + boolean isProject = isProjectDirectory(packageInfo); + Optional wheel; + + /* + * Current project is a directory, not a package, so version may be null. + * Compensate for that. + */ + if (isProject) { + name = project.getName(); + version = project.getVersion().toString(); + } + + // set the flag for doPipOperation + customBuild = packageSettings.requiresSourceBuild(packageInfo) + || packageSettings.isCustomized(packageInfo) + || isCustomEnvironment(name); + + // Look in cache layers first when applicable. + if (!packageSettings.requiresSourceBuild(packageInfo)) { + wheel = wheelCache.findWheel(name, version, pythonDetails, WheelCacheLayer.PROJECT_LAYER); + if (wheel.isPresent()) { + packageFile = wheel.get(); + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("{} from wheel: {}", + packageInfo.toShortHand(), packageFile.getAbsolutePath()); + } + return packageFile; + } else if (!customBuild) { + wheel = wheelCache.findWheel(name, version, pythonDetails, WheelCacheLayer.HOST_LAYER); + if (wheel.isPresent()) { + packageFile = wheel.get(); + wheelCache.storeWheel(packageFile, WheelCacheLayer.PROJECT_LAYER); + if (PythonHelpers.isPlainOrVerbose(project)) { + logger.lifecycle("{} from wheel: {}", + packageInfo.toShortHand(), packageFile.getAbsolutePath()); + } + return packageFile; + } + } + } + + // Build the wheel into the project layer by default. + try { + execute(packageInfo, extraArgs); + } catch (PipExecutionException e) { + if (!customBuild) { + /* + * The users may need PythonEnvironment for their wheel build. + * We must treat this as a custom build and set a flag for + * doPipOperation to merge PythonEnvironment in accordingly. + * Then retry. + */ + customBuild = true; + execute(packageInfo, extraArgs); + } else { + throw e; + } + } + + wheel = wheelCache.findWheel(name, version, pythonDetails, WheelCacheLayer.PROJECT_LAYER); + if (wheel.isPresent()) { + packageFile = wheel.get(); + if (!customBuild) { + wheelCache.storeWheel(packageFile, WheelCacheLayer.HOST_LAYER); + } + } + + /* + * After ensuring we have a wheel built for our project, + * we still return the project directory back to PipInstallAction + * so that it can install it in editable (development) mode + * into virtualenv. + */ + if (isProject) { + return packageInfo.getPackageFile(); + } + + return packageFile; + } + + private List makeCommandLine(PackageInfo packageInfo, List extraArgs) { + List commandLine = new ArrayList<>(); + Optional targetDir = wheelCache.getTargetDirectory(); + + if (targetDir.isPresent()) { + String wheelDirPath = targetDir.get().toString(); + + commandLine.addAll(Arrays.asList( + pythonDetails.getVirtualEnvInterpreter().toString(), + pythonDetails.getVirtualEnvironment().getPip().toString(), + "wheel", + "--disable-pip-version-check", + "--wheel-dir", wheelDirPath, + "--no-deps" + )); + commandLine.addAll(cleanupArgs(extraArgs)); + commandLine.addAll(getGlobalOptions(packageInfo)); + commandLine.addAll(getBuildOptions(packageInfo)); + + commandLine.add(packageInfo.getPackageFile().toString()); + } + + return commandLine; + } + + private List cleanupArgs(List args) { + List cleanArgs = new ArrayList<>(args); + cleanArgs.removeAll(NOT_WHEEL_OPTIONS); + return cleanArgs; + } + + private boolean isProjectDirectory(PackageInfo packageInfo) { + File packageDir = packageInfo.getPackageFile(); + String version = packageInfo.getVersion(); + return version == null && Files.isDirectory(packageDir.toPath()) && project.getProjectDir().equals(packageDir); + } + + // Use of pythonEnvironment may hide really customized packages. Catch them! + private boolean isCustomEnvironment(String name) { + if (CUSTOM_ENVIRONMENT.containsKey(name)) { + for (String entry : CUSTOM_ENVIRONMENT.get(name)) { + if (baseEnvironment.containsKey(entry)) { + return true; + } + } + } + return false; + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy index 1c67cbb8..a65dd539 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipActionHelpers.groovy @@ -40,6 +40,20 @@ class PipActionHelpers { } } + static class CustomizedOverridePackageSettings extends DefaultPackageSettings { + private final List override + + CustomizedOverridePackageSettings(TemporaryFolder temporaryFolder, List override) { + super(temporaryFolder.root) + this.override = override + } + + @Override + boolean isCustomized(PackageInfo packageInfo) { + return override.contains(packageInfo.name) + } + } + static class SupportedLanguageVersionOverridePackageSettings extends DefaultPackageSettings { private final Map> override @@ -66,12 +80,17 @@ class PipActionHelpers { List getInstallOptions(PackageInfo packageInfo) { return override.getOrDefault(packageInfo.name, []) } + + @Override + boolean isCustomized(PackageInfo packageInfo) { + return override.containsKey(packageInfo.name) + } } static class GlobalOptionOverridePackageSettings extends DefaultPackageSettings { private final Map> override - GlobalOptionOverridePackageSettings(TemporaryFolder temporaryFolder, override) { + GlobalOptionOverridePackageSettings(TemporaryFolder temporaryFolder, Map> override) { super(temporaryFolder.root) this.override = override } @@ -80,6 +99,11 @@ class PipActionHelpers { List getGlobalOptions(PackageInfo packageInfo) { return override.getOrDefault(packageInfo.name, []) } + + @Override + boolean isCustomized(PackageInfo packageInfo) { + return override.containsKey(packageInfo.name) + } } static class EnvOverridePackageSettings extends DefaultPackageSettings { @@ -94,19 +118,30 @@ class PipActionHelpers { Map getEnvironment(PackageInfo packageInfo) { envOverride.getOrDefault(packageInfo.name, [:]) } + + @Override + boolean isCustomized(PackageInfo packageInfo) { + return envOverride.containsKey(packageInfo.name) + } + } static class BuildOptionOverridePackageSetting extends DefaultPackageSettings { - private final Map> envOverride + private final Map> override - BuildOptionOverridePackageSetting(TemporaryFolder temporaryFolder, Map> envOverride) { + BuildOptionOverridePackageSetting(TemporaryFolder temporaryFolder, Map> override) { super(temporaryFolder.root) - this.envOverride = envOverride + this.override = override } @Override List getBuildOptions(PackageInfo packageInfo) { - return envOverride.getOrDefault(packageInfo.name, []) + return override.getOrDefault(packageInfo.name, []) + } + + @Override + boolean isCustomized(PackageInfo packageInfo) { + return override.containsKey(packageInfo.name) } } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy index c7809ad3..6925b9ee 100644 --- a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/PipInstallActionTest.groovy @@ -111,8 +111,7 @@ class PipInstallActionTest extends Specification { List override = ['pyflakes'] def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, override) def execSpec = Mock(ExecSpec) - def mockWheelCache = Mock(WheelCache) - def pipInstallAction = createPipInstallAction(settings, execSpec, mockWheelCache) + def pipInstallAction = createPipInstallAction(settings, execSpec) def eggFile = pipInstallAction.sitePackagesPath.resolve("pyflakes-1.6.0-py3.6.egg-info").toFile() eggFile.parentFile.mkdirs() @@ -123,7 +122,7 @@ class PipInstallActionTest extends Specification { distInfo.createNewFile() when: - pipInstallAction.execute(packageInGradleCache("pyflakes-1.0.0.tar.gz"), []) + pipInstallAction.execute(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) then: 1 * execSpec.commandLine(_) >> { List> args -> @@ -134,8 +133,6 @@ class PipInstallActionTest extends Specification { assert it[2] == 'install' assert it[idx + 1] == '--ignore-installed' } - - 0 * mockWheelCache._ } def 'will skip if package (egg) is installed'() { @@ -170,6 +167,23 @@ class PipInstallActionTest extends Specification { 0 * execSpec._ } + def 'does not rebuild if only customized but present'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.CustomizedOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def pipInstallAction = createPipInstallAction(settings, execSpec) + + def distInfo = pipInstallAction.sitePackagesPath.resolve("pyflakes-1.6.0.dist-info").toFile() + distInfo.parentFile.mkdirs() + distInfo.createNewFile() + + when: + pipInstallAction.execute(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: + 0 * execSpec._ + } + private PipInstallAction createPipInstallAction(PackageSettings settings, ExecSpec execSpec) { return createPipInstallAction(settings, execSpec, new EmptyWheelCache()) } diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilderTest.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilderTest.groovy new file mode 100644 index 00000000..eb25e38b --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/action/pip/WheelBuilderTest.groovy @@ -0,0 +1,442 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.action.pip + +import com.linkedin.gradle.python.exception.PipExecutionException +import com.linkedin.gradle.python.extension.PythonDetailsFactory +import com.linkedin.gradle.python.extension.PythonDetailsTestDouble +import com.linkedin.gradle.python.extension.internal.DefaultVirtualEnvironment +import com.linkedin.gradle.python.tasks.exec.ExternalExec +import com.linkedin.gradle.python.tasks.exec.ExternalExecFailTestDouble +import com.linkedin.gradle.python.tasks.exec.ExternalExecTestDouble +import com.linkedin.gradle.python.util.DefaultEnvironmentMerger +import com.linkedin.gradle.python.util.DefaultPackageSettings +import com.linkedin.gradle.python.util.PackageInfo +import com.linkedin.gradle.python.util.PackageSettings +import com.linkedin.gradle.python.wheel.WheelCache +import com.linkedin.gradle.python.wheel.WheelCacheLayer +import org.gradle.process.ExecSpec +import org.gradle.testfixtures.ProjectBuilder +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import spock.lang.Specification + +import static com.linkedin.gradle.python.tasks.action.pip.PipActionHelpers.packageInGradleCache + + +/** + * Unit tests for WheelBuilder action class. + */ +class WheelBuilderTest extends Specification { + + @Rule + TemporaryFolder temporaryFolder + + def "wheel found in project layer is returned"() { + setup: "return wheel stub from project layer" + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >> Optional.of(new File(expected)) + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request package that is present in project layer" + def pkg = wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "we get the wheel from project layer without any rebuilding" + assert pkg.toString() == expected + 0 * execSpec._ + } + + def "wheel found in host layer is returned"() { + setup: "return wheel stub from host layer only" + def execSpec = Mock(ExecSpec) + def expected = 'fake/host-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >> Optional.empty() + findWheel(!null, !null, !null, WheelCacheLayer.HOST_LAYER) >> Optional.of(new File(expected)) + storeWheel(!null, WheelCacheLayer.PROJECT_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request package that is present only in host layer" + def pkg = wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "we get the wheel from host layer without any rebuilding and store it into project layer too" + assert pkg.toString() == expected + assert storeCounter == 1 + 0 * execSpec._ + } + + def "wheel not found in cache is built"() { + setup: "do not return wheel from cache layers" + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request package that is not in cache" + def pkg = wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "wheel is built but if it's not present after that, the source package is returned as fallback" + assert pkg == packageInGradleCache("foo-1.0.0.tar.gz").getPackageFile() + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + } + } + + def "wheel build ignores environment when not customized"() { + setup: "do not return wheel from cache layers on first attempt" + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >>> [ + Optional.empty(), Optional.of(new File(expected))] + findWheel(!null, !null, !null, WheelCacheLayer.HOST_LAYER) >> Optional.empty() + storeWheel(!null, WheelCacheLayer.HOST_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request package that is not in cache" + def pkg = wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "wheel is built excluding PythonEnvironment, then stored into host layer and returned" + assert pkg.toString() == expected + assert storeCounter == 1 + 1 * execSpec.environment([:]) + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + } + } + + def "wheel build ignores extraArgs not used with pip wheel command"() { + setup: "do not return wheel from cache layers on first attempt" + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >>> [ + Optional.empty(), Optional.of(new File(expected))] + findWheel(!null, !null, !null, WheelCacheLayer.HOST_LAYER) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request package that is not in cache" + def pkg = wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), ["--upgrade", "--ignore-installed"]) + + then: "wheel is built and there are no incompatible options" + assert pkg.toString() == expected + 1 * execSpec.environment([:]) + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + assert !it.any { entry -> entry == '--upgrade' } + assert !it.any { entry -> entry == '--ignore-installed' } + } + } + + def "wheel build adds environment after failed build without customization"() { + setup: "do not return wheel from cache layers" + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(null, new ExternalExecFailTestDouble(execSpec), stubWheelCache) + + when: "we request package that is not cache" + wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "wheel is rebuilt without PythonEnvironment, then with the environment, and error thrown if both fail" + thrown(PipExecutionException) + 1 * execSpec.environment([:]) + 1 * execSpec.environment(['CPPFLAGS':'bogus', 'LDFLAGS':'bogus']) + } + + def "uses environment"() { + setup: + def override = [foo: ['CPPFLAGS': '-I/some/custom/path/include', + 'LDFLAGS' : '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib'] + ] + def settings = new PipActionHelpers.EnvOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request package that's not in cache" + wheelBuilder.getPackage(packageInGradleCache("foo-1.0.0.tar.gz"), []) + + then: "wheel is built with the custom environment" + 1 * execSpec.environment([ + 'CPPFLAGS': '-I/some/custom/path/include', + 'LDFLAGS': '-L/some/custom/path/lib -Wl,-rpath,/some/custom/path/lib' + ]) + } + + def 'uses global options'() { + Map> override = ['setuptools': ['--global-option', '--dummy-global-option']] + def settings = new PipActionHelpers.GlobalOptionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request package that's not in cache" + wheelBuilder.getPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: "wheel is built with the global options" + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--global-option') + assert idx != -1 + assert it[2] == 'wheel' + assert it[idx + 1] == '--dummy-global-option' + } + } + + def 'uses build options'() { + Map> override = ['setuptools': ['--build-option', '--disable-something']] + def settings = new PipActionHelpers.BuildOptionOverridePackageSetting(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request package that's not in cache" + wheelBuilder.getPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: "wheel is built with build options" + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--build-option') + assert idx != -1 + assert it[2] == 'wheel' + assert it[idx + 1] == '--disable-something' + } + } + + def "does not use supported language versions"() { + Map> override = ['setuptools': ['2.8']] + def settings = new PipActionHelpers.SupportedLanguageVersionOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(*_) >> Optional.empty() + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request the package that's not in cache" + wheelBuilder.getPackage(packageInGradleCache("setuptools-1.0.0.tar.gz"), []) + + then: "the supported versions are ignored; the caller PipInstallAction should throw exception for this!" + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + } + } + + def 'requires source rebuild'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.RequiresRebuildOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >> Optional.of(new File(expected)) + storeWheel(!null, WheelCacheLayer.HOST_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request the package that is in wheel cache" + wheelBuilder.getPackage(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: "we still rebuild it because of the required source build and we do not store it into host layer" + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--no-deps') + assert idx != -1 + assert it[2] == 'wheel' + assert !it.any { entry -> entry == '--ignore-installed' } + } + assert storeCounter == 0 + } + + def 'does not rebuild if customized but present'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.CustomizedOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >> Optional.of(new File(expected)) + storeWheel(!null, WheelCacheLayer.HOST_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request the package that is in project layer" + def pkg = wheelBuilder.getPackage(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: "we do not rebuild it or store it into host layer" + assert pkg.toString() == expected + assert storeCounter == 0 + 0 * execSpec._ + } + + def 'rebuilds if customized and not in project layer'() { + List override = ['pyflakes'] + def settings = new PipActionHelpers.CustomizedOverridePackageSettings(temporaryFolder, override) + def execSpec = Mock(ExecSpec) + def expected = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >>> [Optional.empty(), Optional.of(new File(expected))] + storeWheel(!null, WheelCacheLayer.HOST_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(settings, execSpec, stubWheelCache) + + when: "we request the package that is not in project layer" + def pkg = wheelBuilder.getPackage(packageInGradleCache("pyflakes-1.6.0.tar.gz"), []) + + then: "we build it and get the wheel, but do not store it into host layer" + assert pkg.toString() == expected + assert storeCounter == 0 + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + def idx = it.indexOf('--no-deps') + assert idx != -1 + assert it[2] == 'wheel' + assert !it.any { entry -> entry == '--ignore-installed' } + } + } + + def 'builds project wheel but returns the project itself'() { + setup: "do not return wheel from cache layers on first attempt" + def execSpec = Mock(ExecSpec) + def fakeWheel = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >>> [ + Optional.empty(), Optional.of(new File(fakeWheel))] + findWheel(!null, !null, !null, WheelCacheLayer.HOST_LAYER) >> Optional.empty() + storeWheel(!null, WheelCacheLayer.HOST_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache) + + when: "we request project to be built" + def pkg = wheelBuilder.getPackage(PackageInfo.fromPath(wheelBuilder.project.getProjectDir()), []) + + then: "wheel is built but not stored to host layer and project directory returned for editable install" + assert pkg.toString() == wheelBuilder.project.getProjectDir().toString() + assert storeCounter == 0 + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + } + } + + def "wheel with matching custom environment is built"() { + setup: "return wheel stub from host layer only" + def execSpec = Mock(ExecSpec) + def expected = 'fake/host-dir/wheel' + def fakeWheel = 'fake/project-dir/wheel' + // Cardinality of calls on stubs cannot be asserted, as opposed to mocks, so we keep the counter. + def storeCounter = 0 + def stubWheelCache = Stub(WheelCache) { + getTargetDirectory() >> Optional.of(new File('fake/project-dir')) + findWheel(!null, !null, !null, WheelCacheLayer.PROJECT_LAYER) >>> [ + Optional.empty(), Optional.of(new File(fakeWheel))] + findWheel(!null, !null, !null, WheelCacheLayer.HOST_LAYER) >> Optional.of(new File(expected)) + storeWheel(!null, WheelCacheLayer.PROJECT_LAYER) >> { storeCounter++ } + } + def wheelBuilder = createWheelBuilder(execSpec, stubWheelCache, ['OPENBLAS': 'None']) + + when: "we request package that is present only in host layer but has custom pythonEnvironment" + def pkg = wheelBuilder.getPackage(packageInGradleCache("numpy-12.0.0.tar.gz"), []) + + then: "we do not get it from the host layer but build the custom wheel instead and get it from project layer" + assert pkg.toString() == fakeWheel + assert storeCounter == 0 + 1 * execSpec.commandLine(_) >> { List> args -> + println args + def it = args[0] + assert it[2] == 'wheel' + } + } + + private WheelBuilder createWheelBuilder(PackageSettings settings, ExecSpec execSpec, WheelCache wheelCache) { + return createWheelBuilder(settings, new ExternalExecTestDouble(execSpec), wheelCache) + } + + private WheelBuilder createWheelBuilder(ExecSpec execSpec, WheelCache wheelCache) { + createWheelBuilder(null, new ExternalExecTestDouble(execSpec), wheelCache) + } + + private WheelBuilder createWheelBuilder(ExecSpec execSpec, WheelCache wheelCache, Map environment) { + createWheelBuilder(null, new ExternalExecTestDouble(execSpec), wheelCache, environment) + } + + private WheelBuilder createWheelBuilder(PackageSettings settings, ExternalExec executor, WheelCache wheelCache) { + createWheelBuilder(settings, executor, wheelCache, ['CPPFLAGS': 'bogus', 'LDFLAGS': 'bogus']) + } + + private WheelBuilder createWheelBuilder( + PackageSettings settings, ExternalExec executor, WheelCache wheelCache, Map environment) { + def project = new ProjectBuilder().withProjectDir(temporaryFolder.root).build() + def packageSettings = settings ?: new DefaultPackageSettings(project.getProjectDir()) + def binDir = temporaryFolder.newFolder('build', 'venv', PythonDetailsFactory.getPythonApplicationDirectory()) + DefaultVirtualEnvironment.findExecutable(binDir.toPath(), "pip").createNewFile() + DefaultVirtualEnvironment.findExecutable(binDir.toPath(), "python").createNewFile() + def details = new PythonDetailsTestDouble(project, binDir.parentFile) + return new WheelBuilder(packageSettings, project, executor, environment, details, wheelCache, + new DefaultEnvironmentMerger(), { it -> false }) + } +} diff --git a/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecFailTestDouble.groovy b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecFailTestDouble.groovy new file mode 100644 index 00000000..6764e18d --- /dev/null +++ b/pygradle-plugin/src/test/groovy/com/linkedin/gradle/python/tasks/exec/ExternalExecFailTestDouble.groovy @@ -0,0 +1,36 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.linkedin.gradle.python.tasks.exec + +import org.gradle.api.Action +import org.gradle.process.ExecResult +import org.gradle.process.ExecSpec + + +class ExternalExecFailTestDouble implements ExternalExec { + + private final ExecSpec spec + + ExternalExecFailTestDouble(ExecSpec spec) { + this.spec = spec + } + + @Override + ExecResult exec(Action action) { + action.execute(spec) + return ['getExitValue': { -> 1 }] as ExecResult + } +}