Skip to content

Commit

Permalink
Merge pull request #288 from zvezdan/backing-storage-find-store
Browse files Browse the repository at this point in the history 
Complete the wheel cache API and backing storage.
  • Loading branch information
@zvezdan
zvezdan authored Apr 13, 2019
2 parents 86627eb + 28d7635 commit 87e09e4
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 113 deletions.
2 changes: 1 addition & 1 deletion pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/PipInstallTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ class PipInstallTask extends DefaultTask implements FailureReasonProvider, Suppo
void pipInstall() {
def extension = ExtensionUtils.getPythonExtension(project)

ProgressLoggerFactory progressLoggerFactory = getServices().get(ProgressLoggerFactory)
ProgressLoggerFactory progressLoggerFactory = (ProgressLoggerFactory) getServices().get(ProgressLoggerFactory)
ProgressLogger progressLogger = progressLoggerFactory.newOperation(PipInstallTask)
progressLogger.setDescription("Installing Libraries")

Expand Down
14 changes: 11 additions & 3 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/EmptyWheelCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,23 @@

public class EmptyWheelCache implements WheelCache {
@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails) {
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails) {
return Optional.empty();
}

@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
return Optional.empty();
}

@Override
public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { }
public void storeWheel(File wheel) { }

@Override
public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { }

@Override
public Optional<File> getTargetDirectory() {
return Optional.empty();
}
}
45 changes: 23 additions & 22 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/FileBackedWheelCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,36 +38,37 @@ public FileBackedWheelCache(File cacheDir, PythonAbiContainer pythonAbiContainer
this.pythonAbiContainer = pythonAbiContainer;
}

/**
* Find's a wheel in the wheel cache.
*
* @param library name of the library
* @param version version of the library
* @param pythonDetails details on the python to find a wheel for
* @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
*/
@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails) {
return findWheel(library, version, pythonDetails.getVirtualEnvInterpreter());
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails) {
return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter());
}

@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails,
WheelCacheLayer wheelCacheLayer) {
return Optional.empty();
}

@Override
public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) { }
public void storeWheel(File wheel) { }

@Override
public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) { }

@Override
public Optional<File> getTargetDirectory() {
return Optional.empty();
}

/**
* Find's a wheel in the wheel cache.
* Finds a wheel in the cache.
*
* @param library name of the library
* @param version version of the library
* @param pythonExecutable Python Executable
* @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
* @param name package name
* @param version package version
* @param pythonExecutable Python interpreter executable file
* @return the wheel if found in the cache, otherwise {@code Optional.empty()}
*/
public Optional<File> findWheel(String library, String version, File pythonExecutable) {
public Optional<File> findWheel(String name, String version, File pythonExecutable) {
if (cacheDir == null) {
return Optional.empty();
}
Expand All @@ -79,13 +80,13 @@ public Optional<File> findWheel(String library, String version, File pythonExecu
* See PEP 427: https://www.python.org/dev/peps/pep-0427/
*/
String wheelPrefix = (
library.replace("-", "_")
name.replace("-", "_")
+ "-"
+ version.replace("-", "_")
+ "-"
);
logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix);
File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl"));
logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix);
File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl"));

if (files == null) {
return Optional.empty();
Expand All @@ -96,7 +97,7 @@ public Optional<File> findWheel(String library, String version, File pythonExecu
.map(Optional::get)
.collect(Collectors.toList());

logger.info("Wheels for version of library: {}", wheelDetails);
logger.info("Wheels for version of package: {}", wheelDetails);

Optional<PythonWheelDetails> foundWheel = wheelDetails.stream()
.filter(it -> wheelMatches(pythonExecutable, it))
Expand Down
96 changes: 44 additions & 52 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/LayeredWheelCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,52 +42,63 @@ public LayeredWheelCache(Map<WheelCacheLayer, File> layeredCacheMap, PythonAbiCo
this.pythonAbiContainer = pythonAbiContainer;
}

/**
* Find a wheel from all wheel cache layers.
*
* @param library name of the library
* @param version version of the library
* @param pythonDetails details on the python to find a wheel for
* @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
*/
@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails) {
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails) {
// TODO: Make sure layeredCacheMap is a LinkedHashMap when we initialize it in the plugin.
for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) {
Optional<File> layerResult = findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);
Optional<File> wheel = findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);

if (layerResult.isPresent()) {
return layerResult;
if (wheel.isPresent()) {
return wheel;
}
}

return Optional.empty();
}

/**
* Find wheel based on cache layer.
*
* @param library name of the library
* @param version version of the library
* @param pythonDetails details on the python to find a wheel for
* @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel
* @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()}
*/
@Override
public Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer) {
return findWheelInLayer(library, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);
public Optional<File> findWheel(String name, String version, PythonDetails pythonDetails,
WheelCacheLayer wheelCacheLayer) {
return findWheel(name, version, pythonDetails.getVirtualEnvInterpreter(), wheelCacheLayer);
}


@Override
public void storeWheel(File wheel) {
for (WheelCacheLayer wheelCacheLayer : layeredCacheMap.keySet()) {
storeWheel(wheel, wheelCacheLayer);
}
}

@Override
public void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer) {
File cacheDir = layeredCacheMap.get(wheelCacheLayer);

if (wheel != null && cacheDir != null) {
try {
Files.copy(wheel.toPath(), new File(cacheDir, wheel.getName()).toPath());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

@Override
public Optional<File> getTargetDirectory() {
return Optional.of(layeredCacheMap.get(WheelCacheLayer.PROJECT_LAYER));
}

/**
* Find a wheel from target layer.
*
* @param library name of the library
* @param version version of the library
* @param pythonExecutable python executable
* @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel
* @return A wheel that could be used in it's place. If not found, {@code Optional.empty()}
* @param name package name
* @param version package version
* @param pythonExecutable Python interpreter executable file
* @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from
* @return the wheel if found in the specified layer, otherwise {@code Optional.empty()}
*/
public Optional<File> findWheelInLayer(String library, String version, File pythonExecutable, WheelCacheLayer wheelCacheLayer) {
private Optional<File> findWheel(String name, String version, File pythonExecutable,
WheelCacheLayer wheelCacheLayer) {
File cacheDir = layeredCacheMap.get(wheelCacheLayer);

if (cacheDir == null) {
Expand All @@ -101,13 +112,13 @@ public Optional<File> findWheelInLayer(String library, String version, File pyth
* See PEP 427: https://www.python.org/dev/peps/pep-0427/
*/
String wheelPrefix = (
library.replace("-", "_")
name.replace("-", "_")
+ "-"
+ version.replace("-", "_")
+ "-"
);
logger.info("Searching for {} {} with prefix {}", library, version, wheelPrefix);
File[] files = cacheDir.listFiles((dir, name) -> name.startsWith(wheelPrefix) && name.endsWith(".whl"));
logger.info("Searching for {} {} with prefix {}", name, version, wheelPrefix);
File[] files = cacheDir.listFiles((dir, entry) -> entry.startsWith(wheelPrefix) && entry.endsWith(".whl"));

if (files == null) {
return Optional.empty();
Expand All @@ -118,7 +129,7 @@ public Optional<File> findWheelInLayer(String library, String version, File pyth
.map(Optional::get)
.collect(Collectors.toList());

logger.info("Wheels for version of library: {}", wheelDetails);
logger.info("Wheels for version of package: {}", wheelDetails);

Optional<PythonWheelDetails> foundWheel = wheelDetails.stream()
.filter(it -> wheelMatches(pythonExecutable, it))
Expand All @@ -129,25 +140,6 @@ public Optional<File> findWheelInLayer(String library, String version, File pyth
return foundWheel.map(it -> it.getFile());
}

/**
* Store given wheel file to target layer.
*
* @param wheelFile the wheel file to store
* @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel
*/
@Override
public void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer) {
File cacheDir = layeredCacheMap.get(wheelCacheLayer);

if (wheelFile != null && cacheDir != null) {
try {
Files.copy(wheelFile.toPath(), new File(cacheDir, wheelFile.getName()).toPath());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

private boolean wheelMatches(File pythonExecutable, PythonWheelDetails wheelDetails) {
return pythonAbiContainer.matchesSupportedVersion(
pythonExecutable,
Expand Down
50 changes: 38 additions & 12 deletions pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/wheel/WheelCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,50 @@
import java.util.Optional;

public interface WheelCache extends Serializable {
Optional<File> findWheel(String library, String version, PythonDetails pythonDetails);
/**
* Finds a wheel in the cache.
*
* <p>The wheel can be stored in any cache layer.</p>
*
* @param name package name
* @param version package version
* @param pythonDetails the Python interpreter and other details
* @return the wheel if found, otherwise {@code Optional.empty()}
*/
Optional<File> findWheel(String name, String version, PythonDetails pythonDetails);

/**
* Find wheel based on cache layer.
* Finds a wheel in the cache layer.
*
* @param library name of the library
* @param version version of the library
* @param pythonDetails details on the python to find a wheel for
* @param wheelCacheLayer which {@link WheelCacheLayer} to fetch wheel
* @return a wheel that could be used in the target layer. If not found, {@code Optional.empty()}
* @param name package name
* @param version package version
* @param pythonDetails the Python interpreter and other details
* @param wheelCacheLayer the {@link WheelCacheLayer} to fetch the wheel from
* @return the wheel if found in the specified layer, otherwise {@code Optional.empty()}
*/
Optional<File> findWheel(String library, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer);
Optional<File> findWheel(String name, String version, PythonDetails pythonDetails, WheelCacheLayer wheelCacheLayer);

/**
* Store given wheel file to target layer.
* Stores the wheel file into all cache layers.
*
* @param wheel the wheel file to store
*/
void storeWheel(File wheel);

/**
* Stores the wheel file into the cache layer.
*
* @param wheel the wheel file to store
* @param wheelCacheLayer the {@link WheelCacheLayer} to store the wheel in
*/
void storeWheel(File wheel, WheelCacheLayer wheelCacheLayer);

/**
* Gets the default directory for wheel build target.
*
* <p>This should be project layer cache directory</p>
*
* @param wheelFile the wheel file to store
* @param wheelCacheLayer which {@link WheelCacheLayer} to store wheel
* @return the directory used for wheel build target
*/
void storeWheel(File wheelFile, WheelCacheLayer wheelCacheLayer);
Optional<File> getTargetDirectory();
}
Loading

0 comments on commit 87e09e4

Please sign in to comment.