Skip to content

Commit

Permalink
Improved handling of cached dependencies and added cache also for Cla…
Browse files Browse the repository at this point in the history
…ssLoaders.
  • Loading branch information
renatoathaydes committed Apr 24, 2024
1 parent 044fd4a commit d92c749
Show file tree
Hide file tree
Showing 11 changed files with 157 additions and 80 deletions.
60 changes: 60 additions & 0 deletions jgrab-runner/src/main/java/com/athaydes/jgrab/Classpath.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.athaydes.jgrab;

import java.io.File;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;

import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableSortedSet;

public final class Classpath {
private static final Classpath EMPTY = new Classpath( new TreeSet<>(), List.of() );

public final SortedSet<Dependency> dependencies;
public final List<File> resolvedArtifacts;
public final String hash;

public Classpath( SortedSet<Dependency> dependencies, List<File> resolvedArtifacts ) {
this( dependencies, resolvedArtifacts, Dependency.hashOf( dependencies ) );
}

public Classpath( SortedSet<Dependency> dependencies,
List<File> resolvedArtifacts,
String hash ) {
this.dependencies = unmodifiableSortedSet( dependencies );
this.resolvedArtifacts = unmodifiableList( resolvedArtifacts );
this.hash = hash;
}

public static Classpath empty() {
return EMPTY;
}

public boolean isEmpty() {
return resolvedArtifacts.isEmpty();
}

@Override
public boolean equals( Object o ) {
if ( this == o ) return true;
if ( o == null || getClass() != o.getClass() ) return false;

Classpath classpath = ( Classpath ) o;

return hash.equals( classpath.hash );
}

@Override
public int hashCode() {
return hash.hashCode();
}

@Override
public String toString() {
return "Classpath{" +
"dependencies=" + dependencies +
", resolvedArtifacts=" + resolvedArtifacts +
'}';
}
}
6 changes: 3 additions & 3 deletions jgrab-runner/src/main/java/com/athaydes/jgrab/Dependency.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static Dependency of( String declaration ) {
return new Dependency( parts[ 0 ], parts[ 1 ], parts.length == 2 ? "latest" : parts[ 2 ] );
}

public static String hashOf( Collection<Dependency> dependencies ) {
public static String hashOf( SortedSet<Dependency> dependencies ) {
var list = new ArrayList<>( dependencies );
list.sort( COMPARATOR );

Expand All @@ -60,15 +60,15 @@ public static String hashOf( Collection<Dependency> dependencies ) {
return Base64.getUrlEncoder().encodeToString( digest.digest() );
}

public static Set<Dependency> parseDependencies( Stream<String> codeLines ) {
public static SortedSet<Dependency> parseDependencies( Stream<String> codeLines ) {
return codeLines.flatMap( line -> {
Matcher matcher = JGRAB_PATTERN.matcher( line );
if ( matcher.matches() ) {
return Stream.of( Dependency.of( matcher.group( 1 ) ) );
} else {
return Stream.empty();
}
} ).collect( Collectors.toSet() );
} ).collect( Collectors.toCollection( TreeSet::new ) );
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.athaydes.jgrab.Dependency;

import java.util.Set;
import java.util.SortedSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -18,7 +18,7 @@ public interface JavaCode {
Pattern CLASS_PATTERN = Pattern.compile(
"\\s*((public|static|private|abstract|final)\\s+)*\\s*class\\s+(?<name>[a-zA-Z_0-9.$]+).*" );

Set<Dependency> extractDependencies();
SortedSet<Dependency> extractDependencies();

String getClassName();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.Set;
import java.util.SortedSet;
import java.util.stream.Stream;

/**
Expand Down Expand Up @@ -49,7 +49,7 @@ public boolean isSnippet() {
}

@Override
public Set<Dependency> extractDependencies() {
public SortedSet<Dependency> extractDependencies() {
return Dependency.parseDependencies( Stream.of( lines ) );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.athaydes.jgrab.Dependency;

import java.util.Set;
import java.util.SortedSet;
import java.util.stream.Stream;

/**
Expand All @@ -21,7 +21,7 @@ public StringJavaCode( String code ) {
}

@Override
public Set<Dependency> extractDependencies() {
public SortedSet<Dependency> extractDependencies() {
return Dependency.parseDependencies( Stream.of( codeLines ) );
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.athaydes.jgrab.daemon;

import com.athaydes.jgrab.Dependency;
import com.athaydes.jgrab.Classpath;
import com.athaydes.jgrab.code.JavaCode;
import com.athaydes.jgrab.code.StringJavaCode;
import com.athaydes.jgrab.ivy.IvyGrabber;
Expand All @@ -9,11 +9,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;

/**
Expand Down Expand Up @@ -51,6 +52,8 @@ public static void start( RunArgs runArgs ) {
return;
}

JGrabRunner.populateClassLoaderCache( libsCache.loadCache().values() );

while ( true ) {
try ( Socket clientSocket = serverSocket.accept();
final PrintStream out = new PrintStream( clientSocket.getOutputStream(), true );
Expand Down Expand Up @@ -117,20 +120,19 @@ public static void start( RunArgs runArgs ) {

logSourceCode( code );

Set<Dependency> deps = code.extractDependencies();
var deps = code.extractDependencies();

logger.debug( "Dependencies to grab: {}", deps );

List<File> libs = libsCache.libsFor( deps,
() -> grabber.grab( deps ) );
var classpath = libsCache.classpathOf( deps, () -> grabber.grab( deps ) );

System.setIn( clientSocket.getInputStream() );
System.setOut( out );
System.setErr( out );

// run this synchronously, which means only one program can run per daemon at a time
try {
runArgs.accept( code, args, libs );
runArgs.accept( code, args, classpath );
} catch ( Throwable t ) {
t.printStackTrace( out );
}
Expand All @@ -154,7 +156,7 @@ private static void logSourceCode( Object source ) {
public interface RunArgs {
void accept( JavaCode javaCode,
String[] args,
List<File> libs );
Classpath classpath );
}

}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.athaydes.jgrab.daemon;

import com.athaydes.jgrab.Classpath;
import com.athaydes.jgrab.Dependency;
import com.athaydes.jgrab.JGrabHome;
import org.slf4j.Logger;
Expand All @@ -20,13 +21,13 @@
/**
* A persistent cache for resolved dependencies
*/
class PersistentCache {
final class PersistentCache {

private static final Logger logger = LoggerFactory.getLogger( PersistentCache.class );

private final File cacheFile;
private final AtomicBoolean isCacheLoaded = new AtomicBoolean( false );
private final Map<Set<Dependency>, List<File>> cache = new HashMap<>();
private final Map<String, Classpath> cache = new HashMap<>();

PersistentCache() {
this( new File( JGrabHome.getDir(), "deps-cache" ) );
Expand Down Expand Up @@ -62,13 +63,11 @@ void save() throws IOException {
tempCacheFile.createNewFile();

try ( BufferedWriter writer = Files.newBufferedWriter( tempCacheFile.toPath(), StandardOpenOption.WRITE ) ) {
for (Map.Entry<Set<Dependency>, List<File>> entry : cache.entrySet()) {
var dependencies = new ArrayList<>( entry.getKey() );
dependencies.sort( Dependency.COMPARATOR );
String deps = dependencies.stream()
for ( var classpath : cache.values() ) {
String deps = classpath.dependencies.stream()
.map( Dependency::canonicalNotation )
.collect( Collectors.joining( "," ) );
String libs = entry.getValue().stream()
String libs = classpath.resolvedArtifacts.stream()
.map( File::getAbsolutePath )
.collect( Collectors.joining( File.pathSeparator ) );

Expand All @@ -90,22 +89,25 @@ void save() throws IOException {
}
}

List<File> libsFor( Set<Dependency> dependencies,
Supplier<List<File>> compute ) {
Classpath classpathOf( SortedSet<Dependency> dependencies,
Supplier<List<File>> compute ) {
if ( dependencies.isEmpty() ) {
return Collections.emptyList();
return Classpath.empty();
}

if ( !isCacheLoaded.getAndSet( true ) ) {
cache.putAll( loadCache() );
}

return cache.computeIfAbsent( dependencies, ( ignore ) -> compute.get() );
return cache.computeIfAbsent( Dependency.hashOf( dependencies ), ( hash ) ->
new Classpath( dependencies, compute.get(), hash ) );
}

Map<Set<Dependency>, List<File>> loadCache() {
logger.debug( "Initializing dependencies cache" );
return cacheFrom( loadCacheEntries() );
Map<String, Classpath> loadCache() {
logger.debug( "Loading dependencies cache" );
var result = cacheFrom( loadCacheEntries() );
isCacheLoaded.set( true );
return result;
}

private List<String> loadCacheEntries() {
Expand All @@ -126,24 +128,24 @@ private List<String> loadCacheEntries() {
return cacheEntries;
}

private Map<Set<Dependency>, List<File>> cacheFrom( List<String> cacheEntries ) {
Map<Set<Dependency>, List<File>> cache = new HashMap<>();
private Map<String, Classpath> cacheFrom( List<String> cacheEntries ) {
Map<String, Classpath> cache = new HashMap<>();

for (String entry : cacheEntries) {
for ( String entry : cacheEntries ) {
String[] parts = entry.split( " " );
if ( parts.length == 2 ) {
try {
Set<Dependency> deps = Stream.of( parts[ 0 ].split( "," ) )
var deps = Stream.of( parts[ 0 ].split( "," ) )
.map( Dependency::of )
.collect( Collectors.toSet() );
.collect( Collectors.toCollection( TreeSet::new ) );

List<File> libs = Stream.of( parts[ 1 ].split( File.pathSeparator ) )
.map( File::new )
.collect( Collectors.toList() );

if ( libs.stream().allMatch( File::isFile ) ) {
logger.debug( "Loading dependency entry from cache: {} -> {}", deps, libs );
cache.put( deps, libs );
cache.put( Dependency.hashOf( deps ), new Classpath( deps, libs ) );
} else {
logger.info( "Ignoring cache entry because not all lib files exist" );
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public List<File> grab( Collection<Dependency> toGrab ) {
NonEmptyCollection.of( List.of(
new FileArtifactRetriever( MavenUtils.mavenHome() ),
new HttpArtifactRetriever( log, MavenUtils.MAVEN_CENTRAL_URL ) ) ) );
var outputDir = outputDirHashOf( toGrab );
var outputDir = outputDirHashOf( new TreeSet<>( toGrab ) );

if ( outputDir.isDirectory() ) {
// the dir already exists, reuse that!
Expand Down Expand Up @@ -78,7 +78,7 @@ private static Void throwErrors( NonEmptyCollection<Throwable> errors ) {
throw new JGrabError( "Errors occurred while grabbing dependencies: " + errors );
}

private File outputDirHashOf( Collection<Dependency> toGrab ) {
private File outputDirHashOf( SortedSet<Dependency> toGrab ) {
//noinspection ResultOfMethodCallIgnored
cacheDir.mkdirs();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.regex.Matcher;

/**
Expand Down Expand Up @@ -43,7 +43,7 @@ public boolean isSnippet() {
}

@Override
public Set<Dependency> extractDependencies() {
public SortedSet<Dependency> extractDependencies() {
return Dependency.parseDependencies( lines.stream() );
}

Expand Down
Loading

0 comments on commit d92c749

Please sign in to comment.