Skip to content

Commit

Permalink
DEF-2888 Added support for collecting multiple classesN.dex files (#58)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mathiaswking
mathiaswking authored Jan 18, 2018
1 parent 15a64e2 commit c59ea2c
Show file tree
Hide file tree
Showing 12 changed files with 208 additions and 63 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
/.gradle
/.idea
/tmp
/debugsdk
.DS_Store
*.class
!server/test-data/sdk/
server/test-data/sdk/*
!server/test-data/sdk/a
Expand Down
5 changes: 2 additions & 3 deletions server/docker-base/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,9 +254,8 @@ RUN mkdir -p ${ANDROID_HOME} && \
# The sdk manager wants user access folder
USER extender
RUN cd ${ANDROID_HOME} && \
echo y | ./tools/bin/sdkmanager --verbose "tools" "platform-tools" "platforms;android-${ANDROID_TARGET_API_LEVEL}" "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" && \
# make sure it exists!
ls -la ./build-tools/${ANDROID_BUILD_TOOLS_VERSION}/dx
echo y | ./tools/bin/sdkmanager --verbose "tools" "platform-tools" "extras;android;m2repository" "platforms;android-${ANDROID_TARGET_API_LEVEL}" "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" && \
ls -la ./build-tools/${ANDROID_BUILD_TOOLS_VERSION}/dx # make sure it exists!

# Remove the rights again from the extender user
USER root
Expand Down
60 changes: 17 additions & 43 deletions server/src/main/java/com/defold/extender/Extender.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class Extender {
this.platform = platform;
this.sdk = sdk;
this.platformConfig = getPlatformConfig();

// LEGACY: Make sure the Emscripten compiler doesn't pollute the environment
processExecutor.putEnv("EM_CACHE", buildDirectory.toString());

Expand Down Expand Up @@ -379,34 +379,6 @@ private void buildExtension(File manifest, Map<String, Object> manifestContext)
processExecutor.execute(command);
}

static List<String> getAppManifestItems(AppManifestConfiguration manifest, String platform, String name) throws ExtenderException {
List<String> items = new ArrayList<>();

if( manifest == null || manifest.platforms == null )
return items;

if (manifest.platforms.containsKey("common")) {
Object v = manifest.platforms.get("common").context.get(name);
if( v != null ) {
if (!Extender.isListOfStrings((List<Object>) v)) {
throw new ExtenderException(String.format("The context variables only support lists of strings. Got %s (type %s)", v.toString(), v.getClass().getCanonicalName()));
}
items.addAll((List<String>) v);
}
}

if (manifest.platforms.containsKey(platform)) {
Object v = manifest.platforms.get(platform).context.get(name);
if( v != null ) {
if (!Extender.isListOfStrings((List<Object>) v)) {
throw new ExtenderException(String.format("The context variables only support lists of strings. Got %s (type %s)", v.toString(), v.getClass().getCanonicalName()));
}
items.addAll((List<String>) v);
}
}
return items;
}

private File linkEngine(List<String> symbols, Map<String, Object> manifestContext) throws IOException, InterruptedException, ExtenderException {
File maincpp = new File(buildDirectory , "main.cpp");

Expand All @@ -415,8 +387,8 @@ private File linkEngine(List<String> symbols, Map<String, Object> manifestContex

Map<String, Object> mainContext = context(manifestContext);

extSymbols = ExtenderUtil.pruneItems( extSymbols, getAppManifestItems(appManifest, platform, "includeSymbols"), getAppManifestItems(appManifest, platform, "excludeSymbols") );
mainContext.put("symbols", ExtenderUtil.pruneItems( (List<String>)mainContext.get("symbols"), getAppManifestItems(appManifest, platform, "includeSymbols"), getAppManifestItems(appManifest, platform, "excludeSymbols")));
extSymbols = ExtenderUtil.pruneItems( extSymbols, ExtenderUtil.getAppManifestItems(appManifest, platform, "includeSymbols"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeSymbols") );
mainContext.put("symbols", ExtenderUtil.pruneItems( (List<String>)mainContext.get("symbols"), ExtenderUtil.getAppManifestItems(appManifest, platform, "includeSymbols"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeSymbols")));
mainContext.put("ext", ImmutableMap.of("symbols", extSymbols));

String main = templateExecutor.execute(config.main, mainContext);
Expand Down Expand Up @@ -460,8 +432,8 @@ private File linkEngine(List<String> symbols, Map<String, Object> manifestContex
}
}

extLibs = ExtenderUtil.pruneItems( extLibs, getAppManifestItems(appManifest, platform, "includeLibs"), getAppManifestItems(appManifest, platform, "excludeLibs"));
extJsLibs = ExtenderUtil.pruneItems( extJsLibs, getAppManifestItems(appManifest, platform, "includeJsLibs"), getAppManifestItems(appManifest, platform, "excludeJsLibs"));
extLibs = ExtenderUtil.pruneItems( extLibs, ExtenderUtil.getAppManifestItems(appManifest, platform, "includeLibs"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeLibs"));
extJsLibs = ExtenderUtil.pruneItems( extJsLibs, ExtenderUtil.getAppManifestItems(appManifest, platform, "includeJsLibs"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeJsLibs"));

File exe;
if (platformConfig.writeExePattern != null ) {
Expand All @@ -474,8 +446,8 @@ private File linkEngine(List<String> symbols, Map<String, Object> manifestContex
context.put("src", ExtenderUtil.getRelativePath(jobDirectory, mainObject));
context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, exe));
context.put("ext", ImmutableMap.of("libs", extLibs, "libPaths", extLibPaths, "frameworks", extFrameworks, "frameworkPaths", extFrameworkPaths, "jsLibs", extJsLibs));
context.put("engineLibs", ExtenderUtil.pruneItems((List<String>) context.getOrDefault("engineLibs", new ArrayList<>()), getAppManifestItems(appManifest, platform, "includeLibs"), getAppManifestItems(appManifest, platform, "excludeLibs")) );
context.put("engineJsLibs", ExtenderUtil.pruneItems((List<String>) context.getOrDefault("engineJsLibs", new ArrayList<>()), getAppManifestItems(appManifest, platform, "includeJsLibs"), getAppManifestItems(appManifest, platform, "excludeJsLibs")) );
context.put("engineLibs", ExtenderUtil.pruneItems((List<String>) context.getOrDefault("engineLibs", new ArrayList<>()), ExtenderUtil.getAppManifestItems(appManifest, platform, "includeLibs"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeLibs")) );
context.put("engineJsLibs", ExtenderUtil.pruneItems((List<String>) context.getOrDefault("engineJsLibs", new ArrayList<>()), ExtenderUtil.getAppManifestItems(appManifest, platform, "includeJsLibs"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeJsLibs")) );

String command = templateExecutor.execute(platformConfig.linkCmd, context);
processExecutor.execute(command);
Expand Down Expand Up @@ -671,12 +643,12 @@ private Map<String, Object> getManifestContext(ManifestConfiguration manifestCon
return new HashMap<>();
}

private File buildClassesDex(List<File> extraJars) throws ExtenderException {
private File[] buildClassesDex(List<File> extraJars) throws ExtenderException {
LOGGER.info("Building classes.dex with extension source {}", uploadDirectory);

// To support older versions of build.yml where dxCmd is not defined:
if (platformConfig.dxCmd == null || platformConfig.dxCmd.isEmpty()) {
return null;
return new File[0];
}

File classesDex = new File(buildDirectory, "classes.dex");
Expand All @@ -692,8 +664,9 @@ private File buildClassesDex(List<File> extraJars) throws ExtenderException {
HashMap<String, Object> empty = new HashMap<>();
Map<String, Object> context = context(empty);
context.put("classes_dex", classesDex.getAbsolutePath());
context.put("jars", ExtenderUtil.pruneItems( extJars, getAppManifestItems(appManifest, platform, "includeJars"), getAppManifestItems(appManifest, platform, "excludeJars")));
context.put("engineJars", ExtenderUtil.pruneItems( (List<String>)context.get("engineJars"), getAppManifestItems(appManifest, platform, "includeJars"), getAppManifestItems(appManifest, platform, "excludeJars")) );
context.put("classes_dex_dir", buildDirectory.getAbsolutePath());
context.put("jars", ExtenderUtil.pruneItems( extJars, ExtenderUtil.getAppManifestItems(appManifest, platform, "includeJars"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeJars")));
context.put("engineJars", ExtenderUtil.pruneItems( (List<String>)context.get("engineJars"), ExtenderUtil.getAppManifestItems(appManifest, platform, "includeJars"), ExtenderUtil.getAppManifestItems(appManifest, platform, "excludeJars")) );

String command = templateExecutor.execute(platformConfig.dxCmd, context);
try {
Expand All @@ -702,7 +675,8 @@ private File buildClassesDex(List<File> extraJars) throws ExtenderException {
throw new ExtenderException(e, processExecutor.getOutput());
}

return classesDex;
File[] classes = ExtenderUtil.listFilesMatching(buildDirectory, "^classes(|[0-9]+)\\.dex$");
return classes;
}

private void buildWin32Manifest(File exe, Map<String, Object> mergedExtensionContext) throws ExtenderException {
Expand Down Expand Up @@ -785,9 +759,9 @@ List<File> build() throws ExtenderException {
File rJar = buildRJar();
List<File> extraJars = buildJava(rJar);

File classesDex = buildClassesDex(extraJars);
if (classesDex != null) {
outputFiles.add(classesDex);
File[] classesDex = buildClassesDex(extraJars);
if (classesDex.length > 0) {
outputFiles.addAll(Arrays.asList(classesDex));
}
}

Expand Down
43 changes: 43 additions & 0 deletions server/src/main/java/com/defold/extender/ExtenderUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.defold.extender;

import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -108,4 +109,46 @@ static void debugPrint(Map<String, Object> map, int indent) {
}
}
}

static File[] listFilesMatching(File dir, String regex) {
if(!dir.isDirectory()) {
throw new IllegalArgumentException(dir+" is not a directory.");
}
final Pattern p = Pattern.compile(regex);
return dir.listFiles(new FileFilter(){
@Override
public boolean accept(File file) {
return p.matcher(file.getName()).matches();
}
});
}

static List<String> getAppManifestItems(AppManifestConfiguration manifest, String platform, String name) throws ExtenderException {
List<String> items = new ArrayList<>();

if( manifest == null || manifest.platforms == null )
return items;

if (manifest.platforms.containsKey("common")) {
Object v = manifest.platforms.get("common").context.get(name);
if( v != null ) {
if (!Extender.isListOfStrings((List<Object>) v)) {
throw new ExtenderException(String.format("The context variables only support lists of strings. Got %s (type %s)", v.toString(), v.getClass().getCanonicalName()));
}
items.addAll((List<String>) v);
}
}

if (manifest.platforms.containsKey(platform)) {
Object v = manifest.platforms.get(platform).context.get(name);
if( v != null ) {
if (!Extender.isListOfStrings((List<Object>) v)) {
throw new ExtenderException(String.format("The context variables only support lists of strings. Got %s (type %s)", v.toString(), v.getClass().getCanonicalName()));
}
items.addAll((List<String>) v);
}
}
return items;
}

}
16 changes: 9 additions & 7 deletions server/src/test/java/com/defold/extender/ExtenderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,15 +287,17 @@ public void testCollectLibraries() {
@Test
public void testCollectJars() {
List<String> paths = Extender.collectFilesByPath(new File("test-data/ext/lib/armv7-android"), Extender.JAR_RE);
assertEquals(2, paths.size());
assertEquals(4, paths.size());

String[] suffixes = {"test-data/ext/lib/armv7-android/Dummy.jar", "test-data/ext/lib/armv7-android/JarDep.jar"};
String[] endings = {"test-data/ext/lib/armv7-android/Dummy.jar", "test-data/ext/lib/armv7-android/JarDep.jar",
"test-data/ext/lib/armv7-android/VeryLarge1.jar", "test-data/ext/lib/armv7-android/VeryLarge2.jar"};

for (String suffix : suffixes) {
for (String p : endings) {
boolean exists = false;
for (String path : paths) {
if (path.endsWith(suffix)) {
if (path.endsWith(p)) {
exists = true;
break;
}
}
assertTrue(exists);
Expand All @@ -321,15 +323,15 @@ public void testExcludeItems() throws IOException, InterruptedException, Extende

// Make sure it handles platforms
{
List<String> items = Extender.getAppManifestItems(appManifest, "x86-osx", "excludeSymbols");
List<String> items = ExtenderUtil.getAppManifestItems(appManifest, "x86-osx", "excludeSymbols");
assertTrue( items.contains("SymbolA") );
assertTrue( items.contains("SymbolB") );
assertFalse( items.contains("SymbolC") );
}

{
List<String> includePatterns = Extender.getAppManifestItems(appManifest, "x86-osx", "includeSymbols");
List<String> excludePatterns = Extender.getAppManifestItems(appManifest, "x86-osx", "excludeSymbols");
List<String> includePatterns = ExtenderUtil.getAppManifestItems(appManifest, "x86-osx", "includeSymbols");
List<String> excludePatterns = ExtenderUtil.getAppManifestItems(appManifest, "x86-osx", "excludeSymbols");
List<String> allItems = new ArrayList<>();
allItems.add("SymbolA");
allItems.add("SymbolB");
Expand Down
Loading

0 comments on commit c59ea2c

Please sign in to comment.