diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index c958462b2fb..9b5811b8d38 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -115,6 +115,10 @@ jobs: fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform uses: actions/checkout@v3 with: diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index 8ae50e90f78..2609c13bffc 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -99,6 +99,10 @@ jobs: fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} uses: actions/checkout@v3 diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index d6ba0b3bdb0..5db7bfce4e7 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -307,7 +307,7 @@ class HelperDll : implements ILoadedDllEntry, public CInterface virtual const byte * getResource(unsigned id) const; virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const; virtual IPropertyTree &queryManifest() const override; - virtual const StringArray &queryManifestFiles(const char *type, const char *wuid) const override; + virtual const StringArray &queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const override; bool load(bool isGlobal, bool raiseOnError); bool loadCurrentExecutable(); bool loadResources(); @@ -492,7 +492,7 @@ IPropertyTree &HelperDll::queryManifest() const return *querySingleton(manifest, manifestLock, [this]{ return getEmbeddedManifestPTree(this); }); } -const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid) const +const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const { CriticalBlock b(manifestLock); Linked list = manifestFiles.find(type); @@ -500,8 +500,18 @@ const StringArray &HelperDll::queryManifestFiles(const char *type, const char *w { // The temporary path we unpack to is based on so file's current location and workunit // MORE - this is good for deployed cases, may not be so good for standalone executables. + // Doesn't really work for cloud either - it needs to be in ephemeral there StringBuffer tempDir; - splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr); + if (!isEmptyString(tempRoot)) + { + tempDir.append(tempRoot); + addPathSepChar(tempDir); + splitFilename(name, nullptr, nullptr, &tempDir, nullptr); + } + else + { + splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr); + } list.setown(new ManifestFileList(type, tempDir)); tempDir.append(".tmp").append(PATHSEPCHAR).append(wuid); VStringBuffer xpath("Resource[@type='%s']", type); diff --git a/common/dllserver/thorplugin.hpp b/common/dllserver/thorplugin.hpp index fc1f200c642..717bcbdbd9d 100644 --- a/common/dllserver/thorplugin.hpp +++ b/common/dllserver/thorplugin.hpp @@ -32,7 +32,7 @@ interface ILoadedDllEntry : extends IInterface virtual const byte * getResource(unsigned id) const = 0; virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace=true) const = 0; virtual IPropertyTree &queryManifest() const = 0; - virtual const StringArray &queryManifestFiles(const char *type, const char *tempDir) const = 0; + virtual const StringArray &queryManifestFiles(const char *type, const char *id, const char *tempRoot = nullptr) const = 0; }; extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile, bool resourcesOnly); diff --git a/plugins/javaembed/HpccClassLoader.java b/plugins/javaembed/HpccClassLoader.java index 052e909f077..6578bf0535c 100644 --- a/plugins/javaembed/HpccClassLoader.java +++ b/plugins/javaembed/HpccClassLoader.java @@ -21,47 +21,60 @@ HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems(R). import java.util.Hashtable; import java.util.List; import java.util.ArrayList; +import java.util.Arrays; +import java.util.stream.Collectors; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.lang.Throwable; +import com.HPCCSystems.HpccUtils; + public class HpccClassLoader extends java.lang.ClassLoader { private long bytecode; private int bytecodeLen; + private Boolean trace = false; private native Class defineClassForEmbed(int bytecodeLen, long bytecode, String name); private Hashtable> classes = new Hashtable<>(); - static private Hashtable pathLoaders = new Hashtable<>(); - private java.net.URLClassLoader pathLoader; + static private Hashtable sharedPathLoaders = new Hashtable<>(); + private List libraryPathLoaders = new ArrayList<>(); private HpccClassLoader(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname) { super(parent); if (classPath != null && !classPath.isEmpty()) { - synchronized(pathLoaders) + String[] libraryPaths = classPath.split("\\|"); // Careful - the param is a regex so need to escape the | + synchronized(sharedPathLoaders) { - pathLoader = pathLoaders.get(classPath); - if (pathLoader == null) + for (String libraryPath : libraryPaths) { - List urls = new ArrayList<>(); - String[] paths = classPath.split(";"); - for (String path : paths) + URLClassLoader libraryPathLoader = sharedPathLoaders.get(libraryPath); + if (libraryPathLoader == null) { - try - { - if (path.contains(":")) - urls.add(new URL(path)); - else - urls.add(new URL("file:" + path)); - } - catch (MalformedURLException E) + List urls = new ArrayList<>(); + String[] paths = libraryPath.split(";"); + for (String path : paths) { - // Ignore any that we don't recognize - // System.out.print(E.toString()); + try + { + if (path.contains(":")) + urls.add(new URL(path)); + else + urls.add(new URL("file:" + path)); + } + catch (MalformedURLException E) + { + // Ignore any that we don't recognize + if (trace) + HpccUtils.log("Malformed URL: " + E.toString()); + } } + libraryPathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()])); + if (trace) + HpccUtils.log("Created new URLClassLoader " + libraryPath); + sharedPathLoaders.put(libraryPath, libraryPathLoader); } - pathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()])); - pathLoaders.put(classPath, pathLoader); + libraryPathLoaders.add(libraryPathLoader); } } } @@ -74,25 +87,55 @@ public synchronized Class findClass(String className) throws ClassNotFoundExc { String luName = className.replace(".","/"); Class result = classes.get(luName); - if (result == null) + if (result != null) + return result; + if (trace) + HpccUtils.log("In findClass for " + className); + if (bytecodeLen != 0) + { + result = defineClassForEmbed(bytecodeLen, bytecode, luName); + if (result != null) + return result; + } + for (URLClassLoader libraryLoader: libraryPathLoaders) { - if (bytecodeLen != 0) - result = defineClassForEmbed(bytecodeLen, bytecode, luName); - if ( result == null && pathLoader != null) - result = pathLoader.loadClass(className); - if (result == null) - return super.findClass(className); - classes.put(luName, result); + try + { + if (trace) + HpccUtils.log("Looking in loader " + + Arrays.stream(libraryLoader.getURLs()) + .map(URL::toString) + .collect(Collectors.joining(";"))); + result = libraryLoader.loadClass(className); + if (result != null) + { + classes.put(luName, result); + return result; + } + } + catch (Exception E) + { + } } - return result; + throw new ClassNotFoundException(); } @Override public URL getResource(String path) { - URL ret = pathLoader.getResource(path); - if (ret == null) - ret = super.getResource(path); - return ret; + URL ret = null; + for (URLClassLoader libraryLoader: libraryPathLoaders) + { + try + { + ret = libraryLoader.getResource(path); + if (ret != null) + return ret; + } + catch (Exception E) + { + } + } + return super.getResource(path); } public static HpccClassLoader newInstance(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname) { diff --git a/plugins/javaembed/javaembed.cpp b/plugins/javaembed/javaembed.cpp index 39c5c8e3f5d..cde57fef00f 100644 --- a/plugins/javaembed/javaembed.cpp +++ b/plugins/javaembed/javaembed.cpp @@ -3257,6 +3257,7 @@ class JavaEmbedImportContext : public CInterfaceOf } } StringBuffer lclassPath; + bool needSep = false; if (engine) { StringArray manifestJars; @@ -3265,7 +3266,22 @@ class JavaEmbedImportContext : public CInterfaceOf { if (doTrace(traceJava)) DBGLOG("Adding manifest file %s to classpath", manifestJars.item(idx)); - lclassPath.append(';').append(manifestJars.item(idx)); + const char *thisJar = manifestJars.item(idx); + if (thisJar) + { + if (needSep) + lclassPath.append(';'); + lclassPath.append(manifestJars.item(idx)); + needSep = true; + } + else + { + if (needSep) + { + lclassPath.append('|'); + needSep = false; + } + } } } ForEachItemIn(idx, opts) @@ -3277,7 +3293,12 @@ class JavaEmbedImportContext : public CInterfaceOf StringBuffer optName(val-opt, opt); val++; if (stricmp(optName, "classpath")==0) - lclassPath.append(';').append(val); + { + if (needSep) + lclassPath.append('|'); // The | means we use a common classloader for the bit named by classpath + lclassPath.append(val); + needSep = true; + } else if (strieq(optName, "globalscope")) globalScopeKey.set(val); else if (strieq(optName, "persist")) @@ -3299,8 +3320,8 @@ class JavaEmbedImportContext : public CInterfaceOf throw MakeStringException(0, "javaembed: Unknown option %s", optName.str()); } } - if (lclassPath.length()>1) - classpath.set(lclassPath.str()+1); + if (lclassPath.length()) + classpath.set(lclassPath); if (flags & EFthreadlocal) sharedCtx->registerContext(this); // Do at end - otherwise an exception thrown during construction will leave this reference dangling } diff --git a/plugins/py3embed/py3embed.cpp b/plugins/py3embed/py3embed.cpp index f4d87a35096..283df68716f 100644 --- a/plugins/py3embed/py3embed.cpp +++ b/plugins/py3embed/py3embed.cpp @@ -738,11 +738,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx) ForEachItemIn(idx, manifestModules) { const char *path = manifestModules.item(idx); - DBGLOG("Manifest zip %s", path); - OwnedPyObject newPath = PyUnicode_FromString(path); - PyList_Insert(sysPath, 0, newPath); - checkPythonError(); - engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true); + if (path) + { + DBGLOG("Manifest zip %s", path); + OwnedPyObject newPath = PyUnicode_FromString(path); + PyList_Insert(sysPath, 0, newPath); + checkPythonError(); + engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true); + } } } } diff --git a/plugins/pyembed/pyembed.cpp b/plugins/pyembed/pyembed.cpp index 63d36f139e7..e99dcdbcadf 100644 --- a/plugins/pyembed/pyembed.cpp +++ b/plugins/pyembed/pyembed.cpp @@ -641,11 +641,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx) ForEachItemIn(idx, manifestModules) { const char *path = manifestModules.item(idx); - DBGLOG("Manifest zip %s", path); - OwnedPyObject newPath = PyString_FromString(path); - PyList_Insert(sysPath, 0, newPath); - checkPythonError(); - engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true); + if (path) + { + DBGLOG("Manifest zip %s", path); + OwnedPyObject newPath = PyString_FromString(path); + PyList_Insert(sysPath, 0, newPath); + checkPythonError(); + engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true); + } } } } diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index f8d3c0ae9a7..a81d5bf9ec4 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -456,6 +456,7 @@ extern StringBuffer pluginsList; extern StringBuffer queryDirectory; extern StringBuffer codeDirectory; extern StringBuffer tempDirectory; +extern StringBuffer spillDirectory; #undef UNIMPLEMENTED #undef throwUnexpected diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 6cd9f641852..aebc319cadc 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -3074,15 +3074,19 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon { ILoadedDllEntry *dll = factory->queryDll(); StringBuffer id; - const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str()); + const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str(), tempDirectory); ForEachItemIn(idx, dllFiles) files.append(dllFiles.item(idx)); ForEachItemIn(lidx, loadedLibraries) { IQueryFactory &lfactory = loadedLibraries.item(lidx); ILoadedDllEntry *ldll = lfactory.queryDll(); + // Libraries share the same copy of the jar (and the classloader) for all queries that use the library StringBuffer lid; - const StringArray &ldllFiles = ldll->queryManifestFiles(type, getQueryId(lid, true).str()); + lid.append('Q').append(lfactory.queryHash()); + const StringArray &ldllFiles = ldll->queryManifestFiles(type, lid.str(), tempDirectory); + if (ldllFiles.length()) + files.append(nullptr); ForEachItemIn(ldidx, ldllFiles) files.append(ldllFiles.item(ldidx)); } diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 015d5981bba..9c3563d1b4b 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -207,6 +207,7 @@ StringBuffer logDirectory; StringBuffer pluginDirectory; StringBuffer queryDirectory; StringBuffer codeDirectory; +StringBuffer spillDirectory; StringBuffer tempDirectory; ClientCertificate clientCert; @@ -1284,7 +1285,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) queryDirectory.append(codeDirectory).append("queries"); } addNonEmptyPathSepChar(queryDirectory); - getSpillFilePath(tempDirectory, "roxie", topology); + getSpillFilePath(spillDirectory, "roxie", topology); + getTempFilePath(tempDirectory, "roxie", topology); #ifdef _WIN32 topology->addPropBool("@linuxOS", false); diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index a2eb6349273..b960a9296fa 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -8567,7 +8567,7 @@ class CRoxieServerSortActivity : public CRoxieServerActivity if (sortAlgorithm==unknownSortAlgorithm) sorter.clear(); else - sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId)); + sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), spillDirectory, activityId)); } virtual void doStart(unsigned parentExtractSize, const byte *parentExtract, bool paused) @@ -8653,7 +8653,7 @@ class CRoxieServerSortActivity : public CRoxieServerActivity sorter.clear(); OwnedRoxieString algorithmName(helper.getAlgorithm()); sortAlgorithm = useAlgorithm(algorithmName, sortFlags); - sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId)); + sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), spillDirectory, activityId)); } sorter->prepare(inputStream); noteStatistic(StTimeSortElapsed, cycle_to_nanosec(sorter->getElapsedCycles(true))); @@ -12994,12 +12994,12 @@ class CRoxieServerJoinActivity : public CRoxieServerTwoInputActivity if (helper.isLeftAlreadySorted()) sortedLeft.setown(createDegroupedInputReader(inputStream)); else - sortedLeft.setown(createSortedInputReader(inputStream, createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + sortedLeft.setown(createSortedInputReader(inputStream, createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); ICompare *compareRight = helper.queryCompareRight(); if (helper.isRightAlreadySorted()) groupedSortedRight.setown(createGroupedInputReader(inputStream1, compareRight)); else - groupedSortedRight.setown(createSortedGroupedInputReader(inputStream1, compareRight, createSortAlgorithm(sortAlgorithm, compareRight, ctx->queryRowManager(), input1->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + groupedSortedRight.setown(createSortedGroupedInputReader(inputStream1, compareRight, createSortAlgorithm(sortAlgorithm, compareRight, ctx->queryRowManager(), input1->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) { //limited match join (s[1..n]) limitedhelper.setown(createRHLimitedCompareHelper()); @@ -18737,7 +18737,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm; else sortAlgorithm = isStable ? stableQuickSortAlgorithm : quickSortAlgorithm; - groupedInput.setown(createSortedGroupedInputReader(inputStream, compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + groupedInput.setown(createSortedGroupedInputReader(inputStream, compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); } if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) { //limited match join (s[1..n]) diff --git a/testing/regress/ecl/key/libraryjava.xml b/testing/regress/ecl/key/libraryjava.xml index d73a6eee541..a11d459e9ac 100644 --- a/testing/regress/ecl/key/libraryjava.xml +++ b/testing/regress/ecl/key/libraryjava.xml @@ -8,3 +8,6 @@ George Smith Baby Smith + + Hello World + diff --git a/testing/regress/ecl/libraryjava.ecl b/testing/regress/ecl/libraryjava.ecl index ec38d574751..b5c58ad99a9 100644 --- a/testing/regress/ecl/libraryjava.ecl +++ b/testing/regress/ecl/libraryjava.ecl @@ -19,6 +19,10 @@ //nothor //The library is defined and built in aaalibraryjava.ecl +IMPORT java; + +STRING cat(SET OF STRING s) := IMPORT(java, 'javaembed_ex7.cat:([Ljava/lang/String;)Ljava/lang/String;'); + namesRecord := RECORD string20 surname; @@ -50,3 +54,5 @@ integer init := appendDataset(namesTable).initialized : ONCE; output(init); appended := appendDataset(namesTable); output(appended.result); +// Call java from both query and library +output(cat(['Hello','World'])); diff --git a/testing/regress/ecl/libraryjava.manifest b/testing/regress/ecl/libraryjava.manifest new file mode 100644 index 00000000000..ad5b99ab861 --- /dev/null +++ b/testing/regress/ecl/libraryjava.manifest @@ -0,0 +1,3 @@ + + + diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 9d52f3bbd1d..92c2108f484 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -862,8 +862,8 @@ void CJobManager::reply(IConstWorkUnit *workunit, const char *wuid, IException * we->setExceptionMessage(errStr); we->setExceptionSource("thormasterexception"); we->setExceptionCode(e->errorCode()); - - w->setState(WUStateWait); + WUState newState = (WUStateRunning == w->getState()) ? WUStateWait : WUStateFailed; + w->setState(newState); } return; } @@ -1346,9 +1346,10 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam jobManager->execute(workunit, currentWuid, currentGraphName, dummyAgentEp); IException *e = jobManager->queryExitException(); Owned w = &workunit->lock(); + WUState newState = (WUStateRunning == w->getState()) ? WUStateWait : WUStateFailed; if (e) { - if (WUStateWait != w->getState()) // if set already, can only mean exception has been set already (see CJobManager::reply) + if (WUStateWait != w->getState()) // if set already, CJobManager::reply may have already set to WUStateWait { Owned we = w->createException(); we->setSeverity(SeverityInformation); @@ -1358,7 +1359,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam we->setExceptionSource("thormasterexception"); we->setExceptionCode(e->errorCode()); - w->setState(WUStateWait); + w->setState(newState); } break; } @@ -1366,7 +1367,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam if (!multiJobLinger && lingerPeriod) w->setDebugValue(instance, "1", true); - w->setState(WUStateWait); + w->setState(newState); } currentGraphName.clear();