Merge remote-tracking branch 'origin/candidate-9.0.x' into candidate-…

…9.2.x

.github/workflows/build-assets.yml

@@ -115,6 +115,10 @@ jobs:
       fail-fast: false
 
     steps:
+      - name: Free additional disk space (remove Android SDK + Tools)
+        run: |
+          sudo rm -rf /usr/local/lib/android
+
       - name: Checkout HPCC-Platform
         uses: actions/checkout@v3
         with:

.github/workflows/build-vcpkg.yml

@@ -99,6 +99,10 @@ jobs:
       fail-fast: false
 
     steps:
+      - name: Free additional disk space (remove Android SDK + Tools)
+        run: |
+          sudo rm -rf /usr/local/lib/android
+
       - name: Checkout HPCC-Platform
         if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         uses: actions/checkout@v3

common/dllserver/thorplugin.cpp

@@ -307,7 +307,7 @@ class HelperDll : implements ILoadedDllEntry, public CInterface
     virtual const byte * getResource(unsigned id) const;
     virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const;
     virtual IPropertyTree &queryManifest() const override;
-    virtual const StringArray &queryManifestFiles(const char *type, const char *wuid) const override;
+    virtual const StringArray &queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const override;
     bool load(bool isGlobal, bool raiseOnError);
     bool loadCurrentExecutable();
     bool loadResources();
@@ -492,16 +492,26 @@ IPropertyTree &HelperDll::queryManifest() const
     return *querySingleton(manifest, manifestLock, [this]{ return getEmbeddedManifestPTree(this); });
 }
 
-const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid) const
+const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const
 {
     CriticalBlock b(manifestLock);
     Linked<ManifestFileList> list = manifestFiles.find(type);
     if (!list)
     {
         // The temporary path we unpack to is based on so file's current location and workunit
         // MORE - this is good for deployed cases, may not be so good for standalone executables.
+        // Doesn't really work for cloud either - it needs to be in ephemeral there
         StringBuffer tempDir;
-        splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr);
+        if (!isEmptyString(tempRoot))
+        {
+            tempDir.append(tempRoot);
+            addPathSepChar(tempDir);
+            splitFilename(name, nullptr, nullptr, &tempDir, nullptr);
+        }
+        else
+        {
+            splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr);
+        }
         list.setown(new ManifestFileList(type, tempDir));
         tempDir.append(".tmp").append(PATHSEPCHAR).append(wuid);
         VStringBuffer xpath("Resource[@type='%s']", type);

common/dllserver/thorplugin.hpp

@@ -32,7 +32,7 @@ interface ILoadedDllEntry : extends IInterface
     virtual const byte * getResource(unsigned id) const = 0;
     virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace=true) const = 0;
     virtual IPropertyTree &queryManifest() const = 0;
-    virtual const StringArray &queryManifestFiles(const char *type, const char *tempDir) const = 0;
+    virtual const StringArray &queryManifestFiles(const char *type, const char *id, const char *tempRoot = nullptr) const = 0;
 };
 
 extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile, bool resourcesOnly);

plugins/javaembed/HpccClassLoader.java

@@ -21,47 +21,60 @@ HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems(R).
 import java.util.Hashtable;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.stream.Collectors;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.lang.Throwable;
+import com.HPCCSystems.HpccUtils;
+
 
 public class HpccClassLoader extends java.lang.ClassLoader
 {
     private long bytecode;
     private int bytecodeLen;
+    private Boolean trace = false;
     private native Class<?> defineClassForEmbed(int bytecodeLen, long bytecode, String name);
     private Hashtable<String, Class<?>> classes = new Hashtable<>();
-    static private Hashtable<String, java.net.URLClassLoader> pathLoaders = new Hashtable<>();
-    private java.net.URLClassLoader pathLoader;
+    static private Hashtable<String, java.net.URLClassLoader> sharedPathLoaders = new Hashtable<>();
+    private List<java.net.URLClassLoader> libraryPathLoaders = new ArrayList<>();
     private HpccClassLoader(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname)
     {
         super(parent);
         if (classPath != null && !classPath.isEmpty())
         {
-            synchronized(pathLoaders)
+            String[] libraryPaths = classPath.split("\\|");  // Careful - the param is a regex so need to escape the |
+            synchronized(sharedPathLoaders)
             {
-                pathLoader = pathLoaders.get(classPath);
-                if (pathLoader == null)
+                for (String libraryPath : libraryPaths)
                 {
-                    List<URL> urls = new ArrayList<>();
-                    String[] paths = classPath.split(";");
-                    for (String path : paths)
+                    URLClassLoader libraryPathLoader = sharedPathLoaders.get(libraryPath);
+                    if (libraryPathLoader == null)
                     {
-                        try
-                        {
-                            if (path.contains(":"))
-                                urls.add(new URL(path));
-                            else
-                                urls.add(new URL("file:" + path));
-                        }
-                        catch (MalformedURLException E)
+                        List<URL> urls = new ArrayList<>();
+                        String[] paths = libraryPath.split(";");
+                        for (String path : paths)
                         {
-                            // Ignore any that we don't recognize
-                            // System.out.print(E.toString());
+                            try
+                            {
+                                if (path.contains(":"))
+                                    urls.add(new URL(path));
+                                else
+                                    urls.add(new URL("file:" + path));
+                            }
+                            catch (MalformedURLException E)
+                            {
+                                // Ignore any that we don't recognize
+                                if (trace)
+                                    HpccUtils.log("Malformed URL: " + E.toString());
+                            }
                         }
+                        libraryPathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]));
+                        if (trace)
+                            HpccUtils.log("Created new URLClassLoader " + libraryPath);
+                        sharedPathLoaders.put(libraryPath, libraryPathLoader);
                     }
-                    pathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]));
-                    pathLoaders.put(classPath, pathLoader);
+                    libraryPathLoaders.add(libraryPathLoader);
                 }
             }
         }
@@ -74,25 +87,55 @@ public synchronized Class<?> findClass(String className) throws ClassNotFoundExc
     {
         String luName = className.replace(".","/");
         Class<?> result = classes.get(luName);
-        if (result == null)
+        if (result != null)
+            return result;
+        if (trace)
+            HpccUtils.log("In findClass for " + className);
+        if (bytecodeLen != 0)
+        {
+            result = defineClassForEmbed(bytecodeLen, bytecode, luName);
+            if (result != null)
+                return result;
+        }
+        for (URLClassLoader libraryLoader: libraryPathLoaders)
         {
-            if (bytecodeLen != 0)
-                result = defineClassForEmbed(bytecodeLen, bytecode, luName);
-            if ( result == null && pathLoader != null)
-                result = pathLoader.loadClass(className);
-            if (result == null)
-                return super.findClass(className);
-            classes.put(luName, result);
+            try
+            {
+                if (trace)
+                    HpccUtils.log("Looking in loader " + 
+                        Arrays.stream(libraryLoader.getURLs())
+                            .map(URL::toString)
+                            .collect(Collectors.joining(";")));
+                result = libraryLoader.loadClass(className);
+                if (result != null)
+                {
+                    classes.put(luName, result);
+                    return result;
+                }
+            }
+            catch (Exception E)
+            {
+            }
         }
-        return result; 
+        throw new ClassNotFoundException();
     }
     @Override
     public URL getResource(String path)
     {
-        URL ret = pathLoader.getResource(path);
-        if (ret == null)
-            ret = super.getResource(path);
-        return ret;
+        URL ret = null;
+        for (URLClassLoader libraryLoader: libraryPathLoaders)
+        {
+            try
+            {
+                ret = libraryLoader.getResource(path);
+                if (ret != null)
+                    return ret;
+            }
+            catch (Exception E)
+            {
+            }
+        }
+        return super.getResource(path);
     }
     public static HpccClassLoader newInstance(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname)
     {

plugins/javaembed/javaembed.cpp

@@ -3257,6 +3257,7 @@ class JavaEmbedImportContext : public CInterfaceOf<IJavaEmbedFunctionContext>
             }
         }
         StringBuffer lclassPath;
+        bool needSep = false;
         if (engine)
         {
             StringArray manifestJars;
@@ -3265,7 +3266,22 @@ class JavaEmbedImportContext : public CInterfaceOf<IJavaEmbedFunctionContext>
             {
                 if (doTrace(traceJava))
                     DBGLOG("Adding manifest file %s to classpath", manifestJars.item(idx));
-                lclassPath.append(';').append(manifestJars.item(idx));
+                const char *thisJar = manifestJars.item(idx);
+                if (thisJar)
+                {
+                    if (needSep)
+                        lclassPath.append(';');
+                    lclassPath.append(manifestJars.item(idx));
+                    needSep = true;
+                }
+                else
+                {
+                    if (needSep)
+                    {
+                        lclassPath.append('|');
+                        needSep = false;
+                    }
+                }
             }
         }
         ForEachItemIn(idx, opts)
@@ -3277,7 +3293,12 @@ class JavaEmbedImportContext : public CInterfaceOf<IJavaEmbedFunctionContext>
                 StringBuffer optName(val-opt, opt);
                 val++;
                 if (stricmp(optName, "classpath")==0)
-                    lclassPath.append(';').append(val);
+                {
+                    if (needSep)
+                        lclassPath.append('|');   // The | means we use a common classloader for the bit named by classpath
+                    lclassPath.append(val);
+                    needSep = true;
+                }
                 else if (strieq(optName, "globalscope"))
                     globalScopeKey.set(val);
                 else if (strieq(optName, "persist"))
@@ -3299,8 +3320,8 @@ class JavaEmbedImportContext : public CInterfaceOf<IJavaEmbedFunctionContext>
                     throw MakeStringException(0, "javaembed: Unknown option %s", optName.str());
             }
         }
-        if (lclassPath.length()>1)
-            classpath.set(lclassPath.str()+1);
+        if (lclassPath.length())
+            classpath.set(lclassPath);
         if (flags & EFthreadlocal)
             sharedCtx->registerContext(this);  // Do at end - otherwise an exception thrown during construction will leave this reference dangling
     }

plugins/py3embed/py3embed.cpp

@@ -738,11 +738,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx)
                 ForEachItemIn(idx, manifestModules)
                 {
                     const char *path = manifestModules.item(idx);
-                    DBGLOG("Manifest zip %s", path);
-                    OwnedPyObject newPath = PyUnicode_FromString(path);
-                    PyList_Insert(sysPath, 0, newPath);
-                    checkPythonError();
-                    engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true);
+                    if (path)
+                    {
+                        DBGLOG("Manifest zip %s", path);
+                        OwnedPyObject newPath = PyUnicode_FromString(path);
+                        PyList_Insert(sysPath, 0, newPath);
+                        checkPythonError();
+                        engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true);
+                    }
                 }
             }
         }

plugins/pyembed/pyembed.cpp

@@ -641,11 +641,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx)
                 ForEachItemIn(idx, manifestModules)
                 {
                     const char *path = manifestModules.item(idx);
-                    DBGLOG("Manifest zip %s", path);
-                    OwnedPyObject newPath = PyString_FromString(path);
-                    PyList_Insert(sysPath, 0, newPath);
-                    checkPythonError();
-                    engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true);
+                    if (path)
+                    {
+                        DBGLOG("Manifest zip %s", path);
+                        OwnedPyObject newPath = PyString_FromString(path);
+                        PyList_Insert(sysPath, 0, newPath);
+                        checkPythonError();
+                        engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true);
+                    }
                 }
             }
         }

roxie/ccd/ccd.hpp

@@ -456,6 +456,7 @@ extern StringBuffer pluginsList;
 extern StringBuffer queryDirectory;
 extern StringBuffer codeDirectory;
 extern StringBuffer tempDirectory;
+extern StringBuffer spillDirectory;
 
 #undef UNIMPLEMENTED
 #undef throwUnexpected

roxie/ccd/ccdcontext.cpp

@@ -3074,15 +3074,19 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon
     {
         ILoadedDllEntry *dll = factory->queryDll();
         StringBuffer id;
-        const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str());
+        const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str(), tempDirectory);
         ForEachItemIn(idx, dllFiles)
             files.append(dllFiles.item(idx));
         ForEachItemIn(lidx, loadedLibraries)
         {
             IQueryFactory &lfactory = loadedLibraries.item(lidx);
             ILoadedDllEntry *ldll = lfactory.queryDll();
+            // Libraries share the same copy of the jar (and the classloader) for all queries that use the library
             StringBuffer lid;
-            const StringArray &ldllFiles = ldll->queryManifestFiles(type, getQueryId(lid, true).str());
+            lid.append('Q').append(lfactory.queryHash());
+            const StringArray &ldllFiles = ldll->queryManifestFiles(type, lid.str(), tempDirectory);
+            if (ldllFiles.length())
+                files.append(nullptr);
             ForEachItemIn(ldidx, ldllFiles)
                 files.append(ldllFiles.item(ldidx));
         }

roxie/ccd/ccdmain.cpp

@@ -207,6 +207,7 @@ StringBuffer logDirectory;
 StringBuffer pluginDirectory;
 StringBuffer queryDirectory;
 StringBuffer codeDirectory;
+StringBuffer spillDirectory;
 StringBuffer tempDirectory;
 
 ClientCertificate clientCert;
@@ -1284,7 +1285,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                 queryDirectory.append(codeDirectory).append("queries");
         }
         addNonEmptyPathSepChar(queryDirectory);
-        getSpillFilePath(tempDirectory, "roxie", topology);
+        getSpillFilePath(spillDirectory, "roxie", topology);
+        getTempFilePath(tempDirectory, "roxie", topology);
 
 #ifdef _WIN32
         topology->addPropBool("@linuxOS", false);