From 11a10fc9ea4deb965a406efb24cdae59cab767da Mon Sep 17 00:00:00 2001
From: Jake Smith <jake.smith@lexisnexisrisk.com>
Date: Fri, 14 Jul 2023 17:06:29 +0100
Subject: [PATCH] HPCC-29746 Fix hthor bug reading remote compressed files.

HThor was treating all external files (including remote) as if
they were external files (~file::), for the purpose of calculating
whether they were compressed on disk.
As a consequence, hthor disk reads would read compressed files
as if they were uncompressed, and consequently fail with various
errors.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>
---
 dali/base/dautils.hpp                     | 1 +
 ecl/hthor/hthor.cpp                       | 4 ++--
 esp/clients/ws_dfsclient/ws_dfsclient.cpp | 7 ++++++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/dali/base/dautils.hpp b/dali/base/dautils.hpp
index 5ea1f8b447c..d42b78fe0bd 100644
--- a/dali/base/dautils.hpp
+++ b/dali/base/dautils.hpp
@@ -448,6 +448,7 @@ interface ILocalOrDistributedFile: extends IInterface
     virtual bool getPartCrc(unsigned partnum, unsigned &crc) = 0;
     virtual bool exists() const = 0;   // if created for writing, this may be false
     virtual bool isExternal() const = 0;
+    virtual bool isExternalFile() const = 0;
 };
 
 typedef __int64 ConnectionId;
diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
index c80fcd65e44..66c47fd80b3 100644
--- a/ecl/hthor/hthor.cpp
+++ b/ecl/hthor/hthor.cpp
@@ -490,7 +490,7 @@ void CHThorDiskWriteActivity::resolve()
                 else
                     throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", lfn.str());
             }
-            else if (f->exists() || f->isExternal() || agent.queryResolveFilesLocally())
+            else if (f->exists() || f->isExternalFile() || agent.queryResolveFilesLocally())
             {
                 // special/local/external file
                 if (f->numParts()!=1)
@@ -8370,7 +8370,7 @@ void CHThorDiskReadBaseActivity::resolve()
             Owned<IFileDescriptor> fdesc;
             fdesc.setown(ldFile->getFileDescriptor());
             gatherInfo(fdesc);
-            if (ldFile->isExternal())
+            if (ldFile->isExternalFile())
                 compressed = checkWriteIsCompressed(helper.getFlags(), fixedDiskRecordSize, false);//grouped=FALSE because fixedDiskRecordSize already includes grouped
             IDistributedFile *dFile = ldFile->queryDistributedFile();
             if (dFile)  //only makes sense for distributed (non local) files
diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp
index 59df048c19d..6905c574e74 100644
--- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp
+++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp
@@ -920,7 +920,7 @@ class CLocalOrDistributedFile: implements ILocalOrDistributedFile, public CInter
         }
         if (!onlylocal)
         {
-            if (lfn.isExternal() && !lfn.isRemote())
+            if (lfn.isExternalFile() || lfn.isExternalPlane())
             {
                 Owned<IFileDescriptor> fDesc = createExternalFileDescriptor(lfn.get());
                 dfile.setown(queryDistributedFileDirectory().createExternal(fDesc, lfn.get()));
@@ -1126,6 +1126,11 @@ class CLocalOrDistributedFile: implements ILocalOrDistributedFile, public CInter
     {
         return lfn.isExternal();
     }
+
+    virtual bool isExternalFile() const override
+    {
+        return lfn.isExternalFile() || lfn.isExternalPlane();
+    }
 };