Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.4.0 Signed-off-by: Gavin Halliday <[email protected]>
hpcc-systems · Sep 26, 2023 · a0ab774 · a0ab774
2 parents 67d1de3 + 0448af1
commit a0ab774
Show file tree

Hide file tree

Showing 15 changed files with 271 additions and 14 deletions.
diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp
@@ -1378,9 +1378,23 @@ static void checkLogicalScope(const char *scopename,IUserDescriptor *user,bool r
     SecAccessFlags perm = getScopePermissions(scopename,user,auditflags);
     IDFS_Exception *e = NULL;
     if (readreq&&!HASREADPERMISSION(perm))
-        e = new CDFS_Exception(DFSERR_LookupAccessDenied,scopename);
+    {
+        StringBuffer scopeDescription;
+        StringBuffer username("");
+        if (user)
+            user->getUserName(username);
+        scopeDescription.appendf("%s user '%s', assigned access %s (%d)", scopename, username.str(), getSecAccessFlagName(perm), perm);
+        e = new CDFS_Exception(DFSERR_LookupAccessDenied,scopeDescription);
+    }
     else if (createreq&&!HASWRITEPERMISSION(perm))
-        e = new CDFS_Exception(DFSERR_CreateAccessDenied,scopename);
+    {
+        StringBuffer scopeDescription;
+        StringBuffer username("");
+        if (user)
+            user->getUserName(username);
+        scopeDescription.appendf("%s user '%s', assigned access %s (%d)", scopename, username.str(), getSecAccessFlagName(perm), perm);
+        e = new CDFS_Exception(DFSERR_CreateAccessDenied,scopeDescription);
+    }
     if (e)
         throw e;
 }

diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx
@@ -175,8 +175,8 @@ export const Files: React.FunctionComponent<FilesProps> = ({
                 },
             },
             Owner: { label: nlsHPCC.Owner },
-            SuperOwners: { label: nlsHPCC.SuperOwner },
-            Description: { label: nlsHPCC.Description },
+            SuperOwners: { label: nlsHPCC.SuperOwner, sortable: false },
+            Description: { label: nlsHPCC.Description, sortable: false },
             NodeGroup: { label: nlsHPCC.Cluster },
             Records: {
                 label: nlsHPCC.Records,

diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx
@@ -144,7 +144,7 @@ export const Workunits: React.FunctionComponent<WorkunitsProps> = ({
             Owner: { label: nlsHPCC.Owner, width: 80 },
             Jobname: { label: nlsHPCC.JobName },
             Cluster: { label: nlsHPCC.Cluster },
-            RoxieCluster: { label: nlsHPCC.RoxieCluster },
+            RoxieCluster: { label: nlsHPCC.RoxieCluster, sortable: false },
             State: { label: nlsHPCC.State, width: 60 },
             TotalClusterTime: {
                 label: nlsHPCC.TotalClusterTime, width: 120,

diff --git a/esp/src/src-react/components/forms/AddToSuperfile.tsx b/esp/src/src-react/components/forms/AddToSuperfile.tsx
@@ -123,7 +123,7 @@ export const AddToSuperfile: React.FunctionComponent<AddToSuperfileProps> = ({
                     <table className={`${componentStyles.twoColumnTable} ${componentStyles.selectionTable}`}>
                         <thead>
                             <tr>
-                                <th>{nlsHPCC.TargetName}</th>
+                                <th>{nlsHPCC.LogicalName}</th>
                             </tr>
                         </thead>
                         <tbody>

diff --git a/system/jlib/jsocket.cpp b/system/jlib/jsocket.cpp
@@ -3598,7 +3598,6 @@ bool IpAddress::ipset(const char *text)
 {
     if (text&&*text) {
         if ((text[0]=='.')&&(text[1]==0)) {
-            hostname.set(GetCachedHostName()); // Is this better than '.'?
             ipset(queryHostIP());
             return true;
         }

diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp
@@ -1434,7 +1434,7 @@ StringAttr& StringAttr::operator = (StringAttr && from)
 
 StringAttr& StringAttr::operator = (const StringAttr & from)
 {
-    set(from.str());
+    set(from.get());
     return *this;
 }
 

diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp
@@ -658,8 +658,6 @@ class CTraceManager : implements ITraceManager, public CInterface
                     DBGLOG("Tracing exporter type not specified");
             }
         }
-        else
-            DBGLOG("Tracing exporter type 'InMemory'");
 
         //Administrator can choose to process spans in batches or one at a time
         std::unique_ptr<opentelemetry::v1::sdk::trace::SpanProcessor> processor;
@@ -756,7 +754,6 @@ class CTraceManager : implements ITraceManager, public CInterface
             else
             {
                 initTracerProviderAndGlobalInternals(traceConfig);
-                DBGLOG("OpenTel tracing enabled!!");
             }
 
             // The global propagator should be set regardless of whether tracing is enabled or not.

diff --git a/testing/regress/ecl/common/SoapTextTest.ecl b/testing/regress/ecl/common/SoapTextTest.ecl
@@ -0,0 +1,132 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version multiPart=false,variant='default'
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, true);
+variant := #IFDEFINED(root.variant, '');
+
+//--- end of version configuration ---
+
+import $.^.setup;
+files := setup.files(multiPart, false);
+
+EXPORT SoapTextTest := MODULE
+
+    import Std, Setup.Ts;
+    EXPORT wordRec := { string word; };
+
+    //External service 1: How many documents does this word appear in?
+    EXPORT doDocumentCount(string search) := FUNCTION
+        cleanedSearch := Std.Str.ToLowerCase(TRIM(search));
+        searchIndex := files.getSearchIndexVariant(variant);
+        matches := searchIndex(KEYED(kind = Ts.KindType.TextEntry AND word = cleanedSearch)) : onwarning(4523, ignore);
+        bydocs := TABLE(matches, { cnt := COUNT(GROUP), doc }, doc);
+        numDocs := COUNT(byDocs);
+        RETURN numDocs;
+    END;
+
+    EXPORT countServiceRequest := RECORD
+        STRING search;
+    END;
+    EXPORT countServiceResponse := RECORD
+        UNSIGNED cnt;
+    END;
+
+    EXPORT documentCountService() := FUNCTION
+        string searchWord := '' : stored('search');
+        RETURN OUTPUT(ROW(TRANSFORM(countServiceResponse, SELF.cnt := doDocumentCount(searchWord))));
+    END;
+
+    //External searvice2: Given a list of words, find the documents they all occur in, and return the number of counts of each word
+    EXPORT joinServiceResponseRecord := RECORD
+        Ts.DocumentId doc;
+        STRING word;
+        UNSIGNED cnt;
+    END;
+
+    EXPORT doSearchWords(DATASET(wordRec) searchWords) := FUNCTION
+
+        searchIndex := files.getSearchIndexVariant(variant);
+        outputRecord := RECORDOF(searchIndex);
+
+        doAction(SET OF DATASET(outputRecord) prev, UNSIGNED step) := FUNCTION
+            searchWord := searchWords[step].word;
+            cleanedSearch := Std.Str.ToLowerCase(TRIM(searchWord));
+            matches := searchIndex(KEYED(kind = Ts.KindType.TextEntry AND word = cleanedSearch)) : onwarning(4523, ignore);
+            doJoin := MERGEJOIN([prev[step-1], matches], STEPPED(LEFT.doc = RIGHT.doc), SORTED(doc));
+            RETURN IF (step =1, matches, doJoin);
+        END;
+
+        nullInput := DATASET([], outputRecord);
+        results := GRAPH(nullInput, count(searchWords), doAction(ROWSET(LEFT), COUNTER), PARALLEL);
+        summary := TABLE(results, {doc, word, cnt := COUNT(GROUP)}, doc, word);
+        RETURN PROJECT(summary, TRANSFORM(joinServiceResponseRecord, SELF.word := TRIM(LEFT.word), SELF := LEFT));
+    END;
+
+    EXPORT searchWordsService() := FUNCTION
+        DATASET(wordRec) searchWords := DATASET([], wordRec) : stored('search');
+        RETURN OUTPUT(doSearchWords(searchWords));
+    END;
+
+    EXPORT doMain(string serviceUrl, string searchWords, unsigned documentLimit) := FUNCTION
+
+        soapcallDocumentCount(string searchWord) := SOAPCALL(serviceUrl, 'soaptest_getdocumentcount', countServiceRequest, transform(countServiceRequest, SELF.search := searchWord), countServiceResponse).cnt;
+        callDocumentCount(string search) := IF((serviceUrl != ''), soapcallDocumentCount(search), doDocumentCount(search));
+
+        soapcallSearchWords(DATASET(wordRec) searchWords) := SOAPCALL(serviceUrl, 'soaptest_getsearchwords', { DATASET(wordRec) search := searchWords }, DATASET(joinServiceResponseRecord));
+        callSearchWords(DATASET(wordRec) searchWords) := IF((serviceUrl != ''), soapcallSearchWords(searchWords), doSearchWords(searchWords));
+
+        splitWords := Std.Str.SplitWords(searchWords, ',', false);
+        splitWordsDs := DATASET(splitwords, wordRec);
+
+        wordsWithDocCounts := TABLE(splitWordsDs, { string word := word; numDocs := callDocumentCount(word); });
+        //
+        leastCommon := TOPN(wordsWithDocCounts, documentLimit, numDocs);
+
+        searchAgain := PROJECT(leastCommon, TRANSFORM(wordRec, SELF.word := LEFT.word));
+        joinLeastCommon := callSearchWords(searchAgain);
+
+        rollupRecord := RECORD
+            Ts.DocumentId doc;
+            SET OF STRING words;
+            SET OF UNSIGNED counts;
+        END;
+
+        rollupRecord rollupWords(joinServiceResponseRecord l, DATASET(joinServiceResponseRecord) matches) := TRANSFORM
+            SELF.doc := l.doc;
+            sortedWords := SORT(matches, -cnt);
+            SELF.words := SET(sortedWords, word);
+            SELF.counts := SET(sortedWords, cnt);
+        END;
+
+        rolledUp := ROLLUP(GROUP(joinLeastCommon, doc), GROUP, rollupWords(LEFT, ROWS(LEFT)));
+        RETURN rolledUp;
+    END;
+
+    EXPORT mainService() := FUNCTION
+        // The published search service take a list of words, and a maximum number of significant documents
+        string searchWords := '' : stored('searchWords');
+        unsigned documentLimit := 3 : stored('documentLimit');
+        serviceUrl := '' : stored('url');
+        unsigned maxResults := 50;
+
+        RETURN OUTPUT(CHOOSEN(doMain(serviceUrl, searchWords, documentLimit), maxResults));
+    END;
+
+END;
diff --git a/testing/regress/ecl/key/soaptext1.xml b/testing/regress/ecl/key/soaptext1.xml
@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><doc>562949953421345</doc><words><Item>king</Item><Item>sheep</Item><Item>richard</Item></words><counts><Item>3</Item><Item>2</Item><Item>1</Item></counts></Row>
+ <Row><doc>562949953421587</doc><words><Item>king</Item><Item>sheep</Item><Item>richard</Item></words><counts><Item>3</Item><Item>2</Item><Item>1</Item></counts></Row>
+ <Row><doc>562949953421614</doc><words><Item>king</Item><Item>richard</Item><Item>sheep</Item></words><counts><Item>65</Item><Item>2</Item><Item>2</Item></counts></Row>
+ <Row><doc>562949953421651</doc><words><Item>sheep</Item><Item>richard</Item><Item>king</Item></words><counts><Item>130</Item><Item>4</Item><Item>2</Item></counts></Row>
+ <Row><doc>562949953421681</doc><words><Item>king</Item><Item>richard</Item><Item>sheep</Item></words><counts><Item>2</Item><Item>1</Item><Item>1</Item></counts></Row>
+</Dataset>
diff --git a/testing/regress/ecl/setup/soaptest_getdocumentcount.ecl b/testing/regress/ecl/setup/soaptest_getdocumentcount.ecl
@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+//nohthor
+//publish
+
+import common.SoapTextTest;
+
+SoapTextTest.documentCountService();
diff --git a/testing/regress/ecl/setup/soaptest_getsearchwords.ecl b/testing/regress/ecl/setup/soaptest_getsearchwords.ecl
@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+//nohthor
+//publish
+
+import common.SoapTextTest;
+
+SoapTextTest.searchWordsService();
diff --git a/testing/regress/ecl/setup/soaptest_mainservice.ecl b/testing/regress/ecl/setup/soaptest_mainservice.ecl
@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+//nohthor
+//publish
+
+import common.SoapTextTest;
+
+SoapTextTest.mainService();
diff --git a/testing/regress/ecl/soaptext1.ecl b/testing/regress/ecl/soaptext1.ecl
@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//xversion url='http://.:9876'
+//nothor
+//nohthor
+
+import ^ as root;
+
+serviceUrl := #IFDEFINED(root.url, 'http://.:9876');
+
+//--- end of version configuration ---
+
+import common.SoapTextTest;
+
+#stored ('searchWords', 'one,and,sheep,when,richard,king');
+#stored ('url', serviceUrl);
+
+SoapTextTest.mainService();
diff --git a/testing/regress/ecl/stresstext.ecl b/testing/regress/ecl/stresstext.ecl
@@ -33,6 +33,7 @@
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);
 variant := #IFDEFINED(root.variant, '');
+numJoins := #IFDEFINED(root.numJoins, 40);
 
 //--- end of version configuration ---
 
@@ -71,4 +72,4 @@ createSamples(iters, numRows) := FUNCTIONMACRO
     RETURN o;
 ENDMACRO;
 
-createSamples(40, 60000);
+createSamples(numJoins, 60000);
diff --git a/thorlcr/shared/thwatchdog.hpp b/thorlcr/shared/thwatchdog.hpp
@@ -30,11 +30,13 @@
 struct HeartBeatPacketHeader
 {
 public:
+    // NB: packetSize and progressSize are back-patched and must remain at fixed offsets from the start of this struct
     size32_t packetSize = 0;   // used as validity check must be first
-    SocketEndpoint sender;
-    unsigned tick = 0;         // sequence check
     size32_t progressSize = 0; // size of progress data (following performance data)
 
+    unsigned tick = 0;         // sequence check
+    SocketEndpoint sender;
+
 public:
     void serialize(MemoryBuffer & out) const
     {