From ffd9db71d565f453e4a107af390b75eaba778d11 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 13 Jul 2023 19:23:23 +0100 Subject: [PATCH 1/5] HPCC-28924 Avoid spurious workunit wait state Thor uses as a transitional state between itself and the agent. However, if the state changed outside of Thor, e.g. due to failure, Thor would spuriously have set the state to wait. Ensure it is only set if it is in an expected state, otherwise set to fail. Signed-off-by: Jake Smith --- thorlcr/master/thgraphmanager.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index aed713cef0f..49d2537f783 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -860,8 +860,8 @@ void CJobManager::reply(IConstWorkUnit *workunit, const char *wuid, IException * we->setExceptionMessage(errStr); we->setExceptionSource("thormasterexception"); we->setExceptionCode(e->errorCode()); - - w->setState(WUStateWait); + WUState newState = (WUStateRunning == w->getState()) ? WUStateWait : WUStateFailed; + w->setState(newState); } return; } @@ -1342,9 +1342,10 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam jobManager->execute(workunit, currentWuid, currentGraphName, dummyAgentEp); IException *e = jobManager->queryExitException(); Owned w = &workunit->lock(); + WUState newState = (WUStateRunning == w->getState()) ? WUStateWait : WUStateFailed; if (e) { - if (WUStateWait != w->getState()) // if set already, can only mean exception has been set already (see CJobManager::reply) + if (WUStateWait != w->getState()) // if set already, CJobManager::reply may have already set to WUStateWait { Owned we = w->createException(); we->setSeverity(SeverityInformation); @@ -1354,7 +1355,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam we->setExceptionSource("thormasterexception"); we->setExceptionCode(e->errorCode()); - w->setState(WUStateWait); + w->setState(newState); } break; } @@ -1362,7 +1363,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam if (!multiJobLinger && lingerPeriod) w->setDebugValue(instance, "1", true); - w->setState(WUStateWait); + w->setState(newState); } currentGraphName.clear(); From 543b6d1fd488baf69cdf7c7f53c6b0cb41a313e7 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 20 Jul 2023 17:45:59 +0100 Subject: [PATCH 2/5] Split off 8.12.38 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 9360998a034..db2d75e4ca1 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.37-closedown0 +version: 8.12.39-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.37-closedown0 +appVersion: 8.12.39-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 56524440d28..ec51a0868af 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1225,7 +1225,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index bb28665d2ce..50a8c237233 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 592de934fc9..8d6f2d13970 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 89b0ac5dd09..c828460bc17 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 94cf5152947..5be52a59015 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index a802a4faf10..a29c78697c9 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index e0f98b4f467..de2c81a7c16 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 6ed4312344c..b184b2ce11a 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 7ed52833ea5..a5326579dfa 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index fd9100652d2..e6ea180e9ac 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index fefd95b901e..b056a554c7f 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 1e840c3cd21..bd58fd894d7 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.37-closedown0 + helmVersion: 8.12.39-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index a45cae5ab0c..5ada4907ac9 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 37 ) +set ( HPCC_POINT 39 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 61fe9654dcfce059e388f3ba80b43f0cca4e36c8 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 20 Jul 2023 17:47:26 +0100 Subject: [PATCH 3/5] Split off 9.0.28 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index c5080319496..cca749026da 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.27-closedown0 +version: 9.0.29-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.27-closedown0 +appVersion: 9.0.29-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 6b2d13a4569..7ef894332d8 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1299,7 +1299,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index b48b2c24519..deb647fdf74 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 3d3ef2a8488..e429c54e39d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index a5326da4026..feb5ff4f50a 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 340a8bffa88..f8d16e716f8 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index fca16720401..16539dbe516 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index cdae0c352c7..de772cc8321 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index c054c0cc2f2..c50e21f08f5 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 439f0b05b0c..48c6321e639 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 76d7a11d326..bf3dce7bae5 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 1a807d9fd4d..b593d72fc9d 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index a01ee6039f0..5353331f222 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.27-closedown0 + helmVersion: 9.0.29-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 8b66a01e974..ee6dd256f1d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 27 ) +set ( HPCC_POINT 29 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 33b7f615b3d3b66a0a336c0623d60fcd4e9fe605 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Wed, 26 Jul 2023 17:29:38 +0100 Subject: [PATCH 4/5] HPCC-29982 GH Actions running out of disk space Removing Android SDK and related tooling to double the amount of capacity Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 4 ++++ .github/workflows/build-vcpkg.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 362583c2fe9..b852634abd2 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -102,6 +102,10 @@ jobs: fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform uses: actions/checkout@v3 with: diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index f8b610710f4..5aab20df5ed 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -95,6 +95,10 @@ jobs: fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} uses: actions/checkout@v3 From 34c8c89c3a3a90f91ff9faa99ea4b5b2f6e5f40c Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 19 Jul 2023 15:39:09 +0100 Subject: [PATCH 5/5] HPCC-29957 Queries using embedded Java with libraries have unexpected class loaders Split the URLClassLoaders used according to the source of the corresponding jar files, so that we use the same classloader whenever loading classes from a given manifest jar file (and therefore from a given workunit or library). Signed-off-by: Richard Chapman --- common/dllserver/thorplugin.cpp | 16 +++- common/dllserver/thorplugin.hpp | 2 +- plugins/javaembed/HpccClassLoader.java | 109 ++++++++++++++++------- plugins/javaembed/javaembed.cpp | 29 +++++- plugins/py3embed/py3embed.cpp | 13 +-- plugins/pyembed/pyembed.cpp | 13 +-- roxie/ccd/ccd.hpp | 1 + roxie/ccd/ccdcontext.cpp | 8 +- roxie/ccd/ccdmain.cpp | 4 +- roxie/ccd/ccdserver.cpp | 10 +-- testing/regress/ecl/key/libraryjava.xml | 3 + testing/regress/ecl/libraryjava.ecl | 6 ++ testing/regress/ecl/libraryjava.manifest | 3 + 13 files changed, 158 insertions(+), 59 deletions(-) create mode 100644 testing/regress/ecl/libraryjava.manifest diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index df567e246c2..d43792b4743 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -307,7 +307,7 @@ class HelperDll : implements ILoadedDllEntry, public CInterface virtual const byte * getResource(unsigned id) const; virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const; virtual IPropertyTree &queryManifest() const override; - virtual const StringArray &queryManifestFiles(const char *type, const char *wuid) const override; + virtual const StringArray &queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const override; bool load(bool isGlobal, bool raiseOnError); bool loadCurrentExecutable(); bool loadResources(); @@ -492,7 +492,7 @@ IPropertyTree &HelperDll::queryManifest() const return *querySingleton(manifest, manifestLock, [this]{ return getEmbeddedManifestPTree(this); }); } -const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid) const +const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid, const char *tempRoot) const { CriticalBlock b(manifestLock); Linked list = manifestFiles.find(type); @@ -500,8 +500,18 @@ const StringArray &HelperDll::queryManifestFiles(const char *type, const char *w { // The temporary path we unpack to is based on so file's current location and workunit // MORE - this is good for deployed cases, may not be so good for standalone executables. + // Doesn't really work for cloud either - it needs to be in ephemeral there StringBuffer tempDir; - splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr); + if (!isEmptyString(tempRoot)) + { + tempDir.append(tempRoot); + addPathSepChar(tempDir); + splitFilename(name, nullptr, nullptr, &tempDir, nullptr); + } + else + { + splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr); + } list.setown(new ManifestFileList(type, tempDir)); tempDir.append(".tmp").append(PATHSEPCHAR).append(wuid); VStringBuffer xpath("Resource[@type='%s']", type); diff --git a/common/dllserver/thorplugin.hpp b/common/dllserver/thorplugin.hpp index fc1f200c642..717bcbdbd9d 100644 --- a/common/dllserver/thorplugin.hpp +++ b/common/dllserver/thorplugin.hpp @@ -32,7 +32,7 @@ interface ILoadedDllEntry : extends IInterface virtual const byte * getResource(unsigned id) const = 0; virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace=true) const = 0; virtual IPropertyTree &queryManifest() const = 0; - virtual const StringArray &queryManifestFiles(const char *type, const char *tempDir) const = 0; + virtual const StringArray &queryManifestFiles(const char *type, const char *id, const char *tempRoot = nullptr) const = 0; }; extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile, bool resourcesOnly); diff --git a/plugins/javaembed/HpccClassLoader.java b/plugins/javaembed/HpccClassLoader.java index 052e909f077..6578bf0535c 100644 --- a/plugins/javaembed/HpccClassLoader.java +++ b/plugins/javaembed/HpccClassLoader.java @@ -21,47 +21,60 @@ HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems(R). import java.util.Hashtable; import java.util.List; import java.util.ArrayList; +import java.util.Arrays; +import java.util.stream.Collectors; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.lang.Throwable; +import com.HPCCSystems.HpccUtils; + public class HpccClassLoader extends java.lang.ClassLoader { private long bytecode; private int bytecodeLen; + private Boolean trace = false; private native Class defineClassForEmbed(int bytecodeLen, long bytecode, String name); private Hashtable> classes = new Hashtable<>(); - static private Hashtable pathLoaders = new Hashtable<>(); - private java.net.URLClassLoader pathLoader; + static private Hashtable sharedPathLoaders = new Hashtable<>(); + private List libraryPathLoaders = new ArrayList<>(); private HpccClassLoader(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname) { super(parent); if (classPath != null && !classPath.isEmpty()) { - synchronized(pathLoaders) + String[] libraryPaths = classPath.split("\\|"); // Careful - the param is a regex so need to escape the | + synchronized(sharedPathLoaders) { - pathLoader = pathLoaders.get(classPath); - if (pathLoader == null) + for (String libraryPath : libraryPaths) { - List urls = new ArrayList<>(); - String[] paths = classPath.split(";"); - for (String path : paths) + URLClassLoader libraryPathLoader = sharedPathLoaders.get(libraryPath); + if (libraryPathLoader == null) { - try - { - if (path.contains(":")) - urls.add(new URL(path)); - else - urls.add(new URL("file:" + path)); - } - catch (MalformedURLException E) + List urls = new ArrayList<>(); + String[] paths = libraryPath.split(";"); + for (String path : paths) { - // Ignore any that we don't recognize - // System.out.print(E.toString()); + try + { + if (path.contains(":")) + urls.add(new URL(path)); + else + urls.add(new URL("file:" + path)); + } + catch (MalformedURLException E) + { + // Ignore any that we don't recognize + if (trace) + HpccUtils.log("Malformed URL: " + E.toString()); + } } + libraryPathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()])); + if (trace) + HpccUtils.log("Created new URLClassLoader " + libraryPath); + sharedPathLoaders.put(libraryPath, libraryPathLoader); } - pathLoader = new URLClassLoader(urls.toArray(new URL[urls.size()])); - pathLoaders.put(classPath, pathLoader); + libraryPathLoaders.add(libraryPathLoader); } } } @@ -74,25 +87,55 @@ public synchronized Class findClass(String className) throws ClassNotFoundExc { String luName = className.replace(".","/"); Class result = classes.get(luName); - if (result == null) + if (result != null) + return result; + if (trace) + HpccUtils.log("In findClass for " + className); + if (bytecodeLen != 0) + { + result = defineClassForEmbed(bytecodeLen, bytecode, luName); + if (result != null) + return result; + } + for (URLClassLoader libraryLoader: libraryPathLoaders) { - if (bytecodeLen != 0) - result = defineClassForEmbed(bytecodeLen, bytecode, luName); - if ( result == null && pathLoader != null) - result = pathLoader.loadClass(className); - if (result == null) - return super.findClass(className); - classes.put(luName, result); + try + { + if (trace) + HpccUtils.log("Looking in loader " + + Arrays.stream(libraryLoader.getURLs()) + .map(URL::toString) + .collect(Collectors.joining(";"))); + result = libraryLoader.loadClass(className); + if (result != null) + { + classes.put(luName, result); + return result; + } + } + catch (Exception E) + { + } } - return result; + throw new ClassNotFoundException(); } @Override public URL getResource(String path) { - URL ret = pathLoader.getResource(path); - if (ret == null) - ret = super.getResource(path); - return ret; + URL ret = null; + for (URLClassLoader libraryLoader: libraryPathLoaders) + { + try + { + ret = libraryLoader.getResource(path); + if (ret != null) + return ret; + } + catch (Exception E) + { + } + } + return super.getResource(path); } public static HpccClassLoader newInstance(String classPath, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname) { diff --git a/plugins/javaembed/javaembed.cpp b/plugins/javaembed/javaembed.cpp index bb5192a5128..33c519c4778 100644 --- a/plugins/javaembed/javaembed.cpp +++ b/plugins/javaembed/javaembed.cpp @@ -3257,13 +3257,29 @@ class JavaEmbedImportContext : public CInterfaceOf } } StringBuffer lclassPath; + bool needSep = false; if (engine) { StringArray manifestJars; engine->getManifestFiles("jar", manifestJars); ForEachItemIn(idx, manifestJars) { - lclassPath.append(';').append(manifestJars.item(idx)); + const char *thisJar = manifestJars.item(idx); + if (thisJar) + { + if (needSep) + lclassPath.append(';'); + lclassPath.append(manifestJars.item(idx)); + needSep = true; + } + else + { + if (needSep) + { + lclassPath.append('|'); + needSep = false; + } + } } } ForEachItemIn(idx, opts) @@ -3275,7 +3291,12 @@ class JavaEmbedImportContext : public CInterfaceOf StringBuffer optName(val-opt, opt); val++; if (stricmp(optName, "classpath")==0) - lclassPath.append(';').append(val); + { + if (needSep) + lclassPath.append('|'); // The | means we use a common classloader for the bit named by classpath + lclassPath.append(val); + needSep = true; + } else if (strieq(optName, "globalscope")) globalScopeKey.set(val); else if (strieq(optName, "persist")) @@ -3297,8 +3318,8 @@ class JavaEmbedImportContext : public CInterfaceOf throw MakeStringException(0, "javaembed: Unknown option %s", optName.str()); } } - if (lclassPath.length()>1) - classpath.set(lclassPath.str()+1); + if (lclassPath.length()) + classpath.set(lclassPath); if (flags & EFthreadlocal) sharedCtx->registerContext(this); // Do at end - otherwise an exception thrown during construction will leave this reference dangling } diff --git a/plugins/py3embed/py3embed.cpp b/plugins/py3embed/py3embed.cpp index 07b91e3d933..efaf020ce5d 100644 --- a/plugins/py3embed/py3embed.cpp +++ b/plugins/py3embed/py3embed.cpp @@ -738,11 +738,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx) ForEachItemIn(idx, manifestModules) { const char *path = manifestModules.item(idx); - DBGLOG("Manifest zip %s", path); - OwnedPyObject newPath = PyUnicode_FromString(path); - PyList_Insert(sysPath, 0, newPath); - checkPythonError(); - engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true); + if (path) + { + DBGLOG("Manifest zip %s", path); + OwnedPyObject newPath = PyUnicode_FromString(path); + PyList_Insert(sysPath, 0, newPath); + checkPythonError(); + engine->onTermination(Python3xGlobalState::removePath, manifestModules.item(idx), true); + } } } } diff --git a/plugins/pyembed/pyembed.cpp b/plugins/pyembed/pyembed.cpp index 63d36f139e7..e99dcdbcadf 100644 --- a/plugins/pyembed/pyembed.cpp +++ b/plugins/pyembed/pyembed.cpp @@ -641,11 +641,14 @@ void PythonThreadContext::addManifestFiles(ICodeContext *codeCtx) ForEachItemIn(idx, manifestModules) { const char *path = manifestModules.item(idx); - DBGLOG("Manifest zip %s", path); - OwnedPyObject newPath = PyString_FromString(path); - PyList_Insert(sysPath, 0, newPath); - checkPythonError(); - engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true); + if (path) + { + DBGLOG("Manifest zip %s", path); + OwnedPyObject newPath = PyString_FromString(path); + PyList_Insert(sysPath, 0, newPath); + checkPythonError(); + engine->onTermination(Python27GlobalState::removePath, manifestModules.item(idx), true); + } } } } diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 9505636f259..d2849192ff6 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -458,6 +458,7 @@ extern StringBuffer pluginsList; extern StringBuffer queryDirectory; extern StringBuffer codeDirectory; extern StringBuffer tempDirectory; +extern StringBuffer spillDirectory; #undef UNIMPLEMENTED #undef throwUnexpected diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 169bbaead87..fd3f5aef80f 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -3076,15 +3076,19 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon { ILoadedDllEntry *dll = factory->queryDll(); StringBuffer id; - const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str()); + const StringArray &dllFiles = dll->queryManifestFiles(type, getQueryId(id, true).str(), tempDirectory); ForEachItemIn(idx, dllFiles) files.append(dllFiles.item(idx)); ForEachItemIn(lidx, loadedLibraries) { IQueryFactory &lfactory = loadedLibraries.item(lidx); ILoadedDllEntry *ldll = lfactory.queryDll(); + // Libraries share the same copy of the jar (and the classloader) for all queries that use the library StringBuffer lid; - const StringArray &ldllFiles = ldll->queryManifestFiles(type, getQueryId(lid, true).str()); + lid.append('Q').append(lfactory.queryHash()); + const StringArray &ldllFiles = ldll->queryManifestFiles(type, lid.str(), tempDirectory); + if (ldllFiles.length()) + files.append(nullptr); ForEachItemIn(ldidx, ldllFiles) files.append(ldllFiles.item(ldidx)); } diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index f2e996a4031..e558d3936eb 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -203,6 +203,7 @@ StringBuffer logDirectory; StringBuffer pluginDirectory; StringBuffer queryDirectory; StringBuffer codeDirectory; +StringBuffer spillDirectory; StringBuffer tempDirectory; ClientCertificate clientCert; @@ -1253,7 +1254,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) queryDirectory.append(codeDirectory).append("queries"); } addNonEmptyPathSepChar(queryDirectory); - getSpillFilePath(tempDirectory, "roxie", topology); + getSpillFilePath(spillDirectory, "roxie", topology); + getTempFilePath(tempDirectory, "roxie", topology); #ifdef _WIN32 topology->addPropBool("@linuxOS", false); diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index aa9af59d4ca..d4926356ee1 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -8576,7 +8576,7 @@ class CRoxieServerSortActivity : public CRoxieServerActivity if (sortAlgorithm==unknownSortAlgorithm) sorter.clear(); else - sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId)); + sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), spillDirectory, activityId)); } virtual void doStart(unsigned parentExtractSize, const byte *parentExtract, bool paused) @@ -8662,7 +8662,7 @@ class CRoxieServerSortActivity : public CRoxieServerActivity sorter.clear(); OwnedRoxieString algorithmName(helper.getAlgorithm()); sortAlgorithm = useAlgorithm(algorithmName, sortFlags); - sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId)); + sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), spillDirectory, activityId)); } sorter->prepare(inputStream); noteStatistic(StTimeSortElapsed, cycle_to_nanosec(sorter->getElapsedCycles(true))); @@ -12986,12 +12986,12 @@ class CRoxieServerJoinActivity : public CRoxieServerTwoInputActivity if (helper.isLeftAlreadySorted()) sortedLeft.setown(createDegroupedInputReader(inputStream)); else - sortedLeft.setown(createSortedInputReader(inputStream, createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + sortedLeft.setown(createSortedInputReader(inputStream, createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); ICompare *compareRight = helper.queryCompareRight(); if (helper.isRightAlreadySorted()) groupedSortedRight.setown(createGroupedInputReader(inputStream1, compareRight)); else - groupedSortedRight.setown(createSortedGroupedInputReader(inputStream1, compareRight, createSortAlgorithm(sortAlgorithm, compareRight, ctx->queryRowManager(), input1->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + groupedSortedRight.setown(createSortedGroupedInputReader(inputStream1, compareRight, createSortAlgorithm(sortAlgorithm, compareRight, ctx->queryRowManager(), input1->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) { //limited match join (s[1..n]) limitedhelper.setown(createRHLimitedCompareHelper()); @@ -18729,7 +18729,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm; else sortAlgorithm = isStable ? stableQuickSortAlgorithm : quickSortAlgorithm; - groupedInput.setown(createSortedGroupedInputReader(inputStream, compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId))); + groupedInput.setown(createSortedGroupedInputReader(inputStream, compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), spillDirectory, activityId))); } if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) { //limited match join (s[1..n]) diff --git a/testing/regress/ecl/key/libraryjava.xml b/testing/regress/ecl/key/libraryjava.xml index d73a6eee541..a11d459e9ac 100644 --- a/testing/regress/ecl/key/libraryjava.xml +++ b/testing/regress/ecl/key/libraryjava.xml @@ -8,3 +8,6 @@ George Smith Baby Smith + + Hello World + diff --git a/testing/regress/ecl/libraryjava.ecl b/testing/regress/ecl/libraryjava.ecl index ec38d574751..b5c58ad99a9 100644 --- a/testing/regress/ecl/libraryjava.ecl +++ b/testing/regress/ecl/libraryjava.ecl @@ -19,6 +19,10 @@ //nothor //The library is defined and built in aaalibraryjava.ecl +IMPORT java; + +STRING cat(SET OF STRING s) := IMPORT(java, 'javaembed_ex7.cat:([Ljava/lang/String;)Ljava/lang/String;'); + namesRecord := RECORD string20 surname; @@ -50,3 +54,5 @@ integer init := appendDataset(namesTable).initialized : ONCE; output(init); appended := appendDataset(namesTable); output(appended.result); +// Call java from both query and library +output(cat(['Hello','World'])); diff --git a/testing/regress/ecl/libraryjava.manifest b/testing/regress/ecl/libraryjava.manifest new file mode 100644 index 00000000000..ad5b99ab861 --- /dev/null +++ b/testing/regress/ecl/libraryjava.manifest @@ -0,0 +1,3 @@ + + +