Merge remote-tracking branch 'origin/candidate-9.2.x' into candidate-…

…9.4.x

Signed-off-by: Gordon Smith <[email protected]>

ecl/eclccserver/eclccserver.cpp

@@ -831,7 +831,9 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         }
         else
         {
-            getHomeFolder(repoRootPath);
+            char dir[_MAX_PATH];
+            if (GetCurrentDirectory(sizeof(dir), dir))
+                repoRootPath.append(dir);
         }
         if (repoRootPath.length())
         {

esp/src/src-react/components/Files.tsx

@@ -31,7 +31,7 @@ const FilterFields: Fields = {
     "LogicalFiles": { type: "checkbox", label: nlsHPCC.LogicalFiles },
     "SuperFiles": { type: "checkbox", label: nlsHPCC.SuperFiles },
     "Indexes": { type: "checkbox", label: nlsHPCC.Indexes },
-    "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles },
+    "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles, disabled: (params: Fields) => !!params?.SuperFiles?.value || !!params?.LogicalFiles?.value },
     "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster },
     "FileSizeFrom": { type: "string", label: nlsHPCC.FromSizes, placeholder: "4096" },
     "FileSizeTo": { type: "string", label: nlsHPCC.ToSizes, placeholder: "16777216" },

esp/src/src-react/components/Workunits.tsx

@@ -150,25 +150,25 @@ export const Workunits: React.FunctionComponent<WorkunitsProps> = ({
                 label: nlsHPCC.TotalClusterTime, width: 120,
                 justify: "right",
             },
-            CompileCost: {
+            "Compile Cost": {
                 label: nlsHPCC.CompileCost, width: 100,
                 justify: "right",
                 formatter: (cost, row) => {
-                    return `${formatCost(cost)}`;
+                    return `${formatCost(row.CompileCost)}`;
                 }
             },
-            ExecuteCost: {
+            "Execute Cost": {
                 label: nlsHPCC.ExecuteCost, width: 100,
                 justify: "right",
                 formatter: (cost, row) => {
-                    return `${formatCost(cost)}`;
+                    return `${formatCost(row.ExecuteCost)}`;
                 }
             },
-            FileAccessCost: {
+            "File Access Cost": {
                 label: nlsHPCC.FileAccessCost, width: 100,
                 justify: "right",
                 formatter: (cost, row) => {
-                    return `${formatCost(cost)}`;
+                    return `${formatCost(row.FileAccessCost)}`;
                 }
             }
         };

esp/src/src-react/components/forms/Fields.tsx

@@ -490,12 +490,18 @@ export const TargetGroupTextField: React.FunctionComponent<TargetGroupTextFieldP
 
     React.useEffect(() => {
         TpGroupQuery({}).then(({ TpGroupQueryResponse }) => {
-            setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(n => {
-                return {
-                    key: n.Name,
-                    text: n.Name + (n.Name !== n.Kind ? ` (${n.Kind})` : "")
-                };
-            }));
+            setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(group => {
+                switch (group?.Kind) {
+                    case "Thor":
+                    case "hthor":
+                    case "Roxie":
+                    case "Plane":
+                        return {
+                            key: group.Name,
+                            text: group.Name + (group.Name !== group.Kind ? ` (${group.Kind})` : "")
+                        };
+                }
+            }).filter(group => group));
         }).catch(err => logger.error(err));
     }, []);
 
@@ -780,6 +786,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a
                     label: field.label,
                     field: <TextField
                         key={fieldID}
+                        id={fieldID}
                         type={field.type}
                         name={fieldID}
                         value={field.value}
@@ -803,6 +810,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a
                     label: field.label,
                     field: <TextField
                         key={fieldID}
+                        id={fieldID}
                         type={field.type}
                         name={fieldID}
                         value={`${field.value}`}
@@ -821,8 +829,9 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a
                     label: field.label,
                     field: <Checkbox
                         key={fieldID}
+                        id={fieldID}
                         name={fieldID}
-                        disabled={field.disabled("") ? true : false}
+                        disabled={field.disabled(fields) ? true : false}
                         checked={field.value === true ? true : false}
                         onChange={(evt, newValue) => onChange(fieldID, newValue)}
                     />

esp/src/src/store/Memory.ts

@@ -69,7 +69,7 @@ export class Memory<T extends BaseRow = BaseRow> extends BaseStore<T, T> {
         }
     }
 
-    protected fetchData(request: QueryRequest<T>, options: QueryOptions<T>): ThenableResponse<T> {
+    protected fetchData(request: QueryRequest<T>, options: QueryOptions<T> = {}): ThenableResponse<T> {
         options.alphanumColumns = this.alphanumSort;
         const data = this.queryEngine(request, options)(this.data);
         data.total = this.data.length;

esp/src/src/store/Store.ts

@@ -21,7 +21,7 @@ export interface QueryOptions<T> {
     start?: number;
     count?: number;
     sort?: QuerySort<T>;
-    alphanumColumns: { [id: string]: boolean };
+    alphanumColumns?: { [id: string]: boolean };
 }
 
 export abstract class BaseStore<R extends BaseRow, T extends BaseRow> {

helm/hpcc/docs/expert.md

@@ -18,6 +18,8 @@ global:
 NB: Some components (e.g. DfuServer and Thor) also have an 'expert' settings area (see values schema) that can be used for relavent settings
 on a per component instance basis, rather than setting them globally.
 
+Planes can also have an expert section (see Plane Expert Settings section)
+
 
 The following options are currently available:
 
@@ -70,3 +72,12 @@ Foreign file reads (~foreign::) are forbidden by default since the official sant
 service via remote file reads with the ~remote:: syntax.
 Setting expert.allowForeign to true, enables foreign access for compatibility with legacy bare-metal environments
 that have their Dali and Dafilesrv's open.
+
+
+# Plane Expert Settings
+
+## validatePlaneScript (list of { string })
+
+Optional list of bash commands to execute within an init container in pods that use this plane.
+This can be used to validate that the plane is healthy, e.g. that it is mounted as expected.
+If the script returns a non-zero result, the init container and therefore the pod will fail.

helm/hpcc/templates/_helpers.tpl

@@ -327,7 +327,7 @@ Add ConfigMap volume for a component
 
 {{/*
 Add volume mounts
-Pass in root, me (the component), includeCategories (optional) and/or includeNames (optional)
+Pass in root, me (the component), includeCategories (optional) and/or includeNames (optional), container identifier (optional)
 Note: if there are multiple planes (other than dll, dali and spill planes), they should be all called with a single call
 to addVolumeMounts so that if a plane can be used for multiple purposes then duplicate volume mounts are not created.
 */}}
@@ -339,6 +339,7 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
 {{- $includeNames := .includeNames | default list -}}
 {{- $component := .me -}}
 {{- $previousMounts := dict -}}
+{{- $id := .id | default "" -}}
 {{- range $plane := $planes -}}
  {{- if not $plane.disabled }}
   {{- $componentMatches := or (not (hasKey $plane "components")) (has $component.name $plane.components) -}}
@@ -349,16 +350,24 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
      {{- $mountPath := $plane.prefix }}
      {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }}
      {{- if le $numMounts 1 }}
-- name: {{ lower $plane.name }}-pv
+- name: {{ lower $plane.name }}-volume
   mountPath: {{ $mountPath | quote }}
      {{- else }}
       {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }}
-- name: {{ lower $plane.name }}-pv-many-{{ $elem }}
+- name: {{ lower $plane.name }}-volume-many-{{ $elem }}
   mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }}
       {{- end }}
      {{- end }}
     {{- end }}
     {{- $_ := set $previousMounts $plane.prefix true -}}
+   {{- else if $plane.hostPath }}
+    {{- if not (hasKey $previousMounts $plane.prefix) }}
+- name: {{ lower $plane.name }}-volume
+  mountPath: {{ $plane.prefix | quote }}
+     {{- if $id }}
+  subPath: {{ printf "%s-%s" $component.name $id }}
+     {{- end }}
+    {{- end }}
    {{- end }}
 
    {{- /*Generate entries for each alias of the plane*/ -}}
@@ -370,11 +379,11 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
       {{- $mountPath := $alias.prefix }}
       {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }}
       {{- if le $numMounts 1 }}
-- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}
+- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}
   mountPath: {{ $mountPath | quote }}
       {{- else }}
        {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }}
-- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ $elem }}
+- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}-many-{{ $elem }}
   mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }}
        {{- end }}
       {{- end }}
@@ -411,18 +420,25 @@ The plane will generate a volume if it matches either an includeLabel or an incl
      {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) }}
      {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }}
      {{- if le $numMounts 1 }}
-- name: {{ lower $plane.name }}-pv
+- name: {{ lower $plane.name }}-volume
   persistentVolumeClaim:
     claimName: {{ $pvc }}
      {{- else }}
       {{- range $elem := until $numMounts }}
-- name: {{ lower $plane.name }}-pv-many-{{ add $elem 1 }}
+- name: {{ lower $plane.name }}-volume-many-{{ add $elem 1 }}
   persistentVolumeClaim:
     claimName: {{ $pvc }}-{{ add $elem 1 }}
       {{- end }}
      {{- end }}
      {{- $_ := set $previousMounts $plane.prefix true }}
     {{- end }}
+   {{- else if $plane.hostPath }}
+    {{- if not (hasKey $previousMounts $plane.prefix) }}
+- name: {{ lower $plane.name }}-volume
+  hostPath:
+    path: {{ $plane.hostPath }}
+    type: Directory
+    {{- end }}
    {{- end }}
 
    {{- /*Generate entries for each alias of the plane*/ -}}
@@ -434,12 +450,12 @@ The plane will generate a volume if it matches either an includeLabel or an incl
       {{- $pvc := $alias.pvc }}
       {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }}
       {{- if le $numMounts 1 }}
-- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}
+- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}
   persistentVolumeClaim:
     claimName: {{ $pvc }}
       {{- else }}
        {{- range $elem := until $numMounts }}
-- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ add $elem 1 }}
+- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}-many-{{ add $elem 1 }}
   persistentVolumeClaim:
     claimName: {{ $pvc }}-{{ add $elem 1 }}
        {{- end }}
@@ -814,9 +830,9 @@ Specifically for now (but could be extended), this container generates sysctl co
 A kludge to ensure until the mount of a PVC appears (this can happen with some types of host storage)
 */}}
 {{- define "hpcc.waitForMount" -}}
-- name: wait-mount-container
+- name: {{ printf "wait-mount-container-%s" .volumeName }}
   {{- include "hpcc.addImageAttrs" . | nindent 2 }}
-  command: ["/bin/sh"]
+  command: ["/bin/bash"]
   args:
   - "-c"
   - {{ printf "until mountpoint -q %s; do sleep 5; done" .volumePath }}
@@ -825,6 +841,25 @@ A kludge to ensure until the mount of a PVC appears (this can happen with some t
       mountPath: {{ .volumePath | quote }}
 {{- end }}
 
+{{/*
+Inject container to perform any post plane initialization validation
+Pass in dict with volumeName, volumePath and cmds
+*/}}
+{{- define "hpcc.validatePlaneScript" -}}
+- name: {{ printf "validate-plane-script-container-%s" .volumeName }}
+  {{- include "hpcc.addImageAttrs" . | nindent 2 }}
+  command: ["/bin/bash"]
+  args:
+  - -c
+  - |
+{{- range $cmd := .cmds }}
+    {{ $cmd }}
+{{- end }}
+  volumeMounts:
+    - name: {{ .volumeName | quote}}
+      mountPath: {{ .volumePath | quote }}
+{{- end }}
+
 
 {{/*
 A kludge to ensure mounted storage (e.g. for nfs, minikube or docker for desktop) has correct permissions for PV
@@ -878,24 +913,30 @@ NB: uid=10000 and gid=10001 are the uid/gid of the hpcc user, built into platfor
 {{- $component := .me -}}
 {{- range $plane := $planes -}}
  {{- if not $plane.disabled -}}
-  {{- if (or ($plane.pvc) (hasKey $plane "storageClass")) -}}
+  {{- if (or ($plane.pvc) (or (hasKey $plane "storageClass") (hasKey $plane "hostPath"))) -}}
    {{- $componentMatches := or (not (hasKey $plane "components")) (has $component.name $plane.components) -}}
    {{- if and (or (has $plane.category $includeCategories) (has $plane.name $includeNames)) $componentMatches }}
     {{- if $plane.forcePermissions -}}
      {{- $planesToChown = append $planesToChown $plane -}}
     {{- end -}}
     {{- if $plane.waitForMount -}}
-     {{- $volumeName := (printf "%s-pv" $plane.name) -}}
+     {{- $volumeName := (printf "%s-volume" $plane.name) -}}
      {{- include "hpcc.waitForMount" (dict "root" $root "me" $component "uid" $uid "gid" $gid "volumeName" $volumeName "volumePath" $plane.prefix) | nindent 0 }}
     {{- end -}}
+    {{- if hasKey $plane "expert" -}}
+     {{- if $plane.expert.validatePlaneScript -}}
+      {{- $volumeName := (printf "%s-volume" $plane.name) -}}
+      {{- include "hpcc.validatePlaneScript" (dict "root" $root "me" $component "uid" $uid "gid" $gid "volumeName" $volumeName "volumePath" $plane.prefix "cmds" $plane.expert.validatePlaneScript) | nindent 0 }}
+     {{- end -}}
+    {{- end -}}
    {{- end -}}
   {{- end -}}
  {{- end -}}
 {{- end -}}
 {{- $volumes := list -}}
 {{- if len $planesToChown -}}
  {{- range $plane := $planesToChown -}}
-  {{- $volumeName := (printf "%s-pv" $plane.name) -}}
+  {{- $volumeName := (printf "%s-volume" $plane.name) -}}
   {{- $volumes = append $volumes (dict "name" $volumeName "path" $plane.prefix) -}}
  {{- end -}}
  {{- include "hpcc.changeMountPerms" (dict "root" $root "uid" $uid "gid" $gid "volumes" $volumes) | nindent 0 }}
@@ -1636,12 +1677,14 @@ args:
 {{- end }}
 - >-
     {{ $check_cmd.command }};
+    exitCode=$?;
     k8s_postjob_clearup.sh;
-{{- if $misc.postJobCommandViaSidecar -}} ;
-    touch /wait-and-run/{{ .me.name }}.jobdone
-{{- else if $postJobCommand -}} ;
-    {{ $postJobCommand }}
+{{- if $misc.postJobCommandViaSidecar -}}
+    touch /wait-and-run/{{ .me.name }}.jobdone;
+{{- else if $postJobCommand -}}
+    {{ $postJobCommand }} ;
 {{- end }}
+    exit $exitCode;
 {{- end -}}
 
 {{- define "hpcc.addCertificateImpl" }}

helm/hpcc/templates/thor.yaml

@@ -251,7 +251,7 @@ data:
             workingDir: /var/lib/HPCCSystems
             volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" $configCtx.me | indent 12 }}
-{{ include "hpcc.addVolumeMounts" $configCtx | indent 12 }}
+{{ include "hpcc.addVolumeMounts" (deepCopy $configCtx | merge (dict "id" (toString $containerNum))) | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" $configCtx | indent 12 }}
 {{ include "hpcc.addCertificateVolumeMount" (dict "root" $configCtx.root "name" $configCtx.me.name "component" "thorworker" "includeRemote" true) | indent 12 }}
 {{- if and ($misc.postJobCommandViaSidecar) (eq $containerNum 1) }}

helm/hpcc/values.schema.json

@@ -544,6 +544,17 @@
             "waitForMount": {
               "type": "boolean"
             },
+            "expert": {
+              "type": "object",
+              "description": "Custom internal options usually reserved for internal testing",
+              "properties": {
+                "validatePlaneScript": {
+                  "description": "a list of bash commands to run to validate the plane is healthy",
+                  "type": "array",
+                  "items": { "type": "string" }
+                }
+              }
+            },
             "blockedFileIOKB": {
               "description": "Optimal block size for efficient reading from this plane. Implementations will use if they can",
               "type": "integer",
@@ -557,6 +568,7 @@
             "subPath": {},
             "secret": {},
             "pvc": {},
+            "hostPath": {},
             "hostGroup": {},
             "hosts": {},
             "umask": {},
@@ -584,6 +596,10 @@
           "description": "optional name of any secret required to access this storage plane",
           "type": "string"
         },
+        "hostPath": {
+          "description": "optional, this will create a hostPath volume (mutually exclusive with using pvc)",
+          "type": "string"
+        },
         "pvc": {
           "description": "optional name of the persistent volume claim for this plane",
           "type": "string"
@@ -689,6 +705,7 @@
             "subPath": {},
             "secret": {},
             "pvc": {},
+            "hostPath": {},
             "hostGroup": {},
             "hosts": {},
             "umask": {},