diff --git a/ecl/eclccserver/eclccserver.cpp b/ecl/eclccserver/eclccserver.cpp index 7426ee8c197..858eb2dc67d 100644 --- a/ecl/eclccserver/eclccserver.cpp +++ b/ecl/eclccserver/eclccserver.cpp @@ -831,7 +831,9 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter, } else { - getHomeFolder(repoRootPath); + char dir[_MAX_PATH]; + if (GetCurrentDirectory(sizeof(dir), dir)) + repoRootPath.append(dir); } if (repoRootPath.length()) { diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 03f60dff097..b2664d6b119 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -31,7 +31,7 @@ const FilterFields: Fields = { "LogicalFiles": { type: "checkbox", label: nlsHPCC.LogicalFiles }, "SuperFiles": { type: "checkbox", label: nlsHPCC.SuperFiles }, "Indexes": { type: "checkbox", label: nlsHPCC.Indexes }, - "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles }, + "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles, disabled: (params: Fields) => !!params?.SuperFiles?.value || !!params?.LogicalFiles?.value }, "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster }, "FileSizeFrom": { type: "string", label: nlsHPCC.FromSizes, placeholder: "4096" }, "FileSizeTo": { type: "string", label: nlsHPCC.ToSizes, placeholder: "16777216" }, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 0f1e9314ed6..f2bfc4d4d84 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -150,25 +150,25 @@ export const Workunits: React.FunctionComponent = ({ label: nlsHPCC.TotalClusterTime, width: 120, justify: "right", }, - CompileCost: { + "Compile Cost": { label: nlsHPCC.CompileCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.CompileCost)}`; } }, - ExecuteCost: { + "Execute Cost": { label: nlsHPCC.ExecuteCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.ExecuteCost)}`; } }, - FileAccessCost: { + "File Access Cost": { label: nlsHPCC.FileAccessCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.FileAccessCost)}`; } } }; diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index 3d191882b36..9f394b0aacc 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -490,12 +490,18 @@ export const TargetGroupTextField: React.FunctionComponent { TpGroupQuery({}).then(({ TpGroupQueryResponse }) => { - setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(n => { - return { - key: n.Name, - text: n.Name + (n.Name !== n.Kind ? ` (${n.Kind})` : "") - }; - })); + setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(group => { + switch (group?.Kind) { + case "Thor": + case "hthor": + case "Roxie": + case "Plane": + return { + key: group.Name, + text: group.Name + (group.Name !== group.Kind ? ` (${group.Kind})` : "") + }; + } + }).filter(group => group)); }).catch(err => logger.error(err)); }, []); @@ -780,6 +786,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a label: field.label, field: onChange(fieldID, newValue)} /> diff --git a/esp/src/src/store/Memory.ts b/esp/src/src/store/Memory.ts index ec36b30414c..1bec0fee3e6 100644 --- a/esp/src/src/store/Memory.ts +++ b/esp/src/src/store/Memory.ts @@ -69,7 +69,7 @@ export class Memory extends BaseStore { } } - protected fetchData(request: QueryRequest, options: QueryOptions): ThenableResponse { + protected fetchData(request: QueryRequest, options: QueryOptions = {}): ThenableResponse { options.alphanumColumns = this.alphanumSort; const data = this.queryEngine(request, options)(this.data); data.total = this.data.length; diff --git a/esp/src/src/store/Store.ts b/esp/src/src/store/Store.ts index 41dea790b39..d41d24e893c 100644 --- a/esp/src/src/store/Store.ts +++ b/esp/src/src/store/Store.ts @@ -21,7 +21,7 @@ export interface QueryOptions { start?: number; count?: number; sort?: QuerySort; - alphanumColumns: { [id: string]: boolean }; + alphanumColumns?: { [id: string]: boolean }; } export abstract class BaseStore { diff --git a/helm/hpcc/docs/expert.md b/helm/hpcc/docs/expert.md index 506e9404c41..a42fc346ba5 100644 --- a/helm/hpcc/docs/expert.md +++ b/helm/hpcc/docs/expert.md @@ -18,6 +18,8 @@ global: NB: Some components (e.g. DfuServer and Thor) also have an 'expert' settings area (see values schema) that can be used for relavent settings on a per component instance basis, rather than setting them globally. +Planes can also have an expert section (see Plane Expert Settings section) + The following options are currently available: @@ -70,3 +72,12 @@ Foreign file reads (~foreign::) are forbidden by default since the official sant service via remote file reads with the ~remote:: syntax. Setting expert.allowForeign to true, enables foreign access for compatibility with legacy bare-metal environments that have their Dali and Dafilesrv's open. + + +# Plane Expert Settings + +## validatePlaneScript (list of { string }) + +Optional list of bash commands to execute within an init container in pods that use this plane. +This can be used to validate that the plane is healthy, e.g. that it is mounted as expected. +If the script returns a non-zero result, the init container and therefore the pod will fail. diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e2a8fa431cf..511ea27f3f4 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -327,7 +327,7 @@ Add ConfigMap volume for a component {{/* Add volume mounts -Pass in root, me (the component), includeCategories (optional) and/or includeNames (optional) +Pass in root, me (the component), includeCategories (optional) and/or includeNames (optional), container identifier (optional) Note: if there are multiple planes (other than dll, dali and spill planes), they should be all called with a single call to addVolumeMounts so that if a plane can be used for multiple purposes then duplicate volume mounts are not created. */}} @@ -339,6 +339,7 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup {{- $includeNames := .includeNames | default list -}} {{- $component := .me -}} {{- $previousMounts := dict -}} +{{- $id := .id | default "" -}} {{- range $plane := $planes -}} {{- if not $plane.disabled }} {{- $componentMatches := or (not (hasKey $plane "components")) (has $component.name $plane.components) -}} @@ -349,16 +350,24 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup {{- $mountPath := $plane.prefix }} {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }} {{- if le $numMounts 1 }} -- name: {{ lower $plane.name }}-pv +- name: {{ lower $plane.name }}-volume mountPath: {{ $mountPath | quote }} {{- else }} {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }} -- name: {{ lower $plane.name }}-pv-many-{{ $elem }} +- name: {{ lower $plane.name }}-volume-many-{{ $elem }} mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }} {{- end }} {{- end }} {{- end }} {{- $_ := set $previousMounts $plane.prefix true -}} + {{- else if $plane.hostPath }} + {{- if not (hasKey $previousMounts $plane.prefix) }} +- name: {{ lower $plane.name }}-volume + mountPath: {{ $plane.prefix | quote }} + {{- if $id }} + subPath: {{ printf "%s-%s" $component.name $id }} + {{- end }} + {{- end }} {{- end }} {{- /*Generate entries for each alias of the plane*/ -}} @@ -370,11 +379,11 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup {{- $mountPath := $alias.prefix }} {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }} {{- if le $numMounts 1 }} -- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }} +- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }} mountPath: {{ $mountPath | quote }} {{- else }} {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }} -- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ $elem }} +- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}-many-{{ $elem }} mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }} {{- end }} {{- end }} @@ -411,18 +420,25 @@ The plane will generate a volume if it matches either an includeLabel or an incl {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) }} {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }} {{- if le $numMounts 1 }} -- name: {{ lower $plane.name }}-pv +- name: {{ lower $plane.name }}-volume persistentVolumeClaim: claimName: {{ $pvc }} {{- else }} {{- range $elem := until $numMounts }} -- name: {{ lower $plane.name }}-pv-many-{{ add $elem 1 }} +- name: {{ lower $plane.name }}-volume-many-{{ add $elem 1 }} persistentVolumeClaim: claimName: {{ $pvc }}-{{ add $elem 1 }} {{- end }} {{- end }} {{- $_ := set $previousMounts $plane.prefix true }} {{- end }} + {{- else if $plane.hostPath }} + {{- if not (hasKey $previousMounts $plane.prefix) }} +- name: {{ lower $plane.name }}-volume + hostPath: + path: {{ $plane.hostPath }} + type: Directory + {{- end }} {{- end }} {{- /*Generate entries for each alias of the plane*/ -}} @@ -434,12 +450,12 @@ The plane will generate a volume if it matches either an includeLabel or an incl {{- $pvc := $alias.pvc }} {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }} {{- if le $numMounts 1 }} -- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }} +- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }} persistentVolumeClaim: claimName: {{ $pvc }} {{- else }} {{- range $elem := until $numMounts }} -- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ add $elem 1 }} +- name: {{ lower $plane.name }}-volume-alias-{{ $curAlias.num }}-many-{{ add $elem 1 }} persistentVolumeClaim: claimName: {{ $pvc }}-{{ add $elem 1 }} {{- end }} @@ -814,9 +830,9 @@ Specifically for now (but could be extended), this container generates sysctl co A kludge to ensure until the mount of a PVC appears (this can happen with some types of host storage) */}} {{- define "hpcc.waitForMount" -}} -- name: wait-mount-container +- name: {{ printf "wait-mount-container-%s" .volumeName }} {{- include "hpcc.addImageAttrs" . | nindent 2 }} - command: ["/bin/sh"] + command: ["/bin/bash"] args: - "-c" - {{ printf "until mountpoint -q %s; do sleep 5; done" .volumePath }} @@ -825,6 +841,25 @@ A kludge to ensure until the mount of a PVC appears (this can happen with some t mountPath: {{ .volumePath | quote }} {{- end }} +{{/* +Inject container to perform any post plane initialization validation +Pass in dict with volumeName, volumePath and cmds +*/}} +{{- define "hpcc.validatePlaneScript" -}} +- name: {{ printf "validate-plane-script-container-%s" .volumeName }} + {{- include "hpcc.addImageAttrs" . | nindent 2 }} + command: ["/bin/bash"] + args: + - -c + - | +{{- range $cmd := .cmds }} + {{ $cmd }} +{{- end }} + volumeMounts: + - name: {{ .volumeName | quote}} + mountPath: {{ .volumePath | quote }} +{{- end }} + {{/* A kludge to ensure mounted storage (e.g. for nfs, minikube or docker for desktop) has correct permissions for PV @@ -878,16 +913,22 @@ NB: uid=10000 and gid=10001 are the uid/gid of the hpcc user, built into platfor {{- $component := .me -}} {{- range $plane := $planes -}} {{- if not $plane.disabled -}} - {{- if (or ($plane.pvc) (hasKey $plane "storageClass")) -}} + {{- if (or ($plane.pvc) (or (hasKey $plane "storageClass") (hasKey $plane "hostPath"))) -}} {{- $componentMatches := or (not (hasKey $plane "components")) (has $component.name $plane.components) -}} {{- if and (or (has $plane.category $includeCategories) (has $plane.name $includeNames)) $componentMatches }} {{- if $plane.forcePermissions -}} {{- $planesToChown = append $planesToChown $plane -}} {{- end -}} {{- if $plane.waitForMount -}} - {{- $volumeName := (printf "%s-pv" $plane.name) -}} + {{- $volumeName := (printf "%s-volume" $plane.name) -}} {{- include "hpcc.waitForMount" (dict "root" $root "me" $component "uid" $uid "gid" $gid "volumeName" $volumeName "volumePath" $plane.prefix) | nindent 0 }} {{- end -}} + {{- if hasKey $plane "expert" -}} + {{- if $plane.expert.validatePlaneScript -}} + {{- $volumeName := (printf "%s-volume" $plane.name) -}} + {{- include "hpcc.validatePlaneScript" (dict "root" $root "me" $component "uid" $uid "gid" $gid "volumeName" $volumeName "volumePath" $plane.prefix "cmds" $plane.expert.validatePlaneScript) | nindent 0 }} + {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -895,7 +936,7 @@ NB: uid=10000 and gid=10001 are the uid/gid of the hpcc user, built into platfor {{- $volumes := list -}} {{- if len $planesToChown -}} {{- range $plane := $planesToChown -}} - {{- $volumeName := (printf "%s-pv" $plane.name) -}} + {{- $volumeName := (printf "%s-volume" $plane.name) -}} {{- $volumes = append $volumes (dict "name" $volumeName "path" $plane.prefix) -}} {{- end -}} {{- include "hpcc.changeMountPerms" (dict "root" $root "uid" $uid "gid" $gid "volumes" $volumes) | nindent 0 }} @@ -1636,12 +1677,14 @@ args: {{- end }} - >- {{ $check_cmd.command }}; + exitCode=$?; k8s_postjob_clearup.sh; -{{- if $misc.postJobCommandViaSidecar -}} ; - touch /wait-and-run/{{ .me.name }}.jobdone -{{- else if $postJobCommand -}} ; - {{ $postJobCommand }} +{{- if $misc.postJobCommandViaSidecar -}} + touch /wait-and-run/{{ .me.name }}.jobdone; +{{- else if $postJobCommand -}} + {{ $postJobCommand }} ; {{- end }} + exit $exitCode; {{- end -}} {{- define "hpcc.addCertificateImpl" }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 27492e25249..e66feb175a0 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -251,7 +251,7 @@ data: workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" $configCtx.me | indent 12 }} -{{ include "hpcc.addVolumeMounts" $configCtx | indent 12 }} +{{ include "hpcc.addVolumeMounts" (deepCopy $configCtx | merge (dict "id" (toString $containerNum))) | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" $configCtx | indent 12 }} {{ include "hpcc.addCertificateVolumeMount" (dict "root" $configCtx.root "name" $configCtx.me.name "component" "thorworker" "includeRemote" true) | indent 12 }} {{- if and ($misc.postJobCommandViaSidecar) (eq $containerNum 1) }} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 7b507f27451..5525f1916c2 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -544,6 +544,17 @@ "waitForMount": { "type": "boolean" }, + "expert": { + "type": "object", + "description": "Custom internal options usually reserved for internal testing", + "properties": { + "validatePlaneScript": { + "description": "a list of bash commands to run to validate the plane is healthy", + "type": "array", + "items": { "type": "string" } + } + } + }, "blockedFileIOKB": { "description": "Optimal block size for efficient reading from this plane. Implementations will use if they can", "type": "integer", @@ -557,6 +568,7 @@ "subPath": {}, "secret": {}, "pvc": {}, + "hostPath": {}, "hostGroup": {}, "hosts": {}, "umask": {}, @@ -584,6 +596,10 @@ "description": "optional name of any secret required to access this storage plane", "type": "string" }, + "hostPath": { + "description": "optional, this will create a hostPath volume (mutually exclusive with using pvc)", + "type": "string" + }, "pvc": { "description": "optional name of the persistent volume claim for this plane", "type": "string" @@ -689,6 +705,7 @@ "subPath": {}, "secret": {}, "pvc": {}, + "hostPath": {}, "hostGroup": {}, "hosts": {}, "umask": {}, diff --git a/system/jlib/jcontainerized.cpp b/system/jlib/jcontainerized.cpp index c4172e4828a..093e647834b 100644 --- a/system/jlib/jcontainerized.cpp +++ b/system/jlib/jcontainerized.cpp @@ -19,7 +19,6 @@ namespace k8s { -#ifdef _CONTAINERIZED static StringBuffer myPodName; const char *queryMyPodName() @@ -63,13 +62,45 @@ void deleteResource(const char *componentName, const char *resourceType, const c remove(k8sResourcesFilename); } +bool checkExitCodes(StringBuffer &output, const char *podStatuses) +{ + const char *startOfPodStatus = podStatuses; + while (*startOfPodStatus) + { + const char *endOfPodStatus = strchr(startOfPodStatus, '|'); + StringBuffer podStatus; + if (endOfPodStatus) + podStatus.append((size_t)(endOfPodStatus-startOfPodStatus), startOfPodStatus); + else + podStatus.append(startOfPodStatus); + StringArray fields; + fields.appendList(podStatus, ","); + if (3 == fields.length()) // should be 3 fields {,<"initContainer"|"container">,} + { + const char *exitCodeStr = fields.item(0); + if (strlen(exitCodeStr)) + { + unsigned exitCode = atoi(exitCodeStr); + if (exitCode) // non-zero = failure + { + output.appendf(" %s '%s' failed with exitCode = %u", fields.item(1), fields.item(2), exitCode); + return true; + } + } + } + if (!endOfPodStatus) + break; + startOfPodStatus = endOfPodStatus+1; + } + return false; +} + void waitJob(const char *componentName, const char *resourceType, const char *job, unsigned pendingTimeoutSecs, KeepJobs keepJob) { VStringBuffer jobName("%s-%s-%s", componentName, resourceType, job); jobName.toLowerCase(); VStringBuffer waitJob("kubectl get jobs %s -o jsonpath={.status.active}", jobName.str()); VStringBuffer getScheduleStatus("kubectl get pods --selector=job-name=%s --output=jsonpath={.items[*].status.conditions[?(@.type=='PodScheduled')].status}", jobName.str()); - VStringBuffer checkJobExitCode("kubectl get pods --selector=job-name=%s --output=jsonpath={.items[*].status.containerStatuses[?(@.name==\"%s\")].state.terminated.exitCode}", jobName.str(), jobName.str()); unsigned delay = 100; unsigned start = msTick(); @@ -82,14 +113,30 @@ void waitJob(const char *componentName, const char *resourceType, const char *jo { StringBuffer output; runKubectlCommand(componentName, waitJob, nullptr, &output); - if (!streq(output, "1")) // status.active value + if ((0 == output.length()) || streq(output, "0")) // status.active value { // Job is no longer active - we can terminate DBGLOG("kubectl jobs output: %s", output.str()); - runKubectlCommand(componentName, checkJobExitCode, nullptr, &output.clear()); - if (output.length() && !streq(output, "0")) // state.terminated.exitCode - throw makeStringExceptionV(0, "Failed to run %s: pod exited with error: %s", jobName.str(), output.str()); - break; + VStringBuffer checkJobExitStatus("kubectl get jobs %s '-o=jsonpath={range .status.conditions[*]}{.type}: {.status} - {.message}|{end}'", jobName.str()); + runKubectlCommand(componentName, checkJobExitStatus, nullptr, &output.clear()); + if (strstr(output.str(), "Failed: ")) + { + VStringBuffer errMsg("Job %s failed [%s].", jobName.str(), output.str()); + VStringBuffer checkInitContainerExitCodes("kubectl get pods --selector=job-name=%s '-o=jsonpath={range .items[*].status.initContainerStatuses[*]}{.state.terminated.exitCode},{\"initContainer\"},{.name}{\"|\"}{end}'", jobName.str()); + runKubectlCommand(componentName, checkInitContainerExitCodes, nullptr, &output.clear()); + DBGLOG("checkInitContainerExitCodes - output = %s", output.str()); + if (!checkExitCodes(errMsg, output)) + { + // no init container failures, check regular containers + VStringBuffer checkContainerExitCodes("kubectl get pods --selector=job-name=%s '-o=jsonpath={range .items[*].status.containerStatuses[*]}{.state.terminated.exitCode},{\"container\"},{.name}{\"|\"}{end}'", jobName.str()); + runKubectlCommand(componentName, checkContainerExitCodes, nullptr, &output.clear()); + DBGLOG("checkContainerExitCodes - output = %s", output.str()); + checkExitCodes(errMsg, output); + } + throw makeStringException(0, errMsg); + } + else // assume success, either .status.conditions type of "Complete" or "Succeeded" + break; } runKubectlCommand(nullptr, getScheduleStatus, nullptr, &output.clear()); @@ -261,51 +308,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) } MODULE_EXIT() { - removeConfigUpdateHook(podInfoInitCBId); -} - -#else - -const char *queryMyPodName() -{ - throwUnexpected(); -} - -KeepJobs translateKeepJobs(const char *keepJobs) -{ - throwUnexpected(); -} - -bool isActiveService(const char *serviceName) -{ - throwUnexpected(); -} - -void deleteResource(const char *componentName, const char *job, const char *resource) -{ - throwUnexpected(); -} - -void waitJob(const char *componentName, const char *resourceType, const char *job, unsigned pendingTimeoutSecs, KeepJobs keepJob) -{ - throwUnexpected(); -} - -bool applyYaml(const char *componentName, const char *wuid, const char *job, const char *resourceType, const std::list> &extraParams, bool optional, bool autoCleanup) -{ - throwUnexpected(); -} - -void runJob(const char *componentName, const char *wuid, const char *job, const std::list> &extraParams) -{ - throwUnexpected(); -} - -std::vector> getPodNodes(const char *selector) -{ - throwUnexpected(); + if (isContainerized()) + removeConfigUpdateHook(podInfoInitCBId); } -#endif // _CONTAINERIZED - } // end of k8s namespace