Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-…
Browse files Browse the repository at this point in the history 
…9.8.x

Signed-off-by: Jake Smith <[email protected]>
  • Loading branch information
@jakesmith
jakesmith committed Jul 26, 2024
2 parents d806773 + 6dc3989 commit 1c853a5
Show file tree
Hide file tree
Showing 46 changed files with 2,277 additions and 1,251 deletions.
11 changes: 8 additions & 3 deletions common/thorhelper/thorsoapcall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ class BlackLister : public CInterface, implements IThreadFactory
{
if (e->errorCode() == ROXIE_ABORT_EVENT)
throw;
// MCK - do we checkTimeLimitExceeded(&remainingMS) and possibly error out if timelimit exceeded ?
// TODO: do we checkTimeLimitExceeded(&remainingMS) and possibly error out if timelimit exceeded ?
if (numAttemptsRemaining > 0)
{
e->Release();
Expand Down Expand Up @@ -2457,7 +2457,10 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
{
checkTimeLimitExceeded(&remainingMS);
Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0);
ep.set(connUrl.host.get(), connUrl.port);

// TODO: for DNS, do we use timeoutMS or remainingMS or remainingMS / maxRetries+1 or ?
ep.set(connUrl.host.get(), connUrl.port, master->timeoutMS);

if (ep.isNull())
throw MakeStringException(-1, "Failed to resolve host '%s'", nullText(connUrl.host.get()));

Expand All @@ -2477,7 +2480,10 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
{
isReused = false;
keepAlive = true;

// TODO: for each connect attempt, do we use timeoutMS or remainingMS or remainingMS / maxRetries or ?
socket.setown(blacklist->connect(ep, master->logctx, (unsigned)master->maxRetries, master->timeoutMS, master->roxieAbortMonitor, master->rowProvider));

if (proto == PersistentProtocol::ProtoTLS)
{
#ifdef _USE_OPENSSL
Expand All @@ -2503,7 +2509,6 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
err.append(": OpenSSL disabled in build");
throw makeStringException(0, err.str());
#endif

}
}
break;
Expand Down
10 changes: 8 additions & 2 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1494,7 +1494,14 @@ class GraphScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
{
if (whichProperties & PTattributes)
{
playAttribute(visitor, WaLabel);
Owned<IAttributeIterator> nodeAttrs = cur.getAttributes();
ForEach(*nodeAttrs)
{
const char * name = nodeAttrs->queryName();
WuAttr attr = queryGraphAttrToWuAttr(name+1);
if (attr != WaNone)
visitor.noteAttribute(attr, nodeAttrs->queryValue());
}
Owned<IPropertyTreeIterator> attrs = cur.getElements("att");
ForEach(*attrs)
{
Expand All @@ -1515,7 +1522,6 @@ class GraphScopeIterator : public CInterfaceOf<IConstWUScopeIterator>
}
if (whichProperties & PTstatistics)
{
playAttribute(visitor, WaLabel);
Owned<IPropertyTreeIterator> attrs = cur.getElements("att");
ForEach(*attrs)
{
Expand Down
1 change: 1 addition & 0 deletions common/workunit/wuattr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ const static WuAttrInfo attrInfo[] = {
CHILD(RecordFormat, SMeasureText, "recordFormat"),
CHILD(ServiceName, SMeasureText, "serviceName"),
CHILD(SizeComplexHelper, SMeasureSize, "complexHelperSize"),
CHILD(IsBalanced, SMeasureBool, "balanced"),
{ WaMax, SMeasureNone, nullptr, nullptr, nullptr, nullptr, nullptr, WaNone, WaNone }
};

Expand Down
1 change: 1 addition & 0 deletions common/workunit/wuattr.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ enum WuAttr : unsigned
WaRecordFormat,
WaServiceName,
WaSizeComplexHelper,
WaIsBalanced,
WaMax
};
inline WuAttr & operator++(WuAttr & x) { assert(x<WaMax); x = (WuAttr)(x+1); return x; }
Expand Down
2 changes: 1 addition & 1 deletion dockerfiles/platform-build/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ RUN groupadd -g 10001 hpcc
RUN useradd -s /bin/bash -r -m -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc
RUN passwd -l hpcc

RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.10/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin

WORKDIR /hpcc-dev
RUN mkdir build
Expand Down
7 changes: 5 additions & 2 deletions dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ RUN apt-get clean -y && \
expect \
g++ \
git \
git-lfs \
locales \
jq \
openssh-client \
Expand All @@ -59,10 +58,14 @@ RUN apt-get install -y \
gdb \
nano

RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.10/bin/linux/amd64/kubectl && \
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/bin/linux/amd64/kubectl && \
chmod +x ./kubectl && \
mv ./kubectl /usr/local/bin

RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.5.1_amd64.deb/download && \
dpkg -i download && \
rm download

# Set the locale
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
Expand Down
4 changes: 2 additions & 2 deletions dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@ RUN apt-get install -y \
gdb \
nano

RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.10/bin/linux/amd64/kubectl && \
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/bin/linux/amd64/kubectl && \
chmod +x ./kubectl && \
mv ./kubectl /usr/local/bin

RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.4.1_amd64.deb/download && \
RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.5.1_amd64.deb/download && \
dpkg -i download && \
rm download

Expand Down
48 changes: 33 additions & 15 deletions ecl/hql/hqlfold.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4610,7 +4610,8 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
{
if (isConstantTransform(transform))
{
DBGLOG("Folder: Replace %s with ROW", getOpString(op));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s with ROW", getOpString(op));
return createDataset(no_datasetfromrow, createRow(no_createrow, LINK(transform)));
}
}
Expand Down Expand Up @@ -4691,7 +4692,8 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
args.append(*newTransform.getClear());
args.append(*LINK(selSeq));
OwnedHqlExpr ret = createDataset(no_hqlproject, args);
DBGLOG("Folder: Replace %s%s with PROJECT", getOpString(op), reason);
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s%s with PROJECT", getOpString(op), reason);
return ret.getClear();
}

Expand Down Expand Up @@ -4720,7 +4722,8 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
args.append(*newTransform.getClear());
args.append(*LINK(newSelSeq));
OwnedHqlExpr ret = createDataset(no_hqlproject, args);
DBGLOG("Folder: Replace JOIN(<empty>, ds) with PROJECT");
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace JOIN(<empty>, ds) with PROJECT");
return ret.getClear();
}
break;
Expand Down Expand Up @@ -4761,19 +4764,22 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
case 1:
if (op == no_cogroup)
{
DBGLOG("Folder: Replace %s with group", getOpString(op));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s with group", getOpString(op));
IHqlExpression * grouping = queryAttributeChild(expr, groupAtom, 0);
IHqlExpression * mappedGrouping = replaceSelector(grouping, queryActiveTableSelector(), lastInput);
OwnedHqlExpr group = createDataset(no_group, LINK(lastInput), mappedGrouping);
return expr->cloneAllAnnotations(group);
}
else
{
DBGLOG("Folder: Replace %s with child", getOpString(op));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s with child", getOpString(op));
return LINK(lastInput);
}
default:
DBGLOG("Folder: Remove %d inputs from %s", expr->numChildren()-args.ordinality(), getOpString(op));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Remove %d inputs from %s", expr->numChildren()-args.ordinality(), getOpString(op));
return expr->clone(args);
}
}
Expand Down Expand Up @@ -5632,9 +5638,12 @@ IHqlExpression * CExprFolderTransformer::doFoldTransformed(IHqlExpression * unfo
if (expandedFilter->isConstant())
{
//Following would be sensible, but can't call transform at this point, so replace arg, and wait for it to re-iterate
IIdAtom * nameF = expr->queryId();
IIdAtom * nameP = child->queryId();
DBGLOG("Folder: Combining FILTER %s with %s %s produces constant filter", nameF ? str(nameF) : "", getOpString(child->getOperator()), nameP ? str(nameP) : "");
if (doTrace(traceOptimizations))
{
IIdAtom * nameF = expr->queryId();
IIdAtom * nameP = child->queryId();
DBGLOG("Folder: Combining FILTER %s with %s %s produces constant filter", nameF ? str(nameF) : "", getOpString(child->getOperator()), nameP ? str(nameP) : "");
}
expandedFilter.setown(transformExpanded(expandedFilter));
IValue * value = expandedFilter->queryValue();
if (value)
Expand Down Expand Up @@ -5685,8 +5694,11 @@ IHqlExpression * CExprFolderTransformer::doFoldTransformed(IHqlExpression * unfo
return replaceWithNull(expr);
if (filtered.ordinality() == values->numChildren())
return removeParentNode(expr);
StringBuffer s1, s2;
DBGLOG("Folder: Node %s reduce values in child: %s from %d to %d", queryChildNodeTraceText(s1, expr), queryChildNodeTraceText(s2, child), values->numChildren(), filtered.ordinality());
if (doTrace(traceOptimizations))
{
StringBuffer s1, s2;
DBGLOG("Folder: Node %s reduce values in child: %s from %d to %d", queryChildNodeTraceText(s1, expr), queryChildNodeTraceText(s2, child), values->numChildren(), filtered.ordinality());
}
HqlExprArray args;
args.append(*values->clone(filtered));
unwindChildren(args, child, 1);
Expand Down Expand Up @@ -6587,7 +6599,10 @@ IHqlExpression * CExprFolderTransformer::createTransformed(IHqlExpression * expr

#ifdef LOG_ALL_FOLDING
if ((op != transformed->getOperator()) || (expr->numChildren() != transformed->numChildren()))
DBGLOG("Folding %s to %s", getOpString(updated->getOperator()), getOpString(transformed->getOperator()));
{
if (doTrace(traceOptimizations))
DBGLOG("Folding %s to %s", getOpString(updated->getOperator()), getOpString(transformed->getOperator()));
}
#endif

#ifdef _DEBUG
Expand Down Expand Up @@ -7183,21 +7198,24 @@ IHqlExpression * CExprFolderTransformer::percolateConstants(IHqlExpression * exp
IHqlExpression * CExprFolderTransformer::removeParentNode(IHqlExpression * expr)
{
IHqlExpression * child = expr->queryChild(0);
DBGLOG("Folder: Node %s remove self (now %s)", queryNode0Text(expr), queryNode1Text(child));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Node %s remove self (now %s)", queryNode0Text(expr), queryNode1Text(child));
return LINK(child);
}

IHqlExpression * CExprFolderTransformer::replaceWithNull(IHqlExpression * expr)
{
IHqlExpression * ret = createNullExpr(expr);
DBGLOG("Folder: Replace %s with %s", queryNode0Text(expr), queryNode1Text(ret));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s with %s", queryNode0Text(expr), queryNode1Text(ret));
return ret;
}

IHqlExpression * CExprFolderTransformer::replaceWithNullRow(IHqlExpression * expr)
{
IHqlExpression * ret = createRow(no_null, LINK(expr->queryRecord()));
DBGLOG("Folder: Replace %s with %s", queryNode0Text(expr), queryNode1Text(ret));
if (doTrace(traceOptimizations))
DBGLOG("Folder: Replace %s with %s", queryNode0Text(expr), queryNode1Text(ret));
return ret;
}

Expand Down
Loading

0 comments on commit 1c853a5

Please sign in to comment.