From 34513d235ff00e026ce9e69650eadd9650ac64dc Mon Sep 17 00:00:00 2001 From: xwang2713 Date: Fri, 17 Jan 2025 10:12:54 -0500 Subject: [PATCH 1/6] HPCC-23332 Correct ECL CMD shortcut path in Clienttools uninstall --- CMakeLists.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0ce50bb013..d44ee8b5790 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -526,13 +526,11 @@ if(TOP_LEVEL_PROJECT) !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR_WITHOUT_TRADEMARK}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\" !define MUI_FINISHPAGE_NOAUTOCLOSE ") - - list ( APPEND CPACK_NSIS_CREATE_ICONS_EXTRA " + set (CPACK_NSIS_CREATE_ICONS_EXTRA " CreateShortCut \\\"$SMPROGRAMS\\\\$STARTMENU_FOLDER\\\\ECL Command Prompt.lnk\\\" '%comspec%' '/k \\\"$INSTDIR\\\\ECLCmd.bat\\\"' - ") - list ( APPEND CPACK_NSIS_DELETE_ICONS_EXTRA " - Delete \\\"$SMPROGRAMS\\\\$MUI_TEMP\\\\ECL CMD.lnk\\\" - ") + ") + set (CPACK_NSIS_DELETE_ICONS_EXTRA "Delete '$SMPROGRAMS\\\\$START_MENU\\\\ECL Command Prompt.lnk'") + set(CPACK_NSIS_MODIFY_PATH "ON") From 14199b18a5f5cd3cc8e36564266ac6a4310650a1 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 22 Jan 2025 18:16:45 +0000 Subject: [PATCH 2/6] HPCC-33291 Ensure exceptions during file exists check are properly reported When Thor opens a physical file, it scans for parts that exist. If all IFile::exists() calls hit an exception (e.g. due to access errors), Thor was incorrectly reporting that all physical files were not found, instead of the underlying error. Signed-off-by: Jake Smith --- thorlcr/thorutil/thormisc.cpp | 89 +++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 3144eabcdd6..81c245e4b70 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -1068,7 +1068,11 @@ bool getBestFilePart(CActivityBase *activity, IPartDescriptor &partDesc, OwnedIF RemoteFilename rfn; StringBuffer locationName, primaryName; //First check for local matches - for (l=0; l accessException; + IMultiException *multiException = nullptr; + for (l=0; lqueryContainer(), e, "getBestFilePart"); - e->Release(); + if (!accessException) + accessException.setown(e); + else + { + if (!multiException) + { + multiException = makeMultiException(); + multiException->append(*accessException.getClear()); + accessException.setown(multiException); + } + multiException->append(*e); + } } } } - //Now check for a remote match... - for (l=0; l file = createIFile(locationName.str()); - try + rfn.clear(); + partDesc.getFilename(l, rfn); + if (!rfn.isLocal()) { - if (file->exists()) + rfn.getPath(locationName.clear()); + assertex(locationName.length()); + Owned file = createIFile(locationName.str()); + try { - ifile.set(file); - location = l; - if (0 != l) + if (file->exists()) { - Owned e = MakeActivityWarning(activity, 0, "Primary file missing: %s, using remote copy: %s", primaryName.str(), locationName.str()); - if (!eHandler) - throw e.getClear(); - eHandler->fireException(e); + ifile.set(file); + location = l; + if (0 != l) + { + Owned e = MakeActivityWarning(activity, 0, "Primary file missing: %s, using remote copy: %s", primaryName.str(), locationName.str()); + if (!eHandler) + throw e.getClear(); + eHandler->fireException(e); + } + path.append(locationName); + return true; + } + } + catch (IException *e) + { + ActPrintLog(&activity->queryContainer(), e, "In getBestFilePart"); + if (!accessException) + accessException.setown(e); + else + { + if (!multiException) + { + multiException = makeMultiException(); + multiException->append(*accessException.getClear()); + accessException.setown(multiException); + } + multiException->append(*e); } - path.append(locationName); - return true; } - } - catch (IException *e) - { - ActPrintLog(&activity->queryContainer(), e, "In getBestFilePart"); - e->Release(); } } } + if (accessException) + { + if (!eHandler) + throw accessException.getClear(); + + eHandler->fireException(accessException); + } return false; } From dbbf319af2a600e019813af3bbee18d127426dc6 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:17:15 -0500 Subject: [PATCH 3/6] HPCC-33174 ECL Watch v9 ZAP dialog disable log filters this change disables the ZAP dialog's logging filter inputs when the environment is either bare-metal (log filtering isn't supported), or in a containerized env where the logging engine has not been configured Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- .../src-react/components/forms/ZAPDialog.tsx | 145 ++++++++++++++---- esp/src/src/nls/hpcc.ts | 2 + 2 files changed, 116 insertions(+), 31 deletions(-) diff --git a/esp/src/src-react/components/forms/ZAPDialog.tsx b/esp/src/src-react/components/forms/ZAPDialog.tsx index 8d728ae5f8a..a72511c0602 100644 --- a/esp/src/src-react/components/forms/ZAPDialog.tsx +++ b/esp/src/src-react/components/forms/ZAPDialog.tsx @@ -1,10 +1,10 @@ import * as React from "react"; -import { Checkbox, DefaultButton, Dropdown, Icon, IDropdownProps, IOnRenderComboBoxLabelProps, IStackTokens, ITextFieldProps, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField, TooltipHost } from "@fluentui/react"; +import { Checkbox, DefaultButton, Dropdown, Icon, IDropdownProps, IOnRenderComboBoxLabelProps, IStackTokens, ITextFieldProps, Label, mergeStyleSets, PrimaryButton, Spinner, Stack, TextField, TooltipHost } from "@fluentui/react"; import { useForm, Controller } from "react-hook-form"; import { LogType } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; import * as WsWorkunits from "src/WsWorkunits"; -import { useBuildInfo } from "../../hooks/platform"; +import { useBuildInfo, useLogAccessInfo } from "../../hooks/platform"; import { MessageBox } from "../../layouts/MessageBox"; import { CloudContainerNameField } from "../forms/Fields"; import nlsHPCC from "src/nlsHPCC"; @@ -36,7 +36,7 @@ type CustomLabelProps = ITextFieldProps & IDropdownProps & IOnRenderComboBoxLabe const CustomLabel = (props: CustomLabelProps): JSX.Element => { return - + @@ -56,6 +56,8 @@ interface ZAPDialogValues { WhereSlow: string; Password: string; IncludeThorSlaveLog: boolean; + IncludeRelatedLogs: boolean; + IncludePerComponentLogs: boolean; SendEmail: boolean; EmailTo: string; EmailFrom: string; @@ -92,6 +94,8 @@ const defaultValues: ZAPDialogValues = { WhereSlow: "", Password: "", IncludeThorSlaveLog: true, + IncludeRelatedLogs: true, + IncludePerComponentLogs: false, SendEmail: false, EmailTo: "", EmailFrom: "", @@ -155,6 +159,10 @@ export const ZAPDialog: React.FunctionComponent = ({ textOverflow: "ellipsis", outline: 0 }, + disabled: { + background: theme.semanticColors.disabledBackground, + border: `1px solid ${theme.semanticColors.disabledBackground}` + }, "errorMessage": { fontSize: 12, margin: 0, @@ -175,6 +183,20 @@ export const ZAPDialog: React.FunctionComponent = ({ const [, { isContainer }] = useBuildInfo(); + const [logFiltersUnavailable, setLogFiltersUnavailable] = React.useState(false); + const { logsEnabled, logsStatusMessage } = useLogAccessInfo(); + + const logFiltersUnavailableMessage = React.useMemo(() => { + let retVal = ""; + if (!isContainer) { + retVal = nlsHPCC.UnavailableInBareMetal; + } + if (logsStatusMessage !== "") { + retVal = logsStatusMessage; + } + return retVal; + }, [isContainer, logsStatusMessage]); + const closeForm = React.useCallback(() => { setShowForm(false); }, [setShowForm]); @@ -189,22 +211,29 @@ export const ZAPDialog: React.FunctionComponent = ({ setSubmitDisabled(true); setSpinnerHidden(false); - for (const key in data) { - formData.append(key, data[key]); - } - for (const key in logFilter) { - if (key === "AbsoluteTimeRange") { - const startDate = logFilter.AbsoluteTimeRange.StartDate ? new Date(logFilter.AbsoluteTimeRange.StartDate) : null; - let endDate = logFilter.AbsoluteTimeRange.EndDate ? new Date(logFilter.AbsoluteTimeRange.EndDate) : null; - if (startDate) { - endDate = endDate === null ? new Date(startDate.getTime() + (8 * 3600 * 1000)) : endDate; - formData.append("LogFilter_AbsoluteTimeRange_StartDate", startDate.toISOString()); - formData.append("LogFilter_AbsoluteTimeRange_EndDate", endDate.toISOString()); - delete logFilter.RelativeTimeRangeBuffer; + if (logsEnabled) { + for (const key in logFilter) { + if (key === "AbsoluteTimeRange") { + const startDate = logFilter.AbsoluteTimeRange.StartDate ? new Date(logFilter.AbsoluteTimeRange.StartDate) : null; + let endDate = logFilter.AbsoluteTimeRange.EndDate ? new Date(logFilter.AbsoluteTimeRange.EndDate) : null; + if (startDate) { + endDate = endDate === null ? new Date(startDate.getTime() + (8 * 3600 * 1000)) : endDate; + formData.append("LogFilter_AbsoluteTimeRange_StartDate", startDate.toISOString()); + formData.append("LogFilter_AbsoluteTimeRange_EndDate", endDate.toISOString()); + delete logFilter.RelativeTimeRangeBuffer; + } + } else { + formData.append(`LogFilter_${key}`, logFilter[key]); } - } else { - formData.append(`LogFilter_${key}`, logFilter[key]); } + } else { + data.IncludePerComponentLogs = false; + data.IncludeRelatedLogs = false; + data.IncludeThorSlaveLog = false; + } + + for (const key in data) { + formData.append(key, data[key]); } fetch("/WsWorkunits/WUCreateAndDownloadZAPInfo", { @@ -244,7 +273,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }, logger.info )(); - }, [closeForm, handleSubmit, logAccessorMessage]); + }, [closeForm, handleSubmit, logAccessorMessage, logsEnabled]); React.useEffect(() => { WsWorkunits.WUGetZAPInfo({ request: { WUID: wuid } }).then(response => { @@ -261,6 +290,11 @@ export const ZAPDialog: React.FunctionComponent = ({ }).catch(err => logger.error(err)); }, [wuid, reset]); + React.useEffect(() => { + if (!logsEnabled) { setLogFiltersUnavailable(true); } + else { setLogFiltersUnavailable(false); } + }, [logsEnabled]); + return @@ -384,14 +418,34 @@ export const ZAPDialog: React.FunctionComponent = ({ />} />
-
- } - /> -
+ {!isContainer + ?
+ } + /> +
+ :
+
+ } + /> +
+
+ } + /> +
+
+ }
= ({ />} />
- {nlsHPCC.LogFilters} + + {nlsHPCC.LogFilters} + {logFiltersUnavailable && + + + + } + = ({ fieldState: { error } }) =>
- + @@ -465,7 +526,8 @@ export const ZAPDialog: React.FunctionComponent = ({ key={fieldName} type="datetime-local" name={fieldName} - className={formClasses.input} + disabled={logFiltersUnavailable} + className={[formClasses.input, logFiltersUnavailable ? formClasses.disabled : ""].join(" ")} defaultValue={value} onChange={onChange} /> @@ -491,7 +553,7 @@ export const ZAPDialog: React.FunctionComponent = ({ fieldState: { error } }) =>
- + @@ -500,7 +562,8 @@ export const ZAPDialog: React.FunctionComponent = ({ key={fieldName} type="datetime-local" name={fieldName} - className={formClasses.input} + disabled={logFiltersUnavailable} + className={[formClasses.input, logFiltersUnavailable ? formClasses.disabled : ""].join(" ")} defaultValue={value} onChange={onChange} /> @@ -516,8 +579,10 @@ export const ZAPDialog: React.FunctionComponent = ({ name={fieldName} onChange={onChange} label={nlsHPCC.RelativeTimeRange} + disabled={logFiltersUnavailable} onRenderLabel={(props: CustomLabelProps) => } @@ -545,8 +610,10 @@ export const ZAPDialog: React.FunctionComponent = ({ name={fieldName} onChange={onChange} label={nlsHPCC.LogLineLimit} + disabled={logFiltersUnavailable} onRenderLabel={(props: CustomLabelProps) => } @@ -562,8 +629,10 @@ export const ZAPDialog: React.FunctionComponent = ({ name={fieldName} onChange={onChange} label={nlsHPCC.LogLineStartFrom} + disabled={logFiltersUnavailable} onRenderLabel={(props: CustomLabelProps) => } @@ -578,6 +647,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }) => = ({ selectedKey={columnMode} onRenderLabel={(props: CustomLabelProps) => } @@ -606,8 +677,10 @@ export const ZAPDialog: React.FunctionComponent = ({ name={fieldName} onChange={onChange} label={nlsHPCC.CustomLogColumns} + disabled={logFiltersUnavailable} onRenderLabel={(props: CustomLabelProps) => } @@ -624,6 +697,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }) => { const selectedKeys = value ? [...value] : []; const selected = option?.key ?? _value; @@ -640,6 +714,7 @@ export const ZAPDialog: React.FunctionComponent = ({ onRenderLabel={(props: CustomLabelProps) => } @@ -653,6 +728,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }) => = ({ selectedKey={logFormat} onRenderLabel={(props: CustomLabelProps) => } @@ -678,8 +755,10 @@ export const ZAPDialog: React.FunctionComponent = ({ name={fieldName} onChange={onChange} label={nlsHPCC.WildcardFilter} + disabled={logFiltersUnavailable} onRenderLabel={(props: CustomLabelProps) => } @@ -694,6 +773,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }) => = ({ selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } @@ -718,6 +799,7 @@ export const ZAPDialog: React.FunctionComponent = ({ }) => = ({ selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 814b1b547d0..74cf69bc1ba 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -515,6 +515,7 @@ export = { LogFilterStartDateTooltip: "Include log lines from time range start", LogFilterTimeRequired: "Choose either \"From and To Date\" or \"Relative Log Time Buffer\"", LogFilterWildcardFilterTooltip: "A string of text upon which to filter log messages", + LogFiltersUnavailable: "Log Filters are unavailable", LogFormat: "Log Format", LogLineLimit: "Log Line Limit", LogLineStartFrom: "Log Line Start From", @@ -1087,6 +1088,7 @@ export = { Text: "Text", Tree: "Tree", Type: "Type", + UnavailableInBareMetal: "This is unavailable in a non-containerized environment.", UnavailableInContainerized: "This is unavailable in a containerized environment.", Unbound: "unbound", undefined: "undefined", From 551ddb8ce5b0bb1c5e893774bcc9e6f26b3448ec Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 28 Jan 2025 13:36:11 +0000 Subject: [PATCH 4/6] HPCC-33129 Fix sysinfologger interator Signed-off-by: Shamser Ahmed --- dali/base/sysinfologger.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp index 242d303733f..1d744a69bac 100644 --- a/dali/base/sysinfologger.cpp +++ b/dali/base/sysinfologger.cpp @@ -467,7 +467,7 @@ class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOfnext()) + if (!msgIter->next()) return false; return ensureMatch(); } @@ -603,7 +603,6 @@ unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigne // With visibleOnly/hiddenOnly option, use createSysInfoLoggerMsgFilter() if (visibleOnly || hiddenOnly || day) { - unsigned count = 0; Owned msgFilter = createSysInfoLoggerMsgFilter(source); if (hiddenOnly) msgFilter->setHiddenOnly(); From 65eb60c401ffdd7f69d10fa8b88a67e3ac12541d Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 29 Jan 2025 00:07:28 +0000 Subject: [PATCH 5/6] HPCC-32989 Fix potential stall in spilling distributors Distributors spill if their receiving thread starts to receive too much (e.g. if unbalanced). The new lookahead implementation (CCompressedSpillingRowStream) is conditionally used by distributors in this situation [dependent on #option('newlookahead') ]. If a spill occurs, and the consumers last content is from disk (as opposed to from the queue), then it fails to signal the completion correctly. If the provider/writer to CCompressedSpillingRowStream calls flush after this event, it will block waiting for the consumer to signal that is complete, and stall indefinitely. NB: this could happen in any context a distributor is used, e.g. regular hash distribute, but only if there is memory pressure whilst it's actively reading. A smart join that has failed over to a local (hash distributed) smart join is a common case where that can happen. NB2: standard lookahead that also uses CCompressedSpillingRowStream inside CRowStreamLookAhead, is unaffected, due to the way it asynchronously ensures stop() is explicitly called (which signals a blocked flush to release). NB3: although the bug is in CCompressedSpillingRowStream introduced in 9.6.26 (HPCC-32132), in practice this should only affect distributors, and distributors only started to use this new spilling container (via option, and not in by default in BM) since 9.8.16 (HPCC-32486). Signed-off-by: Jake Smith --- thorlcr/thorutil/thbuf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 8edf9dfd942..68e469082db 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -1107,7 +1107,8 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf, { if (nextInputRow == nextOutputRow) { - readState = rs_stopped; + CriticalBlock b(readerWriterCS); + handleInputComplete(); // sets readState to rs_stopped return nullptr; } const void *row = readRowFromStream(); From 70a428ccb4235e7a7917b0feb45111a966418061 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 29 Jan 2025 00:07:28 +0000 Subject: [PATCH 6/6] HPCC-32989 Fix potential stall in spilling distributors Distributors spill if their receiving thread starts to receive too much (e.g. if unbalanced). The new lookahead implementation (CCompressedSpillingRowStream) is conditionally used by distributors in this situation [dependent on #option('newlookahead') ]. If a spill occurs, and the consumers last content is from disk (as opposed to from the queue), then it fails to signal the completion correctly. If the provider/writer to CCompressedSpillingRowStream calls flush after this event, it will block waiting for the consumer to signal that is complete, and stall indefinitely. NB: this could happen in any context a distributor is used, e.g. regular hash distribute, but only if there is memory pressure whilst it's actively reading. A smart join that has failed over to a local (hash distributed) smart join is a common case where that can happen. NB2: standard lookahead that also uses CCompressedSpillingRowStream inside CRowStreamLookAhead, is unaffected, due to the way it asynchronously ensures stop() is explicitly called (which signals a blocked flush to release). NB3: although the bug is in CCompressedSpillingRowStream introduced in 9.6.26 (HPCC-32132), in practice this should only affect distributors, and distributors only started to use this new spilling container (via option, and not in by default in BM) since 9.8.16 (HPCC-32486). Signed-off-by: Jake Smith --- thorlcr/thorutil/thbuf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index b25879ee69e..fb926b27777 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -1107,7 +1107,8 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf, { if (nextInputRow == nextOutputRow) { - readState = rs_stopped; + CriticalBlock b(readerWriterCS); + handleInputComplete(); // sets readState to rs_stopped return nullptr; } const void *row = readRowFromStream();