HPCC-33436 Fix issues caused by HPCC 33279 changes

Address regression introduce by HPCC 33279 and improve the changes: - Fix incorrect change to update to offset branches (regression) - Use DuplicateKeys instead of DuplicateKeyCount - Reword comment relating to StSizeBranchMemory and StSizeLeafMemory Signed-off-by: Shamser Ahmed <[email protected]>
hpcc-systems · Feb 14, 2025 · 6f0984b · 6f0984b
1 parent 1bf6354
commit 6f0984b
Show file tree

Hide file tree

Showing 8 changed files with 8 additions and 10 deletions.
diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
@@ -1280,7 +1280,7 @@ void CHThorIndexWriteActivity::execute()
             reccount++;
         }
         builder->finish(metadata, &fileCrc, maxRecordSizeSeen);
-        duplicateKeyCount = builder->getStatistic(StNumDuplicateKeyCount);
+        duplicateKeyCount = builder->getStatistic(StNumDuplicateKeys);
         cummulativeDuplicateKeyCount += duplicateKeyCount;
         numLeafNodes = builder->getStatistic(StNumLeafCacheAdds);
         numBranchNodes = builder->getStatistic(StNumNodeCacheAdds);

diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp
@@ -12839,7 +12839,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
                 }
                 reccount++;
             }
-            duplicateKeyCount = builder->getStatistic(StNumDuplicateKeyCount);
+            duplicateKeyCount = builder->getStatistic(StNumDuplicateKeys);
             cummulativeDuplicateKeyCount += duplicateKeyCount;
             builder->finish(metadata, &fileCrc, maxRecordSizeSeen);
             numLeafNodes = builder->getStatistic(StNumLeafCacheAdds);

diff --git a/system/jhtree/keybuild.cpp b/system/jhtree/keybuild.cpp
@@ -602,7 +602,7 @@ class CKeyBuilder : public CInterfaceOf<IKeyBuilder>
     {
         switch (kind)
         {
-        case StNumDuplicateKeyCount:
+        case StNumDuplicateKeys:
             return duplicateCount;
         case StSizeOffsetBranches:
             return offsetBranches;

diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h
@@ -326,7 +326,6 @@ enum StatisticKind
     StNumPeakCacheObjects,              // Peak number of objects in a generic cache
     StNumCacheDuplicates,
     StNumCacheEvictions,
-    StNumDuplicateKeyCount,
     StSizeOffsetBranches,
     StSizeBranchMemory,
     StSizeLeafMemory,

diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp
@@ -999,10 +999,9 @@ static const constexpr StatisticMeta statsMetaData[StMax] = {
     { PEAKNUMSTAT(PeakCacheObjects), "High water mark for number of objects in a cache"},
     { NUMSTAT(CacheDuplicates), "The number of times an item was added to a cache by two threads at the same time" },
     { NUMSTAT(CacheEvictions), "The number of times an item was evicted from a cache" },
-    { NUMSTAT(DuplicateKeyCount), "The number of duplicate keys" },
     { SIZESTAT(OffsetBranches), "The 1st branch node offset position in the index" },
-    { SIZESTAT(BranchMemory), "The size of branch memory" },
-    { SIZESTAT(LeafMemory), "The size of leaf memory"},
+    { SIZESTAT(BranchMemory), "The estimated size of the branch nodes when stored in memory" },
+    { SIZESTAT(LeafMemory), "The estimated size of the leaf nodes when stored in memory"},
 
 };
 

diff --git a/thorlcr/activities/indexwrite/thindexwriteslave.cpp b/thorlcr/activities/indexwrite/thindexwriteslave.cpp
@@ -602,7 +602,7 @@ class IndexWriteSlaveActivity : public ProcessSlaveActivity, public ILookAheadSt
             return;
         rowcount_t _processed = processed & THORDATALINK_COUNT_MASK;
         mb.append(_processed);
-        mb.append(inactiveStats.getStatisticValue(StNumDuplicateKeyCount));
+        mb.append(inactiveStats.getStatisticValue(StNumDuplicateKeys));
         if (!singlePartKey || firstNode())
         {
             StringBuffer partFname;

diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp
@@ -85,7 +85,7 @@ const StatisticsMapping basicActivityStatistics({StNumParallelExecute, StTimeLoo
 const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics);
 const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics);
 const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed}, indexReadFileStatistics, basicActivityStatistics);
-const StatisticsMapping indexWriteActivityStatistics({StPerReplicated, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds, StNumDuplicateKeyCount, StSizeOffsetBranches, StSizeBranchMemory, StSizeLeafMemory}, basicActivityStatistics, diskWriteRemoteStatistics);
+const StatisticsMapping indexWriteActivityStatistics({StPerReplicated, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds, StNumDuplicateKeys, StSizeOffsetBranches, StSizeBranchMemory, StSizeLeafMemory}, basicActivityStatistics, diskWriteRemoteStatistics);
 const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexAccepted, StNumPreFiltered, StNumDiskSeeks, StNumDiskAccepted, StNumDiskRejected}, basicActivityStatistics, indexReadFileStatistics);
 const StatisticsMapping commonJoinActivityStatistics({StNumMatchLeftRowsMax, StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax}, basicActivityStatistics);
 const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics);

diff --git a/tools/dumpkey/dumpkey.cpp b/tools/dumpkey/dumpkey.cpp
@@ -508,7 +508,7 @@ int main(int argc, const char **argv)
                 if (keyBuilder)
                 {
                     keyBuilder->finish(metadata, nullptr, maxSizeSeen);
-                    printf("New key has %" I64F "u leaves, %" I64F "u branches, %" I64F "u duplicates\n", keyBuilder->getStatistic(StNumLeafCacheAdds), keyBuilder->getStatistic(StNumNodeCacheAdds), keyBuilder->getStatistic(StNumDuplicateKeyCount));
+                    printf("New key has %" I64F "u leaves, %" I64F "u branches, %" I64F "u duplicates\n", keyBuilder->getStatistic(StNumLeafCacheAdds), keyBuilder->getStatistic(StNumNodeCacheAdds), keyBuilder->getStatistic(StNumDuplicateKeys));
                     printf("Original key size: %" I64F "u bytes\n", const_cast<IFileIO *>(index->queryFileIO())->size());
                     printf("New key size: %" I64F "u bytes (%" I64F "u bytes written in %" I64F "u writes)\n", outFileStream->size(), outFileStream->getStatistic(StSizeDiskWrite), outFileStream->getStatistic(StNumDiskWrites));
                     keyBuilder.clear();