diff --git a/plugins/ExtendedTools/PresentMon/PresentMon.cpp b/plugins/ExtendedTools/PresentMon/PresentMon.cpp index 3327ac81a6d1..03d5457f8fdf 100644 --- a/plugins/ExtendedTools/PresentMon/PresentMon.cpp +++ b/plugins/ExtendedTools/PresentMon/PresentMon.cpp @@ -48,7 +48,7 @@ static void CheckForTerminatedRealtimeProcesses( ) { LARGE_INTEGER performanceCounter; - PhQueryPerformanceCounter(&performanceCounter, NULL); + PhQueryPerformanceCounter(&performanceCounter, nullptr); terminatedProcesses->emplace_back(processId, performanceCounter.QuadPart); PhClearReference(reinterpret_cast(&processInfo->ProcessItem)); @@ -347,14 +347,14 @@ VOID PresentMonUpdateProcessStats( EtAddGpuFrameToHashTable( ProcessId, - frameLatency, - framesPerSecond, - displayLatency, - displayFramesPerSecond, - msBetweenPresents, - msInPresentApi, - msUntilRenderComplete, - msUntilDisplayed, + static_cast(frameLatency), + static_cast(framesPerSecond), + static_cast(displayLatency), + static_cast(displayFramesPerSecond), + static_cast(msBetweenPresents), + static_cast(msInPresentApi), + static_cast(msUntilRenderComplete), + static_cast(msUntilDisplayed), runtime, presentMode ); @@ -427,7 +427,7 @@ VOID StartOutputThread( { HANDLE threadHandle; - if (NT_SUCCESS(PhCreateThreadEx(&threadHandle, PresentMonOutputThread, NULL))) + if (NT_SUCCESS(PhCreateThreadEx(&threadHandle, PresentMonOutputThread, nullptr))) { PhSetThreadName(threadHandle, L"FpsEtwOutputThread"); NtClose(threadHandle); @@ -442,7 +442,7 @@ VOID StopOutputThread( ) { InterlockedExchange(&QuitOutputThread, 1); - //NtWaitForSingleObject(OutputThreadHandle, FALSE, NULL); + //NtWaitForSingleObject(OutputThreadHandle, FALSE, nullptr); } static NTSTATUS PresentMonTraceThread( @@ -454,7 +454,7 @@ static NTSTATUS PresentMonTraceThread( while (TRUE) { - while (!QuitOutputThread && (result = ProcessTrace(&traceHandle, 1, NULL, NULL)) == ERROR_SUCCESS) + while (!QuitOutputThread && (result = ProcessTrace(&traceHandle, 1, nullptr, nullptr)) == ERROR_SUCCESS) NOTHING; if (QuitOutputThread) @@ -480,7 +480,7 @@ VOID StartConsumerThread( { HANDLE threadHandle; - if (NT_SUCCESS(PhCreateThreadEx(&threadHandle, PresentMonTraceThread, (PVOID)traceHandle))) + if (NT_SUCCESS(PhCreateThreadEx(&threadHandle, PresentMonTraceThread, reinterpret_cast(traceHandle)))) { PhSetThreadName(threadHandle, L"FpsEtwConsumerThread"); NtClose(threadHandle); @@ -495,5 +495,5 @@ VOID WaitForConsumerThreadToExit( ) { InterlockedExchange(&QuitOutputThread, 1); - //NtWaitForSingleObject(ConsumerThreadHandle, FALSE, NULL); + //NtWaitForSingleObject(ConsumerThreadHandle, FALSE, nullptr); } diff --git a/plugins/ExtendedTools/etwstat.c b/plugins/ExtendedTools/etwstat.c index 9373d982365b..ee92d23bb5ee 100644 --- a/plugins/ExtendedTools/etwstat.c +++ b/plugins/ExtendedTools/etwstat.c @@ -3,7 +3,7 @@ * ETW statistics collection * * Copyright (C) 2010-2011 wj32 - * Copyright (C) 2019-2020 dmex + * Copyright (C) 2019-2022 dmex * * This file is part of Process Hacker. * @@ -34,10 +34,17 @@ VOID NTAPI EtEtwNetworkItemsUpdatedCallback( _In_opt_ PVOID Context ); +VOID EtpUpdateProcessInformation( + VOID + ); + BOOLEAN EtDiskExtEnabled = FALSE; static PH_CALLBACK_REGISTRATION EtpProcessesUpdatedCallbackRegistration; static PH_CALLBACK_REGISTRATION EtpNetworkItemsUpdatedCallbackRegistration; +static PVOID EtpProcessInformation = NULL; +static PH_QUEUED_LOCK EtpProcessInformationLock = PH_QUEUED_LOCK_INIT; + ULONG EtpDiskReadRaw; ULONG EtpDiskWriteRaw; ULONG EtpNetworkReceiveRaw; @@ -396,6 +403,8 @@ VOID NTAPI EtEtwProcessesUpdatedCallback( // Since Windows 8, we no longer get the correct process/thread IDs in the // event headers for disk events. We need to update our process information since // etwmon uses our EtThreadIdToProcessId function. (wj32) + if (PhWindowsVersion >= WINDOWS_8) + EtpUpdateProcessInformation(); // ETW is extremely lazy when it comes to flushing buffers, so we must do it manually. (wj32) //EtFlushEtwSession(); @@ -548,43 +557,56 @@ VOID NTAPI EtEtwNetworkItemsUpdatedCallback( } } +VOID EtpUpdateProcessInformation( + VOID + ) +{ + PhAcquireQueuedLockExclusive(&EtpProcessInformationLock); + + if (EtpProcessInformation) + { + PhFree(EtpProcessInformation); + EtpProcessInformation = NULL; + } + + PhEnumProcesses(&EtpProcessInformation); + + PhReleaseQueuedLockExclusive(&EtpProcessInformationLock); +} + HANDLE EtThreadIdToProcessId( _In_ HANDLE ThreadId ) { - // Note: no lock is needed because we use the list on the same thread (EtpEtwMonitorThreadStart). (dmex) - static PVOID processInfo = NULL; - static ULONG64 lastTickTotal = 0; PSYSTEM_PROCESS_INFORMATION process; - ULONG64 tickCount; + ULONG i; + HANDLE processId; - tickCount = NtGetTickCount64(); + PhAcquireQueuedLockShared(&EtpProcessInformationLock); - if (tickCount - lastTickTotal >= 2 * CLOCKS_PER_SEC) + if (!EtpProcessInformation) { - lastTickTotal = tickCount; - - if (processInfo) - { - PhFree(processInfo); - processInfo = NULL; - } - - PhEnumProcesses(&processInfo); + PhReleaseQueuedLockShared(&EtpProcessInformationLock); + return SYSTEM_PROCESS_ID; } - process = PH_FIRST_PROCESS(processInfo); + process = PH_FIRST_PROCESS(EtpProcessInformation); do { - for (ULONG i = 0; i < process->NumberOfThreads; i++) + for (i = 0; i < process->NumberOfThreads; i++) { if (process->Threads[i].ClientId.UniqueThread == ThreadId) { - return process->UniqueProcessId; + processId = process->UniqueProcessId; + PhReleaseQueuedLockShared(&EtpProcessInformationLock); + + return processId; } } } while (process = PH_NEXT_PROCESS(process)); + PhReleaseQueuedLockShared(&EtpProcessInformationLock); + return SYSTEM_PROCESS_ID; } diff --git a/plugins/ExtendedTools/framemon.cpp b/plugins/ExtendedTools/framemon.cpp index 28bd43e2a76e..338542cf932f 100644 --- a/plugins/ExtendedTools/framemon.cpp +++ b/plugins/ExtendedTools/framemon.cpp @@ -24,7 +24,7 @@ #include "framemon.h" BOOLEAN EtFramesEnabled = FALSE; -static PPH_HASHTABLE EtFramesHashTable = NULL; +static PPH_HASHTABLE EtFramesHashTable = nullptr; static PH_QUEUED_LOCK EtFramesHashTableLock = PH_QUEUED_LOCK_INIT; static BOOLEAN NTAPI EtFramesEqualFunction( @@ -94,7 +94,7 @@ VOID EtFramesMonitorStart( { if (EtFramesEnabled) { - PhQueueItemWorkQueue(PhGetGlobalWorkQueue(), EtStartFpsTraceSession, NULL); + PhQueueItemWorkQueue(PhGetGlobalWorkQueue(), EtStartFpsTraceSession, nullptr); } } @@ -132,14 +132,14 @@ VOID EtUnlockGpuFrameHashTable( VOID EtAddGpuFrameToHashTable( _In_ ULONG ProcessId, - _In_ DOUBLE FrameMs, - _In_ DOUBLE FramesPerSecond, - _In_ DOUBLE DisplayLatency, - _In_ DOUBLE DisplayFramesPerSecond, - _In_ DOUBLE MsBetweenPresents, - _In_ DOUBLE MsInPresentApi, - _In_ DOUBLE MsUntilRenderComplete, - _In_ DOUBLE MsUntilDisplayed, + _In_ FLOAT FrameMs, + _In_ FLOAT FramesPerSecond, + _In_ FLOAT DisplayLatency, + _In_ FLOAT DisplayFramesPerSecond, + _In_ FLOAT MsBetweenPresents, + _In_ FLOAT MsInPresentApi, + _In_ FLOAT MsUntilRenderComplete, + _In_ FLOAT MsUntilDisplayed, _In_ USHORT Runtime, _In_ USHORT PresentMode ) @@ -229,14 +229,14 @@ VOID EtProcessFramesUpdateProcessBlock( if (entry = EtLookupProcessGpuFrameEntry(ProcessBlock->ProcessItem->ProcessId)) { - ProcessBlock->FramesPerSecond = (FLOAT)entry->FramesPerSecond; - ProcessBlock->FramesLatency = (FLOAT)entry->FrameMs; - ProcessBlock->FramesMsBetweenPresents = (FLOAT)entry->MsBetweenPresents; - ProcessBlock->FramesMsInPresentApi = (FLOAT)entry->MsInPresentApi; - ProcessBlock->FramesMsUntilRenderComplete = (FLOAT)entry->MsUntilRenderComplete; - ProcessBlock->FramesMsUntilDisplayed = (FLOAT)entry->MsUntilDisplayed; - ProcessBlock->FramesDisplayLatency = (FLOAT)entry->DisplayLatency; - //ProcessBlock->FramesDisplayFramesPerSecond = (FLOAT)entry->DisplayFramesPerSecond; + ProcessBlock->FramesPerSecond = entry->FramesPerSecond; + ProcessBlock->FramesLatency = entry->FrameMs; + ProcessBlock->FramesMsBetweenPresents = entry->MsBetweenPresents; + ProcessBlock->FramesMsInPresentApi = entry->MsInPresentApi; + ProcessBlock->FramesMsUntilRenderComplete = entry->MsUntilRenderComplete; + ProcessBlock->FramesMsUntilDisplayed = entry->MsUntilDisplayed; + ProcessBlock->FramesDisplayLatency = entry->DisplayLatency; + //ProcessBlock->FramesDisplayFramesPerSecond = entry->DisplayFramesPerSecond; ProcessBlock->FramesRuntime = entry->Runtime; ProcessBlock->FramesPresentMode = entry->PresentMode; diff --git a/plugins/ExtendedTools/framemon.h b/plugins/ExtendedTools/framemon.h index e2b06fb68c88..c8e178a48404 100644 --- a/plugins/ExtendedTools/framemon.h +++ b/plugins/ExtendedTools/framemon.h @@ -30,14 +30,14 @@ extern "C" { typedef struct _ET_FPS_COUNTER { HANDLE ProcessId; - DOUBLE FrameMs; - DOUBLE FramesPerSecond; - DOUBLE MsBetweenPresents; - DOUBLE MsInPresentApi; - DOUBLE MsUntilRenderComplete; - DOUBLE MsUntilDisplayed; - DOUBLE DisplayLatency; - //DOUBLE DisplayFramesPerSecond; + FLOAT FrameMs; + FLOAT FramesPerSecond; + FLOAT MsBetweenPresents; + FLOAT MsInPresentApi; + FLOAT MsUntilRenderComplete; + FLOAT MsUntilDisplayed; + FLOAT DisplayLatency; + //FLOAT DisplayFramesPerSecond; USHORT Runtime; USHORT PresentMode; } ET_FPS_COUNTER, *PET_FPS_COUNTER; @@ -60,14 +60,14 @@ PET_FPS_COUNTER EtLookupProcessGpuFrameEntry( VOID EtAddGpuFrameToHashTable( _In_ ULONG ProcessId, - _In_ DOUBLE FrameMs, - _In_ DOUBLE FramesPerSecond, - _In_ DOUBLE DisplayLatency, - _In_ DOUBLE DisplayFramesPerSecond, - _In_ DOUBLE MsBetweenPresents, - _In_ DOUBLE MsInPresentApi, - _In_ DOUBLE MsUntilRenderComplete, - _In_ DOUBLE MsUntilDisplayed, + _In_ FLOAT FrameMs, + _In_ FLOAT FramesPerSecond, + _In_ FLOAT DisplayLatency, + _In_ FLOAT DisplayFramesPerSecond, + _In_ FLOAT MsBetweenPresents, + _In_ FLOAT MsInPresentApi, + _In_ FLOAT MsUntilRenderComplete, + _In_ FLOAT MsUntilDisplayed, _In_ USHORT Runtime, _In_ USHORT PresentMode ); diff --git a/tools/peview/clrprp.c b/tools/peview/clrprp.c index 932878ba565c..b62accf43744 100644 --- a/tools/peview/clrprp.c +++ b/tools/peview/clrprp.c @@ -22,7 +22,7 @@ */ #include -#include "metahost.h" +#include typedef struct _PVP_PE_CLR_CONTEXT { diff --git a/tools/peview/clrtableimportprp.c b/tools/peview/clrtableimportprp.c index 90a69ad46e71..593485a1fa53 100644 --- a/tools/peview/clrtableimportprp.c +++ b/tools/peview/clrtableimportprp.c @@ -142,6 +142,21 @@ VOID PvpEnumerateClrImports( if (importDll->Functions) { + if (importDll->ImportName) + { + PPH_STRING importDllName; + + if (importDllName = PhApiSetResolveToHost(&importDll->ImportName->sr)) + { + PhMoveReference(&importDll->ImportName, PhFormatString( + L"%s (%s)", + PhGetString(importDll->ImportName), + PhGetString(importDllName)) + ); + PhDereferenceObject(importDllName); + } + } + for (j = 0; j < importDll->Functions->Count; j++) { PPV_CLR_IMAGE_IMPORT_FUNCTION importFunction = importDll->Functions->Items[j]; diff --git a/tools/peview/expprp.c b/tools/peview/expprp.c index 42fc139cb491..66ddb5a9918d 100644 --- a/tools/peview/expprp.c +++ b/tools/peview/expprp.c @@ -206,8 +206,11 @@ NTSTATUS PvpPeExportsEnumerateThread( } else { - PhPrintPointer(value, exportFunction.Function); - exportNode->AddressString = PhCreateString(value); + if (exportFunction.Function) + { + PhPrintPointer(value, exportFunction.Function); + exportNode->AddressString = PhCreateString(value); + } } if (exportEntry.Name) @@ -257,11 +260,12 @@ NTSTATUS PvpPeExportsEnumerateThread( ); } - if (exportSymbolName) + if (!PhIsNullOrEmptyString(exportSymbolName)) { exportNode->NameString = PhConcatStringRefZ(&exportSymbolName->sr, L" (unnamed)"); } - + + PhClearReference(&exportSymbolName); PhClearReference(&exportSymbol); } } @@ -746,12 +750,7 @@ BOOLEAN NTAPI PvExportTreeNewCallback( getCellText->Text = PhGetStringRef(node->AddressString); break; case PV_EXPORT_TREE_COLUMN_ITEM_NAME: - { - if (node->NameString) - getCellText->Text = PhGetStringRef(node->NameString); - else - PhInitializeStringRefLongHint(&getCellText->Text, L"(unnamed)"); - } + getCellText->Text = PhGetStringRef(node->NameString); break; case PV_EXPORT_TREE_COLUMN_ITEM_ORDINAL: getCellText->Text = PhGetStringRef(node->OrdinalString); diff --git a/tools/peview/impprp.c b/tools/peview/impprp.c index e7794ee4264c..54f7664a364c 100644 --- a/tools/peview/impprp.c +++ b/tools/peview/impprp.c @@ -216,22 +216,29 @@ PPH_STRING PvpQueryModuleOrdinalName( if (NT_SUCCESS(PhGetProcessMappedFileName(NtCurrentProcess(), (PVOID)mappedImage.ViewBase, &exportFileName))) { - if (PhLoadModuleSymbolProvider( - PvSymbolProvider, - exportFileName, - (ULONG64)mappedImage.ViewBase, - (ULONG)mappedImage.Size - )) + PPH_SYMBOL_PROVIDER moduleSymbolProvider = NULL; + + if (PvpLoadDbgHelp(&moduleSymbolProvider)) { - // Try find the export name using symbols. - exportSymbol = PhGetSymbolFromAddress( - PvSymbolProvider, - (ULONG64)PTR_ADD_OFFSET(mappedImage.ViewBase, exportFunction.Function), - NULL, - NULL, - &exportSymbolName, - NULL - ); + if (PhLoadModuleSymbolProvider( + moduleSymbolProvider, + exportFileName, + (ULONG64)mappedImage.ViewBase, + (ULONG)mappedImage.Size + )) + { + // Try find the export name using symbols. + exportSymbol = PhGetSymbolFromAddress( + moduleSymbolProvider, + (ULONG64)PTR_ADD_OFFSET(mappedImage.ViewBase, exportFunction.Function), + NULL, + NULL, + &exportSymbolName, + NULL + ); + } + + PhDereferenceObject(moduleSymbolProvider); } PhDereferenceObject(exportFileName); @@ -339,8 +346,13 @@ VOID PvpProcessImports( if (exportDllName = PhConvertUtf8ToUtf16(importDll.Name)) { PPH_STRING filePath; + PPH_STRING importDllName; + + if (importDllName = PhApiSetResolveToHost(&exportDllName->sr)) + { + PhMoveReference(&exportDllName, importDllName); + } - // TODO: Implement ApiSet mappings for exportDllName. (dmex) // TODO: Add DLL directory to PhSearchFilePath for locating non-system images. (dmex) if (filePath = PhSearchFilePath(exportDllName->Buffer, L".dll")) diff --git a/tools/peview/pdbprp.c b/tools/peview/pdbprp.c index 2c7ea81c07dd..d6f12efd1128 100644 --- a/tools/peview/pdbprp.c +++ b/tools/peview/pdbprp.c @@ -2,7 +2,7 @@ * PE viewer - * pdb support * - * Copyright (C) 2017-2021 dmex + * Copyright (C) 2017-2022 dmex * * This file is part of Process Hacker. * @@ -187,13 +187,13 @@ END_SORT_FUNCTION BEGIN_SORT_FUNCTION(Symbol) { - sortResult = PhCompareStringWithNull(node1->Name, node2->Name, FALSE); + sortResult = PhCompareStringWithNull(node1->Name, node2->Name, TRUE); } END_SORT_FUNCTION BEGIN_SORT_FUNCTION(Data) { - sortResult = PhCompareStringWithNull(node1->Data, node2->Data, FALSE); + sortResult = PhCompareStringWithNull(node1->Data, node2->Data, TRUE); } END_SORT_FUNCTION diff --git a/tools/peview/peprp.c b/tools/peview/peprp.c index fa7e845cb4da..0459c7012074 100644 --- a/tools/peview/peprp.c +++ b/tools/peview/peprp.c @@ -1126,7 +1126,7 @@ static NTSTATUS PvpEntryPointImageThreadStart( { ULONG addressOfEntryPoint; PPH_STRING string; - PPH_STRING symbol; + PPH_STRING symbol = NULL; PPH_STRING symbolName = NULL; PH_SYMBOL_RESOLVE_LEVEL symbolResolveLevel = PhsrlInvalid; @@ -1135,30 +1135,38 @@ static NTSTATUS PvpEntryPointImageThreadStart( else addressOfEntryPoint = PvMappedImage.NtHeaders->OptionalHeader.AddressOfEntryPoint; - if (PvMappedImage.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) + if (addressOfEntryPoint) { - symbol = PhGetSymbolFromAddress( - PvSymbolProvider, - (ULONG64)PTR_ADD_OFFSET(PvMappedImage.NtHeaders32->OptionalHeader.ImageBase, addressOfEntryPoint), - &symbolResolveLevel, - NULL, - &symbolName, - NULL - ); - } - else - { - symbol = PhGetSymbolFromAddress( - PvSymbolProvider, - (ULONG64)PTR_ADD_OFFSET(PvMappedImage.NtHeaders->OptionalHeader.ImageBase, addressOfEntryPoint), - &symbolResolveLevel, - NULL, - &symbolName, - NULL - ); + if (PvMappedImage.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) + { + symbol = PhGetSymbolFromAddress( + PvSymbolProvider, + (ULONG64)PTR_ADD_OFFSET(PvMappedImage.NtHeaders32->OptionalHeader.ImageBase, addressOfEntryPoint), + &symbolResolveLevel, + NULL, + &symbolName, + NULL + ); + } + else + { + symbol = PhGetSymbolFromAddress( + PvSymbolProvider, + (ULONG64)PTR_ADD_OFFSET(PvMappedImage.NtHeaders->OptionalHeader.ImageBase, addressOfEntryPoint), + &symbolResolveLevel, + NULL, + &symbolName, + NULL + ); + } } - if (symbolName && symbolResolveLevel == PhsrlFunction || symbolResolveLevel == PhsrlModule || symbolResolveLevel == PhsrlAddress) + if ( + !PhIsNullOrEmptyString(symbolName) && ( + symbolResolveLevel == PhsrlFunction || + symbolResolveLevel == PhsrlModule || + symbolResolveLevel == PhsrlAddress + )) { string = PhFormatString(L"0x%I32x (%s)", addressOfEntryPoint, PhGetStringOrEmpty(symbolName)); PhSetListViewSubItem(Parameter, PVP_IMAGE_GENERAL_INDEX_ENTRYPOINT, 1, string->Buffer); @@ -1171,9 +1179,8 @@ static NTSTATUS PvpEntryPointImageThreadStart( PhDereferenceObject(string); } - if (symbolName) - PhDereferenceObject(symbolName); - PhDereferenceObject(symbol); + PhClearReference(&symbolName); + PhClearReference(&symbol); return STATUS_SUCCESS; } diff --git a/tools/thirdparty/bin/Debug32/thirdparty.lib b/tools/thirdparty/bin/Debug32/thirdparty.lib index 9c580ce4f13a..a77c790ddbf4 100644 Binary files a/tools/thirdparty/bin/Debug32/thirdparty.lib and b/tools/thirdparty/bin/Debug32/thirdparty.lib differ diff --git a/tools/thirdparty/bin/Debug64/thirdparty.lib b/tools/thirdparty/bin/Debug64/thirdparty.lib index 6db3c7afed8b..f44334aedc6d 100644 Binary files a/tools/thirdparty/bin/Debug64/thirdparty.lib and b/tools/thirdparty/bin/Debug64/thirdparty.lib differ diff --git a/tools/thirdparty/bin/DebugARM64/thirdparty.lib b/tools/thirdparty/bin/DebugARM64/thirdparty.lib index 1d957c96e46a..94a323246748 100644 Binary files a/tools/thirdparty/bin/DebugARM64/thirdparty.lib and b/tools/thirdparty/bin/DebugARM64/thirdparty.lib differ diff --git a/tools/thirdparty/bin/Release32/thirdparty.lib b/tools/thirdparty/bin/Release32/thirdparty.lib index 3b4cfec31c5e..909cf36ad3cc 100644 Binary files a/tools/thirdparty/bin/Release32/thirdparty.lib and b/tools/thirdparty/bin/Release32/thirdparty.lib differ diff --git a/tools/thirdparty/bin/Release64/thirdparty.lib b/tools/thirdparty/bin/Release64/thirdparty.lib index 8da8c26c8d83..4e36bd7dfc4a 100644 Binary files a/tools/thirdparty/bin/Release64/thirdparty.lib and b/tools/thirdparty/bin/Release64/thirdparty.lib differ diff --git a/tools/thirdparty/bin/ReleaseARM64/thirdparty.lib b/tools/thirdparty/bin/ReleaseARM64/thirdparty.lib index 435f63459a18..b47da11ca31d 100644 Binary files a/tools/thirdparty/bin/ReleaseARM64/thirdparty.lib and b/tools/thirdparty/bin/ReleaseARM64/thirdparty.lib differ diff --git a/tools/thirdparty/thirdparty.vcxproj b/tools/thirdparty/thirdparty.vcxproj index badb2fef4997..62583566dfd3 100644 --- a/tools/thirdparty/thirdparty.vcxproj +++ b/tools/thirdparty/thirdparty.vcxproj @@ -515,8 +515,8 @@ + - diff --git a/tools/thirdparty/thirdparty.vcxproj.filters b/tools/thirdparty/thirdparty.vcxproj.filters index 0abce24fb1f1..46e894278c25 100644 --- a/tools/thirdparty/thirdparty.vcxproj.filters +++ b/tools/thirdparty/thirdparty.vcxproj.filters @@ -752,9 +752,6 @@ tlsh - - tlsh - gzip\Headers @@ -821,6 +818,9 @@ miniz + + tlsh + diff --git a/tools/thirdparty/tlsh/WinFunctions.h b/tools/thirdparty/tlsh/WinFunctions.h deleted file mode 100644 index 58ea99ab774e..000000000000 --- a/tools/thirdparty/tlsh/WinFunctions.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifdef WINDOWS -#ifndef WINFUNCTIONS_H -#define WINFUNCTIONS_H - -#define TLSH_LIB - -#include -#include -#include - -#ifndef TLSH_LIB -# ifdef TLSH_EXPORTS -# define TLSH_API __declspec(dllexport) -# else -# define TLSH_API __declspec(dllimport) -# endif -#else -# define TLSH_API -#endif - -#define strdup _strdup -#define NAME_LENGTH MAX_PATH -#define snprintf _snprintf -#define strcasecmp _stricmp -#define random rand -#define srandom srand - -struct dirent { - char d_name[NAME_LENGTH]; -}; - -typedef struct _DIR -{ - char dirname[NAME_LENGTH]; - HANDLE hFind; - WIN32_FIND_DATA findFileData; - struct dirent findDirEnt; -} DIR; - -extern DIR *opendir(const char *dirname); -extern struct dirent *readdir(DIR *dir); -extern int closedir(DIR *dir); -extern struct tm *localtime_r(const time_t *timep, struct tm *results); -extern bool read_file_win(const char *fname, int sizefile, unsigned char* data); - -#endif // #ifndef WINFUNCTIONS_H - -#endif // #ifdef WINDOWS diff --git a/tools/thirdparty/tlsh/tlsh.cpp b/tools/thirdparty/tlsh/tlsh.cpp index 301e89bdbec2..204746795f73 100644 --- a/tools/thirdparty/tlsh/tlsh.cpp +++ b/tools/thirdparty/tlsh/tlsh.cpp @@ -55,13 +55,11 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ -#define _CRT_SECURE_NO_WARNINGS 1 - #include "tlsh.h" #include "tlsh_impl.h" -//#include -//#include -//#include +#include +#include +#include ///////////////////////////////////////////////////// // C++ Implementation @@ -135,17 +133,23 @@ Tlsh::~Tlsh() void Tlsh::update(const unsigned char* data, unsigned int len) { - if ( NULL != impl ) - impl->update(data, len); + // + // threaded and private options only available to + // windowsize == 5 + // calling final - without calling update first + // + int tlsh_option = 0; + if (impl != NULL) + impl->update(data, len, tlsh_option); } -void Tlsh::final(const unsigned char* data, unsigned int len, int fc_cons_option) +void Tlsh::final(const unsigned char* data, unsigned int len, int tlsh_option) { - if ( NULL != impl ){ - if ( NULL != data && len > 0 ) - impl->update(data, len); - impl->final(fc_cons_option); - } + if (NULL != impl) { + if ((data != NULL) && (len > 0)) + impl->update(data, len, tlsh_option); + impl->final(tlsh_option); + } } const char* Tlsh::getHash(int showvers) const @@ -217,6 +221,11 @@ int Tlsh::BucketValue(int bucket) return( impl->BucketValue(bucket) ); } +int Tlsh::HistogramCount(int bucket) +{ + return( impl->HistogramCount(bucket) ); +} + int Tlsh::totalDiff(const Tlsh *other, bool len_diff) const { if( NULL==impl || NULL == other || NULL == other->impl ) diff --git a/tools/thirdparty/tlsh/tlsh.h b/tools/thirdparty/tlsh/tlsh.h index 5904214d9391..97ae9ebf620e 100644 --- a/tools/thirdparty/tlsh/tlsh.h +++ b/tools/thirdparty/tlsh/tlsh.h @@ -60,12 +60,19 @@ #ifndef HEADER_TLSH_H #define HEADER_TLSH_H -#define WINDOWS +#define WINDOWS // dmex + +// -force option no longer used +// #define TLSH_OPTION_FORCE 1 +#define TLSH_OPTION_CONSERVATIVE 2 +#define TLSH_OPTION_KEEP_BUCKET 4 +#define TLSH_OPTION_PRIVATE 8 +#define TLSH_OPTION_THREADED 16 #if defined WINDOWS || defined MINGW -#include "win_version.h" +#include "tlsh_win_version.h" #else -#include "version.h" +#include "tlsh_version.h" #endif #ifndef NULL @@ -108,7 +115,9 @@ class TlshImpl; #define TLSH_STRING_BUFFER_LEN (TLSH_STRING_LEN_REQ+1) #ifdef WINDOWS -#include "WinFunctions.h" +// 27/Nov/2020 +// #include + #define TLSH_API #else #if defined(__SPARC) || defined(_AS_MK_OS_RH73) #define TLSH_API @@ -144,6 +153,7 @@ class TLSH_API Tlsh{ int Q2ratio(); int Checksum(int k); int BucketValue(int bucket); + int HistogramCount(int bucket); /* calculate difference */ /* The len_diff parameter specifies if the file length is to be included in the difference calculation (len_diff=true) or if it */ diff --git a/tools/thirdparty/tlsh/tlsh_impl.cpp b/tools/thirdparty/tlsh/tlsh_impl.cpp index 811f00704ea6..49a093eb7e91 100644 --- a/tools/thirdparty/tlsh/tlsh_impl.cpp +++ b/tools/thirdparty/tlsh/tlsh_impl.cpp @@ -67,6 +67,7 @@ #include #include //#include +#include #define RANGE_LVALUE 256 #define RANGE_QRATIO 16 @@ -122,6 +123,25 @@ static unsigned char v_table[256] = { 51, 65, 28, 144, 254, 221, 93, 189, 194, 139, 112, 43, 71, 109, 184, 209 }; +static unsigned char v_table48[256] = { + 1, 39, 1, 12, 32, 34, 6, 22, 25, 1, 6, 36, 48, 38, 44, 19, + 14, 5, 21, 37, 17, 37, 26, 32, 16, 47, 24, 34, 44, 46, 38, 8, + 14, 33, 8, 7, 45, 48, 48, 2, 29, 5, 33, 18, 45, 0, 31, 30, + 25, 11, 46, 22, 38, 45, 48, 34, 24, 48, 20, 22, 48, 35, 5, 43, + 1, 42, 9, 22, 12, 48, 34, 31, 16, 5, 31, 7, 15, 14, 39, 48, + 30, 25, 19, 10, 18, 10, 10, 3, 48, 27, 17, 43, 38, 35, 0, 48, + 36, 8, 4, 27, 32, 37, 14, 4, 34, 30, 43, 13, 9, 48, 24, 27, + 23, 20, 31, 30, 35, 40, 9, 3, 26, 11, 44, 32, 40, 18, 4, 10, + 42, 30, 0, 39, 12, 35, 13, 26, 47, 26, 48, 46, 33, 18, 15, 8, + 26, 7, 19, 23, 48, 14, 3, 6, 7, 11, 7, 28, 42, 5, 23, 35, + 29, 29, 15, 46, 31, 47, 41, 16, 9, 41, 33, 32, 25, 16, 37, 27, + 22, 25, 2, 13, 46, 20, 9, 1, 38, 36, 15, 20, 10, 23, 21, 37, + 27, 44, 19, 28, 24, 48, 42, 4, 29, 12, 21, 48, 19, 13, 39, 11, + 41, 40, 42, 3, 6, 0, 11, 33, 20, 47, 2, 12, 21, 36, 21, 28, + 44, 36, 18, 28, 41, 6, 15, 8, 41, 40, 17, 4, 39, 47, 2, 24, + 3, 17, 28, 0, 48, 29, 45, 45, 2, 43, 16, 43, 23, 13, 40, 17, +}; + // Pearson's algorithm unsigned char b_mapping(unsigned char salt, unsigned char i, unsigned char j, unsigned char k) { unsigned char h = 0; @@ -145,7 +165,11 @@ unsigned char faster_b_mapping(unsigned char mod_salt, unsigned char i, unsigned } */ -#define fast_b_mapping(ms,i,j,k) (v_table[ v_table[ v_table[ms^i] ^ j] ^ k ]) +#if defined BUCKETS_48 +#define fast_b_mapping(ms,i,j,k) (v_table48[ v_table[ v_table[ms^i] ^ j] ^ k ]) +#else +#define fast_b_mapping(ms,i,j,k) (v_table [ v_table[ v_table[ms^i] ^ j] ^ k ]) +#endif //////////////////////////////////////////////////////////////////////////////////////////// @@ -161,15 +185,15 @@ unsigned char faster_b_mapping(unsigned char mod_salt, unsigned char i, unsigned #define SLIDING_WND_SIZE_M1 7 #endif -void TlshImpl::update(const unsigned char* data, unsigned int len) +#define RNG_SIZE SLIDING_WND_SIZE +#define RNG_IDX(i) ((i+RNG_SIZE)%RNG_SIZE) + +void TlshImpl::update(const unsigned char* data, unsigned int len, int tlsh_option) { if (this->lsh_code_valid) { //fprintf(stderr, "call to update() on a tlsh that is already valid\n"); return; } - - #define RNG_SIZE SLIDING_WND_SIZE - #define RNG_IDX(i) ((i+RNG_SIZE)%RNG_SIZE) unsigned int fed_len = this->data_len; @@ -180,7 +204,12 @@ void TlshImpl::update(const unsigned char* data, unsigned int len) #if SLIDING_WND_SIZE==5 if (TLSH_CHECKSUM_LEN == 1) { - fast_update(data, len); + fast_update5(data, len, tlsh_option); +#ifndef CHECKSUM_0B + if ((tlsh_option & TLSH_OPTION_THREADED) || (tlsh_option & TLSH_OPTION_PRIVATE)) { + this->lsh_bin.checksum[0] = 0; + } +#endif return; } #endif @@ -294,104 +323,396 @@ void TlshImpl::update(const unsigned char* data, unsigned int len) } } this->data_len += len; +#ifndef CHECKSUM_0B + if ((tlsh_option & TLSH_OPTION_THREADED) || (tlsh_option & TLSH_OPTION_PRIVATE)) { + for (int k = 0; k < TLSH_CHECKSUM_LEN; k++) { + this->lsh_bin.checksum[k] = 0; + } + } +#endif } ///////////////////////////////////////////////////////////////////////////// -// update for the case when SLIDING_WND_SIZE==5 && (TLSH_CHECKSUM_LEN == 1) +// update for the case when SLIDING_WND_SIZE==5 +// have different optimized functions for +// default TLSH +// threaded TLSH +// private TLSH ///////////////////////////////////////////////////////////////////////////// +static void raw_fast_update5( + // inputs + const unsigned char* data, + unsigned int len, + unsigned int fed_len, + // outputs + unsigned int *a_bucket, + unsigned char *ret_checksum, + unsigned char *slide_window + ); + +static void raw_fast_update5_private( + // inputs + const unsigned char* data, + unsigned int len, + unsigned int fed_len, + // outputs + unsigned int *a_bucket, + unsigned char *slide_window + ); + +#ifdef THREADING_IMPLEMENTED +#include +#endif + +struct raw_args { + // inputs + const unsigned char* data; + unsigned int len; + unsigned int fed_len; + // outputs + unsigned int bucket[256]; + unsigned char slide_window[5]; +}; + +static void raw_fast_update5_nochecksum( struct raw_args *ra ); + +static struct raw_args call1; +static struct raw_args call2; + +void thread1() +{ + raw_fast_update5_nochecksum( &call1 ); +} +void thread2() +{ + raw_fast_update5_nochecksum( &call2 ); +} + +void TlshImpl::fast_update5(const unsigned char* data, unsigned int len, int tlsh_option) +{ +#ifdef THREADING_IMPLEMENTED + if ((len >= 10000) && (tlsh_option & TLSH_OPTION_THREADED)) { + unsigned len2A = len / 2; + unsigned len2B = len - len2A; + // printf("method 2 len=%d len2A=%d len2B=%d\n", len, len2A, len2B); + + for (int bi=0; bi<256; bi++) { + call1.bucket[bi] = 0; + call2.bucket[bi] = 0; + } + for (int di=0; di<5; di++) { + call1.slide_window[di] = 0; + int didx = len2A - 5 + di; + int wi = didx % 5; + call2.slide_window[wi] = data[didx]; + } + + call1.data = data; + call1.len = len2A; + call1.fed_len = 0; + + call2.data = &data[len2A]; + call2.len = len2B; + call2.fed_len = len2A; + + std::thread t1(thread1); + std::thread t2(thread2); + t1.join(); + t2.join(); + + this->data_len += len; -void TlshImpl::fast_update(const unsigned char* data, unsigned int len) + for (int bi=0; bi<128; bi++) { + this->a_bucket[bi] += (call1.bucket[bi] + call2.bucket[bi]) ; + // printf("bucket %d =%d\n", bi, this->a_bucket[bi] ); + } + return; + } +#endif + if (tlsh_option & TLSH_OPTION_PRIVATE) { + raw_fast_update5_private(data, len, this->data_len, this->a_bucket, this->slide_window); + this->data_len += len; + this->lsh_bin.checksum[0] = 0; + } else { + raw_fast_update5 (data, len, this->data_len, this->a_bucket, &(this->lsh_bin.checksum[0]), this->slide_window); + this->data_len += len; + } +} + +static void raw_fast_update5( + // inputs + const unsigned char* data, + unsigned int len, + unsigned int fed_len, + // outputs + unsigned int *a_bucket, + unsigned char *ret_checksum, + unsigned char *slide_window + ) +{ + int j = (int)(fed_len % RNG_SIZE); + unsigned char checksum = *ret_checksum; + + unsigned int start_i=0; + if ( fed_len < SLIDING_WND_SIZE_M1 ) { + int extra = SLIDING_WND_SIZE_M1 - fed_len; + start_i = extra; + j = (j + extra) % RNG_SIZE; + } + for( unsigned int i=start_i; i= 5 bytes + if ((i >= 4) && (i+5 < len)) { + unsigned char a0 = data[i-4]; + unsigned char a1 = data[i-3]; + unsigned char a2 = data[i-2]; + unsigned char a3 = data[i-1]; + unsigned char a4 = data[i]; + unsigned char a5 = data[i+1]; + unsigned char a6 = data[i+2]; + unsigned char a7 = data[i+3]; + unsigned char a8 = data[i+4]; + + checksum = fast_b_mapping(1, a4, a3, checksum ); + a_bucket[ fast_b_mapping(49, a4, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(12, a4, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(178, a4, a2, a1 ) ]++; + a_bucket[ fast_b_mapping(166, a4, a2, a0 ) ]++; + a_bucket[ fast_b_mapping(84, a4, a3, a0 ) ]++; + a_bucket[ fast_b_mapping(230, a4, a1, a0 ) ]++; + + checksum = fast_b_mapping(1, a5, a4, checksum ); + a_bucket[ fast_b_mapping(49, a5, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(12, a5, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(178, a5, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(166, a5, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(84, a5, a4, a1 ) ]++; + a_bucket[ fast_b_mapping(230, a5, a2, a1 ) ]++; + + checksum = fast_b_mapping(1, a6, a5, checksum ); + a_bucket[ fast_b_mapping(49, a6, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(12, a6, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(178, a6, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(166, a6, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(84, a6, a5, a2 ) ]++; + a_bucket[ fast_b_mapping(230, a6, a3, a2 ) ]++; + + checksum = fast_b_mapping(1, a7, a6, checksum ); + a_bucket[ fast_b_mapping(49, a7, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(12, a7, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(178, a7, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(166, a7, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(84, a7, a6, a3 ) ]++; + a_bucket[ fast_b_mapping(230, a7, a4, a3 ) ]++; + + checksum = fast_b_mapping(1, a8, a7, checksum ); + a_bucket[ fast_b_mapping(49, a8, a7, a6 ) ]++; + a_bucket[ fast_b_mapping(12, a8, a7, a5 ) ]++; + a_bucket[ fast_b_mapping(178, a8, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(166, a8, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(84, a8, a7, a4 ) ]++; + a_bucket[ fast_b_mapping(230, a8, a5, a4 ) ]++; + + i=i+5; + j=RNG_IDX(j+5); + } else { + slide_window[j] = data[i]; + int j_1 = RNG_IDX(j-1); if (i >= 1) { slide_window[j_1] = data[i-1]; } + int j_2 = RNG_IDX(j-2); if (i >= 2) { slide_window[j_2] = data[i-2]; } + int j_3 = RNG_IDX(j-3); if (i >= 3) { slide_window[j_3] = data[i-3]; } + int j_4 = RNG_IDX(j-4); if (i >= 4) { slide_window[j_4] = data[i-4]; } + + checksum = fast_b_mapping(1, slide_window[j], slide_window[j_1], checksum ); + a_bucket[ fast_b_mapping(49, slide_window[j], slide_window[j_1], slide_window[j_2] ) ]++; + a_bucket[ fast_b_mapping(12, slide_window[j], slide_window[j_1], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(178, slide_window[j], slide_window[j_2], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(166, slide_window[j], slide_window[j_2], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(84, slide_window[j], slide_window[j_1], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(230, slide_window[j], slide_window[j_3], slide_window[j_4] ) ]++; + i++; + j=RNG_IDX(j+1); + } + } + *ret_checksum = checksum; +} + +static void raw_fast_update5_private( + // inputs + const unsigned char* data, + unsigned int len, + unsigned int fed_len, + // outputs + unsigned int *a_bucket, + unsigned char *slide_window + ) { - unsigned int fed_len = this->data_len; - int j = (int)(this->data_len % RNG_SIZE); - unsigned char checksum = this->lsh_bin.checksum[0]; - - for( unsigned int i=0; i= SLIDING_WND_SIZE_M1 ) { - //only calculate when input >= 5 bytes - if ((i >= 4) && (i+5 < len)) { - unsigned a0 = data[i-4]; - unsigned a1 = data[i-3]; - unsigned a2 = data[i-2]; - unsigned a3 = data[i-1]; - unsigned a4 = data[i]; - unsigned a5 = data[i+1]; - unsigned a6 = data[i+2]; - unsigned a7 = data[i+3]; - unsigned a8 = data[i+4]; - - checksum = fast_b_mapping(1, a4, a3, checksum ); - this->a_bucket[ fast_b_mapping(49, a4, a3, a2 ) ]++; - this->a_bucket[ fast_b_mapping(12, a4, a3, a1 ) ]++; - this->a_bucket[ fast_b_mapping(178, a4, a2, a1 ) ]++; - this->a_bucket[ fast_b_mapping(166, a4, a2, a0 ) ]++; - this->a_bucket[ fast_b_mapping(84, a4, a3, a0 ) ]++; - this->a_bucket[ fast_b_mapping(230, a4, a1, a0 ) ]++; - - checksum = fast_b_mapping(1, a5, a4, checksum ); - this->a_bucket[ fast_b_mapping(49, a5, a4, a3 ) ]++; - this->a_bucket[ fast_b_mapping(12, a5, a4, a2 ) ]++; - this->a_bucket[ fast_b_mapping(178, a5, a3, a2 ) ]++; - this->a_bucket[ fast_b_mapping(166, a5, a3, a1 ) ]++; - this->a_bucket[ fast_b_mapping(84, a5, a4, a1 ) ]++; - this->a_bucket[ fast_b_mapping(230, a5, a2, a1 ) ]++; - - checksum = fast_b_mapping(1, a6, a5, checksum ); - this->a_bucket[ fast_b_mapping(49, a6, a5, a4 ) ]++; - this->a_bucket[ fast_b_mapping(12, a6, a5, a3 ) ]++; - this->a_bucket[ fast_b_mapping(178, a6, a4, a3 ) ]++; - this->a_bucket[ fast_b_mapping(166, a6, a4, a2 ) ]++; - this->a_bucket[ fast_b_mapping(84, a6, a5, a2 ) ]++; - this->a_bucket[ fast_b_mapping(230, a6, a3, a2 ) ]++; - - checksum = fast_b_mapping(1, a7, a6, checksum ); - this->a_bucket[ fast_b_mapping(49, a7, a6, a5 ) ]++; - this->a_bucket[ fast_b_mapping(12, a7, a6, a4 ) ]++; - this->a_bucket[ fast_b_mapping(178, a7, a5, a4 ) ]++; - this->a_bucket[ fast_b_mapping(166, a7, a5, a3 ) ]++; - this->a_bucket[ fast_b_mapping(84, a7, a6, a3 ) ]++; - this->a_bucket[ fast_b_mapping(230, a7, a4, a3 ) ]++; - - checksum = fast_b_mapping(1, a8, a7, checksum ); - this->a_bucket[ fast_b_mapping(49, a8, a7, a6 ) ]++; - this->a_bucket[ fast_b_mapping(12, a8, a7, a5 ) ]++; - this->a_bucket[ fast_b_mapping(178, a8, a6, a5 ) ]++; - this->a_bucket[ fast_b_mapping(166, a8, a6, a4 ) ]++; - this->a_bucket[ fast_b_mapping(84, a8, a7, a4 ) ]++; - this->a_bucket[ fast_b_mapping(230, a8, a5, a4 ) ]++; - - i=i+5; - fed_len=fed_len+5; - j=RNG_IDX(j+5); - } else { - this->slide_window[j] = data[i]; - int j_1 = RNG_IDX(j-1); if (i >= 1) { this->slide_window[j_1] = data[i-1]; } - int j_2 = RNG_IDX(j-2); if (i >= 2) { this->slide_window[j_2] = data[i-2]; } - int j_3 = RNG_IDX(j-3); if (i >= 3) { this->slide_window[j_3] = data[i-3]; } - int j_4 = RNG_IDX(j-4); if (i >= 4) { this->slide_window[j_4] = data[i-4]; } - - checksum = fast_b_mapping(1, this->slide_window[j], this->slide_window[j_1], checksum ); - this->a_bucket[ fast_b_mapping(49, this->slide_window[j], this->slide_window[j_1], this->slide_window[j_2] ) ]++; - this->a_bucket[ fast_b_mapping(12, this->slide_window[j], this->slide_window[j_1], this->slide_window[j_3] ) ]++; - this->a_bucket[ fast_b_mapping(178, this->slide_window[j], this->slide_window[j_2], this->slide_window[j_3] ) ]++; - this->a_bucket[ fast_b_mapping(166, this->slide_window[j], this->slide_window[j_2], this->slide_window[j_4] ) ]++; - this->a_bucket[ fast_b_mapping(84, this->slide_window[j], this->slide_window[j_1], this->slide_window[j_4] ) ]++; - this->a_bucket[ fast_b_mapping(230, this->slide_window[j], this->slide_window[j_3], this->slide_window[j_4] ) ]++; - i++; - fed_len++; - j=RNG_IDX(j+1); - } + int j = (int)(fed_len % RNG_SIZE); + + unsigned int start_i=0; + if ( fed_len < SLIDING_WND_SIZE_M1 ) { + int extra = SLIDING_WND_SIZE_M1 - fed_len; + start_i = extra; + j = (j + extra) % RNG_SIZE; + } + for( unsigned int i=start_i; i= 5 bytes + if ((i >= 4) && (i+5 < len)) { + unsigned char a0 = data[i-4]; + unsigned char a1 = data[i-3]; + unsigned char a2 = data[i-2]; + unsigned char a3 = data[i-1]; + unsigned char a4 = data[i]; + unsigned char a5 = data[i+1]; + unsigned char a6 = data[i+2]; + unsigned char a7 = data[i+3]; + unsigned char a8 = data[i+4]; + + a_bucket[ fast_b_mapping(49, a4, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(12, a4, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(178, a4, a2, a1 ) ]++; + a_bucket[ fast_b_mapping(166, a4, a2, a0 ) ]++; + a_bucket[ fast_b_mapping(84, a4, a3, a0 ) ]++; + a_bucket[ fast_b_mapping(230, a4, a1, a0 ) ]++; + + a_bucket[ fast_b_mapping(49, a5, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(12, a5, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(178, a5, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(166, a5, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(84, a5, a4, a1 ) ]++; + a_bucket[ fast_b_mapping(230, a5, a2, a1 ) ]++; + + a_bucket[ fast_b_mapping(49, a6, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(12, a6, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(178, a6, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(166, a6, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(84, a6, a5, a2 ) ]++; + a_bucket[ fast_b_mapping(230, a6, a3, a2 ) ]++; + + a_bucket[ fast_b_mapping(49, a7, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(12, a7, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(178, a7, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(166, a7, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(84, a7, a6, a3 ) ]++; + a_bucket[ fast_b_mapping(230, a7, a4, a3 ) ]++; + + a_bucket[ fast_b_mapping(49, a8, a7, a6 ) ]++; + a_bucket[ fast_b_mapping(12, a8, a7, a5 ) ]++; + a_bucket[ fast_b_mapping(178, a8, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(166, a8, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(84, a8, a7, a4 ) ]++; + a_bucket[ fast_b_mapping(230, a8, a5, a4 ) ]++; + + i=i+5; + j=RNG_IDX(j+5); + } else { + slide_window[j] = data[i]; + int j_1 = RNG_IDX(j-1); if (i >= 1) { slide_window[j_1] = data[i-1]; } + int j_2 = RNG_IDX(j-2); if (i >= 2) { slide_window[j_2] = data[i-2]; } + int j_3 = RNG_IDX(j-3); if (i >= 3) { slide_window[j_3] = data[i-3]; } + int j_4 = RNG_IDX(j-4); if (i >= 4) { slide_window[j_4] = data[i-4]; } + + a_bucket[ fast_b_mapping(49, slide_window[j], slide_window[j_1], slide_window[j_2] ) ]++; + a_bucket[ fast_b_mapping(12, slide_window[j], slide_window[j_1], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(178, slide_window[j], slide_window[j_2], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(166, slide_window[j], slide_window[j_2], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(84, slide_window[j], slide_window[j_1], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(230, slide_window[j], slide_window[j_3], slide_window[j_4] ) ]++; + i++; + j=RNG_IDX(j+1); + } + } +} + +static void raw_fast_update5_nochecksum( struct raw_args *ra ) +{ + const unsigned char* data = ra->data; + unsigned int len = ra->len; + // outputs + unsigned int *a_bucket = ra->bucket; + unsigned char *slide_window = ra->slide_window; + + int j = (int)(ra->fed_len % RNG_SIZE); + + unsigned int start_i=0; + if ( ra->fed_len < SLIDING_WND_SIZE_M1 ) { + int extra = SLIDING_WND_SIZE_M1 - ra->fed_len; + start_i = extra; + j = (j + extra) % RNG_SIZE; + } + for( unsigned int i=start_i; i= 5 bytes + if ((i >= 4) && (i+5 < len)) { + unsigned char a0 = data[i-4]; + unsigned char a1 = data[i-3]; + unsigned char a2 = data[i-2]; + unsigned char a3 = data[i-1]; + unsigned char a4 = data[i]; + unsigned char a5 = data[i+1]; + unsigned char a6 = data[i+2]; + unsigned char a7 = data[i+3]; + unsigned char a8 = data[i+4]; + + a_bucket[ fast_b_mapping(49, a4, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(12, a4, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(178, a4, a2, a1 ) ]++; + a_bucket[ fast_b_mapping(166, a4, a2, a0 ) ]++; + a_bucket[ fast_b_mapping(84, a4, a3, a0 ) ]++; + a_bucket[ fast_b_mapping(230, a4, a1, a0 ) ]++; + + a_bucket[ fast_b_mapping(49, a5, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(12, a5, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(178, a5, a3, a2 ) ]++; + a_bucket[ fast_b_mapping(166, a5, a3, a1 ) ]++; + a_bucket[ fast_b_mapping(84, a5, a4, a1 ) ]++; + a_bucket[ fast_b_mapping(230, a5, a2, a1 ) ]++; + + a_bucket[ fast_b_mapping(49, a6, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(12, a6, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(178, a6, a4, a3 ) ]++; + a_bucket[ fast_b_mapping(166, a6, a4, a2 ) ]++; + a_bucket[ fast_b_mapping(84, a6, a5, a2 ) ]++; + a_bucket[ fast_b_mapping(230, a6, a3, a2 ) ]++; + + a_bucket[ fast_b_mapping(49, a7, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(12, a7, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(178, a7, a5, a4 ) ]++; + a_bucket[ fast_b_mapping(166, a7, a5, a3 ) ]++; + a_bucket[ fast_b_mapping(84, a7, a6, a3 ) ]++; + a_bucket[ fast_b_mapping(230, a7, a4, a3 ) ]++; + + a_bucket[ fast_b_mapping(49, a8, a7, a6 ) ]++; + a_bucket[ fast_b_mapping(12, a8, a7, a5 ) ]++; + a_bucket[ fast_b_mapping(178, a8, a6, a5 ) ]++; + a_bucket[ fast_b_mapping(166, a8, a6, a4 ) ]++; + a_bucket[ fast_b_mapping(84, a8, a7, a4 ) ]++; + a_bucket[ fast_b_mapping(230, a8, a5, a4 ) ]++; + + i=i+5; + j=RNG_IDX(j+5); } else { + slide_window[j] = data[i]; + int j_1 = RNG_IDX(j-1); if (i >= 1) { slide_window[j_1] = data[i-1]; } + int j_2 = RNG_IDX(j-2); if (i >= 2) { slide_window[j_2] = data[i-2]; } + int j_3 = RNG_IDX(j-3); if (i >= 3) { slide_window[j_3] = data[i-3]; } + int j_4 = RNG_IDX(j-4); if (i >= 4) { slide_window[j_4] = data[i-4]; } + + a_bucket[ fast_b_mapping(49, slide_window[j], slide_window[j_1], slide_window[j_2] ) ]++; + a_bucket[ fast_b_mapping(12, slide_window[j], slide_window[j_1], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(178, slide_window[j], slide_window[j_2], slide_window[j_3] ) ]++; + a_bucket[ fast_b_mapping(166, slide_window[j], slide_window[j_2], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(84, slide_window[j], slide_window[j_1], slide_window[j_4] ) ]++; + a_bucket[ fast_b_mapping(230, slide_window[j], slide_window[j_3], slide_window[j_4] ) ]++; i++; - fed_len++; j=RNG_IDX(j+1); } } - this->lsh_bin.checksum[0] = checksum; - this->data_len += len; } ///////////////////////////////////////////////////////////////////////////// +// fc_cons_option - a bitfield +// 0 default +// 1 force (now the default) +// 2 conservative +// 4 do not delete a_bucket ///////////////////////////////////////////////////////////////////////////// /* to signal the class there is no more data to be added */ @@ -402,12 +723,12 @@ void TlshImpl::final(int fc_cons_option) return; } // incoming data must more than or equal to MIN_DATA_LENGTH bytes - if ((fc_cons_option <= 1) && (this->data_len < MIN_DATA_LENGTH)) { + if (((fc_cons_option & TLSH_OPTION_CONSERVATIVE) == 0) && (this->data_len < MIN_DATA_LENGTH)) { // this->lsh_code be empty delete [] this->a_bucket; this->a_bucket = NULL; return; } - if ((fc_cons_option == 2) && (this->data_len < MIN_CONSERVATIVE_DATA_LENGTH)) { + if ((fc_cons_option & TLSH_OPTION_CONSERVATIVE) && (this->data_len < MIN_CONSERVATIVE_DATA_LENGTH)) { // this->lsh_code be empty delete [] this->a_bucket; this->a_bucket = NULL; return; @@ -454,13 +775,15 @@ void TlshImpl::final(int fc_cons_option) h += 2 << (j*2); } else if( q1 < k ) { h += 1 << (j*2); - } + } } this->lsh_bin.tmp_code[i] = h; } - //Done with a_bucket so deallocate - delete [] this->a_bucket; this->a_bucket = NULL; + if ((fc_cons_option & TLSH_OPTION_KEEP_BUCKET) == 0) { + //Done with a_bucket so deallocate + delete [] this->a_bucket; this->a_bucket = NULL; + } this->lsh_bin.Lvalue = l_capturing(this->data_len); this->lsh_bin.Q.QR.Q1ratio = (unsigned int) ((float)(q1*100)/(float) q3) % 16; @@ -653,6 +976,12 @@ unsigned char bv; } return(p4); } +int TlshImpl::HistogramCount(int bucket) +{ + if (this->a_bucket == NULL) + return(-1); + return(this->a_bucket[EFF_BUCKETS - 1 - bucket]); +} int TlshImpl::totalDiff(const TlshImpl& other, bool len_diff) const { diff --git a/tools/thirdparty/tlsh/tlsh_impl.h b/tools/thirdparty/tlsh/tlsh_impl.h index c5a006872c29..59ae8b16dcb1 100644 --- a/tools/thirdparty/tlsh/tlsh_impl.h +++ b/tools/thirdparty/tlsh/tlsh_impl.h @@ -56,9 +56,9 @@ */ #if defined WINDOWS || defined MINGW -#include "win_version.h" +#include "tlsh_win_version.h" #else -#include "version.h" +#include "tlsh_version.h" #endif #ifndef HEADER_TLSH_IMPL_H @@ -113,8 +113,8 @@ class TlshImpl TlshImpl(); ~TlshImpl(); public: - void update(const unsigned char* data, unsigned int len); - void fast_update(const unsigned char* data, unsigned int len); + void update(const unsigned char* data, unsigned int len, int tlsh_option); + void fast_update5(const unsigned char* data, unsigned int len, int tlsh_option); void final(int fc_cons_option = 0); void reset(); const char* hash(int showvers) const; @@ -126,6 +126,7 @@ class TlshImpl int Q2ratio(); int Checksum(int k); int BucketValue(int bucket); + int HistogramCount(int bucket); int fromTlshStr(const char* str); bool isValid() const { return lsh_code_valid; } diff --git a/tools/thirdparty/tlsh/tlsh_util.cpp b/tools/thirdparty/tlsh/tlsh_util.cpp index 5214d00bdd78..3ec77ce12daa 100644 --- a/tools/thirdparty/tlsh/tlsh_util.cpp +++ b/tools/thirdparty/tlsh/tlsh_util.cpp @@ -59,7 +59,7 @@ * Util definition and functions */ -//#include +#include // Compile and run gen_arr2.cpp to generate bit_pairs_diff_table static unsigned char bit_pairs_diff_table[][256] = { @@ -4872,7 +4872,7 @@ unsigned int topval[170] = { 4224281216 }; -//#include +#include unsigned char l_capturing(unsigned int len) { diff --git a/tools/thirdparty/tlsh/win_version.h b/tools/thirdparty/tlsh/tlsh_win_version.h similarity index 92% rename from tools/thirdparty/tlsh/win_version.h rename to tools/thirdparty/tlsh/tlsh_win_version.h index 9f04b0398435..3c1c3fba9a55 100644 --- a/tools/thirdparty/tlsh/win_version.h +++ b/tools/thirdparty/tlsh/tlsh_win_version.h @@ -6,8 +6,8 @@ ****************************************************/ #define VERSION_MAJOR 4 -#define VERSION_MINOR 5 -#define VERSION_PATCH 0 +#define VERSION_MINOR 11 +#define VERSION_PATCH 2 #define TLSH_HASH "compact hash" #define TLSH_CHECKSUM "1 byte checksum"