From 2fc9bf31b0189f4c6e8be7be59b59335fee38c46 Mon Sep 17 00:00:00 2001 From: amir Date: Sun, 8 Mar 2026 10:59:13 +0100 Subject: [PATCH] Making the hashing buffer reusable instead of malloc every file --- binaries/changelog.txt | 6 +++--- platform_windows.c | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/binaries/changelog.txt b/binaries/changelog.txt index 9c3bd29..26928d5 100644 --- a/binaries/changelog.txt +++ b/binaries/changelog.txt @@ -7,7 +7,6 @@ v2.0: Multi threaded scan v2.1: Uses AVX2 instead of SSE2 v3.0: Simple mutex/critical section based MPMC queue -reusable hashing buffer v3.1: Lock free MPMC queue Vyukov-style @@ -22,5 +21,6 @@ Reorder helper functions v3.4: Rewriting hash_worker() to export file_hashes.txt v4.0: Instead of writing directly to file_hashes.txt, hash_workers now are using a local arena, writing everything once at the end -using #pragma once to ensure that a given header file is included only once in a single compilation unit -forcing xxhash to use the stack instead of the heap +Using #pragma once to ensure that a given header file is included only once in a single compilation unit +Forcing xxhash to use the stack instead of the heap +Making the hashing buffer reusable instead of malloc every file diff --git a/platform_windows.c b/platform_windows.c index 290d4f4..b899e75 100644 --- a/platform_windows.c +++ b/platform_windows.c @@ -375,7 +375,7 @@ void scan_folder_windows_parallel(const char *base, DirQueue *q) { } // ----------------------------- Hashing helpers ----------------------------- -static void xxh3_hash_file_stream(const char *path, char *out_hex) { +static void xxh3_hash_file_stream(const char *path, char *out_hex, BYTE *buf) { // compute XXH3_128 over file. POSIX and Windows use standard reads in this // helper. // On Windows try to use overlapped synchronous chunked reads for higher @@ -391,7 +391,7 @@ static void xxh3_hash_file_stream(const char *path, char *out_hex) { XXH3_state_t state; XXH3_128bits_reset(&state); - BYTE *buf = (BYTE *)malloc(READ_BLOCK); + // BYTE *buf = (BYTE *)malloc(READ_BLOCK); DWORD read = 0; BOOL ok; while (ReadFile(hFile, buf, READ_BLOCK, &read, NULL) && read > 0) { @@ -400,7 +400,7 @@ static void xxh3_hash_file_stream(const char *path, char *out_hex) { } h = XXH3_128bits_digest(&state); CloseHandle(hFile); - free(buf); + // free(buf); snprintf(out_hex, HASH_STRLEN, "%016llx%016llx", (unsigned long long)h.high64, (unsigned long long)h.low64); } @@ -411,6 +411,7 @@ static DWORD WINAPI hash_worker(LPVOID arg) { WorkerContext *ctx = (WorkerContext *)arg; MPMCQueue *q = ctx->queue; mem_arena *local_arena = ctx->arena; + BYTE *buf = (BYTE *)malloc(READ_BLOCK); for (;;) { FileEntry *fe = mpmc_pop(q); @@ -418,7 +419,7 @@ static DWORD WINAPI hash_worker(LPVOID arg) { break; char hash[HASH_STRLEN]; - xxh3_hash_file_stream(fe->path, hash); + xxh3_hash_file_stream(fe->path, hash, buf); char created[32], modified[32]; format_time(fe->created_time, created, sizeof(created)); @@ -440,6 +441,7 @@ static DWORD WINAPI hash_worker(LPVOID arg) { free(fe->path); free(fe); } + free(buf); return 0; }