hashers now use thread local arena

Instead of writing directly to file_hashes.txt, hash_workers now are
using a local arena, writing everything once at the end

using #pragma once to ensure that a given header file is included only
once in a single compilation unit
This commit is contained in:
2026-03-08 10:46:05 +01:00
parent ee02b83094
commit dd0797df79
7 changed files with 177 additions and 73 deletions

View File

@@ -1,39 +1,10 @@
#pragma once
#pragma once // ensure that a given header file is included only once in a
// single compilation unit
#include <immintrin.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(_WIN32) || defined(_WIN64)
#define PLATFORM_WINDOWS 1
#include <aclapi.h>
#include <fcntl.h>
#include <io.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <windows.h>
#define strdup _strdup
#else
#include <dirent.h>
#include <fcntl.h>
#include <pthread.h>
#include <pwd.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
#define XXH_VECTOR \
XXH_AVX2 // not recommanded to compile with gcc see xxhash.h line 4082
// Must compile with /arch:AVX2 in clang-cl or -mavx2 in clang/gcc
#define XXH_INLINE_ALL
#include "xxhash.c"
#include "xxhash.h"
#include "arena.h"
#include "base.h"
#include "arena.c"
// ----------------------------- Config -------------------------------------
#define FILE_HASHES_TXT "file_hashes.txt"
#define HASH_STRLEN 33 // 128-bit hex (32 chars) + null
@@ -124,6 +95,11 @@ typedef struct {
static MPMCQueue g_file_queue;
typedef struct {
MPMCQueue *queue;
mem_arena *arena;
} WorkerContext;
/* Scan folders */
typedef struct DirQueue DirQueue;