Linux porting

Porting to linux
Reorganising the code
Improving the scan function

.gitignore

@@ -1,3 +1,7 @@
+file_hasher.pdb
+file_hasher.ilk
+file_hasher.rdi
 file_hasher.exe
 file_hashes.txt
 file_list.txt
+temp_code.c

README.md

@@ -1,3 +1,24 @@
 # filehasher
 
-Collects some metadata and hashes files.
+Collects some metadata and hashes files.
+
+## Building:  
+### Windows:  
+#### Release:  
+clang-cl /O3 file_hasher.c xxh_x86dispatch.c advapi32.lib  
+clang -O3 file_hasher.c xxh_x86dispatch.c -ladvapi32 -o file_hasher  
+gcc -O3 file_hasher.c xxh_x86dispatch.c -ladvapi32 -o file_hasher  
+
+#### Debug:  
+clang-cl /Zi /Od file_hasher.c xxh_x86dispatch.c advapi32.lib  
+clang -g -O0 file_hasher.c xxh_x86dispatch.c -ladvapi32 -o file_hasher  
+gcc -g -O0 file_hasher.c xxh_x86dispatch.c -ladvapi32 -o file_hasher  
+
+### Linux:  
+#### Release:  
+clang -O3 -pthread file_hasher.c xxh_x86dispatch.c -o file_hasher  
+gcc -O3 -pthread file_hasher.c xxh_x86dispatch.c -o file_hasher  
+
+#### Debug:  
+clang -g -O0 -pthread file_hasher.c xxh_x86dispatch.c -o file_hasher  
+gcc -g -O0 -pthread file_hasher.c xxh_x86dispatch.c -o file_hasher  

arena.c

@@ -1,7 +1,6 @@
-#include "base.h"
+#pragma once
 
 #include "arena.h"
-
 /* ============================================================
    Helper functions
    ============================================================ */
@@ -132,9 +131,6 @@ mem_arena *arena_create(arena_params *params) { // mk create
 
   u32 pagesize = arena_pagesize();
 
-  u64 align = params->align ? params->align : ARENA_ALIGN;
-  ASSERT((align & (align - 1)) == 0);
-
   u64 reserve_size = ALIGN_UP_POW2(params->reserve_size, pagesize);
   u64 commit_size =
       params->commit_size ? ALIGN_UP_POW2(params->commit_size, pagesize) : 0;
@@ -181,7 +177,7 @@ mem_arena *arena_create(arena_params *params) { // mk create
   arena->commit_size = commit_size;
   arena->commit_pos = commit_size;
 
-  arena->align = align;
+  arena->align = params->align;
   arena->push_size = 0;
 
   arena->allow_free_list = params->allow_free_list;
@@ -360,7 +356,8 @@ void *arena_push(mem_arena **arena_ptr, u64 size, bool zero) { // mk push
   u64 local_pre = ALIGN_UP_POW2(local_pos, selected->align);
   u64 local_post = local_pre + size;
 
-  if (local_post > selected->reserve_size) {
+  if (local_post > selected->reserve_size -
+                       ALIGN_UP_POW2(sizeof(mem_arena), selected->align)) {
 
     if (arena->allow_free_list && arena->push_size == 0) {
       u64 tail_start = selected->pos;
@@ -605,8 +602,8 @@ void *arena_swapback_pop(mem_arena **arena_ptr, u64 index) { // mk swapback
   mem_arena *owner = arena_block_from_index(arena, index);
 
   if (!owner) {
-      fprintf(stderr, "ERROR: Swapback pop failed, index out of range");
-      return NULL;
+    fprintf(stderr, "ERROR: Swapback pop failed, index out of range");
+    return NULL;
   }
   u8 *owner_base = (u8 *)owner + ALIGN_UP_POW2(sizeof(mem_arena), owner->align);
   u8 *arena_base = (u8 *)arena + ALIGN_UP_POW2(sizeof(mem_arena), arena->align);
@@ -823,70 +820,3 @@ mem_arena_temp arena_scratch_get(mem_arena **conflicts, u32 num_conflicts) {
 }
 
 void arena_scratch_release(mem_arena_temp scratch) { arena_temp_end(scratch); }
-
-#if defined(_WIN32) || defined(_WIN64)
-
-#include <windows.h>
-
-u32 plat_get_pagesize(void) {
-  SYSTEM_INFO sysinfo = {0};
-  GetSystemInfo(&sysinfo);
-
-  return sysinfo.dwPageSize;
-}
-
-void *plat_mem_reserve(u64 size) {
-  return VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_READWRITE);
-}
-
-b32 plat_mem_commit(void *ptr, u64 size) {
-  void *ret = VirtualAlloc(ptr, size, MEM_COMMIT, PAGE_READWRITE);
-  return ret != NULL;
-}
-
-b32 plat_mem_decommit(void *ptr, u64 size) {
-  return VirtualFree(ptr, size, MEM_DECOMMIT);
-}
-
-b32 plat_mem_release(void *ptr, u64 size) {
-  return VirtualFree(ptr, size, MEM_RELEASE);
-}
-
-#elif defined(__linux__)
-
-#ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE
-#endif
-
-#include <sys/mman.h>
-#include <unistd.h>
-
-u32 plat_get_pagesize(void) { return (u32)sysconf(_SC_PAGESIZE); }
-
-void *plat_mem_reserve(u64 size) {
-  void *out = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-  if (out == MAP_FAILED) {
-    return NULL;
-  }
-  return out;
-}
-
-b32 plat_mem_commit(void *ptr, u64 size) {
-  i32 ret = mprotect(ptr, size, PROT_READ | PROT_WRITE);
-  return ret == 0;
-}
-
-b32 plat_mem_decommit(void *ptr, u64 size) {
-  i32 ret = mprotect(ptr, size, PROT_NONE);
-  if (ret != 0)
-    return false;
-  ret = madvise(ptr, size, MADV_DONTNEED);
-  return ret == 0;
-}
-
-b32 plat_mem_release(void *ptr, u64 size) {
-  i32 ret = munmap(ptr, size);
-  return ret == 0;
-}
-
-#endif

arena.h

@@ -1,47 +1,7 @@
-#ifndef BASE_ARENA_H
-#define BASE_ARENA_H
+#pragma once
 
 #include "base.h"
 
-// #define _CRT_SECURE_NO_WARNINGS
-//
-// #include <assert.h>
-// #include <stdbool.h>
-// #include <stdint.h>
-// #include <stdio.h>
-// #include <string.h>
-//
-// /* ------------------------------------------------------------
-//    Base types
-//    ------------------------------------------------------------ */
-//
-// typedef uint8_t u8;
-// typedef uint32_t u32;
-// typedef uint64_t u64;
-// typedef int32_t i32;
-// typedef int b32;
-//
-// /* ------------------------------------------------------------
-//    Size helpers
-//    ------------------------------------------------------------ */
-//
-// #define KiB(x) ((u64)(x) * 1024ULL)
-// #define MiB(x) (KiB(x) * 1024ULL)
-//
-// /* ------------------------------------------------------------
-//    Alignment helpers
-//    ------------------------------------------------------------ */
-//
-// #define ALIGN_UP_POW2(x, a) (((x) + ((a) - 1)) & ~((a) - 1))
-//
-// /* ------------------------------------------------------------
-//    Assert
-//    ------------------------------------------------------------ */
-//
-// #ifndef ASSERT
-// #define ASSERT(x) assert(x)
-// #endif
-//
 /*
 ===============================================================================
                                ARENA USAGE GUIDE
@@ -77,7 +37,8 @@ Each block contains:
 - prev/next: Links to neighboring blocks
 - the arena pointer points to the current block
 
-The arena allocates from the current block where the global position is or from the free list.
+The arena allocates from the current block where the global position is or from
+the free list.
 
 Blocks form a single logical address space:
     global_offset = pos
@@ -294,7 +255,8 @@ typedef enum arena_commit_policy {
 typedef struct arena_params {
   u64 reserve_size; // size of one arena block
   u64 commit_size;  // initial commit size
-  u64 align;        // allocation alignment (0 = default)
+  u64 align; // allocation alignment, 0 to disable and ARENA_ALIGN to align
+             // according to architecture
 
   // Element size rules:
   // - stack mode        : push_size > 0 (mandatory)
@@ -335,10 +297,6 @@ typedef struct mem_arena {
 
   // configuration
   u64 align;
-  // Element size:
-  // - stack mode        : fixed > 0
-  // - pointer fixed     : fixed > 0
-  // - pointer variable  : 0
   u64 push_size;
 
   // Pointer mode only
@@ -421,12 +379,3 @@ void arena_scratch_release(mem_arena_temp scratch);
 #define ARENA_PUSH_NZ(arena, size) arena_push((arena), (size), false)
 
 #define arena_pop(arena_ptr) arena_pop_to((arena_ptr), 1)
-
-u32 plat_get_pagesize(void);
-
-void *plat_mem_reserve(u64 size);
-b32 plat_mem_commit(void *ptr, u64 size);
-b32 plat_mem_decommit(void *ptr, u64 size);
-b32 plat_mem_release(void *ptr, u64 size);
-
-#endif // BASE_ARENA_H

arena_base.h

@@ -0,0 +1,129 @@
+#pragma once
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+/* ------------------------------------------------------------
+   Base types
+   ------------------------------------------------------------ */
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t i8;
+typedef int16_t i16;
+typedef int32_t i32;
+typedef int64_t i64;
+
+typedef i8 b8;
+typedef int b32;
+
+typedef float f32;
+typedef double f64;
+/* ------------------------------------------------------------
+   Size helpers
+   ------------------------------------------------------------ */
+
+#define KiB(x) ((u64)(x) * 1024ULL)
+#define MiB(x) (KiB(x) * 1024ULL)
+
+/* ------------------------------------------------------------
+   Min / Max helpers
+   ------------------------------------------------------------ */
+
+#ifndef MIN
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef MAX
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#endif
+
+/* ------------------------------------------------------------
+   Alignment helpers
+   ------------------------------------------------------------ */
+
+#define ALIGN_UP_POW2(x, a) (((x) + ((a) - 1)) & ~((a) - 1))
+
+/* ------------------------------------------------------------
+   Assert
+   ------------------------------------------------------------ */
+
+#ifndef ASSERT
+#define ASSERT(x) assert(x)
+#endif
+
+/* ------------------------------------------------------------
+   Some helper functions
+   ------------------------------------------------------------ */
+
+#if defined(_WIN32) || defined(_WIN64)
+
+#include <Windows.h>
+
+static u32 plat_get_pagesize(void) {
+  SYSTEM_INFO sysinfo = {0};
+  GetSystemInfo(&sysinfo);
+
+  return sysinfo.dwPageSize;
+}
+
+static void *plat_mem_reserve(u64 size) {
+  return VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_READWRITE);
+}
+
+static b32 plat_mem_commit(void *ptr, u64 size) {
+  void *ret = VirtualAlloc(ptr, size, MEM_COMMIT, PAGE_READWRITE);
+  return ret != NULL;
+}
+
+static b32 plat_mem_decommit(void *ptr, u64 size) {
+  return VirtualFree(ptr, size, MEM_DECOMMIT);
+}
+
+static b32 plat_mem_release(void *ptr, u64 size) {
+  return VirtualFree(ptr, size, MEM_RELEASE);
+}
+
+#elif defined(__linux__)
+
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+
+#include <sys/mman.h>
+#include <unistd.h>
+
+static u32 plat_get_pagesize(void) { return (u32)sysconf(_SC_PAGESIZE); }
+
+static void *plat_mem_reserve(u64 size) {
+  void *out = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (out == MAP_FAILED) {
+    return NULL;
+  }
+  return out;
+}
+
+static b32 plat_mem_commit(void *ptr, u64 size) {
+  i32 ret = mprotect(ptr, size, PROT_READ | PROT_WRITE);
+  return ret == 0;
+}
+
+static b32 plat_mem_decommit(void *ptr, u64 size) {
+  i32 ret = mprotect(ptr, size, PROT_NONE);
+  if (ret != 0)
+    return false;
+  ret = madvise(ptr, size, MADV_DONTNEED);
+  return ret == 0;
+}
+
+static b32 plat_mem_release(void *ptr, u64 size) {
+  i32 ret = munmap(ptr, size);
+  return ret == 0;
+}
+
+#endif

base.h

@@ -1,11 +1,33 @@
-#ifndef BASE_H
-#define BASE_H
+#pragma once
 
 #include <assert.h>
+#include <immintrin.h>
+#include <stdatomic.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <time.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#define PLATFORM_WINDOWS 1
+#include <aclapi.h>
+#include <fcntl.h>
+#include <io.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <windows.h>
+
+#define strdup _strdup
+#else
+#include <dirent.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
 
 /* ------------------------------------------------------------
    Base types
@@ -25,12 +47,14 @@ typedef int b32;
 
 typedef float f32;
 typedef double f64;
+
 /* ------------------------------------------------------------
    Size helpers
    ------------------------------------------------------------ */
 
 #define KiB(x) ((u64)(x) * 1024ULL)
 #define MiB(x) (KiB(x) * 1024ULL)
+#define GiB(x) (MiB(x) * 1024ULL)
 
 /* ------------------------------------------------------------
    Min / Max helpers
@@ -48,7 +72,7 @@ typedef double f64;
    Alignment helpers
    ------------------------------------------------------------ */
 
-#define ALIGN_UP_POW2(x, a) (((x) + ((a) - 1)) & ~((a) - 1))
+#define ALIGN_UP_POW2(x, a) ((a) ? (((x) + ((a) - 1)) & ~((a) - 1)) : (x))
 
 /* ------------------------------------------------------------
    Assert
@@ -58,4 +82,139 @@ typedef double f64;
 #define ASSERT(x) assert(x)
 #endif
 
-#endif // Base.h
+#define NDEBUG // Comment to enable asserts
+
+/* ------------------------------------------------------------
+   Some helper functions
+   ------------------------------------------------------------ */
+
+#if defined(_WIN32) || defined(_WIN64)
+
+// Memory allocation
+
+static u32 plat_get_pagesize(void) {
+  SYSTEM_INFO sysinfo = {0};
+  GetSystemInfo(&sysinfo);
+
+  return sysinfo.dwPageSize;
+}
+
+static void *plat_mem_reserve(u64 size) {
+  return VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_READWRITE);
+}
+
+static b32 plat_mem_commit(void *ptr, u64 size) {
+  void *ret = VirtualAlloc(ptr, size, MEM_COMMIT, PAGE_READWRITE);
+  return ret != NULL;
+}
+
+static b32 plat_mem_decommit(void *ptr, u64 size) {
+  return VirtualFree(ptr, size, MEM_DECOMMIT);
+}
+
+static b32 plat_mem_release(void *ptr, u64 size) {
+  return VirtualFree(ptr, size, MEM_RELEASE);
+}
+
+// Semaphores
+typedef struct plat_sem {
+  HANDLE handle;
+} plat_sem;
+
+static b32 plat_sem_init(plat_sem *s, u32 initial) {
+  s->handle = CreateSemaphore(NULL, initial, LONG_MAX, NULL);
+  return s->handle != NULL;
+}
+
+static void plat_sem_wait(plat_sem *s) {
+  WaitForSingleObject(s->handle, INFINITE);
+}
+
+static b32 plat_sem_trywait(HANDLE sem) {
+  DWORD r = WaitForSingleObject(sem, 0);
+  return r == WAIT_OBJECT_0;
+}
+
+static void plat_sem_post(plat_sem *s, u32 count) {
+  ReleaseSemaphore(s->handle, count, NULL);
+}
+
+static void plat_sem_destroy(plat_sem *s) {
+  if (s->handle) {
+    CloseHandle(s->handle);
+    s->handle = NULL;
+  }
+}
+
+// Sleep
+static void sleep_ms(int ms) { Sleep(ms); }
+
+#elif defined(__linux__)
+
+// Memory allocation
+
+#ifndef _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE
+#endif
+
+#include <sys/mman.h>
+#include <unistd.h>
+
+static u32 plat_get_pagesize(void) { return (u32)sysconf(_SC_PAGESIZE); }
+
+static void *plat_mem_reserve(u64 size) {
+  void *out = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (out == MAP_FAILED) {
+    return NULL;
+  }
+  return out;
+}
+
+static b32 plat_mem_commit(void *ptr, u64 size) {
+  i32 ret = mprotect(ptr, size, PROT_READ | PROT_WRITE);
+  return ret == 0;
+}
+
+static b32 plat_mem_decommit(void *ptr, u64 size) {
+  i32 ret = mprotect(ptr, size, PROT_NONE);
+  if (ret != 0)
+    return false;
+  ret = madvise(ptr, size, MADV_DONTNEED);
+  return ret == 0;
+}
+
+static b32 plat_mem_release(void *ptr, u64 size) {
+  i32 ret = munmap(ptr, size);
+  return ret == 0;
+}
+
+// Semaphores
+#include <semaphore.h>
+
+typedef struct plat_sem {
+  sem_t sem;
+} plat_sem;
+
+static b32 plat_sem_init(plat_sem *s, u32 initial) {
+  return sem_init(&s->sem, 0, initial) == 0;
+}
+
+static void plat_sem_wait(plat_sem *s) {
+  while (sem_wait(&s->sem) == -1 && errno == EINTR) {
+  }
+}
+
+static b32 plat_sem_trywait(sem_t *sem) { return sem_trywait(sem) == 0; }
+
+static void plat_sem_post(plat_sem *s, u32 count) {
+  for (u32 i = 0; i < count; i++) {
+    sem_post(&s->sem);
+  }
+}
+
+static void plat_sem_destroy(plat_sem *s) { sem_destroy(&s->sem); }
+
+// Sleep
+static void sleep_ms(int ms) { usleep(ms * 1000); }
+
+#endif

binaries/changelog.txt

@@ -0,0 +1,51 @@
+V1.0: Recursive scan
+
+v1.1: Collects more metadata
+
+v2.0: Multi threaded scan
+
+v2.1: Uses AVX2 instead of SSE2
+
+v3.0: Simple mutex/critical section based MPMC queue
+
+v3.1: Lock free MPMC queue Vyukov-style
+
+v3.2: Making the lock free MPMC queue growable
+Add padding to avoir false sharing
+Add sleep() and SwitchToThread() to limit spinning
+
+v3.3: Fix bug slots used before initialization,compare and swap is protecting updating committed, but it is not protecting the memory initialization. Adding atomic_flag commit_lock to protect against that
+Fix bug multiple threads committing at the same time, fixed by using atomic_flag commit_lock and re-checking committed after acquiring the lock
+Reorder helper functions
+
+v3.4: Rewriting hash_worker() to export file_hashes.txt
+
+3.5: Instead of writing directly to file_hashes.txt, hash_workers now are using a local arena, writing everything once at the end
+Using #pragma once to ensure that a given header file is included only once in a single compilation unit
+Forcing xxhash to use the stack instead of the heap
+Making the hashing buffer reusable instead of malloc every file
+Implementing a general purpose arena to replace small allocations
+Small improvements of the LF MPMC queue
+Making the LF MPMC queue generic and in a seperate header file
+
+4.0: Implementing a semaphore in the LF MPMC queue to wake up consumers when there is items in the queue instead of spinning (busy waiting) or sleeping, this makes the queue spin only when the slots are transitionning (multiple consumers claiming the same slot)
+
+Making the MPMC queue platform agnostic
+
+Align the MPMC queue to pagesize
+
+Getting file size from FindFirstFileA() instead of CreateFileA(), since we already call FindFirstFileA() and it returns the size there is no need to open/close every file to get it's size
+
+Replacing Malloc and strdup in scan helper function with FileEntry and path arenas
+
+Making the MPMC queue support when producers are consumers at the same time by adding a variable work, mpmc_push_work() that increments work and mpmc_task_done() that decrements work, and if work = 0 calls mpmc_producers_finished() that pushes poinsons to wake up sleeping threads and make them return NULL
+
+Replacing DirQueue, a queue growable with realloc with the MPMC queue
+
+4.1: Using xxhash xxh_x86dispatch to select the best SIMD instruction set at runtime, this dispatcher can not be added in a unity build and we must remove AVX2 or AVX512 compilation flags, link xxh_x86dispatch.c in the compilation command. The compilaiton throws two warnings about function with internal linkage but not defined, they are defined in xxh_x86dispatch.c so it's harmless warnings
+
+Fixing user prompt parsing
+
+4.5: Porting to linux
+Reorganising the code
+Improving the scan function

file_hasher.c

@@ -1,7 +1,223 @@
-#define _CRT_SECURE_NO_WARNINGS
+#include "platform.c"
 
-#if defined(_WIN32) || defined(_WIN64)
-#include "platform_windows.c"
-#else
-#include "platform_posix.c"
-#endif
+// ----------------------------- Main ---------------------------------------
+int main(int argc, char **argv) {
+  char folders[64][MAX_PATHLEN]; // up to 64 input folders
+  int folder_count = 0;
+
+  // -------------------------------
+  // Parse arguments
+  // -------------------------------
+  for (int i = 1; i < argc; ++i) {
+    if (folder_count < 64) {
+      normalize_path(argv[i]);
+      strncpy(folders[folder_count], argv[i], MAX_PATHLEN - 1);
+      folders[folder_count][MAX_PATHLEN - 1] = 0;
+      folder_count++;
+    }
+  }
+
+  // -------------------------------
+  // Ask user if no folders provided
+  // -------------------------------
+  if (folder_count == 0) {
+    printf("Enter folders to process (Enter = current folder): ");
+    fflush(stdout);
+
+    char buf[KiB(32)];
+
+    if (!fgets(buf, sizeof(buf), stdin))
+      return 1;
+
+    buf[strcspn(buf, "\r\n")] = 0;
+
+    if (buf[0] == 0) {
+      strcpy(folders[0], ".");
+      folder_count = 1;
+    } else {
+      folder_count = parse_paths(buf, folders, 64);
+    }
+  }
+
+  // Display selected folders
+  printf("Processing %d folder(s):\n", folder_count);
+  for (int i = 0; i < folder_count; ++i) {
+    printf("  - %s\n", folders[i]);
+  }
+
+  // -------------------------------
+  // Scanning and total timer init
+  // -------------------------------
+  timer_init();
+
+  HiResTimer total_timer;
+  HiResTimer scan_timer;
+
+  timer_start(&total_timer);
+  timer_start(&scan_timer);
+
+  // -------------------------------
+  // Creating a general purpose arena
+  // -------------------------------
+  arena_params params = {
+      .reserve_size = GiB(1),
+      .commit_size = MiB(16),
+      .align = 0,
+      .push_size = 0,
+      .allow_free_list = true,
+      .allow_swapback = false,
+      .growth_policy = ARENA_GROWTH_NORMAL,
+      .commit_policy = ARENA_COMMIT_LAZY,
+      .max_nbre_blocks = 1,
+  };
+
+  mem_arena *gp_arena = arena_create(&params);
+
+  // -------------------------------
+  // Detect hardware threads
+  // -------------------------------
+  // --- Windows: detect PHYSICAL cores (not logical threads) ---
+  size_t hw_threads = platform_physical_cores();
+
+  // Logical threads = CPU cores * 2
+  size_t num_threads = hw_threads * 2;
+
+  printf("Starting thread pool: %zu threads (CPU cores: %zu)\n", num_threads,
+         hw_threads);
+  printf("  Selected instruction set: %s\n", get_xxhash_instruction_set());
+
+  // -------------------------------
+  // Scanning and hashing
+  // -------------------------------
+  MPMCQueue dir_queue;
+  mpmc_init(&dir_queue, MiB(1));
+
+  MPMCQueue file_queue;
+  mpmc_init(&file_queue, MiB(1));
+
+  // Starting hash threads
+  size_t num_hash_threads = num_threads;
+
+  WorkerContext workers[num_hash_threads];
+  Thread *hash_threads =
+      arena_push(&gp_arena, sizeof(Thread) * num_hash_threads, true);
+
+  for (size_t i = 0; i < num_hash_threads; ++i) {
+    workers[i].arena = arena_create(&params);
+    workers[i].file_queue = &file_queue;
+
+    if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker, &workers[i]) !=
+        0) {
+      fprintf(stderr, "Failed to create hash thread %zu\n", i);
+      exit(1);
+    }
+  }
+
+  // Starting progress printing thread
+  Thread progress_thread_handle;
+  if (thread_create(&progress_thread_handle, (ThreadFunc)progress_thread,
+                    NULL) != 0) {
+    fprintf(stderr, "Failed to create progress thread\n");
+    exit(1);
+  }
+
+  // Starting scan threads
+  size_t num_scan_threads = num_threads;
+
+  ScannerContext scanners[num_scan_threads];
+  Thread *scan_threads =
+      arena_push(&gp_arena, sizeof(Thread) * num_scan_threads, true);
+
+  for (size_t i = 0; i < num_scan_threads; i++) {
+    scanners[i].num_threads = num_scan_threads;
+    scanners[i].path_arena = arena_create(&params);
+    scanners[i].meta_arena = arena_create(&params);
+    scanners[i].dir_queue = &dir_queue;
+    scanners[i].file_queue = &file_queue;
+
+    if (thread_create(&scan_threads[i], (ThreadFunc)scan_worker,
+                      &scanners[i]) != 0) {
+      fprintf(stderr, "Failed to create scan thread %zu\n", i);
+      exit(1);
+    }
+  }
+
+  // Initial folder push
+  for (int i = 0; i < folder_count; i++) {
+    size_t len = strlen(folders[i]) + 1;
+    char *path = arena_push(&scanners[0].path_arena, len, false);
+    memcpy(path, folders[i], len);
+    mpmc_push_work(&dir_queue, path);
+  }
+
+  // Stop scan threads
+  thread_wait_multiple(scan_threads, num_scan_threads);
+
+  for (size_t i = 0; i < num_scan_threads; ++i) {
+    thread_close(&scan_threads[i]);
+  }
+
+  mpmc_producers_finished(&file_queue, num_hash_threads);
+
+  atomic_store(&g_scan_done, 1);
+
+  arena_free(&gp_arena, (u8 **)&scan_threads,
+             sizeof(Thread) * num_scan_threads);
+
+  double scan_seconds = timer_elapsed(&scan_timer);
+  size_t total_found = atomic_load(&g_files_found);
+
+  printf("\r%*s\r", 120, ""); // clear_console_line
+  printf("Completed scanning in %.2f seconds, found %zu files\n\n",
+         scan_seconds, total_found);
+
+  // If no files found
+  if (total_found == 0) {
+    printf("No files found.\n");
+    return 0;
+  }
+
+  // Stop hashing threads
+  thread_wait_multiple(hash_threads, num_hash_threads);
+
+  for (size_t i = 0; i < num_hash_threads; ++i) {
+    thread_close(&hash_threads[i]);
+  }
+
+  arena_free(&gp_arena, (u8 **)&hash_threads,
+             sizeof(Thread) * num_hash_threads);
+
+  // Stop progress printing thread
+  thread_join(&progress_thread_handle);
+  thread_close(&progress_thread_handle);
+
+  // -------------------------------
+  // Export file_hashes.txt
+  // -------------------------------
+
+  FILE *f = fopen(FILE_HASHES_TXT, "wb");
+
+  for (int i = 0; i < num_threads; i++) {
+    mem_arena *arena = workers[i].arena;
+    u8 *arena_base =
+        (u8 *)arena + ALIGN_UP_POW2(sizeof(mem_arena), arena->align);
+    fwrite(arena_base, 1, arena->pos, f);
+  }
+
+  fclose(f);
+
+  // -------------------------------
+  // Print summary
+  // -------------------------------
+  double total_seconds = timer_elapsed(&total_timer);
+
+  printf("Completed hashing %zu files\n", total_found);
+
+  uint64_t total_bytes = (uint64_t)atomic_load(&g_bytes_processed);
+  double total_mb = (double)total_bytes / (1024.0 * 1024.0);
+  double avg_mbps = total_mb / total_seconds;
+  printf("Total: %.2f MB, Average: %.2f MB/s\n", total_mb, avg_mbps);
+  printf("  Total time  : %.2f seconds\n\n", total_seconds);
+
+  return 0;
+}

lf_mpmc.h

@@ -0,0 +1,326 @@
+#pragma once
+
+#include "base.h"
+
+#define CACHELINE 64
+
+#if defined(_MSC_VER)
+#define CACHE_ALIGN __declspec(align(CACHELINE))
+#else
+#define CACHE_ALIGN __attribute__((aligned(CACHELINE)))
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+#define likely(x) __builtin_expect((x), 1)
+#define unlikely(x) __builtin_expect((x), 0)
+#else
+#define likely(x) (x)
+#define unlikely(x) (x)
+#endif
+
+static void cpu_pause(void) {
+#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__)
+  _mm_pause();
+#endif
+}
+
+typedef struct plat_sem plat_sem;
+
+typedef struct CACHE_ALIGN {
+  atomic_size_t seq;
+  void *data;
+  char pad[64 - sizeof(atomic_size_t) - sizeof(void *)];
+} MPMCSlot;
+
+typedef struct {
+  CACHE_ALIGN atomic_size_t head;
+  CACHE_ALIGN atomic_size_t tail;
+
+  CACHE_ALIGN atomic_size_t work_count;
+
+  size_t capacity;
+  size_t mask;
+
+  atomic_size_t committed;
+  size_t commit_step;
+  atomic_flag commit_lock;
+
+  plat_sem items_sem;
+
+  MPMCSlot *slots;
+} MPMCQueue;
+
+// --------------- functions ----------------
+// static: each translation unit gets its own private copy this will solve the
+// error:  Function defined in a header file; function definitions in header
+// files can lead to ODR violations (multiple definition errors if included in
+// more than one file)
+
+/* ----------------------------------------------------------- */
+/* INIT */
+/* ----------------------------------------------------------- */
+static void mpmc_init(MPMCQueue *q, size_t max_capacity) {
+
+  q->capacity = max_capacity;
+  q->mask = max_capacity - 1;
+
+  u32 pagesize = plat_get_pagesize();
+
+  size_t bytes = ALIGN_UP_POW2(sizeof(MPMCSlot) * max_capacity, pagesize);
+
+  q->slots = (MPMCSlot *)plat_mem_reserve(bytes);
+
+  if (!q->slots) {
+    fprintf(stderr, "VirtualAlloc reserve failed\n");
+    exit(1);
+  }
+
+  u64 commit_bytes = pagesize;
+  commit_bytes = ALIGN_UP_POW2(commit_bytes, pagesize);
+
+  q->commit_step = commit_bytes / sizeof(MPMCSlot);
+
+  atomic_flag_clear(&q->commit_lock);
+
+  q->committed = q->commit_step;
+
+  plat_mem_commit(q->slots, commit_bytes);
+
+  for (size_t i = 0; i < q->committed; i++) {
+    atomic_init(&q->slots[i].seq, i);
+    q->slots[i].data = NULL;
+  }
+
+  atomic_init(&q->head, 0);
+  atomic_init(&q->tail, 0);
+  atomic_init(&q->work_count, 0);
+
+  plat_sem_init(&q->items_sem, 0);
+}
+
+/* ----------------------------------------------------------- */
+/* COMMIT MORE MEMORY */
+/* ----------------------------------------------------------- */
+static void mpmc_commit_more(MPMCQueue *q) {
+
+  if (atomic_flag_test_and_set(&q->commit_lock))
+    return;
+
+  size_t start = atomic_load_explicit(&q->committed, memory_order_acquire);
+  size_t tail = atomic_load_explicit(&q->tail, memory_order_relaxed);
+
+  // another thread already committed enough
+  if (tail < start) {
+    atomic_flag_clear(&q->commit_lock);
+    return;
+  }
+
+  if (start >= q->capacity) {
+    atomic_flag_clear(&q->commit_lock);
+    return;
+  }
+
+  size_t new_commit = start + q->commit_step;
+  if (new_commit > q->capacity)
+    new_commit = q->capacity;
+
+  size_t count = new_commit - start;
+
+  plat_mem_commit(&q->slots[start], count * sizeof(MPMCSlot));
+
+  for (size_t i = start; i < new_commit; i++) {
+    atomic_init(&q->slots[i].seq, i);
+    q->slots[i].data = NULL;
+  }
+
+  atomic_store_explicit(&q->committed, new_commit, memory_order_release);
+
+  atomic_flag_clear(&q->commit_lock);
+}
+
+/* ----------------------------------------------------------- */
+/* PUSH */
+/* ----------------------------------------------------------- */
+// Does not increment work
+static void mpmc_push(MPMCQueue *q, void *item) {
+  MPMCSlot *slot;
+  size_t pos;
+
+  for (;;) {
+
+    pos = atomic_load_explicit(&q->tail, memory_order_relaxed);
+
+    // ensure the slot is committed BEFORE accessing it
+    size_t committed =
+        atomic_load_explicit(&q->committed, memory_order_relaxed);
+
+    if (unlikely(pos >= committed)) {
+      mpmc_commit_more(q);
+      continue;
+    }
+
+    slot = &q->slots[pos & q->mask];
+
+    size_t seq = atomic_load_explicit(&slot->seq, memory_order_acquire);
+    intptr_t diff = (intptr_t)seq - (intptr_t)pos;
+
+    if (likely(diff == 0)) {
+
+      if (atomic_compare_exchange_weak_explicit(&q->tail, &pos, pos + 1,
+                                                memory_order_relaxed,
+                                                memory_order_relaxed))
+        break;
+
+    } else if (diff < 0) { // queue actually full
+
+      sleep_ms(1000);
+
+    } else { // waiting to grow
+
+      sleep_ms(0);
+    }
+  }
+
+  slot->data = item;
+
+  atomic_store_explicit(&slot->seq, pos + 1, memory_order_release);
+
+  plat_sem_post(&q->items_sem, 1);
+}
+
+// Increment work
+static void mpmc_push_work(MPMCQueue *q, void *item) {
+  MPMCSlot *slot;
+  size_t pos;
+
+  for (;;) {
+
+    pos = atomic_load_explicit(&q->tail, memory_order_relaxed);
+
+    // ensure the slot is committed BEFORE accessing it
+    size_t committed =
+        atomic_load_explicit(&q->committed, memory_order_relaxed);
+
+    if (unlikely(pos >= committed)) {
+      mpmc_commit_more(q);
+      continue;
+    }
+
+    slot = &q->slots[pos & q->mask];
+
+    size_t seq = atomic_load_explicit(&slot->seq, memory_order_acquire);
+    intptr_t diff = (intptr_t)seq - (intptr_t)pos;
+
+    if (likely(diff == 0)) {
+
+      if (atomic_compare_exchange_weak_explicit(&q->tail, &pos, pos + 1,
+                                                memory_order_relaxed,
+                                                memory_order_relaxed))
+        break;
+
+    } else if (diff < 0) { // queue actually full
+
+      sleep_ms(1000);
+
+    } else { // waiting to grow
+
+      sleep_ms(0);
+    }
+  }
+
+  slot->data = item;
+
+  atomic_store_explicit(&slot->seq, pos + 1, memory_order_release);
+
+  atomic_fetch_add(&q->work_count, 1);
+  plat_sem_post(&q->items_sem, 1);
+}
+/* ----------------------------------------------------------- */
+/* POP */
+/* ----------------------------------------------------------- */
+static void *mpmc_pop(MPMCQueue *q) {
+
+  plat_sem_wait(&q->items_sem);
+
+  MPMCSlot *slot;
+  size_t pos;
+
+  int spins = 0;
+
+  for (;;) {
+
+    pos = atomic_load_explicit(&q->head, memory_order_relaxed);
+    slot = &q->slots[pos & q->mask];
+
+    size_t seq = atomic_load_explicit(&slot->seq, memory_order_acquire);
+    intptr_t diff = (intptr_t)seq - (intptr_t)(pos + 1);
+
+    if (likely(diff == 0)) {
+
+      if (atomic_compare_exchange_weak_explicit(&q->head, &pos, pos + 1,
+                                                memory_order_relaxed,
+                                                memory_order_relaxed))
+        break;
+
+    } else { // slot is still transitioning (written by another thread)
+      if (++spins > 10) {
+        sleep_ms(0); // yield CPU
+        spins = 0;
+      } else {
+        cpu_pause();
+      }
+    }
+  }
+
+  void *data = slot->data;
+
+  atomic_store_explicit(&slot->seq, pos + q->capacity, memory_order_release);
+
+  return data;
+}
+
+/* ----------------------------------------------------------- */
+/* PUSH POISON */
+/* ----------------------------------------------------------- */
+/*note:
+ After producers finishes, push N poison pills where N = number of consumer
+threads, this is necessary to stop the consumers.
+*/
+
+static void mpmc_producers_finished(MPMCQueue *q, u8 consumer_count) {
+  for (u8 i = 0; i < consumer_count; i++) {
+    mpmc_push(q, NULL);
+  }
+}
+
+/* ----------------------------------------------------------- */
+/* Done */
+/* ----------------------------------------------------------- */
+static void mpmc_task_done(MPMCQueue *q, u8 consumer_count) {
+  size_t prev = atomic_fetch_sub(&q->work_count, 1);
+  if (prev == 1) {
+    mpmc_producers_finished(q, consumer_count);
+  }
+}
+
+/* ----------------------------------------------------------- */
+/* MPMC Cleanup */
+/* ----------------------------------------------------------- */
+static void mpmc_finish(MPMCQueue *q) {
+  if (!q)
+    return;
+
+  if (q->slots) {
+    plat_mem_release(q->slots, 0);
+    q->slots = NULL;
+  }
+
+  plat_sem_destroy(&q->items_sem);
+
+  q->capacity = 0;
+  q->mask = 0;
+
+  atomic_store_explicit(&q->head, 0, memory_order_relaxed);
+  atomic_store_explicit(&q->tail, 0, memory_order_relaxed);
+  atomic_store_explicit(&q->committed, 0, memory_order_relaxed);
+}

platform.c

@@ -0,0 +1,942 @@
+#pragma once // ensure that a given header file is included only once in a
+             // single compilation unit
+#define _CRT_SECURE_NO_WARNINGS
+
+#include "arena.h"
+#include "base.h"
+#include "lf_mpmc.h"
+
+#include "arena.c"
+
+// xxhash include
+#define XXH_INLINE_ALL
+#include "xxh_x86dispatch.h"
+
+// ----------------------------- Config -------------------------------------
+#define FILE_HASHES_TXT "file_hashes.txt"
+#define HASH_STRLEN 33 // 128-bit hex (32 chars) + null
+#define MAX_PATHLEN 4096
+#define READ_BLOCK (KiB(64))
+
+// ----------------------------- Globals ------------------------------------
+static atomic_uint_fast64_t g_files_found = 0;
+static atomic_uint_fast64_t g_files_hashed = 0;
+static atomic_uint_fast64_t g_bytes_processed = 0;
+static atomic_int g_scan_done = 0;
+
+// ================== OS-agnostic functions abstraction =====================
+// ----------------------------- Timer functions --------------
+typedef struct {
+  u64 start;
+  u64 now;
+} HiResTimer;
+
+#if defined(_WIN32) || defined(_WIN64)
+
+static LARGE_INTEGER g_freq;
+
+static void timer_init(void) { QueryPerformanceFrequency(&g_freq); }
+
+static void timer_start(HiResTimer *t) {
+  LARGE_INTEGER v;
+  QueryPerformanceCounter(&v);
+  t->start = v.QuadPart;
+}
+
+static double timer_elapsed(HiResTimer *t) {
+  LARGE_INTEGER v;
+  QueryPerformanceCounter(&v);
+  t->now = v.QuadPart;
+
+  return (double)(t->now - t->start) / (double)g_freq.QuadPart;
+}
+
+#elif defined(__linux__)
+
+void timer_init(void) {}
+
+void timer_start(HiResTimer *t) {
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  t->start = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
+}
+
+double timer_elapsed(HiResTimer *t) {
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+
+  uint64_t now = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
+
+  return (double)(now - t->start) / 1e9;
+}
+
+#endif
+
+// ----------------------------- Get HW info --------------
+#if defined(_WIN32) || defined(_WIN64)
+
+size_t platform_physical_cores(void) {
+  DWORD len = 0;
+  GetLogicalProcessorInformation(NULL, &len);
+
+  SYSTEM_LOGICAL_PROCESSOR_INFORMATION buf[len];
+
+  GetLogicalProcessorInformation(buf, &len);
+  DWORD count = 0;
+  DWORD n = len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+  for (DWORD i = 0; i < n; i++) {
+    if (buf[i].Relationship == RelationProcessorCore)
+      count++;
+  }
+  return count ? count : 1;
+}
+
+#elif defined(__linux__)
+
+size_t platform_physical_cores(void) {
+  long n = sysconf(_SC_NPROCESSORS_ONLN);
+  return n > 0 ? (size_t)n : 1;
+}
+
+#endif
+
+const char *get_xxhash_instruction_set(void) {
+  int vecID = XXH_featureTest();
+
+  switch (vecID) {
+  case XXH_SCALAR:
+    return "Scalar (portable C)";
+  case XXH_SSE2:
+    return "SSE2";
+  case XXH_AVX2:
+    return "AVX2";
+  case XXH_AVX512:
+    return "AVX-512";
+  default:
+    return "Unknown";
+  }
+}
+
+// -------------------- File IO -------------------
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef HANDLE FileHandle;
+#define INVALID_FILE_HANDLE INVALID_HANDLE_VALUE
+
+// File open function
+static FileHandle os_file_open(const char *path) {
+  return CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
+                     NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
+}
+
+// File read function
+static int os_file_read(FileHandle handle, void *buf, size_t count,
+                        uint64_t *bytes_read) {
+  DWORD read = 0;
+  BOOL result = ReadFile(handle, buf, (DWORD)count, &read, NULL);
+  *bytes_read = read;
+  return (result && read > 0) ? 0 : -1;
+}
+
+// File close function
+static void os_file_close(FileHandle handle) { CloseHandle(handle); }
+
+#elif defined(__linux__)
+typedef int FileHandle;
+#define INVALID_FILE_HANDLE (-1)
+
+// File open function
+static FileHandle os_file_open(const char *path) {
+  return open(path, O_RDONLY | O_NOFOLLOW);
+}
+
+// File read function
+static int os_file_read(FileHandle handle, void *buf, size_t count,
+                        uint64_t *bytes_read) {
+  ssize_t result = read(handle, buf, count);
+  if (result >= 0) {
+    *bytes_read = (uint64_t)result;
+    return 0;
+  }
+  *bytes_read = 0;
+  return -1;
+}
+
+// File close function
+static void os_file_close(FileHandle handle) { close(handle); }
+
+#endif
+
+// -------------------- Thread abstraction -------------------
+// Threads context
+typedef struct {
+  u8 num_threads;
+
+  mem_arena *path_arena;
+  mem_arena *meta_arena;
+
+  MPMCQueue *dir_queue;
+  MPMCQueue *file_queue;
+} ScannerContext;
+
+typedef struct {
+  mem_arena *arena;
+  MPMCQueue *file_queue;
+} WorkerContext;
+
+#if defined(_WIN32) || defined(_WIN64)
+typedef HANDLE ThreadHandle;
+typedef DWORD(WINAPI *ThreadFunc)(void *);
+#define THREAD_RETURN DWORD WINAPI
+#define THREAD_RETURN_VALUE 0;
+
+typedef struct {
+  ThreadHandle handle;
+  int valid; // Track if thread was successfully created
+} Thread;
+
+// Thread function wrapper to handle different return types
+#define THREAD_FUNCTION(name) DWORD WINAPI name(LPVOID arg)
+
+// Thread creation function
+static int thread_create(Thread *thread, ThreadFunc func, void *arg) {
+  thread->handle =
+      CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)func, arg, 0, NULL);
+  return (thread->handle != NULL) ? 0 : -1;
+}
+
+// Thread join function
+static int thread_join(Thread *thread) {
+  return (WaitForSingleObject(thread->handle, INFINITE) == WAIT_OBJECT_0) ? 0
+                                                                          : -1;
+}
+
+// Thread close/detach function
+static void thread_close(Thread *thread) { CloseHandle(thread->handle); }
+
+// Wait for multiple threads
+static int thread_wait_multiple(Thread *threads, size_t count) {
+  HANDLE handles[64]; // Max 64 threads for Windows
+  for (size_t i = 0; i < count; i++) {
+    handles[i] = threads[i].handle;
+  }
+  return (WaitForMultipleObjects((DWORD)count, handles, TRUE, INFINITE) ==
+          WAIT_OBJECT_0)
+             ? 0
+             : -1;
+}
+
+#elif defined(__linux__)
+typedef pthread_t ThreadHandle;
+typedef void *(*ThreadFunc)(void *);
+#define THREAD_RETURN void *
+#define THREAD_RETURN_VALUE NULL;
+
+typedef struct {
+  ThreadHandle handle;
+  int valid; // Track if thread was successfully created
+} Thread;
+
+// Thread function wrapper to handle different return types
+typedef struct {
+  void *(*func)(void *);
+  void *arg;
+} ThreadWrapper;
+
+static void *thread_start_routine(void *arg) {
+  ThreadWrapper *wrapper = (ThreadWrapper *)arg;
+  void *result = wrapper->func(wrapper->arg);
+  free(wrapper);
+  return result;
+}
+
+// Thread creation function
+static int thread_create(Thread *thread, ThreadFunc func, void *arg) {
+  int ret = pthread_create(&thread->handle, NULL, func, arg);
+  if (ret == 0) {
+    thread->valid = 1;
+  }
+  return ret;
+}
+
+// Thread join function
+static int thread_join(Thread *thread) {
+  int ret = pthread_join(thread->handle, NULL);
+  thread->valid = 0;
+  return ret;
+}
+
+// Thread close/detach function
+static void thread_close(Thread *thread) {
+  if (thread->valid) {
+    pthread_detach(thread->handle);
+    thread->valid = 0;
+  }
+}
+
+// Wait for multiple threads
+static int thread_wait_multiple(Thread *threads, size_t count) {
+  for (size_t i = 0; i < count; i++) {
+    if (thread_join(&threads[i]) != 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+#endif
+
+// ======================== Get file metadata ========================
+// -------------------- Path parsing -------------------
+static void normalize_path(char *p) {
+  char *src = p;
+  char *dst = p;
+  int prev_slash = 0;
+
+  while (*src) {
+    char c = *src++;
+
+    if (c == '\\' || c == '/') {
+      if (!prev_slash) {
+        *dst++ = '/';
+        prev_slash = 1;
+      }
+    } else {
+      *dst++ = c;
+      prev_slash = 0;
+    }
+  }
+
+  *dst = '\0';
+}
+
+static int parse_paths(char *line, char folders[][MAX_PATHLEN],
+                       int max_folders) {
+  int count = 0;
+  char *p = line;
+
+  while (*p && count < max_folders) {
+
+    while (*p && isspace((unsigned char)*p))
+      p++;
+
+    if (!*p)
+      break;
+
+    char *start;
+    char quote = 0;
+
+    if (*p == '"' || *p == '\'') {
+      quote = *p++;
+      start = p;
+
+      while (*p && *p != quote)
+        p++;
+    } else {
+      start = p;
+
+      while (*p && !isspace((unsigned char)*p))
+        p++;
+    }
+
+    size_t len = p - start;
+    if (len >= MAX_PATHLEN)
+      len = MAX_PATHLEN - 1;
+
+    memcpy(folders[count], start, len);
+    folders[count][len] = 0;
+
+    normalize_path(folders[count]);
+
+    count++;
+
+    if (quote && *p == quote)
+      p++;
+  }
+  return count;
+}
+
+// ----------------------------- File time -------------------------
+#if defined(_WIN32) || defined(_WIN64)
+static void format_time(uint64_t t, char *out, size_t out_sz) {
+  if (t == 0) {
+    snprintf(out, out_sz, "N/A");
+    return;
+  }
+
+  time_t tt = (time_t)t;
+  struct tm tm;
+
+  localtime_s(&tm, &tt);
+
+  strftime(out, out_sz, "%Y-%m-%d %H:%M:%S", &tm);
+}
+// ----------------------------- Convert filetime to epoch --------------
+static uint64_t filetime_to_epoch(const FILETIME *ft) {
+  ULARGE_INTEGER ull;
+  ull.LowPart = ft->dwLowDateTime;
+  ull.HighPart = ft->dwHighDateTime;
+
+  // Windows epoch (1601) ￢ﾆﾒ Unix epoch (1970)
+  return (ull.QuadPart - 116444736000000000ULL) / 10000000ULL;
+}
+
+void platform_get_file_times(const char *path, uint64_t *out_created,
+                             uint64_t *out_modified) {
+  WIN32_FILE_ATTRIBUTE_DATA fad;
+  if (GetFileAttributesExA(path, GetFileExInfoStandard, &fad)) {
+    *out_created = filetime_to_epoch(&fad.ftCreationTime);
+    *out_modified = filetime_to_epoch(&fad.ftLastWriteTime);
+  } else {
+    *out_created = 0;
+    *out_modified = 0;
+  }
+}
+
+#elif defined(__linux__)
+static void format_time(uint64_t t, char *out, size_t out_sz) {
+  if (t == 0) {
+    snprintf(out, out_sz, "N/A");
+    return;
+  }
+
+  time_t tt = (time_t)t;
+  struct tm tm;
+
+  localtime_r(&tt, &tm);
+
+  strftime(out, out_sz, "%Y-%m-%d %H:%M:%S", &tm);
+}
+
+void platform_get_file_times(const char *path, uint64_t *out_created,
+                             uint64_t *out_modified) {
+  struct stat st;
+  if (stat(path, &st) == 0) {
+    *out_created = (uint64_t)st.st_ctime;
+    *out_modified = (uint64_t)st.st_mtime;
+  } else {
+    *out_created = 0;
+    *out_modified = 0;
+  }
+}
+
+#endif
+
+// ----------------------------- File owner ---------------------
+#if defined(_WIN32) || defined(_WIN64)
+static void get_file_owner(const char *path, char *out, size_t out_sz) {
+  PSID sid = NULL;
+  PSECURITY_DESCRIPTOR sd = NULL;
+
+  if (GetNamedSecurityInfoA(path, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION,
+                            &sid, NULL, NULL, NULL, &sd) == ERROR_SUCCESS) {
+
+    char name[64], domain[64];
+    DWORD name_len = sizeof(name);
+    DWORD domain_len = sizeof(domain);
+    SID_NAME_USE use;
+
+    if (LookupAccountSidA(NULL, sid, name, &name_len, domain, &domain_len,
+                          &use)) {
+      snprintf(out, out_sz, "%s\\%s", domain, name);
+    } else {
+      snprintf(out, out_sz, "UNKNOWN");
+    }
+  } else {
+    snprintf(out, out_sz, "UNKNOWN");
+  }
+
+  if (sd)
+    LocalFree(sd);
+}
+
+void platform_get_file_owner(const char *path, char *out_owner,
+                             size_t out_owner_size) {
+  get_file_owner(path, out_owner, out_owner_size);
+}
+
+#elif defined(__linux__)
+static void get_file_owner(uid_t uid, char *out, size_t out_sz) {
+  struct passwd *pw = getpwuid(uid);
+  if (pw) {
+    snprintf(out, out_sz, "%s", pw->pw_name);
+  } else {
+    snprintf(out, out_sz, "UNKNOWN");
+  }
+}
+
+void platform_get_file_owner(const char *path, char *out_owner,
+                             size_t out_owner_size) {
+  struct stat st;
+  if (stat(path, &st) == 0) {
+    get_file_owner(st.st_uid, out_owner, out_owner_size);
+  } else {
+    snprintf(out_owner, out_owner_size, "UNKNOWN");
+  }
+}
+
+#endif
+
+// ----------------------------- Scan helpers -----------------------------
+typedef struct FileEntry {
+  char *path;
+
+  uint64_t size_bytes;
+  uint64_t created_time;  // epoch
+  uint64_t modified_time; // epoch seconds
+  char owner[128];        // resolved owner name
+} FileEntry;
+
+typedef struct {
+  char buffer[MAX_PATHLEN];
+  char *base_end;     // Points to end of base path
+  char *filename_pos; // Points to where filename should be written
+  size_t base_len;
+} PathBuilder;
+
+static void path_builder_init(PathBuilder *pb, const char *base) {
+  pb->base_len = strlen(base);
+  memcpy(pb->buffer, base, pb->base_len);
+  pb->base_end = pb->buffer + pb->base_len;
+
+#if defined(_WIN32) || defined(_WIN64)
+  *pb->base_end = '\\';
+#elif defined(__linux__)
+  *pb->base_end = '/';
+#endif
+
+  // Ensure null termination
+  *(pb->base_end + 1) = '\0';
+  pb->filename_pos = pb->base_end + 1;
+}
+
+static void path_builder_set_filename(PathBuilder *pb, const char *filename,
+                                      size_t name_len) {
+  memcpy(pb->filename_pos, filename, name_len);
+  pb->filename_pos[name_len] = '\0'; // Ensure null termination
+}
+
+static char *path_builder_dup_arena(PathBuilder *pb, mem_arena *arena,
+                                    bool zero) {
+  // Calculate total length including base + separator + filename + null
+  // terminator
+  size_t total_len =
+      (pb->filename_pos - pb->buffer) + strlen(pb->filename_pos) + 1;
+  char *dup = arena_push(&arena, total_len, zero);
+  memcpy(dup, pb->buffer, total_len);
+  return dup;
+}
+
+#if defined(_WIN32) || defined(_WIN64)
+void scan_folder(const char *base, ScannerContext *ctx) {
+  PathBuilder pb;
+  path_builder_init(&pb, base);
+
+  char search[MAX_PATHLEN];
+  memcpy(search, pb.buffer, pb.base_len + 1); // Copy base + separator
+  memcpy(search + pb.base_len + 1, "*", 2);   // Add "*" and null
+
+  WIN32_FIND_DATAA fd;
+  HANDLE h = FindFirstFileA(search, &fd);
+  if (h == INVALID_HANDLE_VALUE)
+    return;
+
+  do {
+    // Skip . and ..
+    if (fd.cFileName[0] == '.' &&
+        (fd.cFileName[1] == 0 ||
+         (fd.cFileName[1] == '.' && fd.cFileName[2] == 0)))
+      continue;
+
+    if (fd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+      continue;
+
+    size_t name_len = strlen(fd.cFileName);
+    path_builder_set_filename(&pb, fd.cFileName, name_len);
+
+    if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+      char *dir = path_builder_dup_arena(&pb, ctx->path_arena, false);
+      mpmc_push_work(ctx->dir_queue, dir);
+    } else {
+      atomic_fetch_add(&g_files_found, 1);
+
+      FileEntry *fe = arena_push(&ctx->meta_arena, sizeof(FileEntry), true);
+
+      // Create a temporary copy for normalization to avoid corrupting pb.buffer
+      char temp_path[MAX_PATHLEN];
+      memcpy(temp_path, pb.buffer,
+             (pb.filename_pos - pb.buffer) + name_len + 1);
+      normalize_path(temp_path);
+
+      fe->path = arena_push(&ctx->path_arena, strlen(temp_path) + 1, false);
+      strcpy(fe->path, temp_path);
+
+      platform_get_file_times(pb.buffer, &fe->created_time, &fe->modified_time);
+      platform_get_file_owner(pb.buffer, fe->owner, sizeof(fe->owner));
+      fe->size_bytes = ((uint64_t)fd.nFileSizeHigh << 32) | fd.nFileSizeLow;
+
+      mpmc_push(ctx->file_queue, fe);
+    }
+
+  } while (FindNextFileA(h, &fd));
+
+  FindClose(h);
+}
+
+#elif defined(__linux__)
+To test
+Choice 1
+static int platform_get_file_times_fd(int dir_fd, const char *name,
+                                      time_t *created, time_t *modified) {
+  struct stat st;
+  if (fstatat(dir_fd, name, &st, 0) == 0) {
+    *created = st.st_ctime; // or st.st_birthtime on systems that support it
+    *modified = st.st_mtime;
+    return 0;
+  }
+  return -1;
+}
+
+static int platform_get_file_owner_fd(int dir_fd, const char *name, char *owner,
+                                      size_t owner_size) {
+  struct stat st;
+  if (fstatat(dir_fd, name, &st, 0) == 0) {
+    struct passwd pw;
+    struct passwd *result;
+    char buffer[4096]; // Sufficiently large buffer for passwd data
+    
+    // Reentrant version (thread-safe)
+    if (getpwuid_r(st.st_uid, &pw, buffer, sizeof(buffer), &result) == 0 && 
+        result != NULL && result->pw_name != NULL) {
+      strncpy(owner, result->pw_name, owner_size - 1);
+      owner[owner_size - 1] = '\0';
+    } else {
+      // Fallback to uid
+      snprintf(owner, owner_size, "uid:%d", st.st_uid);
+    }
+    return 0;
+  }
+  return -1;
+
+
+void scan_folder(const char *base, ScannerContext *ctx) {
+  PathBuilder pb;
+  path_builder_init(&pb, base);
+
+  int dir_fd = open(base, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
+  if (dir_fd == -1)
+    return;
+
+  DIR *dir = fdopendir(dir_fd);
+  if (!dir) {
+    close(dir_fd);
+    return;
+  }
+
+  struct dirent *entry;
+
+  while ((entry = readdir(dir)) != NULL) {
+    if (entry->d_name[0] == '.' &&
+        (entry->d_name[1] == 0 ||
+         (entry->d_name[1] == '.' && entry->d_name[2] == 0)))
+      continue;
+
+    size_t name_len = strlen(entry->d_name);
+    path_builder_set_filename(&pb, entry->d_name, name_len);
+
+    int file_type = DT_UNKNOWN;
+#ifdef _DIRENT_HAVE_D_TYPE
+    file_type = entry->d_type;
+#endif
+
+    // Fast path using d_type
+    if (file_type != DT_UNKNOWN) {
+      if (file_type == DT_LNK)
+        continue; // Skip symlinks
+
+      if (file_type == DT_DIR) {
+        char *dir_path = path_builder_dup_arena(&pb, ctx->path_arena, false);
+        mpmc_push_work(ctx->dir_queue, dir_path);
+        continue;
+      }
+
+      if (file_type == DT_REG) {
+        atomic_fetch_add(&g_files_found, 1);
+        FileEntry *fe = arena_push(&ctx->meta_arena, sizeof(FileEntry),
+        true);
+
+        // Use fstatat for file info
+        struct stat st;
+        if (fstatat(dir_fd, entry->d_name, &st, 0) == 0) {
+          // Convert times using fd variant
+          platform_get_file_times_fd(dir_fd, entry->d_name,
+          &fe->created_time,
+                                     &fe->modified_time);
+          platform_get_file_owner_fd(dir_fd, entry->d_name, fe->owner,
+                                     sizeof(fe->owner));
+          fe->size_bytes = (uint64_t)st.st_size;
+
+          // Normalize path
+          char temp_path[MAX_PATHLEN];
+          memcpy(temp_path, pb.buffer,
+                 (pb.filename_pos - pb.buffer) + name_len + 1);
+          normalize_path(temp_path);
+
+          fe->path = arena_push(&ctx->path_arena, strlen(temp_path) + 1,
+          false); strcpy(fe->path, temp_path);
+
+          mpmc_push(ctx->file_queue, fe);
+        }
+        continue;
+      }
+    }
+
+    // Fallback for unknown types
+    struct stat st;
+    if (fstatat(dir_fd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) == 0) {
+      if (S_ISLNK(st.st_mode))
+        continue;
+
+      if (S_ISDIR(st.st_mode)) {
+        char *dir_path = path_builder_dup_arena(&pb, ctx->path_arena, false);
+        mpmc_push_work(ctx->dir_queue, dir_path);
+      } else if (S_ISREG(st.st_mode)) {
+        atomic_fetch_add(&g_files_found, 1);
+        FileEntry *fe = arena_push(&ctx->meta_arena, sizeof(FileEntry),
+        true);
+
+        platform_get_file_times(pb.buffer, &fe->created_time,
+                                &fe->modified_time);
+        platform_get_file_owner(pb.buffer, fe->owner, sizeof(fe->owner));
+        fe->size_bytes = (uint64_t)st.st_size;
+
+        char temp_path[MAX_PATHLEN];
+        memcpy(temp_path, pb.buffer,
+               (pb.filename_pos - pb.buffer) + name_len + 1);
+        normalize_path(temp_path);
+
+        fe->path = arena_push(&ctx->path_arena, strlen(temp_path) + 1,
+        false); strcpy(fe->path, temp_path);
+
+        mpmc_push(ctx->file_queue, fe);
+      }
+    }
+  }
+
+  closedir(dir); // Closes dir_fd automatically
+}
+
+// Choice 2
+
+// void scan_folder(const char *base, ScannerContext *ctx) {
+//   PathBuilder pb;
+//   path_builder_init(&pb, base);
+//
+//   DIR *dir = opendir(base);
+//   if (!dir)
+//     return;
+//
+//   struct dirent *entry;
+//   struct stat st;
+//
+//   while ((entry = readdir(dir)) != NULL) {
+//     if (entry->d_name[0] == '.' &&
+//         (entry->d_name[1] == 0 ||
+//          (entry->d_name[1] == '.' && entry->d_name[2] == 0)))
+//       continue;
+//
+//     size_t name_len = strlen(entry->d_name);
+//     path_builder_set_filename(&pb, entry->d_name, name_len);
+//
+//     if (lstat(pb.buffer, &st) == 0 && S_ISLNK(st.st_mode))
+//       continue;
+//
+//     if (stat(pb.buffer, &st) == 0) {
+//       if (S_ISDIR(st.st_mode)) {
+//         char *dir_path = path_builder_dup_arena(&pb, ctx->path_arena, false);
+//         mpmc_push_work(ctx->dir_queue, dir_path);
+//       } else {
+//         atomic_fetch_add(&g_files_found, 1);
+//
+//         FileEntry *fe = arena_push(&ctx->meta_arena, sizeof(FileEntry), true);
+//
+//         // Create a temporary copy for normalization
+//         char temp_path[MAX_PATHLEN];
+//         memcpy(temp_path, pb.buffer,
+//                (pb.filename_pos - pb.buffer) + name_len + 1);
+//         normalize_path(temp_path);
+//
+//         fe->path = arena_push(&ctx->path_arena, strlen(temp_path) + 1, false);
+//         strcpy(fe->path, temp_path);
+//
+//         platform_get_file_times(pb.buffer, &fe->created_time,
+//                                 &fe->modified_time);
+//         platform_get_file_owner(pb.buffer, fe->owner, sizeof(fe->owner));
+//         fe->size_bytes = (uint64_t)st.st_size;
+//
+//         mpmc_push(ctx->file_queue, fe);
+//       }
+//     }
+//   }
+//
+//   closedir(dir);
+// }
+
+#endif
+
+// ------------------------- Scan worker --------------------------------
+static THREAD_RETURN scan_worker(void *arg) {
+  ScannerContext *ctx = (ScannerContext *)arg;
+
+  for (;;) {
+    char *dir = mpmc_pop(ctx->dir_queue);
+    if (!dir)
+      break;
+
+    scan_folder(dir, ctx);
+
+    mpmc_task_done(ctx->dir_queue, ctx->num_threads);
+  }
+
+  return THREAD_RETURN_VALUE;
+}
+
+// ----------------------------- Hashing helpers -----------------------------
+static void xxh3_hash_file_stream(const char *path, char *out_hex,
+                                  unsigned char *buf) {
+  XXH128_hash_t h;
+  XXH3_state_t state;
+  XXH3_128bits_reset(&state);
+
+  FileHandle handle = os_file_open(path);
+  if (handle == INVALID_FILE_HANDLE) {
+    strcpy(out_hex, "ERROR");
+    return;
+  }
+
+  uint64_t bytes_read;
+  while (os_file_read(handle, buf, READ_BLOCK, &bytes_read) == 0 &&
+         bytes_read > 0) {
+    XXH3_128bits_update(&state, buf, (size_t)bytes_read);
+    atomic_fetch_add(&g_bytes_processed, bytes_read);
+  }
+
+  os_file_close(handle);
+
+  h = XXH3_128bits_digest(&state);
+  snprintf(out_hex, HASH_STRLEN, "%016llx%016llx", (unsigned long long)h.high64,
+           (unsigned long long)h.low64);
+}
+
+// ------------------------- Hash worker --------------------------------
+static THREAD_RETURN hash_worker(void *arg) {
+  WorkerContext *ctx = (WorkerContext *)arg;
+  unsigned char *buf = (unsigned char *)malloc(READ_BLOCK);
+
+  for (;;) {
+    FileEntry *fe = mpmc_pop(ctx->file_queue);
+    if (!fe)
+      break;
+
+    char hash[HASH_STRLEN];
+    xxh3_hash_file_stream(fe->path, hash, buf);
+
+    char created[32], modified[32];
+    format_time(fe->created_time, created, sizeof(created));
+    format_time(fe->modified_time, modified, sizeof(modified));
+
+    double size_kib = (double)fe->size_bytes / 1024.0;
+
+    char stack_buf[1024];
+
+    int len =
+        snprintf(stack_buf, sizeof(stack_buf), "%s\t%s\t%.2f\t%s\t%s\t%s\n",
+                 hash, fe->path, size_kib, created, modified, fe->owner);
+
+    char *dst = arena_push(&ctx->arena, len, false);
+    memcpy(dst, stack_buf, len);
+
+    atomic_fetch_add(&g_files_hashed, 1);
+  }
+
+  free(buf);
+
+  return THREAD_RETURN_VALUE;
+}
+
+// ----------------------------- Progress display ---------------------------
+static THREAD_RETURN progress_thread(void *arg) {
+  (void)arg; // Unused parameter
+
+  HiResTimer progress_timer;
+  timer_start(&progress_timer);
+
+  uint64_t last_bytes = atomic_load(&g_bytes_processed);
+  double last_time = 0.0;
+
+  double displayed_speed = 0.0;
+  const double sample_interval = 0.5;
+
+  for (;;) {
+    uint64_t found = atomic_load(&g_files_found);
+    uint64_t hashed = atomic_load(&g_files_hashed);
+    uint64_t bytes = atomic_load(&g_bytes_processed);
+    int scan_done = atomic_load(&g_scan_done);
+
+    double t = timer_elapsed(&progress_timer);
+
+    if (last_time == 0.0) {
+      last_time = t;
+      last_bytes = bytes;
+    }
+
+    double dt = t - last_time;
+
+    if (dt >= sample_interval) {
+      uint64_t db = bytes - last_bytes;
+
+      if (db > 0 && dt > 0.0001) {
+        displayed_speed = (double)db / (1024.0 * 1024.0) / dt;
+      }
+
+      last_bytes = bytes;
+      last_time = t;
+    }
+
+    if (!scan_done) {
+      printf("\rScanning: %llu files | Hashed: %llu | %.2f MB/s    ",
+             (unsigned long long)found, (unsigned long long)hashed,
+             displayed_speed);
+    } else {
+      double pct = found ? (double)hashed / (double)found : 0.0;
+      int barw = 40;
+      int filled = (int)(pct * barw);
+
+      char bar[64];
+      int p = 0;
+
+      bar[p++] = '[';
+      for (int i = 0; i < filled; i++)
+        bar[p++] = '#';
+      for (int i = filled; i < barw; i++)
+        bar[p++] = '.';
+      bar[p++] = ']';
+      bar[p] = 0;
+
+      printf("\r%s %6.2f%% (%llu / %llu) %.2f MB/s    ", bar, pct * 100.0,
+             (unsigned long long)hashed, (unsigned long long)found,
+             displayed_speed);
+    }
+
+    fflush(stdout);
+
+    if (scan_done && hashed == found)
+      break;
+
+    sleep_ms(100);
+  }
+
+  printf("\n");
+
+  return THREAD_RETURN_VALUE;
+}

platform.h

@@ -1,153 +0,0 @@
-#pragma once
-
-#include <stdatomic.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#if defined(_WIN32) || defined(_WIN64)
-#define PLATFORM_WINDOWS 1
-#include <aclapi.h>
-#include <fcntl.h>
-#include <io.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <windows.h>
-
-#define strdup _strdup
-#else
-#include <dirent.h>
-#include <fcntl.h>
-#include <pthread.h>
-#include <pwd.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#endif
-
-#define XXH_VECTOR XXH_AVX2 // not recommanded to compile with gcc see xxhash.h line 4082
-                            // Must compile with /arch:AVX2 in clang-cl or -mavx2 in clang/gcc
-#define XXH_INLINE_ALL
-#include "xxhash.c"
-#include "xxhash.h"
-
-// ----------------------------- Config -------------------------------------
-#define FILE_LIST_TXT "file_list.txt"
-#define FILE_HASHES_TXT "file_hashes.txt"
-#define HASH_STRLEN 33 // 128-bit hex (32 chars) + null
-#define MAX_PATHLEN 4096
-#define READ_BLOCK (64 * 1024) // 64KB blocks
-
-// ----------------------------- Data types ---------------------------------
-typedef struct FileEntry {
-  char *path;
-
-  uint64_t size_bytes;
-  uint64_t created_time;  // epoch
-  uint64_t modified_time; // epoch seconds
-  char owner[128];        // resolved owner name
-} FileEntry;
-
-/* File path and metadata */
-static void normalize_path(char *p) {
-  char *src = p;
-  char *dst = p;
-  int prev_slash = 0;
-
-  while (*src) {
-    char c = *src++;
-
-    if (c == '\\' || c == '/') {
-      if (!prev_slash) {
-        *dst++ = '/';
-        prev_slash = 1;
-      }
-    } else {
-      *dst++ = c;
-      prev_slash = 0;
-    }
-  }
-
-  *dst = '\0';
-}
-
-void platform_get_file_times(const char *path, uint64_t *out_created,
-                             uint64_t *out_modified);
-void platform_get_file_owner(const char *path, char *out_owner,
-                             size_t out_owner_size);
-
-/* scan folder timer*/
-typedef struct {
-  LARGE_INTEGER start;
-  LARGE_INTEGER end;
-} HiResTimer;
-
-static LARGE_INTEGER g_qpc_freq;
-
-static void timer_init(void) { QueryPerformanceFrequency(&g_qpc_freq); }
-
-static void timer_start(HiResTimer *t) { QueryPerformanceCounter(&t->start); }
-
-static double timer_stop(HiResTimer *t) {
-  QueryPerformanceCounter(&t->end);
-  return (double)(t->end.QuadPart - t->start.QuadPart) /
-         (double)g_qpc_freq.QuadPart;
-}
-
-/* Scan folders */
-typedef struct EntryBuffer {
-  FileEntry *entries;
-  size_t count;
-  size_t capacity;
-} EntryBuffer;
-
-typedef struct DirQueue DirQueue;
-
-void scan_folder_windows_parallel(const char *base, DirQueue *q,
-                                  EntryBuffer *buf);
-void scan_folder_posix_parallel(const char *base, DirQueue *q);
-
-typedef struct DirJob {
-  char *path;
-  struct DirJob *next;
-} DirJob;
-
-typedef struct DirQueue {
-  char **items;
-  size_t count;
-  size_t cap;
-  size_t active;
-
-  int stop;
-
-#if PLATFORM_WINDOWS
-  CRITICAL_SECTION cs;
-  CONDITION_VARIABLE cv;
-#else
-  pthread_mutex_t mutex;
-  pthread_cond_t cond;
-#endif
-} DirQueue;
-
-/* Hashing */
-typedef struct Job {
-  FileEntry *file;
-  struct Job *next;
-} Job;
-
-typedef struct {
-  Job *head;
-  Job *tail;
-  CRITICAL_SECTION cs;
-  CONDITION_VARIABLE cv;
-  atomic_size_t count; // queued jobs
-  int stop;
-} JobQueue;
-
-typedef struct {
-  JobQueue *queue;
-  atomic_size_t *done_counter;
-  size_t total_jobs;
-  atomic_int *live_workers;
-} WorkerArg;

platform_posix.c

@@ -1,678 +0,0 @@
-#include "platform.h"
-
-// ----------------------------- Globals ------------------------------------
-static atomic_uint_fast64_t g_bytes_processed = 0;
-FileEntry *g_entries = NULL;
-size_t g_entry_count = 0;
-size_t g_entry_capacity = 0;
-
-// ----------------------------- Utils --------------------------------------
-static void perror_exit(const char *msg) {
-  perror(msg);
-  exit(1);
-}
-
-static void *xmalloc(size_t n) {
-  void *p = malloc(n);
-  if (!p)
-    perror_exit("malloc");
-  return p;
-}
-
-static void add_entry(const FileEntry *src) {
-  if (g_entry_count + 1 > g_entry_capacity) {
-    g_entry_capacity = g_entry_capacity ? g_entry_capacity * 2 : 1024;
-    g_entries = realloc(g_entries, sizeof(FileEntry) * g_entry_capacity);
-    if (!g_entries)
-      perror_exit("realloc");
-  }
-
-  FileEntry *dst = &g_entries[g_entry_count++];
-  memset(dst, 0, sizeof(*dst));
-
-  dst->size_bytes = src->size_bytes;
-  dst->created_time = src->created_time;
-  dst->modified_time = src->modified_time;
-
-  if (src->path)
-    dst->path = strdup(src->path);
-
-  strncpy(dst->owner, src->owner, sizeof(dst->owner) - 1);
-  dst->owner[sizeof(dst->owner) - 1] = '\0';
-}
-
-static void free_entries(void) {
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    free(g_entries[i].path);
-  }
-
-  free(g_entries);
-  g_entries = NULL;
-  g_entry_count = 0;
-  g_entry_capacity = 0;
-}
-
-// ----------------------------- Owner lookup ------------------------------
-static void get_file_owner(uid_t uid, char *out, size_t out_sz) {
-  struct passwd *pw = getpwuid(uid);
-  if (pw) {
-    snprintf(out, out_sz, "%s", pw->pw_name);
-  } else {
-    snprintf(out, out_sz, "UNKNOWN");
-  }
-}
-
-// ----------------------------- Format time helper -------------------------
-static void format_time(uint64_t t, char *out, size_t out_sz) {
-  if (t == 0) {
-    snprintf(out, out_sz, "N/A");
-    return;
-  }
-
-  time_t tt = (time_t)t;
-  struct tm tm;
-
-#if PLATFORM_WINDOWS
-  localtime_s(&tm, &tt);
-#else
-  localtime_r(&tt, &tm);
-#endif
-
-  strftime(out, out_sz, "%Y-%m-%d %H:%M:%S", &tm);
-}
-
-// --------------- parallel directory scanning ----------------
-
-// Add queue helper functions
-static void dirqueue_push(DirQueue *q, const char *path) {
-  DirJob *job = malloc(sizeof(*job));
-  job->path = strdup(path);
-  job->next = NULL;
-
-  pthread_mutex_lock(&q->mutex);
-
-  if (q->tail)
-    q->tail->next = job;
-  else
-    q->head = job;
-
-  q->tail = job;
-
-  pthread_cond_signal(&q->cond);
-  pthread_mutex_unlock(&q->mutex);
-}
-
-static char *dirqueue_pop(DirQueue *q) {
-  pthread_mutex_lock(&q->mutex);
-  while (!q->head && !q->stop)
-    pthread_cond_wait(&q->cond, &q->mutex);
-
-  if (q->stop) {
-    pthread_mutex_unlock(&q->mutex);
-    return NULL;
-  }
-
-  DirJob *job = q->head;
-  q->head = job->next;
-  if (!q->head)
-    q->tail = NULL;
-
-  q->active_workers++;
-  pthread_mutex_unlock(&q->mutex);
-
-  char *path = job->path;
-  free(job);
-  return path;
-}
-
-static void dirqueue_done(DirQueue *q) {
-  pthread_mutex_lock(&q->mutex);
-  q->active_workers--;
-
-  if (!q->head && q->active_workers == 0) {
-    q->stop = 1;
-    pthread_cond_broadcast(&q->cond);
-  }
-  pthread_mutex_unlock(&q->mutex);
-}
-
-// Scanning directory worker thread function
-static void scan_worker(void *arg) {
-  DirQueue *q = arg;
-
-  for (;;) {
-    char *dir = dirqueue_pop(q);
-    if (!dir)
-      break;
-
-    scan_folder_posix_parallel(dir, q);
-
-    free(dir);
-    dirqueue_done(q);
-  }
-}
-
-// Scanning directory function
-void scan_folder_posix_parallel(const char *base, DirQueue *q) {
-  DIR *d = opendir(base);
-  if (!d)
-    return;
-
-  struct dirent *ent;
-  while ((ent = readdir(d))) {
-    if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
-      continue;
-
-    char full[MAX_PATHLEN];
-    snprintf(full, sizeof(full), "%s/%s", base, ent->d_name);
-
-    struct stat st;
-    if (lstat(full, &st) != 0)
-      continue;
-
-    if (S_ISDIR(st.st_mode)) {
-      dirqueue_push(q, full);
-    } else if (S_ISREG(st.st_mode)) {
-      FileEntry fe;
-      memset(&fe, 0, sizeof(fe));
-
-      normalize_path(full);
-
-      fe.path = full;
-      fe.size_bytes = (uint64_t)st.st_size;
-      fe.created_time = (uint64_t)st.st_ctime;
-      fe.modified_time = (uint64_t)st.st_mtime;
-
-      get_file_owner(st.st_uid, fe.owner, sizeof(fe.owner));
-
-      add_entry(&fe);
-    }
-  }
-  closedir(d);
-}
-
-// ----------------------------- Job queue ----------------------------------
-static void jobqueue_init(JobQueue *q) {
-  q->head = q->tail = NULL;
-  atomic_store(&q->count, 0);
-  q->stop = 0;
-  pthread_mutex_init(&q->mutex, NULL);
-  pthread_cond_init(&q->cond, NULL);
-}
-
-static void jobqueue_push(JobQueue *q, Job *job) {
-  pthread_mutex_lock(&q->mutex);
-  job->next = NULL;
-  if (q->tail)
-    q->tail->next = job;
-  else
-    q->head = job;
-  q->tail = job;
-  atomic_fetch_add(&q->count, 1);
-  pthread_cond_signal(&q->cond);
-  pthread_mutex_unlock(&q->mutex);
-}
-
-static Job *jobqueue_pop(JobQueue *q) {
-  pthread_mutex_lock(&q->mutex);
-  while (!q->head && !q->stop)
-    pthread_cond_wait(&q->cond, &q->mutex);
-  if (q->stop && !q->head) {
-    pthread_mutex_unlock(&q->mutex);
-    return NULL;
-  }
-  Job *j = q->head;
-  q->head = j->next;
-  if (!q->head)
-    q->tail = NULL;
-  pthread_mutex_unlock(&q->mutex);
-  if (j)
-    atomic_fetch_sub(&q->count, 1);
-  return j;
-}
-
-static void jobqueue_stop(JobQueue *q) {
-  pthread_mutex_lock(&q->mutex);
-  q->stop = 1;
-  pthread_cond_broadcast(&q->cond);
-  pthread_mutex_unlock(&q->mutex);
-}
-
-// ----------------------------- Hashing helpers -----------------------------
-static void xxh3_hash_file_stream(const char *path, char *out_hex) {
-  // compute XXH3_128 over file. POSIX and Windows use standard reads in this
-  // helper.
-  int fd = open(path, O_RDONLY);
-  if (fd < 0) {
-    strcpy(out_hex, "ERROR");
-    return;
-  }
-  XXH128_hash_t h;
-  XXH3_state_t *state = XXH3_createState();
-  XXH3_128bits_reset(state);
-  unsigned char *buf = (unsigned char *)malloc(READ_BLOCK);
-  ssize_t r;
-  while ((r = read(fd, buf, READ_BLOCK)) > 0) {
-    XXH3_128bits_update(state, buf, (size_t)r);
-    atomic_fetch_add(&g_bytes_processed, (uint64_t)r);
-  }
-
-  h = XXH3_128bits_digest(state);
-  XXH3_freeState(state);
-  close(fd);
-  free(buf);
-  snprintf(out_hex, HASH_STRLEN, "%016llx%016llx", (unsigned long long)h.high64,
-           (unsigned long long)h.low64);
-}
-
-// ----------------------------- Worker --------------------------------------
-static void *worker_thread_posix(void *argp) {
-  WorkerArg *w = (WorkerArg *)argp;
-  JobQueue *q = w->queue;
-  for (;;) {
-    Job *job = jobqueue_pop(q);
-    if (!job)
-      break;
-    char hex[HASH_STRLEN];
-    xxh3_hash_file_stream(job->file->path, hex);
-
-    // append to file_hashes.txt atomically: we will store results to a temp
-    // buffer and write them at the end (to avoid synchronization issues). But
-    // for simplicity, here we append directly using a file lock (fopen+fwrite
-    // guarded by mutex). We'll store results in job->file->path? Instead,
-    // simple global append with a mutex. Using a file-level append lock:
-    static pthread_mutex_t append_mutex = PTHREAD_MUTEX_INITIALIZER;
-    pthread_mutex_lock(&append_mutex);
-    FILE *hf = fopen(FILE_HASHES_TXT, "a");
-    if (hf) {
-      char created[32], modified[32];
-
-      format_time(job->file->created_time, created, sizeof(created));
-      format_time(job->file->modified_time, modified, sizeof(modified));
-      double size_kib = (double)job->file->size_bytes / (1024.0);
-
-      fprintf(hf, "%s\t%s\t%.2f\t%s\t%s\t%s\n", hex, job->file->path, size_kib,
-              created, modified, job->file->owner);
-      fclose(hf);
-    }
-    pthread_mutex_unlock(&append_mutex);
-
-    atomic_fetch_add(w->done_counter, 1);
-    free(job);
-  }
-  atomic_fetch_sub(w->live_workers, 1);
-  return NULL;
-}
-
-// ----------------------------- Progress display ---------------------------
-static void print_progress(size_t done, size_t total) {
-  const int barw = 40;
-  double pct = total ? (double)done / (double)total : 0.0;
-  int filled = (int)(pct * barw + 0.5);
-  printf("\r[");
-  for (int i = 0; i < filled; ++i)
-    putchar('#');
-  for (int i = filled; i < barw; ++i)
-    putchar(' ');
-  printf("] %6.2f%%  (%zu / %zu)  ", pct * 100.0, done, total);
-  fflush(stdout);
-}
-
-// ----------------------------- Helpers: load/save --------------------------
-static int file_exists(const char *path) {
-  struct stat st;
-  return (stat(path, &st) == 0);
-}
-
-static void save_file_list(const char *list_path) {
-  FILE *f = fopen(list_path, "w");
-  if (!f) {
-    perror("fopen file_list");
-    return;
-  }
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    fprintf(f, "%s\n", g_entries[i].path);
-  }
-  fclose(f);
-}
-
-static void load_file_list(const char *list_path) {
-  FILE *f = fopen(list_path, "r");
-  if (!f)
-    return;
-
-  char line[MAX_PATHLEN];
-
-  while (fgets(line, sizeof(line), f)) {
-    line[strcspn(line, "\r\n")] = 0;
-
-    FileEntry fe;
-    memset(&fe, 0, sizeof(fe));
-
-    fe.path = line;
-
-    /* Populate metadata from filesystem */
-    platform_get_file_times(line, &fe.created_time, &fe.modified_time);
-
-    platform_get_file_owner(line, fe.owner, sizeof(fe.owner));
-
-    add_entry(&fe);
-  }
-
-  fclose(f);
-}
-
-// Read existing hashes into memory map for resume
-// Simple linear search mapping: returns 1 if path has hash found (and writes
-// into out_hex)
-static int find_hash_in_file(const char *hashfile, const char *path,
-                             char *out_hex) {
-  FILE *f = fopen(hashfile, "r");
-  if (!f)
-    return 0;
-  char p[MAX_PATHLEN];
-  char h[128];
-  int found = 0;
-  while (fscanf(f, "%4095s %127s", p, h) == 2) {
-    if (strcmp(p, path) == 0) {
-      strncpy(out_hex, h, HASH_STRLEN);
-      out_hex[HASH_STRLEN - 1] = 0;
-      found = 1;
-      break;
-    }
-  }
-  fclose(f);
-  return found;
-}
-// ----------------------------- Get file metadata -------------------------
-void platform_get_file_times(const char *path, uint64_t *out_created,
-                             uint64_t *out_modified) {
-  struct stat st;
-  if (stat(path, &st) == 0) {
-    *out_created = (uint64_t)st.st_ctime;
-    *out_modified = (uint64_t)st.st_mtime;
-  } else {
-    *out_created = 0;
-    *out_modified = 0;
-  }
-}
-
-void platform_get_file_owner(const char *path, char *out_owner,
-                             size_t out_owner_size) {
-  struct stat st;
-  if (stat(path, &st) == 0) {
-    get_file_owner(st.st_uid, out_owner, out_owner_size);
-  } else {
-    snprintf(out_owner, out_owner_size, "UNKNOWN");
-  }
-}
-
-// ----------------------------- Main ---------------------------------------
-int main(int argc, char **argv) {
-  char folders[64][MAX_PATHLEN]; // up to 64 input folders
-  int folder_count = 0;
-  int resume = 0;
-
-  // -------------------------------
-  // Parse arguments
-  // -------------------------------
-  for (int i = 1; i < argc; ++i) {
-    if (strcmp(argv[i], "-resume") == 0) {
-      resume = 1;
-    } else {
-      if (folder_count < 64) {
-        strncpy(folders[folder_count], argv[i], MAX_PATHLEN - 1);
-        folders[folder_count][MAX_PATHLEN - 1] = 0;
-        folder_count++;
-      }
-    }
-  }
-
-  // -------------------------------
-  // Ask user if no folders provided
-  // -------------------------------
-  if (folder_count == 0 && !resume) {
-    printf("Enter folder to process (Enter = current folder): ");
-    fflush(stdout);
-
-    char buf[MAX_PATHLEN];
-    if (!fgets(buf, sizeof(buf), stdin))
-      return 1;
-    buf[strcspn(buf, "\r\n")] = 0;
-
-    if (buf[0] == 0)
-      strcpy(folders[0], ".");
-    else
-      strncpy(folders[0], buf, MAX_PATHLEN - 1);
-
-    folder_count = 1;
-  } else if (folder_count == 0 && resume) {
-    strcpy(folders[0], ".");
-    folder_count = 1;
-  }
-
-  // -------------------------------
-  // Display selected folders
-  // -------------------------------
-  printf("Processing %d folder(s):\n", folder_count);
-  for (int i = 0; i < folder_count; ++i) {
-    printf("  - %s\n", folders[i]);
-  }
-
-  // -------------------------------
-  // Detect hardware threads (CPU cores)
-  // -------------------------------
-  size_t hw_threads = 1;
-  long cpus = sysconf(_SC_NPROCESSORS_ONLN);
-  if (cpus > 0)
-    hw_threads = (size_t)cpus;
-
-  // Add some extra threads to overlap I/O more aggressively
-  size_t num_threads = hw_threads * 2;
-  if (num_threads < 2)
-    num_threads = 2;
-
-  // -------------------------------
-  // Step 1: Scan all folders
-  // -------------------------------
-  if (!resume) {
-    DirQueue q = {0};
-    pthread_mutex_init(&q.mutex, NULL);
-    pthread_cond_init(&q.cond, NULL);
-
-    // Seed queue
-    for (int i = 0; i < folder_count; ++i)
-      dirqueue_push(&q, folders[i]);
-
-    pthread_t *threads = malloc(sizeof(pthread_t) * num_threads);
-
-    for (size_t i = 0; i < num_threads; ++i)
-      pthread_create(&threads[i], NULL, (void *(*)(void *))scan_worker, &q);
-
-    for (size_t i = 0; i < num_threads; ++i)
-      pthread_join(threads[i], NULL);
-
-    free(threads);
-
-    pthread_mutex_destroy(&q.mutex);
-    pthread_cond_destroy(&q.cond);
-
-    printf("Found %zu files. Saving to %s\n", g_entry_count, FILE_LIST_TXT);
-    save_file_list(FILE_LIST_TXT);
-  } else {
-    if (!file_exists(FILE_LIST_TXT)) {
-      fprintf(stderr, "Resume requested but %s not found\n", FILE_LIST_TXT);
-      return 1;
-    }
-    load_file_list(FILE_LIST_TXT);
-    printf("Loaded %zu files from %s\n", g_entry_count, FILE_LIST_TXT);
-  }
-
-  if (g_entry_count == 0) {
-    printf("No files to process.\n");
-    return 0;
-  }
-
-  // If resume: create map of which files are already hashed
-  char **existing_hash = calloc(g_entry_count, sizeof(char *));
-  for (size_t i = 0; i < g_entry_count; ++i)
-    existing_hash[i] = NULL;
-
-  if (resume && file_exists(FILE_HASHES_TXT)) {
-    // For simplicity we parse hash file and match lines to list entries.
-    for (size_t i = 0; i < g_entry_count; ++i) {
-      char hex[HASH_STRLEN] = {0};
-      if (find_hash_in_file(FILE_HASHES_TXT, g_entries[i].path, hex)) {
-        existing_hash[i] = strdup(hex);
-      }
-    }
-  }
-
-  // Prepare job queue of only missing files (or all if not resume)
-  JobQueue queue;
-  jobqueue_init(&queue);
-
-  size_t total_jobs = 0;
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    if (resume && existing_hash[i])
-      continue;
-    Job *j = (Job *)malloc(sizeof(Job));
-    j->file = &g_entries[i];
-    j->next = NULL;
-    jobqueue_push(&queue, j);
-    ++total_jobs;
-  }
-
-  if (total_jobs == 0) {
-    printf("Nothing to do — all files already hashed.\n");
-    return 0;
-  }
-
-  // Remove old hashes file if we're recomputing from scratch.
-  if (!resume) {
-    // create/overwrite hashes file
-    FILE *hf = fopen(FILE_HASHES_TXT, "w");
-    if (hf)
-      fclose(hf);
-  } // if resume, we append only missing
-
-  // Starting thread pool
-
-  atomic_size_t done_counter;
-  atomic_store(&done_counter, 0);
-  atomic_int live_workers;
-  atomic_store(&live_workers, (int)num_threads);
-
-  WorkerArg warg = {.queue = &queue,
-                    .done_counter = &done_counter,
-                    .total_jobs = total_jobs,
-                    .live_workers = &live_workers};
-
-  printf("Starting thread pool: %zu threads (CPU cores: %zu)\n", num_threads,
-         hw_threads);
-
-  // Launch threads
-  pthread_t *tids = malloc(sizeof(pthread_t) * num_threads);
-  for (size_t i = 0; i < num_threads; ++i) {
-    pthread_create(&tids[i], NULL, worker_thread_posix, &warg);
-  }
-
-  // Progress / timer
-  struct timespec tstart, tnow;
-  clock_gettime(CLOCK_MONOTONIC, &tstart);
-
-  size_t last_done = 0;
-
-  // ---------- Correct real-time MB/s (stable & accurate) ----------
-  uint64_t last_bytes = atomic_load(&g_bytes_processed);
-  double last_time = 0.0;
-  double displayed_speed = 0.0;
-  const double sample_interval = 0.5;
-  char linebuf[256];
-
-  for (;;) {
-    size_t done = (size_t)atomic_load(&done_counter);
-
-    // ---- monotonic time ----
-    clock_gettime(CLOCK_MONOTONIC, &tnow);
-    double now =
-        (tnow.tv_sec - tstart.tv_sec) + (tnow.tv_nsec - tstart.tv_nsec) / 1e9;
-
-    // ---- bytes so far ----
-    uint64_t bytes = atomic_load(&g_bytes_processed);
-
-    // ---- real sampler (independent of UI sleep) ----
-    if (last_time == 0.0) {
-      last_time = now;
-      last_bytes = bytes;
-    }
-
-    double dt = now - last_time;
-    if (dt >= sample_interval) {
-      uint64_t db = bytes - last_bytes;
-
-      if (db > 0 && dt > 0.0001) {
-        displayed_speed = (double)db / (1024.0 * 1024.0) / dt;
-      }
-
-      last_bytes = bytes;
-      last_time = now;
-    }
-
-    // ---- progress bar build ----
-    const int barw = 40;
-    double pct = total_jobs ? (double)done / (double)total_jobs : 0.0;
-    int filled = (int)(pct * barw + 0.5);
-
-    int p = 0;
-    p += snprintf(linebuf + p, sizeof(linebuf) - p, "[");
-    for (int i = 0; i < filled && p < (int)sizeof(linebuf); ++i)
-      p += snprintf(linebuf + p, sizeof(linebuf) - p, "#");
-    for (int i = filled; i < barw && p < (int)sizeof(linebuf); ++i)
-      p += snprintf(linebuf + p, sizeof(linebuf) - p, ".");
-
-    snprintf(linebuf + p, sizeof(linebuf) - p,
-             "] %6.2f%% (%zu / %zu)  %8.2f MB/s", pct * 100.0, done, total_jobs,
-             displayed_speed);
-
-    printf("\r%s", linebuf);
-    fflush(stdout);
-
-    if (done >= total_jobs)
-      break;
-
-    usleep(100000);
-  }
-
-  printf("\n\n");
-
-  // stop queue and join threads
-  jobqueue_stop(&queue);
-  for (size_t i = 0; i < num_threads; ++i)
-    pthread_join(tids[i], NULL);
-
-  // done time
-  clock_gettime(CLOCK_MONOTONIC, &tnow);
-  double elapsed =
-      (tnow.tv_sec - tstart.tv_sec) + (tnow.tv_nsec - tstart.tv_nsec) / 1e9;
-
-  printf("Completed hashing %zu files in %.2f seconds\n", total_jobs, elapsed);
-  uint64_t total_bytes = (uint64_t)atomic_load(&g_bytes_processed);
-  double total_mb = (double)total_bytes / (1024.0 * 1024.0);
-  double avg_mbps = total_mb / elapsed;
-  printf("Total: %.2f MB, Average: %.2f MB/s\n", total_mb, avg_mbps);
-
-  // If resume: we appended missing entries. If not resume: we wrote all results
-  // during workers. Note: This program appends hashes as workers finish. This
-  // avoids holding all hashes in RAM.
-
-  // Cleanup
-  for (size_t i = 0; i < g_entry_count; ++i)
-    if (existing_hash[i])
-      free(existing_hash[i]);
-  free(existing_hash);
-
-  free_entries();
-
-  return 0;
-}

platform_windows.c

@@ -1,741 +0,0 @@
-#include "platform.h"
-
-// ----------------------------- Globals ------------------------------------
-FileEntry *g_entries = NULL;
-size_t g_entry_count = 0;
-size_t g_entry_capacity = 0;
-static atomic_int g_scan_done = 0;
-static atomic_size_t g_files_found = 0;
-static atomic_uint_fast64_t g_bytes_processed = 0;
-// __________________________________________________________________________
-static CRITICAL_SECTION g_entries_cs;
-
-// ----------------------------- Utils --------------------------------------
-static void perror_exit(const char *msg) {
-  perror(msg);
-  exit(1);
-}
-
-static void *xmalloc(size_t n) {
-  void *p = malloc(n);
-  if (!p)
-    perror_exit("malloc");
-  return p;
-}
-
-static void global_entries_push(const FileEntry *src) {
-  if (g_entry_count == g_entry_capacity) {
-    size_t newcap = g_entry_capacity ? g_entry_capacity * 2 : 1024;
-    g_entries = realloc(g_entries, newcap * sizeof(FileEntry));
-    if (!g_entries)
-      perror_exit("realloc");
-    g_entry_capacity = newcap;
-  }
-
-  FileEntry *dst = &g_entries[g_entry_count++];
-  memset(dst, 0, sizeof(*dst));
-
-  dst->size_bytes = src->size_bytes;
-  dst->created_time = src->created_time;
-  dst->modified_time = src->modified_time;
-  dst->path = strdup(src->path);
-  strncpy(dst->owner, src->owner, sizeof(dst->owner) - 1);
-}
-
-static void free_entries(void) {
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    free(g_entries[i].path);
-  }
-
-  free(g_entries);
-  g_entries = NULL;
-  g_entry_count = 0;
-  g_entry_capacity = 0;
-}
-
-// ----------------------------- Convert filetime to epoch --------------
-static uint64_t filetime_to_epoch(const FILETIME *ft) {
-  ULARGE_INTEGER ull;
-  ull.LowPart = ft->dwLowDateTime;
-  ull.HighPart = ft->dwHighDateTime;
-
-  // Windows epoch (1601) → Unix epoch (1970)
-  return (ull.QuadPart - 116444736000000000ULL) / 10000000ULL;
-}
-
-// ----------------------------- Resolve file owner ---------------------
-static void get_file_owner(const char *path, char *out, size_t out_sz) {
-  PSID sid = NULL;
-  PSECURITY_DESCRIPTOR sd = NULL;
-
-  if (GetNamedSecurityInfoA(path, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION,
-                            &sid, NULL, NULL, NULL, &sd) == ERROR_SUCCESS) {
-
-    char name[64], domain[64];
-    DWORD name_len = sizeof(name);
-    DWORD domain_len = sizeof(domain);
-    SID_NAME_USE use;
-
-    if (LookupAccountSidA(NULL, sid, name, &name_len, domain, &domain_len,
-                          &use)) {
-      snprintf(out, out_sz, "%s\\%s", domain, name);
-    } else {
-      snprintf(out, out_sz, "UNKNOWN");
-    }
-  } else {
-    snprintf(out, out_sz, "UNKNOWN");
-  }
-
-  if (sd)
-    LocalFree(sd);
-}
-
-// ----------------------------- Format time helper -------------------------
-static void format_time(uint64_t t, char *out, size_t out_sz) {
-  if (t == 0) {
-    snprintf(out, out_sz, "N/A");
-    return;
-  }
-
-  time_t tt = (time_t)t;
-  struct tm tm;
-
-#if PLATFORM_WINDOWS
-  localtime_s(&tm, &tt);
-#else
-  localtime_r(&tt, &tm);
-#endif
-
-  strftime(out, out_sz, "%Y-%m-%d %H:%M:%S", &tm);
-}
-
-// --------------- parallel directory scanning ----------------
-static void entrybuf_init(EntryBuffer *b) {
-  b->entries = NULL;
-  b->count = 0;
-  b->capacity = 0;
-}
-
-static void entrybuf_push(EntryBuffer *b, const FileEntry *src) {
-  if (b->count == b->capacity) {
-    size_t newcap = b->capacity ? b->capacity * 2 : 256;
-    b->entries = realloc(b->entries, newcap * sizeof(FileEntry));
-    if (!b->entries)
-      perror_exit("realloc");
-    b->capacity = newcap;
-  }
-
-  FileEntry *dst = &b->entries[b->count++];
-  memset(dst, 0, sizeof(*dst));
-
-  dst->size_bytes = src->size_bytes;
-  dst->created_time = src->created_time;
-  dst->modified_time = src->modified_time;
-  dst->path = strdup(src->path);
-  strncpy(dst->owner, src->owner, sizeof(dst->owner) - 1);
-}
-
-// Add queue helper functions
-static void dirqueue_push(DirQueue *q, const char *path) {
-  EnterCriticalSection(&q->cs);
-
-  if (q->count + 1 > q->cap) {
-    q->cap = q->cap ? q->cap * 2 : 1024;
-    q->items = realloc(q->items, q->cap * sizeof(char *));
-  }
-
-  q->items[q->count++] = _strdup(path);
-
-  WakeConditionVariable(&q->cv);
-  LeaveCriticalSection(&q->cs);
-}
-
-static char *dirqueue_pop(DirQueue *q) {
-  EnterCriticalSection(&q->cs);
-
-  while (q->count == 0 && q->active > 0) {
-    SleepConditionVariableCS(&q->cv, &q->cs, INFINITE);
-  }
-
-  if (q->count == 0 && q->active == 0) {
-    LeaveCriticalSection(&q->cs);
-    return NULL; // truly done
-  }
-
-  char *dir = q->items[--q->count];
-  q->active++;
-
-  LeaveCriticalSection(&q->cs);
-  return dir;
-}
-
-static void dirqueue_done(DirQueue *q) {
-  EnterCriticalSection(&q->cs);
-  q->active--;
-  WakeAllConditionVariable(&q->cv);
-  LeaveCriticalSection(&q->cs);
-}
-static DWORD WINAPI scan_worker(LPVOID arg) {
-  DirQueue *q = (DirQueue *)arg;
-
-  EntryBuffer local;
-  entrybuf_init(&local);
-
-  for (;;) {
-    char *dir = dirqueue_pop(q);
-    if (!dir)
-      break;
-
-    scan_folder_windows_parallel(dir, q, &local);
-    // debug
-    // printf("[T%lu] scanning %s\n", GetCurrentThreadId(), dir);
-    // debug
-
-    free(dir);
-    dirqueue_done(q);
-  }
-
-  // merge once at end
-  EnterCriticalSection(&g_entries_cs);
-
-  if (g_entry_count + local.count > g_entry_capacity) {
-    g_entry_capacity = g_entry_count + local.count;
-    g_entries = realloc(g_entries, g_entry_capacity * sizeof(FileEntry));
-    if (!g_entries)
-      perror_exit("realloc");
-  }
-
-  memcpy(&g_entries[g_entry_count], local.entries,
-         local.count * sizeof(FileEntry));
-  g_entry_count += local.count;
-
-  LeaveCriticalSection(&g_entries_cs);
-
-  free(local.entries);
-  return 0;
-}
-
-// Scanning directory function
-void scan_folder_windows_parallel(const char *base, DirQueue *q,
-                                  EntryBuffer *buf) {
-  char search[MAX_PATHLEN];
-  snprintf(search, sizeof(search), "%s\\*", base);
-
-  WIN32_FIND_DATAA fd;
-  HANDLE h = FindFirstFileA(search, &fd);
-  if (h == INVALID_HANDLE_VALUE)
-    return;
-
-  do {
-    if (!strcmp(fd.cFileName, ".") || !strcmp(fd.cFileName, ".."))
-      continue;
-
-    char full[MAX_PATHLEN];
-    snprintf(full, sizeof(full), "%s\\%s", base, fd.cFileName);
-
-    if (fd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
-      continue;
-
-    if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
-      dirqueue_push(q, full);
-    } else {
-
-      atomic_fetch_add(&g_files_found, 1);
-
-      FileEntry fe;
-      memset(&fe, 0, sizeof(fe));
-
-      char norm[MAX_PATHLEN];
-      strncpy(norm, full, sizeof(norm) - 1);
-      norm[sizeof(norm) - 1] = 0;
-      normalize_path(norm);
-      fe.path = norm;
-
-      platform_get_file_times(full, &fe.created_time, &fe.modified_time);
-
-      platform_get_file_owner(full, fe.owner, sizeof(fe.owner));
-
-      LARGE_INTEGER size;
-      HANDLE hf =
-          CreateFileA(full, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
-                      NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-
-      if (hf != INVALID_HANDLE_VALUE) {
-        if (GetFileSizeEx(hf, &size))
-          fe.size_bytes = (uint64_t)size.QuadPart;
-        CloseHandle(hf);
-      }
-
-      entrybuf_push(buf, &fe);
-    }
-  } while (FindNextFileA(h, &fd));
-
-  FindClose(h);
-}
-
-// Scan progress thread
-static DWORD WINAPI scan_progress_thread(LPVOID arg) {
-  (void)arg;
-
-  for (;;) {
-    if (atomic_load(&g_scan_done))
-      break;
-
-    Sleep(100); // 0.2 seconds
-
-    size_t count = atomic_load(&g_files_found);
-
-    printf("\rScanning... %zu files found", count);
-    fflush(stdout);
-  }
-
-  return 0;
-}
-
-// ----------------------------- Job queue ----------------------------------
-static void jobqueue_init(JobQueue *q) {
-  q->head = q->tail = NULL;
-  atomic_store(&q->count, 0);
-  q->stop = 0;
-  InitializeCriticalSection(&q->cs);
-  InitializeConditionVariable(&q->cv);
-}
-
-static void jobqueue_push(JobQueue *q, Job *job) {
-  EnterCriticalSection(&q->cs);
-  job->next = NULL;
-  if (q->tail)
-    q->tail->next = job;
-  else
-    q->head = job;
-  q->tail = job;
-  atomic_fetch_add(&q->count, 1);
-  WakeConditionVariable(&q->cv);
-  LeaveCriticalSection(&q->cs);
-}
-
-static Job *jobqueue_pop(JobQueue *q) {
-  EnterCriticalSection(&q->cs);
-  while (!q->head && !q->stop)
-    SleepConditionVariableCS(&q->cv, &q->cs, INFINITE);
-  if (q->stop && !q->head) {
-    LeaveCriticalSection(&q->cs);
-    return NULL;
-  }
-  Job *j = q->head;
-  q->head = j->next;
-  if (!q->head)
-    q->tail = NULL;
-  LeaveCriticalSection(&q->cs);
-  if (j)
-    atomic_fetch_sub(&q->count, 1);
-  return j;
-}
-
-static void jobqueue_stop(JobQueue *q) {
-  EnterCriticalSection(&q->cs);
-  q->stop = 1;
-  WakeAllConditionVariable(&q->cv);
-  LeaveCriticalSection(&q->cs);
-}
-
-// ----------------------------- Hashing helpers -----------------------------
-static void xxh3_hash_file_stream(const char *path, char *out_hex) {
-  // compute XXH3_128 over file. POSIX and Windows use standard reads in this
-  // helper.
-  // On Windows try to use overlapped synchronous chunked reads for higher
-  // throughput.
-  HANDLE hFile =
-      CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
-                  OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
-  if (hFile == INVALID_HANDLE_VALUE) {
-    strcpy(out_hex, "ERROR");
-    return;
-  }
-  XXH128_hash_t h;
-  XXH3_state_t *state = XXH3_createState();
-  XXH3_128bits_reset(state);
-
-  BYTE *buf = (BYTE *)malloc(READ_BLOCK);
-  DWORD read = 0;
-  BOOL ok;
-  while (ReadFile(hFile, buf, READ_BLOCK, &read, NULL) && read > 0) {
-    XXH3_128bits_update(state, buf, (size_t)read);
-    atomic_fetch_add(&g_bytes_processed, (uint64_t)read);
-  }
-  h = XXH3_128bits_digest(state);
-  XXH3_freeState(state);
-  CloseHandle(hFile);
-  free(buf);
-  snprintf(out_hex, HASH_STRLEN, "%016llx%016llx", (unsigned long long)h.high64,
-           (unsigned long long)h.low64);
-}
-
-// ----------------------------- Worker --------------------------------------
-static DWORD WINAPI worker_thread_windows(LPVOID argp) {
-  WorkerArg *w = (WorkerArg *)argp;
-  JobQueue *q = w->queue;
-  for (;;) {
-    Job *job = jobqueue_pop(q);
-    if (!job)
-      break;
-    char hex[HASH_STRLEN];
-    // On Windows we use overlapped ReadFile for large files would be better,
-    // but ReadFile with NULL overlapped is sufficient inside parallel threads.
-    xxh3_hash_file_stream(job->file->path, hex);
-
-    // append to hashes file using a critical section to avoid races
-    static CRITICAL_SECTION append_cs;
-    static LONG init = 0;
-    if (InterlockedCompareExchange(&init, 1, 1) == 0) {
-      // first time initialize
-      InitializeCriticalSection(&append_cs);
-      InterlockedExchange(&init, 1);
-    }
-    EnterCriticalSection(&append_cs);
-    FILE *hf = fopen(FILE_HASHES_TXT, "a");
-    if (hf) {
-      char created[32], modified[32];
-
-      format_time(job->file->created_time, created, sizeof(created));
-      format_time(job->file->modified_time, modified, sizeof(modified));
-      double size_kib = (double)job->file->size_bytes / (1024.0);
-
-      fprintf(hf, "%s\t%s\t%.2f\t%s\t%s\t%s\n", hex, job->file->path, size_kib,
-              created, modified, job->file->owner);
-      fclose(hf);
-    }
-    LeaveCriticalSection(&append_cs);
-
-    atomic_fetch_add(w->done_counter, 1);
-    free(job);
-  }
-  atomic_fetch_sub(w->live_workers, 1);
-  return 0;
-}
-
-// ----------------------------- Progress display ---------------------------
-static void print_progress(size_t done, size_t total) {
-  const int barw = 40;
-  double pct = total ? (double)done / (double)total : 0.0;
-  int filled = (int)(pct * barw + 0.5);
-  printf("\r[");
-  for (int i = 0; i < filled; ++i)
-    putchar('#');
-  for (int i = filled; i < barw; ++i)
-    putchar(' ');
-  printf("] %6.2f%%  (%zu / %zu)  ", pct * 100.0, done, total);
-  fflush(stdout);
-}
-
-// ----------------------------- Helpers: load/save --------------------------
-static int file_exists(const char *path) {
-  DWORD attr = GetFileAttributesA(path);
-  return attr != INVALID_FILE_ATTRIBUTES;
-}
-
-static void save_file_list(const char *list_path) {
-  FILE *f = fopen(list_path, "w");
-  if (!f) {
-    perror("fopen file_list");
-    return;
-  }
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    fprintf(f, "%s\n", g_entries[i].path);
-  }
-  fclose(f);
-}
-
-// ----------------------------- Get file metadata -------------------------
-void platform_get_file_times(const char *path, uint64_t *out_created,
-                             uint64_t *out_modified) {
-  WIN32_FILE_ATTRIBUTE_DATA fad;
-  if (GetFileAttributesExA(path, GetFileExInfoStandard, &fad)) {
-    *out_created = filetime_to_epoch(&fad.ftCreationTime);
-    *out_modified = filetime_to_epoch(&fad.ftLastWriteTime);
-  } else {
-    *out_created = 0;
-    *out_modified = 0;
-  }
-}
-
-void platform_get_file_owner(const char *path, char *out_owner,
-                             size_t out_owner_size) {
-  get_file_owner(path, out_owner, out_owner_size);
-}
-
-// ----------------------------- Main ---------------------------------------
-int main(int argc, char **argv) {
-  char folders[64][MAX_PATHLEN]; // up to 64 input folders
-  int folder_count = 0;
-
-  // -------------------------------
-  // Scanning and total timer init
-  // -------------------------------
-  timer_init();
-
-  HiResTimer total_timer;
-  HiResTimer scan_timer;
-  HiResTimer hash_timer;
-
-  timer_start(&total_timer);
-  timer_start(&scan_timer);
-
-  // -------------------------------
-  // Parse arguments
-  // -------------------------------
-  for (int i = 1; i < argc; ++i) {
-    if (folder_count < 64) {
-      strncpy(folders[folder_count], argv[i], MAX_PATHLEN - 1);
-      folders[folder_count][MAX_PATHLEN - 1] = 0;
-      folder_count++;
-    }
-  }
-
-  // -------------------------------
-  // Ask user if no folders provided
-  // -------------------------------
-  if (folder_count == 0) {
-    printf("Enter folder to process (Enter = current folder): ");
-    fflush(stdout);
-
-    char buf[MAX_PATHLEN];
-    if (!fgets(buf, sizeof(buf), stdin))
-      return 1;
-    buf[strcspn(buf, "\r\n")] = 0;
-
-    if (buf[0] == 0)
-      strcpy(folders[0], ".");
-    else
-      strncpy(folders[0], buf, MAX_PATHLEN - 1);
-
-    folder_count = 1;
-  }
-
-  // -------------------------------
-  // Display selected folders
-  // -------------------------------
-  printf("Processing %d folder(s):\n", folder_count);
-  for (int i = 0; i < folder_count; ++i) {
-    printf("  - %s\n", folders[i]);
-  }
-
-  // -------------------------------
-  // Detect hardware threads (CPU cores)
-  // -------------------------------
-  size_t hw_threads = 1;
-  // --- Windows: detect PHYSICAL cores (not logical threads) ---
-  DWORD len = 0;
-  GetLogicalProcessorInformation(NULL, &len);
-
-  SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf =
-      (SYSTEM_LOGICAL_PROCESSOR_INFORMATION *)malloc(len);
-
-  if (GetLogicalProcessorInformation(buf, &len)) {
-    DWORD count = 0;
-    DWORD n = len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
-    for (DWORD i = 0; i < n; i++) {
-      if (buf[i].Relationship == RelationProcessorCore)
-        count++;
-    }
-    if (count > 0)
-      hw_threads = count;
-  }
-  free(buf);
-
-  // Add some extra threads to overlap I/O more aggressively
-  size_t num_threads = hw_threads * 2;
-  if (num_threads < 2)
-    num_threads = 2;
-
-  // -------------------------------
-  // Step 1: Scan all folders
-  // -------------------------------
-  InitializeCriticalSection(&g_entries_cs);
-
-  DirQueue q;
-  memset(&q, 0, sizeof(q));
-  InitializeCriticalSection(&q.cs);
-  InitializeConditionVariable(&q.cv);
-  q.active = 0;
-
-  HANDLE scan_progress =
-      CreateThread(NULL, 0, scan_progress_thread, NULL, 0, NULL);
-
-  for (int i = 0; i < folder_count; ++i) {
-    dirqueue_push(&q, folders[i]);
-  }
-
-  size_t scan_threads = hw_threads;
-  if (scan_threads < 2)
-    scan_threads = 2;
-
-  HANDLE *scan_tids = malloc(sizeof(HANDLE) * scan_threads);
-  for (size_t i = 0; i < scan_threads; ++i) {
-    scan_tids[i] =
-        CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)scan_worker, &q, 0, NULL);
-  }
-
-  WaitForMultipleObjects((DWORD)scan_threads, scan_tids, TRUE, INFINITE);
-
-  atomic_store(&g_scan_done, 1);
-  WaitForSingleObject(scan_progress, INFINITE);
-  CloseHandle(scan_progress);
-
-  for (size_t i = 0; i < scan_threads; ++i)
-    CloseHandle(scan_tids[i]);
-  free(scan_tids);
-
-  double scan_seconds = timer_stop(&scan_timer);
-  double scan_rate = (double)g_entry_count / scan_seconds;
-
-  printf(". Scan rate   : %.1f files/sec\n", scan_rate);
-  printf("Completed scanning in %.2f seconds. Saving to %s\n\n", scan_seconds,
-         FILE_LIST_TXT);
-  save_file_list(FILE_LIST_TXT);
-
-  if (g_entry_count == 0) {
-    printf("No files to process.\n");
-    return 0;
-  }
-
-  DeleteCriticalSection(&g_entries_cs);
-
-  // Prepare job queue
-  JobQueue queue;
-  jobqueue_init(&queue);
-
-  size_t total_jobs = 0;
-  for (size_t i = 0; i < g_entry_count; ++i) {
-    Job *j = (Job *)malloc(sizeof(Job));
-    j->file = &g_entries[i];
-    j->next = NULL;
-    jobqueue_push(&queue, j);
-    ++total_jobs;
-  }
-
-  if (total_jobs == 0) {
-    printf("Nothing to do — all files already hashed.\n");
-    return 0;
-  }
-
-  FILE *hf = fopen(FILE_HASHES_TXT, "w");
-  if (hf)
-    fclose(hf);
-
-  // Starting thread pool
-  atomic_size_t done_counter;
-  atomic_store(&done_counter, 0);
-  atomic_int live_workers;
-  atomic_store(&live_workers, (int)num_threads);
-
-  WorkerArg warg = {.queue = &queue,
-                    .done_counter = &done_counter,
-                    .total_jobs = total_jobs,
-                    .live_workers = &live_workers};
-
-  printf("Starting thread pool: %zu threads (CPU cores: %zu)\n", num_threads,
-         hw_threads);
-
-  // Launch threads
-  HANDLE *tids = malloc(sizeof(HANDLE) * num_threads);
-  for (size_t i = 0; i < num_threads; ++i) {
-    tids[i] = CreateThread(NULL, 0, worker_thread_windows, &warg, 0, NULL);
-  }
-
-  // Progress / timer
-  struct timespec tstart, tnow;
-  // fallback for windows
-  LARGE_INTEGER freq, start_li;
-  QueryPerformanceFrequency(&freq);
-  QueryPerformanceCounter(&start_li);
-
-  size_t last_done = 0;
-
-  // --------------- Hashing speed MB/s ----------------
-  uint64_t last_bytes = atomic_load(&g_bytes_processed);
-  double last_time = 0.0;
-  double displayed_speed = 0.0;
-  const double sample_interval = 0.5;
-  char linebuf[256];
-
-  for (;;) {
-    size_t done = (size_t)atomic_load(&done_counter);
-
-    // ---- monotonic time ----
-    LARGE_INTEGER now_li;
-    QueryPerformanceCounter(&now_li);
-    double now =
-        (double)(now_li.QuadPart - start_li.QuadPart) / (double)freq.QuadPart;
-
-    // ---- total processed bytes ----
-    uint64_t bytes = atomic_load(&g_bytes_processed);
-
-    // ---- real sampler (independent of UI sleep) ----
-    if (last_time == 0.0) {
-      last_time = now;
-      last_bytes = bytes;
-    }
-
-    double dt = now - last_time;
-    if (dt >= sample_interval) {
-      uint64_t db = bytes - last_bytes;
-
-      if (db > 0 && dt > 0.0001) {
-        displayed_speed = (double)db / (1024.0 * 1024.0) / dt;
-      }
-
-      last_bytes = bytes;
-      last_time = now;
-    }
-
-    // ---- progress bar build ----
-    const int barw = 40;
-    double pct = total_jobs ? (double)done / (double)total_jobs : 0.0;
-    int filled = (int)(pct * barw + 0.5);
-
-    int p = 0;
-    p += snprintf(linebuf + p, sizeof(linebuf) - p, "[");
-    for (int i = 0; i < filled && p < (int)sizeof(linebuf); ++i)
-      p += snprintf(linebuf + p, sizeof(linebuf) - p, "#");
-    for (int i = filled; i < barw && p < (int)sizeof(linebuf); ++i)
-      p += snprintf(linebuf + p, sizeof(linebuf) - p, ".");
-
-    snprintf(linebuf + p, sizeof(linebuf) - p,
-             "] %6.2f%% (%zu / %zu)  %8.2f MB/s", pct * 100.0, done, total_jobs,
-             displayed_speed);
-
-    printf("\r%s", linebuf);
-    fflush(stdout);
-
-    if (done >= total_jobs)
-      break;
-
-    Sleep(100);
-  }
-
-  printf("\n\n");
-
-  // stop queue and join threads
-  jobqueue_stop(&queue);
-  WaitForMultipleObjects((DWORD)num_threads, tids, TRUE, INFINITE);
-  for (size_t i = 0; i < num_threads; ++i)
-    CloseHandle(tids[i]);
-
-  // done time
-  LARGE_INTEGER end_li;
-  QueryPerformanceCounter(&end_li);
-  double elapsed =
-      (double)(end_li.QuadPart - start_li.QuadPart) / (double)freq.QuadPart;
-  double total_seconds = timer_stop(&total_timer);
-
-  printf("Completed hashing %zu files in %.2f seconds\n", total_jobs, elapsed);
-
-  uint64_t total_bytes = (uint64_t)atomic_load(&g_bytes_processed);
-  double total_mb = (double)total_bytes / (1024.0 * 1024.0);
-  double avg_mbps = total_mb / elapsed;
-  printf("Total: %.2f MB, Average: %.2f MB/s\n", total_mb, avg_mbps);
-  printf("  Total time  : %.2f seconds\n", total_seconds);
-
-  return 0;
-}

xxh_x86dispatch.c

@@ -0,0 +1,821 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2020-2021 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other materials provided with the
+ *      distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ *   - xxHash homepage: https://www.xxhash.com
+ *   - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+
+/*!
+ * @file xxh_x86dispatch.c
+ *
+ * Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
+ *
+ * Optional add-on.
+ *
+ * **Compile this file with the default flags for your target.**
+ * Note that compiling with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`
+ * will make the resulting binary incompatible with cpus not supporting the requested instruction set.
+ *
+ * @defgroup dispatch x86 Dispatcher
+ * @{
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64))
+#  error "Dispatching is currently only supported on x86 and x86_64."
+#endif
+
+/*! @cond Doxygen ignores this part */
+#ifndef XXH_HAS_INCLUDE
+#  ifdef __has_include
+/*
+ * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
+ * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
+ */
+#    define XXH_HAS_INCLUDE __has_include
+#  else
+#    define XXH_HAS_INCLUDE(x) 0
+#  endif
+#endif
+/*! @endcond */
+
+/*!
+ * @def XXH_DISPATCH_SCALAR
+ * @brief Enables/dispatching the scalar code path.
+ *
+ * If this is defined to 0, SSE2 support is assumed. This reduces code size
+ * when the scalar path is not needed.
+ *
+ * This is automatically defined to 0 when...
+ *   - SSE2 support is enabled in the compiler
+ *   - Targeting x86_64
+ *   - Targeting Android x86
+ *   - Targeting macOS
+ */
+#ifndef XXH_DISPATCH_SCALAR
+#  if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
+     || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
+     || defined(__ANDROID__) || defined(__APPLE__) /* Android or macOS */
+#     define XXH_DISPATCH_SCALAR 0 /* disable */
+#  else
+#     define XXH_DISPATCH_SCALAR 1
+#  endif
+#endif
+/*!
+ * @def XXH_DISPATCH_AVX2
+ * @brief Enables/disables dispatching for AVX2.
+ *
+ * This is automatically detected if it is not defined.
+ *  - GCC 4.7 and later are known to support AVX2, but >4.9 is required for
+ *    to get the AVX2 intrinsics and typedefs without -mavx -mavx2.
+ *  - Visual Studio 2013 Update 2 and later are known to support AVX2.
+ *  - The GCC/Clang internal header `<avx2intrin.h>` is detected. While this is
+ *    not allowed to be included directly, it still appears in the builtin
+ *    include path and is detectable with `__has_include`.
+ *
+ * @see XXH_AVX2
+ */
+#ifndef XXH_DISPATCH_AVX2
+#  if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \
+   || (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \
+   || (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \
+   || XXH_HAS_INCLUDE(<avx2intrin.h>) /* GCC/Clang internal header */
+#    define XXH_DISPATCH_AVX2 1   /* enable dispatch towards AVX2 */
+#  else
+#    define XXH_DISPATCH_AVX2 0
+#  endif
+#endif /* XXH_DISPATCH_AVX2 */
+
+/*!
+ * @def XXH_DISPATCH_AVX512
+ * @brief Enables/disables dispatching for AVX512.
+ *
+ * Automatically detected if one of the following conditions is met:
+ *  - GCC 4.9 and later are known to support AVX512.
+ *  - Visual Studio 2017  and later are known to support AVX2.
+ *  - The GCC/Clang internal header `<avx512fintrin.h>` is detected. While this
+ *    is not allowed to be included directly, it still appears in the builtin
+ *    include path and is detectable with `__has_include`.
+ *
+ * @see XXH_AVX512
+ */
+#ifndef XXH_DISPATCH_AVX512
+#  if (defined(__GNUC__) \
+       && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \
+   || (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \
+   || XXH_HAS_INCLUDE(<avx512fintrin.h>) /* GCC/Clang internal header */
+#    define XXH_DISPATCH_AVX512 1   /* enable dispatch towards AVX512 */
+#  else
+#    define XXH_DISPATCH_AVX512 0
+#  endif
+#endif /* XXH_DISPATCH_AVX512 */
+
+/*!
+ * @def XXH_TARGET_SSE2
+ * @brief Allows a function to be compiled with SSE2 intrinsics.
+ *
+ * Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used
+ * even with `-mno-sse2`.
+ *
+ * @def XXH_TARGET_AVX2
+ * @brief Like @ref XXH_TARGET_SSE2, but for AVX2.
+ *
+ * @def XXH_TARGET_AVX512
+ * @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
+ *
+ */
+#if defined(__GNUC__)
+#  include <emmintrin.h> /* SSE2 */
+#  if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
+#    include <immintrin.h> /* AVX2, AVX512F */
+#  endif
+#  define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
+#  define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
+#  define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
+#elif defined(__clang__) && defined(_MSC_VER) /* clang-cl.exe */
+#  include <emmintrin.h> /* SSE2 */
+#  if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
+#    include <immintrin.h> /* AVX2, AVX512F */
+#    include <smmintrin.h>
+#    include <avxintrin.h>
+#    include <avx2intrin.h>
+#    include <avx512fintrin.h>
+#  endif
+#  define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
+#  define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
+#  define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
+#elif defined(_MSC_VER)
+#  include <intrin.h>
+#  define XXH_TARGET_SSE2
+#  define XXH_TARGET_AVX2
+#  define XXH_TARGET_AVX512
+#else
+#  error "Dispatching is currently not supported for your compiler."
+#endif
+
+/*! @cond Doxygen ignores this part */
+#ifdef XXH_DISPATCH_DEBUG
+/* debug logging */
+#  include <stdio.h>
+#  define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); }
+#else
+#  define XXH_debugPrint(str) ((void)0)
+#  undef NDEBUG /* avoid redefinition */
+#  define NDEBUG
+#endif
+/*! @endcond */
+#include <assert.h>
+
+#ifndef XXH_DOXYGEN
+#define XXH_INLINE_ALL
+#define XXH_X86DISPATCH
+#include "xxhash.h"
+#endif
+
+/*! @cond Doxygen ignores this part */
+#ifndef XXH_HAS_ATTRIBUTE
+#  ifdef __has_attribute
+#    define XXH_HAS_ATTRIBUTE(...) __has_attribute(__VA_ARGS__)
+#  else
+#    define XXH_HAS_ATTRIBUTE(...) 0
+#  endif
+#endif
+/*! @endcond */
+
+/*! @cond Doxygen ignores this part */
+#if XXH_HAS_ATTRIBUTE(constructor)
+#  define XXH_CONSTRUCTOR __attribute__((constructor))
+#  define XXH_DISPATCH_MAYBE_NULL 0
+#else
+#  define XXH_CONSTRUCTOR
+#  define XXH_DISPATCH_MAYBE_NULL 1
+#endif
+/*! @endcond */
+
+
+/*! @cond Doxygen ignores this part */
+/*
+ * Support both AT&T and Intel dialects
+ *
+ * GCC doesn't convert AT&T syntax to Intel syntax, and will error out if
+ * compiled with -masm=intel. Instead, it supports dialect switching with
+ * curly braces: { AT&T syntax | Intel syntax }
+ *
+ * Clang's integrated assembler automatically converts AT&T syntax to Intel if
+ * needed, making the dialect switching useless (it isn't even supported).
+ *
+ * Note: Comments are written in the inline assembly itself.
+ */
+#ifdef __clang__
+#  define XXH_I_ATT(intel, att) att "\n\t"
+#else
+#  define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
+#endif
+/*! @endcond */
+
+/*!
+ * @private
+ * @brief Runs CPUID.
+ *
+ * @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
+ * @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
+ */
+static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
+{
+#if defined(_MSC_VER)
+    __cpuidex((int*)abcd, eax, ecx);
+#else
+    xxh_u32 ebx, edx;
+# if defined(__i386__) && defined(__PIC__)
+    __asm__(
+        "# Call CPUID\n\t"
+        "#\n\t"
+        "# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t"
+        "# EBX, so we use EDI instead.\n\t"
+        XXH_I_ATT("mov     edi, ebx",   "movl    %%ebx, %%edi")
+        XXH_I_ATT("cpuid",              "cpuid"               )
+        XXH_I_ATT("xchg    edi, ebx",   "xchgl   %%ebx, %%edi")
+        : "=D" (ebx),
+# else
+    __asm__(
+        "# Call CPUID\n\t"
+        XXH_I_ATT("cpuid",              "cpuid")
+        : "=b" (ebx),
+# endif
+              "+a" (eax), "+c" (ecx), "=d" (edx));
+    abcd[0] = eax;
+    abcd[1] = ebx;
+    abcd[2] = ecx;
+    abcd[3] = edx;
+#endif
+}
+
+/*
+ * Modified version of Intel's guide
+ * https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
+ */
+
+#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
+/*!
+ * @private
+ * @brief Runs `XGETBV`.
+ *
+ * While the CPU may support AVX2, the operating system might not properly save
+ * the full YMM/ZMM registers.
+ *
+ * xgetbv is used for detecting this: Any compliant operating system will define
+ * a set of flags in the xcr0 register indicating how it saves the AVX registers.
+ *
+ * You can manually disable this flag on Windows by running, as admin:
+ *
+ *   bcdedit.exe /set xsavedisable 1
+ *
+ * and rebooting. Run the same command with 0 to re-enable it.
+ */
+static xxh_u64 XXH_xgetbv(void)
+{
+#if defined(_MSC_VER)
+    return _xgetbv(0);  /* min VS2010 SP1 compiler is required */
+#else
+    xxh_u32 xcr0_lo, xcr0_hi;
+    __asm__(
+        "# Call XGETBV\n\t"
+        "#\n\t"
+        "# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t"
+        "# the XGETBV opcode, so we encode it by hand instead.\n\t"
+        "# See <https://github.com/asmjit/asmjit/issues/78> for details.\n\t"
+        ".byte   0x0f, 0x01, 0xd0\n\t"
+       : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
+    return xcr0_lo | ((xxh_u64)xcr0_hi << 32);
+#endif
+}
+#endif
+
+/*! @cond Doxygen ignores this part */
+#define XXH_SSE2_CPUID_MASK (1 << 26)
+#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
+#define XXH_AVX2_CPUID_MASK (1 << 5)
+#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
+#define XXH_AVX512F_CPUID_MASK (1 << 16)
+#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
+/*! @endcond */
+
+/*!
+ * @private
+ * @brief Returns the best XXH3 implementation.
+ *
+ * Runs various CPUID/XGETBV tests to try and determine the best implementation.
+ *
+ * @return The best @ref XXH_VECTOR implementation.
+ * @see XXH_VECTOR_TYPES
+ */
+int XXH_featureTest(void)
+{
+    xxh_u32 abcd[4];
+    xxh_u32 max_leaves;
+    int best = XXH_SCALAR;
+#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
+    xxh_u64 xgetbv_val;
+#endif
+#if defined(__GNUC__) && defined(__i386__)
+    xxh_u32 cpuid_supported;
+    __asm__(
+        "# For the sake of ruthless backwards compatibility, check if CPUID\n\t"
+        "# is supported in the EFLAGS on i386.\n\t"
+        "# This is not necessary on x86_64 - CPUID is mandatory.\n\t"
+        "#   The ID flag (bit 21) in the EFLAGS register indicates support\n\t"
+        "#   for the CPUID instruction. If a software procedure can set and\n\t"
+        "#   clear this flag, the processor executing the procedure supports\n\t"
+        "#   the CPUID instruction.\n\t"
+        "#   <https://c9x.me/x86/html/file_module_x86_id_45.html>\n\t"
+        "#\n\t"
+        "# Routine is from <https://wiki.osdev.org/CPUID>.\n\t"
+
+        "# Save EFLAGS\n\t"
+        XXH_I_ATT("pushfd",                           "pushfl"                    )
+        "# Store EFLAGS\n\t"
+        XXH_I_ATT("pushfd",                           "pushfl"                    )
+        "# Invert the ID bit in stored EFLAGS\n\t"
+        XXH_I_ATT("xor     dword ptr[esp], 0x200000", "xorl    $0x200000, (%%esp)")
+        "# Load stored EFLAGS (with ID bit inverted)\n\t"
+        XXH_I_ATT("popfd",                            "popfl"                     )
+        "# Store EFLAGS again (ID bit may or not be inverted)\n\t"
+        XXH_I_ATT("pushfd",                           "pushfl"                    )
+        "# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t"
+        XXH_I_ATT("pop     eax",                      "popl    %%eax"             )
+        "# eax = whichever bits were changed\n\t"
+        XXH_I_ATT("xor     eax, dword ptr[esp]",      "xorl    (%%esp), %%eax"    )
+        "# Restore original EFLAGS\n\t"
+        XXH_I_ATT("popfd",                            "popfl"                     )
+        "# eax = zero if ID bit can't be changed, else non-zero\n\t"
+        XXH_I_ATT("and     eax, 0x200000",            "andl    $0x200000, %%eax"  )
+        : "=a" (cpuid_supported) :: "cc");
+
+    if (XXH_unlikely(!cpuid_supported)) {
+        XXH_debugPrint("CPUID support is not detected!");
+        return best;
+    }
+
+#endif
+    /* Check how many CPUID pages we have */
+    XXH_cpuid(0, 0, abcd);
+    max_leaves = abcd[0];
+
+    /* Shouldn't happen on hardware, but happens on some QEMU configs. */
+    if (XXH_unlikely(max_leaves == 0)) {
+        XXH_debugPrint("Max CPUID leaves == 0!");
+        return best;
+    }
+
+    /* Check for SSE2, OSXSAVE and xgetbv */
+    XXH_cpuid(1, 0, abcd);
+
+    /*
+     * Test for SSE2. The check is redundant on x86_64, but it doesn't hurt.
+     */
+    if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK))
+        return best;
+
+    XXH_debugPrint("SSE2 support detected.");
+
+    best = XXH_SSE2;
+#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
+    /* Make sure we have enough leaves */
+    if (XXH_unlikely(max_leaves < 7))
+        return best;
+
+    /* Test for OSXSAVE and XGETBV */
+    if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK)
+        return best;
+
+    /* CPUID check for AVX features */
+    XXH_cpuid(7, 0, abcd);
+
+    xgetbv_val = XXH_xgetbv();
+#if XXH_DISPATCH_AVX2
+    /* Validate that AVX2 is supported by the CPU */
+    if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK)
+        return best;
+
+    /* Validate that the OS supports YMM registers */
+    if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) {
+        XXH_debugPrint("AVX2 supported by the CPU, but not the OS.");
+        return best;
+    }
+
+    /* AVX2 supported */
+    XXH_debugPrint("AVX2 support detected.");
+    best = XXH_AVX2;
+#endif
+#if XXH_DISPATCH_AVX512
+    /* Check if AVX512F is supported by the CPU */
+    if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) {
+        XXH_debugPrint("AVX512F not supported by CPU");
+        return best;
+    }
+
+    /* Validate that the OS supports ZMM registers */
+    if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) {
+        XXH_debugPrint("AVX512F supported by the CPU, but not the OS.");
+        return best;
+    }
+
+    /* AVX512F supported */
+    XXH_debugPrint("AVX512F support detected.");
+    best = XXH_AVX512;
+#endif
+#endif
+    return best;
+}
+
+
+/* ===   Vector implementations   === */
+
+/*! @cond PRIVATE */
+/*!
+ * @private
+ * @brief Defines the various dispatch functions.
+ *
+ * TODO: Consolidate?
+ *
+ * @param suffix The suffix for the functions, e.g. sse2 or scalar
+ * @param target XXH_TARGET_* or empty.
+ */
+
+#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target)                             \
+                                                                              \
+/* ===   XXH3, default variants   === */                                      \
+                                                                              \
+XXH_NO_INLINE target XXH64_hash_t                                             \
+XXHL64_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input,          \
+                        size_t len)                                           \
+{                                                                             \
+    return XXH3_hashLong_64b_internal(                                        \
+               input, len, XXH3_kSecret, sizeof(XXH3_kSecret),                \
+               XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix            \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+/* ===   XXH3, Seeded variants   === */                                       \
+                                                                              \
+XXH_NO_INLINE target XXH64_hash_t                                             \
+XXHL64_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len, \
+                     XXH64_hash_t seed)                                       \
+{                                                                             \
+    return XXH3_hashLong_64b_withSeed_internal(                               \
+                    input, len, seed, XXH3_accumulate_##suffix,               \
+                    XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+/* ===   XXH3, Secret variants   === */                                       \
+                                                                              \
+XXH_NO_INLINE target XXH64_hash_t                                             \
+XXHL64_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input,           \
+                       size_t len, XXH_NOESCAPE const void* secret,           \
+                       size_t secretLen)                                      \
+{                                                                             \
+    return XXH3_hashLong_64b_internal(                                        \
+                    input, len, secret, secretLen,                            \
+                    XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix       \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+/* ===   XXH3 update variants   === */                                        \
+                                                                              \
+XXH_NO_INLINE target XXH_errorcode                                            \
+XXH3_update_##suffix(XXH_NOESCAPE XXH3_state_t* state,                        \
+                     XXH_NOESCAPE const void* input, size_t len)              \
+{                                                                             \
+    return XXH3_update(state, (const xxh_u8*)input, len,                      \
+                    XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix);     \
+}                                                                             \
+                                                                              \
+/* ===   XXH128 default variants   === */                                     \
+                                                                              \
+XXH_NO_INLINE target XXH128_hash_t                                            \
+XXHL128_default_##suffix(XXH_NOESCAPE  const void* XXH_RESTRICT input,        \
+                         size_t len)                                          \
+{                                                                             \
+    return XXH3_hashLong_128b_internal(                                       \
+                    input, len, XXH3_kSecret, sizeof(XXH3_kSecret),           \
+                    XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix       \
+    );                                                                        \
+}                                                                             \
+                                                                              \
+/* ===   XXH128 Secret variants   === */                                      \
+                                                                              \
+XXH_NO_INLINE target XXH128_hash_t                                            \
+XXHL128_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input,          \
+                        size_t len,                                           \
+                        XXH_NOESCAPE const void* XXH_RESTRICT secret,         \
+                        size_t secretLen)                                     \
+{                                                                             \
+    return XXH3_hashLong_128b_internal(                                       \
+                    input, len, (const xxh_u8*)secret, secretLen,             \
+                    XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix);     \
+}                                                                             \
+                                                                              \
+/* ===   XXH128 Seeded variants   === */                                      \
+                                                                              \
+XXH_NO_INLINE target XXH128_hash_t                                            \
+XXHL128_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len,\
+                      XXH64_hash_t seed)                                      \
+{                                                                             \
+    return XXH3_hashLong_128b_withSeed_internal(input, len, seed,             \
+                    XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix,      \
+                    XXH3_initCustomSecret_##suffix);                          \
+}
+
+/*! @endcond */
+/* End XXH_DEFINE_DISPATCH_FUNCS */
+
+/*! @cond Doxygen ignores this part */
+#if XXH_DISPATCH_SCALAR
+XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
+#endif
+XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2)
+#if XXH_DISPATCH_AVX2
+XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
+#endif
+#if XXH_DISPATCH_AVX512
+XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
+#endif
+#undef XXH_DEFINE_DISPATCH_FUNCS
+/*! @endcond */
+
+/* ====    Dispatchers    ==== */
+
+/*! @cond Doxygen ignores this part */
+typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
+
+typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
+
+typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
+
+typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH_NOESCAPE XXH3_state_t*, XXH_NOESCAPE const void*, size_t);
+
+typedef struct {
+    XXH3_dispatchx86_hashLong64_default    hashLong64_default;
+    XXH3_dispatchx86_hashLong64_withSeed   hashLong64_seed;
+    XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret;
+    XXH3_dispatchx86_update                update;
+} XXH_dispatchFunctions_s;
+
+#define XXH_NB_DISPATCHES 4
+/*! @endcond */
+
+/*!
+ * @private
+ * @brief Table of dispatchers for @ref XXH3_64bits().
+ *
+ * @pre The indices must match @ref XXH_VECTOR_TYPE.
+ */
+static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
+#if XXH_DISPATCH_SCALAR
+    /* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar },
+#else
+    /* Scalar */ { NULL, NULL, NULL, NULL },
+#endif
+    /* SSE2   */ { XXHL64_default_sse2,   XXHL64_seed_sse2,   XXHL64_secret_sse2,   XXH3_update_sse2 },
+#if XXH_DISPATCH_AVX2
+    /* AVX2   */ { XXHL64_default_avx2,   XXHL64_seed_avx2,   XXHL64_secret_avx2,   XXH3_update_avx2 },
+#else
+    /* AVX2   */ { NULL, NULL, NULL, NULL },
+#endif
+#if XXH_DISPATCH_AVX512
+    /* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 }
+#else
+    /* AVX512 */ { NULL, NULL, NULL, NULL }
+#endif
+};
+/*!
+ * @private
+ * @brief The selected dispatch table for @ref XXH3_64bits().
+ */
+static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
+
+
+/*! @cond Doxygen ignores this part */
+typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
+
+typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
+
+typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
+
+typedef struct {
+    XXH3_dispatchx86_hashLong128_default    hashLong128_default;
+    XXH3_dispatchx86_hashLong128_withSeed   hashLong128_seed;
+    XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
+    XXH3_dispatchx86_update                 update;
+} XXH_dispatch128Functions_s;
+/*! @endcond */
+
+
+/*!
+ * @private
+ * @brief Table of dispatchers for @ref XXH3_128bits().
+ *
+ * @pre The indices must match @ref XXH_VECTOR_TYPE.
+ */
+static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
+#if XXH_DISPATCH_SCALAR
+    /* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar },
+#else
+    /* Scalar */ { NULL, NULL, NULL, NULL },
+#endif
+    /* SSE2   */ { XXHL128_default_sse2,   XXHL128_seed_sse2,   XXHL128_secret_sse2,   XXH3_update_sse2 },
+#if XXH_DISPATCH_AVX2
+    /* AVX2   */ { XXHL128_default_avx2,   XXHL128_seed_avx2,   XXHL128_secret_avx2,   XXH3_update_avx2 },
+#else
+    /* AVX2   */ { NULL, NULL, NULL, NULL },
+#endif
+#if XXH_DISPATCH_AVX512
+    /* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 }
+#else
+    /* AVX512 */ { NULL, NULL, NULL, NULL }
+#endif
+};
+
+/*!
+ * @private
+ * @brief The selected dispatch table for @ref XXH3_64bits().
+ */
+static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
+
+/*!
+ * @private
+ * @brief Runs a CPUID check and sets the correct dispatch tables.
+ */
+static XXH_CONSTRUCTOR void XXH_setDispatch(void)
+{
+    int vecID = XXH_featureTest();
+    XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
+    assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512);
+#if !XXH_DISPATCH_SCALAR
+    assert(vecID != XXH_SCALAR);
+#endif
+#if !XXH_DISPATCH_AVX512
+    assert(vecID != XXH_AVX512);
+#endif
+#if !XXH_DISPATCH_AVX2
+    assert(vecID != XXH_AVX2);
+#endif
+    XXH_g_dispatch = XXH_kDispatch[vecID];
+    XXH_g_dispatch128 = XXH_kDispatch128[vecID];
+}
+
+
+/* ====    XXH3 public functions    ==== */
+/*! @cond Doxygen ignores this part */
+
+static XXH64_hash_t
+XXH3_hashLong_64b_defaultSecret_selection(const void* XXH_RESTRICT input, size_t len,
+                                          XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_default == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch.hashLong64_default(input, len);
+}
+
+XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
+{
+    return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
+}
+
+static XXH64_hash_t
+XXH3_hashLong_64b_withSeed_selection(const void* XXH_RESTRICT input, size_t len,
+                                     XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_seed == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
+}
+
+XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
+}
+
+static XXH64_hash_t
+XXH3_hashLong_64b_withSecret_selection(const void* XXH_RESTRICT input, size_t len,
+                                       XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
+{
+    (void)seed64;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_secret == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
+}
+
+XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
+{
+    return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
+}
+
+XXH_errorcode
+XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
+{
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.update == NULL)
+        XXH_setDispatch();
+
+    return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
+}
+
+/*! @endcond */
+
+
+/* ====    XXH128 public functions    ==== */
+/*! @cond Doxygen ignores this part */
+
+static XXH128_hash_t
+XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
+                                           XXH64_hash_t seed64, const void* secret, size_t secretLen)
+{
+    (void)seed64; (void)secret; (void)secretLen;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_default == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch128.hashLong128_default(input, len);
+}
+
+XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
+{
+    return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
+}
+
+static XXH128_hash_t
+XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
+                                      XXH64_hash_t seed64, const void* secret, size_t secretLen)
+{
+    (void)secret; (void)secretLen;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_seed == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
+}
+
+XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
+{
+    return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
+}
+
+static XXH128_hash_t
+XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
+                                        XXH64_hash_t seed64, const void* secret, size_t secretLen)
+{
+    (void)seed64;
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_secret == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
+}
+
+XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
+{
+    return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
+}
+
+XXH_errorcode
+XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
+{
+    if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.update == NULL)
+        XXH_setDispatch();
+    return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
+}
+
+/*! @endcond */
+
+#if defined (__cplusplus)
+}
+#endif
+/*! @} */

xxh_x86dispatch.h

@@ -0,0 +1,93 @@
+/*
+ * xxHash - XXH3 Dispatcher for x86-based targets
+ * Copyright (C) 2020-2024 Yann Collet
+ *
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other materials provided with the
+ *      distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at:
+ *   - xxHash homepage: https://www.xxhash.com
+ *   - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+#ifndef XXH_X86DISPATCH_H_13563687684
+#define XXH_X86DISPATCH_H_13563687684
+
+#include "xxhash.h"  /* XXH64_hash_t, XXH3_state_t */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*!
+ * @brief Returns the best XXH3 implementation for x86
+ *
+ * @return The best @ref XXH_VECTOR implementation.
+ * @see XXH_VECTOR_TYPES
+ */
+XXH_PUBLIC_API int XXH_featureTest(void);
+
+XXH_PUBLIC_API XXH64_hash_t  XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
+XXH_PUBLIC_API XXH64_hash_t  XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH64_hash_t  XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
+
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
+XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
+
+#if defined (__cplusplus)
+}
+#endif
+
+
+/* automatic replacement of XXH3 functions.
+ * can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */
+#ifndef XXH_DISPATCH_DISABLE_REPLACE
+
+# undef  XXH3_64bits
+# define XXH3_64bits XXH3_64bits_dispatch
+# undef  XXH3_64bits_withSeed
+# define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch
+# undef  XXH3_64bits_withSecret
+# define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch
+# undef  XXH3_64bits_update
+# define XXH3_64bits_update XXH3_64bits_update_dispatch
+
+# undef  XXH128
+# define XXH128 XXH3_128bits_withSeed_dispatch
+# undef  XXH3_128bits
+# define XXH3_128bits XXH3_128bits_dispatch
+# undef  XXH3_128bits_withSeed
+# define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch
+# undef  XXH3_128bits_withSecret
+# define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch
+# undef  XXH3_128bits_update
+# define XXH3_128bits_update XXH3_128bits_update_dispatch
+
+#endif /* XXH_DISPATCH_DISABLE_REPLACE */
+
+#endif /* XXH_X86DISPATCH_H_13563687684 */