forked from amir/filehasher
hashers now use thread local arena
Instead of writing directly to file_hashes.txt, hash_workers now are using a local arena, writing everything once at the end using #pragma once to ensure that a given header file is included only once in a single compilation unit
This commit is contained in:
44
platform.h
44
platform.h
@@ -1,39 +1,10 @@
|
||||
#pragma once
|
||||
#pragma once // ensure that a given header file is included only once in a
|
||||
// single compilation unit
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
#define PLATFORM_WINDOWS 1
|
||||
#include <aclapi.h>
|
||||
#include <fcntl.h>
|
||||
#include <io.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <windows.h>
|
||||
|
||||
#define strdup _strdup
|
||||
#else
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
#include <pthread.h>
|
||||
#include <pwd.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#define XXH_VECTOR \
|
||||
XXH_AVX2 // not recommanded to compile with gcc see xxhash.h line 4082
|
||||
// Must compile with /arch:AVX2 in clang-cl or -mavx2 in clang/gcc
|
||||
#define XXH_INLINE_ALL
|
||||
#include "xxhash.c"
|
||||
#include "xxhash.h"
|
||||
#include "arena.h"
|
||||
#include "base.h"
|
||||
|
||||
#include "arena.c"
|
||||
// ----------------------------- Config -------------------------------------
|
||||
#define FILE_HASHES_TXT "file_hashes.txt"
|
||||
#define HASH_STRLEN 33 // 128-bit hex (32 chars) + null
|
||||
@@ -124,6 +95,11 @@ typedef struct {
|
||||
|
||||
static MPMCQueue g_file_queue;
|
||||
|
||||
typedef struct {
|
||||
MPMCQueue *queue;
|
||||
mem_arena *arena;
|
||||
} WorkerContext;
|
||||
|
||||
/* Scan folders */
|
||||
typedef struct DirQueue DirQueue;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user