amir/filehasher
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reworking IO Ring pipeline to fully support multiple infilght files

Browse Source 
Reworking the filequeue, the buffer chaining logic and the error
handling.
Renaming functions.
Fix bug in arena.
This commit is contained in:
amir
2026-04-23 14:13:48 +01:00
parent b8e577b5bb
commit ab31776658
7 changed files with 365 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
file_hasher.c
View File

@@ -77,14 +77,19 @@ int main(int argc, char **argv) {
// Detect hardware
// -------------------------------
// --- Windows: detect PHYSICAL cores (not logical threads) ---
size_t hw_threads = platform_physical_cores();
uint32_t cpu_cores = platform_physical_cores();
// Logical threads = CPU cores * 2
size_t num_threads = hw_threads * 2;
uint32_t cpu_threads = cpu_cores * 2;
printf("Starting thread pool: %zu threads (CPU cores: %zu)\n", num_threads,
hw_threads);
printf(" Selected instruction set: %s\n", get_xxhash_instruction_set());
uint32_t num_scan_threads = cpu_threads;
uint32_t num_hash_threads = cpu_threads;
// uint32_t num_hash_threads = 1;
printf("%d cores %d threads CPU detected with %s instruction set\n"
"Starting thread pool: %d scanning and %d hashing threads\n",
cpu_cores, cpu_threads, get_xxhash_instruction_set(), num_scan_threads,
num_hash_threads);
// Align IO Ring block size to the system page size
g_ioring_buffer_size = ALIGN_UP_POW2(g_ioring_buffer_size, g_pagesize);
@@ -119,9 +124,6 @@ int main(int argc, char **argv) {
// }
// Starting hash threads
size_t num_hash_threads = num_threads;
// size_t num_hash_threads = 1;
WorkerContext workers[num_hash_threads];
Thread *hash_threads =
arena_push(&gp_arena, sizeof(Thread) * num_hash_threads, true);
@@ -130,7 +132,7 @@ int main(int argc, char **argv) {
workers[i].arena = arena_create(&params);
workers[i].file_queue = &file_queue;
if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker_io_ring,
if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker_ioring,
&workers[i]) != 0) {
fprintf(stderr, "Failed to create hash thread %zu\n", i);
exit(1);
@@ -178,8 +180,6 @@ int main(int argc, char **argv) {
}
// Starting scan threads
size_t num_scan_threads = num_threads;
ScannerContext scanners[num_scan_threads];
Thread *scan_threads =
arena_push(&gp_arena, sizeof(Thread) * num_scan_threads, true);