amir/duplicatefinder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implementing registered files in io_uring

Browse Source 
The windows implementation is disabled, currently registering files in
IO Ring when there is inflight IO operations causes corruptions.

Implementing a config file.

Some code cleanup
This commit is contained in:
amir
2026-04-24 15:30:04 +01:00
parent ab31776658
commit 3393129c5f
5 changed files with 194 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
file_hasher.c
View File

@@ -82,17 +82,31 @@ int main(int argc, char **argv) {
// Logical threads = CPU cores * 2
uint32_t cpu_threads = cpu_cores * 2;
#if MULTI_THREADED
uint32_t num_scan_threads = cpu_threads;
uint32_t num_hash_threads = cpu_threads;
// uint32_t num_hash_threads = 1;
printf("%d cores %d threads CPU detected with %s instruction set\n"
"Starting thread pool: %d scanning and %d hashing threads\n",
cpu_cores, cpu_threads, get_xxhash_instruction_set(), num_scan_threads,
num_hash_threads);
#else
uint32_t num_scan_threads = 1;
uint32_t num_hash_threads = 1;
printf(
"%d cores %d threads CPU detected with %s instruction set\n"
"Starting thread pool: %d scanning and %d hashing threads(Debug mode)\n",
cpu_cores, cpu_threads, get_xxhash_instruction_set(), num_scan_threads,
num_hash_threads);
#endif
// Align IO Ring block size to the system page size
#if USE_IORING
g_ioring_buffer_size = ALIGN_UP_POW2(g_ioring_buffer_size, g_pagesize);
#endif
// -------------------------------
// Scanning and hashing
// -------------------------------
@@ -104,25 +118,6 @@ int main(int argc, char **argv) {
MPMCQueue file_queue;
mpmc_init(&file_queue, MiB(1));
// Starting hash threads
// size_t num_hash_threads = num_threads;
//
// WorkerContext workers[num_hash_threads];
// Thread *hash_threads =
// arena_push(&gp_arena, sizeof(Thread) * num_hash_threads, true);
//
// for (size_t i = 0; i < num_hash_threads; ++i) {
// workers[i].arena = arena_create(&params);
// workers[i].file_queue = &file_queue;
//
// if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker, &workers[i])
// !=
// 0) {
// fprintf(stderr, "Failed to create hash thread %zu\n", i);
// exit(1);
// }
// }
// Starting hash threads
WorkerContext workers[num_hash_threads];
Thread *hash_threads =
@@ -132,45 +127,19 @@ int main(int argc, char **argv) {
workers[i].arena = arena_create(&params);
workers[i].file_queue = &file_queue;
#if USE_IORING
if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker_ioring,
&workers[i]) != 0) {
&workers[i]) != 0)
#else
if (thread_create(&hash_threads[i], (ThreadFunc)hash_worker, &workers[i]) !=
0)
#endif
{
fprintf(stderr, "Failed to create hash thread %zu\n", i);
exit(1);
}
}
// Starting hash threads
// size_t num_hash_threads = num_threads;
//
// WorkerContext workers[num_hash_threads];
// Thread *hash_threads =
// arena_push(&gp_arena, sizeof(Thread) * num_hash_threads, true);
//
// // Check if I/O Ring is available
// bool io_ring_available = false;
// HIORING test_ring = io_ring_init();
// if (test_ring) {
// io_ring_available = true;
// io_ring_cleanup(test_ring);
// // printf("I/O Ring is available, using high-performance async I/O\n");
// } else {
// printf("I/O Ring not available, using buffered I/O\n");
// }
//
// for (size_t i = 0; i < num_hash_threads; ++i) {
// workers[i].arena = arena_create(&params);
// workers[i].file_queue = &file_queue;
//
// // Select the appropriate worker function
// ThreadFunc fn = io_ring_available ? (ThreadFunc)hash_worker_io_ring
// : (ThreadFunc)hash_worker;
//
// if (thread_create(&hash_threads[i], fn, &workers[i]) != 0) {
// fprintf(stderr, "Failed to create hash thread %zu\n", i);
// exit(1);
// }
// }
// Starting progress printing thread
Thread progress_thread_handle;
if (thread_create(&progress_thread_handle, (ThreadFunc)progress_thread,
@@ -265,12 +234,14 @@ int main(int argc, char **argv) {
// -------------------------------
// Print summary
// -------------------------------
#if USE_IORING
uint64_t incomplete = atomic_load(&g_io_ring_fallbacks);
if (incomplete > 0) {
printf("\nWARNING: I/O Ring incomplete files: %llu (fallback to buffered "
"I/O used)\n",
(unsigned long long)incomplete);
}
#endif
double total_seconds = timer_elapsed(&total_timer);