amir/filehasher
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allocate both ioring buffers and fallback buffer memory at the same time + bug fix

Browse Source
This commit is contained in:
amir
2026-06-19 23:16:17 +01:00
parent 4fac135dce
commit 2a7bed5036
2 changed files with 23 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
file_hasher.c
View File

@@ -118,8 +118,10 @@ int main(int argc, char **argv) {
#endif #endif
// Align IO Ring block size to the system page size // Align IO Ring block size to the system page size
g_pagesize = plat_get_pagesize();
g_read_block = ALIGN_UP_POW2(READ_BLOCK, g_pagesize);
#if USE_IORING #if USE_IORING
g_ioring_buffer_size = ALIGN_UP_POW2(g_ioring_buffer_size, g_pagesize); g_ioring_buffer_size = ALIGN_UP_POW2(IORING_BUFFER_SIZE, g_pagesize);
#endif #endif
// ------------------------------- // -------------------------------

29
platform.c
View File

@@ -5,8 +5,6 @@
#include "mt_mpmc.h" #include "mt_mpmc.h"
#include "arena.c" #include "arena.c"
#include <stdint.h>
#include <stdio.h>
// xxhash include // xxhash include
#define XXH_STATIC_LINKING_ONLY #define XXH_STATIC_LINKING_ONLY
@@ -14,6 +12,8 @@
#include "config.h" #include "config.h"
// ----------------------------- Globals ------------------------------------ // ----------------------------- Globals ------------------------------------
u64 g_read_block;
static atomic_uint_fast64_t g_files_found = 0; static atomic_uint_fast64_t g_files_found = 0;
static atomic_uint_fast64_t g_files_hashed = 0; static atomic_uint_fast64_t g_files_hashed = 0;
static atomic_uint_fast64_t g_bytes_processed = 0; static atomic_uint_fast64_t g_bytes_processed = 0;
@@ -1082,7 +1082,16 @@ static void xxh3_hash_file_stream(char *path, char *out_hex,
// ------------------------- Hash worker -------------------------------- // ------------------------- Hash worker --------------------------------
static THREAD_RETURN hash_worker(void *arg) { static THREAD_RETURN hash_worker(void *arg) {
HasherContext *ctx = (HasherContext *)arg; HasherContext *ctx = (HasherContext *)arg;
void *buf = malloc(READ_BLOCK);
void *buf = plat_mem_reserve(g_read_block);
if (!buf) {
fprintf(stderr, "Failed to reserve hashing buffer memory\n");
exit(1);
}
plat_mem_commit(buf, g_read_block);
char *separator; char *separator;
for (;;) { for (;;) {
@@ -2023,16 +2032,15 @@ static ThreadIoContext *ioring_init_thread(void) {
} }
// Initialize buffer pool // Initialize buffer pool
thread_ctx->fallback_buffer = malloc(READ_BLOCK);
IORING_BUFFER_INFO buf_info[NUM_BUFFERS_PER_THREAD]; IORING_BUFFER_INFO buf_info[NUM_BUFFERS_PER_THREAD];
u64 buf_pool_size = g_ioring_buffer_size * NUM_BUFFERS_PER_THREAD; u64 buf_pool_size = g_ioring_buffer_size * NUM_BUFFERS_PER_THREAD;
u64 total_buf_size = buf_pool_size + g_read_block;
// Reserve and Commit memory for buffers // Reserve and Commit memory for buffers
void *base_ptr = plat_mem_reserve(buf_pool_size); void *base_ptr = plat_mem_reserve(total_buf_size);
if (base_ptr) { if (base_ptr) {
if (!plat_mem_commit(base_ptr, buf_pool_size)) { if (!plat_mem_commit(base_ptr, total_buf_size)) {
plat_mem_release(base_ptr, 0); plat_mem_release(base_ptr, 0);
close_ioring(thread_ctx); close_ioring(thread_ctx);
free(thread_ctx); free(thread_ctx);
@@ -2058,6 +2066,8 @@ static ThreadIoContext *ioring_init_thread(void) {
thread_ctx->free_count = NUM_BUFFERS_PER_THREAD; thread_ctx->free_count = NUM_BUFFERS_PER_THREAD;
thread_ctx->fallback_buffer = (u8 *)base_ptr + buf_pool_size;
// Register buffers // Register buffers
ioring_register_buffers(thread_ctx, buf_info); ioring_register_buffers(thread_ctx, buf_info);
@@ -2081,8 +2091,9 @@ static void ioring_cleanup_thread(ThreadIoContext *thread_ctx) {
// Free the buffer pool memory // Free the buffer pool memory
if (thread_ctx->buffers[0].data) { if (thread_ctx->buffers[0].data) {
u64 buf_pool_size = g_ioring_buffer_size * NUM_BUFFERS_PER_THREAD; u64 total_buf_size =
plat_mem_release(thread_ctx->buffers[0].data, buf_pool_size); g_ioring_buffer_size * NUM_BUFFERS_PER_THREAD + g_read_block;
plat_mem_release(thread_ctx->buffers[0].data, total_buf_size);
} }
free(thread_ctx); free(thread_ctx);