diff --git a/.gitignore b/.gitignore index 670e490..f61a95e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,13 @@ file_hasher.ilk file_hasher.rdi file_hasher.exe file_hashes.txt -Binaries/file_hashes.txt +/Binaries file_list.txt temp_code.c -/.cache/clangd/index +/.cache /file_hasher /io_uring_test /file_hasher /io_uring_test +/compile_commands.json +/build diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..2e6f72d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,284 @@ +cmake_minimum_required(VERSION 3.20) +project(filehasher + VERSION 1.0.0 + DESCRIPTION "High-performance file hasher with I/O Ring/io_uring support" + LANGUAGES C +) + +# --------------------------------------------------------------------------- +# Force compiler search order +# --------------------------------------------------------------------------- + +# On Windows, prefer clang-cl, then GCC, then Clang +if(WIN32) + # Try to force compiler order if not already set + if(NOT CMAKE_C_COMPILER) + # Search in preferred order + find_program(CLANG_CL_COMPILER NAMES clang-cl) + find_program(GCC_COMPILER NAMES gcc) + find_program(CLANG_COMPILER NAMES clang) + + if(CLANG_CL_COMPILER) + message(STATUS "Found clang-cl as preferred compiler: ${CLANG_CL_COMPILER}") + set(CMAKE_C_COMPILER "${CLANG_CL_COMPILER}" CACHE STRING "" FORCE) + elseif(GCC_COMPILER) + message(STATUS "Found GCC as fallback compiler: ${GCC_COMPILER}") + set(CMAKE_C_COMPILER "${GCC_COMPILER}" CACHE STRING "" FORCE) + elseif(CLANG_COMPILER) + message(STATUS "Found Clang as last-resort compiler: ${CLANG_COMPILER}") + set(CMAKE_C_COMPILER "${CLANG_COMPILER}" CACHE STRING "" FORCE) + endif() + endif() +else() + # On Linux, prefer GCC, then Clang + if(NOT CMAKE_C_COMPILER) + find_program(GCC_COMPILER NAMES gcc) + find_program(CLANG_COMPILER NAMES clang) + + if(GCC_COMPILER) + message(STATUS "Found GCC as preferred compiler: ${GCC_COMPILER}") + set(CMAKE_C_COMPILER "${GCC_COMPILER}" CACHE STRING "" FORCE) + elseif(CLANG_COMPILER) + message(STATUS "Found Clang as fallback compiler: ${CLANG_COMPILER}") + set(CMAKE_C_COMPILER "${CLANG_COMPILER}" CACHE STRING "" FORCE) + endif() + endif() +endif() + +# Now project() will use the compiler we found +# However, since we needed project() first to get C support, +# we check what we actually got +message(STATUS "Using compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID})") + +# --------------------------------------------------------------------------- +# Platform and Compiler Detection +# --------------------------------------------------------------------------- + +if(WIN32) + set(PLATFORM_WINDOWS TRUE) + set(PLATFORM_NAME "Windows") +else() + set(PLATFORM_LINUX TRUE) + set(PLATFORM_NAME "Linux") +endif() + +# Compiler type +if(CMAKE_C_COMPILER_ID STREQUAL "Clang") + # Check if it's clang-cl + if(CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + set(COMPILER_CLANG_CL TRUE) + message(STATUS "Detected clang-cl (MSVC-compatible frontend)") + else() + set(COMPILER_CLANG_GNU TRUE) + message(STATUS "Detected Clang (GNU-compatible frontend)") + endif() +elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(COMPILER_GCC TRUE) + message(STATUS "Detected GCC") +elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC") + # We don't want MSVC, but if it's found, warn user + message(FATAL_ERROR + "MSVC (cl.exe) detected!\n" + "This project requires clang-cl, GCC, or Clang.\n" + "Please install one of these compilers or specify manually:\n" + " cmake .. -DCMAKE_C_COMPILER=clang-cl\n" + " cmake .. -DCMAKE_C_COMPILER=gcc\n" + " cmake .. -DCMAKE_C_COMPILER=clang\n" + ) +endif() + +# --------------------------------------------------------------------------- +# Build System Selection +# --------------------------------------------------------------------------- + +if(NOT CMAKE_GENERATOR OR CMAKE_GENERATOR STREQUAL "") + find_program(NINJA_EXECUTABLE NAMES ninja) + if(NINJA_EXECUTABLE) + message(STATUS "Using Ninja build system") + set(CMAKE_GENERATOR "Ninja") + else() + message(STATUS "Ninja not found, using default generator: ${CMAKE_GENERATOR}") + endif() +endif() + +# --------------------------------------------------------------------------- +# Source Files +# --------------------------------------------------------------------------- + +set(SOURCES + file_hasher.c + xxhash.c + xxh_x86dispatch.c +) + +# Headers for dependency tracking and IDE +set(HEADERS + arena.h + base.h + xxhash.h + lf_mpmc.h +) + +# --------------------------------------------------------------------------- +# Create Executable +# --------------------------------------------------------------------------- + +add_executable(${PROJECT_NAME} + ${SOURCES} + ${HEADERS} +) + +# Include directories +target_include_directories(${PROJECT_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) + +# --------------------------------------------------------------------------- +# Compiler Flags - Exact match to your commands +# --------------------------------------------------------------------------- + +if(PLATFORM_WINDOWS) + if(COMPILER_CLANG_CL) + # === clang-cl flags === + # Release: /O2 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:/O2> + ) + # Debug: /Zi /Od + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:/Zi /Od> + ) + # Common warnings + target_compile_options(${PROJECT_NAME} PRIVATE /W4) + + elseif(COMPILER_GCC) + # === GCC flags (Windows/MinGW) === + # Release: -O3 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O3> + ) + # Debug: -g -O0 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-g -O0> + ) + # Common warnings + target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) + + elseif(COMPILER_CLANG_GNU) + # === Clang flags (Windows, GNU frontend) === + # Release: -O3 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O3> + ) + # Debug: -g -O0 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-g -O0> + ) + # Common warnings + target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) + endif() + + # Windows-specific libraries + target_link_libraries(${PROJECT_NAME} PRIVATE + kernel32 + ) + + # Windows-specific defines + target_compile_definitions(${PROJECT_NAME} PRIVATE + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 # Windows 10+ + ) + + # Set output name with .exe + set_target_properties(${PROJECT_NAME} PROPERTIES + SUFFIX ".exe" + ) + +elseif(PLATFORM_LINUX) + # === Linux GCC/Clang flags === + if(COMPILER_GCC OR COMPILER_CLANG_GNU) + # Release: -O3 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O3> + ) + # Debug: -g -O0 + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-g -O0> + ) + # Common warnings + target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wextra) + endif() + + # Linux-specific libraries + find_package(Threads REQUIRED) + find_library(LIBURING_LIBRARY NAMES uring) + + if(LIBURING_LIBRARY) + message(STATUS "Found liburing: ${LIBURING_LIBRARY}") + target_link_libraries(${PROJECT_NAME} PRIVATE + Threads::Threads + ${LIBURING_LIBRARY} + ) + else() + message(FATAL_ERROR "liburing not found! Install liburing-dev or equivalent") + endif() + + # Linux-specific defines + target_compile_definitions(${PROJECT_NAME} PRIVATE + _GNU_SOURCE + ) +endif() + +# --------------------------------------------------------------------------- +# C Standard +# --------------------------------------------------------------------------- + +set_target_properties(${PROJECT_NAME} PROPERTIES + C_STANDARD 11 + C_STANDARD_REQUIRED ON + C_EXTENSIONS OFF +) + +# --------------------------------------------------------------------------- +# Build Configurations +# --------------------------------------------------------------------------- + +# Set default build type if not specified (matching your Release command) +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING + "Choose the type of build: Release or Debug" FORCE) + message(STATUS "No build type specified, defaulting to Release") +endif() + +# --------------------------------------------------------------------------- +# IDE Support +# --------------------------------------------------------------------------- + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# --------------------------------------------------------------------------- +# Info Target +# --------------------------------------------------------------------------- + +add_custom_target(info + COMMAND ${CMAKE_COMMAND} -E echo "=== Build Configuration ===" + COMMAND ${CMAKE_COMMAND} -E echo "Project: ${PROJECT_NAME}" + COMMAND ${CMAKE_COMMAND} -E echo "Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID})" + COMMAND ${CMAKE_COMMAND} -E echo "Frontend: $,clang-cl,GNU>" + COMMAND ${CMAKE_COMMAND} -E echo "Generator: ${CMAKE_GENERATOR}" + COMMAND ${CMAKE_COMMAND} -E echo "Build Type: ${CMAKE_BUILD_TYPE}" + COMMAND ${CMAKE_COMMAND} -E echo "Platform: ${PLATFORM_NAME}" + COMMAND ${CMAKE_COMMAND} -E echo "============================" +) + +# --------------------------------------------------------------------------- +# Print final configuration +# --------------------------------------------------------------------------- + +message(STATUS "----------------------------------------") +message(STATUS "Configuration Summary:") +message(STATUS " Compiler: ${CMAKE_C_COMPILER}") +message(STATUS " Build Type: ${CMAKE_BUILD_TYPE}") +message(STATUS " Generator: ${CMAKE_GENERATOR}") +message(STATUS " Platform: ${PLATFORM_NAME}") +message(STATUS "----------------------------------------") \ No newline at end of file diff --git a/README.md b/README.md index c0fd7e6..56fe817 100644 --- a/README.md +++ b/README.md @@ -20,36 +20,55 @@ It is a high performance cross platform Windows and Linux compatible program, it * Fallback to buffered I/O if there is errors in the IO Ring path. # Building -## Windows -### Release -**Note**: Make sur to use UCRT64 environment from MSYS2 instead of the standard MinGW environment. +## Windows +**Requirements**: Make sur to use UCRT64 environment from MSYS2 instead of the standard MinGW environment. UCRT64 uses the modern Universal C Runtime (ucrtbase.dll), which supports the newest APIs, the standard MSYS2 uses the legacy msvcrt.dll and does not support IO Ring. To install: pacman -S mingw-w64-ucrt-x86_64-gcc -pacman -S mingw-w64-ucrt-x86_64-clang -pacman -Syu +pacman -S mingw-w64-ucrt-x86_64-clang +pacman -S mingw-w64-ucrt-x86_64-cmake +pacman -Syu And add to path: C:\msys64\ucrt64\bin -gcc -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -o file_hasher -clang -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -o file_hasher +### Using a build system +| Command | Description| +| :--- | :--- | +| ./build.bat | Build Release with best available compiler | +| ./build.bat Debug | Build Debug | +| ./build.bat clean | Clean and build Release | +| ./build.bat Debug clean | Clean and build Debug | + +### Release +gcc -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -o filehasher +clang -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -o filehasher clang-cl /O2 file_hasher.c xxhash.c xxh_x86dispatch.c ### Debug -gcc -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -o file_hasher -clang -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -o file_hasher +gcc -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -o filehasher +clang -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -o filehasher clang-cl /Zi /Od file_hasher.c xxhash.c xxh_x86dispatch.c ## Linux +**Requirements**: GCC, CMake and Ninja + +### Using a build system +| Command | Description| +| :--- | :--- | +| ./build.sh | Build Release with best available compiler | +| ./build.sh Debug | Build Debug | +| ./build.sh clean | Clean and build Release | +| ./build.sh Debug clean | Clean and build Debug | + ### Release -gcc -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o file_hasher -clang -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o file_hasher +gcc -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o filehasher +clang -O3 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o filehasher ### Debug -gcc -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o file_hasher -clang -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o file_hasher +gcc -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o filehasher +clang -g -O0 file_hasher.c xxhash.c xxh_x86dispatch.c -pthread -luring -o filehasher # Notes about the IO Ring implementations ## IO Ring diff --git a/arena.c b/arena.c index 61767d5..beb646b 100644 --- a/arena.c +++ b/arena.c @@ -83,8 +83,7 @@ u64 arena_pos_from_ptr(mem_arena *arena, void *ptr) { void *arena_ptr_from_pos(mem_arena *arena, u64 global_pos) { ASSERT(arena); - ASSERT(global_pos >= 0); - if (!arena || global_pos < 0) { + if (!arena) { return NULL; } @@ -488,7 +487,6 @@ void *arena_free(mem_arena **arena_ptr, u8 **ptr, u64 size) { // mk free Find owning block ------------------------------------------------------------ */ - mem_arena *selected = arena; mem_arena *owner = arena_block_from_ptr(arena, *ptr); ASSERT(owner); if (!owner) { @@ -500,7 +498,7 @@ void *arena_free(mem_arena **arena_ptr, u8 **ptr, u64 size) { // mk free ------------------------------------------------------------ */ u64 global_offset = arena_pos_from_ptr(arena, *ptr); - if (global_offset == -1) { + if (global_offset == UINT64_MAX) { return NULL; } @@ -607,9 +605,6 @@ void *arena_swapback_pop(mem_arena **arena_ptr, u64 index) { // mk swapback fprintf(stderr, "ERROR: Swapback pop failed, index out of range"); return NULL; } - u8 *owner_base = (u8 *)owner + ALIGN_UP_POW2(sizeof(mem_arena), owner->align); - u8 *arena_base = (u8 *)arena + ALIGN_UP_POW2(sizeof(mem_arena), arena->align); - u8 *dst = arena_ptr_from_index(arena, index); u8 *src = arena_ptr_from_index(arena, count); diff --git a/base.h b/base.h index e209c28..3acc537 100644 --- a/base.h +++ b/base.h @@ -104,7 +104,9 @@ typedef double f64; #define ASSERT(x) assert(x) #endif -#define NDEBUG // Comment to enable asserts +#ifndef NDEBUG +#define NDEBUG 1 // 0 to enable asserts +#endif /* ------------------------------------------------------------ Some helper functions @@ -130,9 +132,9 @@ static b32 plat_mem_commit(void *ptr, u64 size) { return ret != NULL; } -static b32 plat_mem_decommit(void *ptr, u64 size) { - return VirtualFree(ptr, size, MEM_DECOMMIT); -} +// static b32 plat_mem_decommit(void *ptr, u64 size) { // Comment to prevent warning: unused function +// return VirtualFree(ptr, size, MEM_DECOMMIT); +// } static b32 plat_mem_release(void *ptr, u64 size) { return VirtualFree(ptr, size, MEM_RELEASE); @@ -152,21 +154,21 @@ static void plat_sem_wait(plat_sem *s) { WaitForSingleObject(s->handle, INFINITE); } -static b32 plat_sem_trywait(HANDLE sem) { - DWORD r = WaitForSingleObject(sem, 0); - return r == WAIT_OBJECT_0; -} +// static b32 plat_sem_trywait(HANDLE sem) { // Comment to prevent warning: unused function +// DWORD r = WaitForSingleObject(sem, 0); +// return r == WAIT_OBJECT_0; +// } static void plat_sem_post(plat_sem *s, u32 count) { ReleaseSemaphore(s->handle, count, NULL); } -static void plat_sem_destroy(plat_sem *s) { - if (s->handle) { - CloseHandle(s->handle); - s->handle = NULL; - } -} +// static void plat_sem_destroy(plat_sem *s) { // Comment to prevent warning: unused function +// if (s->handle) { +// CloseHandle(s->handle); +// s->handle = NULL; +// } +// } // Sleep static void sleep_ms(int ms) { Sleep(ms); } @@ -197,13 +199,13 @@ static b32 plat_mem_commit(void *ptr, u64 size) { return ret == 0; } -static b32 plat_mem_decommit(void *ptr, u64 size) { - i32 ret = mprotect(ptr, size, PROT_NONE); - if (ret != 0) - return false; - ret = madvise(ptr, size, MADV_DONTNEED); - return ret == 0; -} +// static b32 plat_mem_decommit(void *ptr, u64 size) { // Comment to prevent warning: unused function +// i32 ret = mprotect(ptr, size, PROT_NONE); +// if (ret != 0) +// return false; +// ret = madvise(ptr, size, MADV_DONTNEED); +// return ret == 0; +// } static b32 plat_mem_release(void *ptr, u64 size) { i32 ret = munmap(ptr, size); @@ -226,7 +228,7 @@ static void plat_sem_wait(plat_sem *s) { } } -static b32 plat_sem_trywait(sem_t *sem) { return sem_trywait(sem) == 0; } +// static b32 plat_sem_trywait(sem_t *sem) { return sem_trywait(sem) == 0; } // Comment to prevent warning: unused function static void plat_sem_post(plat_sem *s, u32 count) { for (u32 i = 0; i < count; i++) { @@ -234,7 +236,7 @@ static void plat_sem_post(plat_sem *s, u32 count) { } } -static void plat_sem_destroy(plat_sem *s) { sem_destroy(&s->sem); } +// static void plat_sem_destroy(plat_sem *s) { sem_destroy(&s->sem); } // Comment to prevent warning: unused function // Sleep static void sleep_ms(int ms) { usleep(ms * 1000); } diff --git a/binaries/file_hasher_v1.0.exe b/binaries/file_hasher_v1.0.exe deleted file mode 100644 index 97b90a0..0000000 Binary files a/binaries/file_hasher_v1.0.exe and /dev/null differ diff --git a/binaries/file_hasher_v1.1.exe b/binaries/file_hasher_v1.1.exe deleted file mode 100644 index 33c2f77..0000000 Binary files a/binaries/file_hasher_v1.1.exe and /dev/null differ diff --git a/binaries/file_hasher_v2.0.exe b/binaries/file_hasher_v2.0.exe deleted file mode 100644 index f00eaf7..0000000 Binary files a/binaries/file_hasher_v2.0.exe and /dev/null differ diff --git a/binaries/file_hasher_v2.1.exe b/binaries/file_hasher_v2.1.exe deleted file mode 100644 index e4c2ad2..0000000 Binary files a/binaries/file_hasher_v2.1.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.0.exe b/binaries/file_hasher_v3.0.exe deleted file mode 100644 index 661ffa2..0000000 Binary files a/binaries/file_hasher_v3.0.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.1.exe b/binaries/file_hasher_v3.1.exe deleted file mode 100644 index 205bafa..0000000 Binary files a/binaries/file_hasher_v3.1.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.2.exe b/binaries/file_hasher_v3.2.exe deleted file mode 100644 index 8364396..0000000 Binary files a/binaries/file_hasher_v3.2.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.3.exe b/binaries/file_hasher_v3.3.exe deleted file mode 100644 index 1712169..0000000 Binary files a/binaries/file_hasher_v3.3.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.4.exe b/binaries/file_hasher_v3.4.exe deleted file mode 100644 index 3ad25de..0000000 Binary files a/binaries/file_hasher_v3.4.exe and /dev/null differ diff --git a/binaries/file_hasher_v3.5.exe b/binaries/file_hasher_v3.5.exe deleted file mode 100644 index 4915feb..0000000 Binary files a/binaries/file_hasher_v3.5.exe and /dev/null differ diff --git a/binaries/file_hasher_v4.0.exe b/binaries/file_hasher_v4.0.exe deleted file mode 100644 index 75e93c8..0000000 Binary files a/binaries/file_hasher_v4.0.exe and /dev/null differ diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..fddea92 --- /dev/null +++ b/build.bat @@ -0,0 +1,120 @@ +@echo off +setlocal enabledelayedexpansion + +:: ============================================================================ +:: build.bat - Build script with compiler preference: clang-cl > gcc > clang +:: Usage: build [Release|Debug] [clean] +:: ============================================================================ + +set BUILD_TYPE=Release +set CLEAN_BUILD=0 + +:: Parse arguments +:parse_args +if "%~1"=="" goto :main + +if /i "%~1"=="Release" ( + set BUILD_TYPE=Release + shift + goto :parse_args +) +if /i "%~1"=="Debug" ( + set BUILD_TYPE=Debug + shift + goto :parse_args +) +if /i "%~1"=="clean" ( + set CLEAN_BUILD=1 + shift + goto :parse_args +) + +:: Unknown argument fallback (the *) +echo Unknown argument: %~1 +echo Usage: .\%~nx0 [Release^|Debug] [clean] +exit /b 1 + +:main +set BUILD_DIR=build\windows\%BUILD_TYPE% +echo === Building filehasher (%BUILD_TYPE%) === + +:: Clean if requested +if %CLEAN_BUILD%==1 ( + echo Cleaning... + if exist "%BUILD_DIR%" rmdir /s /q "%BUILD_DIR%" 2>nul +) + +:: Create build dir +if not exist "%BUILD_DIR%" mkdir "%BUILD_DIR%" +pushd "%BUILD_DIR%" + +:: Find compiler in preferred order +set CC= +where clang-cl >nul 2>&1 +if !ERRORLEVEL! equ 0 ( + echo Compiler: clang-cl ^(preferred^) + set "CC=-DCMAKE_C_COMPILER=clang-cl" + goto :find_generator +) + +where gcc >nul 2>&1 +if !ERRORLEVEL! equ 0 ( + echo Compiler: gcc ^(fallback^) + set "CC=-DCMAKE_C_COMPILER=gcc" + goto :find_generator +) + +where clang >nul 2>&1 +if !ERRORLEVEL! equ 0 ( + echo Compiler: clang ^(last resort^) + set "CC=-DCMAKE_C_COMPILER=clang" + goto :find_generator +) + +echo ERROR: No suitable compiler found! (clang-cl, gcc, or clang required) +popd +exit /b 1 + +:find_generator +:: Find Ninja for build system +set GEN= +where ninja >nul 2>&1 +if !ERRORLEVEL! equ 0 ( + echo Generator: Ninja + set "GEN=-G Ninja" +) else ( + echo Generator: Default +) + +:: Configure +echo. +echo Configuring CMake... +set CMD=cmake ../../.. %GEN% %CC% -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +echo !CMD! +!CMD! +if !ERRORLEVEL! neq 0 (echo ERROR: Configuration failed & popd & exit /b 1) + +:: Build +echo. +echo Building... +cmake --build . --config %BUILD_TYPE% +if !ERRORLEVEL! neq 0 (echo ERROR: Build failed & popd & exit /b 1) + +:: Check if compile_commands.json exists in the current build directory +if exist "compile_commands.json" ( + echo. + echo clangd: compile_commands.json generated + + :: Copy from current build dir up two levels to the project root + copy /Y "compile_commands.json" "..\..\..\compile_commands.json" >nul 2>&1 + if !ERRORLEVEL! equ 0 ( + echo clangd: Copied to project root + ) else ( + echo clangd: Could not copy to project root + ) +) + +popd +echo. +echo === Build Complete === +echo Executable: %BUILD_DIR%\filehasher.exe \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..9734457 --- /dev/null +++ b/build.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# ============================================================================ +# build.sh - Build script for filehasher (Linux) +# Usage: ./build.sh [Release|Debug] [clean] +# +# Compiler preference: gcc > clang +# Build system: Ninja (fallback to Make) +# ============================================================================ + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colors +# --------------------------------------------------------------------------- +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly CYAN='\033[0;36m' +readonly NC='\033[0m' + +# --------------------------------------------------------------------------- +# Default values +# --------------------------------------------------------------------------- +BUILD_TYPE="Release" +CLEAN_BUILD=0 + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + Release|release) + BUILD_TYPE="Release" + shift + ;; + Debug|debug) + BUILD_TYPE="Debug" + shift + ;; + clean|-clean|--clean) + CLEAN_BUILD=1 + shift + ;; + *) + echo -e "${RED}Unknown argument: $1${NC}" + echo "Usage: $0 [Release|Debug] [clean]" + exit 1 + ;; + esac +done + +# --------------------------------------------------------------------------- +# Setup +# --------------------------------------------------------------------------- +readonly BUILD_DIR="build/linux/${BUILD_TYPE}" +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo -e "${GREEN}=== Building filehasher (${BUILD_TYPE}) ===${NC}" +echo "Project: ${SCRIPT_DIR}" + +# --------------------------------------------------------------------------- +# Clean if requested +# --------------------------------------------------------------------------- +if [[ $CLEAN_BUILD -eq 1 ]]; then + echo -e "${YELLOW}Cleaning build directory...${NC}" + rm -rf "${BUILD_DIR}" + echo +fi + +# --------------------------------------------------------------------------- +# Create build directory +# --------------------------------------------------------------------------- +mkdir -p "${BUILD_DIR}" +cd "${BUILD_DIR}" + +# --------------------------------------------------------------------------- +# Find compiler (prefer gcc, fallback to clang) +# --------------------------------------------------------------------------- +echo -e "${YELLOW}Detecting compiler...${NC}" + +CC_BINARY="" +CC_NAME="" + +if command -v gcc &> /dev/null; then + CC_BINARY="gcc" + CC_VERSION=$(gcc --version | head -n1) + CC_NAME="GCC (${CC_VERSION})" + echo -e " ${GREEN}[OK]${NC} Found GCC (preferred): ${CC_VERSION}" +elif command -v clang &> /dev/null; then + CC_BINARY="clang" + CC_VERSION=$(clang --version | head -n1) + CC_NAME="Clang (${CC_VERSION})" + echo -e " ${YELLOW}[OK]${NC} Found Clang (fallback): ${CC_VERSION}" +else + echo -e "${RED}[FAIL] No suitable compiler found!${NC}" + echo "Please install gcc or clang:" + echo " Ubuntu/Debian: sudo apt install build-essential" + echo " Fedora/RHEL: sudo dnf install gcc" + echo " Arch: sudo pacman -S gcc" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Check dependencies +# --------------------------------------------------------------------------- +echo -e "${YELLOW}Checking dependencies...${NC}" + +# Check for liburing +HAVE_LIBURING=0 +if ldconfig -p | grep -q liburing 2>/dev/null; then + HAVE_LIBURING=1 + echo -e " ${GREEN}[OK]${NC} Found liburing" +elif pkg-config --exists liburing 2>/dev/null; then + HAVE_LIBURING=1 + echo -e " ${GREEN}[OK]${NC} Found liburing (pkg-config)" +elif [[ -f /usr/lib/liburing.so ]] || [[ -f /usr/lib64/liburing.so ]] || [[ -f /usr/local/lib/liburing.so ]]; then + HAVE_LIBURING=1 + echo -e " ${GREEN}[OK]${NC} Found liburing (manual detection)" +else + echo -e "${RED}[FAIL] liburing not found!${NC}" + echo "Please install liburing-dev:" + echo " Ubuntu/Debian: sudo apt install liburing-dev" + echo " Fedora/RHEL: sudo dnf install liburing-devel" + echo " Arch: sudo pacman -S liburing" + exit 1 +fi + +# Check for pthreads +# Check if pthreads is available (either in ldconfig or merged into libc) +if ldconfig -p | grep -q libpthread 2>/dev/null || ldd --version | grep -qP '2\.(3[4-9]|[4-9][0-9])'; then + echo -e " ${GREEN}[OK]${NC} Found pthreads (merged or standalone)" +else + echo -e " ${YELLOW}[NOTE]${NC} pthreads not found, attempting link" +fi +echo + +# --------------------------------------------------------------------------- +# Find build system (prefer ninja) +# --------------------------------------------------------------------------- +echo -e "${YELLOW}Selecting build system...${NC}" + +GENERATOR="" +GENERATOR_NAME="" + +if command -v ninja &> /dev/null; then + GENERATOR="Ninja" + GENERATOR_NAME="Ninja" + echo -e " ${GREEN}[OK]${NC} Using Ninja" +elif command -v make &> /dev/null; then + GENERATOR="Unix Makefiles" + GENERATOR_NAME="Make" + echo -e " ${YELLOW}[OK]${NC} Ninja not found, using Make" +else + echo -e "${RED}[FAIL] No build system found!${NC}" + echo "Please install ninja or make:" + echo " Ubuntu/Debian: sudo apt install ninja-build" + echo " Fedora/RHEL: sudo dnf install ninja-build" + echo " Arch: sudo pacman -S ninja" + exit 1 +fi + +echo + +# --------------------------------------------------------------------------- +# Configure +# --------------------------------------------------------------------------- +echo -e "${YELLOW}Configuring CMake...${NC}" +echo -e " Build type: ${BUILD_TYPE}" +echo -e " Compiler: ${CC_NAME}" +echo -e " Generator: ${GENERATOR_NAME}" + +cmake "${SCRIPT_DIR}" \ + -G "${GENERATOR}" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DCMAKE_C_COMPILER="${CC_BINARY}" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + +if [[ $? -ne 0 ]]; then + echo -e "${RED}CMake configuration failed!${NC}" + exit 1 +fi + +echo -e "${GREEN}Configuration successful!${NC}" +echo + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- +echo -e "${YELLOW}Building...${NC}" + +# Get number of CPU cores +if command -v nproc &> /dev/null; then + CORES=$(nproc) +else + CORES=4 +fi + +cmake --build . --config "${BUILD_TYPE}" --parallel "${CORES}" + +if [[ $? -ne 0 ]]; then + echo -e "${RED}Build failed!${NC}" + exit 1 +fi + +echo -e "${GREEN}Build successful!${NC}" +echo + +# --------------------------------------------------------------------------- +# Verify output +# --------------------------------------------------------------------------- +cd "${SCRIPT_DIR}" + +if [[ -f "${BUILD_DIR}/filehasher" ]]; then + echo -e "${GREEN}Executable: ${BUILD_DIR}/filehasher${NC}" + + if command -v file &> /dev/null; then + echo -e " Type: $(file -b ${BUILD_DIR}/filehasher)" + fi + + if command -v du &> /dev/null; then + echo -e " Size: $(du -h ${BUILD_DIR}/filehasher | cut -f1)" + fi +elif [[ -f "${BUILD_DIR}/filehasher.exe" ]]; then + echo -e "${GREEN}Executable: ${BUILD_DIR}/filehasher.exe${NC}" +else + echo -e "${YELLOW}Note: Could not locate executable${NC}" + echo "Checking build directory:" + find "${BUILD_DIR}" -type f -executable 2>/dev/null || echo " No executables found" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo +echo -e "${CYAN}=== Build Complete ===${NC}" +echo +echo -e "${YELLOW}Build Information:${NC}" +echo -e " Configuration: ${BUILD_TYPE}" +echo -e " Compiler: ${CC_NAME}" +echo -e " Generator: ${GENERATOR_NAME}" +echo -e " Output: ${BUILD_DIR}/" + +# --------------------------------------------------------------------------- +# Copy compile_commands.json for clangd +# --------------------------------------------------------------------------- +if [[ -f "${BUILD_DIR}/compile_commands.json" ]]; then + echo -e " clangd: compile_commands.json generated" + + # Always copy the latest version + cp "${BUILD_DIR}/compile_commands.json" "${SCRIPT_DIR}/compile_commands.json" + echo -e " clangd: Copied to project root" +fi + +echo +echo -e "${GREEN}Ready to run: ./${BUILD_DIR}/filehasher${NC}" diff --git a/binaries/changelog.txt b/changelog.txt similarity index 100% rename from binaries/changelog.txt rename to changelog.txt diff --git a/file_hasher.c b/file_hasher.c index 67bf97d..abb4f6d 100644 --- a/file_hasher.c +++ b/file_hasher.c @@ -77,14 +77,14 @@ int main(int argc, char **argv) { // Detect hardware // ------------------------------- // --- Windows: detect PHYSICAL cores (not logical threads) --- - uint32_t cpu_cores = platform_physical_cores(); + uint8_t cpu_cores = platform_physical_cores(); // Logical threads = CPU cores * 2 - uint32_t cpu_threads = cpu_cores * 2; + uint8_t cpu_threads = cpu_cores * 2; #if MULTI_THREADING - uint32_t num_scan_threads = cpu_threads; - uint32_t num_hash_threads = cpu_threads; + uint8_t num_scan_threads = cpu_threads; + uint8_t num_hash_threads = cpu_threads; printf("%d cores %d threads CPU detected with %s instruction set\n" "Starting thread pool: %d scanning and %d hashing threads\n", @@ -123,7 +123,7 @@ int main(int argc, char **argv) { Thread *hash_threads = arena_push(&gp_arena, sizeof(Thread) * num_hash_threads, true); - for (size_t i = 0; i < num_hash_threads; ++i) { + for (uint8_t i = 0; i < num_hash_threads; ++i) { workers[i].arena = arena_create(¶ms); workers[i].file_queue = &file_queue; @@ -135,7 +135,7 @@ int main(int argc, char **argv) { 0) #endif { - fprintf(stderr, "Failed to create hash thread %zu\n", i); + fprintf(stderr, "Failed to create hash thread %d\n", i); exit(1); } } @@ -153,7 +153,7 @@ int main(int argc, char **argv) { Thread *scan_threads = arena_push(&gp_arena, sizeof(Thread) * num_scan_threads, true); - for (size_t i = 0; i < num_scan_threads; i++) { + for (uint8_t i = 0; i < num_scan_threads; i++) { scanners[i].num_threads = num_scan_threads; scanners[i].path_arena = arena_create(¶ms); scanners[i].meta_arena = arena_create(¶ms); @@ -162,7 +162,7 @@ int main(int argc, char **argv) { if (thread_create(&scan_threads[i], (ThreadFunc)scan_worker, &scanners[i]) != 0) { - fprintf(stderr, "Failed to create scan thread %zu\n", i); + fprintf(stderr, "Failed to create scan thread %d\n", i); exit(1); } } @@ -178,7 +178,7 @@ int main(int argc, char **argv) { // Stop scan threads thread_wait_multiple(scan_threads, num_scan_threads); - for (size_t i = 0; i < num_scan_threads; ++i) { + for (uint8_t i = 0; i < num_scan_threads; ++i) { thread_close(&scan_threads[i]); } @@ -205,7 +205,7 @@ int main(int argc, char **argv) { // Stop hashing threads thread_wait_multiple(hash_threads, num_hash_threads); - for (size_t i = 0; i < num_hash_threads; ++i) { + for (uint8_t i = 0; i < num_hash_threads; ++i) { thread_close(&hash_threads[i]); } @@ -222,7 +222,7 @@ int main(int argc, char **argv) { FILE *f = fopen(FILE_HASHES_TXT, "wb"); - for (int i = 0; i < num_hash_threads; i++) { + for (uint8_t i = 0; i < num_hash_threads; i++) { mem_arena *arena = workers[i].arena; u8 *arena_base = (u8 *)arena + ALIGN_UP_POW2(sizeof(mem_arena), arena->align); diff --git a/lf_mpmc.h b/lf_mpmc.h index 2288f26..05bbd5f 100644 --- a/lf_mpmc.h +++ b/lf_mpmc.h @@ -306,21 +306,21 @@ static void mpmc_task_done(MPMCQueue *q, u8 consumer_count) { /* ----------------------------------------------------------- */ /* MPMC Cleanup */ /* ----------------------------------------------------------- */ -static void mpmc_finish(MPMCQueue *q) { - if (!q) - return; - - if (q->slots) { - plat_mem_release(q->slots, 0); - q->slots = NULL; - } - - plat_sem_destroy(&q->items_sem); - - q->capacity = 0; - q->mask = 0; - - atomic_store_explicit(&q->head, 0, memory_order_relaxed); - atomic_store_explicit(&q->tail, 0, memory_order_relaxed); - atomic_store_explicit(&q->committed, 0, memory_order_relaxed); -} +// static void mpmc_finish(MPMCQueue *q) { // Comment to prevent warning: unused function +// if (!q) +// return; +// +// if (q->slots) { +// plat_mem_release(q->slots, 0); +// q->slots = NULL; +// } +// +// plat_sem_destroy(&q->items_sem); +// +// q->capacity = 0; +// q->mask = 0; +// +// atomic_store_explicit(&q->head, 0, memory_order_relaxed); +// atomic_store_explicit(&q->tail, 0, memory_order_relaxed); +// atomic_store_explicit(&q->committed, 0, memory_order_relaxed); +// } diff --git a/platform.c b/platform.c index 3813d71..0ea8108 100644 --- a/platform.c +++ b/platform.c @@ -247,13 +247,6 @@ typedef struct { void *arg; } ThreadWrapper; -static void *thread_start_routine(void *arg) { - ThreadWrapper *wrapper = (ThreadWrapper *)arg; - void *result = wrapper->func(wrapper->arg); - free(wrapper); - return result; -} - // Thread creation function static int thread_create(Thread *thread, ThreadFunc func, void *arg) { int ret = pthread_create(&thread->handle, NULL, func, arg); @@ -1200,9 +1193,9 @@ static int close_ioring(ThreadIoContext *thread_ctx) { (IORING_BUFFER_INFO) { .Address = (a), .Length = (uint32_t)(l) } static int ioring_submit(ThreadIoContext *thread_ctx, uint32_t wait_count, - uint32_t timeout_ms, uint32_t *submitted) { - HRESULT hr = SubmitIoRing(thread_ctx->ring, 0, timeout_ms, submitted); - // HRESULT hr = SubmitIoRing(ring, wait_count, timeout_ms, submitted); + uint32_t *submitted) { + HRESULT hr = SubmitIoRing(thread_ctx->ring, 0, SUBMIT_TIMEOUT_MS, submitted); + // HRESULT hr = SubmitIoRing(ring, wait_count, SUBMIT_TIMEOUT_MS, submitted); // The wait_count in windows is not implemented yet, so we wait with a // completion event for a single completion @@ -1214,7 +1207,6 @@ static int ioring_submit(ThreadIoContext *thread_ctx, uint32_t wait_count, } static void ioring_register_buffers(ThreadIoContext *thread_ctx, - uint32_t num_buffers, IORING_BUFFER_INFO *buf_info) { HRESULT hr = BuildIoRingRegisterBuffers( @@ -1230,7 +1222,7 @@ static void ioring_register_buffers(ThreadIoContext *thread_ctx, error_msg, (unsigned int)hr); } // Submit registration - ioring_submit(thread_ctx, 0, 0, NULL); + ioring_submit(thread_ctx, 0, NULL); } #if USE_REGISTERED_FILES @@ -1429,11 +1421,10 @@ static int close_ioring(ThreadIoContext *thread_ctx) { (IORING_BUFFER_INFO) { .iov_base = (a), .iov_len = (size_t)(l) } static void ioring_register_buffers(ThreadIoContext *thread_ctx, - uint32_t num_buffers, IORING_BUFFER_INFO *buf_info) { int ret = io_uring_register_buffers(&((IoUring *)thread_ctx->ring)->ring, - buf_info, num_buffers); + buf_info, NUM_BUFFERS_PER_THREAD); if (ret < 0) { if (ret == -ENOMEM) { @@ -1540,7 +1531,7 @@ static int ioring_build_read(ThreadIoContext *thread_ctx, } static int ioring_submit(ThreadIoContext *thread_ctx, uint32_t wait_count, - uint32_t timeout_ms, uint32_t *submitted) { + uint32_t *submitted) { int ret; if (wait_count > 0) { @@ -1745,7 +1736,7 @@ static ThreadIoContext *ioring_init_thread(void) { thread_ctx->free_count = NUM_BUFFERS_PER_THREAD; // Register buffers - ioring_register_buffers(thread_ctx, NUM_BUFFERS_PER_THREAD, buf_info); + ioring_register_buffers(thread_ctx, buf_info); #if USE_REGISTERED_FILES ioring_register_files(thread_ctx); @@ -1799,7 +1790,7 @@ static void return_buffer(ThreadIoContext *ctx, IoBuffer *buf) { } // -------------------------- Process completions --------------------------- -static void process_completions(ThreadIoContext *thread_ctx, FileQueue *fq) { +static void process_completions(ThreadIoContext *thread_ctx) { IoRingCQE cqe; while (ioring_pop_completion(thread_ctx->ring, &cqe) == 1) { @@ -2095,10 +2086,10 @@ static THREAD_RETURN hash_worker_ioring(void *arg) { wait_count = MIN(thread_ctx->num_submissions, NUM_BUFFERS_PER_THREAD - 6); submitted = 0; - ioring_submit(thread_ctx, wait_count, 0, &submitted); + ioring_submit(thread_ctx, wait_count, &submitted); // Process completions - process_completions(thread_ctx, &fq); + process_completions(thread_ctx); #if IORING_DEBUG_STATS printf(