diff --git a/.devcontainer b/.devcontainer index 0e352b9..8c08f0b 160000 --- a/.devcontainer +++ b/.devcontainer @@ -1 +1 @@ -Subproject commit 0e352b953871074477bdddbd9cc0cf1b5c4d6b9d +Subproject commit 8c08f0bbbcff882a932fd81552aa161a829fff15 diff --git a/.gitignore b/.gitignore index 78f309c..f6df72c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ tests.txt *.o *.a *.so + +# externpro +.env +_bld*/ +docker-compose.override.yml diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..30544b8 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,100 @@ +cmake_minimum_required(VERSION 4.3) +project(fecpp) +set(lib_name ${PROJECT_NAME}) +####################################### +set(${lib_name}_libsrcs + cpuid.cpp + fecpp.cpp + fecpp.h + #fecpp_python.cpp # TODO + ) +# Conditionally add SIMD optimized sources +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|x64)$") + # Check for SSE2 support + if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$") + include(CheckCXXSourceCompiles) + set(CMAKE_REQUIRED_FLAGS "-msse2") + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + return 0; + } + " HAVE_SSE2) + if(HAVE_SSE2) + list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp) + set(SSE2_COMPILE_OPTIONS $<$:-msse2>) + endif() + # Check for SSSE3 support + set(CMAKE_REQUIRED_FLAGS "-mssse3") + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + a = _mm_shuffle_epi8(a, a); + return 0; + } + " HAVE_SSSE3) + if(HAVE_SSSE3) + list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp) + set(SSSE3_COMPILE_OPTIONS $<$:-mssse3>) + endif() + unset(CMAKE_REQUIRED_FLAGS) + elseif(MSVC) + # MSVC automatically enables SSE2 on x64, check for SSSE3 availability + include(CheckCXXSourceCompiles) + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + a = _mm_shuffle_epi8(a, a); + return 0; + } + " HAVE_SSSE3) + # Always add SSE2 for MSVC x64 (it's always available) + list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp) + if(HAVE_SSSE3) + list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp) + set(SSSE3_COMPILE_DEFINITIONS __SSSE3__) + endif() + endif() +endif() +source_group("" FILES ${${lib_name}_libsrcs}) +####################################### +add_library(${lib_name} STATIC ${${lib_name}_libsrcs}) +target_include_directories(${lib_name} PUBLIC $ + $ + ) +# Apply SIMD compile options if available +if(DEFINED SSE2_COMPILE_OPTIONS) + target_compile_options(${lib_name} PRIVATE ${SSE2_COMPILE_OPTIONS}) +endif() +if(DEFINED SSSE3_COMPILE_OPTIONS) + target_compile_options(${lib_name} PRIVATE ${SSSE3_COMPILE_OPTIONS}) +endif() +if(DEFINED SSSE3_COMPILE_DEFINITIONS) + target_compile_definitions(${lib_name} PRIVATE ${SSSE3_COMPILE_DEFINITIONS}) +endif() +add_subdirectory(test) +####################################### +set(targetsFile ${PROJECT_NAME}-targets) +if(COMMAND xpExternPackage) + xpExternPackage(TARGETS_FILE ${targetsFile} + LIBRARIES ${lib_name} DEFAULT_TARGETS ${lib_name} + BASE 0.10 XPDIFF "intro" PVT_DEPS boost + WEB "http://www.randombit.net/code/fecpp/" UPSTREAM "github.com/randombit/fecpp" + DESC "C++ forward error correction with SIMD optimizations" + LICENSE "[BSD-2-Clause](http://www.randombit.net/code/fecpp/ 'BSD 2-Clause Simplified License')" + ) +elseif(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR ${CMAKE_INSTALL_DATADIR}/cmake) +endif() +install(TARGETS ${lib_name} EXPORT ${targetsFile} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +install(FILES fecpp.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}) +install(EXPORT ${targetsFile} DESTINATION ${CMAKE_INSTALL_CMAKEDIR} NAMESPACE ${PROJECT_NAME}::) +set(txtFiles format.txt license.txt news.txt readme.txt) +install(FILES ${txtFiles} DESTINATION ${CMAKE_INSTALL_DOCDIR}) diff --git a/cpuid.cpp b/cpuid.cpp index 2554964..663c30a 100644 --- a/cpuid.cpp +++ b/cpuid.cpp @@ -1,10 +1,59 @@ #include "fecpp.h" +#if defined(_MSC_VER) +#include +#endif + namespace fecpp { -bool has_sse2() { return true; } +bool has_sse2() +{ +#if defined(_MSC_VER) + // MSVC on x64 always has SSE2 support + #if defined(_M_X64) || defined(_M_AMD64) + return true; + #else + // For 32-bit MSVC, assume no SSE2 for now + return false; + #endif +#else + // GCC/Clang: use runtime CPU detection + #if defined(__builtin_cpu_supports) + return __builtin_cpu_supports("sse2"); + #else + // Fallback for older compilers: assume SSE2 on x86_64 + #if defined(__x86_64__) || defined(__amd64__) || defined(_AMD64__) || defined(_M_X64) + return true; + #else + return false; + #endif + #endif +#endif +} -bool has_ssse3() { return true; } +bool has_ssse3() +{ +#if defined(_MSC_VER) + // MSVC: check if SSSE3 is enabled via compiler defines + #if defined(__SSSE3__) + return true; + #else + return false; + #endif +#else + // GCC/Clang: use runtime CPU detection + #if defined(__builtin_cpu_supports) + return __builtin_cpu_supports("ssse3"); + #else + // Fallback for older compilers: check if SSSE3 is enabled + #if defined(__SSSE3__) + return true; + #else + return false; + #endif + #endif +#endif +} } diff --git a/fecpp_sse2.cpp b/fecpp_sse2.cpp index 1474ac7..ded0ee0 100644 --- a/fecpp_sse2.cpp +++ b/fecpp_sse2.cpp @@ -6,6 +6,9 @@ #include "fecpp.h" #include +#if defined(_MSC_VER) +#include +#endif namespace fecpp { @@ -13,7 +16,13 @@ size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size) { const __m128i polynomial = _mm_set1_epi8(0x1D); +#if defined(_MSC_VER) + unsigned long y_bits; + _BitScanReverse(&y_bits, y); + y_bits += 1; // _BitScanReverse returns 0-based index +#else const size_t y_bits = 32 - __builtin_clz(y); +#endif // unrolled out to cache line size while(size >= 64) @@ -29,10 +38,10 @@ size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size) __m128i z_4 = _mm_load_si128((const __m128i*)(z + 48)); // prefetch next two x and z blocks - _mm_prefetch(x + 64, _MM_HINT_T0); - _mm_prefetch(z + 64, _MM_HINT_T0); - _mm_prefetch(x + 128, _MM_HINT_T1); - _mm_prefetch(z + 128, _MM_HINT_T1); + _mm_prefetch(reinterpret_cast(x + 64), _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(z + 64), _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(x + 128), _MM_HINT_T1); + _mm_prefetch(reinterpret_cast(z + 128), _MM_HINT_T1); if(y & 0x01) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..5bc05a4 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,14 @@ +find_package(Boost) +set(${PROJECT_NAME}_exes + benchmark + gen_test_vec + test_fec + test_recovery + zfec + ) +set(test_recovery_deps Boost::headers) +foreach(exe ${${PROJECT_NAME}_exes}) + source_group("" FILES ${exe}.cpp) + add_executable(${exe} ${exe}.cpp) + target_link_libraries(${exe} PRIVATE ${lib_name} ${${exe}_deps}) +endforeach() diff --git a/test/test_fec.cpp b/test/test_fec.cpp index ae76d6e..ec5a633 100644 --- a/test/test_fec.cpp +++ b/test/test_fec.cpp @@ -7,12 +7,15 @@ #include #include #include +#include #include "fecpp.h" +using fecpp::byte; + /* * compatibility stuff */ -#ifdef MSDOS /* but also for others, e.g. sun... */ +#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS) /* but also for others, e.g. sun... */ #define NEED_BCOPY #define bcmp(a,b,n) memcmp(a,b,n) #endif @@ -29,26 +32,27 @@ #define DEB(x) #define DDB(x) x #define DEBUG 0 /* minimal debugging */ -#ifdef MSDOS +#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS) #include struct timeval { unsigned long ticks; }; #define gettimeofday(x, dummy) { (x)->ticks = clock() ; } #define DIFF_T(a,b) (1+ 1000000*(a.ticks - b.ticks) / CLOCKS_PER_SEC ) +#define TICK(t) { struct timeval x ; gettimeofday(&x, NULL) ; t = x.ticks ; } typedef unsigned long u_long ; typedef unsigned short u_short ; #else /* typically, unix systems */ #include #define DIFF_T(a,b) \ (1+ 1000000*(a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec) ) -#endif #define TICK(t) \ {struct timeval x ; \ gettimeofday(&x, NULL) ; \ t = x.tv_usec + 1000000* (x.tv_sec & 0xff ) ; \ } +#endif #define TOCK(t) \ { u_long t1 ; TICK(t1) ; \ if (t1 < t) t = 256000000 + t1 - t ; \ @@ -77,7 +81,7 @@ my_malloc(int sz, const char *s) */ int -test_decode(fec_code& code, size_t k, size_t index[], size_t sz, +test_decode(fecpp::fec_code& code, size_t k, size_t index[], size_t sz, const char *s) { int errors; @@ -88,12 +92,12 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz, static byte **d_original = NULL, **d_src = NULL ; if (sz < 1 || sz > 8192) { - fprintf(stderr, "test_decode: size %d invalid, must be 1..8K\n", + fprintf(stderr, "test_decode: size %zd invalid, must be 1..8K\n", sz); return 1 ; } if (k < 1 || k > 255 + 1) { - fprintf(stderr, "test_decode: k %d invalid, must be 1..%d\n", + fprintf(stderr, "test_decode: k %zd invalid, must be 1..%d\n", k, 255 + 1 ); return 2 ; } @@ -134,12 +138,45 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz, if (index[i] >= k ) reconstruct ++ ; TICK(ticks[2]); - for( i = 0 ; i < k ; i++ ) - code.encode(d_original, d_src[i], index[i], sz ); + // Create contiguous input for new API (requires size % K == 0) + std::vector contiguous_input(k * sz); + for( i = 0 ; i < k ; i++ ) { + memcpy(&contiguous_input[i * sz], d_original[i], sz); + } + + // Encode and capture only the shares we need + code.encode(contiguous_input.data(), k * sz, [&](size_t share_id, size_t total_shares, const uint8_t data[], size_t len) { + for( size_t j = 0; j < k; j++ ) { + if( index[j] == share_id ) { + memcpy(d_src[j], data, len); + break; + } + } + }); TOCK(ticks[2]); TICK(ticks[1]); - code.decode(d_src, index, sz); + std::map shares; + for( i = 0 ; i < k ; i++ ) { + shares[index[i]] = d_src[i]; + } + + // Use temp buffers to avoid overwriting share data during decode + byte** d_reconstructed = (byte**)my_malloc(k * sizeof(byte*), "d_reconstructed ptr"); + for( i = 0 ; i < k ; i++ ) { + d_reconstructed[i] = (byte*)my_malloc(sz, "d_reconstructed data"); + } + + code.decode(shares, sz, [&](size_t block_id, size_t k_blocks, const uint8_t data[], size_t len) { + memcpy(d_reconstructed[block_id], data, len); + }); + + // Copy results back to d_src + for( i = 0 ; i < k ; i++ ) { + memcpy(d_src[i], d_reconstructed[i], sz); + free(d_reconstructed[i]); + } + free(d_reconstructed); TOCK(ticks[1]); for (i=0; i 2 ; kk-- ) { - fec_code code(kk, lim); + fecpp::fec_code code(kk, lim); ixs = (size_t*)my_malloc(kk * sizeof(size_t), "ixs" ); for (i=0; i