From 5a95ac663ed6cc16e90faf07f95ed829cd8dbe4d Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 12:35:52 -0600 Subject: [PATCH 01/10] gitignore: externpro ignores --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 78f309c..f6df72c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ tests.txt *.o *.a *.so + +# externpro +.env +_bld*/ +docker-compose.override.yml From b7d2f4964bb7157934ffdd01255592a01615987d Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 13:09:13 -0600 Subject: [PATCH 02/10] Fix test_fec.cpp for new fecpp API - Update encode() to use contiguous input and callback-based output - Update decode() to use std::map and callback-based output - Add temporary buffers to prevent share data corruption during decode - Replace deprecated sprintf() with snprintf() - Fix format specifiers for size_t types - Add missing vector include and fecpp namespace usage --- test/test_fec.cpp | 56 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/test/test_fec.cpp b/test/test_fec.cpp index ae76d6e..8e5adf2 100644 --- a/test/test_fec.cpp +++ b/test/test_fec.cpp @@ -7,8 +7,11 @@ #include #include #include +#include #include "fecpp.h" +using fecpp::byte; + /* * compatibility stuff */ @@ -77,7 +80,7 @@ my_malloc(int sz, const char *s) */ int -test_decode(fec_code& code, size_t k, size_t index[], size_t sz, +test_decode(fecpp::fec_code& code, size_t k, size_t index[], size_t sz, const char *s) { int errors; @@ -88,12 +91,12 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz, static byte **d_original = NULL, **d_src = NULL ; if (sz < 1 || sz > 8192) { - fprintf(stderr, "test_decode: size %d invalid, must be 1..8K\n", + fprintf(stderr, "test_decode: size %zd invalid, must be 1..8K\n", sz); return 1 ; } if (k < 1 || k > 255 + 1) { - fprintf(stderr, "test_decode: k %d invalid, must be 1..%d\n", + fprintf(stderr, "test_decode: k %zd invalid, must be 1..%d\n", k, 255 + 1 ); return 2 ; } @@ -134,12 +137,45 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz, if (index[i] >= k ) reconstruct ++ ; TICK(ticks[2]); - for( i = 0 ; i < k ; i++ ) - code.encode(d_original, d_src[i], index[i], sz ); + // Create contiguous input for new API (requires size % K == 0) + std::vector contiguous_input(k * sz); + for( i = 0 ; i < k ; i++ ) { + memcpy(&contiguous_input[i * sz], d_original[i], sz); + } + + // Encode and capture only the shares we need + code.encode(contiguous_input.data(), k * sz, [&](size_t share_id, size_t total_shares, const uint8_t data[], size_t len) { + for( size_t j = 0; j < k; j++ ) { + if( index[j] == share_id ) { + memcpy(d_src[j], data, len); + break; + } + } + }); TOCK(ticks[2]); TICK(ticks[1]); - code.decode(d_src, index, sz); + std::map shares; + for( i = 0 ; i < k ; i++ ) { + shares[index[i]] = d_src[i]; + } + + // Use temp buffers to avoid overwriting share data during decode + byte** d_reconstructed = (byte**)my_malloc(k * sizeof(byte*), "d_reconstructed ptr"); + for( i = 0 ; i < k ; i++ ) { + d_reconstructed[i] = (byte*)my_malloc(sz, "d_reconstructed data"); + } + + code.decode(shares, sz, [&](size_t block_id, size_t k_blocks, const uint8_t data[], size_t len) { + memcpy(d_reconstructed[block_id], data, len); + }); + + // Copy results back to d_src + for( i = 0 ; i < k ; i++ ) { + memcpy(d_src[i], d_reconstructed[i], sz); + free(d_reconstructed[i]); + } + free(d_reconstructed); TOCK(ticks[1]); for (i=0; i 2 ; kk-- ) { - fec_code code(kk, lim); + fecpp::fec_code code(kk, lim); ixs = (size_t*)my_malloc(kk * sizeof(size_t), "ixs" ); for (i=0; i Date: Mon, 1 Jun 2026 13:10:21 -0600 Subject: [PATCH 03/10] cmake: xpExternPackage, externpro-ready --- CMakeLists.txt | 38 ++++++++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 14 ++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..69c9d11 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 4.3) +project(fecpp) +set(lib_name ${PROJECT_NAME}) +####################################### +set(${lib_name}_libsrcs + fecpp.cpp + fecpp.h + #fecpp_python.cpp # TODO + ) +source_group("" FILES ${${lib_name}_libsrcs}) +####################################### +add_library(${lib_name} STATIC ${${lib_name}_libsrcs}) +target_include_directories(${lib_name} PUBLIC $ + $ + ) +add_subdirectory(test) +####################################### +set(targetsFile ${PROJECT_NAME}-targets) +if(COMMAND xpExternPackage) + xpExternPackage(TARGETS_FILE ${targetsFile} + LIBRARIES ${lib_name} DEFAULT_TARGETS ${lib_name} + BASE 0.10 XPDIFF "intro" PVT_DEPS boost + WEB "http://www.randombit.net/code/fecpp/" UPSTREAM "github.com/randombit/fecpp" + DESC "C++ forward error correction with SIMD optimizations" + LICENSE "[BSD-2-Clause](http://www.randombit.net/code/fecpp/ 'BSD 2-Clause Simplified License')" + ) +elseif(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR ${CMAKE_INSTALL_DATADIR}/cmake) +endif() +install(TARGETS ${lib_name} EXPORT ${targetsFile} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +install(FILES fecpp.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}) +install(EXPORT ${targetsFile} DESTINATION ${CMAKE_INSTALL_CMAKEDIR} NAMESPACE ${PROJECT_NAME}::) +set(txtFiles format.txt license.txt news.txt readme.txt) +install(FILES ${txtFiles} DESTINATION ${CMAKE_INSTALL_DOCDIR}) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..5bc05a4 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,14 @@ +find_package(Boost) +set(${PROJECT_NAME}_exes + benchmark + gen_test_vec + test_fec + test_recovery + zfec + ) +set(test_recovery_deps Boost::headers) +foreach(exe ${${PROJECT_NAME}_exes}) + source_group("" FILES ${exe}.cpp) + add_executable(${exe} ${exe}.cpp) + target_link_libraries(${exe} PRIVATE ${lib_name} ${${exe}_deps}) +endforeach() From 8608a3a1a48b966680905cfe65cd571f1d4717f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 1 Jun 2026 19:16:27 +0000 Subject: [PATCH 04/10] externpro 26.01.1-49-g8c08f0b --- .devcontainer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer b/.devcontainer index 0e352b9..8c08f0b 160000 --- a/.devcontainer +++ b/.devcontainer @@ -1 +1 @@ -Subproject commit 0e352b953871074477bdddbd9cc0cf1b5c4d6b9d +Subproject commit 8c08f0bbbcff882a932fd81552aa161a829fff15 From 06662d008d691ceee3ff6df510020b2124c1b3f6 Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 13:33:23 -0600 Subject: [PATCH 05/10] cmake: Add conditional SIMD source inclusion - Conditionally include fecpp_sse2.cpp and fecpp_ssse3.cpp on x86_64 platforms - Test compiler support for SSE2/SSSE3 intrinsics before inclusion - Apply appropriate compile flags (-msse2, -mssse3) for GCC/Clang - Handle MSVC x64 build with automatic SSE2 and conditional SSSE3 support - Exclude SIMD sources on non-x86 architectures (ARM64, etc.) --- CMakeLists.txt | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69c9d11..9e88105 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,12 +7,73 @@ set(${lib_name}_libsrcs fecpp.h #fecpp_python.cpp # TODO ) +# Conditionally add SIMD optimized sources +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|x64)$") + # Check for SSE2 support + if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$") + include(CheckCXXSourceCompiles) + set(CMAKE_REQUIRED_FLAGS "-msse2") + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + return 0; + } + " HAVE_SSE2) + if(HAVE_SSE2) + list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp) + set(SSE2_COMPILE_OPTIONS $<$:-msse2>) + endif() + # Check for SSSE3 support + set(CMAKE_REQUIRED_FLAGS "-mssse3") + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + a = _mm_shuffle_epi8(a, a); + return 0; + } + " HAVE_SSSE3) + if(HAVE_SSSE3) + list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp) + set(SSSE3_COMPILE_OPTIONS $<$:-mssse3>) + endif() + unset(CMAKE_REQUIRED_FLAGS) + elseif(MSVC) + # MSVC automatically enables SSE2 on x64, check for SSSE3 availability + include(CheckCXXSourceCompiles) + check_cxx_source_compiles(" + #include + int main() { + __m128i a = _mm_set1_epi32(1); + a = _mm_shuffle_epi8(a, a); + return 0; + } + " HAVE_SSSE3) + # Always add SSE2 for MSVC x64 (it's always available) + list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp) + if(HAVE_SSSE3) + list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp) + set(SSSE3_COMPILE_DEFINITIONS __SSSE3__) + endif() + endif() +endif() source_group("" FILES ${${lib_name}_libsrcs}) ####################################### add_library(${lib_name} STATIC ${${lib_name}_libsrcs}) target_include_directories(${lib_name} PUBLIC $ $ ) +# Apply SIMD compile options if available +if(DEFINED SSE2_COMPILE_OPTIONS) + target_compile_options(${lib_name} PRIVATE ${SSE2_COMPILE_OPTIONS}) +endif() +if(DEFINED SSSE3_COMPILE_OPTIONS) + target_compile_options(${lib_name} PRIVATE ${SSSE3_COMPILE_OPTIONS}) +endif() +if(DEFINED SSSE3_COMPILE_DEFINITIONS) + target_compile_definitions(${lib_name} PRIVATE ${SSSE3_COMPILE_DEFINITIONS}) +endif() add_subdirectory(test) ####################################### set(targetsFile ${PROJECT_NAME}-targets) From 00935844011d570bb03024cbb336c091bbce5fc7 Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 13:59:23 -0600 Subject: [PATCH 06/10] cpu detection: enhance cpuid.cpp with cross-platform SSE support - Add cpuid.cpp to main library sources for universal CPU detection - Replace weak has_sse2()/has_ssse3() implementations with comprehensive detection - Support MSVC x64 (always SSE2), MSVC x86, GCC/Clang runtime detection, and fallbacks - Move intrin.h include inside MSVC-specific blocks for cross-platform compatibility - Enable future ARM64 support by centralizing CPU capability detection --- CMakeLists.txt | 1 + cpuid.cpp | 53 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e88105..30544b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,7 @@ project(fecpp) set(lib_name ${PROJECT_NAME}) ####################################### set(${lib_name}_libsrcs + cpuid.cpp fecpp.cpp fecpp.h #fecpp_python.cpp # TODO diff --git a/cpuid.cpp b/cpuid.cpp index 2554964..663c30a 100644 --- a/cpuid.cpp +++ b/cpuid.cpp @@ -1,10 +1,59 @@ #include "fecpp.h" +#if defined(_MSC_VER) +#include +#endif + namespace fecpp { -bool has_sse2() { return true; } +bool has_sse2() +{ +#if defined(_MSC_VER) + // MSVC on x64 always has SSE2 support + #if defined(_M_X64) || defined(_M_AMD64) + return true; + #else + // For 32-bit MSVC, assume no SSE2 for now + return false; + #endif +#else + // GCC/Clang: use runtime CPU detection + #if defined(__builtin_cpu_supports) + return __builtin_cpu_supports("sse2"); + #else + // Fallback for older compilers: assume SSE2 on x86_64 + #if defined(__x86_64__) || defined(__amd64__) || defined(_AMD64__) || defined(_M_X64) + return true; + #else + return false; + #endif + #endif +#endif +} -bool has_ssse3() { return true; } +bool has_ssse3() +{ +#if defined(_MSC_VER) + // MSVC: check if SSSE3 is enabled via compiler defines + #if defined(__SSSE3__) + return true; + #else + return false; + #endif +#else + // GCC/Clang: use runtime CPU detection + #if defined(__builtin_cpu_supports) + return __builtin_cpu_supports("ssse3"); + #else + // Fallback for older compilers: check if SSSE3 is enabled + #if defined(__SSSE3__) + return true; + #else + return false; + #endif + #endif +#endif +} } From 4c19c5ef2805493b7601e137756fed64cf58c4ea Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 14:07:43 -0600 Subject: [PATCH 07/10] sse2: add MSVC compatibility for Windows builds - Add intrin.h include for MSVC intrinsics support - Replace __builtin_clz with _BitScanReverse for MSVC compatibility - Fix _mm_prefetch calls with explicit const char* casts for MSVC - Maintain GCC/Clang compatibility through conditional compilation --- fecpp_sse2.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fecpp_sse2.cpp b/fecpp_sse2.cpp index 1474ac7..6462f94 100644 --- a/fecpp_sse2.cpp +++ b/fecpp_sse2.cpp @@ -6,6 +6,7 @@ #include "fecpp.h" #include +#include namespace fecpp { @@ -13,7 +14,13 @@ size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size) { const __m128i polynomial = _mm_set1_epi8(0x1D); +#if defined(_MSC_VER) + unsigned long y_bits; + _BitScanReverse(&y_bits, y); + y_bits += 1; // _BitScanReverse returns 0-based index +#else const size_t y_bits = 32 - __builtin_clz(y); +#endif // unrolled out to cache line size while(size >= 64) @@ -29,10 +36,10 @@ size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size) __m128i z_4 = _mm_load_si128((const __m128i*)(z + 48)); // prefetch next two x and z blocks - _mm_prefetch(x + 64, _MM_HINT_T0); - _mm_prefetch(z + 64, _MM_HINT_T0); - _mm_prefetch(x + 128, _MM_HINT_T1); - _mm_prefetch(z + 128, _MM_HINT_T1); + _mm_prefetch(reinterpret_cast(x + 64), _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(z + 64), _MM_HINT_T0); + _mm_prefetch(reinterpret_cast(x + 128), _MM_HINT_T1); + _mm_prefetch(reinterpret_cast(z + 128), _MM_HINT_T1); if(y & 0x01) { From 97a1c236c5b302b696a27ebe0dc76603b39f336b Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 14:13:41 -0600 Subject: [PATCH 08/10] build: fix cross-platform compatibility issues - Wrap intrin.h include in MSVC conditional to fix Linux AMD64 builds - Expand Windows detection to include _WIN32/_WIN64 for modern Windows compilers - Fix sys/time.h inclusion error on Windows CI systems - Maintain compatibility across Windows MSVC, Linux GCC/Clang, and macOS --- fecpp_sse2.cpp | 2 ++ test/test_fec.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fecpp_sse2.cpp b/fecpp_sse2.cpp index 6462f94..ded0ee0 100644 --- a/fecpp_sse2.cpp +++ b/fecpp_sse2.cpp @@ -6,7 +6,9 @@ #include "fecpp.h" #include +#if defined(_MSC_VER) #include +#endif namespace fecpp { diff --git a/test/test_fec.cpp b/test/test_fec.cpp index 8e5adf2..771cbbe 100644 --- a/test/test_fec.cpp +++ b/test/test_fec.cpp @@ -32,7 +32,7 @@ using fecpp::byte; #define DEB(x) #define DDB(x) x #define DEBUG 0 /* minimal debugging */ -#ifdef MSDOS +#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS) #include struct timeval { unsigned long ticks; From 2d55ad444ea6e2d48fb3c9afed338cfb32cf1c63 Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 14:23:18 -0600 Subject: [PATCH 09/10] test: fix Windows MSVC compatibility issues - Extend Windows detection to include _WIN32/_WIN64 for modern MSVC compilers - Add Windows-specific TICK macro using ticks instead of tv_sec/tv_usec - Fix bcmp function availability on Windows CI systems - Maintain original code structure with minimal targeted changes --- test/test_fec.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_fec.cpp b/test/test_fec.cpp index 771cbbe..84a87ef 100644 --- a/test/test_fec.cpp +++ b/test/test_fec.cpp @@ -15,7 +15,7 @@ using fecpp::byte; /* * compatibility stuff */ -#ifdef MSDOS /* but also for others, e.g. sun... */ +#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS) /* but also for others, e.g. sun... */ #define NEED_BCOPY #define bcmp(a,b,n) memcmp(a,b,n) #endif @@ -38,6 +38,7 @@ struct timeval { unsigned long ticks; }; #define gettimeofday(x, dummy) { (x)->ticks = clock() ; } +#define TICK(t) { struct timeval x ; gettimeofday(&x, NULL) ; t = x.ticks ; } #define DIFF_T(a,b) (1+ 1000000*(a.ticks - b.ticks) / CLOCKS_PER_SEC ) typedef unsigned long u_long ; typedef unsigned short u_short ; From b55b38679e633f893c514e48c811818518eb9f25 Mon Sep 17 00:00:00 2001 From: Scott M Anderson Date: Mon, 1 Jun 2026 14:34:26 -0600 Subject: [PATCH 10/10] test: fix TICK macro redefinition by moving #endif - Move #endif to include Unix TICK macro definition - Add TICK macro to Windows block for proper platform separation - Prevent macro redefinition warning on Windows CI builds - Maintain original macro ordering and structure --- test/test_fec.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_fec.cpp b/test/test_fec.cpp index 84a87ef..ec5a633 100644 --- a/test/test_fec.cpp +++ b/test/test_fec.cpp @@ -38,21 +38,21 @@ struct timeval { unsigned long ticks; }; #define gettimeofday(x, dummy) { (x)->ticks = clock() ; } -#define TICK(t) { struct timeval x ; gettimeofday(&x, NULL) ; t = x.ticks ; } #define DIFF_T(a,b) (1+ 1000000*(a.ticks - b.ticks) / CLOCKS_PER_SEC ) +#define TICK(t) { struct timeval x ; gettimeofday(&x, NULL) ; t = x.ticks ; } typedef unsigned long u_long ; typedef unsigned short u_short ; #else /* typically, unix systems */ #include #define DIFF_T(a,b) \ (1+ 1000000*(a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec) ) -#endif #define TICK(t) \ {struct timeval x ; \ gettimeofday(&x, NULL) ; \ t = x.tv_usec + 1000000* (x.tv_sec & 0xff ) ; \ } +#endif #define TOCK(t) \ { u_long t1 ; TICK(t1) ; \ if (t1 < t) t = 256000000 + t1 - t ; \