externpro 26.01.1-49-g8c08f0b

.gitignore

@@ -7,3 +7,8 @@ tests.txt
 *.o
 *.a
 *.so
+
+# externpro
+.env
+_bld*/
+docker-compose.override.yml

CMakeLists.txt

@@ -0,0 +1,100 @@
+cmake_minimum_required(VERSION 4.3)
+project(fecpp)
+set(lib_name ${PROJECT_NAME})
+#######################################
+set(${lib_name}_libsrcs
+  cpuid.cpp
+  fecpp.cpp
+  fecpp.h
+  #fecpp_python.cpp # TODO
+  )
+# Conditionally add SIMD optimized sources
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|x64)$")
+  # Check for SSE2 support
+  if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
+    include(CheckCXXSourceCompiles)
+    set(CMAKE_REQUIRED_FLAGS "-msse2")
+    check_cxx_source_compiles("
+      #include <emmintrin.h>
+      int main() {
+        __m128i a = _mm_set1_epi32(1);
+        return 0;
+      }
+    " HAVE_SSE2)
+    if(HAVE_SSE2)
+      list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp)
+      set(SSE2_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:CXX>:-msse2>)
+    endif()
+    # Check for SSSE3 support
+    set(CMAKE_REQUIRED_FLAGS "-mssse3")
+    check_cxx_source_compiles("
+      #include <tmmintrin.h>
+      int main() {
+        __m128i a = _mm_set1_epi32(1);
+        a = _mm_shuffle_epi8(a, a);
+        return 0;
+      }
+    " HAVE_SSSE3)
+    if(HAVE_SSSE3)
+      list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp)
+      set(SSSE3_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:CXX>:-mssse3>)
+    endif()
+    unset(CMAKE_REQUIRED_FLAGS)
+  elseif(MSVC)
+    # MSVC automatically enables SSE2 on x64, check for SSSE3 availability
+    include(CheckCXXSourceCompiles)
+    check_cxx_source_compiles("
+      #include <tmmintrin.h>
+      int main() {
+        __m128i a = _mm_set1_epi32(1);
+        a = _mm_shuffle_epi8(a, a);
+        return 0;
+      }
+    " HAVE_SSSE3)
+    # Always add SSE2 for MSVC x64 (it's always available)
+    list(APPEND ${lib_name}_libsrcs fecpp_sse2.cpp)
+    if(HAVE_SSSE3)
+      list(APPEND ${lib_name}_libsrcs fecpp_ssse3.cpp)
+      set(SSSE3_COMPILE_DEFINITIONS __SSSE3__)
+    endif()
+  endif()
+endif()
+source_group("" FILES ${${lib_name}_libsrcs})
+#######################################
+add_library(${lib_name} STATIC ${${lib_name}_libsrcs})
+target_include_directories(${lib_name} PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  )
+# Apply SIMD compile options if available
+if(DEFINED SSE2_COMPILE_OPTIONS)
+  target_compile_options(${lib_name} PRIVATE ${SSE2_COMPILE_OPTIONS})
+endif()
+if(DEFINED SSSE3_COMPILE_OPTIONS)
+  target_compile_options(${lib_name} PRIVATE ${SSSE3_COMPILE_OPTIONS})
+endif()
+if(DEFINED SSSE3_COMPILE_DEFINITIONS)
+  target_compile_definitions(${lib_name} PRIVATE ${SSSE3_COMPILE_DEFINITIONS})
+endif()
+add_subdirectory(test)
+#######################################
+set(targetsFile ${PROJECT_NAME}-targets)
+if(COMMAND xpExternPackage)
+  xpExternPackage(TARGETS_FILE ${targetsFile}
+    LIBRARIES ${lib_name} DEFAULT_TARGETS ${lib_name}
+    BASE 0.10 XPDIFF "intro" PVT_DEPS boost
+    WEB "http://www.randombit.net/code/fecpp/" UPSTREAM "github.com/randombit/fecpp"
+    DESC "C++ forward error correction with SIMD optimizations"
+    LICENSE "[BSD-2-Clause](http://www.randombit.net/code/fecpp/ 'BSD 2-Clause Simplified License')"
+    )
+elseif(NOT DEFINED CMAKE_INSTALL_CMAKEDIR)
+  set(CMAKE_INSTALL_CMAKEDIR ${CMAKE_INSTALL_DATADIR}/cmake)
+endif()
+install(TARGETS ${lib_name} EXPORT ${targetsFile}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  )
+install(FILES fecpp.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
+install(EXPORT ${targetsFile} DESTINATION ${CMAKE_INSTALL_CMAKEDIR} NAMESPACE ${PROJECT_NAME}::)
+set(txtFiles format.txt license.txt news.txt readme.txt)
+install(FILES ${txtFiles} DESTINATION ${CMAKE_INSTALL_DOCDIR})

cpuid.cpp

@@ -1,10 +1,59 @@
 
 #include "fecpp.h"
 
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
+
 namespace fecpp {
 
-bool has_sse2() { return true; }
+bool has_sse2()
+{
+#if defined(_MSC_VER)
+   // MSVC on x64 always has SSE2 support
+   #if defined(_M_X64) || defined(_M_AMD64)
+   return true;
+   #else
+   // For 32-bit MSVC, assume no SSE2 for now
+   return false;
+   #endif
+#else
+   // GCC/Clang: use runtime CPU detection
+   #if defined(__builtin_cpu_supports)
+   return __builtin_cpu_supports("sse2");
+   #else
+   // Fallback for older compilers: assume SSE2 on x86_64
+   #if defined(__x86_64__) || defined(__amd64__) || defined(_AMD64__) || defined(_M_X64)
+   return true;
+   #else
+   return false;
+   #endif
+   #endif
+#endif
+}
 
-bool has_ssse3() { return true; }
+bool has_ssse3()
+{
+#if defined(_MSC_VER)
+   // MSVC: check if SSSE3 is enabled via compiler defines
+   #if defined(__SSSE3__)
+   return true;
+   #else
+   return false;
+   #endif
+#else
+   // GCC/Clang: use runtime CPU detection
+   #if defined(__builtin_cpu_supports)
+   return __builtin_cpu_supports("ssse3");
+   #else
+   // Fallback for older compilers: check if SSSE3 is enabled
+   #if defined(__SSSE3__)
+   return true;
+   #else
+   return false;
+   #endif
+   #endif
+#endif
+}
 
 }

fecpp_sse2.cpp

@@ -6,14 +6,23 @@
 
 #include "fecpp.h"
 #include <emmintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
 
 namespace fecpp {
 
 size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size)
    {
    const __m128i polynomial = _mm_set1_epi8(0x1D);
 
+#if defined(_MSC_VER)
+   unsigned long y_bits;
+   _BitScanReverse(&y_bits, y);
+   y_bits += 1; // _BitScanReverse returns 0-based index
+#else
    const size_t y_bits = 32 - __builtin_clz(y);
+#endif
 
    // unrolled out to cache line size
    while(size >= 64)
@@ -29,10 +38,10 @@ size_t addmul_sse2(uint8_t z[], const uint8_t x[], uint8_t y, size_t size)
       __m128i z_4 = _mm_load_si128((const __m128i*)(z + 48));
 
       // prefetch next two x and z blocks
-      _mm_prefetch(x + 64, _MM_HINT_T0);
-      _mm_prefetch(z + 64, _MM_HINT_T0);
-      _mm_prefetch(x + 128, _MM_HINT_T1);
-      _mm_prefetch(z + 128, _MM_HINT_T1);
+      _mm_prefetch(reinterpret_cast<const char*>(x + 64), _MM_HINT_T0);
+      _mm_prefetch(reinterpret_cast<const char*>(z + 64), _MM_HINT_T0);
+      _mm_prefetch(reinterpret_cast<const char*>(x + 128), _MM_HINT_T1);
+      _mm_prefetch(reinterpret_cast<const char*>(z + 128), _MM_HINT_T1);
 
       if(y & 0x01)
          {

test/CMakeLists.txt

@@ -0,0 +1,14 @@
+find_package(Boost)
+set(${PROJECT_NAME}_exes
+  benchmark
+  gen_test_vec
+  test_fec
+  test_recovery
+  zfec
+  )
+set(test_recovery_deps Boost::headers)
+foreach(exe ${${PROJECT_NAME}_exes})
+  source_group("" FILES ${exe}.cpp)
+  add_executable(${exe} ${exe}.cpp)
+  target_link_libraries(${exe} PRIVATE ${lib_name} ${${exe}_deps})
+endforeach()

test/test_fec.cpp

@@ -7,12 +7,15 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <vector>
 #include "fecpp.h"
 
+using fecpp::byte;
+
 /*
  * compatibility stuff
  */
-#ifdef MSDOS	/* but also for others, e.g. sun... */
+#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS)	/* but also for others, e.g. sun... */
 #define NEED_BCOPY
 #define bcmp(a,b,n) memcmp(a,b,n)
 #endif
@@ -29,26 +32,27 @@
 #define DEB(x)
 #define DDB(x) x
 #define	DEBUG	0	/* minimal debugging */
-#ifdef	MSDOS
+#if defined(_WIN32) || defined(_WIN64) || defined(MSDOS)
 #include <time.h>
 struct timeval {
     unsigned long ticks;
 };
 #define gettimeofday(x, dummy) { (x)->ticks = clock() ; }
 #define DIFF_T(a,b) (1+ 1000000*(a.ticks - b.ticks) / CLOCKS_PER_SEC )
+#define TICK(t) { struct timeval x ; gettimeofday(&x, NULL) ; t = x.ticks ; }
 typedef unsigned long u_long ;
 typedef unsigned short u_short ;
 #else /* typically, unix systems */
 #include <sys/time.h>
 #define DIFF_T(a,b) \
 	(1+ 1000000*(a.tv_sec - b.tv_sec) + (a.tv_usec - b.tv_usec) )
-#endif
 
 #define TICK(t) \
 	{struct timeval x ; \
 	gettimeofday(&x, NULL) ; \
 	t = x.tv_usec + 1000000* (x.tv_sec & 0xff ) ; \
 	}
+#endif
 #define TOCK(t) \
 	{ u_long t1 ; TICK(t1) ; \
 	  if (t1 < t) t = 256000000 + t1 - t ; \
@@ -77,7 +81,7 @@ my_malloc(int sz, const char *s)
  */
 
 int
-test_decode(fec_code& code, size_t k, size_t index[], size_t sz,
+test_decode(fecpp::fec_code& code, size_t k, size_t index[], size_t sz,
             const char *s)
 {
     int errors;
@@ -88,12 +92,12 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz,
     static byte **d_original = NULL, **d_src = NULL ;
 
     if (sz < 1 || sz > 8192) {
-	fprintf(stderr, "test_decode: size %d invalid, must be 1..8K\n",
+	fprintf(stderr, "test_decode: size %zd invalid, must be 1..8K\n",
 		sz);
 	return 1 ;
     }
     if (k < 1 || k > 255 + 1) {
-	fprintf(stderr, "test_decode: k %d invalid, must be 1..%d\n",
+	fprintf(stderr, "test_decode: k %zd invalid, must be 1..%d\n",
 		k, 255 + 1 );
 	return 2 ;
     }
@@ -134,12 +138,45 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz,
 	if (index[i] >= k ) reconstruct ++ ;
 
     TICK(ticks[2]);
-    for( i = 0 ; i < k ; i++ )
-       code.encode(d_original, d_src[i], index[i], sz );
+    // Create contiguous input for new API (requires size % K == 0)
+    std::vector<uint8_t> contiguous_input(k * sz);
+    for( i = 0 ; i < k ; i++ ) {
+        memcpy(&contiguous_input[i * sz], d_original[i], sz);
+    }
+
+    // Encode and capture only the shares we need
+    code.encode(contiguous_input.data(), k * sz, [&](size_t share_id, size_t total_shares, const uint8_t data[], size_t len) {
+        for( size_t j = 0; j < k; j++ ) {
+            if( index[j] == share_id ) {
+                memcpy(d_src[j], data, len);
+                break;
+            }
+        }
+    });
     TOCK(ticks[2]);
 
     TICK(ticks[1]);
-    code.decode(d_src, index, sz);
+    std::map<size_t, const uint8_t*> shares;
+    for( i = 0 ; i < k ; i++ ) {
+        shares[index[i]] = d_src[i];
+    }
+
+    // Use temp buffers to avoid overwriting share data during decode
+    byte** d_reconstructed = (byte**)my_malloc(k * sizeof(byte*), "d_reconstructed ptr");
+    for( i = 0 ; i < k ; i++ ) {
+        d_reconstructed[i] = (byte*)my_malloc(sz, "d_reconstructed data");
+    }
+
+    code.decode(shares, sz, [&](size_t block_id, size_t k_blocks, const uint8_t data[], size_t len) {
+        memcpy(d_reconstructed[block_id], data, len);
+    });
+
+    // Copy results back to d_src
+    for( i = 0 ; i < k ; i++ ) {
+        memcpy(d_src[i], d_reconstructed[i], sz);
+        free(d_reconstructed[i]);
+    }
+    free(d_reconstructed);
     TOCK(ticks[1]);
 
     for (i=0; i<k; i++)
@@ -148,11 +185,11 @@ test_decode(fec_code& code, size_t k, size_t index[], size_t sz,
 	    fprintf(stderr, "error reconstructing block %d\n", i);
 	}
     if (errors)
-	fprintf(stderr, "Errors reconstructing %d blocks out of %d\n",
+	fprintf(stderr, "Errors reconstructing %d blocks out of %zd\n",
 	    errors, k);
 
     fprintf(stderr,
-	"  k %3d, l %3d  c_enc %10.6f MB/s c_dec %10.6f MB/s     \r",
+	"  k %3zd, l %3d  c_enc %10.6f MB/s c_dec %10.6f MB/s     \r",
 	k, reconstruct,
 	(double)(k * sz * reconstruct)/(double)ticks[2],
 	(double)(k * sz * reconstruct)/(double)ticks[1]);
@@ -206,11 +243,11 @@ main(int argc, char *argv[])
 
     for ( kk = KK ; kk > 2 ; kk-- )
        {
-       fec_code code(kk, lim);
+       fecpp::fec_code code(kk, lim);
        ixs = (size_t*)my_malloc(kk * sizeof(size_t), "ixs" );
 
        for (i=0; i<kk; i++) ixs[i] = kk - i ;
-       sprintf(buf, "kk=%d, kk - i", kk);
+       snprintf(buf, sizeof(buf), "kk=%d, kk - i", kk);
        test_decode(code, kk, ixs, SZ, buf);
 
        for (i=0; i<kk; i++) ixs[i] = i ;