diff --git a/.gitignore b/.gitignore index 0179a56..c66dec1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ !/plugins/** !/ScopeOneCuda/ !/ScopeOneCuda/** +!/ScopeOneInference/ +!/ScopeOneInference/** /ScopeOneCore/install/ /ScopeOneCore/build/ /ScopeOneCore/external/* diff --git a/README.md b/README.md index e5cadfe..0b6a665 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@

Compile Check + BSD 3-Clause License

-ScopeOne is open-source microscopy control software built with C++ and Qt. Hardware is accessed through provider-independent device contracts; [Micro-Manager](https://micro-manager.org/) is the built-in provider, and isolated devices run through ScopeOne DriverHost processes. -It retains compatibility with the Micro-Manager device ecosystem while allowing native vendor providers and adds a modular real-time image processing pipeline with support for background calibration, temporal filtering, FFT analysis, and more. - +ScopeOne is a high-performance open-source microscopy control platform built with C++ and Qt. It is designed to be an extensible platform for advanced microscopy applications, including multiple camera live imaging, on-the-fly image processing, and automated experiments. + It retains compatibility with the [Micro-Manager](https://micro-manager.org/) broad device ecosystem while allowing featuers like online image processing with GPU acceleration, deep learning inference and more.


Graphical User Interface of ScopeOne @@ -17,6 +17,19 @@ It retains compatibility with the Micro-Manager device ecosystem while allowing As an open-source project, ScopeOne builds on existing community efforts to reduce duplicated work and provides an alternative that enriches the microscopy community. While the current development is conducted in close collaboration with the optics and biology teams within our laboratory, we aim to expand engagement with the broader research community to make the platform more practical, accessible, and universal. Any issues or pull requests are greatly appreciated！ +## ✨ New Features + +### Deep Learning Inference + +Deep learning is becoming an increasingly powerful tool for microscopy image restoration and analysis. ScopeOne allows users to import pre-trained AI models through the widely supported ONNX model format and apply them directly to live camera streams or recorded images. Models can run on either the CPU or GPU. + +

+
+ Real-time denoising demo using DnCNN at approximately 150 FPS for 256 × 256 images, tested on an NVIDIA RTX 4070 Ti +

+ +### GPU-Accelerated Online Image Processing +Despite the deep learning, traditional image processing methods remain essential. GPU-accelerated image processing pipeline can be applied to live camera streams or recorded images, including filtering, background subtraction, FFT, and more. Users can customize image processing algorithms and parameters to suit their specific needs via the ScopeOne plugin system. ## 🚀 Quick Start ### For Users diff --git a/ScopeOneCore/include/scopeone/ProcessingPlugin.h b/ScopeOneCore/include/scopeone/ProcessingPlugin.h index d48b9d6..eaba54a 100644 --- a/ScopeOneCore/include/scopeone/ProcessingPlugin.h +++ b/ScopeOneCore/include/scopeone/ProcessingPlugin.h @@ -19,7 +19,8 @@ namespace scopeone::core Integer, Real, Boolean, - Choice + Choice, + FilePath }; struct ProcessingParameterChoice @@ -39,6 +40,7 @@ namespace scopeone::core QVariant step; int decimals{0}; QList choices; + QString fileFilter; }; struct ProcessingModuleDescriptor diff --git a/ScopeOneCore/include/scopeone/SimulatorProvider.h b/ScopeOneCore/include/scopeone/SimulatorProvider.h index c653e04..0ac1f79 100644 --- a/ScopeOneCore/include/scopeone/SimulatorProvider.h +++ b/ScopeOneCore/include/scopeone/SimulatorProvider.h @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include #include +#include #include "scopeone/CameraProvider.h" #include "scopeone/HardwareProvider.h" @@ -64,11 +66,13 @@ namespace scopeone::core enum class ImageMode { Gradient, - Hologram + Hologram, + Beads }; bool accepts(const QString& cameraIdOrAll) const; ImageFrame makeFrame(); + void rebuildBeadsImage(); void updateTimerInterval(); QString m_providerId; @@ -77,8 +81,11 @@ namespace scopeone::core int m_sensorHeight{512}; QRect m_roi; double m_exposureMs{10.0}; + double m_gaussianNoiseSigma{0.0}; ImageMode m_imageMode{ImageMode::Gradient}; + QByteArray m_beadsImage; quint64 m_frameIndex{0}; + std::mt19937 m_randomGenerator{std::random_device{}()}; FrameSink m_frameSink; PreviewStateSink m_previewStateSink; QTimer m_timer; diff --git a/ScopeOneCore/src/ScopeOneCore.cpp b/ScopeOneCore/src/ScopeOneCore.cpp index c832207..cd761d6 100644 --- a/ScopeOneCore/src/ScopeOneCore.cpp +++ b/ScopeOneCore/src/ScopeOneCore.cpp @@ -998,8 +998,8 @@ namespace scopeone::core CameraManager* cameraManager{nullptr}; RecordingManager* recordingManager{nullptr}; ImageProcessingManager* imageProcessingManager{nullptr}; + std::vector> processingExtensionLibraries; std::unique_ptr processingModuleRegistry; - std::unique_ptr cudaLibrary; StageMosaicManager* stageMosaicManager{nullptr}; internal::DaqDeviceManager* daqDeviceManager{nullptr}; internal::SignalSourceManager* signalSourceManager{nullptr}; @@ -1255,24 +1255,27 @@ namespace scopeone::core "scopeone::core::ScanImageConfig"); m_managers->processingModuleRegistry = std::make_unique(); - const QString cudaLibraryPath = QDir(QCoreApplication::applicationDirPath()) - .filePath(QStringLiteral("ScopeOneCuda")); - { - auto cudaLibrary = std::make_unique(cudaLibraryPath); - if (cudaLibrary->load()) + const QStringList processingExtensionNames{ + QStringLiteral("ScopeOneCuda"), + QStringLiteral("ScopeOneInference")}; + for (const QString& extensionName : processingExtensionNames) + { + auto library = std::make_unique( + QDir(QCoreApplication::applicationDirPath()).filePath(extensionName)); + if (library->load()) { using RegisterProcessingModules = void (*)(ScopeOneCore*); const auto registerProcessingModules = reinterpret_cast( - cudaLibrary->resolve("scopeone_register_processing_modules")); + library->resolve("scopeone_register_processing_modules")); if (registerProcessingModules) { registerProcessingModules(this); - m_managers->cudaLibrary = std::move(cudaLibrary); + m_managers->processingExtensionLibraries.push_back(std::move(library)); } else { - cudaLibrary->unload(); + library->unload(); } } } diff --git a/ScopeOneCore/src/SimulatorProvider.cpp b/ScopeOneCore/src/SimulatorProvider.cpp index cadf40e..7acd200 100644 --- a/ScopeOneCore/src/SimulatorProvider.cpp +++ b/ScopeOneCore/src/SimulatorProvider.cpp @@ -5,11 +5,44 @@ #include #include #include +#include #include #include +#include namespace scopeone::core { + namespace + { + struct Bead + { + double x; + double y; + double brightness; + }; + + constexpr std::array Beads{{ + {0.08, 0.10, 0.91}, {0.19, 0.08, 0.84}, {0.32, 0.12, 0.98}, + {0.46, 0.07, 0.88}, {0.62, 0.11, 0.94}, {0.77, 0.08, 0.82}, + {0.91, 0.13, 0.96}, {0.13, 0.25, 0.87}, {0.29, 0.29, 1.00}, + {0.48, 0.23, 0.83}, {0.69, 0.27, 0.92}, {0.87, 0.31, 0.86}, + {0.08, 0.43, 0.95}, {0.24, 0.47, 0.81}, {0.42, 0.39, 0.90}, + {0.61, 0.45, 0.97}, {0.80, 0.42, 0.85}, {0.94, 0.50, 0.93}, + {0.14, 0.63, 0.89}, {0.34, 0.58, 0.99}, {0.53, 0.66, 0.84}, + {0.72, 0.61, 0.96}, {0.89, 0.68, 0.80}, {0.07, 0.82, 0.92}, + {0.22, 0.88, 0.86}, {0.39, 0.79, 0.95}, {0.58, 0.86, 0.82}, + {0.76, 0.81, 0.98}, {0.92, 0.89, 0.88}, {0.47, 0.94, 0.91}, + {0.30, 0.70, 0.85}, {0.67, 0.75, 0.94} + }}; + + constexpr std::array BeadDoublets{{ + {0.175, 0.176, 1.00}, {0.191, 0.176, 0.92}, + {0.535, 0.335, 0.88}, {0.551, 0.335, 0.96}, + {0.365, 0.515, 0.98}, {0.381, 0.515, 0.86}, + {0.705, 0.925, 0.91}, {0.721, 0.925, 1.00} + }}; + } + SimulatorProvider::SimulatorProvider(const QString& logicalCameraId, int width, int height, @@ -152,6 +185,8 @@ namespace scopeone::core return accepts(cameraId) ? QStringList{QStringLiteral("Exposure"), QStringLiteral("ImageMode"), + QStringLiteral("Resolution"), + QStringLiteral("GaussianNoiseSigma"), QStringLiteral("SensorWidth"), QStringLiteral("SensorHeight")} : QStringList{}; @@ -173,16 +208,31 @@ namespace scopeone::core if (name == QStringLiteral("ImageMode")) { QMutexLocker locker(&m_mutex); - return m_imageMode == ImageMode::Hologram - ? QStringLiteral("Hologram") - : QStringLiteral("Gradient"); + switch (m_imageMode) + { + case ImageMode::Gradient: return QStringLiteral("Gradient"); + case ImageMode::Hologram: return QStringLiteral("Hologram"); + case ImageMode::Beads: return QStringLiteral("Fluorescent Beads"); + } + } + if (name == QStringLiteral("Resolution")) + { + QMutexLocker locker(&m_mutex); + return QString::number(m_sensorWidth); + } + if (name == QStringLiteral("GaussianNoiseSigma")) + { + QMutexLocker locker(&m_mutex); + return QString::number(m_gaussianNoiseSigma, 'g', 12); } if (name == QStringLiteral("SensorWidth")) { + QMutexLocker locker(&m_mutex); return QString::number(m_sensorWidth); } if (name == QStringLiteral("SensorHeight")) { + QMutexLocker locker(&m_mutex); return QString::number(m_sensorHeight); } return {}; @@ -208,7 +258,9 @@ namespace scopeone::core if (name == QStringLiteral("ImageMode")) { const QString mode = value.trimmed(); - if (mode != QStringLiteral("Gradient") && mode != QStringLiteral("Hologram")) + if (mode != QStringLiteral("Gradient") + && mode != QStringLiteral("Hologram") + && mode != QStringLiteral("Fluorescent Beads")) { if (errorMessage) { @@ -217,9 +269,41 @@ namespace scopeone::core return false; } QMutexLocker locker(&m_mutex); - m_imageMode = mode == QStringLiteral("Hologram") - ? ImageMode::Hologram - : ImageMode::Gradient; + m_imageMode = mode == QStringLiteral("Hologram") ? ImageMode::Hologram + : mode == QStringLiteral("Fluorescent Beads") ? ImageMode::Beads + : ImageMode::Gradient; + if (m_imageMode == ImageMode::Beads && m_beadsImage.isEmpty()) + { + rebuildBeadsImage(); + } + return true; + } + if (name == QStringLiteral("Resolution")) + { + const int resolution = value.toInt(); + if (!getAllowedPropertyValues(cameraId, name).contains(QString::number(resolution))) + { + if (errorMessage) + { + *errorMessage = QStringLiteral("Invalid resolution"); + } + return false; + } + QMutexLocker locker(&m_mutex); + m_sensorWidth = resolution; + m_sensorHeight = resolution; + m_roi = QRect(0, 0, resolution, resolution); + m_beadsImage.clear(); + if (m_imageMode == ImageMode::Beads) + { + rebuildBeadsImage(); + } + return true; + } + if (name == QStringLiteral("GaussianNoiseSigma")) + { + QMutexLocker locker(&m_mutex); + m_gaussianNoiseSigma = std::clamp(value.toDouble(), 0.0, 100.0); return true; } if (name == QStringLiteral("Exposure")) @@ -246,6 +330,7 @@ namespace scopeone::core { return accepts(cameraId) && listProperties(cameraId).contains(name) ? (name == QStringLiteral("Exposure") + || name == QStringLiteral("GaussianNoiseSigma") ? QStringLiteral("Float") : name == QStringLiteral("ImageMode") ? QStringLiteral("String") @@ -257,7 +342,9 @@ namespace scopeone::core { return !accepts(cameraId) || (name != QStringLiteral("Exposure") - && name != QStringLiteral("ImageMode")); + && name != QStringLiteral("ImageMode") + && name != QStringLiteral("Resolution") + && name != QStringLiteral("GaussianNoiseSigma")); } bool SimulatorProvider::isPropertyPreInit(const QString&, const QString&) @@ -270,24 +357,45 @@ namespace scopeone::core { if (accepts(cameraId) && name == QStringLiteral("ImageMode")) { - return {QStringLiteral("Gradient"), QStringLiteral("Hologram")}; + return {QStringLiteral("Gradient"), + QStringLiteral("Hologram"), + QStringLiteral("Fluorescent Beads")}; + } + if (accepts(cameraId) && name == QStringLiteral("Resolution")) + { + return {QStringLiteral("256"), + QStringLiteral("512"), + QStringLiteral("1024"), + QStringLiteral("2048")}; } return {}; } bool SimulatorProvider::hasPropertyLimits(const QString& cameraId, const QString& name) { - return accepts(cameraId) && name == QStringLiteral("Exposure"); + return accepts(cameraId) + && (name == QStringLiteral("Exposure") + || name == QStringLiteral("GaussianNoiseSigma")); } double SimulatorProvider::getPropertyLowerLimit(const QString& cameraId, const QString& name) { - return accepts(cameraId) && name == QStringLiteral("Exposure") ? 0.1 : 0.0; + if (!accepts(cameraId)) + { + return 0.0; + } + return name == QStringLiteral("Exposure") ? 0.1 : 0.0; } double SimulatorProvider::getPropertyUpperLimit(const QString& cameraId, const QString& name) { - return accepts(cameraId) && name == QStringLiteral("Exposure") ? 1000.0 : 0.0; + if (!accepts(cameraId)) + { + return 0.0; + } + return name == QStringLiteral("Exposure") ? 1000.0 + : name == QStringLiteral("GaussianNoiseSigma") ? 100.0 + : 0.0; } bool SimulatorProvider::setROI(const QString& cameraId, @@ -297,12 +405,15 @@ namespace scopeone::core int height) { const QRect roi(x, y, width, height); - const QRect sensor(0, 0, m_sensorWidth, m_sensorHeight); - if (!accepts(cameraId) || width <= 0 || height <= 0 || !sensor.contains(roi)) + if (!accepts(cameraId) || width <= 0 || height <= 0) { return false; } QMutexLocker locker(&m_mutex); + if (!QRect(0, 0, m_sensorWidth, m_sensorHeight).contains(roi)) + { + return false; + } m_roi = roi; return true; } @@ -372,7 +483,7 @@ namespace scopeone::core frame.sourceRoiWidth = m_roi.width(); frame.sourceRoiHeight = m_roi.height(); frame.bytes.resize(static_cast(frame.width) * frame.height); - const bool hologram = m_imageMode == ImageMode::Hologram; + std::normal_distribution gaussianNoise; constexpr double twoPi = 6.28318530717958647692; for (int y = 0; y < frame.height; ++y) { @@ -380,34 +491,95 @@ namespace scopeone::core + static_cast(y) * frame.stride; for (int x = 0; x < frame.width; ++x) { - if (!hologram) + double intensity = 0.0; + if (m_imageMode == ImageMode::Gradient) { - row[x] = static_cast((x + y + frame.frameIndex) & 0xffu); - continue; + intensity = static_cast((x + y + frame.frameIndex) & 0xffu); + } + else if (m_imageMode == ImageMode::Hologram) + { + const int sensorX = m_roi.x() + x; + const int sensorY = m_roi.y() + y; + const double nx = (sensorX - 0.5 * m_sensorWidth) / m_sensorWidth; + const double ny = (sensorY - 0.5 * m_sensorHeight) / m_sensorHeight; + const double objectAmplitude = + 0.65 * std::exp(-35.0 * (nx * nx + ny * ny)) + + 0.35 * std::exp(-90.0 * ((nx - 0.18) * (nx - 0.18) + + (ny + 0.12) * (ny + 0.12))); + const double objectPhase = 18.0 * (nx * nx + ny * ny) + + 0.015 * static_cast(frame.frameIndex); + const double carrier = twoPi * (48.0 * sensorX / m_sensorWidth + + 32.0 * sensorY / m_sensorHeight); + intensity = 70.0 + + 55.0 * objectAmplitude * objectAmplitude + + 120.0 * objectAmplitude * std::cos(carrier + objectPhase); + } + else + { + const int sensorX = m_roi.x() + x; + const int sensorY = m_roi.y() + y; + intensity = static_cast(m_beadsImage.at( + sensorY * m_sensorWidth + sensorX)); + } + if (m_gaussianNoiseSigma > 0.0) + { + intensity += m_gaussianNoiseSigma * gaussianNoise(m_randomGenerator); } - - const int sensorX = m_roi.x() + x; - const int sensorY = m_roi.y() + y; - const double nx = (sensorX - 0.5 * m_sensorWidth) / m_sensorWidth; - const double ny = (sensorY - 0.5 * m_sensorHeight) / m_sensorHeight; - const double objectAmplitude = - 0.65 * std::exp(-35.0 * (nx * nx + ny * ny)) - + 0.35 * std::exp(-90.0 * ((nx - 0.18) * (nx - 0.18) - + (ny + 0.12) * (ny + 0.12))); - const double objectPhase = 18.0 * (nx * nx + ny * ny) - + 0.015 * static_cast(frame.frameIndex); - const double carrier = twoPi * (48.0 * sensorX / m_sensorWidth - + 32.0 * sensorY / m_sensorHeight); - const double intensity = 70.0 - + 55.0 * objectAmplitude * objectAmplitude - + 120.0 * objectAmplitude - * std::cos(carrier + objectPhase); row[x] = static_cast(std::clamp(intensity, 0.0, 255.0)); } } return frame; } + // Build a static field of Gaussian PSFs for fast live preview + void SimulatorProvider::rebuildBeadsImage() + { + std::vector intensities( + static_cast(m_sensorWidth) * m_sensorHeight, 3.0f); + const double resolutionScale = (std::min)(m_sensorWidth, m_sensorHeight) / 256.0; + const double sigma = 4.0 * resolutionScale / 2.354820045; + const int radius = static_cast(std::ceil(4.0 * sigma)); + + const auto drawBead = [&](const Bead& bead) + { + const double centerX = bead.x * m_sensorWidth; + const double centerY = bead.y * m_sensorHeight; + const int minX = (std::max)(0, static_cast(std::floor(centerX)) - radius); + const int maxX = (std::min)(m_sensorWidth - 1, + static_cast(std::floor(centerX)) + radius); + const int minY = (std::max)(0, static_cast(std::floor(centerY)) - radius); + const int maxY = (std::min)(m_sensorHeight - 1, + static_cast(std::floor(centerY)) + radius); + const double denominator = 2.0 * sigma * sigma; + for (int y = minY; y <= maxY; ++y) + { + for (int x = minX; x <= maxX; ++x) + { + const double dx = x + 0.5 - centerX; + const double dy = y + 0.5 - centerY; + intensities[static_cast(y) * m_sensorWidth + x] + += static_cast(245.0 * bead.brightness + * std::exp(-(dx * dx + dy * dy) / denominator)); + } + } + }; + + for (const Bead& bead : Beads) + { + drawBead(bead); + } + for (const Bead& bead : BeadDoublets) + { + drawBead(bead); + } + + m_beadsImage.resize(m_sensorWidth * m_sensorHeight); + for (qsizetype i = 0; i < m_beadsImage.size(); ++i) + { + m_beadsImage[i] = static_cast(std::clamp(intensities[i], 0.0f, 255.0f)); + } + } + void SimulatorProvider::updateTimerInterval() { double exposureMs = 0.0; diff --git a/ScopeOneInference/CMakeLists.txt b/ScopeOneInference/CMakeLists.txt new file mode 100644 index 0000000..5d920e2 --- /dev/null +++ b/ScopeOneInference/CMakeLists.txt @@ -0,0 +1,111 @@ +cmake_minimum_required(VERSION 3.23) + +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(ScopeOneInference VERSION 1.0.0 LANGUAGES CXX) + find_package(Qt6 REQUIRED COMPONENTS Core) + find_package(ScopeOneCore CONFIG REQUIRED) +endif () + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(ONNXRUNTIME_ROOT + "${CMAKE_CURRENT_SOURCE_DIR}/../ScopeOneCore/external/onnxruntime-win-x64-gpu_cuda12-1.30.0" + CACHE PATH "ONNX Runtime SDK root" +) + +find_path(ONNXRUNTIME_INCLUDE_DIR + NAMES onnxruntime_cxx_api.h + PATHS "${ONNXRUNTIME_ROOT}/include" + NO_DEFAULT_PATH + REQUIRED +) +find_library(ONNXRUNTIME_LIBRARY + NAMES onnxruntime + PATHS "${ONNXRUNTIME_ROOT}/lib" + NO_DEFAULT_PATH + REQUIRED +) + +add_library(ScopeOneInference SHARED + src/OnnxInternal.h + src/OnnxRuntime.cpp + src/OnnxSession.cpp + src/OnnxInferenceModule.cpp + src/OnnxInferenceModule.h + src/InferenceProcessingRegistration.cpp + include/scopeone/inference/ExecutionProvider.h + include/scopeone/inference/InferenceExport.h + include/scopeone/inference/OnnxRuntime.h + include/scopeone/inference/OnnxSession.h + include/scopeone/inference/TensorInfo.h +) +add_library(scopeone::Inference ALIAS ScopeOneInference) + +set_target_properties(ScopeOneInference PROPERTIES + EXPORT_NAME Inference + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" +) +target_compile_features(ScopeOneInference PUBLIC cxx_std_20) +target_compile_definitions(ScopeOneInference PRIVATE SCOPEONE_INFERENCE_EXPORTS) +target_include_directories(ScopeOneInference + PUBLIC + $ + $ + PRIVATE + "${ONNXRUNTIME_INCLUDE_DIR}" +) +target_link_libraries(ScopeOneInference PRIVATE + "${ONNXRUNTIME_LIBRARY}" + scopeone::ScopeOneCore + Qt6::Core +) + +if (WIN32) + file(GLOB ONNXRUNTIME_DLLS + "${ONNXRUNTIME_ROOT}/bin/onnxruntime*.dll" + "${ONNXRUNTIME_ROOT}/lib/onnxruntime*.dll" + ) + if (NOT ONNXRUNTIME_DLLS) + message(FATAL_ERROR "No ONNX Runtime DLLs found under ${ONNXRUNTIME_ROOT}") + endif () + add_custom_command(TARGET ScopeOneInference POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${ONNXRUNTIME_DLLS} + "$" + COMMAND_EXPAND_LISTS + ) +endif () + +install(TARGETS ScopeOneInference + EXPORT ScopeOneInferenceTargets + RUNTIME DESTINATION "." + LIBRARY DESTINATION "." + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" +) +install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +if (ONNXRUNTIME_DLLS) + install(FILES ${ONNXRUNTIME_DLLS} DESTINATION ".") +endif () +install(EXPORT ScopeOneInferenceTargets + FILE ScopeOneInferenceTargets.cmake + NAMESPACE scopeone:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/ScopeOneInference" +) + +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/ScopeOneInferenceConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/ScopeOneInferenceConfig.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/ScopeOneInference" +) +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/ScopeOneInferenceConfigVersion.cmake" + VERSION 1.0.0 + COMPATIBILITY SameMajorVersion +) +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/ScopeOneInferenceConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/ScopeOneInferenceConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/ScopeOneInference" +) diff --git a/ScopeOneInference/cmake/ScopeOneInferenceConfig.cmake.in b/ScopeOneInference/cmake/ScopeOneInferenceConfig.cmake.in new file mode 100644 index 0000000..b2aada3 --- /dev/null +++ b/ScopeOneInference/cmake/ScopeOneInferenceConfig.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(ScopeOneCore CONFIG REQUIRED) + +include("${CMAKE_CURRENT_LIST_DIR}/ScopeOneInferenceTargets.cmake") diff --git a/ScopeOneInference/include/scopeone/inference/ExecutionProvider.h b/ScopeOneInference/include/scopeone/inference/ExecutionProvider.h new file mode 100644 index 0000000..4e1806c --- /dev/null +++ b/ScopeOneInference/include/scopeone/inference/ExecutionProvider.h @@ -0,0 +1,10 @@ +#pragma once + +namespace scopeone::inference +{ + enum class ExecutionProvider + { + Cpu, + Cuda + }; +} diff --git a/ScopeOneInference/include/scopeone/inference/InferenceExport.h b/ScopeOneInference/include/scopeone/inference/InferenceExport.h new file mode 100644 index 0000000..da8c8ef --- /dev/null +++ b/ScopeOneInference/include/scopeone/inference/InferenceExport.h @@ -0,0 +1,11 @@ +#pragma once + +#if defined(_WIN32) +# if defined(SCOPEONE_INFERENCE_EXPORTS) +# define SCOPEONE_INFERENCE_EXPORT __declspec(dllexport) +# else +# define SCOPEONE_INFERENCE_EXPORT __declspec(dllimport) +# endif +#else +# define SCOPEONE_INFERENCE_EXPORT __attribute__((visibility("default"))) +#endif diff --git a/ScopeOneInference/include/scopeone/inference/OnnxRuntime.h b/ScopeOneInference/include/scopeone/inference/OnnxRuntime.h new file mode 100644 index 0000000..d11ce44 --- /dev/null +++ b/ScopeOneInference/include/scopeone/inference/OnnxRuntime.h @@ -0,0 +1,35 @@ +#pragma once + +#include "scopeone/inference/ExecutionProvider.h" +#include "scopeone/inference/InferenceExport.h" + +#include +#include +#include + +namespace scopeone::inference +{ + class OnnxSession; + + class SCOPEONE_INFERENCE_EXPORT OnnxRuntime + { + public: + OnnxRuntime(); + ~OnnxRuntime(); + + OnnxRuntime(const OnnxRuntime&) = delete; + OnnxRuntime& operator=(const OnnxRuntime&) = delete; + OnnxRuntime(OnnxRuntime&&) noexcept; + OnnxRuntime& operator=(OnnxRuntime&&) noexcept; + + std::string version() const; + std::vector availableProviders() const; + bool supports(ExecutionProvider provider) const; + + private: + struct Impl; + std::shared_ptr m_impl; + + friend class OnnxSession; + }; +} diff --git a/ScopeOneInference/include/scopeone/inference/OnnxSession.h b/ScopeOneInference/include/scopeone/inference/OnnxSession.h new file mode 100644 index 0000000..b7d4ba3 --- /dev/null +++ b/ScopeOneInference/include/scopeone/inference/OnnxSession.h @@ -0,0 +1,42 @@ +#pragma once + +#include "scopeone/inference/ExecutionProvider.h" +#include "scopeone/inference/InferenceExport.h" +#include "scopeone/inference/TensorInfo.h" + +#include +#include +#include + +namespace scopeone::inference +{ + class OnnxRuntime; + + struct SessionOptions + { + ExecutionProvider provider{ExecutionProvider::Cpu}; + int deviceId{0}; + }; + + class SCOPEONE_INFERENCE_EXPORT OnnxSession + { + public: + OnnxSession(OnnxRuntime& runtime, + const std::filesystem::path& modelPath, + const SessionOptions& options = {}); + ~OnnxSession(); + + OnnxSession(const OnnxSession&) = delete; + OnnxSession& operator=(const OnnxSession&) = delete; + OnnxSession(OnnxSession&&) noexcept; + OnnxSession& operator=(OnnxSession&&) noexcept; + + const std::vector& inputs() const; + const std::vector& outputs() const; + std::vector run(const std::vector& inputs); + + private: + struct Impl; + std::unique_ptr m_impl; + }; +} diff --git a/ScopeOneInference/include/scopeone/inference/TensorInfo.h b/ScopeOneInference/include/scopeone/inference/TensorInfo.h new file mode 100644 index 0000000..d275feb --- /dev/null +++ b/ScopeOneInference/include/scopeone/inference/TensorInfo.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +namespace scopeone::inference +{ + enum class TensorElementType + { + Unknown, + Float32, + UInt8, + Int8, + UInt16, + Int16, + Int32, + Int64, + String, + Boolean, + Float16, + Float64, + UInt32, + UInt64, + Complex64, + Complex128, + BFloat16 + }; + + struct TensorInfo + { + std::string name; + TensorElementType elementType{TensorElementType::Unknown}; + std::vector shape; + }; + + struct FloatTensor + { + std::vector shape; + std::vector values; + }; +} diff --git a/ScopeOneInference/src/InferenceProcessingRegistration.cpp b/ScopeOneInference/src/InferenceProcessingRegistration.cpp new file mode 100644 index 0000000..3da7548 --- /dev/null +++ b/ScopeOneInference/src/InferenceProcessingRegistration.cpp @@ -0,0 +1,79 @@ +#include "OnnxInferenceModule.h" + +#include "scopeone/ScopeOneCore.h" +#include "scopeone/inference/InferenceExport.h" + +extern "C" SCOPEONE_INFERENCE_EXPORT void scopeone_register_processing_modules( + scopeone::core::ScopeOneCore* core) +{ + scopeone::core::ProcessingParameterDescriptor modelPath{ + QStringLiteral("model_path"), + QStringLiteral("Model"), + scopeone::core::ProcessingParameterType::FilePath, + QString{}}; + modelPath.fileFilter = QStringLiteral("ONNX models (*.onnx)"); + + core->registerProcessingModule( + {QStringLiteral("onnx_inference"), + QStringLiteral("ONNX Inference"), + 2, + {modelPath, + {QStringLiteral("provider"), + QStringLiteral("Provider"), + scopeone::core::ProcessingParameterType::Choice, + 0, + {}, + {}, + {}, + 0, + {{QStringLiteral("CPU"), 0}, {QStringLiteral("CUDA"), 1}}}, + {QStringLiteral("layout"), + QStringLiteral("Layout"), + scopeone::core::ProcessingParameterType::Choice, + 0, + {}, + {}, + {}, + 0, + {{QStringLiteral("Auto"), 0}, + {QStringLiteral("NCHW"), 1}, + {QStringLiteral("NHWC"), 2}}}, + {QStringLiteral("input_scale"), + QStringLiteral("Input scale"), + scopeone::core::ProcessingParameterType::Real, + 1.0, + 0.000001, + 1000000.0, + 0.01, + 6}, + {QStringLiteral("input_mean"), + QStringLiteral("Input mean"), + scopeone::core::ProcessingParameterType::Real, + 0.0, + -1000000.0, + 1000000.0, + 0.01, + 6}, + {QStringLiteral("input_std"), + QStringLiteral("Input std"), + scopeone::core::ProcessingParameterType::Real, + 1.0, + 0.000001, + 1000000.0, + 0.01, + 6}, + {QStringLiteral("output_mode"), + QStringLiteral("Output"), + scopeone::core::ProcessingParameterType::Choice, + 0, + {}, + {}, + {}, + 0, + {{QStringLiteral("Image"), 0}, + {QStringLiteral("Input - Output"), 1}}}}}, + []() + { + return std::make_unique(); + }); +} diff --git a/ScopeOneInference/src/OnnxInferenceModule.cpp b/ScopeOneInference/src/OnnxInferenceModule.cpp new file mode 100644 index 0000000..5d708e2 --- /dev/null +++ b/ScopeOneInference/src/OnnxInferenceModule.cpp @@ -0,0 +1,336 @@ +#include "OnnxInferenceModule.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace +{ + enum class TensorLayout + { + Nchw, + Nhwc + }; + + std::vector resizeImage(const std::vector& source, + int sourceWidth, + int sourceHeight, + int targetWidth, + int targetHeight) + { + if (sourceWidth == targetWidth && sourceHeight == targetHeight) + { + return source; + } + + std::vector target(static_cast(targetWidth) * targetHeight); + const float scaleX = static_cast(sourceWidth) / targetWidth; + const float scaleY = static_cast(sourceHeight) / targetHeight; + for (int y = 0; y < targetHeight; ++y) + { + const float sourceY = std::clamp((y + 0.5f) * scaleY - 0.5f, + 0.0f, + static_cast(sourceHeight - 1)); + const int y0 = static_cast(std::floor(sourceY)); + const int y1 = std::min(y0 + 1, sourceHeight - 1); + const float fy = sourceY - y0; + for (int x = 0; x < targetWidth; ++x) + { + const float sourceX = std::clamp((x + 0.5f) * scaleX - 0.5f, + 0.0f, + static_cast(sourceWidth - 1)); + const int x0 = static_cast(std::floor(sourceX)); + const int x1 = std::min(x0 + 1, sourceWidth - 1); + const float fx = sourceX - x0; + const float top = source[static_cast(y0) * sourceWidth + x0] + + fx * (source[static_cast(y0) * sourceWidth + x1] + - source[static_cast(y0) * sourceWidth + x0]); + const float bottom = source[static_cast(y1) * sourceWidth + x0] + + fx * (source[static_cast(y1) * sourceWidth + x1] + - source[static_cast(y1) * sourceWidth + x0]); + target[static_cast(y) * targetWidth + x] = top + fy * (bottom - top); + } + } + return target; + } + + TensorLayout resolveLayout(const scopeone::inference::TensorInfo& tensor, + int configuredLayout, + TensorLayout fallback) + { + if (configuredLayout == 1) + { + return TensorLayout::Nchw; + } + if (configuredLayout == 2) + { + return TensorLayout::Nhwc; + } + if (tensor.shape[1] == 1 && tensor.shape[3] != 1) + { + return TensorLayout::Nchw; + } + if (tensor.shape[3] == 1 && tensor.shape[1] != 1) + { + return TensorLayout::Nhwc; + } + return fallback; + } + + int tensorHeight(const scopeone::inference::TensorInfo& tensor, TensorLayout layout) + { + return static_cast(tensor.shape[layout == TensorLayout::Nchw ? 2 : 1]); + } + + int tensorWidth(const scopeone::inference::TensorInfo& tensor, TensorLayout layout) + { + return static_cast(tensor.shape[layout == TensorLayout::Nchw ? 3 : 2]); + } +} + +namespace scopeone::inference +{ + QString OnnxInferenceModule::id() const + { + return QStringLiteral("onnx_inference"); + } + + QString OnnxInferenceModule::name() const + { + return QStringLiteral("ONNX Inference"); + } + + QVariantMap OnnxInferenceModule::parameters() const + { + return {{QStringLiteral("model_path"), m_modelPath}, + {QStringLiteral("provider"), m_provider}, + {QStringLiteral("layout"), m_layout}, + {QStringLiteral("input_scale"), m_inputScale}, + {QStringLiteral("input_mean"), m_inputMean}, + {QStringLiteral("input_std"), m_inputStd}, + {QStringLiteral("output_mode"), m_outputMode}}; + } + + void OnnxInferenceModule::setParameters(const QVariantMap& parameters) + { + const QString modelPath = parameters.value(QStringLiteral("model_path"), + m_modelPath).toString(); + const int provider = parameters.value(QStringLiteral("provider"), m_provider).toInt(); + const int layout = parameters.value(QStringLiteral("layout"), m_layout).toInt(); + const double inputScale = parameters.value(QStringLiteral("input_scale"), + m_inputScale).toDouble(); + const double inputMean = parameters.value(QStringLiteral("input_mean"), + m_inputMean).toDouble(); + const double inputStd = parameters.value(QStringLiteral("input_std"), + m_inputStd).toDouble(); + const int outputMode = parameters.value(QStringLiteral("output_mode"), + m_outputMode).toInt(); + if (modelPath == m_modelPath && provider == m_provider && layout == m_layout + && inputScale == m_inputScale && inputMean == m_inputMean + && inputStd == m_inputStd && outputMode == m_outputMode) + { + return; + } + m_modelPath = modelPath; + m_provider = provider; + m_layout = layout; + m_inputScale = inputScale; + m_inputMean = inputMean; + m_inputStd = inputStd; + m_outputMode = outputMode; + m_session.reset(); + m_runtime.reset(); + } + + std::unique_ptr OnnxInferenceModule::createRuntime() const + { + auto module = std::make_unique(); + module->setParameters(parameters()); + return module; + } + + void OnnxInferenceModule::createSession() + { + m_runtime = std::make_unique(); + SessionOptions options; + options.provider = m_provider == 0 + ? ExecutionProvider::Cpu + : ExecutionProvider::Cuda; +#ifdef Q_OS_WIN + const std::filesystem::path path(m_modelPath.toStdWString()); +#else + const std::filesystem::path path(m_modelPath.toStdString()); +#endif + m_session = std::make_unique(*m_runtime, path, options); + + const auto& inputs = m_session->inputs(); + const auto& outputs = m_session->outputs(); + if (inputs.size() != 1) + { + throw std::runtime_error("ONNX image model must have exactly one input"); + } + if (outputs.size() != 1) + { + throw std::runtime_error("ONNX image model must have exactly one output"); + } + if (inputs.front().elementType != TensorElementType::Float32) + { + throw std::runtime_error("ONNX image model input must be float32"); + } + if (outputs.front().elementType != TensorElementType::Float32) + { + throw std::runtime_error("ONNX image model output must be float32"); + } + if (inputs.front().shape.size() != 4) + { + throw std::runtime_error("ONNX image model input must have four dimensions"); + } + if (outputs.front().shape.size() != 4) + { + throw std::runtime_error("ONNX image model output must have four dimensions"); + } + } + + core::ProcessingResult OnnxInferenceModule::process(const core::ImageFrame& frame, int) + { + if (m_modelPath.isEmpty()) + { + return {core::ImageFrame{}, QStringLiteral("Select an ONNX model")}; + } + + try + { + if (!m_session) + { + createSession(); + } + + const auto& inputInfo = m_session->inputs().front(); + const TensorLayout inputLayout = resolveLayout(inputInfo, + m_layout, + TensorLayout::Nchw); + const int channelIndex = inputLayout == TensorLayout::Nchw ? 1 : 3; + if (inputInfo.shape[0] > 0 && inputInfo.shape[0] != 1) + { + throw std::runtime_error("ONNX image model batch size must be one or dynamic"); + } + if (inputInfo.shape[channelIndex] > 0 && inputInfo.shape[channelIndex] != 1) + { + throw std::runtime_error("ONNX image model input must be single-channel"); + } + + std::vector source(static_cast(frame.width) * frame.height); + const float frameScale = 1.0f / static_cast(frame.maxValue()); + for (int y = 0; y < frame.height; ++y) + { + float* destination = source.data() + static_cast(y) * frame.width; + if (frame.isMono16()) + { + const auto* source = reinterpret_cast( + frame.bytes.constData() + static_cast(y) * frame.stride); + for (int x = 0; x < frame.width; ++x) + { + destination[x] = static_cast(source[x]) * frameScale; + } + } + else + { + const auto* source = reinterpret_cast( + frame.bytes.constData() + static_cast(y) * frame.stride); + for (int x = 0; x < frame.width; ++x) + { + destination[x] = static_cast(source[x]) * frameScale; + } + } + } + + const int modelHeight = tensorHeight(inputInfo, inputLayout) > 0 + ? tensorHeight(inputInfo, inputLayout) + : frame.height; + const int modelWidth = tensorWidth(inputInfo, inputLayout) > 0 + ? tensorWidth(inputInfo, inputLayout) + : frame.width; + std::vector modelInput = resizeImage(source, + frame.width, + frame.height, + modelWidth, + modelHeight); + for (float& value : modelInput) + { + value = static_cast((value * m_inputScale - m_inputMean) / m_inputStd); + } + + FloatTensor input; + input.shape = inputLayout == TensorLayout::Nchw + ? std::vector{1, 1, modelHeight, modelWidth} + : std::vector{1, modelHeight, modelWidth, 1}; + input.values = modelInput; + std::vector outputs = m_session->run({std::move(input)}); + FloatTensor& output = outputs.front(); + const TensorLayout outputLayout = resolveLayout(m_session->outputs().front(), + m_layout, + inputLayout); + const int outputChannelIndex = outputLayout == TensorLayout::Nchw ? 1 : 3; + if (output.shape[0] != 1 || output.shape[outputChannelIndex] != 1) + { + throw std::runtime_error("ONNX image model output must be single-channel"); + } + + const int outputHeight = static_cast( + output.shape[outputLayout == TensorLayout::Nchw ? 2 : 1]); + const int outputWidth = static_cast( + output.shape[outputLayout == TensorLayout::Nchw ? 3 : 2]); + if (output.values.size() + != static_cast(outputWidth) * outputHeight) + { + throw std::runtime_error("ONNX output tensor size is invalid"); + } + + if (m_outputMode == 1) + { + if (output.values.size() != modelInput.size()) + { + throw std::runtime_error("Residual output size must match the model input"); + } + for (std::size_t index = 0; index < output.values.size(); ++index) + { + output.values[index] = modelInput[index] - output.values[index]; + } + } + for (float& value : output.values) + { + value = static_cast((value * m_inputStd + m_inputMean) / m_inputScale); + } + + std::vector displayOutput = resizeImage(output.values, + outputWidth, + outputHeight, + frame.width, + frame.height); + QByteArray bytes(static_cast(displayOutput.size() * sizeof(quint16)), + Qt::Uninitialized); + auto* destination = reinterpret_cast(bytes.data()); + for (std::size_t index = 0; index < displayOutput.size(); ++index) + { + destination[index] = static_cast(std::lround( + std::clamp(displayOutput[index], 0.0f, 1.0f) * 65535.0f)); + } + + core::ImageFrame result = frame; + result.stride = frame.width * static_cast(sizeof(quint16)); + result.bitsPerSample = 16; + result.pixelFormat = core::ImagePixelFormat::Mono16; + result.bytes = std::move(bytes); + return {std::move(result), {}}; + } + catch (const std::exception& error) + { + return {core::ImageFrame{}, QString::fromUtf8(error.what())}; + } + } +} diff --git a/ScopeOneInference/src/OnnxInferenceModule.h b/ScopeOneInference/src/OnnxInferenceModule.h new file mode 100644 index 0000000..e83ca94 --- /dev/null +++ b/ScopeOneInference/src/OnnxInferenceModule.h @@ -0,0 +1,35 @@ +#pragma once + +#include "scopeone/ProcessingPlugin.h" +#include "scopeone/inference/OnnxRuntime.h" +#include "scopeone/inference/OnnxSession.h" + +#include + +namespace scopeone::inference +{ + class OnnxInferenceModule final : public core::ProcessingModule + { + public: + QString id() const override; + QString name() const override; + QVariantMap parameters() const override; + void setParameters(const QVariantMap& parameters) override; + std::unique_ptr createRuntime() const override; + core::ProcessingResult process(const core::ImageFrame& frame, + int processingBitDepth) override; + + private: + void createSession(); + + QString m_modelPath; + int m_provider{0}; + int m_layout{0}; + double m_inputScale{1.0}; + double m_inputMean{0.0}; + double m_inputStd{1.0}; + int m_outputMode{0}; + std::unique_ptr m_runtime; + std::unique_ptr m_session; + }; +} diff --git a/ScopeOneInference/src/OnnxInternal.h b/ScopeOneInference/src/OnnxInternal.h new file mode 100644 index 0000000..9c219e3 --- /dev/null +++ b/ScopeOneInference/src/OnnxInternal.h @@ -0,0 +1,16 @@ +#pragma once + +#include "scopeone/inference/OnnxRuntime.h" +#include "scopeone/inference/OnnxSession.h" + +#include + +namespace scopeone::inference +{ + struct OnnxRuntime::Impl + { + Impl(); + + Ort::Env environment; + }; +} diff --git a/ScopeOneInference/src/OnnxRuntime.cpp b/ScopeOneInference/src/OnnxRuntime.cpp new file mode 100644 index 0000000..1127195 --- /dev/null +++ b/ScopeOneInference/src/OnnxRuntime.cpp @@ -0,0 +1,42 @@ +#include "OnnxInternal.h" + +#include +#include + +namespace scopeone::inference +{ + OnnxRuntime::Impl::Impl() + : environment(ORT_LOGGING_LEVEL_WARNING, "ScopeOneInference") + { + } + + OnnxRuntime::OnnxRuntime() + : m_impl(std::make_shared()) + { + } + + OnnxRuntime::~OnnxRuntime() = default; + OnnxRuntime::OnnxRuntime(OnnxRuntime&&) noexcept = default; + OnnxRuntime& OnnxRuntime::operator=(OnnxRuntime&&) noexcept = default; + + std::string OnnxRuntime::version() const + { + return Ort::GetVersionString(); + } + + std::vector OnnxRuntime::availableProviders() const + { + return Ort::GetAvailableProviders(); + } + + bool OnnxRuntime::supports(ExecutionProvider provider) const + { + if (provider == ExecutionProvider::Cpu) + { + return true; + } + const auto providers = availableProviders(); + return std::find(providers.begin(), providers.end(), "CUDAExecutionProvider") + != providers.end(); + } +} diff --git a/ScopeOneInference/src/OnnxSession.cpp b/ScopeOneInference/src/OnnxSession.cpp new file mode 100644 index 0000000..8ef4c9b --- /dev/null +++ b/ScopeOneInference/src/OnnxSession.cpp @@ -0,0 +1,184 @@ +#include "OnnxInternal.h" + +#include +#include +#include +#include + +namespace +{ + scopeone::inference::TensorElementType tensorElementType( + ONNXTensorElementDataType type) + { + using scopeone::inference::TensorElementType; + switch (type) + { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return TensorElementType::Float32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return TensorElementType::UInt8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return TensorElementType::Int8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return TensorElementType::UInt16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return TensorElementType::Int16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return TensorElementType::Int32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return TensorElementType::Int64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: return TensorElementType::String; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return TensorElementType::Boolean; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return TensorElementType::Float16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return TensorElementType::Float64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return TensorElementType::UInt32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return TensorElementType::UInt64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: return TensorElementType::Complex64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: return TensorElementType::Complex128; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return TensorElementType::BFloat16; + default: return TensorElementType::Unknown; + } + } + + std::size_t elementCount(const std::vector& shape) + { + return std::accumulate(shape.begin(), shape.end(), std::size_t{1}, + [](std::size_t count, std::int64_t dimension) + { + return count * static_cast(dimension); + }); + } + + std::vector tensorMetadata( + Ort::Session& session, + bool input) + { + const std::size_t count = input ? session.GetInputCount() : session.GetOutputCount(); + Ort::AllocatorWithDefaultOptions allocator; + std::vector metadata; + metadata.reserve(count); + for (std::size_t index = 0; index < count; ++index) + { + const auto name = input + ? session.GetInputNameAllocated(index, allocator) + : session.GetOutputNameAllocated(index, allocator); + const auto typeInfo = input + ? session.GetInputTypeInfo(index) + : session.GetOutputTypeInfo(index); + const auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); + metadata.push_back({name.get(), + tensorElementType(tensorInfo.GetElementType()), + tensorInfo.GetShape()}); + } + return metadata; + } +} + +namespace scopeone::inference +{ + struct OnnxSession::Impl + { + std::shared_ptr runtime; + Ort::Session session{nullptr}; + std::vector inputMetadata; + std::vector outputMetadata; + }; + + OnnxSession::OnnxSession(OnnxRuntime& runtime, + const std::filesystem::path& modelPath, + const SessionOptions& options) + : m_impl(std::make_unique()) + { + const std::shared_ptr runtimeImpl = runtime.m_impl; + m_impl->runtime = runtimeImpl; + Ort::SessionOptions sessionOptions; + sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + if (options.provider == ExecutionProvider::Cuda) + { + if (!runtime.supports(ExecutionProvider::Cuda)) + { + throw std::runtime_error("CUDAExecutionProvider is not available"); + } + OrtCUDAProviderOptions cudaOptions{}; + cudaOptions.device_id = options.deviceId; + sessionOptions.AppendExecutionProvider_CUDA(cudaOptions); + } + + m_impl->session = Ort::Session(runtimeImpl->environment, + modelPath.c_str(), + sessionOptions); + m_impl->inputMetadata = tensorMetadata(m_impl->session, true); + m_impl->outputMetadata = tensorMetadata(m_impl->session, false); + } + + OnnxSession::~OnnxSession() = default; + OnnxSession::OnnxSession(OnnxSession&&) noexcept = default; + OnnxSession& OnnxSession::operator=(OnnxSession&&) noexcept = default; + + const std::vector& OnnxSession::inputs() const + { + return m_impl->inputMetadata; + } + + const std::vector& OnnxSession::outputs() const + { + return m_impl->outputMetadata; + } + + std::vector OnnxSession::run(const std::vector& inputs) + { + if (inputs.size() != m_impl->inputMetadata.size()) + { + throw std::runtime_error("Input tensor count does not match the model"); + } + + Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu( + OrtArenaAllocator, OrtMemTypeDefault); + std::vector inputValues; + std::vector inputNames; + inputValues.reserve(inputs.size()); + inputNames.reserve(inputs.size()); + for (std::size_t index = 0; index < inputs.size(); ++index) + { + const FloatTensor& input = inputs[index]; + if (m_impl->inputMetadata[index].elementType != TensorElementType::Float32 + || elementCount(input.shape) != input.values.size()) + { + throw std::runtime_error("Only valid float32 input tensors are supported"); + } + inputValues.push_back(Ort::Value::CreateTensor( + memoryInfo, + const_cast(input.values.data()), + input.values.size(), + input.shape.data(), + input.shape.size())); + inputNames.push_back(m_impl->inputMetadata[index].name.c_str()); + } + + std::vector outputNames; + outputNames.reserve(m_impl->outputMetadata.size()); + for (const TensorInfo& output : m_impl->outputMetadata) + { + outputNames.push_back(output.name.c_str()); + } + + std::vector outputValues = m_impl->session.Run( + Ort::RunOptions{nullptr}, + inputNames.data(), + inputValues.data(), + inputValues.size(), + outputNames.data(), + outputNames.size()); + + std::vector outputs; + outputs.reserve(outputValues.size()); + for (Ort::Value& value : outputValues) + { + const auto info = value.GetTensorTypeAndShapeInfo(); + if (info.GetElementType() != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) + { + throw std::runtime_error("Only float32 output tensors are supported"); + } + FloatTensor output; + output.shape = info.GetShape(); + const std::size_t count = info.GetElementCount(); + const float* data = value.GetTensorData(); + output.values.assign(data, data + count); + outputs.push_back(std::move(output)); + } + return outputs; + } +} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 60f9e7e..3d8de74 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -23,6 +23,7 @@ endfunction() add_subdirectory(hardware) add_subdirectory(tools) +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../ScopeOneInference ScopeOneInference) if (SCOPEONE_CUDA_NVCC_EXECUTABLE) set(CMAKE_CUDA_COMPILER "${SCOPEONE_CUDA_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler selected by the environment" FORCE) diff --git a/resources/ONNX.png b/resources/ONNX.png new file mode 100644 index 0000000..dfc9aa8 Binary files /dev/null and b/resources/ONNX.png differ diff --git a/src/DeviceControlWidget.cpp b/src/DeviceControlWidget.cpp index 3cfc7c2..7eea85f 100644 --- a/src/DeviceControlWidget.cpp +++ b/src/DeviceControlWidget.cpp @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -27,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1320,11 +1323,17 @@ namespace scopeone::ui // Open image files as workspace documents void DeviceControlWidget::onOpenImageClicked() { + QSettings settings(QStringLiteral("ScopeOne"), QStringLiteral("ScopeOne")); const QStringList filePaths = QFileDialog::getOpenFileNames( this, tr("Open Image"), - QString(), + settings.value(QStringLiteral("LastImageDirectory"), QDir::homePath()).toString(), tr("Images (*.tif *.tiff *.png *.jpg *.jpeg *.bmp)")); + if (!filePaths.isEmpty()) + { + settings.setValue(QStringLiteral("LastImageDirectory"), + QFileInfo(filePaths.first()).absolutePath()); + } for (const QString& filePath : filePaths) { m_scopeonecore->openImage(filePath); diff --git a/src/ImageProcessingWidget.cpp b/src/ImageProcessingWidget.cpp index 8ba657b..6d3eed9 100644 --- a/src/ImageProcessingWidget.cpp +++ b/src/ImageProcessingWidget.cpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -16,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -218,6 +223,62 @@ namespace scopeone::ui spinBox->setCorrectionMode(QAbstractSpinBox::CorrectToNearestValue); } + class FilePathEditor final : public QWidget + { + public: + FilePathEditor(const QString& path, + const QString& fileFilter, + std::function changed, + QWidget* parent) + : QWidget(parent), m_fileFilter(fileFilter), m_changed(std::move(changed)) + { + auto* layout = new QHBoxLayout(this); + layout->setContentsMargins(0, 0, 0, 0); + m_pathEdit = new QLineEdit(path, this); + auto* browseButton = new QPushButton(tr("Browse"), this); + setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed); + setMaximumWidth(300); + browseButton->setFixedWidth(68); + layout->addWidget(m_pathEdit, 1); + layout->addWidget(browseButton); + connect(m_pathEdit, &QLineEdit::editingFinished, this, [this]() + { + m_changed(); + }); + connect(browseButton, &QPushButton::clicked, this, [this]() + { + QSettings settings(QStringLiteral("ScopeOne"), QStringLiteral("ScopeOne")); + const QString initialPath = m_pathEdit->text().isEmpty() + ? settings.value(QStringLiteral("LastOnnxDirectory"), QDir::homePath()).toString() + : m_pathEdit->text(); + const QString path = QFileDialog::getOpenFileName( + this, tr("Select File"), initialPath, m_fileFilter); + if (!path.isEmpty()) + { + settings.setValue(QStringLiteral("LastOnnxDirectory"), + QFileInfo(path).absolutePath()); + m_pathEdit->setText(path); + m_changed(); + } + }); + } + + QString path() const + { + return m_pathEdit->text(); + } + + void setPath(const QString& path) + { + m_pathEdit->setText(path); + } + + private: + QLineEdit* m_pathEdit; + QString m_fileFilter; + std::function m_changed; + }; + class ModuleConfigWidget final : public QWidget { public: @@ -290,6 +351,9 @@ namespace scopeone::ui combo->setCurrentIndex(combo->findData(value)); break; } + case ProcessingParameterType::FilePath: + static_cast(editor)->setPath(value.toString()); + break; } } if (m_maskPreview) @@ -347,6 +411,11 @@ namespace scopeone::ui this, [this]() { apply(); }); return editor; } + case ProcessingParameterType::FilePath: + return new FilePathEditor(value.toString(), + descriptor.fileFilter, + [this]() { apply(); }, + parent); } return new QWidget(parent); } @@ -364,6 +433,8 @@ namespace scopeone::ui return qobject_cast(editor)->isChecked(); case ProcessingParameterType::Choice: return qobject_cast(editor)->currentData(); + case ProcessingParameterType::FilePath: + return static_cast(editor)->path(); } return {}; } diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index a8507d4..7f1cd09 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -1838,11 +1838,17 @@ namespace scopeone::ui // Open image files as workspace documents void MainWindow::openImageDialog() { + QSettings settings(QStringLiteral("ScopeOne"), QStringLiteral("ScopeOne")); const QStringList filePaths = QFileDialog::getOpenFileNames( this, tr("Open Image"), - QString(), + settings.value(QStringLiteral("LastImageDirectory"), QDir::homePath()).toString(), tr("Images (*.tif *.tiff *.png *.jpg *.jpeg *.bmp)")); + if (!filePaths.isEmpty()) + { + settings.setValue(QStringLiteral("LastImageDirectory"), + QFileInfo(filePaths.first()).absolutePath()); + } openImages(filePaths); }