diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ff7c1671..222046dd 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -109,7 +109,7 @@ flowchart TD START(["๐Ÿš€ ็จ‹ๅบๅฏๅŠจ"]) --> PARSE["่งฃๆžๅ‘ฝไปค่กŒๅ‚ๆ•ฐ
--epochs, --batch-size,
--lr, --optimizer"] PARSE --> LOAD_DATA["ๅŠ ่ฝฝ CSV ๆ•ฐๆฎ
MNIST 784 ็ปดๅƒ็ด "] LOAD_DATA --> SPLIT["ๅˆ†ๅ‰ฒ่ฎญ็ปƒ้›†/ๆต‹่ฏ•้›†"] - SPLIT --> BUILD["ๆž„ๅปบ็ฝ‘็ปœๆจกๅž‹
Linear(784โ†’64)โ†’ReLU
ร— 3 ๅฑ‚
โ†’ Linear(64โ†’10)"] + SPLIT --> BUILD["ๆž„ๅปบ็ฝ‘็ปœๆจกๅž‹
Linear(784โ†’512)โ†’LayerNormโ†’GeLU
โ†’ Linear(512โ†’256)โ†’LayerNormโ†’GeLU
โ†’ Linear(256โ†’128)โ†’LayerNormโ†’GeLU
โ†’ Linear(128โ†’64)โ†’LayerNormโ†’GeLU
โ†’ Linear(64โ†’10)"] BUILD --> LOAD_PREV{"ๆœ‰้ข„่ฎญ็ปƒๆจกๅž‹?"} LOAD_PREV -->|"ๆ˜ฏ"| LOAD_M["ๅŠ ่ฝฝ้ข„่ฎญ็ปƒๆƒ้‡"] LOAD_PREV -->|"ๅฆ"| INIT["Xavier ๅˆๅง‹ๅŒ–ๆƒ้‡"] @@ -117,7 +117,7 @@ flowchart TD INIT --> TRAIN_LOOP subgraph TRAIN_LOOP["๐Ÿ” ่ฎญ็ปƒๅพช็Žฏ (per epoch)"] - BATCH["ๅ–ไธ€ไธช batch
(้ป˜่ฎค 32 ๆ ทๆœฌ)"] + BATCH["ๅ–ไธ€ไธช batch
(้ป˜่ฎค 64 ๆ ทๆœฌ)"] BATCH --> FORWARD["โšก ๅ‰ๅ‘ไผ ๆ’ญ
model.forward(X)"] FORWARD --> LOSS["๐Ÿ“Š ่ฎก็ฎ—ๆŸๅคฑ
CrossEntropyLoss"] LOSS --> BACKWARD["๐Ÿ“‰ ๅๅ‘ไผ ๆ’ญ
model.backward(grad)"] @@ -217,8 +217,37 @@ classDiagram +backward(grad_output) Matrix } + class GeLU { + -Matrix input_cache_ + -Matrix sigmoid_cache_ + -constexpr double BETA = 1.702 + +forward(input) Matrix + +backward(grad_output) Matrix + } + + class LayerNorm { + -size_t normalized_shape_ + -double epsilon_ + -Matrix gamma_ + -Matrix beta_ + -Matrix grad_gamma_ + -Matrix grad_beta_ + +forward(input) Matrix + +backward(grad_output) Matrix + } + + class PositionalEncoding { + -size_t max_len_ + -size_t d_model_ + -Matrix pe_ + +forward(input) Matrix + } + Layer <|-- Linear Layer <|-- ReLU + Layer <|-- GeLU + Layer <|-- LayerNorm + Layer <|-- PositionalEncoding ``` ### Model โ€” ็ฝ‘็ปœๅฎนๅ™จ @@ -311,13 +340,15 @@ flowchart LR end ``` -ๅ‚ๆ•ฐๅ†™ๅ…ฅ้กบๅบ๏ผˆไปฅ 3 ๅฑ‚้š่—ๅฑ‚ไธบไพ‹๏ผ‰๏ผš -1. Wโ‚ (64ร—784), bโ‚ (64ร—1) -2. Wโ‚‚ (64ร—64), bโ‚‚ (64ร—1) -3. Wโ‚ƒ (64ร—64), bโ‚ƒ (64ร—1) -4. Wโ‚„ (10ร—64), bโ‚„ (10ร—1) +ๅ‚ๆ•ฐๅ†™ๅ…ฅ้กบๅบ๏ผˆ้ป˜่ฎค 4 ๅฑ‚้š่—ๅฑ‚ๆžถๆž„๏ผ‰๏ผš +1. Wโ‚ (512ร—784), bโ‚ (512ร—1) +2. Wโ‚‚ (256ร—512), bโ‚‚ (256ร—1) +3. Wโ‚ƒ (128ร—256), bโ‚ƒ (128ร—1) +4. Wโ‚„ (64ร—128), bโ‚„ (64ร—1) +5. Wโ‚… (10ร—64), bโ‚… (10ร—1) +6. ฮณโ‚~ฮณโ‚„, ฮฒโ‚~ฮฒโ‚„ (LayerNorm ๅ‚ๆ•ฐ) -ๅ…ฑ **8 ไธช็Ÿฉ้˜ตๅ‚ๆ•ฐ**ใ€‚ +ๅ…ฑ **10 ไธช Linear ๅ‚ๆ•ฐ็Ÿฉ้˜ต + 8 ไธช LayerNorm ๅ‚ๆ•ฐ็Ÿฉ้˜ต**ใ€‚ --- @@ -350,12 +381,9 @@ flowchart TD ```mermaid flowchart TD - CMAKE["CMakeLists.txt"] --> OPT1["benchmark_naive
ๆœด็ด ็Ÿฉ้˜ตไน˜ๆณ•ๅŸบๅ‡†"] - CMAKE --> OPT2["benchmark_b32
32 ็บฟ็จ‹็Ÿฉ้˜ตไน˜ๆณ•ๅŸบๅ‡†"] - CMAKE --> OPT3["benchmark_b64
64 ็บฟ็จ‹็Ÿฉ้˜ตไน˜ๆณ•ๅŸบๅ‡†"] - CMAKE --> TARGET1["mnist_train
่ฎญ็ปƒๅฏๆ‰ง่กŒๆ–‡ไปถ"] + CMAKE["CMakeLists.txt
C++26, Clang++"] --> TARGET1["mnist_train
่ฎญ็ปƒๅฏๆ‰ง่กŒๆ–‡ไปถ"] CMAKE --> TARGET2["mnist_infer
ๆŽจ็†ๅฏๆ‰ง่กŒๆ–‡ไปถ"] - CMAKE --> DEPS["ไพ่ต–: OpenMP (ๅฏ้€‰)
C++26 ๆ ‡ๅ‡†"] + CMAKE --> DEPS["็ผ–่ฏ‘้€‰้กน: -O3 -ffast-math
-fexperimental-library
-Wall -Wextra -Wpedantic -Werror"] ``` --- @@ -388,23 +416,29 @@ flowchart LR ## ๐ŸŽฏ ็ฝ‘็ปœ็ป“ๆž„ๆ€ป็ป“ +้ป˜่ฎค MNIST ๆžถๆž„๏ผˆๅฎšไน‰ๅœจ `mnist_common.hpp` ็š„ `MNIST_LAYER_DIMS`๏ผ‰๏ผš + ``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ MNIST ๆ‰‹ๅ†™ๆ•ฐๅญ—่ฏ†ๅˆซ็ฝ‘็ปœ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Input: 784 (28ร—28 ๅƒ็ด ๅฑ•ๅนณ) โ”‚ -โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ -โ”‚ Layer 1: Linear(784 โ†’ 64) + ReLU โ”‚ -โ”‚ Layer 2: Linear(64 โ†’ 64) + ReLU โ”‚ -โ”‚ Layer 3: Linear(64 โ†’ 64) + ReLU โ”‚ -โ”‚ Layer 4: Linear(64 โ†’ 10) โ”‚ -โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ -โ”‚ Loss: CrossEntropy (ๅซ Softmax) โ”‚ -โ”‚ Optimizer: Adam / SGD / SGD+Momentum โ”‚ -โ”‚ ่พ“ๅ‡บ: 10 ็ปดๅ‘้‡ โ†’ argmax โ†’ ๆ•ฐๅญ— 0-9 โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ MNIST ๆ‰‹ๅ†™ๆ•ฐๅญ—่ฏ†ๅˆซ็ฝ‘็ปœ โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ Input: 784 (28ร—28 ๅƒ็ด ๅฑ•ๅนณ) โ”‚ +โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ +โ”‚ Layer 1: Linear(784 โ†’ 512) + LayerNorm + GeLU โ”‚ +โ”‚ Layer 2: Linear(512 โ†’ 256) + LayerNorm + GeLU โ”‚ +โ”‚ Layer 3: Linear(256 โ†’ 128) + LayerNorm + GeLU โ”‚ +โ”‚ Layer 4: Linear(128 โ†’ 64) + LayerNorm + GeLU โ”‚ +โ”‚ Layer 5: Linear(64 โ†’ 10) โ”‚ +โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ +โ”‚ Loss: CrossEntropy (ๅซๆ•ฐๅ€ผ็จณๅฎš Softmax) โ”‚ +โ”‚ Optimizer: Adam / SGD / SGD+Momentum โ”‚ +โ”‚ ่พ“ๅ‡บ: 10 ็ปดๅ‘้‡ โ†’ argmax โ†’ ๆ•ฐๅญ— 0-9 โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ ``` +> ๐Ÿ“ ็ฝ‘็ปœๆžถๆž„็”ฑ `MNIST_LAYER_DIMS = {784, 512, 256, 128, 64, 10}` ้ฉฑๅŠจ๏ผŒ +> ่‡ชๅŠจๅœจ้š่—ๅฑ‚ไน‹้—ดๆ’ๅ…ฅ LayerNorm + GeLU ๆฟ€ๆดปใ€‚ + --- *Generated by GitHub Copilot โ€” MiMo V2.5* diff --git a/DEVELOPMENT_STANDARDS.md b/DEVELOPMENT_STANDARDS.md index 6f78b4fe..88f0205a 100644 --- a/DEVELOPMENT_STANDARDS.md +++ b/DEVELOPMENT_STANDARDS.md @@ -100,7 +100,22 @@ void process(std::span data); void process(const double* data, std::size_t size); ``` -#### 2.4 ้ข„ๅˆ†้…็ผ“ๅ†ฒๅŒบๆจกๅผ +#### 2.4 ไฝฟ็”จ std::span ่ฎฟ้—ฎ็Ÿฉ้˜ตๆ•ฐๆฎ +```cpp +// โœ… ๆŽจ่๏ผšไฝฟ็”จ span() ่ฎฟ้—ฎ็Ÿฉ้˜ตๆ•ฐๆฎ๏ผˆC++20 ้›ถๅผ€้”€ๆŠฝ่ฑก๏ผ‰ +auto data = matrix.span(); // std::span +auto cdata = const_matrix.span(); // std::span + +// ๅœจ lambda ไธญๆ•่Žท span๏ผˆ็ฑปๅž‹ๅฎ‰ๅ…จ๏ผŒ็ผ–่ฏ‘ๆœŸๅคงๅฐๆฃ€ๆŸฅ๏ผ‰ +auto func = [data](std::size_t i) noexcept { + return data[i] * 2.0; +}; + +// โŒ ้ฟๅ…๏ผšไฝฟ็”จๅทฒๅบŸๅผƒ็š„ data_ptr() +double* ptr = matrix.data_ptr(); // [[deprecated]] +``` + +#### 2.5 ้ข„ๅˆ†้…็ผ“ๅ†ฒๅŒบๆจกๅผ ```cpp // โœ… ๆŽจ่๏ผš้ข„ๅˆ†้…ๅนถๅœจๅฐฑๅœฐๆ“ไฝœ void multiply_to(Matrix& result, const Matrix& other) const { @@ -123,7 +138,7 @@ void multiply_to(Matrix& result, const Matrix& other) const { - [ ] ๆ˜ฏๅฆๆœ‰ `new` / `delete` ๆ“ไฝœ๏ผŸ - [ ] ๆ˜ฏๅฆไฝฟ็”จ `std::vector` ๆ›ฟไปฃๅŠจๆ€ๆ•ฐ็ป„๏ผŸ - [ ] ๆ˜ฏๅฆไฝฟ็”จ `std::unique_ptr` ็ฎก็†ๅคšๆ€ๅฏน่ฑก๏ผŸ -- [ ] ๆ˜ฏๅฆไฝฟ็”จ `std::span` ไผ ้€’ๅช่ฏป่ฟž็ปญๆ•ฐๆฎ๏ผŸ +- [ ] ๆ˜ฏๅฆไฝฟ็”จ `std::span` ๆ›ฟไปฃ่ฃธๆŒ‡้’ˆ่ฎฟ้—ฎ่ฟž็ปญๆ•ฐๆฎ๏ผŸ - [ ] ็ƒญ่ทฏๅพ„ไธญๆ˜ฏๅฆ้ข„ๅˆ†้…็ผ“ๅ†ฒๅŒบ๏ผŸ --- @@ -378,32 +393,17 @@ set(CMAKE_CXX_STANDARD 26) set(CMAKE_CXX_STANDARD_REQUIRED ON) ``` -### 2. ๅฟ…้กปไฝฟ็”จ็š„ C++26 ็‰นๆ€ง - -| ็‰นๆ€ง | ็”จ้€” | ็คบไพ‹ | -|------|------|------| -| `std::generator` | ๅ็จ‹็”Ÿๆˆๅ™จ | ๆ•ฐๆฎๅŠ ่ฝฝๅ™จ | -| `std::print` / `std::println` | ๆ ผๅผๅŒ–่พ“ๅ‡บ | ่ฐƒ่ฏ•ไฟกๆฏ | -| `std::expected` | ้”™่ฏฏๅค„็† | ่ฟ”ๅ›žๅ€ผ็ฑปๅž‹ | -| `std::flat_map` | ้ซ˜ๆ€ง่ƒฝๆ˜ ๅฐ„ | ้…็ฝฎ็ฎก็† | -| `std::mdspan` | ๅคš็ปดๆ•ฐ็ป„่ง†ๅ›พ | ็Ÿฉ้˜ตๆ“ไฝœ | -| `std::stacktrace` | ่ฐƒ่ฏ•ๅ †ๆ ˆ่ทŸ่ธช | ้”™่ฏฏๆŠฅๅ‘Š | - -### 3. ็งฏๆžไฝฟ็”จ็š„ C++23/20 ็‰นๆ€ง +### 2. ็งฏๆžไฝฟ็”จ็š„ C++20/23 ็‰นๆ€ง | ็‰นๆ€ง | ๆ ‡ๅ‡† | ็”จ้€” | |------|------|------| -| `std::ranges` | C++20 | ๆ•ฐๆฎๅค„็†็ฎก้“ | -| `std::views` | C++20 | ๅปถ่ฟŸๆฑ‚ๅ€ผ | -| `std::format` | C++20 | ๅญ—็ฌฆไธฒๆ ผๅผๅŒ– | -| `std::span` | C++20 | ้žๆ‹ฅๆœ‰่ง†ๅ›พ | -| `std::jthread` | C++20 | ่‡ชๅŠจ่ฟžๆŽฅ็บฟ็จ‹ | -| `std::source_location` | C++20 | ้”™่ฏฏๅฎšไฝ | -| `std::print` | C++23 | ๆ ผๅผๅŒ–่พ“ๅ‡บ | -| `std::generator` | C++23 | ๅ็จ‹็”Ÿๆˆๅ™จ | -| `std::mdspan` | C++23 | ๅคš็ปดๆ•ฐ็ป„ | - -### 4. ๆ ‡ๅ‡†่ทŸ่ฟ›ๆฃ€ๆŸฅๆธ…ๅ• +| `std::span` | C++20 | ้žๆ‹ฅๆœ‰ๅž‹่ฟž็ปญๅ†…ๅญ˜่ง†ๅ›พ๏ผŒๆ›ฟไปฃ่ฃธๆŒ‡้’ˆ | +| `std::ranges` | C++20 | ๆ•ฐๆฎๅค„็†็ฎก้“๏ผˆๅฆ‚ `ranges::generate` Xavier ๅˆๅง‹ๅŒ–๏ผ‰ | +| `std::views::iota` | C++20 | ๅปถ่ฟŸๆ•ดๆ•ฐๅบๅˆ—๏ผˆๅนถ่กŒ็ดขๅผ•็”Ÿๆˆ๏ผ‰ | +| `std::views::zip` | C++20 | ๅคš่Œƒๅ›ดๅนถ่กŒ่ฟญไปฃ๏ผˆไผ˜ๅŒ–ๅ™จๅ‚ๆ•ฐๆ›ดๆ–ฐ๏ผ‰ | +| `std::execution::par_unseq` | C++17 | ๅนถ่กŒๆ‰ง่กŒ็ญ–็•ฅ๏ผˆ็Ÿฉ้˜ต่ฟ็ฎ—๏ผ‰ | + +### 3. ๆ ‡ๅ‡†่ทŸ่ฟ›ๆฃ€ๆŸฅๆธ…ๅ• - [ ] ๆ˜ฏๅฆไฝฟ็”จๆœ€ๆ–ฐ็š„ C++ ๆ ‡ๅ‡†๏ผˆC++26๏ผ‰๏ผŸ - [ ] ๆ˜ฏๅฆ็งฏๆžไฝฟ็”จ C++20/23/26 ๆ–ฐ็‰นๆ€ง๏ผŸ diff --git a/README.md b/README.md index c7f18bc0..c7f54ca0 100644 --- a/README.md +++ b/README.md @@ -8,28 +8,40 @@ neuralnet.cpp/ โ”œโ”€โ”€ .gitignore โ”œโ”€โ”€ CMakeLists.txt +โ”œโ”€โ”€ ARCHITECTURE.md โ† ๆžถๆž„ๆ–‡ๆกฃ +โ”œโ”€โ”€ DEVELOPMENT_STANDARDS.md โ† ๅผ€ๅ‘่ง„่Œƒ โ”œโ”€โ”€ README.md -โ”œโ”€โ”€ gui.py โ† ๅ›พๅฝขๅŒ–ๆ“ไฝœ็•Œ้ข -โ”œโ”€โ”€ csv_png.py -โ”œโ”€โ”€ extract_digits.py -โ”œโ”€โ”€ save_dataset.py -โ”œโ”€โ”€ build/ -โ”‚ โ”œโ”€โ”€ mnist_infer.exe +โ”œโ”€โ”€ gui.py โ† ๅ›พๅฝขๅŒ–ๆ“ไฝœ็•Œ้ข (Tkinter) +โ”œโ”€โ”€ csv_png.py โ† CSV ่ฝฌ PNG ๅ›พๅƒ +โ”œโ”€โ”€ extract_digits.py โ† ไปŽ MNIST CSV ๆๅ–ๅ•ไธชๆ•ฐๅญ— +โ”œโ”€โ”€ save_dataset.py โ† ไธ‹่ฝฝ MNIST ๆ•ฐๆฎ้›† +โ”œโ”€โ”€ build/ โ† CMake ๆž„ๅปบ่พ“ๅ‡บ โ”‚ โ”œโ”€โ”€ mnist_train.exe -โ”‚ โ””โ”€โ”€ mnist_data/ -โ”‚ โ”œโ”€โ”€ test.csv -โ”‚ โ””โ”€โ”€ train.csv +โ”‚ โ””โ”€โ”€ mnist_infer.exe โ”œโ”€โ”€ include/ โ”‚ โ””โ”€โ”€ neuralnet.cpp/ -โ”‚ โ”œโ”€โ”€ layer.hpp -โ”‚ โ”œโ”€โ”€ loss.hpp -โ”‚ โ”œโ”€โ”€ matrix.hpp -โ”‚ โ””โ”€โ”€ optimizer.hpp -โ””โ”€โ”€ src/ - โ”œโ”€โ”€ infer.cpp - โ””โ”€โ”€ train.cpp +โ”‚ โ”œโ”€โ”€ nn.hpp โ† ็ปŸไธ€ๅ…ฅๅฃๅคดๆ–‡ไปถ +โ”‚ โ”œโ”€โ”€ nn_config.hpp โ† SmartPolicyใ€BLOCK_SIZE ็ญ‰้…็ฝฎ +โ”‚ โ”œโ”€โ”€ thread_pool.hpp โ† ๅ…จๅฑ€็บฟ็จ‹ๆฑ  +โ”‚ โ”œโ”€โ”€ matrix.hpp โ† Matrix ็ฑป๏ผˆๅˆ—ไธปๅบ๏ผ‰ +โ”‚ โ”œโ”€โ”€ layer.hpp โ† Layer ๅŸบ็ฑป + Linear/ReLU/GeLU/LayerNorm/PositionalEncoding +โ”‚ โ”œโ”€โ”€ loss.hpp โ† Loss ๅŸบ็ฑป + MSELoss/CrossEntropyLoss +โ”‚ โ”œโ”€โ”€ optimizer.hpp โ† SGD / SGD_w_Momentum / Adam +โ”‚ โ”œโ”€โ”€ model.hpp โ† Model ๅฎนๅ™จ๏ผˆ้“พๅผ add<>๏ผ‰ +โ”‚ โ”œโ”€โ”€ model_io.hpp โ† ไบŒ่ฟ›ๅˆถๆจกๅž‹ๅบๅˆ—ๅŒ– +โ”‚ โ””โ”€โ”€ mnist_common.hpp โ† MNIST ๅธธ้‡ไธŽ build_mnist_model() +โ”œโ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ train.cpp +โ”‚ โ””โ”€โ”€ infer.cpp +โ”œโ”€โ”€ datasets/ +โ”‚ โ”œโ”€โ”€ mnist_data/ โ† MNIST CSV ๆ•ฐๆฎ +โ”‚ โ””โ”€โ”€ test/ โ† ๆŒ‰ๆ•ฐๅญ—ๅˆ†็ฑป็š„ๆต‹่ฏ•ๅ›พ็‰‡ +โ””โ”€โ”€ pretrained/ + โ””โ”€โ”€ mnist_model.bin โ† ้ข„่ฎญ็ปƒๆจกๅž‹ ``` +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/EthanPeng-2048/neuralnet.cpp) + ## ไพ่ต– - **็ผ–่ฏ‘ๅ™จ**: LLVM Clang++ 22.1+๏ผˆ`C:/Program Files/LLVM/bin/clang++.exe`๏ผ‰ @@ -56,14 +68,17 @@ python save_dataset.py ### ่ฟ่กŒ่ฎญ็ปƒ ```bash -# ไปŽๅคดๅผ€ๅง‹่ฎญ็ปƒ +# ไปŽๅคดๅผ€ๅง‹่ฎญ็ปƒ๏ผˆ้ป˜่ฎค: 10่ฝฎ, ๅญฆไน ็Ž‡0.001, ๆ‰นๅคงๅฐ64, Adamไผ˜ๅŒ–ๅ™จ๏ผ‰ ./build/mnist_train # ไปŽๅทฒๆœ‰ๆจกๅž‹ๆขๅค่ฎญ็ปƒ -./build/mnist_train --load mnist_model.bin +./build/mnist_train --resume pretrained/mnist_model.bin + +# ่‡ชๅฎšไน‰ๅ‚ๆ•ฐ +./build/mnist_train --epochs 20 --lr 0.001 --batch-size 32 --optimizer adam --save my_model.bin -# ๆŒ‡ๅฎšไฟๅญ˜่ทฏๅพ„ -./build/mnist_train --save my_model.bin +# ๆ”ฏๆŒ็š„ไผ˜ๅŒ–ๅ™จ: sgd / sgd_w_momentum / adam +./build/mnist_train --optimizer sgd_w_momentum ``` ### ๐Ÿ–ฅ๏ธ ๅ›พๅฝขๅŒ–็•Œ้ข (GUI) @@ -87,21 +102,37 @@ python gui.py ## ็ฝ‘็ปœ็ป“ๆž„ +้ป˜่ฎค MNIST ๆžถๆž„๏ผˆๅฎšไน‰ๅœจ `mnist_common.hpp`๏ผ‰๏ผš + ``` -่พ“ๅ…ฅ (784) โ†’ Linear(64) โ†’ ReLU โ†’ Linear(64) โ†’ ReLU โ†’ Linear(64) โ†’ ReLU โ†’ Linear(10) +่พ“ๅ…ฅ (784) +โ†’ Linear(784 โ†’ 512) + LayerNorm + GeLU +โ†’ Linear(512 โ†’ 256) + LayerNorm + GeLU +โ†’ Linear(256 โ†’ 128) + LayerNorm + GeLU +โ†’ Linear(128 โ†’ 64) + LayerNorm + GeLU +โ†’ Linear(64 โ†’ 10) +โ†’ CrossEntropy Loss (ๅซ Softmax) ``` ## ๆไพ›็š„็ป„ไปถ | ็ป„ไปถ | ่ฏดๆ˜Ž | |------|------| -| `nn::Matrix` | ๅˆ—ไธปๅบ็Ÿฉ้˜ต๏ผŒๆ”ฏๆŒๅนถ่กŒๅŠ /ๅ‡/ไน˜/่ฝฌ็ฝฎ | -| `nn::Linear` | ๅ…จ่ฟžๆŽฅๅฑ‚๏ผˆๅซ Xavier ๅˆๅง‹ๅŒ–๏ผ‰ | +| `nn::Matrix` | ๅˆ—ไธปๅบ็Ÿฉ้˜ต๏ผŒๆ”ฏๆŒๅนถ่กŒๅŠ /ๅ‡/ไน˜/่ฝฌ็ฝฎใ€้ข„ๅˆ†้…็ผ“ๅ†ฒๅŒบ | +| `nn::Model` | ็ฝ‘็ปœๅฎนๅ™จ๏ผŒๆ”ฏๆŒ้“พๅผ `add()` ๆž„ๅปบ | +| `nn::Linear` | ๅ…จ่ฟžๆŽฅๅฑ‚๏ผˆๅซ Xavier ๅ‡ๅŒ€ๅˆๅง‹ๅŒ–ใ€่žๅˆ bias ่ฎก็ฎ—๏ผ‰ | | `nn::ReLU` | ReLU ๆฟ€ๆดปๅ‡ฝๆ•ฐ | +| `nn::GeLU` | QuickGeLU ๆฟ€ๆดปๅ‡ฝๆ•ฐ๏ผˆ`x ยท ฯƒ(1.702x)`๏ผ‰ | +| `nn::LayerNorm` | ๅฑ‚ๅฝ’ไธ€ๅŒ–๏ผˆๅซๅฏๅญฆไน  ฮณ/ฮฒ ๅ‚ๆ•ฐ๏ผ‰ | +| `nn::PositionalEncoding` | ๆญฃๅผฆ/ไฝ™ๅผฆไฝ็ฝฎ็ผ–็ ๏ผˆTransformer ็”จ๏ผ‰ | | `nn::MSELoss` | ๅ‡ๆ–น่ฏฏๅทฎๆŸๅคฑ | -| `CrossEntropyLoss` | ไบคๅ‰็†ตๆŸๅคฑ๏ผˆๅซๆ•ฐๅ€ผ็จณๅฎš Softmax๏ผŒๅœจ `train.cpp` ไธญๅฎšไน‰๏ผ‰ | +| `nn::CrossEntropyLoss` | ไบคๅ‰็†ตๆŸๅคฑ๏ผˆๅซๆ•ฐๅ€ผ็จณๅฎš Softmax๏ผŒ`loss.hpp` ไธญๅฎšไน‰๏ผ‰ | | `nn::SGD` | ้šๆœบๆขฏๅบฆไธ‹้™ไผ˜ๅŒ–ๅ™จ | -| `save_model` / `load_model` | ไบŒ่ฟ›ๅˆถๆจกๅž‹ๅบๅˆ—ๅŒ– | +| `nn::SGD_w_Momentum` | ๅŠจ้‡ SGD ไผ˜ๅŒ–ๅ™จ | +| `nn::Adam` | Adam ไผ˜ๅŒ–ๅ™จ๏ผˆไธ€้˜ถ/ไบŒ้˜ถ็Ÿฉไผฐ่ฎก๏ผ‰ | +| `nn::SmartPolicy` | ่‡ช้€‚ๅบ”ๅนถ่กŒ็ญ–็•ฅ๏ผšๅฐ็Ÿฉ้˜ตไธฒ่กŒ๏ผŒๅคง็Ÿฉ้˜ต็บฟ็จ‹ๆฑ ๅนถ่กŒ | +| `nn::ThreadPool` | ๅ…จๅฑ€ๅ•ไพ‹็บฟ็จ‹ๆฑ ๏ผˆๆ‡’ๅˆๅง‹ๅŒ–๏ผ‰ | +| `nn::save_model` / `nn::load_model` | ไบŒ่ฟ›ๅˆถๆจกๅž‹ๅบๅˆ—ๅŒ– | ## ๐Ÿ“ ๅผ€ๅ‘่ง„่Œƒ diff --git a/gui.py b/gui.py index 27e85d53..1edb2ccb 100644 --- a/gui.py +++ b/gui.py @@ -144,6 +144,12 @@ def _build_train_tab(self): opt_combo = ttk.Combobox(param_frame, textvariable=self.train_opt_var, width=14, state="readonly", values=["sgd", "sgd_w_momentum", "adam"]) opt_combo.grid(row=1, column=1, sticky="w", pady=(6, 0)) + + ttk.Label(param_frame, text="ๆจกๅž‹็ฑปๅž‹:").grid(row=1, column=2, sticky="w", pady=(6, 0)) + self.train_model_type_var = tk.StringVar(value="mlp") + model_type_combo = ttk.Combobox(param_frame, textvariable=self.train_model_type_var, width=14, state="readonly", + values=["mlp", "transformer"]) + model_type_combo.grid(row=1, column=3, sticky="w", pady=(6, 0)) row += 1 # ๆŒ‰้’ฎ @@ -201,7 +207,8 @@ def _start_training(self): "--epochs", str(self.train_epochs_var.get()), "--lr", str(self.train_lr_var.get()), "--batch-size", str(self.train_batch_var.get()), - "--optimizer", self.train_opt_var.get()] + "--optimizer", self.train_opt_var.get(), + "--model-type", self.train_model_type_var.get()] if self.train_resume_var.get(): cmd += ["--resume", self.train_resume_path_var.get()] @@ -249,7 +256,7 @@ def _build_infer_tab(self): ttk.Button(f, text="ๆต่งˆโ€ฆ", command=self._browse_input).grid(row=row, column=2) row += 1 - # Top-K + # Top-K + ๆจกๅž‹็ฑปๅž‹ param_frame = ttk.Frame(f) param_frame.grid(row=row, column=0, columnspan=3, sticky="w", pady=4) row += 1 @@ -257,12 +264,54 @@ def _build_infer_tab(self): self.infer_topk_var = tk.IntVar(value=3) ttk.Spinbox(param_frame, from_=1, to=10, width=4, textvariable=self.infer_topk_var).pack(side="left", padx=4) + ttk.Label(param_frame, text=" ๆจกๅž‹็ฑปๅž‹:").pack(side="left") + self.infer_model_type_var = tk.StringVar(value="mlp") + infer_model_type_combo = ttk.Combobox(param_frame, textvariable=self.infer_model_type_var, + width=12, state="readonly", + values=["mlp", "transformer"]) + infer_model_type_combo.pack(side="left", padx=4) row += 1 # ๆŒ‰้’ฎ ttk.Button(f, text="โ–ถ ๅผ€ๅง‹ๆŽจ็†", command=self._start_infer).grid(row=row, column=0, columnspan=3, pady=4) row += 1 + # โ”€โ”€ ๆ‰‹ๅ†™ๆฟ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + self._handwriting_infer = False + DRAW_SCALE = 10 + DRAW_SIZE = IMG_DIM * DRAW_SCALE # 280 + self._draw_size = DRAW_SIZE + self._draw_scale = DRAW_SCALE + self._brush_radius = 14 + self._draw_pixels = [0.0] * (DRAW_SIZE * DRAW_SIZE) + self._drawing = False + + draw_frame = ttk.LabelFrame(f, text="โœ๏ธ ๆ‰‹ๅ†™่ฏ†ๅˆซ", padding=6) + draw_frame.grid(row=row, column=0, columnspan=3, sticky="ew", pady=4) + + draw_left = ttk.Frame(draw_frame) + draw_left.pack(side="left", padx=(0, 12)) + + self._draw_canvas = tk.Canvas(draw_left, width=DRAW_SIZE, height=DRAW_SIZE, + bg="white", cursor="pencil", relief="sunken", bd=2) + self._draw_canvas.pack() + self._draw_canvas.bind("", self._on_draw_start) + self._draw_canvas.bind("", self._on_draw_move) + self._draw_canvas.bind("", self._on_draw_end) + + btn_row = ttk.Frame(draw_left) + btn_row.pack(fill="x", pady=4) + ttk.Button(btn_row, text="๐Ÿ—‘๏ธ ๆธ…้™ค", command=self._clear_drawing).pack(side="left", padx=4) + ttk.Button(btn_row, text="๐Ÿ” ่ฏ†ๅˆซๆ‰‹ๅ†™ๆ•ฐๅญ—", command=self._recognize_drawing).pack(side="left", padx=4) + + # ๅณไพง๏ผš28ร—28 ้ข„่งˆ + draw_right = ttk.Frame(draw_frame) + draw_right.pack(side="left", fill="both", expand=True) + ttk.Label(draw_right, text="28ร—28 ้ข„่งˆ:", font=("Segoe UI", 9)).pack(anchor="w") + preview_size = IMG_DIM * PIXEL_SIZE + self._draw_preview = tk.Canvas(draw_right, width=preview_size, height=preview_size, bg="black") + self._draw_preview.pack(pady=4) + row += 1 # ็ป“ๆžœๅŒบๅŸŸ result_frame = ttk.Frame(f) result_frame.grid(row=row, column=0, columnspan=3, sticky="ew") @@ -306,7 +355,114 @@ def _browse_input(self): if path: self.infer_input_var.set(path) + # ---- ๆ‰‹ๅ†™ๆฟไบ‹ไปถ ---- + def _on_draw_start(self, event): + self._drawing = True + self._paint_at(event.x, event.y) + + def _on_draw_move(self, event): + if self._drawing: + self._paint_at(event.x, event.y) + + def _on_draw_end(self, event): + self._drawing = False + + def _paint_at(self, cx, cy): + """ๅœจ็”ปๆฟ (cx, cy) ๅค„็ป˜ๅˆถ็ฌ”่งฆ""" + r = self._brush_radius + for dy in range(-r, r + 1): + for dx in range(-r, r + 1): + dist_sq = dx * dx + dy * dy + if dist_sq > r * r: + continue + px, py = cx + dx, cy + dy + if 0 <= px < self._draw_size and 0 <= py < self._draw_size: + dist = math.sqrt(dist_sq) + sigma = r / 2.0 + intensity = math.exp(-dist_sq / (2 * sigma * sigma)) + idx = py * self._draw_size + px + self._draw_pixels[idx] = max(self._draw_pixels[idx], intensity) + self._draw_canvas.create_oval(cx - r, cy - r, cx + r, cy + r, + fill="black", outline="") + + def _clear_drawing(self): + """ๆธ…้™คๆ‰‹ๅ†™ๆฟ""" + self._draw_pixels = [0.0] * (self._draw_size * self._draw_size) + self._draw_canvas.delete("all") + self._draw_preview.delete("all") + + def _downsample_drawing(self) -> list[float]: + """ๅฐ† 280ร—280 ็”ปๆฟไธ‹้‡‡ๆ ทไธบ 28ร—28 ๅƒ็ด ๅ€ผ""" + scale = self._draw_scale + pixels = [] + for r in range(IMG_DIM): + for c in range(IMG_DIM): + total = 0.0 + for dy in range(scale): + for dx in range(scale): + y = r * scale + dy + x = c * scale + dx + total += self._draw_pixels[y * self._draw_size + x] + avg = total / (scale * scale) + pixels.append(avg) + return pixels + + def _recognize_drawing(self): + """่ฏ†ๅˆซๆ‰‹ๅ†™ๆฟไธŠ็š„ๆ•ฐๅญ—""" + exe = str(INFER_EXE) + if not Path(exe).exists(): + messagebox.showerror("้”™่ฏฏ", f"ๆ‰พไธๅˆฐๆŽจ็†ๅฏๆ‰ง่กŒๆ–‡ไปถ:\n{exe}\n่ฏทๅ…ˆๆž„ๅปบ้กน็›ฎ") + return + + # ๆฃ€ๆŸฅๆ˜ฏๅฆๆœ‰ๅ†…ๅฎน + if max(self._draw_pixels) < 0.01: + messagebox.showwarning("ๆ็คบ", "่ฏทๅ…ˆๅœจๆ‰‹ๅ†™ๆฟไธŠๅ†™ไธ€ไธชๆ•ฐๅญ—") + return + + pixels = self._downsample_drawing() + + # ๆ˜พ็คบ 28ร—28 ้ข„่งˆ + draw_digit(self._draw_preview, pixels) + draw_digit(self.infer_canvas, pixels) + self.infer_filename_label.config(text="ๆ‰‹ๅ†™่พ“ๅ…ฅ") + + # ๅ†™ๅ…ฅไธดๆ—ถ CSV + import tempfile + tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, newline="") + tmp.write(",".join(f"{v:.6f}" for v in pixels)) + tmp.close() + + cmd = [exe, tmp.name, + "--model", self.infer_model_var.get(), + "--model-type", self.infer_model_type_var.get(), + "--topk", str(self.infer_topk_var.get())] + + self._log_infer(f"[ๆ‰‹ๅ†™่ฏ†ๅˆซ] $ {' '.join(cmd)}\n") + self._handwriting_infer = True + + def _run(): + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + self._log_infer(result.stdout) + if result.stderr: + self._log_infer(result.stderr) + self._parse_and_display_infer(result.stdout) + except FileNotFoundError: + self._log_infer("้”™่ฏฏ: ๅฏๆ‰ง่กŒๆ–‡ไปถๆœชๆ‰พๅˆฐ\n") + except subprocess.TimeoutExpired: + self._log_infer("้”™่ฏฏ: ๆŽจ็†่ถ…ๆ—ถ\n") + except Exception as e: + self._log_infer(f"้”™่ฏฏ: {e}\n") + finally: + try: + os.unlink(tmp.name) + except OSError: + pass + + threading.Thread(target=_run, daemon=True).start() + def _start_infer(self): + self._handwriting_infer = False exe = str(INFER_EXE) if not Path(exe).exists(): messagebox.showerror("้”™่ฏฏ", f"ๆ‰พไธๅˆฐๆŽจ็†ๅฏๆ‰ง่กŒๆ–‡ไปถ:\n{exe}\n่ฏทๅ…ˆๆž„ๅปบ้กน็›ฎ:\n cmake -B build -G Ninja && ninja -C build") @@ -319,6 +475,7 @@ def _start_infer(self): cmd = [exe, input_path, "--model", self.infer_model_var.get(), + "--model-type", self.infer_model_type_var.get(), "--topk", str(self.infer_topk_var.get())] self._log_infer(f"$ {' '.join(cmd)}\n") @@ -350,7 +507,10 @@ def _parse_and_display_infer(self, output: str): predictions_str = parts[1].strip() # ๆ˜พ็คบๆ–‡ไปถๅ - self.after(0, lambda f=filename: self.infer_filename_label.config(text=f)) + if self._handwriting_infer: + self.after(0, lambda: self.infer_filename_label.config(text="ๆ‰‹ๅ†™่พ“ๅ…ฅ")) + else: + self.after(0, lambda f=filename: self.infer_filename_label.config(text=f)) # ่งฃๆž้ข„ๆต‹็ป“ๆžœ predictions = [] @@ -361,14 +521,15 @@ def _parse_and_display_infer(self, output: str): # ๆ˜พ็คบ็ป“ๆžœๆกๅฝขๅ›พ self.after(0, lambda p=predictions: self._show_prediction_bars(p)) - # ๅฐ่ฏ•ๆ˜พ็คบๅ›พ็‰‡ - input_path = self.infer_input_var.get().strip() - if os.path.isfile(input_path): - self.after(0, lambda p=input_path: self._display_csv_image(p)) - elif os.path.isdir(input_path): - img_path = os.path.join(input_path, filename) - if os.path.isfile(img_path): - self.after(0, lambda p=img_path: self._display_csv_image(p)) + # ๅฐ่ฏ•ๆ˜พ็คบๅ›พ็‰‡๏ผˆๆ‰‹ๅ†™ๆจกๅผ่ทณ่ฟ‡๏ผŒๅทฒๆœ‰้ข„่งˆ๏ผ‰ + if not self._handwriting_infer: + input_path = self.infer_input_var.get().strip() + if os.path.isfile(input_path): + self.after(0, lambda p=input_path: self._display_csv_image(p)) + elif os.path.isdir(input_path): + img_path = os.path.join(input_path, filename) + if os.path.isfile(img_path): + self.after(0, lambda p=img_path: self._display_csv_image(p)) break def _show_prediction_bars(self, predictions: list[tuple[int, float]]): diff --git a/include/neuralnet.cpp/layer.hpp b/include/neuralnet.cpp/layer.hpp index 2ee4b4a8..5bc2eb92 100644 --- a/include/neuralnet.cpp/layer.hpp +++ b/include/neuralnet.cpp/layer.hpp @@ -3,18 +3,20 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include -#include "matrix.hpp" -#include +#include "matrix.hpp" +#include "nn_config.hpp" namespace nn { @@ -26,10 +28,6 @@ namespace nn virtual Matrix backward(const Matrix &grad_output) = 0; virtual std::vector> parameters() { return {}; } virtual std::vector> param_gradients() { return {}; } - - // ๆทปๅŠ ๅ‚ๆ•ฐๆ›ดๆ–ฐ่พ…ๅŠฉๆ–นๆณ•๏ผŒ้ฟๅ…่™šๅ‡ฝๆ•ฐ่ฐƒ็”จๅผ€้”€ - virtual void update_params(double /*lr*/) noexcept {} - virtual void zero_grad() noexcept {} }; class Linear final : public Layer @@ -91,16 +89,16 @@ namespace nn // result = product + bias๏ผˆ่ฟ”ๅ›žๆ–ฐ็Ÿฉ้˜ต๏ผŒNRVO ไผ˜ๅŒ–๏ผ‰ Matrix result(out_feat, batch); - const double *prod_ptr = product_buf_.data_ptr(); - const double *bias_ptr = b_.data_ptr(); - double *res_ptr = result.data_ptr(); + const auto prod_span = product_buf_.span(); + const auto bias_span = b_.span(); + auto res_span = result.span(); const auto total = static_cast(out_feat * batch); auto indices = std::views::iota(std::size_t{0}, total); SmartPolicy::for_each(indices.begin(), indices.end(), - [prod_ptr, bias_ptr, res_ptr, batch](std::size_t idx) noexcept + [prod_span, bias_span, res_span, batch](std::size_t idx) noexcept { - res_ptr[idx] = prod_ptr[idx] + bias_ptr[idx / batch]; + res_span[idx] = prod_span[idx] + bias_span[idx / batch]; }); return result; @@ -125,31 +123,33 @@ namespace nn // grad_W += grad_output * input_cache_^T๏ผˆ้€ๅ…ƒ็ด ็ดฏๅŠ ๏ผŒ้ฟๅ…ไธดๆ—ถ็Ÿฉ้˜ต๏ผ‰ // ๆ‰‹ๅŠจ่ฎก็ฎ—๏ผšgrad_W[i][j] += sum_k(grad_output[i][k] * input_cache_[j][k]) { - const double *go_ptr = grad_output.data_ptr(); - const double *ic_ptr = input_cache_.data_ptr(); - double *gw_ptr = grad_W_.data_ptr(); - for (std::size_t of = 0; of < out_feat; ++of) - { - for (std::size_t inf = 0; inf < in_feat; ++inf) + const auto go_span = grad_output.span(); + const auto ic_span = input_cache_.span(); + auto gw_span = grad_W_.span(); + const std::size_t gw_size = out_feat * in_feat; + auto gw_indices = std::views::iota(std::size_t{0}, gw_size); + SmartPolicy::for_each(gw_indices.begin(), gw_indices.end(), + [go_span, ic_span, gw_span, in_feat, batch](std::size_t idx) noexcept { + const std::size_t of = idx / in_feat; + const std::size_t inf = idx % in_feat; double sum = 0.0; for (std::size_t b = 0; b < batch; ++b) - sum += go_ptr[of * batch + b] * ic_ptr[inf * batch + b]; - gw_ptr[of * in_feat + inf] += sum; - } - } + sum += go_span[of * batch + b] * ic_span[inf * batch + b]; + gw_span[idx] += sum; + }); } // grad_b += sum(grad_output, dim=batch) { - const double *go_ptr = grad_output.data_ptr(); - double *gb_ptr = grad_b_.data_ptr(); + const auto go_span = grad_output.span(); + auto gb_span = grad_b_.span(); for (std::size_t of = 0; of < out_feat; ++of) { double sum = 0.0; for (std::size_t b = 0; b < batch; ++b) - sum += go_ptr[of * batch + b]; - gb_ptr[of] += sum; + sum += go_span[of * batch + b]; + gb_span[of] += sum; } } @@ -169,20 +169,20 @@ namespace nn { input_cache_ = input; - const double *in_ptr = input.data_ptr(); - const auto n = static_cast(input.size()); + const auto in_span = input.span(); + const auto n = input.size(); Matrix result(input.rows(), input.cols()); - double *out_ptr = result.data_ptr(); + auto out_span = result.span(); if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(0LL, n); + auto indices = std::views::iota(std::size_t{0}, n); SmartPolicy::for_each(indices.begin(), indices.end(), - [in_ptr, out_ptr](long long i) noexcept - { out_ptr[i] = in_ptr[i] > 0.0 ? in_ptr[i] : 0.0; }); + [in_span, out_span](std::size_t i) noexcept + { out_span[i] = in_span[i] > 0.0 ? in_span[i] : 0.0; }); } else { - for (long long i = 0; i < n; ++i) - out_ptr[i] = in_ptr[i] > 0.0 ? in_ptr[i] : 0.0; + for (std::size_t i = 0; i < n; ++i) + out_span[i] = in_span[i] > 0.0 ? in_span[i] : 0.0; } return result; } @@ -192,21 +192,1172 @@ namespace nn if (input_cache_.rows() != grad_output.rows() || input_cache_.cols() != grad_output.cols()) throw std::invalid_argument("relu backward shape mismatch"); - const double *in_ptr = input_cache_.data_ptr(); - const double *go_ptr = grad_output.data_ptr(); - const auto n = static_cast(grad_output.size()); + const auto in_span = input_cache_.span(); + const auto go_span = grad_output.span(); + const auto n = grad_output.size(); + + Matrix grad_input(grad_output.rows(), grad_output.cols()); + auto out_span = grad_input.span(); + + if (n >= SmartPolicy::PARALLEL_THRESHOLD) { + auto indices = std::views::iota(std::size_t{0}, n); + SmartPolicy::for_each(indices.begin(), indices.end(), + [in_span, go_span, out_span](std::size_t i) noexcept + { out_span[i] = in_span[i] > 0.0 ? go_span[i] : 0.0; }); + } else { + for (std::size_t i = 0; i < n; ++i) + out_span[i] = in_span[i] > 0.0 ? go_span[i] : 0.0; + } + return grad_input; + } + }; + + class GeLU final : public Layer + { + private: + Matrix input_cache_; + Matrix sigmoid_cache_; // ็ผ“ๅญ˜ sigmoid(1.702 * x) ็”จไบŽๅๅ‘ไผ ๆ’ญ + static constexpr double BETA = 1.702; + + public: + GeLU() = default; + + // QuickGeLU: x * sigmoid(1.702 * x) + Matrix forward(const Matrix &input) override + { + const auto n = input.size(); + input_cache_ = input; + sigmoid_cache_.resize(input.rows(), input.cols()); + + const auto in_span = input.span(); + auto sig_span = sigmoid_cache_.span(); + + Matrix result(input.rows(), input.cols()); + auto out_span = result.span(); + + if (n >= SmartPolicy::PARALLEL_THRESHOLD) { + auto indices = std::views::iota(std::size_t{0}, n); + SmartPolicy::for_each(indices.begin(), indices.end(), + [in_span, out_span, sig_span](std::size_t i) noexcept + { + double sigmoid_input = BETA * in_span[i]; + double sigmoid_val = 1.0 / (1.0 + std::exp(-sigmoid_input)); + sig_span[i] = sigmoid_val; + out_span[i] = in_span[i] * sigmoid_val; + }); + } else { + for (std::size_t i = 0; i < n; ++i) { + double sigmoid_input = BETA * in_span[i]; + double sigmoid_val = 1.0 / (1.0 + std::exp(-sigmoid_input)); + sig_span[i] = sigmoid_val; + out_span[i] = in_span[i] * sigmoid_val; + } + } + return result; + } + + // d/dx [x * sigmoid(ฮฒx)] = sigmoid(ฮฒx) + x * ฮฒ * sigmoid(ฮฒx) * (1 - sigmoid(ฮฒx)) + // = sigmoid(ฮฒx) * [1 + ฮฒx * (1 - sigmoid(ฮฒx))] + Matrix backward(const Matrix &grad_output) override + { + if (input_cache_.rows() != grad_output.rows() || input_cache_.cols() != grad_output.cols()) + throw std::invalid_argument("gelu backward shape mismatch"); + + const auto in_span = input_cache_.span(); + const auto sig_span = sigmoid_cache_.span(); + const auto go_span = grad_output.span(); + const auto n = grad_output.size(); Matrix grad_input(grad_output.rows(), grad_output.cols()); - double *out_ptr = grad_input.data_ptr(); + auto out_span = grad_input.span(); if (n >= SmartPolicy::PARALLEL_THRESHOLD) { - auto indices = std::views::iota(0LL, n); + auto indices = std::views::iota(std::size_t{0}, n); SmartPolicy::for_each(indices.begin(), indices.end(), - [in_ptr, go_ptr, out_ptr](long long i) noexcept - { out_ptr[i] = in_ptr[i] > 0.0 ? go_ptr[i] : 0.0; }); + [in_span, sig_span, go_span, out_span](std::size_t i) noexcept + { + double s = sig_span[i]; + out_span[i] = go_span[i] * s * (1.0 + BETA * in_span[i] * (1.0 - s)); + }); + } else { + for (std::size_t i = 0; i < n; ++i) { + double s = sig_span[i]; + out_span[i] = go_span[i] * s * (1.0 + BETA * in_span[i] * (1.0 - s)); + } + } + return grad_input; + } + }; + + class LayerNorm final : public Layer + { + private: + std::size_t normalized_shape_; + double epsilon_; + + // ๅฏๅญฆไน ๅ‚ๆ•ฐ + Matrix gamma_; // ็ผฉๆ”พๅ‚ๆ•ฐ (normalized_shape_, 1) + Matrix beta_; // ๅ็งปๅ‚ๆ•ฐ (normalized_shape_, 1) + Matrix grad_gamma_; // gamma ๆขฏๅบฆ + Matrix grad_beta_; // beta ๆขฏๅบฆ + + // ็ผ“ๅญ˜็”จไบŽๅๅ‘ไผ ๆ’ญ + Matrix input_cache_; + Matrix normalized_cache_; // ๅฝ’ไธ€ๅŒ–ๅŽ็š„ๅ€ผ + Matrix std_cache_; // ๆ ‡ๅ‡†ๅทฎๅ€’ๆ•ฐ (1/sqrt(ฯƒยฒ + ฮต)) + Matrix mean_cache_; // ๅ‡ๅ€ผ + + // ๆ•ฐๅ€ผ็จณๅฎšๆ€ง + static constexpr double EPSILON = 1e-5; + + public: + explicit LayerNorm(std::size_t normalized_shape, double epsilon = EPSILON) + : normalized_shape_(normalized_shape), + epsilon_(epsilon), + gamma_(normalized_shape, 1, 1.0), // ๅˆๅง‹ๅŒ–ไธบ1 + beta_(normalized_shape, 1, 0.0), // ๅˆๅง‹ๅŒ–ไธบ0 + grad_gamma_(normalized_shape, 1), + grad_beta_(normalized_shape, 1) + { + // ๅ‚ๆ•ฐๅˆๅง‹ๅŒ–ๅทฒๅœจๆž„้€ ๅ‡ฝๆ•ฐไธญๅฎŒๆˆ + } + + std::vector> parameters() override + { + return {std::ref(gamma_), std::ref(beta_)}; + } + + std::vector> param_gradients() override + { + return {std::ref(grad_gamma_), std::ref(grad_beta_)}; + } + + Matrix forward(const Matrix &input) override + { + // ่พ“ๅ…ฅๅฝข็Šถ: (normalized_shape_, batch_size) + if (input.rows() != normalized_shape_) + throw std::invalid_argument("layer_norm forward input shape mismatch"); + + input_cache_ = input; + const std::size_t batch_size = input.cols(); + const std::size_t features = input.rows(); + + // ็ผ“ๅญ˜ไธญ้—ด็ป“ๆžœ๏ผˆๅค็”จๅทฒๆœ‰ๅ†…ๅญ˜๏ผ‰ + mean_cache_.resize(1, batch_size); + std_cache_.resize(1, batch_size); + normalized_cache_.resize(features, batch_size); + + Matrix result(features, batch_size); + + const auto in_span = input.span(); + const auto gamma_span = gamma_.span(); + const auto beta_span = beta_.span(); + auto mean_span = mean_cache_.span(); + auto std_span = std_cache_.span(); + auto norm_span = normalized_cache_.span(); + auto res_span = result.span(); + + // ๅฏนๆฏไธชๆ ทๆœฌ๏ผˆๅˆ—๏ผ‰็‹ฌ็ซ‹ๅค„็† + auto batch_indices = std::views::iota(std::size_t{0}, batch_size); + + if (batch_size >= SmartPolicy::PARALLEL_THRESHOLD) { + SmartPolicy::for_each(batch_indices.begin(), batch_indices.end(), + [in_span, gamma_span, beta_span, mean_span, std_span, norm_span, res_span, + features, batch_size, epsilon = epsilon_](std::size_t b) noexcept + { + // ่ฎก็ฎ—ๅ‡ๅ€ผ + double sum = 0.0; + for (std::size_t f = 0; f < features; ++f) + sum += in_span[f * batch_size + b]; + double mean = sum / static_cast(features); + mean_span[b] = mean; + + // ่ฎก็ฎ—ๆ–นๅทฎ + double var_sum = 0.0; + for (std::size_t f = 0; f < features; ++f) { + double diff = in_span[f * batch_size + b] - mean; + var_sum += diff * diff; + } + double variance = var_sum / static_cast(features); + double std_inv = 1.0 / std::sqrt(variance + epsilon); + std_span[b] = std_inv; + + // ๅฝ’ไธ€ๅŒ–ๅ’Œไปฟๅฐ„ๅ˜ๆข + for (std::size_t f = 0; f < features; ++f) { + double normalized = (in_span[f * batch_size + b] - mean) * std_inv; + norm_span[f * batch_size + b] = normalized; + res_span[f * batch_size + b] = gamma_span[f] * normalized + beta_span[f]; + } + }); + } else { + for (std::size_t b = 0; b < batch_size; ++b) { + // ่ฎก็ฎ—ๅ‡ๅ€ผ + double sum = 0.0; + for (std::size_t f = 0; f < features; ++f) + sum += in_span[f * batch_size + b]; + double mean = sum / static_cast(features); + mean_span[b] = mean; + + // ่ฎก็ฎ—ๆ–นๅทฎ + double var_sum = 0.0; + for (std::size_t f = 0; f < features; ++f) { + double diff = in_span[f * batch_size + b] - mean; + var_sum += diff * diff; + } + double variance = var_sum / static_cast(features); + double std_inv = 1.0 / std::sqrt(variance + epsilon_); + std_span[b] = std_inv; + + // ๅฝ’ไธ€ๅŒ–ๅ’Œไปฟๅฐ„ๅ˜ๆข + for (std::size_t f = 0; f < features; ++f) { + double normalized = (in_span[f * batch_size + b] - mean) * std_inv; + norm_span[f * batch_size + b] = normalized; + res_span[f * batch_size + b] = gamma_span[f] * normalized + beta_span[f]; + } + } + } + + return result; + } + + Matrix backward(const Matrix &grad_output) override + { + if (grad_output.rows() != normalized_shape_) + throw std::invalid_argument("layer_norm backward shape mismatch"); + + const std::size_t features = normalized_shape_; + const std::size_t batch_size = grad_output.cols(); + + Matrix grad_input(features, batch_size); + + const auto go_span = grad_output.span(); + const auto norm_span = normalized_cache_.span(); + const auto std_span = std_cache_.span(); + const auto gamma_span = gamma_.span(); + auto gi_span = grad_input.span(); + auto gg_span = grad_gamma_.span(); + auto gb_span = grad_beta_.span(); + + // ่ฎก็ฎ—ๆขฏๅบฆ + auto batch_indices = std::views::iota(std::size_t{0}, batch_size); + + if (batch_size >= SmartPolicy::PARALLEL_THRESHOLD) { + // ๅ…ˆๅนถ่กŒ่ฎก็ฎ—ๆฏไธชๆ ทๆœฌ็š„ dL/dx๏ผˆๆ— ็ซžไบ‰๏ผ‰๏ผŒ + // ๅ†ไธฒ่กŒ็ดฏๅŠ  dL/dฮณ ๅ’Œ dL/dฮฒ๏ผˆ้ฟๅ…ๆ•ฐๆฎ็ซžไบ‰๏ผ‰ + SmartPolicy::for_each(batch_indices.begin(), batch_indices.end(), + [go_span, norm_span, std_span, gamma_span, gi_span, + features, batch_size](std::size_t b) noexcept + { + double std_inv = std_span[b]; + double sum_grad = 0.0; + double sum_grad_norm = 0.0; + for (std::size_t f = 0; f < features; ++f) { + double g = go_span[f * batch_size + b] * gamma_span[f]; + sum_grad += g; + sum_grad_norm += g * norm_span[f * batch_size + b]; + } + const double inv_features = 1.0 / static_cast(features); + for (std::size_t f = 0; f < features; ++f) { + double grad_out = go_span[f * batch_size + b]; + double g = grad_out * gamma_span[f]; + gi_span[f * batch_size + b] = (g - sum_grad * inv_features - + norm_span[f * batch_size + b] * sum_grad_norm * inv_features) * std_inv; + } + }); + // ไธฒ่กŒ็ดฏๅŠ  dL/dฮณ ๅ’Œ dL/dฮฒ + for (std::size_t b = 0; b < batch_size; ++b) { + for (std::size_t f = 0; f < features; ++f) { + double grad_out = go_span[f * batch_size + b]; + gg_span[f] += grad_out * norm_span[f * batch_size + b]; + gb_span[f] += grad_out; + } + } } else { - for (long long i = 0; i < n; ++i) - out_ptr[i] = in_ptr[i] > 0.0 ? go_ptr[i] : 0.0; + for (std::size_t b = 0; b < batch_size; ++b) { + double std_inv = std_span[b]; + + // ้ข„่ฎก็ฎ—็ปŸ่ฎก้‡๏ผšO(N) ่€Œ้ž O(Nยฒ) + double sum_grad = 0.0; + double sum_grad_norm = 0.0; + for (std::size_t f = 0; f < features; ++f) { + double g = go_span[f * batch_size + b] * gamma_span[f]; + sum_grad += g; + sum_grad_norm += g * norm_span[f * batch_size + b]; + } + + // ๅ•ๆฌก้ๅŽ†๏ผšๅŒๆ—ถ่ฎก็ฎ— dL/dฮณใ€dL/dฮฒใ€dL/dx + const double inv_features = 1.0 / static_cast(features); + for (std::size_t f = 0; f < features; ++f) { + double grad_out = go_span[f * batch_size + b]; + double g = grad_out * gamma_span[f]; + gg_span[f] += grad_out * norm_span[f * batch_size + b]; + gb_span[f] += grad_out; + gi_span[f * batch_size + b] = (g - sum_grad * inv_features - + norm_span[f * batch_size + b] * sum_grad_norm * inv_features) * std_inv; + } + } + } + + return grad_input; + } + + }; + + // โ”€โ”€ Softmax ๆฟ€ๆดปๅฑ‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๅฏนๆฏไธ€่กŒ็‹ฌ็ซ‹ๅบ”็”จ softmax๏ผšout[i][j] = exp(in[i][j]) / ฮฃ_k exp(in[i][k]) + // ่พ“ๅ…ฅๅฝข็Šถ: (rows, cols)๏ผŒ่พ“ๅ‡บๅฝข็Šถ็›ธๅŒใ€‚ + // ไฝฟ็”จๆœ€ๅคงๅ€ผๅ‡ๆณ•ไฟ่ฏๆ•ฐๅ€ผ็จณๅฎšๆ€งใ€‚ + // ๅๅ‘ไผ ๆ’ญ: grad[i][j] = out[i][j] * (grad_out[i][j] - ฮฃ_k out[i][k] * grad_out[i][k]) + // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + class Softmax final : public Layer + { + private: + Matrix output_cache_; + + public: + Softmax() = default; + + Matrix forward(const Matrix &input) override + { + const std::size_t rows = input.rows(); + const std::size_t cols = input.cols(); + output_cache_.resize(rows, cols); + + Matrix result(rows, cols); + const auto in_span = input.span(); + auto out_span = result.span(); + auto cache_span = output_cache_.span(); + + auto row_indices = std::views::iota(std::size_t{0}, rows); + const std::size_t total = rows * cols; + + auto process_row = [in_span, out_span, cache_span, cols](std::size_t r) noexcept + { + const std::size_t offset = r * cols; + const auto row_in = in_span.subspan(offset, cols); + auto row_out = out_span.subspan(offset, cols); + auto row_cache = cache_span.subspan(offset, cols); + + // ๆ•ฐๅ€ผ็จณๅฎš๏ผšๅ‡ๅŽป่กŒๅ†…ๆœ€ๅคงๅ€ผ + double max_val = row_in[0]; + for (std::size_t c = 1; c < cols; ++c) + max_val = std::max(max_val, row_in[c]); + + // ่ฎก็ฎ— exp ๅ’Œๆฑ‚ๅ’Œ + double sum = 0.0; + for (std::size_t c = 0; c < cols; ++c) + { + double e = std::exp(row_in[c] - max_val); + row_out[c] = e; + row_cache[c] = e; + sum += e; + } + + // ๅฝ’ไธ€ๅŒ– + const double inv_sum = 1.0 / sum; + for (std::size_t c = 0; c < cols; ++c) + { + row_out[c] *= inv_sum; + row_cache[c] *= inv_sum; + } + }; + + if (total >= SmartPolicy::PARALLEL_THRESHOLD) + SmartPolicy::for_each(row_indices.begin(), row_indices.end(), process_row); + else + for (std::size_t r = 0; r < rows; ++r) + process_row(r); + + return result; + } + + Matrix backward(const Matrix &grad_output) override + { + const std::size_t rows = output_cache_.rows(); + const std::size_t cols = output_cache_.cols(); + + if (grad_output.rows() != rows || grad_output.cols() != cols) + throw std::invalid_argument("softmax backward shape mismatch"); + + Matrix grad_input(rows, cols); + + const auto go_span = grad_output.span(); + const auto out_span = output_cache_.span(); + auto gi_span = grad_input.span(); + + auto row_indices = std::views::iota(std::size_t{0}, rows); + const std::size_t total = rows * cols; + + auto process_row = [go_span, out_span, gi_span, cols](std::size_t r) noexcept + { + const std::size_t offset = r * cols; + const auto row_go = go_span.subspan(offset, cols); + const auto row_out = out_span.subspan(offset, cols); + auto row_gi = gi_span.subspan(offset, cols); + + // dot = ฮฃ_k out[k] * grad_out[k] + double dot = 0.0; + for (std::size_t c = 0; c < cols; ++c) + dot += row_out[c] * row_go[c]; + + // grad = out * (grad_out - dot) + for (std::size_t c = 0; c < cols; ++c) + row_gi[c] = row_out[c] * (row_go[c] - dot); + }; + + if (total >= SmartPolicy::PARALLEL_THRESHOLD) + SmartPolicy::for_each(row_indices.begin(), row_indices.end(), process_row); + else + for (std::size_t r = 0; r < rows; ++r) + process_row(r); + + return grad_input; + } + }; + + // โ”€โ”€ ๅคšๅคดๆณจๆ„ๅŠ›ๅฑ‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๆฅๆบ: "Attention Is All You Need" (Vaswani et al., 2017) + // + // MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O + // head_i = Attention(QW_i^Q, KW_i^K, VW_i^V) + // Attention(Q, K, V) = softmax(QK^T / โˆšd_k) V + // + // ่พ“ๅ…ฅๅฝข็Šถ: (d_model, seq_len)๏ผŒ่พ“ๅ‡บๅฝข็Šถ: (d_model, seq_len) + // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + class MultiHeadAttention final : public Layer + { + private: + std::size_t d_model_; + std::size_t num_heads_; + std::size_t d_k_; + double scale_; // 1.0 / sqrt(d_k) + + // ๆŠ•ๅฝฑๅฑ‚ + Linear W_q_; + Linear W_k_; + Linear W_v_; + Linear W_o_; + + // ๅ‰ๅ‘ไผ ๆ’ญ็ผ“ๅญ˜ + Matrix Q_cache_; // (d_model, seq_len) + Matrix K_cache_; // (d_model, seq_len) + Matrix V_cache_; // (d_model, seq_len) + std::vector attn_; // num_heads_ ร— (seq_len, seq_len) โ€” softmax ่พ“ๅ‡บ + + // Per-head ๅˆ‡็‰‡็ผ“ๅญ˜๏ผˆforward ๆๅ–๏ผŒbackward ไฝฟ็”จ๏ผ‰ + std::vector Q_heads_; // num_heads_ ร— (d_k_, seq_len) + std::vector K_heads_; + std::vector V_heads_; + std::vector O_heads_; // num_heads_ ร— (d_k_, seq_len) โ€” ๅ‰ๅ‘่พ“ๅ‡บ็ผ“ๅญ˜ + + // ่พ…ๅŠฉ็ผ“ๅ†ฒๅŒบ๏ผˆ้ฟๅ…ๅพช็Žฏๅ†…้‡ๅคๅˆ†้…๏ผ‰ + Matrix grad_scores_buf_; // (seq_len, seq_len) + Matrix grad_A_buf_; // (seq_len, seq_len) + Matrix grad_O_h_buf_; // (d_k_, seq_len) โ€” backward ไธญๅค็”จ + Softmax softmax_; // ๅค็”จ๏ผŒ้ฟๅ…ๆฏๅคดๆž„้€ /ๆžๆž„ + + // โ”€โ”€ ไปŽ็Ÿฉ้˜ตไธญๆๅ–่กŒๅˆ‡็‰‡ๅˆฐ dst โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + static void extract_rows(const Matrix &src, std::size_t row_start, + std::size_t row_count, Matrix &dst) + { + const std::size_t cols = src.cols(); + dst.resize(row_count, cols); + const auto src_span = src.span(); + auto dst_span = dst.span(); + for (std::size_t r = 0; r < row_count; ++r) + std::copy_n(src_span.begin() + (row_start + r) * cols, cols, dst_span.begin() + r * cols); + } + + // โ”€โ”€ ๅฐ† src ๅ†™ๅ…ฅ dst ็š„ๆŒ‡ๅฎš่กŒ่Œƒๅ›ด โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + static void insert_rows(Matrix &dst, std::size_t row_start, + const Matrix &src) + { + const std::size_t row_count = src.rows(); + const std::size_t cols = src.cols(); + auto dst_span = dst.span(); + const auto src_span = src.span(); + for (std::size_t r = 0; r < row_count; ++r) + std::copy_n(src_span.begin() + r * cols, cols, dst_span.begin() + (row_start + r) * cols); + } + + // โ”€โ”€ ้€ๅ…ƒ็ด ็ผฉๆ”พ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + static void scale_inplace(Matrix &m, double s) + { + auto m_span = m.span(); + const auto n = m.size(); + if (n >= SmartPolicy::PARALLEL_THRESHOLD) + { + auto indices = std::views::iota(std::size_t{0}, n); + SmartPolicy::for_each(indices.begin(), indices.end(), + [m_span, s](std::size_t i) noexcept { m_span[i] *= s; }); + } + else + { + for (std::size_t i = 0; i < n; ++i) + m_span[i] *= s; + } + } + + public: + MultiHeadAttention(std::size_t d_model, std::size_t num_heads) + : d_model_(d_model), + num_heads_(num_heads), + d_k_(d_model / num_heads), + scale_(1.0 / std::sqrt(static_cast(d_model / num_heads))), + W_q_(d_model, d_model), + W_k_(d_model, d_model), + W_v_(d_model, d_model), + W_o_(d_model, d_model) + { + if (d_model % num_heads != 0) + throw std::invalid_argument("MultiHeadAttention: d_model must be divisible by num_heads"); + } + + std::vector> parameters() override + { + auto params = W_q_.parameters(); + auto wk = W_k_.parameters(); + auto wv = W_v_.parameters(); + auto wo = W_o_.parameters(); + params.insert(params.end(), wk.begin(), wk.end()); + params.insert(params.end(), wv.begin(), wv.end()); + params.insert(params.end(), wo.begin(), wo.end()); + return params; + } + + std::vector> param_gradients() override + { + auto grads = W_q_.param_gradients(); + auto gk = W_k_.param_gradients(); + auto gv = W_v_.param_gradients(); + auto go = W_o_.param_gradients(); + grads.insert(grads.end(), gk.begin(), gk.end()); + grads.insert(grads.end(), gv.begin(), gv.end()); + grads.insert(grads.end(), go.begin(), go.end()); + return grads; + } + + Matrix forward(const Matrix &input) override + { + if (input.rows() != d_model_) + throw std::invalid_argument("MultiHeadAttention forward input shape mismatch"); + + const std::size_t seq_len = input.cols(); + + // โ”€โ”€ 1. ็บฟๆ€งๆŠ•ๅฝฑ Q, K, V โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Q_cache_ = W_q_.forward(input); + K_cache_ = W_k_.forward(input); + V_cache_ = W_v_.forward(input); + + // โ”€โ”€ 2. ๅˆ†้… per-head ็ผ“ๅ†ฒๅŒบ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Q_heads_.resize(num_heads_); + K_heads_.resize(num_heads_); + V_heads_.resize(num_heads_); + O_heads_.resize(num_heads_); + attn_.resize(num_heads_); + + // โ”€โ”€ 3. ้€ๅคด่ฎก็ฎ—็ผฉๆ”พ็‚น็งฏๆณจๆ„ๅŠ› โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + for (std::size_t h = 0; h < num_heads_; ++h) + { + const std::size_t row_start = h * d_k_; + + // ๆๅ– Q_h, K_h, V_h โ†’ (d_k_, seq_len) + extract_rows(Q_cache_, row_start, d_k_, Q_heads_[h]); + extract_rows(K_cache_, row_start, d_k_, K_heads_[h]); + extract_rows(V_cache_, row_start, d_k_, V_heads_[h]); + + // S_h = Q_h^T @ K_h โ†’ (seq_len, seq_len) + Matrix QhT = Q_heads_[h].transpose(); + attn_[h].resize(seq_len, seq_len); + QhT.multiply_to(attn_[h], K_heads_[h]); + + // S_h *= 1/โˆšd_k + scale_inplace(attn_[h], scale_); + + // A_h = softmax(S_h) โ€” ๆŒ‰่กŒ softmax๏ผˆๅค็”จๆˆๅ‘˜ softmax_๏ผ‰ + attn_[h] = softmax_.forward(attn_[h]); + + // O_h = V_h @ A_h โ†’ (d_k_, seq_len)๏ผŒ็ผ“ๅญ˜ๅˆฐ O_heads_ + O_heads_[h].resize(d_k_, seq_len); + V_heads_[h].multiply_to(O_heads_[h], attn_[h]); + } + + // โ”€โ”€ 4. ๆ‹ผๆŽฅๆ‰€ๆœ‰ๅคด็š„่พ“ๅ‡บๅนถๆŠ•ๅฝฑ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix output(d_model_, seq_len); + for (std::size_t h = 0; h < num_heads_; ++h) + insert_rows(output, h * d_k_, O_heads_[h]); + + return W_o_.forward(output); + } + + Matrix backward(const Matrix &grad_output) override + { + const std::size_t seq_len = grad_output.cols(); + + // โ”€โ”€ 1. ่พ“ๅ‡บๆŠ•ๅฝฑๅๅ‘ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix grad_concat = W_o_.backward(grad_output); // (d_model, seq_len) + + // โ”€โ”€ 2. ๅˆๅง‹ๅŒ–ๅ„ๅคดๆขฏๅบฆ็ดฏๅŠ ็Ÿฉ้˜ต โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix grad_Q_all(d_model_, seq_len); + Matrix grad_K_all(d_model_, seq_len); + Matrix grad_V_all(d_model_, seq_len); + { + grad_Q_all.zero(); + grad_K_all.zero(); + grad_V_all.zero(); + } + + // โ”€โ”€ 3. ้€ๅคด่ฎก็ฎ—ๆณจๆ„ๅŠ›ๅๅ‘ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + for (std::size_t h = 0; h < num_heads_; ++h) + { + const std::size_t row_start = h * d_k_; + const Matrix &Q_h = Q_heads_[h]; // (d_k_, seq_len) + const Matrix &K_h = K_heads_[h]; // (d_k_, seq_len) + const Matrix &V_h = V_heads_[h]; // (d_k_, seq_len) + const Matrix &A_h = attn_[h]; // (seq_len, seq_len) + + // ๆๅ– grad_O_h๏ผˆๅค็”จๆˆๅ‘˜็ผ“ๅ†ฒๅŒบ๏ผ‰ + extract_rows(grad_concat, row_start, d_k_, grad_O_h_buf_); + + // grad_V_h = grad_O_h @ A_h^T โ†’ (d_k_, seq_len) + Matrix AhT = A_h.transpose(); + Matrix grad_V_h(d_k_, seq_len); + grad_O_h_buf_.multiply_to(grad_V_h, AhT); + + // grad_A_h = V_h^T @ grad_O_h โ†’ (seq_len, seq_len) + Matrix VhT = V_h.transpose(); + grad_A_buf_.resize(seq_len, seq_len); + VhT.multiply_to(grad_A_buf_, grad_O_h_buf_); + + // grad_S_h = A_h โŠ™ (grad_A_h - row_sum(A_h โŠ™ grad_A_h)) + grad_scores_buf_.resize(seq_len, seq_len); + { + const auto a_span = A_h.span(); + const auto ga_span = grad_A_buf_.span(); + auto gs_span = grad_scores_buf_.span(); + + for (std::size_t i = 0; i < seq_len; ++i) + { + double dot = 0.0; + for (std::size_t j = 0; j < seq_len; ++j) + dot += a_span[i * seq_len + j] * ga_span[i * seq_len + j]; + for (std::size_t j = 0; j < seq_len; ++j) + gs_span[i * seq_len + j] = + a_span[i * seq_len + j] * (ga_span[i * seq_len + j] - dot); + } + } + + // grad_Q_h = K_h @ grad_S_h^T * scale โ†’ (d_k_, seq_len) + Matrix gsT = grad_scores_buf_.transpose(); + Matrix grad_Q_h(d_k_, seq_len); + K_h.multiply_to(grad_Q_h, gsT); + scale_inplace(grad_Q_h, scale_); + + // grad_K_h = Q_h @ grad_S_h * scale โ†’ (d_k_, seq_len) + Matrix grad_K_h(d_k_, seq_len); + Q_h.multiply_to(grad_K_h, grad_scores_buf_); + scale_inplace(grad_K_h, scale_); + + // ็ดฏๅŠ ๅˆฐๅ…จๅฑ€ๆขฏๅบฆ๏ผˆๆฏไธชๅคดๅ†™ๅ…ฅไธๅŒ่กŒ๏ผŒๆ— ้‡ๅ ๏ผ‰ + insert_rows(grad_Q_all, row_start, grad_Q_h); + insert_rows(grad_K_all, row_start, grad_K_h); + insert_rows(grad_V_all, row_start, grad_V_h); + } + + // โ”€โ”€ 4. ๆŠ•ๅฝฑๅฑ‚ๅๅ‘๏ผŒ็ดฏๅŠ ่พ“ๅ…ฅๆขฏๅบฆ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix grad_input = W_q_.backward(grad_Q_all); + grad_input = grad_input + W_k_.backward(grad_K_all); + grad_input = grad_input + W_v_.backward(grad_V_all); + + return grad_input; + } + }; + + // โ”€โ”€ ๆญฃๅผฆๆณขๅ›บๅฎšไฝ็ฝฎ็ผ–็  โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๆฅๆบ: "Attention Is All You Need" (Vaswani et al., 2017) + // + // PE(pos, 2i) = sin(pos / 10000^(2i/d_model)) + // PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model)) + // + // ่พ“ๅ…ฅๅฝข็Šถ: (d_model, seq_len)๏ผŒ่พ“ๅ‡บๅฝข็Šถ็›ธๅŒใ€‚ + // ็ผ–็ ็Ÿฉ้˜ตๅœจๆž„้€ ๆ—ถไธ€ๆฌกๆ€ง้ข„่ฎก็ฎ—๏ผŒๅ‰ๅ‘ไผ ๆ’ญไป…ๅš้€ๅ…ƒ็ด ๅŠ ๆณ•ใ€‚ + // ๅๅ‘ไผ ๆ’ญ็›ดๆŽฅ็ฉฟ้€๏ผˆ็ผ–็ ไธๅฏๅญฆไน ๏ผ‰ใ€‚ + // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + class PositionalEncoding final : public Layer + { + private: + std::size_t d_model_; + std::size_t max_len_; + Matrix encoding_; // (d_model, max_len) โ€” ้ข„่ฎก็ฎ—็š„ๆญฃๅผฆๆณข็ผ–็  + + public: + PositionalEncoding(std::size_t d_model, std::size_t max_len = 5000) + : d_model_(d_model), + max_len_(max_len), + encoding_(d_model, max_len) + { + // โ”€โ”€ ไธ€ๆฌกๆ€ง้ข„่ฎก็ฎ—้ข‘็Ž‡ไธŽ็ผ–็  โ”€โ”€ + // ๅ…ˆ่ฎก็ฎ—ๆฏไธช็‰นๅพๅฏน็š„่ง’้ข‘็Ž‡๏ผˆ้ฟๅ…ๅœจ position ๅพช็Žฏไธญ้‡ๅค pow๏ผ‰ + const std::size_t half = d_model / 2; + std::vector freqs(half); + for (std::size_t i = 0; i < half; ++i) + freqs[i] = 1.0 / std::pow(10000.0, static_cast(2 * i) / d_model); + + auto e_span = encoding_.span(); + for (std::size_t pos = 0; pos < max_len; ++pos) + { + const double pos_d = static_cast(pos); + for (std::size_t i = 0; i < half; ++i) + { + const double angle = pos_d * freqs[i]; + e_span[(2 * i) * max_len + pos] = std::sin(angle); + e_span[(2 * i + 1) * max_len + pos] = std::cos(angle); + } + // ๅฅ‡ๆ•ฐ็ปดๅบฆ๏ผšๆœ€ๅŽไธ€ไธช็‰นๅพไป…ไฝฟ็”จ sin + if (d_model % 2 == 1) + { + const double freq_last = 1.0 / std::pow(10000.0, static_cast(2 * half) / d_model); + const double angle = pos_d * freq_last; + e_span[(d_model - 1) * max_len + pos] = std::sin(angle); + } + } + } + + std::vector> parameters() override { return {}; } + std::vector> param_gradients() override { return {}; } + + Matrix forward(const Matrix &input) override + { + if (input.rows() != d_model_) + throw std::invalid_argument("positional encoding forward: d_model mismatch"); + + const std::size_t seq_len = input.cols(); + + if (seq_len > max_len_) + throw std::invalid_argument("positional encoding forward: sequence length exceeds max_len"); + + // result = input + encoding[:, 0:seq_len] + Matrix result(d_model_, seq_len); + const auto in_span = input.span(); + const auto e_span = encoding_.span(); + auto out_span = result.span(); + + const auto total = static_cast(d_model_ * seq_len); + const std::size_t max_len = max_len_; + auto indices = std::views::iota(std::size_t{0}, total); + SmartPolicy::for_each(indices.begin(), indices.end(), + [in_span, e_span, out_span, seq_len, max_len](std::size_t idx) noexcept + { + // idx = row * seq_len + col + const std::size_t row = idx / seq_len; + const std::size_t col = idx % seq_len; + out_span[idx] = in_span[idx] + e_span[row * max_len + col]; + }); + + return result; + } + + // ไฝ็ฝฎ็ผ–็ ไธบๅ›บๅฎšๅ€ผ๏ผŒๆขฏๅบฆ็›ดๆŽฅ็ฉฟ้€ + Matrix backward(const Matrix &grad_output) override { return grad_output; } + }; + + // โ”€โ”€ ๅ‰้ฆˆ็ฝ‘็ปœ (Feed-Forward Network) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // FFN(x) = Linear(d_ff โ†’ d_model)(GeLU(Linear(d_model โ†’ d_ff)(x))) + // ่พ“ๅ…ฅ/่พ“ๅ‡บๅฝข็Šถ: (d_model, seq_len) + class FeedForward final : public Layer + { + private: + Linear fc1_; // (d_ff, d_model) + Linear fc2_; // (d_model, d_ff) + GeLU activation_; + Matrix input_cache_; + + public: + FeedForward(std::size_t d_model, std::size_t d_ff) + : fc1_(d_model, d_ff), fc2_(d_ff, d_model) + {} + + std::vector> parameters() override + { + auto params = fc1_.parameters(); + auto p2 = fc2_.parameters(); + params.insert(params.end(), p2.begin(), p2.end()); + return params; + } + + std::vector> param_gradients() override + { + auto grads = fc1_.param_gradients(); + auto g2 = fc2_.param_gradients(); + grads.insert(grads.end(), g2.begin(), g2.end()); + return grads; + } + + Matrix forward(const Matrix &input) override + { + input_cache_ = input; + auto x = activation_.forward(fc1_.forward(input)); + return fc2_.forward(x); + } + + Matrix backward(const Matrix &grad_output) override + { + auto grad = fc2_.backward(grad_output); + grad = activation_.backward(grad); + return fc1_.backward(grad); + } + }; + + // โ”€โ”€ Transformer ็ผ–็ ๅ™จๅฑ‚ (Pre-Norm ๆžถๆž„) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // x = x + SelfAttn(LayerNormโ‚(x)) โ† ๆฎ‹ๅทฎ่ฟžๆŽฅ + // x = x + FFN(LayerNormโ‚‚(x)) โ† ๆฎ‹ๅทฎ่ฟžๆŽฅ + // ่พ“ๅ…ฅ/่พ“ๅ‡บๅฝข็Šถ: (d_model, seq_len) + class TransformerEncoderLayer final : public Layer + { + private: + MultiHeadAttention self_attn_; + LayerNorm norm1_; + FeedForward ff_; + LayerNorm norm2_; + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ็ผ“ๅญ˜ โ”€โ”€ + Matrix residual1_cache_; // ็ฌฌไธ€ไธชๆฎ‹ๅทฎ่ฟžๆŽฅๅ‰็š„ๅŽŸๅง‹่พ“ๅ…ฅ + Matrix residual2_cache_; // ็ฌฌไบŒไธชๆฎ‹ๅทฎ่ฟžๆŽฅๅ‰็š„่พ“ๅ…ฅ (= residual1) + + public: + TransformerEncoderLayer(std::size_t d_model, std::size_t num_heads, std::size_t d_ff) + : self_attn_(d_model, num_heads), + norm1_(d_model), + ff_(d_model, d_ff), + norm2_(d_model) + {} + + std::vector> parameters() override + { + auto params = self_attn_.parameters(); + auto n1 = norm1_.parameters(); + auto f = ff_.parameters(); + auto n2 = norm2_.parameters(); + params.insert(params.end(), n1.begin(), n1.end()); + params.insert(params.end(), f.begin(), f.end()); + params.insert(params.end(), n2.begin(), n2.end()); + return params; + } + + std::vector> param_gradients() override + { + auto grads = self_attn_.param_gradients(); + auto gn1 = norm1_.param_gradients(); + auto gf = ff_.param_gradients(); + auto gn2 = norm2_.param_gradients(); + grads.insert(grads.end(), gn1.begin(), gn1.end()); + grads.insert(grads.end(), gf.begin(), gf.end()); + grads.insert(grads.end(), gn2.begin(), gn2.end()); + return grads; + } + + Matrix forward(const Matrix &input) override + { + // ๅญๅฑ‚1: Self-Attention + ๆฎ‹ๅทฎ + residual1_cache_ = input; + auto norm1_out = norm1_.forward(input); + auto attn_out = self_attn_.forward(norm1_out); + residual2_cache_ = input + attn_out; // ๆฎ‹ๅทฎ่ฟžๆŽฅ + + // ๅญๅฑ‚2: FFN + ๆฎ‹ๅทฎ + auto norm2_out = norm2_.forward(residual2_cache_); + auto ff_out = ff_.forward(norm2_out); + return residual2_cache_ + ff_out; + } + + Matrix backward(const Matrix &grad_output) override + { + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ็ฌฌ2ไธชๆฎ‹ๅทฎ่ฟžๆŽฅ: grad_r2 = grad_output (ๅˆ†ๆตๅˆฐไธคๆก่ทฏๅพ„) โ”€โ”€ + // ่ทฏๅพ„A: ็›ดๆŽฅๆตๅ‘ residual2 (= residual1) + // ่ทฏๅพ„B: ๆตๅ‘ FFN + Matrix grad_residual1 = grad_output; // ่ทฏๅพ„A + Matrix grad_ff_out = grad_output; // ่ทฏๅพ„B + + // ๅๅ‘ FFN โ†’ LayerNormโ‚‚ + auto grad_norm2 = ff_.backward(grad_ff_out); + grad_residual1 = grad_residual1 + norm2_.backward(grad_norm2); + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ็ฌฌ1ไธชๆฎ‹ๅทฎ่ฟžๆŽฅ โ”€โ”€ + // ่ทฏๅพ„A: ็›ดๆŽฅๆตๅ‘่พ“ๅ…ฅ x + // ่ทฏๅพ„B: ๆตๅ‘ Self-Attention + Matrix grad_input = grad_residual1; // ่ทฏๅพ„A + Matrix grad_attn_out = grad_residual1; // ่ทฏๅพ„B + + // ๅๅ‘ Self-Attention โ†’ LayerNormโ‚ + auto grad_norm1 = self_attn_.backward(grad_attn_out); + grad_input = grad_input + norm1_.backward(grad_norm1); + + return grad_input; + } + }; + + // โ”€โ”€ Transformer ็ผ–็ ๅ™จ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๅ †ๅ  N ไธช TransformerEncoderLayer๏ผŒๅ†…ๅซไฝ็ฝฎ็ผ–็  + ๅ…จๅฑ€ๅนณๅ‡ๆฑ ๅŒ– + // ่พ“ๅ…ฅ: (d_model ร— num_patches, batch_size) โ€” PatchEmbedding ็š„่พ“ๅ‡บ + // ่พ“ๅ‡บ: (d_model, batch_size) โ€” ๆฑ ๅŒ–ๅŽ็š„ๅบๅˆ—่กจ็คบ + // + // ๅๅ‘ไผ ๆ’ญ็ญ–็•ฅ: ๅฏนๆฏไธชๆ ทๆœฌ re-forward ไปฅ้‡ๅปบๅ†…้ƒจ็ผ“ๅญ˜ (checkpointing) + // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + class TransformerEncoder final : public Layer + { + private: + std::size_t d_model_; + std::size_t num_patches_; + + std::vector layers_; + PositionalEncoding pos_encoding_; + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ็ผ“ๅญ˜: ๆฏไธชๆ ทๆœฌๅœจไฝ็ฝฎ็ผ–็ ๅŽ็š„่พ“ๅ…ฅ๏ผˆไพ› re-forward ้‡ๅปบ๏ผ‰ โ”€โ”€ + std::vector stored_inputs_; // [sample] โ€” ๆฏไธชๆ ทๆœฌ PE ๅŽ็š„่พ“ๅ…ฅ + std::size_t batch_size_{0}; + + public: + TransformerEncoder(std::size_t d_model, std::size_t num_heads, + std::size_t d_ff, std::size_t num_layers, + std::size_t num_patches) + : d_model_(d_model), num_patches_(num_patches), + pos_encoding_(d_model, num_patches) + { + for (std::size_t i = 0; i < num_layers; ++i) + layers_.emplace_back(d_model, num_heads, d_ff); + } + + std::vector> parameters() override + { + std::vector> params; + for (auto &layer : layers_) + { + auto lp = layer.parameters(); + params.insert(params.end(), lp.begin(), lp.end()); + } + return params; + } + + std::vector> param_gradients() override + { + std::vector> grads; + for (auto &layer : layers_) + { + auto lg = layer.param_gradients(); + grads.insert(grads.end(), lg.begin(), lg.end()); + } + return grads; + } + + // โ”€โ”€ ๅ‰ๅ‘ไผ ๆ’ญ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix forward(const Matrix &input) override + { + batch_size_ = input.cols(); + stored_inputs_.resize(batch_size_); + + Matrix output(d_model_, batch_size_); + + for (std::size_t b = 0; b < batch_size_; ++b) + { + // ไปŽๅฑ•ๅนณๅ‘้‡ไธญๆๅ–ๆ ทๆœฌ b โ†’ (d_model, num_patches) + Matrix x(d_model_, num_patches_); + for (std::size_t r = 0; r < d_model_; ++r) + for (std::size_t c = 0; c < num_patches_; ++c) + x.set_value_unchecked(r, c, + input.at_unchecked(r * num_patches_ + c, b)); + + // ๆทปๅŠ ไฝ็ฝฎ็ผ–็  + x = pos_encoding_.forward(x); + + // ็ผ“ๅญ˜ PE ๅŽ็š„่พ“ๅ…ฅ๏ผŒไพ› backward ไธญ re-forward ้‡ๅปบ็ผ“ๅญ˜ + stored_inputs_[b] = x; + + // ไพๆฌก้€š่ฟ‡็ผ–็ ๅ™จๅฑ‚ + for (std::size_t l = 0; l < layers_.size(); ++l) + x = layers_[l].forward(x); + + // ๅ…จๅฑ€ๅนณๅ‡ๆฑ ๅŒ–: (d_model, num_patches) โ†’ (d_model,) + const double inv_n = 1.0 / static_cast(num_patches_); + for (std::size_t r = 0; r < d_model_; ++r) + { + double sum = 0.0; + for (std::size_t c = 0; c < num_patches_; ++c) + sum += x.at_unchecked(r, c); + output.set_value_unchecked(r, b, sum * inv_n); + } + } + return output; + } + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๅฏนๆฏไธชๆ ทๆœฌ: re-forward ้‡ๅปบ็ผ“ๅญ˜ โ†’ ๅๅ‘ไผ ๆ’ญ โ†’ ็ดฏๅŠ ๅ‚ๆ•ฐๆขฏๅบฆ + Matrix backward(const Matrix &grad_output) override + { + Matrix grad_input(d_model_ * num_patches_, batch_size_); + grad_input.zero(); + + for (std::size_t b = 0; b < batch_size_; ++b) + { + // โ”€โ”€ Re-forward: ้‡ๅปบ่ฏฅๆ ทๆœฌ็š„ๅ…จ้ƒจ็ผ“ๅญ˜ โ”€โ”€ + // stored_inputs_[b] ๅทฒๅซไฝ็ฝฎ็ผ–็ ๏ผŒๆ— ้œ€้‡ๅคๆ–ฝๅŠ  + Matrix x = stored_inputs_[b]; + for (std::size_t l = 0; l < layers_.size(); ++l) + x = layers_[l].forward(x); // ้‡ๅปบๆฏๅฑ‚็ผ“ๅญ˜ + + // โ”€โ”€ ๅ…จๅฑ€ๅนณๅ‡ๆฑ ๅŒ–ๆขฏๅบฆ: ๅฑ•ๅผ€ โ”€โ”€ + const double inv_n = 1.0 / static_cast(num_patches_); + Matrix grad(d_model_, num_patches_); + for (std::size_t r = 0; r < d_model_; ++r) + { + double g = grad_output.at_unchecked(r, b) * inv_n; + for (std::size_t c = 0; c < num_patches_; ++c) + grad.set_value_unchecked(r, c, g); + } + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ็ผ–็ ๅ™จๅฑ‚ โ”€โ”€ + for (int l = static_cast(layers_.size()) - 1; l >= 0; --l) + grad = layers_[l].backward(grad); + + // โ”€โ”€ ๅ†™ๅ…ฅ grad_input โ”€โ”€ + for (std::size_t r = 0; r < d_model_; ++r) + for (std::size_t c = 0; c < num_patches_; ++c) + grad_input.set_value_unchecked(r * num_patches_ + c, b, + grad.at_unchecked(r, c)); + } + return grad_input; + } + }; + + // โ”€โ”€ Patch ๅตŒๅ…ฅๅฑ‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๅฐ†ๅฑ•ๅนณ็š„ๅ›พๅƒๅˆ†ๅ‰ฒไธบไธ้‡ๅ ็š„ patch ๅนถๆŠ•ๅฝฑๅˆฐ d_model ็ปด็ฉบ้—ด + // ่พ“ๅ…ฅ: (img_sizeยฒ, batch_size) โ€” ๅฑ•ๅนณ็š„ 28ร—28 ๅ›พๅƒ + // ่พ“ๅ‡บ: (d_model ร— num_patches, batch_size) โ€” ๅฑ•ๅนณ็š„ patch ๅบๅˆ— + // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + class PatchEmbedding final : public Layer + { + private: + std::size_t img_size_; + std::size_t patch_size_; + std::size_t grid_size_; // img_size / patch_size + std::size_t num_patches_; // grid_sizeยฒ + std::size_t patch_dim_; // patch_sizeยฒ + std::size_t d_model_; + Linear projection_; // (patch_dim, d_model) + Matrix input_cache_; + + public: + PatchEmbedding(std::size_t img_size, std::size_t patch_size, std::size_t d_model) + : img_size_(img_size), patch_size_(patch_size), + grid_size_(img_size / patch_size), + num_patches_((img_size / patch_size) * (img_size / patch_size)), + patch_dim_(patch_size * patch_size), + d_model_(d_model), + projection_(patch_dim_, d_model) + { + if (img_size % patch_size != 0) + throw std::invalid_argument( + "PatchEmbedding: img_size must be divisible by patch_size"); + } + + [[nodiscard]] std::size_t num_patches() const noexcept { return num_patches_; } + [[nodiscard]] std::size_t d_model() const noexcept { return d_model_; } + + std::vector> parameters() override + { + return projection_.parameters(); + } + + std::vector> param_gradients() override + { + return projection_.param_gradients(); + } + + // โ”€โ”€ ๅ‰ๅ‘ไผ ๆ’ญ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๆ‰น้‡ๆๅ– patch ๅนถไธ€ๆฌกๆ€งๆŠ•ๅฝฑ + Matrix forward(const Matrix &input) override + { + input_cache_ = input; + const std::size_t batch = input.cols(); + + // Step 1: ๆๅ–ๆ‰€ๆœ‰ patch โ†’ (patch_dim, num_patches ร— batch) + // ๅธƒๅฑ€: patch p ็š„ sample b ไฝไบŽๅˆ— p ร— batch + b + Matrix all_patches(patch_dim_, num_patches_ * batch); + + for (std::size_t b = 0; b < batch; ++b) + { + for (std::size_t p = 0; p < num_patches_; ++p) + { + const std::size_t gr = (p / grid_size_) * patch_size_; + const std::size_t gc = (p % grid_size_) * patch_size_; + const std::size_t col_idx = p * batch + b; + + for (std::size_t pr = 0; pr < patch_size_; ++pr) + { + for (std::size_t pc = 0; pc < patch_size_; ++pc) + { + const std::size_t flat = pr * patch_size_ + pc; + const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); + all_patches.set_value_unchecked(flat, col_idx, + input.at_unchecked(pix, b)); + } + } + } + } + + // Step 2: ๆŠ•ๅฝฑ โ†’ (d_model, num_patches ร— batch) + Matrix projected = projection_.forward(all_patches); + + // Step 3: ้‡ๆŽ’ไธบ (d_model ร— num_patches, batch) + Matrix output(d_model_ * num_patches_, batch); + for (std::size_t r = 0; r < d_model_; ++r) + for (std::size_t p = 0; p < num_patches_; ++p) + for (std::size_t b = 0; b < batch; ++b) + output.set_value_unchecked(r * num_patches_ + p, b, + projected.at_unchecked(r, p * batch + b)); + + return output; + } + + // โ”€โ”€ ๅๅ‘ไผ ๆ’ญ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + Matrix backward(const Matrix &grad_output) override + { + const std::size_t batch = grad_output.cols(); + + // Step 1: ้‡ๆŽ’ๆขฏๅบฆ โ†’ (d_model, num_patches ร— batch) + Matrix grad_projected(d_model_, num_patches_ * batch); + for (std::size_t r = 0; r < d_model_; ++r) + for (std::size_t p = 0; p < num_patches_; ++p) + for (std::size_t b = 0; b < batch; ++b) + grad_projected.set_value_unchecked(r, p * batch + b, + grad_output.at_unchecked(r * num_patches_ + p, b)); + + // Step 2: ๆŠ•ๅฝฑๅฑ‚ๅๅ‘ โ†’ (patch_dim, num_patches ร— batch) + Matrix grad_patches = projection_.backward(grad_projected); + + // Step 3: ๆ•ฃๅฐ„ๆขฏๅบฆๅ›ž่พ“ๅ…ฅ โ†’ (img_sizeยฒ, batch) + Matrix grad_input(img_size_ * img_size_, batch); + grad_input.zero(); + + for (std::size_t b = 0; b < batch; ++b) + { + for (std::size_t p = 0; p < num_patches_; ++p) + { + const std::size_t gr = (p / grid_size_) * patch_size_; + const std::size_t gc = (p % grid_size_) * patch_size_; + const std::size_t col_idx = p * batch + b; + + for (std::size_t pr = 0; pr < patch_size_; ++pr) + { + for (std::size_t pc = 0; pc < patch_size_; ++pc) + { + const std::size_t flat = pr * patch_size_ + pc; + const std::size_t pix = (gr + pr) * img_size_ + (gc + pc); + const double val = grad_input.at_unchecked(pix, b) + + grad_patches.at_unchecked(flat, col_idx); + grad_input.set_value_unchecked(pix, b, val); + } + } + } } return grad_input; } diff --git a/include/neuralnet.cpp/loss.hpp b/include/neuralnet.cpp/loss.hpp index dba3fdd3..bd31bc09 100644 --- a/include/neuralnet.cpp/loss.hpp +++ b/include/neuralnet.cpp/loss.hpp @@ -2,12 +2,14 @@ #define LOSS_HPP #include +#include #include +#include #include #include -#include -#include +#include "nn_config.hpp" +#include "matrix.hpp" namespace nn { @@ -99,7 +101,13 @@ namespace nn // ่ถ…่ฟ‡ 128 ็ฑปๆ—ถๆ”น็”จ vector๏ผˆ้ž็ƒญ่ทฏๅพ„๏ผŒๅผ€้”€ๅฏๆŽฅๅ—๏ผ‰ std::array exp_vals_fixed{}; std::vector exp_vals_heap; - double* exp_vals = (classes <= 128) ? exp_vals_fixed.data() : (exp_vals_heap.resize(classes), exp_vals_heap.data()); + std::span exp_vals; + if (classes <= 128) { + exp_vals = exp_vals_fixed; + } else { + exp_vals_heap.resize(classes); + exp_vals = exp_vals_heap; + } double sum_exp = 0.0; for (std::size_t c = 0; c < classes; ++c) diff --git a/include/neuralnet.cpp/matrix.hpp b/include/neuralnet.cpp/matrix.hpp index 7abfbad8..ff47cd44 100644 --- a/include/neuralnet.cpp/matrix.hpp +++ b/include/neuralnet.cpp/matrix.hpp @@ -10,11 +10,12 @@ #include #include #include +#include #include #include #include -#include +#include "nn_config.hpp" namespace nn { @@ -71,8 +72,15 @@ namespace nn data_.resize(rows * cols); } - // โ”€โ”€ ๅŽŸๅง‹ๆŒ‡้’ˆ่ฎฟ้—ฎ๏ผˆไพ› SIMD / ๅ†…ๆ ธไฝฟ็”จ๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // โ”€โ”€ std::span ่ฎฟ้—ฎ๏ผˆC++20 ็ŽฐไปฃๆŽฅๅฃ๏ผŒๆŽจ่ไฝฟ็”จ๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ้›ถๅผ€้”€ๆŠฝ่ฑก๏ผš็ผ–่ฏ‘ๅŽ็ญ‰ไปทไบŽ่ฃธๆŒ‡้’ˆ + ๅคงๅฐ๏ผŒๅฏๆ›ฟไปฃๆ‰€ๆœ‰ data_ptr() ๅœบๆ™ฏ + [[nodiscard]] std::span span() const noexcept { return {data_.data(), data_.size()}; } + [[nodiscard]] std::span span() noexcept { return {data_.data(), data_.size()}; } + + // โ”€โ”€ ๅŽŸๅง‹ๆŒ‡้’ˆ่ฎฟ้—ฎ๏ผˆไป…ไพ›้œ€่ฆๆŒ‡้’ˆ็ฎ—ๆœฏ็š„ๅˆ†ๅ—็ฎ—ๆณ•ๅ†…้ƒจไฝฟ็”จ๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + [[deprecated("use span() instead for safer access")]] [[nodiscard]] const double *data_ptr() const noexcept { return data_.data(); } + [[deprecated("use span() instead for safer access")]] [[nodiscard]] double *data_ptr() noexcept { return data_.data(); } // ่ฎฟ้—ฎๅ™จ @@ -160,29 +168,42 @@ namespace nn result.resize(cols_, rows_); if (rows_ == 0 || cols_ == 0) return; - const auto *src = data_ptr(); - auto *dst = result.data_ptr(); + const auto src = span(); + auto dst = result.span(); const std::size_t R = rows_; const std::size_t C = cols_; const std::size_t i_blocks = (R + BLOCK_SIZE - 1) / BLOCK_SIZE; const std::size_t j_blocks = (C + BLOCK_SIZE - 1) / BLOCK_SIZE; - auto block_indices = std::views::iota(std::size_t{0}, i_blocks * j_blocks); - - SmartPolicy::for_each(block_indices.begin(), block_indices.end(), - [src, dst, R, C, j_blocks](std::size_t block_idx) noexcept - { - const std::size_t ib = block_idx / j_blocks; - const std::size_t jb = block_idx % j_blocks; - const std::size_t i0 = ib * BLOCK_SIZE; - const std::size_t j0 = jb * BLOCK_SIZE; - const std::size_t i1 = std::min(i0 + BLOCK_SIZE, R); - const std::size_t j1 = std::min(j0 + BLOCK_SIZE, C); - for (std::size_t i = i0; i < i1; ++i) - for (std::size_t j = j0; j < j1; ++j) - dst[j * R + i] = src[i * C + j]; - }); + // ็ป•่ฟ‡ SmartPolicy๏ผšๅˆ†ๅ—ๆ•ฐๅพˆๅฐ‘๏ผˆ1~ๅ‡ ๅ๏ผ‰๏ผŒไฝ†ๅ…ถๅ†…ๅพช็Žฏ่ฎก็ฎ—้‡ๅคงใ€‚ + // SmartPolicy ๆŒ‰ๅ…ƒ็ด ๆ•ฐๅˆคๆ–ญไผš้”™่ฏฏๅœฐๅฐ†ๅˆ†ๅ—็บงๅˆซ็š„ๅนถ่กŒ้€€ๅŒ–ไธบไธฒ่กŒใ€‚ + // ่ฟ™้‡Œ็›ดๆŽฅ่ฐƒ็”จ็บฟ็จ‹ๆฑ ๏ผˆ>1 ๅ—ๆ—ถๅนถ่กŒ๏ผ‰๏ผŒ้›ถๅˆ†้… latch ่ฎพ่ฎกๅผ€้”€ๆžไฝŽใ€‚ + const auto n_blocks = i_blocks * j_blocks; + if (n_blocks <= 1) + { + for (std::size_t i = 0; i < R; ++i) + for (std::size_t j = 0; j < C; ++j) + dst[j * R + i] = src[i * C + j]; + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + global_thread_pool().parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [src, dst, R, C, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t ib = block_idx / j_blocks; + const std::size_t jb = block_idx % j_blocks; + const std::size_t i0 = ib * BLOCK_SIZE; + const std::size_t j0 = jb * BLOCK_SIZE; + const std::size_t i1 = std::min(i0 + BLOCK_SIZE, R); + const std::size_t j1 = std::min(j0 + BLOCK_SIZE, C); + for (std::size_t i = i0; i < i1; ++i) + for (std::size_t j = j0; j < j1; ++j) + dst[j * R + i] = src[i * C + j]; + }); + } } [[nodiscard]] Matrix operator+(const Matrix &other) const @@ -227,7 +248,7 @@ namespace nn } // โ”€โ”€ ็Ÿฉ้˜ตไน˜ๆณ•ๅˆฐ้ข„ๅˆ†้…็ผ“ๅ†ฒๅŒบ๏ผˆ้›ถๅˆ†้…็ƒญ่ทฏๅพ„๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - // ไฝฟ็”จๅŽŸๅง‹ๆŒ‡้’ˆ + restrict ๆ็คบ๏ผŒๅธฎๅŠฉ็ผ–่ฏ‘ๅ™จ่‡ชๅŠจๅ‘้‡ๅŒ– + // ไฝฟ็”จ std::span๏ผˆC++20๏ผ‰ๆไพ›็ฑปๅž‹ๅฎ‰ๅ…จ็š„้žๆ‹ฅๆœ‰่ง†ๅ›พ void multiply_to(Matrix &result, const Matrix &other) const { assert(&result != this && "multiply_to: self-referencing not supported"); @@ -237,55 +258,90 @@ namespace nn result.resize(M, N); if (M == 0 || N == 0 || K == 0) return; - // ๆธ…้›ถ็ป“ๆžœ็Ÿฉ้˜ต - auto *r_ptr = result.data_ptr(); - std::fill(r_ptr, r_ptr + M * N, 0.0); + // ๆธ…้›ถ็ป“ๆžœ็Ÿฉ้˜ต๏ผˆไฝฟ็”จ RAII ๅฐ่ฃ…็š„ๆ–นๆณ•๏ผ‰ + result.zero(); - const double *a_ptr = data_ptr(); - const double *b_ptr = other.data_ptr(); + // std::span ้žๆ‹ฅๆœ‰่ง†ๅ›พ๏ผšไปŽ std::vector ๅ€Ÿๅ‡บ๏ผŒ็”Ÿๅ‘ฝๅ‘จๆœŸ็”ฑ่ฐƒ็”จๆ ˆไฟ่ฏ + const auto a = span(); + const auto b = other.span(); + auto r = result.span(); const std::size_t i_blocks = (M + BLOCK_SIZE - 1) / BLOCK_SIZE; const std::size_t j_blocks = (N + BLOCK_SIZE - 1) / BLOCK_SIZE; - auto block_indices = std::views::iota(std::size_t{0}, i_blocks * j_blocks); - SmartPolicy::for_each(block_indices.begin(), block_indices.end(), - [a_ptr, b_ptr, r_ptr, M, N, K, j_blocks](std::size_t block_idx) noexcept - { - const std::size_t i_block = block_idx / j_blocks; - const std::size_t j_block = block_idx % j_blocks; - const std::size_t i_start = i_block * BLOCK_SIZE; - const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); - const std::size_t j_start = j_block * BLOCK_SIZE; - const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); - - for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) - { - const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); - const std::size_t k_len = k_end - k_start; - const std::size_t j_len = j_end - j_start; - - // ๅŠ ่ฝฝ B ็š„ๅญๅ—ๅˆฐๆ ˆๆ•ฐ็ป„ๅนถ่ฝฌ็ฝฎ๏ผŒๆ”นๅ–„่ฎฟ้—ฎๅฑ€้ƒจๆ€ง - std::array b_block{}; - for (std::size_t jj = 0; jj < j_len; ++jj) - for (std::size_t kk = 0; kk < k_len; ++kk) - b_block[jj * k_len + kk] = b_ptr[(k_start + kk) * N + (j_start + jj)]; - - // ็ดฏๅŠ ๅฝ“ๅ‰ K ๅ—ๅฏน C ๅ—็š„่ดก็Œฎ - for (std::size_t i = i_start; i < i_end; ++i) - { - const double *a_row = a_ptr + i * K + k_start; - double *r_row = r_ptr + i * N + j_start; - for (std::size_t j = 0; j < j_len; ++j) - { - const double *b_col = b_block.data() + j * k_len; - double sum = 0.0; - for (std::size_t kk = 0; kk < k_len; ++kk) - sum += a_row[kk] * b_col[kk]; - r_row[j] += sum; - } - } - } - }); + // ็ป•่ฟ‡ SmartPolicy๏ผšๅˆ†ๅ—ๆ•ฐๅฏ่ƒฝๅพˆๅฐ‘ไฝ†ๅ…ถๅ†…ๅพช็Žฏ่ฎก็ฎ—้‡ๅทจๅคง๏ผˆไธ‰ๅฑ‚ๅพช็Žฏ O(nยณ)๏ผ‰ใ€‚ + // ็›ดๆŽฅ่ฐƒ็”จ็บฟ็จ‹ๆฑ ๏ผŒ้›ถๅˆ†้… latch ๅผ€้”€ๆžไฝŽใ€‚ + const auto n_blocks = i_blocks * j_blocks; + if (n_blocks <= 1) + { + // ๅ•ๅ—ไธฒ่กŒโ€”โ€”ไธŽไธ‹ๆ–น lambda ้€ป่พ‘็›ธๅŒ๏ผŒไฝ†้ฟๅ…ไธๅฟ…่ฆ็š„็บฟ็จ‹ๆฑ ่ฐƒ็”จ + const std::size_t i_start = 0, i_end = M; + const std::size_t j_start = 0, j_end = N; + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const double *b_col = b_block.data() + j * k_len; + double sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + } + else + { + auto block_indices = std::views::iota(std::size_t{0}, n_blocks); + global_thread_pool().parallel_for_blocks( + block_indices.begin(), block_indices.end(), + [a, b, r, M, N, K, j_blocks](std::size_t block_idx) noexcept + { + const std::size_t i_block = block_idx / j_blocks; + const std::size_t j_block = block_idx % j_blocks; + const std::size_t i_start = i_block * BLOCK_SIZE; + const std::size_t i_end = std::min(i_start + BLOCK_SIZE, M); + const std::size_t j_start = j_block * BLOCK_SIZE; + const std::size_t j_end = std::min(j_start + BLOCK_SIZE, N); + + for (std::size_t k_start = 0; k_start < K; k_start += BLOCK_SIZE) + { + const std::size_t k_end = std::min(k_start + BLOCK_SIZE, K); + const std::size_t k_len = k_end - k_start; + const std::size_t j_len = j_end - j_start; + + std::array b_block{}; + for (std::size_t jj = 0; jj < j_len; ++jj) + for (std::size_t kk = 0; kk < k_len; ++kk) + b_block[jj * k_len + kk] = b[(k_start + kk) * N + (j_start + jj)]; + + for (std::size_t i = i_start; i < i_end; ++i) + { + const auto a_row = a.subspan(i * K + k_start); + auto r_row = r.subspan(i * N + j_start); + for (std::size_t j = 0; j < j_len; ++j) + { + const double *b_col = b_block.data() + j * k_len; + double sum = 0.0; + for (std::size_t kk = 0; kk < k_len; ++kk) + sum += a_row[kk] * b_col[kk]; + r_row[j] += sum; + } + } + } + }); + } } void scale_inplace(double scalar) noexcept diff --git a/include/neuralnet.cpp/mnist_common.hpp b/include/neuralnet.cpp/mnist_common.hpp new file mode 100644 index 00000000..50b06c57 --- /dev/null +++ b/include/neuralnet.cpp/mnist_common.hpp @@ -0,0 +1,96 @@ +#ifndef MNIST_COMMON_HPP +#define MNIST_COMMON_HPP + +#include +#include +#include +#include + +#include "model.hpp" + +namespace nn { + +// โ”€โ”€ MNIST ๅธธ้‡ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +inline constexpr std::size_t MNIST_INPUT_DIM = 784; +inline constexpr std::size_t MNIST_NUM_CLASSES = 10; + +// ้ป˜่ฎค MLP ็ฝ‘็ปœๆžถๆž„๏ผš่พ“ๅ…ฅๅฑ‚ -> ้š่—ๅฑ‚1 -> ้š่—ๅฑ‚2 -> ้š่—ๅฑ‚3 -> ่พ“ๅ‡บๅฑ‚ +inline const std::vector MNIST_LAYER_DIMS = { + MNIST_INPUT_DIM, 512, 256, 128, 64, MNIST_NUM_CLASSES +}; + +// ้ป˜่ฎค Transformer ่ถ…ๅ‚ๆ•ฐ +inline constexpr std::size_t TRANSFORMER_D_MODEL = 64; +inline constexpr std::size_t TRANSFORMER_NUM_HEADS = 4; +inline constexpr std::size_t TRANSFORMER_D_FF = 128; +inline constexpr std::size_t TRANSFORMER_NUM_LAYERS = 2; +inline constexpr std::size_t TRANSFORMER_PATCH_SIZE = 7; // 28 / 7 = 4 โ†’ 4ร—4 = 16 patches + +// โ”€โ”€ ๆž„ๅปบ MLP ๆจกๅž‹ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// ๆ นๆฎ MNIST_LAYER_DIMS ่‡ชๅŠจๆž„ๅปบ็ฝ‘็ปœ (Linear + LayerNorm + GeLU) +[[nodiscard]] inline Model build_mnist_mlp_model() +{ + Model model; + for (std::size_t i = 0; i < MNIST_LAYER_DIMS.size() - 1; ++i) + { + std::size_t in_dim = MNIST_LAYER_DIMS[i]; + std::size_t out_dim = MNIST_LAYER_DIMS[i + 1]; + + model.add(in_dim, out_dim); + + if (i < MNIST_LAYER_DIMS.size() - 2) + { + model.add(out_dim) + .add(); + } + } + return model; +} + +// โ”€โ”€ ๆž„ๅปบ Transformer ๆจกๅž‹ (ViT-like) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// ๆžถๆž„: PatchEmbedding โ†’ TransformerEncoder โ†’ Linear(ๅˆ†็ฑปๅคด) +// +// ่พ“ๅ…ฅ: (784, batch) โ€” ๅฑ•ๅนณ็š„ 28ร—28 ๅ›พๅƒ +// PatchEmbedding: 28ร—28 โ†’ 16 ไธช 7ร—7 patch โ†’ ๆŠ•ๅฝฑๅˆฐ d_model +// TransformerEncoder: N ๅฑ‚ Self-Attention + FFN (Pre-Norm) + ไฝ็ฝฎ็ผ–็  + ๆฑ ๅŒ– +// ๅˆ†็ฑปๅคด: Linear(d_model โ†’ 10) +// +// ไผ ๆ’ญๆต็จ‹: +// (784, batch) +// โ†’ PatchEmbedding: (d_modelร—16, batch) +// โ†’ TransformerEncoder: (d_model, batch) +// โ†’ Linear: (10, batch) +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +[[nodiscard]] inline Model build_mnist_transformer_model( + std::size_t d_model = TRANSFORMER_D_MODEL, + std::size_t num_heads = TRANSFORMER_NUM_HEADS, + std::size_t d_ff = TRANSFORMER_D_FF, + std::size_t num_layers = TRANSFORMER_NUM_LAYERS, + std::size_t patch_size = TRANSFORMER_PATCH_SIZE) +{ + constexpr std::size_t img_size = 28; + const std::size_t num_patches = (img_size / patch_size) * (img_size / patch_size); + + Model model; + model.add(img_size, patch_size, d_model) + .add(d_model, num_heads, d_ff, num_layers, num_patches) + .add(d_model, MNIST_NUM_CLASSES); + return model; +} + +// โ”€โ”€ ็ปŸไธ€ๆž„ๅปบๅ…ฅๅฃ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// model_type: "mlp" ๆˆ– "transformer" +[[nodiscard]] inline Model build_mnist_model(const std::string &model_type = "mlp") +{ + if (model_type == "mlp") + return build_mnist_mlp_model(); + else if (model_type == "transformer") + return build_mnist_transformer_model(); + else + throw std::invalid_argument("Unknown model type: " + model_type + + " (available: mlp, transformer)"); +} + +} // namespace nn + +#endif // MNIST_COMMON_HPP \ No newline at end of file diff --git a/include/neuralnet.cpp/model.hpp b/include/neuralnet.cpp/model.hpp index 0663aff9..b66a3abe 100644 --- a/include/neuralnet.cpp/model.hpp +++ b/include/neuralnet.cpp/model.hpp @@ -6,8 +6,8 @@ #include #include -#include -#include +#include "nn_config.hpp" +#include "layer.hpp" namespace nn { diff --git a/include/neuralnet.cpp/model_io.hpp b/include/neuralnet.cpp/model_io.hpp index 7f6338d1..4aa5ec27 100644 --- a/include/neuralnet.cpp/model_io.hpp +++ b/include/neuralnet.cpp/model_io.hpp @@ -7,9 +7,9 @@ #include #include -#include -#include -#include +#include "nn_config.hpp" +#include "layer.hpp" +#include "model.hpp" namespace nn { diff --git a/include/neuralnet.cpp/nn.hpp b/include/neuralnet.cpp/nn.hpp index b68938e0..38f284d3 100644 --- a/include/neuralnet.cpp/nn.hpp +++ b/include/neuralnet.cpp/nn.hpp @@ -5,11 +5,12 @@ #include #include -#include -#include -#include -#include -#include +#include "nn_config.hpp" +#include "layer.hpp" +#include "loss.hpp" +#include "model.hpp" +#include "optimizer.hpp" +#include "mnist_common.hpp" namespace nn { diff --git a/include/neuralnet.cpp/nn_config.hpp b/include/neuralnet.cpp/nn_config.hpp index 116f1f2a..d464b70c 100644 --- a/include/neuralnet.cpp/nn_config.hpp +++ b/include/neuralnet.cpp/nn_config.hpp @@ -26,15 +26,19 @@ namespace nn inline constexpr double EPSILON = 1e-8; // โ”€โ”€ ๆ™บ่ƒฝๆ‰ง่กŒ็ญ–็•ฅ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - // ๆ นๆฎๅ…ƒ็ด ๆ•ฐ้‡่‡ชๅŠจ้€‰ๆ‹ฉไธฒ่กŒๆˆ–ไฝฟ็”จๅ…จๅฑ€็บฟ็จ‹ๆฑ ๅนถ่กŒ๏ผŒ้ฟๅ…ๅฐ็Ÿฉ้˜ต็š„็บฟ็จ‹ๆฑ ่ฐƒๅบฆๅผ€้”€ - // ็บฟ็จ‹ๆฑ ๅœจ้ฆ–ๆฌกไฝฟ็”จๆ—ถๆ‡’ๅˆๅง‹ๅŒ–๏ผŒ่ฎญ็ปƒๆœŸ้—ดๅธธ้ฉป๏ผŒ้ฟๅ…ๅๅคๅˆ›ๅปบ/้”€ๆฏ + // ๆ นๆฎๅ…ƒ็ด ๆ•ฐ้‡่‡ชๅŠจ้€‰ๆ‹ฉไธฒ่กŒๆˆ–ไฝฟ็”จๅ…จๅฑ€็บฟ็จ‹ๆฑ ๅนถ่กŒใ€‚ + // ็บฟ็จ‹ๆฑ ้‡‡็”จ latch + ่ฐƒ็”จ่€…ๅ‚ไธŽ่ฎพ่ฎก๏ผšไธ€ๆฌกๆ‰น้‡ๅ…ฅ้˜Ÿ + ๅŽŸๅญ่ฎกๆ•ฐๅ™จ๏ผŒ + // ๆถˆ้™คไบ†ๆ—ง็‰ˆ Nร—submit + Nร—future ็š„ๅ †ๅˆ†้…ๅ’ŒๅŒๆญฅๅผ€้”€ใ€‚ struct SmartPolicy { - static constexpr long long PARALLEL_THRESHOLD = 100000; // 100Kๅ…ƒ็ด ๏ผˆๅฎžๆต‹ๆœ€ไผ˜๏ผ‰ + // ้˜ˆๅ€ผ = ๆฏๆ ธๆœ€ๅฐๅˆ†ๅ— ร— 2๏ผŒ็กฎไฟ่‡ณๅฐ‘ๆœ‰ 2 ไธชๅˆ†ๅ—ๅฏๅนถ่กŒใ€‚ + // chunk_count() ๅทฒ่‡ชๅŠจๆŒ‰ๆ ธๅฟƒๆ•ฐ็ผฉๆ”พ๏ผˆmin(cores, total/4096)๏ผ‰๏ผŒ + // ้˜ˆๅ€ผๅช้œ€่ฎพๆœ€ไฝŽ้—จๆง›๏ผŒๆ ธๅฟƒๅˆฉ็”จ้šๆ•ฐๆฎ้‡่‡ช็„ถๅขž้•ฟใ€‚ + inline static const std::size_t PARALLEL_THRESHOLD = 4096 * 2; // 8192 // for_each ็‰ˆๆœฌ template static void for_each(Iterator first, Iterator last, Func&& func) { - const auto n = static_cast(std::distance(first, last)); + const auto n = static_cast(std::distance(first, last)); if (n >= PARALLEL_THRESHOLD) { global_thread_pool().parallel_for_each(first, last, std::forward(func)); } else { @@ -46,7 +50,7 @@ namespace nn // transform ็‰ˆๆœฌ๏ผˆไธ€ๅ…ƒ๏ผ‰ template static void transform(InputIt first, InputIt last, OutputIt d_first, UnaryOp&& op) { - const auto n = static_cast(std::distance(first, last)); + const auto n = static_cast(std::distance(first, last)); if (n >= PARALLEL_THRESHOLD) { global_thread_pool().parallel_transform(first, last, d_first, std::forward(op)); } else { @@ -57,7 +61,7 @@ namespace nn // transform ็‰ˆๆœฌ๏ผˆไบŒๅ…ƒ๏ผ‰ template static void transform(InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp&& op) { - const auto n = static_cast(std::distance(first1, last1)); + const auto n = static_cast(std::distance(first1, last1)); if (n >= PARALLEL_THRESHOLD) { global_thread_pool().parallel_transform(first1, last1, first2, d_first, std::forward(op)); } else { @@ -68,10 +72,10 @@ namespace nn // transform_reduce ็‰ˆๆœฌ๏ผˆไธ€ๅ…ƒ transform๏ผ‰ template static T transform_reduce(InputIt first, InputIt last, T init, BinaryOp&& reduce_op, UnaryOp&& transform_op) { - const auto n = static_cast(std::distance(first, last)); + const auto n = static_cast(std::distance(first, last)); if (n >= PARALLEL_THRESHOLD) { return global_thread_pool().parallel_transform_reduce( - first, last, first, init, + first, last, init, std::forward(reduce_op), std::forward(transform_op)); } else { return std::transform_reduce(std::execution::seq, first, last, init, @@ -82,7 +86,7 @@ namespace nn // transform_reduce ็‰ˆๆœฌ๏ผˆไบŒๅ…ƒ transform๏ผŒไธคไธช่พ“ๅ…ฅ่Œƒๅ›ด๏ผ‰ template static T transform_reduce(InputIt1 first1, InputIt1 last1, InputIt2 first2, T init, BinaryOp&& reduce_op, UnaryOp&& transform_op) { - const auto n = static_cast(std::distance(first1, last1)); + const auto n = static_cast(std::distance(first1, last1)); if (n >= PARALLEL_THRESHOLD) { return global_thread_pool().parallel_transform_reduce(first1, last1, first2, init, std::forward(reduce_op), diff --git a/include/neuralnet.cpp/optimizer.hpp b/include/neuralnet.cpp/optimizer.hpp index 1e8e9b38..f5021a3b 100644 --- a/include/neuralnet.cpp/optimizer.hpp +++ b/include/neuralnet.cpp/optimizer.hpp @@ -5,7 +5,7 @@ #include #include -#include +#include "matrix.hpp" namespace nn { diff --git a/include/neuralnet.cpp/thread_pool.hpp b/include/neuralnet.cpp/thread_pool.hpp index c6aa515b..9b330382 100644 --- a/include/neuralnet.cpp/thread_pool.hpp +++ b/include/neuralnet.cpp/thread_pool.hpp @@ -9,15 +9,18 @@ #include #include #include -#include // for std::distance +#include // for std::transform (serial fallback) +#include // for std::transform_reduce (serial fallback) +#include // for std::distance #include #include namespace nn { - // โ”€โ”€ ็ฎ€ๆ˜“็บฟ็จ‹ๆฑ  โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - // ่ฎญ็ปƒๅผ€ๅง‹ๆ—ถๅˆ›ๅปบๅ›บๅฎšๆ•ฐ้‡็บฟ็จ‹๏ผŒ้ฟๅ…ๅๅคๅˆ›ๅปบ/้”€ๆฏ็š„่ฐƒๅบฆๅผ€้”€ - // ๆ”ฏๆŒๆไบคไปปๆ„ๅฏ่ฐƒ็”จๅฏน่ฑก๏ผŒ่ฟ”ๅ›ž std::future ไปฅ่Žทๅ–็ป“ๆžœ + // โ”€โ”€ ็ฎ€ๆ˜“็บฟ็จ‹ๆฑ ๏ผˆlatch ้›ถๅˆ†้…่ฎพ่ฎก๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๆ ธๅฟƒๆ”น่ฟ›๏ผšๅฐ†ๅŽŸๆฅ็š„ "N ๆฌก submit + N ไธช future + N ๆฌกๅŠ ้”" ๆ›ฟๆขไธบ + // "1 ๆฌกๆ‰น้‡ๅ…ฅ้˜Ÿ + 1 ไธชๅŽŸๅญ่ฎกๆ•ฐๅ™จ + ่ฐƒ็”จ่€…ๅ‚ไธŽๅค„็†" + // ๆถˆ้™คๆฏๅˆ†ๅ—ไธ€ๆฌก shared_ptr ๅ †ๅˆ†้…ๅ’Œ future ๅŒๆญฅๅผ€้”€ class ThreadPool { private: @@ -29,7 +32,6 @@ namespace nn std::atomic stop_{false}; public: - // ๆž„้€ ๏ผšๅฏๅŠจๆŒ‡ๅฎšๆ•ฐ้‡็š„ๅทฅไฝœ็บฟ็จ‹ explicit ThreadPool(std::size_t num_threads = std::thread::hardware_concurrency()) { if (num_threads == 0) num_threads = 1; @@ -58,7 +60,7 @@ namespace nn } } - // ๆไบคไปปๅŠก๏ผŒ่ฟ”ๅ›ž future + // โ”€โ”€ ้€š็”จๅ•ไปปๅŠกๆไบค๏ผˆไฟ็•™็ป™้žๆ€ง่ƒฝๆ•ๆ„Ÿ่ทฏๅพ„๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ template [[nodiscard]] auto submit(F&& f, Args&&... args) -> std::future> @@ -74,102 +76,224 @@ namespace nn std::unique_lock lock(queue_mutex_); if (stop_.load(std::memory_order_acquire)) throw std::runtime_error("submit on stopped ThreadPool"); - tasks_.emplace([task]() - { - (*task)(); - }); + tasks_.emplace([task]() { (*task)(); }); } condition_.notify_one(); return result; } - // ๅนถ่กŒ for_each๏ผšๅฐ† [first, last) ๅ‡ๅŒ€ๅˆ†ๅ—๏ผŒๆฏๅ—ๆไบคไธ€ไธชไปปๅŠก + private: + // โ”€โ”€ ๅˆ†ๅ—่พ…ๅŠฉ๏ผš่ฎก็ฎ—ๅˆ็†็š„ๅˆ†ๅ—ๆ•ฐ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ๆฏๅˆ†ๅ—่‡ณๅฐ‘ 4096 ไธช double๏ผˆ32 KB๏ผ‰๏ผŒๅฏ่ฃ…ๅ…ฅ L1 ็ผ“ๅญ˜ใ€‚ + // 256โ†’4096๏ผšๆ—งๅ€ผๅฏผ่‡ด 8192 ๅ…ƒ็ด ่ขซๅˆ‡ๆˆ 16 ไธชๅพฎๅ—๏ผŒ่ฐƒๅบฆๅผ€้”€่ฟœ่ถ…่ฎก็ฎ—ใ€‚ + [[nodiscard]] std::size_t chunk_count(std::size_t total) const noexcept + { + constexpr std::size_t MIN_CHUNK = 4096; + const auto nw = workers_.size(); + if (nw <= 1 || total < MIN_CHUNK * 2) + return 1; + auto n = std::min(nw, total / MIN_CHUNK); + return n < 1 ? 1 : n; + } + + // โ”€โ”€ work-stealing ็ญ‰ๅพ…๏ผš่ฐƒ็”จ่€…ไธ็ฉบ่ฝฌ๏ผŒๅธฎๅฟ™ๅค„็†้˜Ÿๅˆ—ไปปๅŠก โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + void wait_for_latch(std::atomic& latch) + { + while (latch.load(std::memory_order_acquire) > 0) + { + std::function task; + { + std::lock_guard lock(queue_mutex_); + if (!tasks_.empty()) + { + task = std::move(tasks_.front()); + tasks_.pop(); + } + } + if (task) + task(); + else + std::this_thread::yield(); + } + } + + public: + // โ”€โ”€ ๅนถ่กŒ for_each๏ผˆlatch + ่ฐƒ็”จ่€…ๅ‚ไธŽ๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ไธŽๆ—ง็‰ˆ็›ธๆฏ”๏ผš้›ถ future ๅˆ†้…ใ€ไธ€ๆฌกๅŠ ้”ๅ…ฅ้˜Ÿใ€่ฐƒ็”จ่€…ไธ็ฉบ็ญ‰ template void parallel_for_each(Iterator first, Iterator last, Func&& func) { const auto total = static_cast(std::distance(first, last)); if (total == 0) return; - const auto num_threads = workers_.size(); - if (num_threads <= 1 || total < num_threads) + const auto n_chunks = chunk_count(total); + if (n_chunks <= 1) { - // ไธฒ่กŒๅ›ž้€€ for (auto it = first; it != last; ++it) func(*it); return; } - const std::size_t chunk_size = total / num_threads; - std::vector> futures; - futures.reserve(num_threads); + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + + // ๅŽŸๅญ่ฎกๆ•ฐๅ™จ๏ผšๅˆๅง‹ๅ€ผ = n_chunks + std::atomic latch{static_cast(n_chunks)}; + + // ๅฐ†ๅ‰ n_chunks-1 ไธชๅˆ†ๅ—ๆ‰น้‡ๅ…ฅ้˜Ÿ๏ผˆไป…ไธ€ๆฌกๅŠ ้”๏ผ‰ + { + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) + { + const std::size_t len = base + (c < rem ? 1 : 0); + auto beg = first; + std::advance(beg, static_cast(off)); + auto end = beg; + std::advance(end, static_cast(len)); + off += len; + + tasks_.emplace([beg, end, &func, &latch]() + { + for (auto it = beg; it != end; ++it) + func(*it); + latch.fetch_sub(1, std::memory_order_release); + }); + } + } + condition_.notify_all(); + + // ่ฐƒ็”จ่€…ๅค„็†ๆœ€ๅŽไธ€ไธชๅˆ†ๅ—๏ผˆไธ็ป่ฟ‡้˜Ÿๅˆ—๏ผŒ้›ถๅˆ†้…๏ผ‰ + { + const std::size_t c = n_chunks - 1; + const std::size_t len = base + (c < rem ? 1 : 0); + const std::size_t off = total - len; + auto beg = first; + std::advance(beg, static_cast(off)); + for (auto it = beg; it != last; ++it) + func(*it); + latch.fetch_sub(1, std::memory_order_release); + } - for (std::size_t t = 0; t < num_threads; ++t) + // work-stealing ็ญ‰ๅพ…๏ผš่ฐƒ็”จ่€…ๅธฎๅฟ™ๅค„็†้˜Ÿๅˆ—ไปปๅŠก่€Œ้ž็ฉบ่ฝฌ + wait_for_latch(latch); + } + + // โ”€โ”€ ๅ—็บงๅนถ่กŒ for_each๏ผˆไพ›็Ÿฉ้˜ตไน˜ๆณ•/่ฝฌ็ฝฎ็š„ๅˆ†ๅ—ๅพช็Žฏไฝฟ็”จ๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // ไธŽๆ™ฎ้€š parallel_for_each ไธๅŒ๏ผšๆฏไธช"ๅ…ƒ็ด "ๆœฌ่บซๅฐฑๆ˜ฏไธ€ๆฌก้‡้‡็บง่ฎก็ฎ— + // ๏ผˆๅฆ‚ 64ร—64ร—K ็š„็Ÿฉ้˜ตไน˜ๆณ•ๅˆ†ๅ—๏ผ‰๏ผŒๅ› ๆญคไธ้€‚็”จ MIN_CHUNK ไฟๆŠคใ€‚ + // ็›ดๆŽฅๆŒ‰ๅ—ๆ•ฐๅˆ†็ป™ๅ„็บฟ็จ‹๏ผŒ1 ๅ— = 1 ๅˆ†็‰‡ใ€‚ + template + void parallel_for_blocks(Iterator first, Iterator last, Func&& func) + { + const auto total = static_cast(std::distance(first, last)); + if (total <= 1) { - auto chunk_start = first; - std::advance(chunk_start, static_cast(t * chunk_size)); - auto chunk_end = (t == num_threads - 1) ? last : chunk_start; - if (t < num_threads - 1) - std::advance(chunk_end, static_cast(chunk_size)); + for (auto it = first; it != last; ++it) + func(*it); + return; + } + + const auto n_chunks = std::min(workers_.size(), total); + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + std::atomic latch{static_cast(n_chunks)}; - futures.push_back(submit([&func, chunk_start, chunk_end]() + { + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) { - for (auto it = chunk_start; it != chunk_end; ++it) - func(*it); - })); + const std::size_t len = base + (c < rem ? 1 : 0); + auto beg = first; + std::advance(beg, static_cast(off)); + auto end = beg; + std::advance(end, static_cast(len)); + off += len; + + tasks_.emplace([beg, end, &func, &latch]() + { + for (auto it = beg; it != end; ++it) + func(*it); + latch.fetch_sub(1, std::memory_order_release); + }); + } } + condition_.notify_all(); - // ็ญ‰ๅพ…ๆ‰€ๆœ‰ๅ—ๅฎŒๆˆ - for (auto& fut : futures) - fut.get(); + { + const std::size_t c = n_chunks - 1; + const std::size_t off = total - (base + (c < rem ? 1 : 0)); + auto beg = first; + std::advance(beg, static_cast(off)); + for (auto it = beg; it != last; ++it) + func(*it); + latch.fetch_sub(1, std::memory_order_release); + } + + wait_for_latch(latch); } - // ๅนถ่กŒ transform๏ผšๅฐ† [first1, last1) ๅ‡ๅŒ€ๅˆ†ๅ— + // โ”€โ”€ ๅนถ่กŒ transform๏ผˆไธ€ๅ…ƒ๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ template void parallel_transform(InputIt first, InputIt last, OutputIt d_first, UnaryOp&& op) { const auto total = static_cast(std::distance(first, last)); if (total == 0) return; - const auto num_threads = workers_.size(); - if (num_threads <= 1 || total < num_threads) + const auto n_chunks = chunk_count(total); + if (n_chunks <= 1) { std::transform(first, last, d_first, std::forward(op)); return; } - const std::size_t chunk_size = total / num_threads; - std::vector> futures; - futures.reserve(num_threads); + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + std::atomic latch{static_cast(n_chunks)}; - for (std::size_t t = 0; t < num_threads; ++t) { - auto chunk_start = first; - auto out_start = d_first; - std::advance(chunk_start, static_cast(t * chunk_size)); - std::advance(out_start, static_cast(t * chunk_size)); - - auto chunk_end = (t == num_threads - 1) ? last : chunk_start; - auto out_end = (t == num_threads - 1) ? d_first : out_start; - if (t < num_threads - 1) + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) { - std::advance(chunk_end, static_cast(chunk_size)); - std::advance(out_end, static_cast(chunk_size)); + const std::size_t len = base + (c < rem ? 1 : 0); + auto in_beg = first; + auto out_beg = d_first; + std::advance(in_beg, static_cast(off)); + std::advance(out_beg, static_cast(off)); + auto in_end = in_beg; + std::advance(in_end, static_cast(len)); + off += len; + + tasks_.emplace([in_beg, in_end, out_beg, &op, &latch]() + { + auto in = in_beg; + auto out = out_beg; + for (; in != in_end; ++in, ++out) + *out = op(*in); + latch.fetch_sub(1, std::memory_order_release); + }); } + } + condition_.notify_all(); - futures.push_back(submit([&op, chunk_start, chunk_end, out_start]() - { - auto in_it = chunk_start; - auto out_it = out_start; - for (; in_it != chunk_end; ++in_it, ++out_it) - *out_it = op(*in_it); - })); + { + const std::size_t c = n_chunks - 1; + const std::size_t off = total - (base + (c < rem ? 1 : 0)); + auto in_beg = first; + auto out_beg = d_first; + std::advance(in_beg, static_cast(off)); + std::advance(out_beg, static_cast(off)); + for (; in_beg != last; ++in_beg, ++out_beg) + *out_beg = op(*in_beg); + latch.fetch_sub(1, std::memory_order_release); } - for (auto& fut : futures) - fut.get(); + wait_for_latch(latch); } - // ๅนถ่กŒ transform๏ผˆไบŒๅ…ƒ็‰ˆๆœฌ๏ผ‰ + // โ”€โ”€ ๅนถ่กŒ transform๏ผˆไบŒๅ…ƒ๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ template void parallel_transform(InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp&& op) @@ -177,49 +301,131 @@ namespace nn const auto total = static_cast(std::distance(first1, last1)); if (total == 0) return; - const auto num_threads = workers_.size(); - if (num_threads <= 1 || total < num_threads) + const auto n_chunks = chunk_count(total); + if (n_chunks <= 1) { std::transform(first1, last1, first2, d_first, std::forward(op)); return; } - const std::size_t chunk_size = total / num_threads; - std::vector> futures; - futures.reserve(num_threads); + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + std::atomic latch{static_cast(n_chunks)}; - for (std::size_t t = 0; t < num_threads; ++t) { - auto in1_start = first1; - auto in2_start = first2; - auto out_start = d_first; - std::advance(in1_start, static_cast(t * chunk_size)); - std::advance(in2_start, static_cast(t * chunk_size)); - std::advance(out_start, static_cast(t * chunk_size)); - - auto in1_end = (t == num_threads - 1) ? last1 : in1_start; - auto out_end = (t == num_threads - 1) ? d_first : out_start; - if (t < num_threads - 1) + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) { - std::advance(in1_end, static_cast(chunk_size)); - std::advance(out_end, static_cast(chunk_size)); + const std::size_t len = base + (c < rem ? 1 : 0); + auto i1 = first1; + auto i2 = first2; + auto o = d_first; + std::advance(i1, static_cast(off)); + std::advance(i2, static_cast(off)); + std::advance(o, static_cast(off)); + auto i1_end = i1; + std::advance(i1_end, static_cast(len)); + off += len; + + tasks_.emplace([i1, i1_end, i2, o, &op, &latch]() + { + auto it1 = i1; + auto it2 = i2; + auto out = o; + for (; it1 != i1_end; ++it1, ++it2, ++out) + *out = op(*it1, *it2); + latch.fetch_sub(1, std::memory_order_release); + }); } + } + condition_.notify_all(); + + { + const std::size_t c = n_chunks - 1; + const std::size_t off = total - (base + (c < rem ? 1 : 0)); + auto i1 = first1; + auto i2 = first2; + auto o = d_first; + std::advance(i1, static_cast(off)); + std::advance(i2, static_cast(off)); + std::advance(o, static_cast(off)); + for (; i1 != last1; ++i1, ++i2, ++o) + *o = op(*i1, *i2); + latch.fetch_sub(1, std::memory_order_release); + } + + wait_for_latch(latch); + } - futures.push_back(submit([&op, in1_start, in1_end, in2_start, out_start]() + // โ”€โ”€ ๅนถ่กŒ transform_reduce๏ผˆไธ€ๅ…ƒ transform๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + template + T parallel_transform_reduce(InputIt first, InputIt last, T init, + BinaryOp&& reduce_op, UnaryOp&& transform_op) + { + const auto total = static_cast(std::distance(first, last)); + if (total == 0) return init; + + const auto n_chunks = chunk_count(total); + if (n_chunks <= 1) + { + return std::transform_reduce(first, last, init, + std::forward(reduce_op), + std::forward(transform_op)); + } + + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + std::atomic latch{static_cast(n_chunks)}; + + // ้ข„ๅˆ†้…็ป“ๆžœๆ•ฐ็ป„๏ผˆๆ ˆไธŠๅฐ vector๏ผ‰๏ผŒ้ฟๅ… future ๅ †ๅˆ†้… + std::vector partials(n_chunks); + + { + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) { - auto it1 = in1_start; - auto it2 = in2_start; - auto out_it = out_start; - for (; it1 != in1_end; ++it1, ++it2, ++out_it) - *out_it = op(*it1, *it2); - })); + const std::size_t len = base + (c < rem ? 1 : 0); + auto beg = first; + std::advance(beg, static_cast(off)); + auto end = beg; + std::advance(end, static_cast(len)); + off += len; + + tasks_.emplace([beg, end, &reduce_op, &transform_op, &partials, &latch, c]() + { + T local{}; // ่ฆๆฑ‚ T{} ๆ˜ฏ reduce_op ็š„ๅ•ไฝๅ…ƒ + for (auto it = beg; it != end; ++it) + local = reduce_op(local, transform_op(*it)); + partials[c] = std::move(local); + latch.fetch_sub(1, std::memory_order_release); + }); + } + } + condition_.notify_all(); + + { + const std::size_t c = n_chunks - 1; + const std::size_t off = total - (base + (c < rem ? 1 : 0)); + auto beg = first; + std::advance(beg, static_cast(off)); + T local{}; + for (auto it = beg; it != last; ++it) + local = reduce_op(local, transform_op(*it)); + partials[c] = std::move(local); + latch.fetch_sub(1, std::memory_order_release); } - for (auto& fut : futures) - fut.get(); + wait_for_latch(latch); + + T result = init; + for (auto& p : partials) + result = reduce_op(result, p); + return result; } - // ๅนถ่กŒ transform_reduce๏ผˆไบŒๅ…ƒ่พ“ๅ…ฅ่Œƒๅ›ด๏ผ‰ + // โ”€โ”€ ๅนถ่กŒ transform_reduce๏ผˆไบŒๅ…ƒ่พ“ๅ…ฅ่Œƒๅ›ด๏ผ‰โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ template T parallel_transform_reduce(InputIt1 first1, InputIt1 last1, InputIt2 first2, T init, BinaryOp&& reduce_op, UnaryOp&& transform_op) @@ -227,48 +433,70 @@ namespace nn const auto total = static_cast(std::distance(first1, last1)); if (total == 0) return init; - const auto num_threads = workers_.size(); - if (num_threads <= 1 || total < num_threads) + const auto n_chunks = chunk_count(total); + if (n_chunks <= 1) { return std::transform_reduce(first1, last1, first2, init, std::forward(reduce_op), std::forward(transform_op)); } - const std::size_t chunk_size = total / num_threads; - std::vector> futures; - futures.reserve(num_threads); + const std::size_t base = total / n_chunks; + const std::size_t rem = total % n_chunks; + std::atomic latch{static_cast(n_chunks)}; + std::vector partials(n_chunks); - for (std::size_t t = 0; t < num_threads; ++t) { - auto in1_start = first1; - auto in2_start = first2; - std::advance(in1_start, static_cast(t * chunk_size)); - std::advance(in2_start, static_cast(t * chunk_size)); - - auto in1_end = (t == num_threads - 1) ? last1 : in1_start; - if (t < num_threads - 1) - std::advance(in1_end, static_cast(chunk_size)); - - futures.push_back(submit([&reduce_op, &transform_op, in1_start, in1_end, in2_start]() + std::lock_guard lock(queue_mutex_); + std::size_t off = 0; + for (std::size_t c = 0; c < n_chunks - 1; ++c) { - T local_sum{}; - auto it1 = in1_start; - auto it2 = in2_start; - for (; it1 != in1_end; ++it1, ++it2) - local_sum = reduce_op(local_sum, transform_op(*it1, *it2)); - return local_sum; - })); + const std::size_t len = base + (c < rem ? 1 : 0); + auto i1 = first1; + auto i2 = first2; + std::advance(i1, static_cast(off)); + std::advance(i2, static_cast(off)); + auto i1_end = i1; + std::advance(i1_end, static_cast(len)); + off += len; + + tasks_.emplace([i1, i1_end, i2, &reduce_op, &transform_op, &partials, &latch, c]() + { + T local{}; + auto it1 = i1; + auto it2 = i2; + for (; it1 != i1_end; ++it1, ++it2) + local = reduce_op(local, transform_op(*it1, *it2)); + partials[c] = std::move(local); + latch.fetch_sub(1, std::memory_order_release); + }); + } } + condition_.notify_all(); + + { + const std::size_t c = n_chunks - 1; + const std::size_t off = total - (base + (c < rem ? 1 : 0)); + auto i1 = first1; + auto i2 = first2; + std::advance(i1, static_cast(off)); + std::advance(i2, static_cast(off)); + T local{}; + for (; i1 != last1; ++i1, ++i2) + local = reduce_op(local, transform_op(*i1, *i2)); + partials[c] = std::move(local); + latch.fetch_sub(1, std::memory_order_release); + } + + wait_for_latch(latch); - // ๆ”ถ้›†ๅ„ๅ—็ป“ๆžœ T result = init; - for (auto& fut : futures) - result = reduce_op(result, fut.get()); + for (auto& p : partials) + result = reduce_op(result, p); return result; } - // ๆžๆž„๏ผš็ญ‰ๅพ…ๆ‰€ๆœ‰ไปปๅŠกๅฎŒๆˆๅŽๅ…ณ้—ญ็บฟ็จ‹ + // โ”€โ”€ ๆžๆž„ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ ~ThreadPool() { stop_.store(true, std::memory_order_release); @@ -280,7 +508,6 @@ namespace nn } } - // ็ฆๆญขๆ‹ท่ด/็งปๅŠจ ThreadPool(const ThreadPool&) = delete; ThreadPool& operator=(const ThreadPool&) = delete; ThreadPool(ThreadPool&&) = delete; @@ -289,8 +516,7 @@ namespace nn [[nodiscard]] std::size_t size() const noexcept { return workers_.size(); } }; - // โ”€โ”€ ๅ…จๅฑ€็บฟ็จ‹ๆฑ ๅ•ไพ‹๏ผˆๆ‡’ๅˆๅง‹ๅŒ–๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - // ่ฎญ็ปƒๅผ€ๅง‹ๆ—ถ่‡ชๅŠจๅˆ›ๅปบ๏ผŒ็จ‹ๅบ็ป“ๆŸๆ—ถ่‡ชๅŠจ้”€ๆฏ๏ผŒ้ฟๅ…ๅๅคๅˆ›ๅปบ/้”€ๆฏ + // โ”€โ”€ ๅ…จๅฑ€็บฟ็จ‹ๆฑ ๅ•ไพ‹ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ inline ThreadPool& global_thread_pool() { static ThreadPool pool{std::thread::hardware_concurrency()}; diff --git a/mnist_model.bin b/mnist_model.bin new file mode 100644 index 00000000..0262ec14 Binary files /dev/null and b/mnist_model.bin differ diff --git a/src/infer.cpp b/src/infer.cpp index fd7b05e1..41b6bbf5 100644 --- a/src/infer.cpp +++ b/src/infer.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -14,10 +15,7 @@ namespace fs = std::filesystem; // ==================== ๅธธ้‡ ==================== -static constexpr std::size_t INPUT_DIM = 784; -static constexpr std::size_t HIDDEN_DIM = 64; -static constexpr std::size_t NUM_CLASSES = 10; -static constexpr std::size_t NUM_HIDDEN_LAYERS = 3; +// ็Žฐๅœจไฝฟ็”จ nn::MNIST_INPUT_DIM, nn::MNIST_NUM_CLASSES, nn::MNIST_LAYER_DIMS // ==================== ๅธฎๅŠฉไฟกๆฏ ==================== void print_usage(const char *prog) @@ -28,16 +26,18 @@ void print_usage(const char *prog) << " " << prog << " [้€‰้กน] ๆŽจ็†ๅ•ๅผ ๅ›พ็‰‡\n" << " " << prog << " <็›ฎๅฝ•> [้€‰้กน] ๆ‰น้‡ๆŽจ็†็›ฎๅฝ•ไธ‹ๆ‰€ๆœ‰ CSV\n\n" << "้€‰้กน:\n" - << " --model ๆจกๅž‹ๆ–‡ไปถ่ทฏๅพ„ (้ป˜่ฎค: pretrained/model.bin)\n" - << " --topk ๆ˜พ็คบๅ‰ n ไธช้ข„ๆต‹็ป“ๆžœ (้ป˜่ฎค: 3)\n" - << " --show-pixels ๆ˜พ็คบๅƒ็ด ็Ÿฉ้˜ต (่ฐƒ่ฏ•็”จ)\n" - << " --help ๆ˜พ็คบๆญคๅธฎๅŠฉไฟกๆฏ\n"; + << " --model ๆจกๅž‹ๆ–‡ไปถ่ทฏๅพ„ (้ป˜่ฎค: pretrained/model.bin)\n" + << " --model-type ๆจกๅž‹็ฑปๅž‹: mlp/transformer (้ป˜่ฎค: mlp)\n" + << " --topk ๆ˜พ็คบๅ‰ n ไธช้ข„ๆต‹็ป“ๆžœ (้ป˜่ฎค: 3)\n" + << " --show-pixels ๆ˜พ็คบๅƒ็ด ็Ÿฉ้˜ต (่ฐƒ่ฏ•็”จ)\n" + << " --help ๆ˜พ็คบๆญคๅธฎๅŠฉไฟกๆฏ\n"; } // ==================== ๅ‘ฝไปค่กŒๅ‚ๆ•ฐ ==================== struct InferConfig { std::string model_path = "pretrained/model.bin"; + std::string model_type = "mlp"; std::string input_path; int topk = 3; bool show_pixels = false; @@ -60,6 +60,16 @@ InferConfig parse_args(int argc, char *argv[]) { cfg.model_path = argv[++i]; } + else if (arg == "--model-type" && i + 1 < argc) + { + cfg.model_type = argv[++i]; + if (cfg.model_type != "mlp" && cfg.model_type != "transformer") + { + std::cerr << "ๆœช็Ÿฅๆจกๅž‹็ฑปๅž‹: " << cfg.model_type + << "๏ผŒๅฏ้€‰: mlp, transformer\n"; + std::exit(1); + } + } else if (arg == "--topk" && i + 1 < argc) { cfg.topk = std::stoi(argv[++i]); @@ -90,19 +100,7 @@ InferConfig parse_args(int argc, char *argv[]) } // ==================== ๆž„ๅปบ็ฝ‘็ปœ ==================== -nn::Model build_model() -{ - nn::Model model; - model.add(INPUT_DIM, HIDDEN_DIM) - .add(); - for (std::size_t i = 0; i < NUM_HIDDEN_LAYERS - 1; ++i) - { - model.add(HIDDEN_DIM, HIDDEN_DIM) - .add(); - } - model.add(HIDDEN_DIM, NUM_CLASSES); - return model; -} +// build_model ๅ‡ฝๆ•ฐๅทฒ็งป่‡ณ neuralnet.cpp/mnist_common.hpp ไธญ็š„ nn::build_mnist_model() // ==================== ๆ•ฐๆฎ่ฏปๅ– ==================== nn::Matrix load_image_from_csv(const std::string &csv_line) @@ -114,11 +112,11 @@ nn::Matrix load_image_from_csv(const std::string &csv_line) { pixels.push_back(std::stod(token)); } - if (pixels.size() != INPUT_DIM) - throw std::runtime_error("CSV ๅฟ…้กปๅŒ…ๅซๆฐๅฅฝ " + std::to_string(INPUT_DIM) + " ไธชๅ€ผ๏ผŒๅฎž้™…: " + std::to_string(pixels.size())); + if (pixels.size() != nn::MNIST_INPUT_DIM) + throw std::runtime_error("CSV ๅฟ…้กปๅŒ…ๅซๆฐๅฅฝ " + std::to_string(nn::MNIST_INPUT_DIM) + " ไธชๅ€ผ๏ผŒๅฎž้™…: " + std::to_string(pixels.size())); - nn::Matrix img(INPUT_DIM, 1); - for (std::size_t i = 0; i < INPUT_DIM; ++i) + nn::Matrix img(nn::MNIST_INPUT_DIM, 1); + for (std::size_t i = 0; i < nn::MNIST_INPUT_DIM; ++i) img.set_value_unchecked(i, 0, pixels[i]); return img; } @@ -136,12 +134,12 @@ std::vector predict_with_confidence(nn::Model &model, const nn::Matr // Softmax ่ฎก็ฎ—ๆฆ‚็Ž‡ double max_val = logits.at_unchecked(0, 0); - for (int c = 1; c < static_cast(NUM_CLASSES); ++c) + for (int c = 1; c < static_cast(nn::MNIST_NUM_CLASSES); ++c) max_val = std::max(max_val, logits.at_unchecked(c, 0)); double sum_exp = 0.0; - std::vector probs(NUM_CLASSES); - for (std::size_t c = 0; c < NUM_CLASSES; ++c) + std::vector probs(nn::MNIST_NUM_CLASSES); + for (std::size_t c = 0; c < nn::MNIST_NUM_CLASSES; ++c) { probs[c] = std::exp(logits.at_unchecked(c, 0) - max_val); sum_exp += probs[c]; @@ -150,7 +148,7 @@ std::vector predict_with_confidence(nn::Model &model, const nn::Matr p /= sum_exp; // ๆŒ‰ๆฆ‚็Ž‡ๆŽ’ๅบๅ– top-k - std::vector indices(NUM_CLASSES); + std::vector indices(nn::MNIST_NUM_CLASSES); std::iota(indices.begin(), indices.end(), 0); std::partial_sort(indices.begin(), indices.begin() + topk, indices.end(), [&](int a, int b) { return probs[a] > probs[b]; }); @@ -220,7 +218,7 @@ int main(int argc, char *argv[]) InferConfig cfg = parse_args(argc, argv); // ๆž„ๅปบๅนถๅŠ ่ฝฝๆจกๅž‹ - auto model = build_model(); + auto model = nn::build_mnist_model(cfg.model_type); nn::load_model(cfg.model_path, model); std::cout << "ๆจกๅž‹ๅทฒๅŠ ่ฝฝ: " << cfg.model_path << "\n" << std::endl; diff --git a/src/train.cpp b/src/train.cpp index c6ab1f88..b37da529 100644 --- a/src/train.cpp +++ b/src/train.cpp @@ -1,5 +1,6 @@ #include #include +#include #include // for std::memcpy #include // for std::unique_ptr #include @@ -14,10 +15,7 @@ #include // ==================== ๅธธ้‡ ==================== -static constexpr std::size_t INPUT_DIM = 784; -static constexpr std::size_t HIDDEN_DIM = 64; -static constexpr std::size_t NUM_CLASSES = 10; -static constexpr std::size_t NUM_HIDDEN_LAYERS = 3; +// ็Žฐๅœจไฝฟ็”จ nn::MNIST_INPUT_DIM, nn::MNIST_NUM_CLASSES, nn::MNIST_LAYER_DIMS // ==================== ๅธฎๅŠฉไฟกๆฏ ==================== void print_usage(const char *prog) @@ -26,11 +24,12 @@ void print_usage(const char *prog) << "MNIST ๆ‰‹ๅ†™ๆ•ฐๅญ—่ฎญ็ปƒ็จ‹ๅบ\n\n" << "็”จๆณ•: " << prog << " [้€‰้กน]\n\n" << "้€‰้กน:\n" + << " --model-type ๆจกๅž‹็ฑปๅž‹: mlp/transformer (้ป˜่ฎค: mlp)\n" << " --resume ไปŽๅทฒๆœ‰ๆจกๅž‹ๆขๅค่ฎญ็ปƒ\n" << " --save ๆจกๅž‹ไฟๅญ˜่ทฏๅพ„ (้ป˜่ฎค: mnist_model.bin)\n" << " --dataset ๆ•ฐๆฎ้›†็›ฎๅฝ• (้ป˜่ฎค: datasets/mnist_data)\n" - << " --epochs ่ฎญ็ปƒ่ฝฎๆ•ฐ (้ป˜่ฎค: 5)\n" - << " --lr ๅญฆไน ็Ž‡ (้ป˜่ฎค: 0.01)\n" + << " --epochs ่ฎญ็ปƒ่ฝฎๆ•ฐ (้ป˜่ฎค: 10)\n" + << " --lr ๅญฆไน ็Ž‡ (้ป˜่ฎค: 0.001)\n" << " --batch-size ๆ‰นๅคงๅฐ (้ป˜่ฎค: 64)\n" << " --optimizer ไผ˜ๅŒ–ๅ™จ: sgd/momentum/adam (้ป˜่ฎค: adam)\n" << " --help ๆ˜พ็คบๆญคๅธฎๅŠฉไฟกๆฏ\n"; @@ -43,8 +42,9 @@ struct TrainConfig std::string dataset_path = "datasets/mnist_data"; std::string resume_path; std::string optimizer_name = "adam"; - int epochs = 5; - double lr = 0.01; + std::string model_type = "mlp"; + int epochs = 10; // ๅขžๅŠ ่ฎญ็ปƒ่ฝฎๆ•ฐ + double lr = 0.001; // ่ฐƒๆ•ดๅญฆไน ็Ž‡ std::size_t batch_size = 64; bool load_existing = false; }; @@ -100,6 +100,16 @@ TrainConfig parse_args(int argc, char *argv[]) std::exit(1); } } + else if (arg == "--model-type" && i + 1 < argc) + { + cfg.model_type = argv[++i]; + if (cfg.model_type != "mlp" && cfg.model_type != "transformer") + { + std::cerr << "ๆœช็Ÿฅๆจกๅž‹็ฑปๅž‹: " << cfg.model_type + << "๏ผŒๅฏ้€‰: mlp, transformer\n"; + std::exit(1); + } + } else { std::cerr << "ๆœช็Ÿฅๅ‚ๆ•ฐ: " << arg << "\nไฝฟ็”จ --help ๆŸฅ็œ‹็”จๆณ•\n"; @@ -109,44 +119,101 @@ TrainConfig parse_args(int argc, char *argv[]) return cfg; } -// ==================== ๆ•ฐๆฎๅŠ ่ฝฝ ==================== +// ==================== ๆ•ฐๆฎๅŠ ่ฝฝ๏ผˆไผ˜ๅŒ–็‰ˆ๏ผ‰ ==================== +// ไผ˜ๅŒ–็ญ–็•ฅ๏ผš +// 1. ๆ•ดๆ–‡ไปถไธ€ๆฌกๆ€ง่ฏปๅ…ฅๅ†…ๅญ˜๏ผŒ้ฟๅ…้€่กŒ I/O ๅผ€้”€ +// 2. ไฝฟ็”จ std::from_chars (C++17) ๆ›ฟไปฃ std::stod๏ผŒๆ— ไธดๆ—ถ string ๅˆ†้… +// 3. ๅ‘้‡้ข„ๅˆ†้… reserve()๏ผŒ้ฟๅ…ๅคšๆฌกๆ‰ฉๅฎนๆ‹ท่ด +// 4. ็›ดๆŽฅๆŒ‡้’ˆ้ๅŽ† buffer๏ผŒ็œๅŽป std::stringstream ๅผ€้”€ std::pair load_csv(const std::string &filename, int max_samples = -1) { - std::ifstream file(filename); + // โ”€โ”€ 1. ๆ•ดๆ–‡ไปถ่ฏปๅ…ฅ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + std::ifstream file(filename, std::ios::binary | std::ios::ate); if (!file.is_open()) throw std::runtime_error("Cannot open file: " + filename); - std::vector features; - std::vector labels; - std::string line; + const auto file_size = file.tellg(); + file.seekg(0); + + std::string buffer(static_cast(file_size), '\0'); + file.read(buffer.data(), file_size); + file.close(); + + // โ”€โ”€ 2. ้ข„ๆ‰ซๆ่กŒๆ•ฐไปฅ้ข„ๅˆ†้…ๅ‘้‡ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + std::size_t row_count = 0; + for (char c : buffer) + if (c == '\n') ++row_count; + if (row_count == 0) + throw std::runtime_error("CSV file is empty or malformed: " + filename); + + if (max_samples > 0 && static_cast(max_samples) < row_count) + row_count = static_cast(max_samples); + + // โ”€โ”€ 3. ่งฃๆž๏ผšไฝฟ็”จ from_chars ็›ดๆŽฅไปŽ buffer ่ฏปๅ–ๆ•ฐๅ€ผ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + const char *ptr = buffer.data(); + const char *end = buffer.data() + buffer.size(); - while (std::getline(file, line)) + // ๅ…ˆ่งฃๆž็ฌฌไธ€่กŒ็กฎๅฎš feat_dim + int first_label = 0; + std::size_t feat_dim = 0; { - std::stringstream ss(line); - std::string token; - int label; - std::getline(ss, token, ','); - label = std::stoi(token); - labels.push_back(label); - - while (std::getline(ss, token, ',')) + // ่ทณ่ฟ‡้ฆ–่กŒ็”จไบŽ่ฎกๆ•ฐๅˆ—ๆ•ฐ + const char *p = ptr; + auto [p1, ec1] = std::from_chars(p, end, first_label); + p = p1; + std::size_t cnt = 0; + while (p < end && *p != '\n' && *p != '\r') { - features.push_back(std::stod(token)); // ๆ•ฐๆฎๅทฒ็”ฑ ToTensor() ๅฝ’ไธ€ๅŒ–ๅˆฐ [0,1] + if (*p == ',') + { + ++cnt; + double tmp; + auto [p2, ec2] = std::from_chars(p + 1, end, tmp); + p = p2; + } + else + ++p; } + feat_dim = cnt; // ๆ ‡็ญพๅŽ็š„้€—ๅทๆ•ฐ = ็‰นๅพๆ•ฐ } - std::size_t N = labels.size(); - if (N == 0) - throw std::runtime_error("CSV file is empty or malformed: " + filename); - std::size_t feat_dim = features.size() / N; // 784 + std::vector features(row_count * feat_dim); + std::vector labels(row_count); + + std::size_t row = 0; + ptr = buffer.data(); // ้‡็ฝฎๆŒ‡้’ˆ - if (max_samples > 0 && static_cast(max_samples) < N) + while (ptr < end && row < row_count) { - N = max_samples; - features.resize(N * feat_dim); - labels.resize(N); + // ่งฃๆžๆ ‡็ญพ + auto [p_label, ec_label] = std::from_chars(ptr, end, labels[row]); + if (ec_label != std::errc{}) + throw std::runtime_error("Failed to parse label at row " + std::to_string(row)); + ptr = p_label; + + // ่งฃๆž็‰นๅพ + for (std::size_t j = 0; j < feat_dim; ++j) + { + // ่ทณ่ฟ‡้€—ๅท + if (ptr < end && *ptr == ',') ++ptr; + + double val; + auto [p_feat, ec_feat] = std::from_chars(ptr, end, val); + if (ec_feat != std::errc{}) + throw std::runtime_error("Failed to parse feature at row " + std::to_string(row)); + features[row * feat_dim + j] = val; + ptr = p_feat; + } + + // ่ทณ่ฟ‡่กŒๅฐพ (\r\n ๆˆ– \n) + while (ptr < end && (*ptr == '\r' || *ptr == '\n')) ++ptr; + ++row; } + // ๅฎž้™…่ฏปๅ–็š„่กŒๆ•ฐๅฏ่ƒฝๅฐ‘ไบŽ้ข„ๆ‰ซๆ๏ผˆไพ‹ๅฆ‚ๆ–‡ไปถๆœซๅฐพ็ฉบ่กŒ๏ผ‰ + std::size_t N = row; + + // โ”€โ”€ 4. ็›ดๆŽฅๅ†™ๅ…ฅ็Ÿฉ้˜ต๏ผˆๅˆ—ไผ˜ๅ…ˆ๏ผŒ้€่กŒๅกซๅ……๏ผ‰ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // ็‰นๅพ็Ÿฉ้˜ต๏ผšๅฝข็Šถ (feat_dim, N) nn::Matrix feat_mat(feat_dim, N); for (std::size_t i = 0; i < N; ++i) @@ -163,7 +230,7 @@ std::pair load_csv(const std::string &filename, int max_ { int lbl = labels[i]; if (lbl < 0 || lbl >= 10) - throw std::out_of_range("Label out of range"); + throw std::out_of_range("Label out of range: " + std::to_string(lbl)); label_mat.set_value_unchecked(lbl, i, 1.0); } @@ -180,7 +247,7 @@ double evaluate(nn::Model &model, const nn::Matrix &x, const nn::Matrix &y_oneho { double max_val = out.at_unchecked(0, i); int pred = 0; - for (int j = 1; j < static_cast(NUM_CLASSES); ++j) + for (int j = 1; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) { double val = out.at_unchecked(j, i); if (val > max_val) @@ -190,7 +257,7 @@ double evaluate(nn::Model &model, const nn::Matrix &x, const nn::Matrix &y_oneho } } int true_label = -1; - for (int j = 0; j < static_cast(NUM_CLASSES); ++j) + for (int j = 0; j < static_cast(nn::MNIST_NUM_CLASSES); ++j) { if (y_onehot.at_unchecked(j, i) == 1.0) { @@ -205,19 +272,7 @@ double evaluate(nn::Model &model, const nn::Matrix &x, const nn::Matrix &y_oneho } // -------------------- ๆž„ๅปบ็ฝ‘็ปœ -------------------- -nn::Model build_model() -{ - nn::Model model; - model.add(INPUT_DIM, HIDDEN_DIM) - .add(); - for (std::size_t i = 0; i < NUM_HIDDEN_LAYERS - 1; ++i) - { - model.add(HIDDEN_DIM, HIDDEN_DIM) - .add(); - } - model.add(HIDDEN_DIM, NUM_CLASSES); - return model; -} +// build_model ๅ‡ฝๆ•ฐๅทฒ็งป่‡ณ neuralnet.cpp/mnist_common.hpp ไธญ็š„ nn::build_mnist_model() // ==================== ไธปๅ‡ฝๆ•ฐ ==================== int main(int argc, char *argv[]) @@ -230,10 +285,29 @@ int main(int argc, char *argv[]) std::cout << "========================================\n"; std::cout << " MNIST ๆ‰‹ๅ†™ๆ•ฐๅญ—่ฎญ็ปƒ\n"; std::cout << "========================================\n"; - std::cout << " ็ฝ‘็ปœ: " << INPUT_DIM; - for (std::size_t i = 0; i < NUM_HIDDEN_LAYERS; ++i) - std::cout << " -> " << HIDDEN_DIM << "(ReLU)"; - std::cout << " -> " << NUM_CLASSES << "\n"; + std::cout << " ๆจกๅž‹็ฑปๅž‹: " << cfg.model_type << "\n"; + if (cfg.model_type == "mlp") + { + std::cout << " ็ฝ‘็ปœ: "; + for (std::size_t i = 0; i < nn::MNIST_LAYER_DIMS.size(); ++i) + { + std::cout << nn::MNIST_LAYER_DIMS[i]; + if (i < nn::MNIST_LAYER_DIMS.size() - 2) + std::cout << "(LayerNorm+GeLU)"; + if (i < nn::MNIST_LAYER_DIMS.size() - 1) + std::cout << " -> "; + } + std::cout << "\n"; + } + else + { + std::cout << " ็ฝ‘็ปœ: PatchEmbedding(7ร—7) -> TransformerEncoder(" + << nn::TRANSFORMER_NUM_LAYERS << "ๅฑ‚, d=" + << nn::TRANSFORMER_D_MODEL << ", heads=" + << nn::TRANSFORMER_NUM_HEADS << ") -> Linear(" + << nn::TRANSFORMER_D_MODEL << "โ†’" + << nn::MNIST_NUM_CLASSES << ")\n"; + } std::cout << " ไผ˜ๅŒ–ๅ™จ: " << cfg.optimizer_name << " ๅญฆไน ็Ž‡: " << cfg.lr << "\n"; std::cout << " ่ฝฎๆ•ฐ: " << cfg.epochs << " ๆ‰นๅคงๅฐ: " << cfg.batch_size << "\n"; std::cout << " ๆจกๅž‹: " << (cfg.load_existing ? cfg.resume_path : "(ไปŽๅคด่ฎญ็ปƒ)") @@ -247,7 +321,7 @@ int main(int argc, char *argv[]) std::cout << "่ฎญ็ปƒ้›†: " << train_x.cols() << " ๆ ทๆœฌ, ๆต‹่ฏ•้›†: " << test_x.cols() << " ๆ ทๆœฌ\n" << std::endl; // โ”€โ”€ ๆž„ๅปบๆจกๅž‹ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - auto model = build_model(); + auto model = nn::build_mnist_model(cfg.model_type); if (cfg.load_existing) { @@ -291,12 +365,12 @@ int main(int argc, char *argv[]) // โ”€โ”€ ่กŒไผ˜ๅ…ˆ memcpy ๆๅ– batch๏ผˆๆฏ”้€ๅˆ—ๅคๅˆถๆ›ด็ผ“ๅญ˜ๅ‹ๅฅฝ๏ผ‰ โ”€โ”€โ”€โ”€โ”€ for (std::size_t r = 0; r < train_x.rows(); ++r) - std::memcpy(x_batch.data_ptr() + r * cfg.batch_size, - train_x.data_ptr() + r * train_x.cols() + start, + std::memcpy(x_batch.span().data() + r * cfg.batch_size, + train_x.span().data() + r * train_x.cols() + start, cfg.batch_size * sizeof(double)); for (std::size_t r = 0; r < train_y.rows(); ++r) - std::memcpy(y_batch.data_ptr() + r * cfg.batch_size, - train_y.data_ptr() + r * train_y.cols() + start, + std::memcpy(y_batch.span().data() + r * cfg.batch_size, + train_y.span().data() + r * train_y.cols() + start, cfg.batch_size * sizeof(double)); auto out = model.forward(x_batch);