From 9f21f26fa93bc9a1ef3eb8d81415285e80ae20be Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Fri, 7 Aug 2026 10:48:36 +0200 Subject: [PATCH 01/19] Preliminary experimentations with on-device machine learning --- dsppp/.clangd | 18 + dsppp/Examples/autodiff_example.cpp | 55 ++ dsppp/Examples/autodiff_regression.cpp | 157 ++++ dsppp/Include/dsppp/autodiff/README.md | 684 ++++++++++++++++++ .../Include/dsppp/autodiff/operators/add.hpp | 100 +++ .../Include/dsppp/autodiff/operators/dot.hpp | 124 ++++ .../autodiff/operators/fully_connected.hpp | 210 ++++++ .../autodiff/operators/quadratic_error.hpp | 104 +++ .../Include/dsppp/autodiff/operators/relu.hpp | 89 +++ .../dsppp/autodiff/operators/scale.hpp | 108 +++ .../dsppp/autodiff/optimizers/adam.hpp | 149 ++++ .../dsppp/autodiff/optimizers/common.hpp | 30 + .../dsppp/autodiff/optimizers/rmsprop.hpp | 131 ++++ dsppp/Include/dsppp/autodiff/reverse.hpp | 684 ++++++++++++++++++ dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h | 55 +- .../SSE-300-MPS3/device_cfg.h.base@1.1.3 | 149 ---- .../RTE_Components.h | 8 +- dsppp/example.cproject.yml | 5 +- dsppp/main.c | 3 + dsppp/run_all.py | 1 + dsppp/test.cbuild-idx.yml | 23 + dsppp/test.cbuild-pack.yml | 13 + dsppp/test.csolution.yml | 34 +- dsppp/tests/.clangd | 18 + dsppp/tests/autodiff_test.cpp | 320 ++++++++ dsppp/tests/test.cproject.yml | 1 + dsppp/tests/test.h | 1 + 27 files changed, 3086 insertions(+), 188 deletions(-) create mode 100644 dsppp/.clangd create mode 100644 dsppp/Examples/autodiff_example.cpp create mode 100644 dsppp/Examples/autodiff_regression.cpp create mode 100644 dsppp/Include/dsppp/autodiff/README.md create mode 100644 dsppp/Include/dsppp/autodiff/operators/add.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/dot.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/relu.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/scale.hpp create mode 100644 dsppp/Include/dsppp/autodiff/optimizers/adam.hpp create mode 100644 dsppp/Include/dsppp/autodiff/optimizers/common.hpp create mode 100644 dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp create mode 100644 dsppp/Include/dsppp/autodiff/reverse.hpp delete mode 100644 dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 create mode 100644 dsppp/test.cbuild-idx.yml create mode 100644 dsppp/tests/.clangd create mode 100644 dsppp/tests/autodiff_test.cpp diff --git a/dsppp/.clangd b/dsppp/.clangd new file mode 100644 index 000000000..9d672aec9 --- /dev/null +++ b/dsppp/.clangd @@ -0,0 +1,18 @@ +CompileFlags: + CompilationDatabase: c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release + +--- +If: + PathMatch: .*\.(c|C|h)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release\compile_macros_c.h + +--- +If: + PathMatch: .*\.(cpp|c\+\+|C\+\+|cxx|cc|CC|hpp)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release\compile_macros_cxx.h diff --git a/dsppp/Examples/autodiff_example.cpp b/dsppp/Examples/autodiff_example.cpp new file mode 100644 index 000000000..dd7ca32e8 --- /dev/null +++ b/dsppp/Examples/autodiff_example.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ + Arena<2048> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + tape.register_operator(); + + float x_value[] = {2.0F, -1.0F}; + float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; + float bias_value[] = {1.0F, 0.0F}; + float linear_value[2] = {}; + float activation_value[2] = {}; + + BufferView x = tape.input(x_value); + MatrixView matrix = tape.parameter(matrix_value); + BufferView bias = tape.parameter(bias_value); + BufferView linear = tape.output(linear_value); + BufferView activation = tape.output(activation_value); + + linear = fully_connected(x, matrix, bias); + activation = relu(linear); + + // Seed both ReLU outputs. The second neuron is negative, so ReLU blocks + // its gradient during the backward pass. + const float seed[] = {1.0F, 1.0F}; + if (!tape.backward(activation, seed, 2)) + { + return 1; + } + + std::printf("linear = {%g, %g}\n", static_cast(linear[0]), + static_cast(linear[1])); + std::printf("relu = {%g, %g}\n", static_cast(activation[0]), + static_cast(activation[1])); + std::printf("db = {%g, %g}\n", static_cast(bias.gradient(0)), + static_cast(bias.gradient(1))); + for (std::size_t row = 0; row < matrix.rows(); ++row) + { + for (std::size_t column = 0; column < matrix.columns(); ++column) + { + std::printf("dm[%u][%u] = %g\n", static_cast(row), + static_cast(column), + static_cast(matrix.gradient(row, column))); + } + } + return 0; +} diff --git a/dsppp/Examples/autodiff_regression.cpp b/dsppp/Examples/autodiff_regression.cpp new file mode 100644 index 000000000..7bbd2b857 --- /dev/null +++ b/dsppp/Examples/autodiff_regression.cpp @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include + +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +/* C-compatible representation suitable for RAM, flash, or application-defined + * nonvolatile storage. It contains values only; optimizer state is not needed + * for inference on a later run. + */ +struct SinePolynomialParameters +{ + float coefficients[3]; // x, x^2, x^3 + float bias; +}; + +static void save_parameters(SinePolynomialParameters &destination, + const SinePolynomialParameters &source) noexcept +{ + for (std::size_t i = 0; i < 3; ++i) + destination.coefficients[i] = source.coefficients[i]; + destination.bias = source.bias; +} + +static void restore_parameters(SinePolynomialParameters &destination, + const SinePolynomialParameters &source) noexcept +{ + save_parameters(destination, source); +} + +static float infer(const SinePolynomialParameters ¶meters, + float x) noexcept +{ + const float x2 = x * x; + return parameters.bias + parameters.coefficients[0] * x + + parameters.coefficients[1] * x2 + + parameters.coefficients[2] * x2 * x; +} + +int main() +{ + + constexpr std::size_t sample_count = 100U; + constexpr std::size_t training_steps = 4000U; + constexpr float pi = 3.14159265358979323846F; + + SinePolynomialParameters model = {{0.0F, 0.0F, 0.0F}, 0.0F}; + // One training graph contains two scalar operations per sample and one + // global loss record. Values stay in caller storage; this arena contains + // gradients and operation records only. + Arena<32768> *arena = new Arena<32768>(); + Tape &tape = arena->tape(); + tape.register_operator(); + tape.register_operator(); + tape.register_operator(); + + float feature_value[sample_count][3] = {}; + float polynomial_value[sample_count] = {}; + float prediction_value[sample_count] = {}; + float target_value[sample_count] = {}; + float loss_value[1] = {}; + + // Construct the complete fixed training set once. No sample storage is + // allocated by the autodiff implementation. + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + const float x = -pi + 2.0F * pi * static_cast(sample) / + static_cast(sample_count - 1U); + const float x2 = x * x; + feature_value[sample][0] = x; + feature_value[sample][1] = x2; + feature_value[sample][2] = x2 * x; + target_value[sample] = std::sin(x); + } + + BufferView coefficients = tape.parameter(model.coefficients); + BufferView bias = tape.parameter(model.bias); + BufferView polynomial = tape.output(polynomial_value); + BufferView prediction = tape.output(prediction_value); + BufferView target = tape.input(target_value); + BufferView loss = tape.output(loss_value); + + // Match the optimizer chosen by the PyTorch example. Adam can be used + // here instead by including adam.hpp and changing only this type. + RMSProp<4, 2> optimizer(1.0e-3F); + optimizer.add(coefficients); + optimizer.add(bias); + + /* Set this to true for bias-only fine tuning. The same mechanism freezes + * all parameters belonging to any selected layer/operator. + */ + constexpr bool fine_tune_bias_only = false; + if (fine_tune_bias_only) + freeze_parameters(optimizer, coefficients); + + /* Views and gradient buffers are persistent. Only operation records after + * this point are rewound for every complete batch. + */ + tape.begin_graph(); + for (std::size_t step = 0; step < training_steps; ++step) + { + tape.rewind_graph(); + + // Build all 100 predictions before constructing the loss. The scalar + // views below share slices of the two arena-managed vector gradients; + // creating them allocates no additional gradient buffers. + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + BufferView features = tape.input(feature_value[sample]); + BufferView polynomial_element = tape.output( + &polynomial_value[sample], &polynomial.gradients()[sample], 1U); + BufferView prediction_element = tape.output( + &prediction_value[sample], &prediction.gradients()[sample], 1U); + + polynomial_element = dot(features, coefficients); + prediction_element = polynomial_element + bias; + } + + // One scalar loss represents the whole sampled period. Consequently + // backward() accumulates parameter gradients from every sample and + // the optimizer performs exactly one global update per step. + loss = quadratic_error(prediction, target); + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if ((step + 1U) % 100U == 0U) + std::printf("step %u, global quadratic error = %g, mean = %g\n", + static_cast(step + 1U), + static_cast(loss_value[0]), + static_cast(loss_value[0] / sample_count)); + } + + SinePolynomialParameters checkpoint{}; + save_parameters(checkpoint, model); + std::printf("learned: y = %g + %g*x + %g*x^2 + %g*x^3\n", + static_cast(checkpoint.bias), + static_cast(checkpoint.coefficients[0]), + static_cast(checkpoint.coefficients[1]), + static_cast(checkpoint.coefficients[2])); + + /* Simulate a later program run. In a real target, checkpoint would first + * be persisted and later loaded from platform-specific nonvolatile memory. + */ + SinePolynomialParameters restored{}; + restore_parameters(restored, checkpoint); + std::printf("restored inference at pi/2: predicted=%g, reference=%g\n", + static_cast(infer(restored, pi * 0.5F)), + static_cast(std::sin(pi * 0.5F))); + + delete arena; + return 0; +} diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md new file mode 100644 index 000000000..820b2b38b --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -0,0 +1,684 @@ +# Reverse automatic differentiation reference + +`reverse.hpp` is a deliberately small reverse-mode automatic differentiation +(AD) implementation for embedded use. It currently handles `float` buffers, +vector addition, vector dot products, affine vector scaling +`alpha * x + beta`, fully connected and ReLU nodes, and a quadratic-error +loss. Fixed-storage Adam and RMSProp optimizers support user-written training +loops. The structure is intended to be extended with tensor operators. + +## Why this implementation uses CMSIS-DSP + +Automatic differentiation needs high performance in both directions. The +forward pass evaluates the model, while the backward pass propagates and +accumulates gradients. This implementation uses CMSIS-DSP for both rather than +treating it only as a collection of forward inference kernels. + +Where an operation maps directly to an optimized CMSIS-DSP C kernel, the +forward pass uses that kernel. For example, dot products use +`arm_dot_prod_f32`, and fully connected matrix-vector products use +`arm_mat_vec_mult_f32`. These kernels provide implementations optimized for +the selected Arm target, including Helium implementations where available. + +The CMSIS-DSP C++ extension is particularly valuable during the backward pass. +Reverse rules frequently combine several element-wise operations with an +accumulation. A typical example is: + +```text +gradient += input * output_gradient +``` + +Calling separate C kernels for the multiplication and addition would normally +require an intermediate buffer and two loops: one loop produces the products +and another accumulates them. The C++ expression system can fuse the complete +expression into one loop. That loop performs more useful computation for each +load and store, avoids the temporary buffer, reduces memory traffic, and gives +the compiler a larger loop body to vectorize effectively. + +The fully connected backward pass illustrates this approach: + +```text +bias_gradient += output_gradient +weight_gradient += outer(output_gradient, input_value) +input_gradient += transpose(weight_value) * output_gradient +``` + +Bias and outer-product accumulation use fused CMSIS-DSP C++ expressions. The +input gradient uses optimized dot products over strided column views, avoiding +a materialized matrix transpose. The same principle applies to future vector, +matrix, and tensor reverse rules: express the whole local gradient update as a +fused accumulation whenever the C++ extension supports it. + +This fusion capability is an important distinction when selecting a Helium +math library for training. A library may provide individually optimized +Helium primitives but still require multiple loops and intermediate buffers +for a compound backward expression. CMSIS-DSP combines optimized kernels with +a C++ expression mechanism capable of fusing those loops, which is especially +important for backward passes because they contain more compound operations +and accumulations than typical forward inference code. + +## What are the arena and the tape? + +Reverse AD needs to remember the operations performed during the forward +calculation. It later visits those operations in the opposite order to +propagate derivatives from the output back to the inputs. This ordered record +of operations is traditionally called a **tape**, by analogy with recording a +sequence on magnetic tape and playing it backward. The name is standard AD +terminology; it is not related to a C++ container type. + +In this implementation, `Tape` does two jobs: + +1. During the forward calculation, it writes one small, fixed-size record for + each operation. A record contains non-owning buffer pointers and the + information needed by that operation's derivative rule. +2. `backward()` follows those records in reverse order and accumulates the + gradients. + +The tape core contains no numerical operators. Each operator is a separate, +self-contained class and header containing its validation, forward rule, tape +record, gradient reset, backward rule, and expression adapter: + +| Header | Operator class | Expression | +| --- | --- | --- | +| `operators/add.hpp` | `AddOperator` | `a + b` | +| `operators/dot.hpp` | `DotOperator` | `dot(a, b)` | +| `operators/scale.hpp` | `ScaleOperator` | `scale(x, alpha, beta)` | +| `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | +| `operators/relu.hpp` | `ReluOperator` | `relu(x)` | +| `operators/quadratic_error.hpp` | `QuadraticErrorOperator` | `quadratic_error(prediction, target)` | + +An application includes and registers only the operators it uses. An operator +header that is not included is not part of that translation unit. Registration +uses an allocation-free, open-addressed hash set of type tokens stored directly +in `Tape`; it does not instantiate or retain an operator object. Registration +and expression checks are expected O(1), with collision resolution by linear +probing and no deletion or tombstones. + +Expression evaluation checks registration before validation or forward +computation. Using an included but unregistered operator leaves the output +unchanged and sets the sticky status to `Status::operator_not_registered`. +The registry defaults to 16 distinct operator types. Define +`DSPPP_AUTODIFF_MAX_OPERATORS` to a larger power of two before including the +core header when an application needs more slots. Exceeding the configured +capacity sets `Status::operator_registry_full`. `Tape::reset()` preserves +registrations, so they normally need to be installed only once during +application setup. + +The tape needs memory for gradients and operation records. An `Arena` +owns exactly `Bytes` bytes of fixed storage and constructs a `Tape` that uses +that storage. It does +**not** allocate a `std::vector`, call the heap, or grow at runtime. For example, +`Arena<2048>` contains a 2048-byte array directly inside the `Arena` object. If +the object is a local variable, that storage is normally on the stack; if it is +static, the storage is static as well. + +All value arrays, including intermediate and final outputs, are allocated by +the user. The way a buffer is registered tells the tape whether it needs a +gradient: + +- `tape.input(values)` registers ordinary algorithm input. It allocates no + gradient because the application does not request derivatives for it. +- `tape.parameter(values)` registers trainable parameters and allocates their + gradients from the arena. +- `tape.output(values)` registers an intermediate or final output and allocates + its adjoint from the arena because it is needed during reverse propagation. + +These functions never copy or take ownership of values. This role-based rule +is used by vector and matrix operators and is intended for future tensors as +well. Operation +records contain only pointers, dimensions, and small operator-specific +metadata, so their cost does not grow with the amount of numerical data. +Gradient storage naturally requires one `float` per parameter or intermediate +element, but no storage is spent on input gradients. + +Applications that need exact placement of every buffer can use the overloads +`tape.parameter(values, gradients, length)` and +`tape.output(values, gradients, length)` with caller-owned gradient storage. +These advanced overloads perform no arena allocation for the view. + +Large buffers may live on the stack, in static memory, or in an +application-owned memory pool. They must remain alive until `backward()` +returns. Input values must not change between their use in the forward pass and +the backward pass, because derivative rules may read them. Output buffers must +be distinct from input buffers in this reference implementation. Recorded +computations use SSA-style storage: do not reuse or overwrite an earlier live +output buffer for another operation before `backward()` or `Tape::reset()`. + +## Basic use + +```cpp +#include +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +Arena<2048> arena; // Gradient and operation-record storage. +Tape &tape = arena.tape(); +tape.register_operator(); +tape.register_operator(); +tape.register_operator(); + +float x_value[] = {1.0F, 2.0F}; +float alpha_value = 2.0F; +float beta_value[] = {3.0F, 4.0F}; +float scaled_value[2] = {}; +float sum_value[2] = {}; +float result_value[1] = {}; + +BufferView x = tape.input(x_value); // No gradient for x. +BufferView alpha = tape.parameter(alpha_value); // Scalar parameter. +BufferView beta = tape.parameter(beta_value); // Vector parameter. +BufferView scaled = tape.output(scaled_value); +BufferView sum = tape.output(sum_value); +BufferView result = tape.output(result_value); + +scaled = scale(x, alpha, beta); // scaled = alpha * x + beta +sum = scaled + x; +result = dot(sum, x); // result[0] == 26 + +if (tape.backward(result)) { + // x.has_gradient() == false + // alpha.gradient(0) == 5 + // beta gradients are {1, 2} +} +``` + +## Training, optimizers, and frozen parameters + +`Adam` and +`RMSProp` keep all optimizer state in +fixed-size arrays inside the optimizer object. `MaximumElements` is the total +number of scalar parameter values and `MaximumParameters` is the maximum +number of separately registered parameter views (16 by default). Neither +optimizer allocates memory or throws exceptions. + +### RMSProp template and constructor arguments + +The two RMSProp template arguments are compile-time storage capacities, not +algorithm hyperparameters: + +```cpp +RMSProp optimizer; +``` + +- `MaximumElements` is the maximum total number of scalar values across all + parameter views added to the optimizer. It has no default. +- `MaximumParameters` is the maximum number of separately added parameter + views. It defaults to 16. + +For example, the polynomial regression has a three-element coefficient vector +and a separate scalar bias: + +```cpp +RMSProp<4, 2> optimizer; +optimizer.add(coefficients); // Three elements; first parameter view. +optimizer.add(bias); // One element; second parameter view. +``` + +This consumes all four element slots and both parameter-view slots. A matrix +counts as one parameter view, while its flattened `rows * columns` values all +count toward `MaximumElements`. Frozen parameters still occupy their original +slots. Adding the same value buffer again is idempotent and does not consume +another slot. + +These capacities determine the optimizer object's static memory footprint. +RMSProp contains one `float square_average_[MaximumElements]` array plus an +`Entry entries_[MaximumParameters]` metadata array. Each entry stores the +parameter value and gradient pointers, its length, its state-array offset, and +whether it is trainable. Parameter values and gradients are not copied into +the optimizer: values remain caller-owned and gradients remain in the tape +arena (or in caller storage when that overload is used). There is no heap +allocation and neither capacity can grow at runtime. + +When only the first argument is specified, the 16-view default applies: + +```cpp +RMSProp<100> optimizer; // Up to 100 scalar values in up to 16 views. +``` + +The constructor arguments configure the numerical RMSProp update: + +```cpp +RMSProp<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.99F, // alpha: squared-gradient moving-average decay + 1.0e-8F // epsilon: denominator stabilization +); +``` + +For each trainable scalar parameter, `step()` performs: + +```text +square_average = alpha * square_average + + (1 - alpha) * gradient^2 +parameter -= learning_rate * gradient / (sqrt(square_average) + epsilon) +``` + +If `add()` would exceed `MaximumParameters`, it returns `false` and sets +`OptimizerStatus::too_many_parameters`. If the total number of scalar values +would exceed `MaximumElements`, it sets `OptimizerStatus::too_many_elements`. +A non-parameter view, a missing gradient, or an unknown view passed to +`freeze()` sets `OptimizerStatus::invalid_parameter`. Optimizer errors are +sticky: after an error, `good()` is false, `status()` reports the first error, +and `step()` returns false. + +### Adam template and constructor arguments + +Adam uses the same two compile-time capacity arguments as RMSProp: + +```cpp +Adam optimizer; +``` + +- `MaximumElements` is the maximum total number of scalar parameter values. +- `MaximumParameters` is the maximum number of parameter views and defaults + to 16. + +Consequently, the same regression parameters fit in: + +```cpp +#include + +Adam<4, 2> optimizer; +optimizer.add(coefficients); // Three scalar values in one view. +optimizer.add(bias); // One scalar value in a second view. +``` + +Matrices, frozen parameters, and duplicate registrations are counted in the +same way as for RMSProp. `Adam<100>` means up to 100 scalar values distributed +across the default maximum of 16 parameter views. + +Adam keeps two state values per scalar parameter, so its principal state +storage is twice that of RMSProp: + +```text +float first_moment_[MaximumElements] +float second_moment_[MaximumElements] +Entry entries_[MaximumParameters] +``` + +It additionally stores scalar configuration values and the current powers of +`beta1` and `beta2` used for bias correction. Parameter values and gradients +are referenced through the metadata entries and are not copied. Adam performs +no heap allocation. + +The constructor arguments are: + +```cpp +Adam<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.9F, // beta1: first-moment decay + 0.999F, // beta2: second-moment decay + 1.0e-8F // epsilon: denominator stabilization +); +``` + +On successful optimizer step `t`, each trainable scalar is updated as follows: + +```text +first_moment = beta1 * first_moment + + (1 - beta1) * gradient +second_moment = beta2 * second_moment + + (1 - beta2) * gradient^2 + +corrected_first = first_moment / (1 - beta1^t) +corrected_second = second_moment / (1 - beta2^t) + +parameter -= learning_rate * corrected_first + / (sqrt(corrected_second) + epsilon) +``` + +The correction compensates for moments initialized to zero, particularly +during the first training steps. Adam's global step advances whenever +`step()` succeeds. Frozen entries are skipped: their parameter values and two +moment arrays remain unchanged, although they continue to occupy capacity. + +Adam reports the same sticky `OptimizerStatus` values as RMSProp: +`too_many_parameters`, `too_many_elements`, and `invalid_parameter`. +`add()`, `freeze()`, `good()`, `status()`, `zero_grad()`, and `step()` therefore +have the same usage pattern for both optimizer types. Switching optimizers in +a training loop normally requires only changing the included header, the +optimizer type, its capacities, and its numerical hyperparameters. + +The user owns the training loop. Register each parameter once, evaluate the +graph, run the reverse pass, and then update the parameters: + +```cpp +#include + +RMSProp<4, 2> optimizer(1.0e-3F); // Four scalar values in two views. +optimizer.add(coefficients); +optimizer.add(bias); + +tape.begin_graph(); // Everything allocated so far remains persistent. +for (std::size_t step = 0; step < number_of_steps; ++step) { + tape.rewind_graph(); // Reclaim records from the preceding iteration. + + for (std::size_t sample = 0; sample < sample_count; ++sample) { + // Scalar output views share the corresponding elements of the + // persistent vector gradient buffers. + BufferView x = tape.input(feature_value[sample]); + BufferView p = tape.output(&polynomial_value[sample], + &polynomial.gradients()[sample], 1); + BufferView y = tape.output(&prediction_value[sample], + &prediction.gradients()[sample], 1); + p = dot(x, coefficients); + y = p + bias; + } + + // prediction and target cover the complete training set. + loss = quadratic_error(prediction, target); + + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) { + handle_error(); + } +} +``` + +`begin_graph()` places an arena mark after persistent gradient buffers. +`rewind_graph()` returns to that mark in constant time, preserving the views, +gradient buffers, parameter values, and operator registrations while releasing +the previous iteration's operation records. + +Parameters can be frozen without rebuilding the graph. A frozen parameter +still participates in forward and backward propagation, but `step()` leaves +its value and optimizer state unchanged: + +```cpp +freeze_parameters(optimizer, coefficients); // Bias-only fine tuning. +unfreeze_parameters(optimizer, coefficients); // Train it again later. +``` + +This API freezes parameter views rather than operator classes because an +operator is stateless and several invocations can use different parameters. +To freeze a layer, pass all parameter views owned by that layer. Both optimizer +implementations return `false` and set a sticky `OptimizerStatus` when an +operation exceeds capacity or receives an invalid parameter view. + +Quadratic error computes the sum, not the mean: + +```text +loss = sum((prediction[i] - target[i])^2) +d(loss)/d(prediction[i]) = 2 * (prediction[i] - target[i]) +``` + +The target must be an input view; gradients are retained only for the +prediction path and ultimately for its parameters. + +### Polynomial sinusoid regression + +`dsppp/Examples/autodiff_regression.cpp` follows the polynomial PyTorch example without a +fully connected or ReLU node. For every sample it constructs the caller-owned +feature vector `{x, x^2, x^3}` and evaluates: + +```cpp +polynomial = dot(features, coefficients); +prediction = polynomial + bias; +loss = quadratic_error(prediction, target); +``` + +RMSProp learns `bias + c1*x + c2*x^2 + c3*x^3` from 100 uniformly spaced +points over `[-pi, pi]`. Each training step first computes all 100 predictions. +It then creates one quadratic-error node over both complete vectors, calls +`backward()` once, and calls the optimizer once. The optimized objective is +therefore the global sum +`sum((prediction[sample] - target[sample])^2)`, rather than 100 independent +online updates. Dividing the reported value by 100 gives its mean quadratic +error without changing the optimum. + +The example also demonstrates bias-only fine tuning and saving and restoring +values through the C-compatible `SinePolynomialParameters` struct. +The checkpoint deliberately contains parameter values only. Continuing +training with exactly the same optimizer trajectory would additionally require +persisting RMSProp or Adam state; inference does not need that state. + +### Fully connected and ReLU + +A fully connected node computes `y = m * x + b`. `m` is a row-major matrix +parameter, `b` is a vector parameter, and the number of matrix columns must +match the input length. The number of rows must match both the bias and output +lengths: + +```cpp +#include +#include + +tape.register_operator(); +tape.register_operator(); +``` + +```cpp +float x_value[] = {2.0F, -1.0F}; +float m_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; +float b_value[] = {1.0F, 0.0F}; +float linear_value[2] = {}; +float activation_value[2] = {}; + +BufferView x = tape.input(x_value); +MatrixView m = tape.parameter(m_value); +BufferView b = tape.parameter(b_value); +BufferView linear = tape.output(linear_value); +BufferView activation = tape.output(activation_value); + +linear = fully_connected(x, m, b); // linear == {1, -7} +activation = relu(linear); // activation == {1, 0} + +const float seed[] = {1.0F, 1.0F}; +tape.backward(activation, seed, 2); +``` + +The fully connected backward rule is: + +```text +m_gradient[row, column] += y_gradient[row] * x_value[column] +b_gradient[row] += y_gradient[row] +x_gradient[column] += y_gradient[row] * m_value[row, column] +``` + +The forward matrix-vector product uses `arm_mat_vec_mult_f32`, followed by a +CMSIS-DSP C++ vector expression that adds the bias. The reverse pass uses fused +C++ expressions for bias accumulation and +`m_gradient += outer(y_gradient, x_value)`. When an input gradient is needed, +each row-major weight column is exposed as a strided view and accumulated with +the C++ dot implementation. This avoids a transposed matrix and all temporary +numerical buffers. Because the C matrix descriptor stores dimensions as +`uint16_t`, larger dimensions are rejected as a shape mismatch. + +The last line is skipped when `x` is an `input`. It is used when `x` is an +intermediate output, allowing several fully connected and activation nodes to +be chained while gradients are ultimately retained only for parameters. + +ReLU is element-wise. It propagates the output gradient when its input value is +strictly positive and propagates zero for negative values and at zero. + +### How buffer length is determined + +In the example, the vector buffers are actual fixed-size C arrays. The overload +below illustrates how each registration function receives an array by +reference, so the compiler deduces `Length` without storing runtime size +information in the array: + +```cpp +template +BufferView Tape::input(float (&values)[Length]); +``` + +This deduction only works while the expression still has an array type. Once +an array is converted to `float *`, its length is not available in C++ and +cannot be inferred safely. Buffers obtained dynamically, from a memory pool, or +through a pointer therefore use the explicit-length overload: + +```cpp +float *values = application_pool_allocate(number_of_elements); +BufferView dynamic_input = tape.input(values, number_of_elements); +``` + +The autodiff implementation does not allocate or free `values`; it only +allocates the corresponding gradient buffer in its fixed arena. The same rule +applies to matrix and future tensor views: dimensions can be deduced from true +array types when available, but pointer-based storage must supply its shape. + +For example, `tape.parameter(float_matrix)` deduces both dimensions from a true +`float[Rows][Columns]` array. A dynamically allocated row-major matrix uses +`tape.parameter(pointer, rows, columns)`. + +The same tape may instead use any caller-owned buffer: + +```cpp +alignas(std::max_align_t) unsigned char memory[2048]; +Tape tape(memory, sizeof(memory)); +``` + +`Tape::reset()` releases all arena-managed gradients and operation records in +constant time, without individual deallocation. It does not release or modify +caller-owned value buffers. All views become invalid after reset and must be +created again. + +## Value-only evaluation + +Recording can be disabled while an output is computed. Operators still write +the numerical result into the user-provided output buffer, but consume no +additional arena space for operation records and that output cannot be used as +the root of `backward()`: + +```cpp +const std::size_t before = tape.used(); +{ + RecordingScope no_gradient(tape, false); + scaled = scale(x, alpha, beta); + sum = scaled + x; + result = dot(sum, x); + use(result_value[0]); +} +// tape.used() == before +``` + +Calling `backward()` on a value-only output fails with +`Status::invalid_output`. The views and buffers may be reused later in a +recorded calculation. + +## Arena and failure model + +The implementation performs no `new`, `delete`, `malloc`, or standard-container +allocation. Placement `new` only starts the lifetime of records inside the +arena supplied by the caller. It does not request memory. + +There are no C++ exceptions. Errors are reported through `Tape::status()` and +the Boolean result of `backward()`. The first error is sticky: + +- `out_of_memory`: a gradient buffer or operation record did not fit in the + arena; +- `tape_mismatch`: views came from different tapes, dimensions or required + roles differ, or a required buffer is null; +- `invalid_output`: `backward()` received a value-only output or an invalid + seed; +- `operator_not_registered`: an expression used an operator type that was not + registered on this tape; +- `operator_registry_full`: the fixed registration list has no free slot. + +If an operation record exhausts the remaining arena, the operation still +computes its numeric value but the result is detached. A view whose gradient +allocation fails is invalid. Always check `tape.good()` or `backward()` before +consuming derivatives. `used()` can be measured on representative worst-case +graphs to select a static arena size. + +## How the reverse pass works + +Each recorded operation appends one fixed-size record. The common `Node` prefix +stores links and pointers to the operation's gradient reset and backward rules. +The rest of an operation record contains non-owning buffer pointers and +dimensions. +`backward()` first clears the associated gradient buffers, then walks the +linked tape in reverse creation order. + +For vector add `z[i] = x[i] + y[i]`, the local rule is: + +```text +x_gradient[i] += z_gradient[i] +y_gradient[i] += z_gradient[i] +``` + +For `z = dot(x, y)`, it is: + +```text +x_gradient[i] += z_gradient[0] * y_value[i] +y_gradient[i] += z_gradient[0] * x_value[i] +``` + +An addition or dot-product operand contributes to a gradient only when it is a +parameter or intermediate with gradient storage. An `input` has a null gradient +pointer, so the same backward rule simply skips that contribution. + +For vector scaling `z[i] = alpha[0] * x[i] + beta[i]`, `x` is required to be an +input while `alpha` and `beta` are required to be parameters. Its rule is: + +```text +alpha_gradient[0] += z_gradient[i] * x_value[i] (summed over i) +beta_gradient[i] += z_gradient[i] +``` + +There is intentionally no `x_gradient`: the role declared by `tape.input(x)` +states that the caller does not request it. + +All operators follow the same ownership rule: inputs and outputs stay in +caller storage, while their records retain non-owning pointers. Future matrix +and tensor operators should use views with shape and stride metadata rather +than copying numerical buffers into the tape. + +## Adding an operator + +The intended extension pattern is: + +1. Create one header in `operators/` and one uniquely named operator class. + That class identity is also its runtime registration token. +2. Define a trivially destructible record whose first member is `detail::Node`. + Store only non-owning buffer pointers and small shape/stride metadata. +3. Keep the forward evaluator, gradient reset, and backward rule in the + operator class. Every path must be `noexcept`. +4. At the start of evaluation, call + `OperatorAccess::require(tape)`. Use `OperatorAccess` for + validation, recording state, status reporting, and appending the record. +5. Add a small expression class with `evaluate(BufferView&)`; the generic + `BufferView::operator=` invokes it, so the core never needs modification. +6. Test registered execution, unregistered failure, value-only execution, + derivatives, tape exhaustion, shapes, aliases, and buffer lifetimes. + +Do not add an operator-specific method or record to `Tape`. Do not store +pointers to temporary caller data. If an operator needs a large forward +intermediate during its backward rule, make it an explicit caller-provided +workspace or output view rather than copying it into the tape arena. + +This implementation is intentionally contiguous and sequential. It is not +thread safe, does not manage buffer lifetimes, and does not yet support strides +or higher derivatives. A training graph is reevaluated on every iteration; +only its arena storage is reused. + +## Building and running the board test + +Autodiff is tested only with the existing dsppp board-test infrastructure; it +has no standalone host CMake project. `dsppp/tests/autodiff_test.cpp` is listed +in `dsppp/tests/test.cproject.yml`, and `AUTODIFF_TEST` is a test category in +`dsppp/run_all.py`. + +From the `dsppp` directory, select the category and its supported datatype: + +```sh +python run_all.py --test AUTODIFF_TEST --dt F32_DT +``` + +`run_all.py` writes `test_config.h` and rebuilds when that generated +configuration changes. The test body is compiled and executed only when all +three generated selections are present: + +```cpp +#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) +``` + +Autodiff currently supports only `float` and the dynamic test mode. Other +datatype or static-mode configurations retain an empty `autodiff_test()` entry +point, so the shared project can still build without executing unsupported +autodiff cases. Board compiler options, CMSIS-DSP sources, linking, and runtime +selection continue to come from the existing solution and layer files. diff --git a/dsppp/Include/dsppp/autodiff/operators/add.hpp b/dsppp/Include/dsppp/autodiff/operators/add.hpp new file mode 100644 index 000000000..ff804b239 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/add.hpp @@ -0,0 +1,100 @@ +#pragma once + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class AddOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + float *left_gradient; + float *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + record.output_gradient[i] = 0.0F; + if (record.left_gradient != nullptr) record.left_gradient[i] = 0.0F; + if (record.right_gradient != nullptr) record.right_gradient[i] = 0.0F; + } + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + if (record.left_gradient != nullptr) + record.left_gradient[i] += record.output_gradient[i]; + if (record.right_gradient != nullptr) + record.right_gradient[i] += record.output_gradient[i]; + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + OperatorAccess::values(output)[i] = + OperatorAccess::values(left)[i] + OperatorAccess::values(right)[i]; + if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class AddExpression +{ +public: + AddExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + AddOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +inline AddExpression operator+(const BufferView &left, + const BufferView &right) noexcept +{ + return AddExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp + diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp new file mode 100644 index 000000000..75e15c84d --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -0,0 +1,124 @@ +#pragma once + +#include + +#include +#include +#include + + +namespace arm_cmsis_dsp { +namespace autodiff { + +class DotOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *left_value; + float *left_gradient; + const float *right_value; + float *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = 0.0F; + + if (record.left_gradient != nullptr) + { + VectorView left_grad(const_cast(record.left_gradient), 0, record.length); + left_grad = 0.0F; + } + if (record.right_gradient != nullptr) + { + VectorView right_grad(const_cast(record.right_gradient), 0, record.length); + right_grad = 0.0F; + } + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const float gradient = record.output_gradient[0]; + if (gradient == 0.0F) return; + + if (record.left_gradient != nullptr) + { + VectorView left_grad(const_cast(record.left_gradient) , 0, record.length); + VectorView right_val(const_cast(record.right_value), 0, record.length); + left_grad += right_val * gradient; + } + if (record.right_gradient != nullptr) + { + VectorView right_grad(const_cast(record.right_gradient), 0, record.length); + VectorView left_val(const_cast(record.left_value), 0, record.length); + right_grad += left_val * gradient; + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, left, right) || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + float value = 0.0F; + //for (std::size_t i = 0; i < OperatorAccess::length(left); ++i) + // value += OperatorAccess::values(left)[i] * OperatorAccess::values(right)[i]; + + arm_dot_prod_f32(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::length(left),&value); + OperatorAccess::values(output)[0] = value; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(left); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class DotExpression +{ +public: + DotExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + DotOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +inline DotExpression dot(const BufferView &left, const BufferView &right) noexcept +{ + return DotExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp + diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp new file mode 100644 index 000000000..7e4b19329 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -0,0 +1,210 @@ +#pragma once + +#include + +#include +#include +#include + +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class FullyConnectedOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *input_value; + float *input_gradient; + const float *weight_value; + float *weight_gradient; + float *bias_gradient; + std::size_t rows; + std::size_t columns; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.rows != 0U) + { + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.rows); + ::arm_cmsis_dsp::VectorView bias_gradient( + record.bias_gradient, 0, record.rows); + output_gradient = 0.0F; + bias_gradient = 0.0F; + } + if (record.rows != 0U && record.columns != 0U) + { + ::arm_cmsis_dsp::MatrixView + weight_gradient(record.weight_gradient, record.rows, + record.columns, record.columns); + weight_gradient = 0.0F; + } + if (record.input_gradient != nullptr && record.columns != 0U) + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.columns); + input_gradient = 0.0F; + } + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.rows == 0U) return; + + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.rows); + ::arm_cmsis_dsp::VectorView bias_gradient( + record.bias_gradient, 0, record.rows); + + // Bias is shared by every input sample, so its gradient must be + // accumulated instead of overwritten. + bias_gradient += output_gradient; + + if (record.columns != 0U) + { + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value), 0, record.columns); + ::arm_cmsis_dsp::MatrixView + weight_gradient(record.weight_gradient, record.rows, + record.columns, record.columns); + + // dW = dy outer x. The C++ expression engine fuses the multiply + // and accumulation without constructing an outer-product buffer. + weight_gradient += + ::arm_cmsis_dsp::outer(output_gradient, input_value); + + if (record.input_gradient == nullptr) return; + + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.columns); + for (std::size_t column = 0; column < record.columns; ++column) + { + // A column is strided in the row-major weight matrix. The + // C++ dot implementation handles that view directly, so no + // transposed matrix or temporary vector is needed. + ::arm_cmsis_dsp::VectorView + weight_column(const_cast(record.weight_value), + column, record.rows * record.columns, + record.columns); + input_gradient[column] += + ::arm_cmsis_dsp::dot(weight_column, output_gradient); + } + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights, + const BufferView &bias) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::require(*tape)) + return false; + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + if (!OperatorAccess::valid(*tape, output) || + !OperatorAccess::valid(*tape, input) || + !OperatorAccess::valid(*tape, weight_buffer) || + !OperatorAccess::valid(*tape, bias) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::length(output) != OperatorAccess::rows(weights) || + OperatorAccess::length(input) != OperatorAccess::columns(weights) || + OperatorAccess::length(bias) != OperatorAccess::rows(weights) || + OperatorAccess::rows(weights) > + std::numeric_limits::max() || + OperatorAccess::columns(weights) > + std::numeric_limits::max() || + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::role(bias) != BufferRole::parameter) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t columns = OperatorAccess::columns(weights); + if (rows != 0U) + { + ::arm_cmsis_dsp::VectorView output_value( + OperatorAccess::values(output), 0, rows); + ::arm_cmsis_dsp::VectorView bias_value( + const_cast(OperatorAccess::values(bias)), 0, rows); + + if (columns != 0U) + { + arm_matrix_instance_f32 weight_matrix; + arm_mat_init_f32( + &weight_matrix, static_cast(rows), + static_cast(columns), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_vec_mult_f32(&weight_matrix, + OperatorAccess::values(input), + OperatorAccess::values(output)); + output_value += bias_value; + } + else + { + // VectorView deliberately deletes copy assignment. This + // dimension-zero edge has no matrix product to optimize. + for (std::size_t row = 0; row < rows; ++row) + OperatorAccess::values(output)[row] = + OperatorAccess::values(bias)[row]; + } + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->weight_value = OperatorAccess::values(weight_buffer); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->bias_gradient = OperatorAccess::gradients(bias); + record->rows = OperatorAccess::rows(weights); + record->columns = OperatorAccess::columns(weights); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class FullyConnectedExpression +{ +public: + FullyConnectedExpression(const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept + : input_(input), weights_(weights), bias_(bias) {} + void evaluate(BufferView &output) const noexcept + { + FullyConnectedOperator::evaluate(output, input_, weights_, bias_); + } +private: + BufferView input_; + MatrixView weights_; + BufferView bias_; +}; + +inline FullyConnectedExpression fully_connected( + const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept +{ + return FullyConnectedExpression(input, weights, bias); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp new file mode 100644 index 000000000..90f67a2bc --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp @@ -0,0 +1,104 @@ +#pragma once + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Sum-of-squared-errors loss: sum((prediction - target)^2). */ +class QuadraticErrorOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *prediction_value; + float *prediction_gradient; + const float *target_value; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = 0.0F; + for (std::size_t i = 0; i < record.length; ++i) + record.prediction_gradient[i] = 0.0F; + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const float seed = record.output_gradient[0]; + if (seed == 0.0F) return; + for (std::size_t i = 0; i < record.length; ++i) + record.prediction_gradient[i] += 2.0F * seed * + (record.prediction_value[i] - record.target_value[i]); + } + +public: + static bool evaluate(BufferView &output, const BufferView &prediction, + const BufferView &target) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, prediction, target) || + OperatorAccess::gradients(prediction) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + float value = 0.0F; + for (std::size_t i = 0; i < OperatorAccess::length(prediction); ++i) + { + const float error = OperatorAccess::values(prediction)[i] - + OperatorAccess::values(target)[i]; + value += error * error; + } + OperatorAccess::values(output)[0] = value; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->prediction_value = OperatorAccess::values(prediction); + record->prediction_gradient = OperatorAccess::gradients(prediction); + record->target_value = OperatorAccess::values(target); + record->length = OperatorAccess::length(prediction); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class QuadraticErrorExpression +{ +public: + QuadraticErrorExpression(const BufferView &prediction, + const BufferView &target) noexcept + : prediction_(prediction), target_(target) {} + void evaluate(BufferView &output) const noexcept + { + QuadraticErrorOperator::evaluate(output, prediction_, target_); + } +private: + BufferView prediction_; + BufferView target_; +}; + +inline QuadraticErrorExpression quadratic_error( + const BufferView &prediction, const BufferView &target) noexcept +{ + return QuadraticErrorExpression(prediction, target); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/relu.hpp b/dsppp/Include/dsppp/autodiff/operators/relu.hpp new file mode 100644 index 000000000..ff15bbb12 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/relu.hpp @@ -0,0 +1,89 @@ +#pragma once + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class ReluOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *input_value; + float *input_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + record.output_gradient[i] = 0.0F; + if (record.input_gradient != nullptr) record.input_gradient[i] = 0.0F; + } + } + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + for (std::size_t i = 0; i < record.length; ++i) + if (record.input_value[i] > 0.0F) + record.input_gradient[i] += record.output_gradient[i]; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + for (std::size_t i = 0; i < OperatorAccess::length(input); ++i) + OperatorAccess::values(output)[i] = + OperatorAccess::values(input)[i] > 0.0F ? + OperatorAccess::values(input)[i] : 0.0F; + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->length = OperatorAccess::length(input); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class ReluExpression +{ +public: + explicit ReluExpression(const BufferView &input) noexcept : input_(input) {} + void evaluate(BufferView &output) const noexcept + { + ReluOperator::evaluate(output, input_); + } +private: + BufferView input_; +}; + +inline ReluExpression relu(const BufferView &input) noexcept +{ + return ReluExpression(input); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/scale.hpp b/dsppp/Include/dsppp/autodiff/operators/scale.hpp new file mode 100644 index 000000000..fecc6f201 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/scale.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class ScaleOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *input_value; + float *alpha_gradient; + float *beta_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.alpha_gradient[0] = 0.0F; + for (std::size_t i = 0; i < record.length; ++i) + { + record.output_gradient[i] = 0.0F; + record.beta_gradient[i] = 0.0F; + } + } + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + const float gradient = record.output_gradient[i]; + if (gradient != 0.0F) + record.alpha_gradient[0] += gradient * record.input_value[i]; + record.beta_gradient[i] += gradient; + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &alpha, + const BufferView &beta) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::valid(*tape, output) || + !OperatorAccess::valid(*tape, input) || + !OperatorAccess::valid(*tape, alpha) || + !OperatorAccess::valid(*tape, beta) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::length(output) != OperatorAccess::length(input) || + OperatorAccess::length(beta) != OperatorAccess::length(input) || + OperatorAccess::length(alpha) != 1U || + OperatorAccess::role(input) != BufferRole::input || + OperatorAccess::role(alpha) != BufferRole::parameter || + OperatorAccess::role(beta) != BufferRole::parameter) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + const float alpha_value = OperatorAccess::values(alpha)[0]; + for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + OperatorAccess::values(output)[i] = alpha_value * + OperatorAccess::values(input)[i] + OperatorAccess::values(beta)[i]; + if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->alpha_gradient = OperatorAccess::gradients(alpha); + record->beta_gradient = OperatorAccess::gradients(beta); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class ScaleExpression +{ +public: + ScaleExpression(const BufferView &input, const BufferView &alpha, + const BufferView &beta) noexcept + : input_(input), alpha_(alpha), beta_(beta) {} + void evaluate(BufferView &output) const noexcept + { + ScaleOperator::evaluate(output, input_, alpha_, beta_); + } +private: + BufferView input_; + BufferView alpha_; + BufferView beta_; +}; + +inline ScaleExpression scale(const BufferView &input, const BufferView &alpha, + const BufferView &beta) noexcept +{ + return ScaleExpression(input, alpha, beta); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp new file mode 100644 index 000000000..f97b203cf --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp @@ -0,0 +1,149 @@ +#pragma once + +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage Adam optimizer for use in a user-written training loop. */ +template +class Adam +{ + static_assert(MaximumElements > 0U, "Adam needs state storage"); + static_assert(MaximumParameters > 0U, "Adam needs parameter slots"); + struct Entry + { + float *values; + float *gradients; + std::size_t length; + std::size_t offset; + bool trainable; + }; + +public: + explicit Adam(float learning_rate = 1.0e-3F, float beta1 = 0.9F, + float beta2 = 0.999F, float epsilon = 1.0e-8F) noexcept + : learning_rate_(learning_rate), beta1_(beta1), beta2_(beta2), + epsilon_(epsilon), beta1_power_(1.0F), beta2_power_(1.0F), + parameter_count_(0U), element_count_(0U), status_(OptimizerStatus::ok), + entries_{}, first_moment_{}, second_moment_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + for (std::size_t i = 0; i < entries_[p].length; ++i) + entries_[p].gradients[i] = 0.0F; + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + beta1_power_ *= beta1_; + beta2_power_ *= beta2_; + const float first_correction = 1.0F - beta1_power_; + const float second_correction = 1.0F - beta2_power_; + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + for (std::size_t i = 0; i < entry.length; ++i) + { + const std::size_t state = entry.offset + i; + const float gradient = entry.gradients[i]; + first_moment_[state] = beta1_ * first_moment_[state] + + (1.0F - beta1_) * gradient; + second_moment_[state] = beta2_ * second_moment_[state] + + (1.0F - beta2_) * gradient * gradient; + const float corrected_first = + first_moment_[state] / first_correction; + const float corrected_second = + second_moment_[state] / second_correction; + entry.values[i] -= learning_rate_ * corrected_first / + (std::sqrt(corrected_second) + epsilon_); + } + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(float *values, float *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = + Entry{values, gradients, length, element_count_, true}; + element_count_ += length; + return true; + } + + bool set_trainable(const float *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + float learning_rate_; + float beta1_; + float beta2_; + float epsilon_; + float beta1_power_; + float beta2_power_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; + float first_moment_[MaximumElements]; + float second_moment_[MaximumElements]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/common.hpp b/dsppp/Include/dsppp/autodiff/optimizers/common.hpp new file mode 100644 index 000000000..3f04db4cf --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/common.hpp @@ -0,0 +1,30 @@ +#pragma once + +namespace arm_cmsis_dsp { +namespace autodiff { + +enum class OptimizerStatus +{ + ok, + too_many_parameters, + too_many_elements, + invalid_parameter +}; + +/** Freeze all parameter views belonging to one logical operator/layer. */ +template +bool freeze_parameters(Optimizer &optimizer, Parameters... parameters) noexcept +{ + return (optimizer.freeze(parameters, true) && ...); +} + +/** Re-enable updates for a previously frozen operator/layer. */ +template +bool unfreeze_parameters(Optimizer &optimizer, + Parameters... parameters) noexcept +{ + return (optimizer.freeze(parameters, false) && ...); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp new file mode 100644 index 000000000..869857af7 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp @@ -0,0 +1,131 @@ +#pragma once + +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage RMSProp optimizer without momentum or centering. */ +template +class RMSProp +{ + static_assert(MaximumElements > 0U, "RMSProp needs state storage"); + static_assert(MaximumParameters > 0U, "RMSProp needs parameter slots"); + + struct Entry + { + float *values; + float *gradients; + std::size_t length; + std::size_t offset; + bool trainable; + }; + +public: + explicit RMSProp(float learning_rate = 1.0e-3F, float alpha = 0.99F, + float epsilon = 1.0e-8F) noexcept + : learning_rate_(learning_rate), alpha_(alpha), epsilon_(epsilon), + parameter_count_(0U), element_count_(0U), + status_(OptimizerStatus::ok), entries_{}, square_average_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + for (std::size_t i = 0; i < entries_[p].length; ++i) + entries_[p].gradients[i] = 0.0F; + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + for (std::size_t i = 0; i < entry.length; ++i) + { + const std::size_t state = entry.offset + i; + const float gradient = entry.gradients[i]; + square_average_[state] = alpha_ * square_average_[state] + + (1.0F - alpha_) * gradient * gradient; + entry.values[i] -= learning_rate_ * gradient / + (std::sqrt(square_average_[state]) + epsilon_); + } + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(float *values, float *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = + Entry{values, gradients, length, element_count_, true}; + element_count_ += length; + return true; + } + bool set_trainable(const float *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + float learning_rate_; + float alpha_; + float epsilon_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; + float square_average_[MaximumElements]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/reverse.hpp b/dsppp/Include/dsppp/autodiff/reverse.hpp new file mode 100644 index 000000000..a42ac6247 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/reverse.hpp @@ -0,0 +1,684 @@ +// -*- C++ -*- +/** @file + * @brief Allocation-free reverse-mode AD core and operator registry. + */ +#pragma once + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +#ifndef DSPPP_AUTODIFF_MAX_OPERATORS +#define DSPPP_AUTODIFF_MAX_OPERATORS 16 +#endif + +class Tape; +class OperatorAccess; + +namespace detail { + +struct Node; +using BackwardFunction = void (*)(Node &) noexcept; +using ResetFunction = void (*)(Node &) noexcept; + +struct Node +{ + Node *previous; + BackwardFunction backward; + ResetFunction reset_gradient; +}; + +template +inline constexpr unsigned char operator_token = 0U; + +} // namespace detail + +enum class Status +{ + ok, + out_of_memory, + tape_mismatch, + invalid_output, + operator_not_registered, + operator_registry_full +}; + +enum class BufferRole +{ + input, + parameter, + intermediate +}; + +/** Non-owning view of caller values and an associated gradient buffer. */ +class BufferView +{ +public: + BufferView() noexcept + : values_(nullptr), gradients_(nullptr), length_(0U), tape_(nullptr), + producer_(nullptr), role_(BufferRole::input) + { + } + + float *values() noexcept { return values_; } + const float *values() const noexcept { return values_; } + float *gradients() noexcept { return gradients_; } + const float *gradients() const noexcept { return gradients_; } + std::size_t length() const noexcept { return length_; } + BufferRole role() const noexcept { return role_; } + bool has_gradient() const noexcept { return gradients_ != nullptr; } + + float &operator[](std::size_t index) noexcept { return values_[index]; } + const float &operator[](std::size_t index) const noexcept + { + return values_[index]; + } + float gradient(std::size_t index) const noexcept + { + return gradients_ == nullptr ? 0.0F : gradients_[index]; + } + + /** Evaluate any expression supplied by a separately included operator. */ + template + BufferView &operator=(const Expression &expression) noexcept + { + expression.evaluate(*this); + return *this; + } + +private: + BufferView(float *values, float *gradients, std::size_t length, + Tape *tape, BufferRole role) noexcept + : values_(values), gradients_(gradients), length_(length), tape_(tape), + producer_(nullptr), role_(role) + { + } + + float *values_; + float *gradients_; + std::size_t length_; + Tape *tape_; + detail::Node *producer_; + BufferRole role_; + + friend class Tape; + friend class OperatorAccess; +}; + +/** Non-owning row-major matrix parameter view. */ +class MatrixView +{ +public: + MatrixView() noexcept : buffer_(), rows_(0U), columns_(0U) {} + + std::size_t rows() const noexcept { return rows_; } + std::size_t columns() const noexcept { return columns_; } + float *values() noexcept { return buffer_.values(); } + const float *values() const noexcept { return buffer_.values(); } + float *gradients() noexcept { return buffer_.gradients(); } + const float *gradients() const noexcept { return buffer_.gradients(); } + std::size_t length() const noexcept { return rows_ * columns_; } + float &operator()(std::size_t row, std::size_t column) noexcept + { + return buffer_.values()[row * columns_ + column]; + } + const float &operator()(std::size_t row, + std::size_t column) const noexcept + { + return buffer_.values()[row * columns_ + column]; + } + float gradient(std::size_t row, std::size_t column) const noexcept + { + return buffer_.gradient(row * columns_ + column); + } + +private: + MatrixView(const BufferView &buffer, std::size_t rows, + std::size_t columns) noexcept + : buffer_(buffer), rows_(rows), columns_(columns) + { + } + + BufferView buffer_; + std::size_t rows_; + std::size_t columns_; + + friend class Tape; + friend class OperatorAccess; +}; + +/** Reverse-mode tape using caller-supplied storage and a fixed operator list. */ +class Tape +{ +public: + static constexpr std::size_t maximum_registered_operators = + DSPPP_AUTODIFF_MAX_OPERATORS; + static_assert(maximum_registered_operators > 0U, + "The operator registry must contain at least one slot"); + static_assert((maximum_registered_operators & + (maximum_registered_operators - 1U)) == 0U, + "The operator registry capacity must be a power of two"); + + Tape(void *storage, std::size_t bytes) noexcept + : storage_(static_cast(storage)), capacity_(bytes), + used_(0U), tail_(nullptr), recording_(true), status_(Status::ok), + graph_begin_(0U), graph_marked_(false), registered_count_(0U), + registered_operators_{} + { + } + + Tape(const Tape &) = delete; + Tape &operator=(const Tape &) = delete; + + /** Register an operator type once. Registration uses no arena storage. */ + template + bool register_operator() noexcept + { + const void *token = &detail::operator_token; + const std::size_t first = operator_hash(token); + for (std::size_t probe = 0U; + probe < maximum_registered_operators; ++probe) + { + const std::size_t slot = + (first + probe) & (maximum_registered_operators - 1U); + if (registered_operators_[slot] == token) + { + return true; + } + if (registered_operators_[slot] == nullptr) + { + registered_operators_[slot] = token; + ++registered_count_; + return true; + } + } + set_error(Status::operator_registry_full); + return false; + } + + template + bool is_operator_registered() const noexcept + { + const void *token = &detail::operator_token; + const std::size_t first = operator_hash(token); + for (std::size_t probe = 0U; + probe < maximum_registered_operators; ++probe) + { + const std::size_t slot = + (first + probe) & (maximum_registered_operators - 1U); + if (registered_operators_[slot] == token) + { + return true; + } + if (registered_operators_[slot] == nullptr) + { + return false; + } + } + return false; + } + + /** Release gradients and records. Operator registrations are preserved. */ + void reset() noexcept + { + used_ = 0U; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + graph_begin_ = 0U; + graph_marked_ = false; + } + + /** Mark all current arena allocations as persistent training state. */ + void begin_graph() noexcept + { + graph_begin_ = used_; + graph_marked_ = true; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + } + + /** Reclaim only operation records, preserving views and their gradients. */ + bool rewind_graph() noexcept + { + if (!graph_marked_) + { + set_error(Status::invalid_output); + return false; + } + used_ = graph_begin_; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + return true; + } + + std::size_t used() const noexcept { return used_; } + std::size_t capacity() const noexcept { return capacity_; } + Status status() const noexcept { return status_; } + bool good() const noexcept { return status_ == Status::ok; } + void clear_status() noexcept { status_ = Status::ok; } + bool recording() const noexcept { return recording_; } + void set_recording(bool enabled) noexcept { recording_ = enabled; } + + /** Generic active view; output() is clearer for application code. */ + BufferView view(float *values, std::size_t length) noexcept + { + if (length != 0U && values == nullptr) + { + set_error(Status::tape_mismatch); + return BufferView(values, nullptr, length, this, + BufferRole::intermediate); + } + if (length > static_cast(-1) / sizeof(float)) + { + set_error(Status::out_of_memory); + return BufferView(values, nullptr, length, this, + BufferRole::intermediate); + } + + float *gradients = nullptr; + if (length != 0U) + { + gradients = static_cast( + allocate(length * sizeof(float), alignof(float))); + if (gradients != nullptr) + { + for (std::size_t i = 0; i < length; ++i) + { + gradients[i] = 0.0F; + } + } + } + return BufferView(values, gradients, length, this, + BufferRole::intermediate); + } + + template + BufferView view(float (&values)[Length]) noexcept + { + return view(values, Length); + } + + BufferView view(float *values, float *gradients, + std::size_t length) noexcept + { + if (length != 0U && (values == nullptr || gradients == nullptr)) + { + set_error(Status::tape_mismatch); + } + return BufferView(values, gradients, length, this, + BufferRole::intermediate); + } + + template + BufferView view(float (&values)[Length], + float (&gradients)[Length]) noexcept + { + return view(values, gradients, Length); + } + + BufferView input(float *values, std::size_t length) noexcept + { + if (length != 0U && values == nullptr) + { + set_error(Status::tape_mismatch); + } + return BufferView(values, nullptr, length, this, BufferRole::input); + } + + template + BufferView input(float (&values)[Length]) noexcept + { + return input(values, Length); + } + + BufferView input(float &value) noexcept { return input(&value, 1U); } + + BufferView parameter(float *values, std::size_t length) noexcept + { + BufferView result = view(values, length); + result.role_ = BufferRole::parameter; + return result; + } + + template + BufferView parameter(float (&values)[Length]) noexcept + { + return parameter(values, Length); + } + + BufferView parameter(float &value) noexcept + { + return parameter(&value, 1U); + } + + BufferView parameter(float *values, float *gradients, + std::size_t length) noexcept + { + BufferView result = view(values, gradients, length); + result.role_ = BufferRole::parameter; + return result; + } + + template + BufferView parameter(float (&values)[Length], + float (&gradients)[Length]) noexcept + { + return parameter(values, gradients, Length); + } + + BufferView parameter(float &value, float &gradient) noexcept + { + return parameter(&value, &gradient, 1U); + } + + MatrixView parameter(float *values, std::size_t rows, + std::size_t columns) noexcept + { + if (columns != 0U && rows > static_cast(-1) / columns) + { + set_error(Status::out_of_memory); + return MatrixView(); + } + return MatrixView(parameter(values, rows * columns), rows, columns); + } + + template + MatrixView parameter(float (&values)[Rows][Columns]) noexcept + { + return parameter(&values[0][0], Rows, Columns); + } + + MatrixView parameter(float *values, float *gradients, std::size_t rows, + std::size_t columns) noexcept + { + if (columns != 0U && rows > static_cast(-1) / columns) + { + set_error(Status::out_of_memory); + return MatrixView(); + } + return MatrixView(parameter(values, gradients, rows * columns), rows, + columns); + } + + template + MatrixView parameter(float (&values)[Rows][Columns], + float (&gradients)[Rows][Columns]) noexcept + { + return parameter(&values[0][0], &gradients[0][0], Rows, Columns); + } + + BufferView output(float *values, std::size_t length) noexcept + { + return view(values, length); + } + + template + BufferView output(float (&values)[Length]) noexcept + { + return output(values, Length); + } + + BufferView output(float &value) noexcept { return output(&value, 1U); } + + BufferView output(float *values, float *gradients, + std::size_t length) noexcept + { + return view(values, gradients, length); + } + + template + BufferView output(float (&values)[Length], + float (&gradients)[Length]) noexcept + { + return output(values, gradients, Length); + } + + BufferView output(float &value, float &gradient) noexcept + { + return output(&value, &gradient, 1U); + } + + bool backward(const BufferView &output, float seed = 1.0F) noexcept + { + if (output.length_ != 1U) + { + set_error(Status::invalid_output); + return false; + } + return backward(output, &seed, 1U); + } + + bool backward(const BufferView &output, const float *seed, + std::size_t seed_length) noexcept + { + if (status_ != Status::ok) + { + return false; + } + if (!valid(output) || output.producer_ == nullptr || seed == nullptr || + seed_length != output.length_) + { + set_error(Status::invalid_output); + return false; + } + + for (detail::Node *node = output.producer_; node != nullptr; + node = node->previous) + { + node->reset_gradient(*node); + } + for (std::size_t i = 0; i < output.length_; ++i) + { + output.gradients_[i] = seed[i]; + } + for (detail::Node *node = output.producer_; node != nullptr; + node = node->previous) + { + node->backward(*node); + } + return true; + } + +private: + static std::size_t operator_hash(const void *token) noexcept + { + std::uintptr_t value = reinterpret_cast(token); + value ^= value >> 4U; + value *= static_cast(0x9E3779B1U); + value ^= value >> (sizeof(std::uintptr_t) * 4U); + return static_cast(value) & + (maximum_registered_operators - 1U); + } + + bool valid(const BufferView &view) const noexcept + { + return view.tape_ == this && + (view.length_ == 0U || + (view.values_ != nullptr && + (view.role_ == BufferRole::input || + view.gradients_ != nullptr))); + } + + void set_error(Status error) noexcept + { + if (status_ == Status::ok) + { + status_ = error; + } + } + + void *allocate(std::size_t bytes, std::size_t alignment) noexcept + { + if (storage_ == nullptr || alignment == 0U) + { + set_error(Status::out_of_memory); + return nullptr; + } + const std::uintptr_t base = reinterpret_cast(storage_); + const std::uintptr_t current = base + used_; + const std::size_t padding = static_cast( + (alignment - (current % alignment)) % alignment); + if (used_ > capacity_) + { + set_error(Status::out_of_memory); + return nullptr; + } + const std::size_t remaining = capacity_ - used_; + if (padding > remaining || bytes > remaining - padding) + { + set_error(Status::out_of_memory); + return nullptr; + } + used_ += padding; + void *result = storage_ + used_; + used_ += bytes; + return result; + } + + template + Record *append(detail::BackwardFunction backward, + detail::ResetFunction reset_gradient) noexcept + { + static_assert(std::is_trivially_destructible::value, + "Tape records are discarded without destructors"); + void *memory = allocate(sizeof(Record), alignof(Record)); + if (memory == nullptr) + { + return nullptr; + } + Record *record = ::new (memory) Record(); + record->node.previous = tail_; + record->node.backward = backward; + record->node.reset_gradient = reset_gradient; + tail_ = &record->node; + return record; + } + + unsigned char *storage_; + std::size_t capacity_; + std::size_t used_; + detail::Node *tail_; + bool recording_; + Status status_; + std::size_t graph_begin_; + bool graph_marked_; + std::size_t registered_count_; + const void *registered_operators_[maximum_registered_operators]; + + friend class OperatorAccess; +}; + +/** Narrow internal interface used by independently defined operators. */ +class OperatorAccess +{ +public: + static Tape *tape(const BufferView &view) noexcept { return view.tape_; } + static float *values(BufferView &view) noexcept { return view.values_; } + static const float *values(const BufferView &view) noexcept + { + return view.values_; + } + static float *gradients(const BufferView &view) noexcept + { + return view.gradients_; + } + static std::size_t length(const BufferView &view) noexcept + { + return view.length_; + } + static BufferRole role(const BufferView &view) noexcept { return view.role_; } + static detail::Node *producer(const BufferView &view) noexcept + { + return view.producer_; + } + static void set_producer(BufferView &view, detail::Node *node) noexcept + { + view.producer_ = node; + } + static bool valid(const Tape &tape, const BufferView &view) noexcept + { + return tape.valid(view); + } + static bool compatible(const Tape &tape, const BufferView &left, + const BufferView &right) noexcept + { + return valid(tape, left) && valid(tape, right) && + length(left) == length(right); + } + static const BufferView &buffer(const MatrixView &matrix) noexcept + { + return matrix.buffer_; + } + static std::size_t rows(const MatrixView &matrix) noexcept + { + return matrix.rows_; + } + static std::size_t columns(const MatrixView &matrix) noexcept + { + return matrix.columns_; + } + static bool recording(const Tape &tape) noexcept { return tape.recording_; } + static Status status(const Tape &tape) noexcept { return tape.status_; } + static void fail(Tape &tape, Status status) noexcept { tape.set_error(status); } + + template + static bool require(Tape &tape) noexcept + { + if (!tape.is_operator_registered()) + { + tape.set_error(Status::operator_not_registered); + return false; + } + return true; + } + + template + static Record *append(Tape &tape, detail::BackwardFunction backward, + detail::ResetFunction reset_gradient) noexcept + { + return tape.append(backward, reset_gradient); + } +}; + +class RecordingScope +{ +public: + RecordingScope(Tape &tape, bool enabled) noexcept + : tape_(tape), previous_(tape.recording()) + { + tape_.set_recording(enabled); + } + ~RecordingScope() noexcept { tape_.set_recording(previous_); } + RecordingScope(const RecordingScope &) = delete; + RecordingScope &operator=(const RecordingScope &) = delete; + +private: + Tape &tape_; + bool previous_; +}; + +template +class Arena +{ +public: + static_assert(Bytes > 0U, "An autodiff arena must contain storage"); + Arena() noexcept : storage_{}, tape_(storage_, Bytes) {} + Arena(const Arena &) = delete; + Arena &operator=(const Arena &) = delete; + Tape &tape() noexcept { return tape_; } + const Tape &tape() const noexcept { return tape_; } + +private: + alignas(std::max_align_t) unsigned char storage_[Bytes]; + Tape tape_; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h index 2ff3eaa77..0e9746a7e 100644 --- a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h +++ b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. All rights reserved. + * Copyright (c) 2020-2024 Arm Limited. All rights reserved. * * Licensed under the Apache License Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,65 +31,65 @@ /* ARM MPS3 IO SCC */ #define MPS3_IO_S -#define MPS3_IO_DEV MPS3_IO_DEV_S +#define MPS3_IO_DEV MPS3_IO_DEV_S /* I2C_SBCon */ #define I2C0_SBCON_S -#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S +#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S /* I2S */ #define MPS3_I2S_S -#define MPS3_I2S_DEV MPS3_I2S_DEV_S +#define MPS3_I2S_DEV MPS3_I2S_DEV_S /* ARM UART Controller PL011 */ #define UART0_CMSDK_S -#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S +#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S #define UART1_CMSDK_S -#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S +#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S -#define DEFAULT_UART_BAUDRATE 115200U +#define DEFAULT_UART_BAUDRATE 115200U /* To be used as CODE and DATA sram */ #define MPC_ISRAM0_S -#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S +#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S #define MPC_ISRAM1_S -#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S +#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S #define MPC_SRAM_S -#define MPC_SRAM_DEV MPC_SRAM_DEV_S +#define MPC_SRAM_DEV MPC_SRAM_DEV_S #define MPC_QSPI_S -#define MPC_QSPI_DEV MPC_QSPI_DEV_S +#define MPC_QSPI_DEV MPC_QSPI_DEV_S /** System Counter Armv8-M */ #define SYSCOUNTER_CNTRL_ARMV8_M_S -#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S +#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S #define SYSCOUNTER_READ_ARMV8_M_S -#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S +#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S /** * Arbitrary scaling values for test purposes */ -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u /* System timer */ #define SYSTIMER0_ARMV8_M_S -#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S +#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S #define SYSTIMER1_ARMV8_M_S -#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S +#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S #define SYSTIMER2_ARMV8_M_S -#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S +#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S #define SYSTIMER3_ARMV8_M_S -#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S +#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S -#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) /* CMSDK GPIO driver structures */ #define GPIO0_CMSDK_S @@ -141,9 +141,8 @@ /* ARM SPI PL022 */ /* Invalid device stubs are not defined */ -#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ +#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ #define SPI1_PL022_S #define SPI1_PL022_DEV SPI1_PL022_DEV_S - -#endif /* __DEVICE_CFG_H__ */ +#endif /* __DEVICE_CFG_H__ */ diff --git a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 deleted file mode 100644 index 2ff3eaa77..000000000 --- a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. All rights reserved. - * - * Licensed under the Apache License Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing software - * distributed under the License is distributed on an "AS IS" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __DEVICE_CFG_H__ -#define __DEVICE_CFG_H__ - -/** - * \file device_cfg.h - * \brief Configuration file native driver re-targeting - * - * \details This file can be used to add native driver specific macro - * definitions to select which peripherals are available in the build. - * - * This is a default device configuration file with all peripherals enabled. - */ - -/* Secure only peripheral configuration */ - -/* ARM MPS3 IO SCC */ -#define MPS3_IO_S -#define MPS3_IO_DEV MPS3_IO_DEV_S - -/* I2C_SBCon */ -#define I2C0_SBCON_S -#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S - -/* I2S */ -#define MPS3_I2S_S -#define MPS3_I2S_DEV MPS3_I2S_DEV_S - -/* ARM UART Controller PL011 */ -#define UART0_CMSDK_S -#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S -#define UART1_CMSDK_S -#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S - -#define DEFAULT_UART_BAUDRATE 115200U - -/* To be used as CODE and DATA sram */ -#define MPC_ISRAM0_S -#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S - -#define MPC_ISRAM1_S -#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S - -#define MPC_SRAM_S -#define MPC_SRAM_DEV MPC_SRAM_DEV_S - -#define MPC_QSPI_S -#define MPC_QSPI_DEV MPC_QSPI_DEV_S - -/** System Counter Armv8-M */ -#define SYSCOUNTER_CNTRL_ARMV8_M_S -#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S - -#define SYSCOUNTER_READ_ARMV8_M_S -#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S -/** - * Arbitrary scaling values for test purposes - */ -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u - -/* System timer */ -#define SYSTIMER0_ARMV8_M_S -#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S -#define SYSTIMER1_ARMV8_M_S -#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S -#define SYSTIMER2_ARMV8_M_S -#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S -#define SYSTIMER3_ARMV8_M_S -#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S - -#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) - -/* CMSDK GPIO driver structures */ -#define GPIO0_CMSDK_S -#define GPIO0_CMSDK_DEV GPIO0_CMSDK_DEV_S -#define GPIO1_CMSDK_S -#define GPIO1_CMSDK_DEV GPIO1_CMSDK_DEV_S -#define GPIO2_CMSDK_S -#define GPIO2_CMSDK_DEV GPIO2_CMSDK_DEV_S -#define GPIO3_CMSDK_S -#define GPIO3_CMSDK_DEV GPIO3_CMSDK_DEV_S - -/* System Watchdogs */ -#define SYSWDOG_ARMV8_M_S -#define SYSWDOG_ARMV8_M_DEV SYSWDOG_ARMV8_M_DEV_S - -/* ARM MPC SIE 300 driver structures */ -#define MPC_VM0_S -#define MPC_VM0_DEV MPC_VM0_DEV_S -#define MPC_VM1_S -#define MPC_VM1_DEV MPC_VM1_DEV_S -#define MPC_SSRAM2_S -#define MPC_SSRAM2_DEV MPC_SSRAM2_DEV_S -#define MPC_SSRAM3_S -#define MPC_SSRAM3_DEV MPC_SSRAM3_DEV_S - -/* ARM PPC driver structures */ -#define PPC_SSE300_MAIN0_S -#define PPC_SSE300_MAIN0_DEV PPC_SSE300_MAIN0_DEV_S -#define PPC_SSE300_MAIN_EXP0_S -#define PPC_SSE300_MAIN_EXP0_DEV PPC_SSE300_MAIN_EXP0_DEV_S -#define PPC_SSE300_MAIN_EXP1_S -#define PPC_SSE300_MAIN_EXP1_DEV PPC_SSE300_MAIN_EXP1_DEV_S -#define PPC_SSE300_MAIN_EXP2_S -#define PPC_SSE300_MAIN_EXP2_DEV PPC_SSE300_MAIN_EXP2_DEV_S -#define PPC_SSE300_MAIN_EXP3_S -#define PPC_SSE300_MAIN_EXP3_DEV PPC_SSE300_MAIN_EXP3_DEV_S -#define PPC_SSE300_PERIPH0_S -#define PPC_SSE300_PERIPH0_DEV PPC_SSE300_PERIPH0_DEV_S -#define PPC_SSE300_PERIPH1_S -#define PPC_SSE300_PERIPH1_DEV PPC_SSE300_PERIPH1_DEV_S -#define PPC_SSE300_PERIPH_EXP0_S -#define PPC_SSE300_PERIPH_EXP0_DEV PPC_SSE300_PERIPH_EXP0_DEV_S -#define PPC_SSE300_PERIPH_EXP1_S -#define PPC_SSE300_PERIPH_EXP1_DEV PPC_SSE300_PERIPH_EXP1_DEV_S -#define PPC_SSE300_PERIPH_EXP2_S -#define PPC_SSE300_PERIPH_EXP2_DEV PPC_SSE300_PERIPH_EXP2_DEV_S -#define PPC_SSE300_PERIPH_EXP3_S -#define PPC_SSE300_PERIPH_EXP3_DEV PPC_SSE300_PERIPH_EXP3_DEV_S - -/* ARM SPI PL022 */ -/* Invalid device stubs are not defined */ -#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ -#define SPI1_PL022_S -#define SPI1_PL022_DEV SPI1_PL022_DEV_S - - -#endif /* __DEVICE_CFG_H__ */ diff --git a/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h b/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h index 5613b81ae..0ce63c7b0 100644 --- a/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h +++ b/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h @@ -1,8 +1,8 @@ /* * CSOLUTION generated file: DO NOT EDIT! - * Generated by: csolution version 2.10.0 + * Generated by: csolution version 2.14.1+p38-gf512b381 * - * Project: 'test.Release+VHT-Corstone-300' + * Project: 'example.Release+VHT-Corstone-300' * Target: 'Release+VHT-Corstone-300' */ @@ -15,6 +15,10 @@ */ #define CMSIS_device_header "SSE300MPS3.h" +/* ARM::Device:Native Driver:SysCounter@1.1.0 */ +#define RTE_SYSCOUNTER 1 +/* ARM::Device:Native Driver:Timeout@1.0.0 */ +#define RTE_TIMEOUT 1 #endif /* RTE_COMPONENTS_H */ diff --git a/dsppp/example.cproject.yml b/dsppp/example.cproject.yml index 0e41fef75..b9f96bbe7 100644 --- a/dsppp/example.cproject.yml +++ b/dsppp/example.cproject.yml @@ -4,7 +4,8 @@ project: files: #- file: Examples/dot_product.cpp #- file: Examples/vector_op.cpp - - file: Examples/matrix_op.cpp + #- file: Examples/matrix_op.cpp + - file: Examples/autodiff_regression.cpp - file: clang_sse300.c for-context: - +MPS3-Corstone-300 @@ -16,7 +17,7 @@ project: components: - component: ARM::CMSIS:CORE - - component: ARM::CMSIS:DSP@1.15.0 + - component: ARM::CMSIS:DSP@1.17.1 - component: ARM::Device:Startup&C Startup for-context: - +VHT-Corstone-300 diff --git a/dsppp/main.c b/dsppp/main.c index 3a7cdcefd..e0e47cb2e 100644 --- a/dsppp/main.c +++ b/dsppp/main.c @@ -58,6 +58,9 @@ int main(void) #if defined(DOT_TEST) dot_test(); #endif + #if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) + autodiff_test(); + #endif #if defined(VECTOR_TEST) vector_test(); #endif diff --git a/dsppp/run_all.py b/dsppp/run_all.py index 3d1d3abf4..c57afd265 100644 --- a/dsppp/run_all.py +++ b/dsppp/run_all.py @@ -181,6 +181,7 @@ def cmd_args(): AVHROOT = args.avh ALL_TESTS=["DOT_TEST", + "AUTODIFF_TEST", "VECTOR_TEST", "ROW_TEST", "COL_TEST", diff --git a/dsppp/test.cbuild-idx.yml b/dsppp/test.cbuild-idx.yml new file mode 100644 index 000000000..211b64606 --- /dev/null +++ b/dsppp/test.cbuild-idx.yml @@ -0,0 +1,23 @@ +build-idx: + generated-by: csolution version 2.14.1+p38-gf512b381 + cdefault: cdefault.yml + csolution: test.csolution.yml + cbuild-run: out/test+VHT-Corstone-300.cbuild-run.yml + tmpdir: tmp/VHT-Corstone-300/default + cprojects: + - cproject: tests/test.cproject.yml + clayers: + - clayer: ../Testing/cmsis_build/dsp.clayer.yml + - cproject: example.cproject.yml + cbuilds: + - cbuild: out/example/VHT-Corstone-300/Release/example.Release+VHT-Corstone-300.cbuild.yml + project: example + configuration: .Release+VHT-Corstone-300 + messages: + info: + - test.cbuild-pack.yml - file is already up-to-date + - test+VHT-Corstone-300.cbuild-run.yml - file is already up-to-date + - example.Release+VHT-Corstone-300.cbuild.yml - file is already up-to-date + packs-unused: + - pack: ARM::CMSIS-Compiler@2.2.0 + - pack: ARM::Cortex_DFP@1.2.0 diff --git a/dsppp/test.cbuild-pack.yml b/dsppp/test.cbuild-pack.yml index 4ffc655f0..ee7f893af 100644 --- a/dsppp/test.cbuild-pack.yml +++ b/dsppp/test.cbuild-pack.yml @@ -3,15 +3,28 @@ cbuild-pack: - resolved-pack: ARM::CMSIS@6.0.0 selected-by-pack: - ARM::CMSIS@6.0.0 + - resolved-pack: ARM::CMSIS@6.3.0 + selected-by-pack: + - ARM::CMSIS@6.3.0 - resolved-pack: ARM::CMSIS-Compiler@2.0.0 selected-by-pack: - ARM::CMSIS-Compiler@2.0.0 + - resolved-pack: ARM::CMSIS-Compiler@2.2.0 + selected-by-pack: + - ARM::CMSIS-Compiler@2.2.0 - resolved-pack: ARM::CMSIS-DSP@1.15.0 selected-by-pack: + - ARM::CMSIS-DSP - ARM::CMSIS-DSP@1.15.0 - resolved-pack: ARM::Cortex_DFP@1.0.0 selected-by-pack: - ARM::Cortex_DFP@1.0.0 + - resolved-pack: ARM::Cortex_DFP@1.2.0 + selected-by-pack: + - ARM::Cortex_DFP@1.2.0 - resolved-pack: ARM::V2M_MPS3_SSE_300_BSP@1.4.0 selected-by-pack: - ARM::V2M_MPS3_SSE_300_BSP@1.4.0 + - resolved-pack: ARM::V2M_MPS3_SSE_300_BSP@1.5.0 + selected-by-pack: + - ARM::V2M_MPS3_SSE_300_BSP@1.5.0 diff --git a/dsppp/test.csolution.yml b/dsppp/test.csolution.yml index 73f4e5e70..57a812e58 100644 --- a/dsppp/test.csolution.yml +++ b/dsppp/test.csolution.yml @@ -6,10 +6,11 @@ solution: cdefault: packs: - - pack: ARM::CMSIS@6.0.0 - - pack: ARM::V2M_MPS3_SSE_300_BSP@1.4.0 - - pack: ARM::CMSIS-Compiler@2.0.0 - - pack: ARM::Cortex_DFP@1.0.0 + - pack: ARM::CMSIS@6.3.0 + - pack: ARM::V2M_MPS3_SSE_300_BSP@1.5.0 + - pack: ARM::CMSIS-Compiler@2.2.0 + - pack: ARM::Cortex_DFP@1.2.0 + - pack: ARM::CMSIS-DSP target-types: - type: MPS3-Corstone-300 @@ -29,7 +30,7 @@ solution: - -Wno-sign-compare - -Wno-unused-parameter Link: - - --specs=nosys.specs + - --specs=nosys.specs - for-compiler: CLANG C: - -Wno-sign-compare @@ -39,6 +40,10 @@ solution: - -Wno-unused-parameter Link: - -lcrt0 + target-set: + - set: + images: + - project-context: test.Release - type: VHT-Corstone-300 device: ARM::SSE-300-MPS3 @@ -57,6 +62,14 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: example.Release + debugger: + name: Arm-FVP + model: FVP_Corstone_SSE-300_Ethos-U55 + config-file: fvp_configs/VHT-Corstone-300.txt - type: VHT-M0P device: ARMCM0P @@ -76,6 +89,10 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: test.Release - type: VHT-M4 device: ARMCM4 @@ -95,13 +112,16 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: test.Release build-types: - type: Release debug: on - projects: - project: ./tests/test.cproject.yml - project: ./example.cproject.yml - \ No newline at end of file + created-for: CMSIS-Toolbox@2.14.1 diff --git a/dsppp/tests/.clangd b/dsppp/tests/.clangd new file mode 100644 index 000000000..1d4dc2e8c --- /dev/null +++ b/dsppp/tests/.clangd @@ -0,0 +1,18 @@ +CompileFlags: + CompilationDatabase: c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release + +--- +If: + PathMatch: .*\.(c|C|h)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release\compile_macros_c.h + +--- +If: + PathMatch: .*\.(cpp|c\+\+|C\+\+|cxx|cc|CC|hpp)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release\compile_macros_cxx.h diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp new file mode 100644 index 000000000..b3bb6795b --- /dev/null +++ b/dsppp/tests/autodiff_test.cpp @@ -0,0 +1,320 @@ +#include "test_config.h" + +extern "C" { + extern void autodiff_test(); +} + +#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace arm_cmsis_dsp::autodiff; + +// Board Release builds define NDEBUG, so the standard assert macro cannot be +// used as a test oracle. Keep the existing concise checks but make failures +// visible to run_all.py, which treats output containing "Error" as a failure. +#define AUTODIFF_CHECK(condition) \ + do \ + { \ + if (!(condition)) \ + { \ + std::printf("Error: autodiff check failed at line %u\r\n", \ + static_cast(__LINE__)); \ + return; \ + } \ + } while (false) + +#ifdef assert +#undef assert +#endif +#define assert(condition) AUTODIFF_CHECK(condition) + +static void run_autodiff_tests() +{ + // Vector add followed by dot. Values and outputs belong to the caller; + // gradients and two fixed-size operation records use the tape arena. + Arena<2048> buffer_arena; + Tape &buffer_tape = buffer_arena.tape(); + buffer_tape.register_operator(); + buffer_tape.register_operator(); + float x_value[] = {1.0F, 2.0F, 3.0F}; + float w_value[] = {4.0F, 5.0F, 6.0F}; + float sum_value[3] = {}; + float result_value[1] = {}; + + BufferView x_view = buffer_tape.view(x_value); + BufferView w_view = buffer_tape.view(w_value); + BufferView sum_view = buffer_tape.view(sum_value); + BufferView result_view = buffer_tape.view(result_value); + const std::size_t gradients_end = buffer_tape.used(); + assert(gradients_end >= 10U * sizeof(float)); + + { + RecordingScope no_gradient(buffer_tape, false); + sum_view = x_view + w_view; + result_view = dot(sum_view, w_view); + assert(result_value[0] == 109.0F); + } + assert(buffer_tape.used() == gradients_end); + + sum_view = x_view + w_view; + result_view = dot(sum_view, w_view); + assert(buffer_tape.good()); + assert(buffer_tape.backward(result_view)); + for (std::size_t i = 0; i < 3; ++i) + { + assert(x_view.gradient(i) == w_value[i]); + assert(w_view.gradient(i) == x_value[i] + 2.0F * w_value[i]); + } + + // A vector output accepts a caller-provided vector-Jacobian seed. + const float sum_seed[] = {1.0F, 2.0F, 3.0F}; + assert(buffer_tape.backward(sum_view, sum_seed, 3)); + for (std::size_t i = 0; i < 3; ++i) + { + assert(x_view.gradient(i) == sum_seed[i]); + assert(w_view.gradient(i) == sum_seed[i]); + } + + // Only parameters receive final gradients. The x input has no gradient + // allocation, while alpha and beta are trainable parameters. + Arena<2048> parameter_arena; + Tape ¶meter_tape = parameter_arena.tape(); + parameter_tape.register_operator(); + parameter_tape.register_operator(); + parameter_tape.register_operator(); + float input_value[] = {1.0F, 2.0F, 3.0F}; + float alpha_value = 2.0F; + float beta_value[] = {10.0F, 20.0F, 30.0F}; + float scaled_value[3] = {}; + float added_value[3] = {}; + float loss_value[1] = {}; + + BufferView input_view = parameter_tape.input(input_value); + const std::size_t after_input = parameter_tape.used(); + BufferView alpha_view = parameter_tape.parameter(alpha_value); + BufferView beta_view = parameter_tape.parameter(beta_value); + BufferView scaled_view = parameter_tape.output(scaled_value); + BufferView added_view = parameter_tape.output(added_value); + BufferView loss_view = parameter_tape.output(loss_value); + + assert(after_input == 0U); + assert(!input_view.has_gradient()); + assert(alpha_view.role() == BufferRole::parameter); + assert(beta_view.role() == BufferRole::parameter); + + scaled_view = scale(input_view, alpha_view, beta_view); + added_view = scaled_view + input_view; + loss_view = dot(added_view, input_view); + assert(loss_value[0] == 182.0F); + assert(parameter_tape.backward(loss_view)); + assert(alpha_view.gradient(0) == 14.0F); + for (std::size_t i = 0; i < 3; ++i) + { + assert(beta_view.gradient(i) == input_value[i]); + assert(input_view.gradient(i) == 0.0F); + } + + // Fully connected followed by ReLU. Only the positive first neuron + // contributes to the matrix and bias parameter gradients. + Arena<2048> network_arena; + Tape &network_tape = network_arena.tape(); + network_tape.register_operator(); + network_tape.register_operator(); + float network_input_value[] = {2.0F, -1.0F}; + float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; + float bias_value[] = {1.0F, 0.0F}; + float linear_value[2] = {}; + float activation_value[2] = {}; + + BufferView network_input = network_tape.input(network_input_value); + MatrixView matrix = network_tape.parameter(matrix_value); + BufferView bias = network_tape.parameter(bias_value); + BufferView linear = network_tape.output(linear_value); + BufferView activation = network_tape.output(activation_value); + + linear = fully_connected(network_input, matrix, bias); + activation = relu(linear); + assert(linear_value[0] == 1.0F); + assert(linear_value[1] == -7.0F); + assert(activation_value[0] == 1.0F); + assert(activation_value[1] == 0.0F); + + const float activation_seed[] = {1.0F, 1.0F}; + assert(network_tape.backward(activation, activation_seed, 2)); + assert(matrix.gradient(0, 0) == 2.0F); + assert(matrix.gradient(0, 1) == -1.0F); + assert(matrix.gradient(1, 0) == 0.0F); + assert(matrix.gradient(1, 1) == 0.0F); + assert(bias.gradient(0) == 1.0F); + assert(bias.gradient(1) == 0.0F); + assert(!network_input.has_gradient()); + + // ReLU uses a zero derivative at exactly zero. + Arena<512> relu_arena; + Tape &relu_tape = relu_arena.tape(); + relu_tape.register_operator(); + float relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; + float relu_output_value[3] = {}; + BufferView relu_parameter = relu_tape.parameter(relu_parameter_value); + BufferView relu_output = relu_tape.output(relu_output_value); + relu_output = relu(relu_parameter); + const float relu_seed[] = {1.0F, 1.0F, 1.0F}; + assert(relu_tape.backward(relu_output, relu_seed, 3)); + assert(relu_parameter.gradient(0) == 0.0F); + assert(relu_parameter.gradient(1) == 0.0F); + assert(relu_parameter.gradient(2) == 1.0F); + + // Including an operator does not enable it. Evaluation fails until that + // operator type is explicitly registered on this tape. + Arena<256> registry_arena; + Tape ®istry_tape = registry_arena.tape(); + float registry_left_value[] = {1.0F}; + float registry_right_value[] = {2.0F}; + float registry_output_value[] = {0.0F}; + BufferView registry_left = registry_tape.input(registry_left_value); + BufferView registry_right = registry_tape.input(registry_right_value); + BufferView registry_output = registry_tape.output(registry_output_value); + registry_output = registry_left + registry_right; + assert(registry_tape.status() == Status::operator_not_registered); + assert(registry_output_value[0] == 0.0F); + registry_tape.clear_status(); + assert(registry_tape.register_operator()); + registry_output = registry_left + registry_right; + assert(registry_tape.good()); + assert(registry_output_value[0] == 3.0F); + + // Quadratic loss, reusable graph records, Adam, and selective freezing. + Arena<1024> training_arena; + Tape &training_tape = training_arena.tape(); + training_tape.register_operator(); + training_tape.register_operator(); + training_tape.register_operator(); + float feature_value[] = {2.0F}; + float coefficient_value[] = {3.0F}; + float bias_parameter_value[] = {1.0F}; + float dot_value[1] = {}; + float prediction_value[1] = {}; + float target_value[] = {0.0F}; + float training_loss_value[1] = {}; + BufferView feature = training_tape.input(feature_value); + BufferView coefficient = training_tape.parameter(coefficient_value); + BufferView bias_parameter = training_tape.parameter(bias_parameter_value); + BufferView dot_output = training_tape.output(dot_value); + BufferView prediction = training_tape.output(prediction_value); + BufferView target = training_tape.input(target_value); + BufferView loss = training_tape.output(training_loss_value); + Adam<2> adam(1.0e-2F); + assert(adam.add(coefficient)); + assert(adam.add(bias_parameter)); + assert(freeze_parameters(adam, coefficient)); + training_tape.begin_graph(); + const std::size_t persistent_training_bytes = training_tape.used(); + dot_output = dot(feature, coefficient); + prediction = dot_output + bias_parameter; + loss = quadratic_error(prediction, target); + adam.zero_grad(); + assert(training_tape.backward(loss)); + assert(training_loss_value[0] == 49.0F); + assert(coefficient.gradient(0) == 28.0F); + assert(bias_parameter.gradient(0) == 14.0F); + assert(adam.step()); + assert(coefficient_value[0] == 3.0F); // Frozen. + assert(bias_parameter_value[0] < 1.0F); + assert(training_tape.rewind_graph()); + assert(training_tape.used() == persistent_training_bytes); + assert(unfreeze_parameters(adam, coefficient)); + + // A single vector loss accumulates contributions from every sample into + // shared polynomial parameters before an optimizer step. + Arena<2048> batch_arena; + Tape &batch_tape = batch_arena.tape(); + batch_tape.register_operator(); + batch_tape.register_operator(); + batch_tape.register_operator(); + float batch_feature_value[2][1] = {{1.0F}, {2.0F}}; + float batch_coefficient_value = 3.0F; + float batch_bias_value = 1.0F; + float batch_polynomial_value[2] = {}; + float batch_prediction_value[2] = {}; + float batch_target_value[2] = {}; + float batch_loss_value = 0.0F; + BufferView batch_coefficient = + batch_tape.parameter(batch_coefficient_value); + BufferView batch_bias = batch_tape.parameter(batch_bias_value); + BufferView batch_polynomial = + batch_tape.output(batch_polynomial_value); + BufferView batch_prediction = + batch_tape.output(batch_prediction_value); + BufferView batch_target = batch_tape.input(batch_target_value); + BufferView batch_loss = batch_tape.output(batch_loss_value); + for (std::size_t sample = 0; sample < 2U; ++sample) + { + BufferView batch_feature = + batch_tape.input(batch_feature_value[sample]); + BufferView batch_polynomial_element = batch_tape.output( + &batch_polynomial_value[sample], + &batch_polynomial.gradients()[sample], 1U); + BufferView batch_prediction_element = batch_tape.output( + &batch_prediction_value[sample], + &batch_prediction.gradients()[sample], 1U); + batch_polynomial_element = dot(batch_feature, batch_coefficient); + batch_prediction_element = batch_polynomial_element + batch_bias; + } + batch_loss = quadratic_error(batch_prediction, batch_target); + assert(batch_tape.backward(batch_loss)); + assert(batch_loss_value == 65.0F); // 4^2 + 7^2. + assert(batch_coefficient.gradient(0) == 36.0F); + assert(batch_bias.gradient(0) == 22.0F); + + // RMSProp uses the same parameter registration and freezing API. + Arena<128> rms_arena; + Tape &rms_tape = rms_arena.tape(); + float rms_value = 1.0F; + BufferView rms_parameter = rms_tape.parameter(rms_value); + RMSProp<1> rmsprop(1.0e-2F); + assert(rmsprop.add(rms_parameter)); + rms_parameter.gradients()[0] = 2.0F; + assert(rmsprop.step()); + assert(rms_value < 1.0F); + + // Arena exhaustion is explicit and backward cannot return partial results. + alignas(std::max_align_t) unsigned char tiny_memory[1]; + Tape tiny(tiny_memory, sizeof(tiny_memory)); + tiny.register_operator(); + float tiny_input_value[1] = {2.0F}; + float tiny_input_gradient[1] = {}; + float tiny_output_value[1] = {}; + float tiny_output_gradient[1] = {}; + BufferView tiny_input = + tiny.view(tiny_input_value, tiny_input_gradient, 1); + BufferView tiny_output = + tiny.view(tiny_output_value, tiny_output_gradient, 1); + tiny_output = tiny_input + tiny_input; + assert(tiny_output_value[0] == 4.0F); + assert(tiny.status() == Status::out_of_memory); + assert(!tiny.backward(tiny_output)); + +} + +#undef assert +#undef AUTODIFF_CHECK + +#endif + +void autodiff_test() +{ +#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) + run_autodiff_tests(); +#endif +} diff --git a/dsppp/tests/test.cproject.yml b/dsppp/tests/test.cproject.yml index 432af5ab7..67e42ee99 100644 --- a/dsppp/tests/test.cproject.yml +++ b/dsppp/tests/test.cproject.yml @@ -4,6 +4,7 @@ project: files: - file: matrix_test.cpp - file: dot_test.cpp + - file: autodiff_test.cpp - file: vector_test.cpp - file: row_test.cpp - file: col_test.cpp diff --git a/dsppp/tests/test.h b/dsppp/tests/test.h index 9273f47f0..3fe33c015 100644 --- a/dsppp/tests/test.h +++ b/dsppp/tests/test.h @@ -6,6 +6,7 @@ extern void matrix_test(void); extern void dot_test(void); +extern void autodiff_test(void); extern void vector_test(void); extern void row_test(void); extern void col_test(void); From 98ea3bdd3c0d820fa338b46cf57b4c0a68fd9ade Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 07:53:56 +0200 Subject: [PATCH 02/19] autodiff : Added new operators and CMSIS-DSP optimizations. --- .gitignore | 1 + dsppp/Include/dsppp/autodiff/README.md | 49 +++-- .../Include/dsppp/autodiff/operators/add.hpp | 34 ++-- .../Include/dsppp/autodiff/operators/dot.hpp | 26 +-- .../autodiff/operators/fully_connected.hpp | 28 +-- .../dsppp/autodiff/operators/multiply.hpp | 122 ++++++++++++ .../dsppp/autodiff/operators/offset.hpp | 108 +++++++++++ .../autodiff/operators/quadratic_error.hpp | 35 ++-- .../Include/dsppp/autodiff/operators/relu.hpp | 21 +- .../dsppp/autodiff/operators/scale.hpp | 92 +++++---- .../dsppp/autodiff/operators/softmax.hpp | 110 +++++++++++ .../Include/dsppp/autodiff/operators/sub.hpp | 103 ++++++++++ .../dsppp/autodiff/optimizers/adam.hpp | 22 ++- .../dsppp/autodiff/optimizers/rmsprop.hpp | 15 +- dsppp/test.cbuild-pack.yml | 3 + dsppp/test.csolution.yml | 2 +- dsppp/test_config.h | 6 +- dsppp/tests/autodiff_test.cpp | 179 +++++++++++++++--- 18 files changed, 778 insertions(+), 178 deletions(-) create mode 100644 dsppp/Include/dsppp/autodiff/operators/multiply.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/offset.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/softmax.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/sub.hpp diff --git a/.gitignore b/.gitignore index 4661b4bbb..42aeb53a0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ build-* .venv/ uv.lock *.cbuild-pack.yml +disasm_* \ No newline at end of file diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 820b2b38b..88ae8fe30 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -2,8 +2,8 @@ `reverse.hpp` is a deliberately small reverse-mode automatic differentiation (AD) implementation for embedded use. It currently handles `float` buffers, -vector addition, vector dot products, affine vector scaling -`alpha * x + beta`, fully connected and ReLU nodes, and a quadratic-error +vector arithmetic, learnable scalar scale and offset operations, vector dot +products, fully connected, ReLU, and softmax nodes, and a quadratic-error loss. Fixed-storage Adam and RMSProp optimizers support user-written training loops. The structure is intended to be extended with tensor operators. @@ -81,10 +81,14 @@ record, gradient reset, backward rule, and expression adapter: | Header | Operator class | Expression | | --- | --- | --- | | `operators/add.hpp` | `AddOperator` | `a + b` | +| `operators/sub.hpp` | `SubOperator` | `a - b` | +| `operators/multiply.hpp` | `MultiplyOperator` | `a * b` | | `operators/dot.hpp` | `DotOperator` | `dot(a, b)` | -| `operators/scale.hpp` | `ScaleOperator` | `scale(x, alpha, beta)` | +| `operators/scale.hpp` | `ScaleOperator` | `scale(x, constant)` | +| `operators/offset.hpp` | `OffsetOperator` | `offset(x, constant)` | | `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | | `operators/relu.hpp` | `ReluOperator` | `relu(x)` | +| `operators/softmax.hpp` | `SoftmaxOperator` | `softmax(x)` | | `operators/quadratic_error.hpp` | `QuadraticErrorOperator` | `quadratic_error(prediction, target)` | An application includes and registers only the operators it uses. An operator @@ -150,6 +154,7 @@ output buffer for another operation before `backward()` or `Tape::reset()`. #include #include #include +#include #include using namespace arm_cmsis_dsp::autodiff; @@ -159,29 +164,33 @@ Tape &tape = arena.tape(); tape.register_operator(); tape.register_operator(); tape.register_operator(); +tape.register_operator(); float x_value[] = {1.0F, 2.0F}; float alpha_value = 2.0F; -float beta_value[] = {3.0F, 4.0F}; +float beta_value = 3.0F; float scaled_value[2] = {}; +float shifted_value[2] = {}; float sum_value[2] = {}; float result_value[1] = {}; BufferView x = tape.input(x_value); // No gradient for x. BufferView alpha = tape.parameter(alpha_value); // Scalar parameter. -BufferView beta = tape.parameter(beta_value); // Vector parameter. +BufferView beta = tape.parameter(beta_value); // Scalar parameter. BufferView scaled = tape.output(scaled_value); +BufferView shifted = tape.output(shifted_value); BufferView sum = tape.output(sum_value); BufferView result = tape.output(result_value); -scaled = scale(x, alpha, beta); // scaled = alpha * x + beta -sum = scaled + x; -result = dot(sum, x); // result[0] == 26 +scaled = scale(x, alpha); // scaled = alpha * x +shifted = offset(scaled, beta); // shifted = scaled + beta +sum = shifted + x; +result = dot(sum, x); // result[0] == 24 if (tape.backward(result)) { // x.has_gradient() == false // alpha.gradient(0) == 5 - // beta gradients are {1, 2} + // beta.gradient(0) == 3 } ``` @@ -548,8 +557,9 @@ the root of `backward()`: const std::size_t before = tape.used(); { RecordingScope no_gradient(tape, false); - scaled = scale(x, alpha, beta); - sum = scaled + x; + scaled = scale(x, alpha); + shifted = offset(scaled, beta); + sum = shifted + x; result = dot(sum, x); use(result_value[0]); } @@ -612,16 +622,23 @@ An addition or dot-product operand contributes to a gradient only when it is a parameter or intermediate with gradient storage. An `input` has a null gradient pointer, so the same backward rule simply skips that contribution. -For vector scaling `z[i] = alpha[0] * x[i] + beta[i]`, `x` is required to be an -input while `alpha` and `beta` are required to be parameters. Its rule is: +For vector scaling `z[i] = alpha[0] * x[i]`, `alpha` is a scalar parameter. +Its rule is: ```text alpha_gradient[0] += z_gradient[i] * x_value[i] (summed over i) -beta_gradient[i] += z_gradient[i] +x_gradient[i] += alpha[0] * z_gradient[i] ``` -There is intentionally no `x_gradient`: the role declared by `tape.input(x)` -states that the caller does not request it. +For vector offset `z[i] = x[i] + beta[0]`, `beta` is also a scalar parameter: + +```text +beta_gradient[0] += z_gradient[i] (summed over i) +x_gradient[i] += z_gradient[i] +``` + +The input-gradient contribution is skipped when `x` was declared with +`tape.input(x)` and therefore has no gradient storage. All operators follow the same ownership rule: inputs and outputs stay in caller storage, while their records retain non-owning pointers. Future matrix diff --git a/dsppp/Include/dsppp/autodiff/operators/add.hpp b/dsppp/Include/dsppp/autodiff/operators/add.hpp index ff804b239..4cb1339dd 100644 --- a/dsppp/Include/dsppp/autodiff/operators/add.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/add.hpp @@ -2,6 +2,9 @@ #include +#include +#include + namespace arm_cmsis_dsp { namespace autodiff { @@ -19,24 +22,22 @@ class AddOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) - { - record.output_gradient[i] = 0.0F; - if (record.left_gradient != nullptr) record.left_gradient[i] = 0.0F; - if (record.right_gradient != nullptr) record.right_gradient[i] = 0.0F; - } + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.left_gradient != nullptr) + arm_fill_f32(0.0F, record.left_gradient, record.length); + if (record.right_gradient != nullptr) + arm_fill_f32(0.0F, record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) - { - if (record.left_gradient != nullptr) - record.left_gradient[i] += record.output_gradient[i]; - if (record.right_gradient != nullptr) - record.right_gradient[i] += record.output_gradient[i]; - } + if (record.left_gradient != nullptr) + arm_add_f32(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); + if (record.right_gradient != nullptr) + arm_add_f32(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); } public: @@ -58,9 +59,9 @@ class AddOperator OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) - OperatorAccess::values(output)[i] = - OperatorAccess::values(left)[i] + OperatorAccess::values(right)[i]; + arm_add_f32(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), + OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) return OperatorAccess::status(*tape) == Status::ok; @@ -97,4 +98,3 @@ inline AddExpression operator+(const BufferView &left, } // namespace autodiff } // namespace arm_cmsis_dsp - diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp index 75e15c84d..2fe2c4404 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dot.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -6,6 +6,9 @@ #include #include +#include +#include + namespace arm_cmsis_dsp { namespace autodiff { @@ -27,17 +30,10 @@ class DotOperator { Record &record = reinterpret_cast(node); record.output_gradient[0] = 0.0F; - if (record.left_gradient != nullptr) - { - VectorView left_grad(const_cast(record.left_gradient), 0, record.length); - left_grad = 0.0F; - } + arm_fill_f32(0.0F, record.left_gradient, record.length); if (record.right_gradient != nullptr) - { - VectorView right_grad(const_cast(record.right_gradient), 0, record.length); - right_grad = 0.0F; - } + arm_fill_f32(0.0F, record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept @@ -48,13 +44,13 @@ class DotOperator if (record.left_gradient != nullptr) { - VectorView left_grad(const_cast(record.left_gradient) , 0, record.length); + VectorView left_grad(record.left_gradient, 0, record.length); VectorView right_val(const_cast(record.right_value), 0, record.length); left_grad += right_val * gradient; } if (record.right_gradient != nullptr) { - VectorView right_grad(const_cast(record.right_gradient), 0, record.length); + VectorView right_grad(record.right_gradient, 0, record.length); VectorView left_val(const_cast(record.left_value), 0, record.length); right_grad += left_val * gradient; } @@ -79,10 +75,9 @@ class DotOperator return false; } float value = 0.0F; - //for (std::size_t i = 0; i < OperatorAccess::length(left); ++i) - // value += OperatorAccess::values(left)[i] * OperatorAccess::values(right)[i]; - - arm_dot_prod_f32(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::length(left),&value); + arm_dot_prod_f32(OperatorAccess::values(left), + OperatorAccess::values(right), + OperatorAccess::length(left), &value); OperatorAccess::values(output)[0] = value; if (!OperatorAccess::recording(*tape)) return OperatorAccess::status(*tape) == Status::ok; @@ -121,4 +116,3 @@ inline DotExpression dot(const BufferView &left, const BufferView &right) noexce } // namespace autodiff } // namespace arm_cmsis_dsp - diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp index 7e4b19329..5507f36c9 100644 --- a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -7,6 +7,7 @@ #include #include +#include #include @@ -31,29 +32,12 @@ class FullyConnectedOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - if (record.rows != 0U) - { - ::arm_cmsis_dsp::VectorView output_gradient( - record.output_gradient, 0, record.rows); - ::arm_cmsis_dsp::VectorView bias_gradient( - record.bias_gradient, 0, record.rows); - output_gradient = 0.0F; - bias_gradient = 0.0F; - } - if (record.rows != 0U && record.columns != 0U) - { - ::arm_cmsis_dsp::MatrixView - weight_gradient(record.weight_gradient, record.rows, - record.columns, record.columns); - weight_gradient = 0.0F; - } + arm_fill_f32(0.0F, record.output_gradient, record.rows); + arm_fill_f32(0.0F, record.bias_gradient, record.rows); + arm_fill_f32(0.0F, record.weight_gradient, + record.rows * record.columns); if (record.input_gradient != nullptr && record.columns != 0U) - { - ::arm_cmsis_dsp::VectorView input_gradient( - record.input_gradient, 0, record.columns); - input_gradient = 0.0F; - } + arm_fill_f32(0.0F, record.input_gradient, record.columns); } static void backward(detail::Node &node) noexcept diff --git a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp new file mode 100644 index 000000000..1370b96e6 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp @@ -0,0 +1,122 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class MultiplyOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *left_value; + float *left_gradient; + const float *right_value; + float *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.left_gradient != nullptr) + arm_fill_f32(0.0F, record.left_gradient, record.length); + if (record.right_gradient != nullptr) + arm_fill_f32(0.0F, record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + if (record.left_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView left_gradient( + record.left_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView right_value( + const_cast(record.right_value), 0, record.length); + left_gradient += output_gradient * right_value; + } + if (record.right_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView right_gradient( + record.right_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView left_value( + const_cast(record.left_value), 0, record.length); + right_gradient += output_gradient * left_value; + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(right)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + arm_mult_f32(OperatorAccess::values(left), + OperatorAccess::values(right), + OperatorAccess::values(output), + OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class MultiplyExpression +{ +public: + MultiplyExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + MultiplyOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +inline MultiplyExpression operator*(const BufferView &left, + const BufferView &right) noexcept +{ + return MultiplyExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/offset.hpp b/dsppp/Include/dsppp/autodiff/operators/offset.hpp new file mode 100644 index 000000000..12f4c5cbc --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/offset.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include + +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class OffsetOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + float *input_gradient; + float *offset_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.input_gradient != nullptr) + arm_fill_f32(0.0F, record.input_gradient, record.length); + record.offset_gradient[0] = 0.0F; + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient != nullptr) + arm_add_f32(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); + float gradient_sum = 0.0F; + arm_accumulate_f32(record.output_gradient, record.length, + &gradient_sum); + record.offset_gradient[0] += gradient_sum; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &offset) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, input) || + !OperatorAccess::valid(*tape, offset) || + OperatorAccess::length(offset) != 1U || + OperatorAccess::role(offset) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(offset) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(offset) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(offset)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + arm_offset_f32(OperatorAccess::values(input), + OperatorAccess::values(offset)[0], + OperatorAccess::values(output), + OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); + record->offset_gradient = OperatorAccess::gradients(offset); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class OffsetExpression +{ +public: + OffsetExpression(const BufferView &input, const BufferView &offset) noexcept + : input_(input), offset_(offset) {} + void evaluate(BufferView &output) const noexcept + { + OffsetOperator::evaluate(output, input_, offset_); + } +private: + BufferView input_; + BufferView offset_; +}; + +inline OffsetExpression offset(const BufferView &input, + const BufferView &constant) noexcept +{ + return OffsetExpression(input, constant); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp index 90f67a2bc..d77a11512 100644 --- a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp @@ -2,6 +2,10 @@ #include +#include + +#include + namespace arm_cmsis_dsp { namespace autodiff { @@ -22,8 +26,7 @@ class QuadraticErrorOperator { Record &record = reinterpret_cast(node); record.output_gradient[0] = 0.0F; - for (std::size_t i = 0; i < record.length; ++i) - record.prediction_gradient[i] = 0.0F; + arm_fill_f32(0.0F, record.prediction_gradient, record.length); } static void backward(detail::Node &node) noexcept @@ -31,9 +34,14 @@ class QuadraticErrorOperator Record &record = reinterpret_cast(node); const float seed = record.output_gradient[0]; if (seed == 0.0F) return; - for (std::size_t i = 0; i < record.length; ++i) - record.prediction_gradient[i] += 2.0F * seed * - (record.prediction_value[i] - record.target_value[i]); + ::arm_cmsis_dsp::VectorView prediction_gradient( + record.prediction_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(record.prediction_value), 0, record.length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(record.target_value), 0, record.length); + prediction_gradient += + (prediction_value - target_value) * (2.0F * seed); } public: @@ -56,14 +64,15 @@ class QuadraticErrorOperator return false; } - float value = 0.0F; - for (std::size_t i = 0; i < OperatorAccess::length(prediction); ++i) - { - const float error = OperatorAccess::values(prediction)[i] - - OperatorAccess::values(target)[i]; - value += error * error; - } - OperatorAccess::values(output)[0] = value; + const std::size_t length = OperatorAccess::length(prediction); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(OperatorAccess::values(prediction)), 0, + length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(OperatorAccess::values(target)), 0, length); + const auto error = prediction_value - target_value; + OperatorAccess::values(output)[0] = + ::arm_cmsis_dsp::dot(error, error); if (!OperatorAccess::recording(*tape)) return OperatorAccess::status(*tape) == Status::ok; diff --git a/dsppp/Include/dsppp/autodiff/operators/relu.hpp b/dsppp/Include/dsppp/autodiff/operators/relu.hpp index ff15bbb12..41121e422 100644 --- a/dsppp/Include/dsppp/autodiff/operators/relu.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/relu.hpp @@ -2,6 +2,11 @@ #include +#include +#include + +#include + namespace arm_cmsis_dsp { namespace autodiff { @@ -19,11 +24,9 @@ class ReluOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) - { - record.output_gradient[i] = 0.0F; - if (record.input_gradient != nullptr) record.input_gradient[i] = 0.0F; - } + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.input_gradient != nullptr) + arm_fill_f32(0.0F, record.input_gradient, record.length); } static void backward(detail::Node &node) noexcept { @@ -49,10 +52,10 @@ class ReluOperator OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - for (std::size_t i = 0; i < OperatorAccess::length(input); ++i) - OperatorAccess::values(output)[i] = - OperatorAccess::values(input)[i] > 0.0F ? - OperatorAccess::values(input)[i] : 0.0F; + arm_clip_f32(OperatorAccess::values(input), + OperatorAccess::values(output), 0.0F, + std::numeric_limits::max(), + OperatorAccess::length(input)); if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) return OperatorAccess::status(*tape) == Status::ok; diff --git a/dsppp/Include/dsppp/autodiff/operators/scale.hpp b/dsppp/Include/dsppp/autodiff/operators/scale.hpp index fecc6f201..c13c6e734 100644 --- a/dsppp/Include/dsppp/autodiff/operators/scale.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/scale.hpp @@ -2,6 +2,11 @@ #include +#include + +#include +#include + namespace arm_cmsis_dsp { namespace autodiff { @@ -12,70 +17,77 @@ class ScaleOperator detail::Node node; float *output_gradient; const float *input_value; - float *alpha_gradient; - float *beta_gradient; + float *input_gradient; + const float *scale_value; + float *scale_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - record.alpha_gradient[0] = 0.0F; - for (std::size_t i = 0; i < record.length; ++i) - { - record.output_gradient[i] = 0.0F; - record.beta_gradient[i] = 0.0F; - } + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.input_gradient != nullptr) + arm_fill_f32(0.0F, record.input_gradient, record.length); + record.scale_gradient[0] = 0.0F; } + static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value), 0, record.length); + record.scale_gradient[0] += + ::arm_cmsis_dsp::dot(output_gradient, input_value); + if (record.input_gradient != nullptr) { - const float gradient = record.output_gradient[i]; - if (gradient != 0.0F) - record.alpha_gradient[0] += gradient * record.input_value[i]; - record.beta_gradient[i] += gradient; + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.length); + input_gradient += output_gradient * record.scale_value[0]; } } public: static bool evaluate(BufferView &output, const BufferView &input, - const BufferView &alpha, - const BufferView &beta) noexcept + const BufferView &scale) noexcept { Tape *tape = OperatorAccess::tape(output); OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::require(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - !OperatorAccess::valid(*tape, input) || - !OperatorAccess::valid(*tape, alpha) || - !OperatorAccess::valid(*tape, beta) || + if (!OperatorAccess::compatible(*tape, output, input) || + !OperatorAccess::valid(*tape, scale) || + OperatorAccess::length(scale) != 1U || + OperatorAccess::role(scale) != BufferRole::parameter || OperatorAccess::gradients(output) == nullptr || - OperatorAccess::length(output) != OperatorAccess::length(input) || - OperatorAccess::length(beta) != OperatorAccess::length(input) || - OperatorAccess::length(alpha) != 1U || - OperatorAccess::role(input) != BufferRole::input || - OperatorAccess::role(alpha) != BufferRole::parameter || - OperatorAccess::role(beta) != BufferRole::parameter) + OperatorAccess::gradients(scale) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(scale) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(scale)) { OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const float alpha_value = OperatorAccess::values(alpha)[0]; - for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) - OperatorAccess::values(output)[i] = alpha_value * - OperatorAccess::values(input)[i] + OperatorAccess::values(beta)[i]; - if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) + arm_scale_f32(OperatorAccess::values(input), + OperatorAccess::values(scale)[0], + OperatorAccess::values(output), + OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) return OperatorAccess::status(*tape) == Status::ok; Record *record = OperatorAccess::append(*tape, backward, reset); if (record == nullptr) return false; record->output_gradient = OperatorAccess::gradients(output); record->input_value = OperatorAccess::values(input); - record->alpha_gradient = OperatorAccess::gradients(alpha); - record->beta_gradient = OperatorAccess::gradients(beta); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->scale_gradient = OperatorAccess::gradients(scale); record->length = OperatorAccess::length(output); OperatorAccess::set_producer(output, &record->node); return true; @@ -85,23 +97,21 @@ class ScaleOperator class ScaleExpression { public: - ScaleExpression(const BufferView &input, const BufferView &alpha, - const BufferView &beta) noexcept - : input_(input), alpha_(alpha), beta_(beta) {} + ScaleExpression(const BufferView &input, const BufferView &scale) noexcept + : input_(input), scale_(scale) {} void evaluate(BufferView &output) const noexcept { - ScaleOperator::evaluate(output, input_, alpha_, beta_); + ScaleOperator::evaluate(output, input_, scale_); } private: BufferView input_; - BufferView alpha_; - BufferView beta_; + BufferView scale_; }; -inline ScaleExpression scale(const BufferView &input, const BufferView &alpha, - const BufferView &beta) noexcept +inline ScaleExpression scale(const BufferView &input, + const BufferView &constant) noexcept { - return ScaleExpression(input, alpha, beta); + return ScaleExpression(input, constant); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp new file mode 100644 index 000000000..a5dcb7690 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp @@ -0,0 +1,110 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class SoftmaxOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *output_value; + float *input_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.input_gradient != nullptr) + arm_fill_f32(0.0F, record.input_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView output_value( + const_cast(record.output_value), 0, record.length); + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.length); + const float projection = + ::arm_cmsis_dsp::dot(output_gradient, output_value); + input_gradient += output_value * (output_gradient - projection); + } + +public: + static bool evaluate(BufferView &output, const BufferView &input) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + const std::size_t length = OperatorAccess::length(input); + if (length == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + const float log_sum = arm_logsumexp_f32( + OperatorAccess::values(input), length); + arm_offset_f32(OperatorAccess::values(input), -log_sum, + OperatorAccess::values(output), length); + arm_vexp_f32(OperatorAccess::values(output), + OperatorAccess::values(output), length); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->output_value = OperatorAccess::values(output); + record->input_gradient = OperatorAccess::gradients(input); + record->length = length; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class SoftmaxExpression +{ +public: + explicit SoftmaxExpression(const BufferView &input) noexcept + : input_(input) {} + void evaluate(BufferView &output) const noexcept + { + SoftmaxOperator::evaluate(output, input_); + } +private: + BufferView input_; +}; + +inline SoftmaxExpression softmax(const BufferView &input) noexcept +{ + return SoftmaxExpression(input); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/sub.hpp b/dsppp/Include/dsppp/autodiff/operators/sub.hpp new file mode 100644 index 000000000..6b637262e --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/sub.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +class SubOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + float *left_gradient; + float *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.left_gradient != nullptr) + arm_fill_f32(0.0F, record.left_gradient, record.length); + if (record.right_gradient != nullptr) + arm_fill_f32(0.0F, record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.left_gradient != nullptr) + arm_add_f32(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); + if (record.right_gradient != nullptr) + arm_sub_f32(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(right)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + arm_sub_f32(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), + OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class SubExpression +{ +public: + SubExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + SubOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +inline SubExpression operator-(const BufferView &left, + const BufferView &right) noexcept +{ + return SubExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp index f97b203cf..ee48103ac 100644 --- a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp +++ b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp @@ -3,6 +3,9 @@ #include #include #include +#include + +#include namespace arm_cmsis_dsp { namespace autodiff { @@ -60,8 +63,8 @@ class Adam void zero_grad() noexcept { for (std::size_t p = 0; p < parameter_count_; ++p) - for (std::size_t i = 0; i < entries_[p].length; ++i) - entries_[p].gradients[i] = 0.0F; + arm_fill_f32(0.0F, entries_[p].gradients, + entries_[p].length); } bool step() noexcept @@ -75,14 +78,19 @@ class Adam { Entry &entry = entries_[p]; if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + ::arm_cmsis_dsp::VectorView first_moment( + first_moment_ + entry.offset, 0, entry.length); + ::arm_cmsis_dsp::VectorView second_moment( + second_moment_ + entry.offset, 0, entry.length); + first_moment = first_moment * beta1_ + + gradients * (1.0F - beta1_); + second_moment = second_moment * beta2_ + + gradients * gradients * (1.0F - beta2_); for (std::size_t i = 0; i < entry.length; ++i) { const std::size_t state = entry.offset + i; - const float gradient = entry.gradients[i]; - first_moment_[state] = beta1_ * first_moment_[state] + - (1.0F - beta1_) * gradient; - second_moment_[state] = beta2_ * second_moment_[state] + - (1.0F - beta2_) * gradient * gradient; const float corrected_first = first_moment_[state] / first_correction; const float corrected_second = diff --git a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp index 869857af7..7d5a95fe4 100644 --- a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp +++ b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp @@ -3,6 +3,9 @@ #include #include #include +#include + +#include namespace arm_cmsis_dsp { namespace autodiff { @@ -58,8 +61,8 @@ class RMSProp void zero_grad() noexcept { for (std::size_t p = 0; p < parameter_count_; ++p) - for (std::size_t i = 0; i < entries_[p].length; ++i) - entries_[p].gradients[i] = 0.0F; + arm_fill_f32(0.0F, entries_[p].gradients, + entries_[p].length); } bool step() noexcept @@ -69,12 +72,16 @@ class RMSProp { Entry &entry = entries_[p]; if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + ::arm_cmsis_dsp::VectorView square_average( + square_average_ + entry.offset, 0, entry.length); + square_average = square_average * alpha_ + + gradients * gradients * (1.0F - alpha_); for (std::size_t i = 0; i < entry.length; ++i) { const std::size_t state = entry.offset + i; const float gradient = entry.gradients[i]; - square_average_[state] = alpha_ * square_average_[state] + - (1.0F - alpha_) * gradient * gradient; entry.values[i] -= learning_rate_ * gradient / (std::sqrt(square_average_[state]) + epsilon_); } diff --git a/dsppp/test.cbuild-pack.yml b/dsppp/test.cbuild-pack.yml index ee7f893af..c22651802 100644 --- a/dsppp/test.cbuild-pack.yml +++ b/dsppp/test.cbuild-pack.yml @@ -16,6 +16,9 @@ cbuild-pack: selected-by-pack: - ARM::CMSIS-DSP - ARM::CMSIS-DSP@1.15.0 + - resolved-pack: ARM::CMSIS-DSP@1.17.1 + selected-by-pack: + - ARM::CMSIS-DSP@1.17.1 - resolved-pack: ARM::Cortex_DFP@1.0.0 selected-by-pack: - ARM::Cortex_DFP@1.0.0 diff --git a/dsppp/test.csolution.yml b/dsppp/test.csolution.yml index 57a812e58..4ed800a5b 100644 --- a/dsppp/test.csolution.yml +++ b/dsppp/test.csolution.yml @@ -10,7 +10,7 @@ solution: - pack: ARM::V2M_MPS3_SSE_300_BSP@1.5.0 - pack: ARM::CMSIS-Compiler@2.2.0 - pack: ARM::Cortex_DFP@1.2.0 - - pack: ARM::CMSIS-DSP + - pack: ARM::CMSIS-DSP@1.17.1 target-types: - type: MPS3-Corstone-300 diff --git a/dsppp/test_config.h b/dsppp/test_config.h index 0d8d3e34b..27e35bb1a 100644 --- a/dsppp/test_config.h +++ b/dsppp/test_config.h @@ -8,9 +8,9 @@ //#define ONLY_BENCHMARKS -#define DOT_TEST -#define COMPLEX_F32_DT -#define STATIC_TEST +#define AUTODIFF_TEST +#define F32_DT +#define DYNAMIC_TEST #endif diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index b3bb6795b..f6cf57722 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -10,8 +10,12 @@ extern "C" { #include #include #include +#include +#include #include #include +#include +#include #include #include #include @@ -30,7 +34,6 @@ using namespace arm_cmsis_dsp::autodiff; { \ std::printf("Error: autodiff check failed at line %u\r\n", \ static_cast(__LINE__)); \ - return; \ } \ } while (false) @@ -39,12 +42,12 @@ using namespace arm_cmsis_dsp::autodiff; #endif #define assert(condition) AUTODIFF_CHECK(condition) -static void run_autodiff_tests() +static void test1() { - // Vector add followed by dot. Values and outputs belong to the caller; + // Vector add followed by dot. Values and outputs belong to the caller; // gradients and two fixed-size operation records use the tape arena. - Arena<2048> buffer_arena; - Tape &buffer_tape = buffer_arena.tape(); + Arena<2048> *buffer_arena = new Arena<2048>(); + Tape &buffer_tape = buffer_arena->tape(); buffer_tape.register_operator(); buffer_tape.register_operator(); float x_value[] = {1.0F, 2.0F, 3.0F}; @@ -59,6 +62,8 @@ static void run_autodiff_tests() const std::size_t gradients_end = buffer_tape.used(); assert(gradients_end >= 10U * sizeof(float)); + + { RecordingScope no_gradient(buffer_tape, false); sum_view = x_view + w_view; @@ -67,6 +72,7 @@ static void run_autodiff_tests() } assert(buffer_tape.used() == gradients_end); + sum_view = x_view + w_view; result_view = dot(sum_view, w_view); assert(buffer_tape.good()); @@ -77,6 +83,8 @@ static void run_autodiff_tests() assert(w_view.gradient(i) == x_value[i] + 2.0F * w_value[i]); } + + // A vector output accepts a caller-provided vector-Jacobian seed. const float sum_seed[] = {1.0F, 2.0F, 3.0F}; assert(buffer_tape.backward(sum_view, sum_seed, 3)); @@ -86,16 +94,23 @@ static void run_autodiff_tests() assert(w_view.gradient(i) == sum_seed[i]); } + delete buffer_arena; +} + +void test2() +{ + + // Only parameters receive final gradients. The x input has no gradient - // allocation, while alpha and beta are trainable parameters. - Arena<2048> parameter_arena; - Tape ¶meter_tape = parameter_arena.tape(); + // allocation, while scale and offset are trainable scalar parameters. + Arena<2048> *parameter_arena = new Arena<2048>(); + Tape ¶meter_tape = parameter_arena->tape(); parameter_tape.register_operator(); - parameter_tape.register_operator(); + parameter_tape.register_operator(); parameter_tape.register_operator(); float input_value[] = {1.0F, 2.0F, 3.0F}; float alpha_value = 2.0F; - float beta_value[] = {10.0F, 20.0F, 30.0F}; + float beta_value = 10.0F; float scaled_value[3] = {}; float added_value[3] = {}; float loss_value[1] = {}; @@ -113,22 +128,26 @@ static void run_autodiff_tests() assert(alpha_view.role() == BufferRole::parameter); assert(beta_view.role() == BufferRole::parameter); - scaled_view = scale(input_view, alpha_view, beta_view); - added_view = scaled_view + input_view; + scaled_view = scale(input_view, alpha_view); + added_view = offset(scaled_view, beta_view); loss_view = dot(added_view, input_view); - assert(loss_value[0] == 182.0F); + assert(loss_value[0] == 88.0F); assert(parameter_tape.backward(loss_view)); assert(alpha_view.gradient(0) == 14.0F); + assert(beta_view.gradient(0) == 6.0F); for (std::size_t i = 0; i < 3; ++i) - { - assert(beta_view.gradient(i) == input_value[i]); assert(input_view.gradient(i) == 0.0F); - } + + delete parameter_arena; +} + +void test3() +{ // Fully connected followed by ReLU. Only the positive first neuron // contributes to the matrix and bias parameter gradients. - Arena<2048> network_arena; - Tape &network_tape = network_arena.tape(); + Arena<2048> *network_arena = new Arena<2048>(); + Tape &network_tape = network_arena->tape(); network_tape.register_operator(); network_tape.register_operator(); float network_input_value[] = {2.0F, -1.0F}; @@ -159,10 +178,16 @@ static void run_autodiff_tests() assert(bias.gradient(0) == 1.0F); assert(bias.gradient(1) == 0.0F); assert(!network_input.has_gradient()); + delete network_arena; + + +} - // ReLU uses a zero derivative at exactly zero. - Arena<512> relu_arena; - Tape &relu_tape = relu_arena.tape(); +void test4() +{ + // ReLU uses a zero derivative at exactly zero. + Arena<512> *relu_arena = new Arena<512>(); + Tape &relu_tape = relu_arena->tape(); relu_tape.register_operator(); float relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; float relu_output_value[3] = {}; @@ -175,10 +200,45 @@ static void run_autodiff_tests() assert(relu_parameter.gradient(1) == 0.0F); assert(relu_parameter.gradient(2) == 1.0F); + delete relu_arena; +} + +void test5() +{ + + // Softmax is normalized and its vector-Jacobian product has zero sum. + Arena<512> *softmax_arena = new Arena<512>(); + Tape &softmax_tape = softmax_arena->tape(); + softmax_tape.register_operator(); + float logits_value[] = {0.0F, 0.0F, 0.0F}; + float probability_value[3] = {}; + BufferView logits = softmax_tape.parameter(logits_value); + BufferView probability = softmax_tape.output(probability_value); + probability = softmax(logits); + const float probability_sum = probability_value[0] + + probability_value[1] + probability_value[2]; + assert(probability_sum > 0.9999F && probability_sum < 1.0001F); + for (std::size_t i = 0; i < 3U; ++i) + assert(probability_value[i] > 0.3332F && + probability_value[i] < 0.3335F); + const float softmax_seed[] = {1.0F, 2.0F, 3.0F}; + assert(softmax_tape.backward(probability, softmax_seed, 3U)); + assert(logits.gradient(0) < -0.3332F && + logits.gradient(0) > -0.3335F); + assert(logits.gradient(1) > -1.0e-6F && + logits.gradient(1) < 1.0e-6F); + assert(logits.gradient(2) > 0.3332F && + logits.gradient(2) < 0.3335F); + delete softmax_arena; +} + +void test6() +{ + // Including an operator does not enable it. Evaluation fails until that // operator type is explicitly registered on this tape. - Arena<256> registry_arena; - Tape ®istry_tape = registry_arena.tape(); + Arena<256> *registry_arena = new Arena<256>(); + Tape ®istry_tape = registry_arena->tape(); float registry_left_value[] = {1.0F}; float registry_right_value[] = {2.0F}; float registry_output_value[] = {0.0F}; @@ -193,10 +253,15 @@ static void run_autodiff_tests() registry_output = registry_left + registry_right; assert(registry_tape.good()); assert(registry_output_value[0] == 3.0F); + delete registry_arena; +} +void test7() +{ + // Quadratic loss, reusable graph records, Adam, and selective freezing. - Arena<1024> training_arena; - Tape &training_tape = training_arena.tape(); + Arena<1024> *training_arena = new Arena<1024>(); + Tape &training_tape = training_arena->tape(); training_tape.register_operator(); training_tape.register_operator(); training_tape.register_operator(); @@ -234,11 +299,16 @@ static void run_autodiff_tests() assert(training_tape.rewind_graph()); assert(training_tape.used() == persistent_training_bytes); assert(unfreeze_parameters(adam, coefficient)); + delete training_arena; +} +void test8() +{ + // A single vector loss accumulates contributions from every sample into // shared polynomial parameters before an optimizer step. - Arena<2048> batch_arena; - Tape &batch_tape = batch_arena.tape(); + Arena<2048> *batch_arena = new Arena<2048>(); + Tape &batch_tape = batch_arena->tape(); batch_tape.register_operator(); batch_tape.register_operator(); batch_tape.register_operator(); @@ -276,10 +346,14 @@ static void run_autodiff_tests() assert(batch_loss_value == 65.0F); // 4^2 + 7^2. assert(batch_coefficient.gradient(0) == 36.0F); assert(batch_bias.gradient(0) == 22.0F); + delete batch_arena; +} +void test9() +{ // RMSProp uses the same parameter registration and freezing API. - Arena<128> rms_arena; - Tape &rms_tape = rms_arena.tape(); + Arena<128> *rms_arena = new Arena<128>(); + Tape &rms_tape = rms_arena->tape(); float rms_value = 1.0F; BufferView rms_parameter = rms_tape.parameter(rms_value); RMSProp<1> rmsprop(1.0e-2F); @@ -287,7 +361,53 @@ static void run_autodiff_tests() rms_parameter.gradients()[0] = 2.0F; assert(rmsprop.step()); assert(rms_value < 1.0F); + delete rms_arena; +} +void test10() +{ + // Subtraction and elementwise multiplication have data operands only. + Arena<1024> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + tape.register_operator(); + float left_value[] = {2.0F, 4.0F, 6.0F}; + float left_gradient[3] = {}; + float right_value[] = {1.0F, 2.0F, 3.0F}; + float right_gradient[3] = {}; + float difference_value[3] = {}; + float product_value[3] = {}; + BufferView left = tape.view(left_value, left_gradient, 3U); + BufferView right = tape.view(right_value, right_gradient, 3U); + BufferView difference = tape.output(difference_value); + BufferView product = tape.output(product_value); + difference = left - right; + product = difference * right; + assert(product_value[0] == 1.0F); + assert(product_value[1] == 4.0F); + assert(product_value[2] == 9.0F); + const float seed[] = {1.0F, 1.0F, 1.0F}; + assert(tape.backward(product, seed, 3U)); + for (std::size_t i = 0; i < 3U; ++i) + { + assert(left.gradient(i) == right_value[i]); + assert(right.gradient(i) == 0.0F); + } +} + +static void run_autodiff_tests() +{ + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + test7(); + test8(); + test9(); + test10(); + // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; Tape tiny(tiny_memory, sizeof(tiny_memory)); @@ -315,6 +435,7 @@ static void run_autodiff_tests() void autodiff_test() { #if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) + printf("Running autodiff tests...\r\n"); run_autodiff_tests(); #endif } From 4c903e3e0f21fb022b116d8e62dde68c7b0f57e9 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 09:13:01 +0200 Subject: [PATCH 03/19] dsppp : Added new transpose_view operator and used it in backward pass of fully connected --- dsppp/Include/dsppp/DSP/matrix_multiply.hpp | 19 +- .../Include/dsppp/Helium/matrix_multiply.hpp | 61 +++- .../Include/dsppp/Scalar/matrix_multiply.hpp | 13 +- .../autodiff/operators/fully_connected.hpp | 20 +- dsppp/Include/dsppp/forward.hpp | 3 + dsppp/Include/dsppp/matrix.hpp | 267 ++++++++++++++++++ dsppp/Include/dsppp/vec.hpp | 51 ++++ dsppp/tests/autodiff_test.cpp | 140 +++++++++ 8 files changed, 558 insertions(+), 16 deletions(-) diff --git a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp index ace09a188..2b8c591ff 100644 --- a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp @@ -253,6 +253,23 @@ inline void _dot_m_v(RES &res, } } +template() && + !is_complex() && + !is_complex() && + !std::is_same::Scalar,Q31>::value && + number_traits::Scalar>::is_fixed,bool>::type = true> +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const DSP* = nullptr) +{ + detail::dot_transposed_unrolled(res,m,v); +} + template() && + has_vector_inst() && + same_nb_lanes() && + same_nb_lanes(),bool>::type = true> +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const Helium* = nullptr) +{ + if constexpr (is_float()) + { + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using T = typename MixedRes::type; + using Acc = typename vector_traits::temp_accumulator; + constexpr int nb_lanes = vector_traits::nb_lanes; + + const auto &original = m.original(); + const vector_length_t rows = original.rows(); + const vector_length_t columns = original.columns(); + vector_length_t column = 0; + + for (; column <= columns - nb_lanes; column += nb_lanes) + { + Acc sum = vector_traits::temp_acc_zero(); + for (index_t row = 0; row < rows; ++row) + sum = inner::vmacc(sum, + original.row(row).vector_op(column), + v[row]); + inner::vstore1<1>(res.ptr() + column,sum); + } + + const vector_length_t remaining = columns - column; + if (remaining > 0) + { + const mve_pred16_t predicate = inner::vctpq::mk(remaining); + Acc sum = vector_traits::temp_acc_zero(); + for (index_t row = 0; row < rows; ++row) + sum = inner::vmacc( + sum, + original.row(row).vector_op_tail(column,remaining), + v[row]); + inner::vstore1_z<1>(res.ptr() + column,sum,remaining,predicate); + } + } + else + { + detail::dot_transposed_unrolled(res,m,v); + } +} + +#endif + #define MATRIX_DIM2 2 #define MATRIX_DIM3 3 #define MATRIX_DIM4 4 @@ -404,4 +463,4 @@ __STATIC_INLINE void _dot_m_m(const MA& pSrcA, #endif -/*! @} */ \ No newline at end of file +/*! @} */ diff --git a/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp b/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp index 3bbfdb714..6f548f8f0 100644 --- a/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp @@ -41,6 +41,17 @@ __STATIC_INLINE void _arm_mat_trans( * @tparam V Vector datatype * @tparam RES Result datatype */ +template +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const Scalar* = nullptr) +{ + detail::dot_transposed_unrolled(res,m,v); +} + template @@ -154,4 +165,4 @@ inline void _dot_m_v(RES &res, #include "matrix_multiply_fixed.hpp" #include "matrix_multiply_float.hpp" -/*! @} */ \ No newline at end of file +/*! @} */ diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp index 5507f36c9..cc37e11d5 100644 --- a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -72,19 +72,13 @@ class FullyConnectedOperator ::arm_cmsis_dsp::VectorView input_gradient( record.input_gradient, 0, record.columns); - for (std::size_t column = 0; column < record.columns; ++column) - { - // A column is strided in the row-major weight matrix. The - // C++ dot implementation handles that view directly, so no - // transposed matrix or temporary vector is needed. - ::arm_cmsis_dsp::VectorView - weight_column(const_cast(record.weight_value), - column, record.rows * record.columns, - record.columns); - input_gradient[column] += - ::arm_cmsis_dsp::dot(weight_column, output_gradient); - } + ::arm_cmsis_dsp::MatrixView + weight_value(const_cast(record.weight_value), + record.rows, record.columns, record.columns); + input_gradient += ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(weight_value), + output_gradient); } } diff --git a/dsppp/Include/dsppp/forward.hpp b/dsppp/Include/dsppp/forward.hpp index e9a17c2ec..22e807388 100644 --- a/dsppp/Include/dsppp/forward.hpp +++ b/dsppp/Include/dsppp/forward.hpp @@ -21,6 +21,9 @@ struct Matrix; template struct MatrixView; +template +struct TransposeView; + template struct NbRows; diff --git a/dsppp/Include/dsppp/matrix.hpp b/dsppp/Include/dsppp/matrix.hpp index e299c3733..9b3e921f7 100644 --- a/dsppp/Include/dsppp/matrix.hpp +++ b/dsppp/Include/dsppp/matrix.hpp @@ -25,6 +25,105 @@ namespace arm_cmsis_dsp { * @{ */ +/** + * @brief Zero-copy transposed view of a matrix. + * + * The view only swaps matrix indexing and dimensions. Algorithms can + * recognize this type and select a kernel which consumes the original + * row-major storage without materializing a transposed matrix. + * + * @tparam M Matrix datatype + */ +template +struct TransposeView +{ + using Scalar = typename traits::Scalar; + using Storage = typename VecRef::type; + + explicit TransposeView(const M &matrix) : matrix_(VecRef::ref(matrix)) {} + + vector_length_t rows() const { return matrix_.columns(); } + vector_length_t columns() const { return matrix_.rows(); } + + Scalar operator()(const index_t row, const index_t column) const + { + return matrix_(column, row); + } + + auto row(const index_t row) const { return matrix_.col(row); } + auto col(const index_t column) const { return matrix_.row(column); } + + const Storage &original() const { return matrix_; } + +private: + Storage matrix_; +}; + +/** + * @brief Create a zero-copy transposed matrix view. + * + * @tparam M Matrix datatype + * @param matrix Matrix whose dimensions and indexing are to be transposed + * @return A non-owning transposed view + */ +template::value, bool>::type = true> +inline TransposeView transpose_view(const M &matrix) +{ + return TransposeView(matrix); +} + +template +struct traits> +{ + using Scalar = typename traits::Scalar; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct HasMatrixIndexing> +{ + constexpr static bool value = true; +}; + +template +struct ElementType> +{ + using type = typename ElementType::type; +}; + +template +struct IsDynamic> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength> +{ + constexpr static vector_length_t value = StaticLength::value; +}; + +template +struct NbRows> +{ + constexpr static vector_length_t value = NbCols::value; +}; + +template +struct NbCols> +{ + constexpr static vector_length_t value = NbRows::value; +}; + +template +struct OutputVectorDim> +{ + constexpr static vector_length_t value = NbCols::value; +}; + template typename A> struct traits> @@ -455,6 +554,174 @@ struct VecRef,((R<0) || (C<0))> }; }; +/** Lazy transposed-matrix times vector expression. */ +template +struct _TransposedMatVec: _Expr<_TransposedMatVec> +{ + using MatrixScalar = typename traits::Scalar; + using VectorScalar = typename traits::Scalar; + using Scalar = typename MixedRes::type; + using Accumulator = typename number_traits::accumulator; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif + + _TransposedMatVec(const TransposeView &matrix, const V &vector) + : matrix_(matrix), vector_(vector) {} + + vector_length_t length() const { return matrix_.rows(); } + + Scalar operator[](const index_t column) const + { + Accumulator sum{}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::mac(sum,original(row,column),vector_[row]); + return inner::from_accumulator(sum); + } + +#if defined(HAS_VECTOR) + auto vector_op(const index_t column) const + { + if constexpr (has_vector_inst() && has_vector_inst() && + same_nb_lanes() && is_float()) + { + using VectorAccumulator = + typename vector_traits::temp_accumulator; + VectorAccumulator sum = vector_traits::temp_acc_zero(); + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::vmacc(sum, + original.row(row).vector_op(column), + vector_[row]); + return sum; + } + else + { + constexpr int lanes = vector_traits::nb_lanes; + Accumulator sums[lanes] = {}; + Scalar values[lanes] = {}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + { + const VectorScalar value = vector_[row]; + for (index_t lane = 0; lane < lanes; ++lane) + sums[lane] = inner::mac( + sums[lane],original(row,column + lane),value); + } + for (index_t lane = 0; lane < lanes; ++lane) + values[lane] = inner::from_accumulator(sums[lane]); + return inner::vload1<1>(values); + } + } + + auto vector_op_tail(const index_t column, + const vector_length_t remaining) const + { + if constexpr (has_vector_inst() && has_vector_inst() && + same_nb_lanes() && is_float()) + { + using VectorAccumulator = + typename vector_traits::temp_accumulator; + VectorAccumulator sum = vector_traits::temp_acc_zero(); + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::vmacc( + sum, + original.row(row).vector_op_tail(column,remaining), + vector_[row]); + return sum; + } + else + { + constexpr int lanes = vector_traits::nb_lanes; + Accumulator sums[lanes] = {}; + Scalar values[lanes] = {}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + { + const VectorScalar value = vector_[row]; + for (index_t lane = 0; lane < remaining; ++lane) + sums[lane] = inner::mac( + sums[lane],original(row,column + lane),value); + } + for (index_t lane = 0; lane < remaining; ++lane) + values[lane] = inner::from_accumulator(sums[lane]); + return inner::vload1<1>(values); + } + } +#endif + +private: + TransposeView matrix_; + V vector_; +}; + +template +struct traits<_TransposedMatVec> +{ + using Scalar = typename MixedRes::Scalar, + typename traits::Scalar>::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct ElementType<_TransposedMatVec> +{ + using type = typename MixedRes::Scalar, + typename traits::Scalar>::type; +}; + +template +struct IsVector<_TransposedMatVec> +{ + constexpr static bool value = true; +}; + +template +struct IsDynamic<_TransposedMatVec> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength<_TransposedMatVec> +{ + constexpr static vector_length_t value = NbCols::value; +}; + +template +struct Complexity<_TransposedMatVec> +{ + constexpr static int value = 1; +}; + +template +struct VecRef<_TransposedMatVec> +{ + using type = _TransposedMatVec; + static type ref(const type &expression) { return expression; } +}; + +#if !defined(ARM_MATH_NEON) + +template,V>::value || + CompatibleDynamicMatVecProduct,V>::value), + bool>::type = true> +inline auto dot(const TransposeView &matrix, const V &vector) +{ + using VectorRef = VecRef; + return _TransposedMatVec( + matrix,VectorRef::ref(vector)); +} + +#endif + /***************** * diff --git a/dsppp/Include/dsppp/vec.hpp b/dsppp/Include/dsppp/vec.hpp index dab176ac4..fbe79faeb 100644 --- a/dsppp/Include/dsppp/vec.hpp +++ b/dsppp/Include/dsppp/vec.hpp @@ -586,6 +586,57 @@ Core algorithms that cannot be expressed only with high level abstractions and need intrinsics. */ +namespace detail { + +/** Four-column fallback for a transposed matrix times vector product. */ +template +inline void dot_transposed_unrolled(RES &res, + const TransposeView &m, + const V &v) +{ + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using T = typename MixedRes::type; + using Acc = typename number_traits::accumulator; + + const auto &original = m.original(); + const vector_length_t rows = original.rows(); + const vector_length_t columns = original.columns(); + vector_length_t column = 0; + + for (; column <= columns - 4; column += 4) + { + Acc sum0{}; + Acc sum1{}; + Acc sum2{}; + Acc sum3{}; + + for (index_t row = 0; row < rows; ++row) + { + const TV value = v[row]; + sum0 = inner::mac(sum0, original(row,column), value); + sum1 = inner::mac(sum1, original(row,column + 1), value); + sum2 = inner::mac(sum2, original(row,column + 2), value); + sum3 = inner::mac(sum3, original(row,column + 3), value); + } + + res[column] = inner::from_accumulator(sum0); + res[column + 1] = inner::from_accumulator(sum1); + res[column + 2] = inner::from_accumulator(sum2); + res[column + 3] = inner::from_accumulator(sum3); + } + + for (; column < columns; ++column) + { + Acc sum{}; + for (index_t row = 0; row < rows; ++row) + sum = inner::mac(sum, original(row,column), v[row]); + res[column] = inner::from_accumulator(sum); + } +} + +} // namespace detail + #include "Helium/matrix_multiply.hpp" #include "DSP/matrix_multiply.hpp" #include "Scalar/matrix_multiply.hpp" diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index f6cf57722..5d87a1c04 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -395,6 +395,145 @@ void test10() } } +template +static void test_transposed_dot_type() +{ + const T one = ::arm_cmsis_dsp::number_traits::one(); + T matrix_value[2][5] = {}; + for (std::size_t column = 0; column < 5; ++column) + matrix_value[0][column] = one; + T vector_value[2] = {one,T{}}; + + ::arm_cmsis_dsp::MatrixView matrix( + &matrix_value[0][0], 2, 5, 5); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 2); + ::arm_cmsis_dsp::Matrix + materialized_transpose(5,2); + ::arm_cmsis_dsp::transposeTo(materialized_transpose,matrix); + auto reference = ::arm_cmsis_dsp::dot(materialized_transpose,vector); + auto result = ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(matrix),vector); + for (std::size_t column = 0; column < 5; ++column) + { + if constexpr (std::is_same::value) + assert(static_cast(result[column]) == + static_cast(reference[column])); + else + assert(result[column] == reference[column]); + } +} + +template +static void test_transposed_dot_mixed_type() +{ + using Result = typename ::arm_cmsis_dsp::MixedRes::type; + const TM matrix_one = ::arm_cmsis_dsp::number_traits::one(); + const TV vector_one = ::arm_cmsis_dsp::number_traits::one(); + TM matrix_value[2][5] = {}; + for (std::size_t column = 0; column < 5; ++column) + matrix_value[0][column] = matrix_one; + TV vector_value[2] = {vector_one,TV{}}; + + ::arm_cmsis_dsp::MatrixView matrix( + &matrix_value[0][0], 2, 5, 5); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 2); + ::arm_cmsis_dsp::Matrix + materialized_transpose(5,2); + ::arm_cmsis_dsp::transposeTo(materialized_transpose,matrix); + ::arm_cmsis_dsp::Vector reference = + ::arm_cmsis_dsp::dot(materialized_transpose,vector); + ::arm_cmsis_dsp::Vector result = + ::arm_cmsis_dsp::dot(::arm_cmsis_dsp::transpose_view(matrix),vector); + for (std::size_t column = 0; column < 5; ++column) + assert(result[column] == reference[column]); +} + +void test11() +{ + // A transposed view selects the fused matrix-vector kernel without + // materializing the transpose. Five columns exercise the MVE tail. + float matrix_value[3][5] = { + {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}, + {6.0F, 7.0F, 8.0F, 9.0F, 10.0F}, + {11.0F, 12.0F, 13.0F, 14.0F, 15.0F}}; + float vector_value[] = {2.0F, -1.0F, 0.5F}; + const float expected[] = {1.5F, 3.0F, 4.5F, 6.0F, 7.5F}; + + ::arm_cmsis_dsp::MatrixView matrix( + &matrix_value[0][0], 3, 5, 5); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 3); + const auto transposed = ::arm_cmsis_dsp::transpose_view(matrix); + assert(transposed.rows() == 5); + assert(transposed.columns() == 3); + assert(transposed(4, 2) == 15.0F); + + auto result = ::arm_cmsis_dsp::dot(transposed, vector); + for (std::size_t column = 0; column < 5; ++column) + assert(result[column] == expected[column]); + + ::arm_cmsis_dsp::Matrix static_matrix; + ::arm_cmsis_dsp::Vector static_vector; + for (std::size_t row = 0; row < 2; ++row) + for (std::size_t column = 0; column < 5; ++column) + static_matrix(row,column) = 1.0F; + static_vector = 1.0F; + ::arm_cmsis_dsp::Vector static_result = + ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(static_matrix),static_vector); + for (std::size_t column = 0; column < 5; ++column) + assert(static_result[column] == 2.0F); + + test_transposed_dot_type(); + test_transposed_dot_type(); + test_transposed_dot_type>(); +#if !defined(DISABLEFLOAT16) + test_transposed_dot_type(); + test_transposed_dot_type>(); +#endif + test_transposed_dot_type<::arm_cmsis_dsp::Q31>(); + test_transposed_dot_type>(); + test_transposed_dot_type<::arm_cmsis_dsp::Q15>(); + test_transposed_dot_type>(); + test_transposed_dot_type<::arm_cmsis_dsp::Q7>(); + test_transposed_dot_mixed_type,float>(); + test_transposed_dot_mixed_type>(); +#if !defined(DISABLEFLOAT16) + test_transposed_dot_mixed_type,float16_t>(); + test_transposed_dot_mixed_type>(); +#endif + test_transposed_dot_mixed_type< + std::complex<::arm_cmsis_dsp::Q31>,::arm_cmsis_dsp::Q31>(); + test_transposed_dot_mixed_type< + ::arm_cmsis_dsp::Q31,std::complex<::arm_cmsis_dsp::Q31>>(); + test_transposed_dot_mixed_type< + std::complex<::arm_cmsis_dsp::Q15>,::arm_cmsis_dsp::Q15>(); + test_transposed_dot_mixed_type< + ::arm_cmsis_dsp::Q15,std::complex<::arm_cmsis_dsp::Q15>>(); + + // Exercise the same kernel through the fully connected backward pass. + Arena<4096> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + float input_value[5] = {}; + float input_gradient[5] = {}; + float weight_gradient[3][5] = {}; + float bias_value[3] = {}; + float bias_gradient[3] = {}; + float output_value[3] = {}; + float output_gradient[3] = {}; + BufferView input = tape.view(input_value, input_gradient, 5); + MatrixView weights = tape.parameter( + &matrix_value[0][0], &weight_gradient[0][0], 3, 5); + BufferView bias = tape.parameter(bias_value, bias_gradient, 3); + BufferView output = tape.view(output_value, output_gradient, 3); + output = fully_connected(input, weights, bias); + assert(tape.backward(output, vector_value, 3)); + for (std::size_t column = 0; column < 5; ++column) + assert(input.gradient(column) == expected[column]); +} + static void run_autodiff_tests() { test1(); @@ -407,6 +546,7 @@ static void run_autodiff_tests() test8(); test9(); test10(); + test11(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; From aa7b998dcd1432ff945e0b3fbc6fd36d4a266a21 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 09:57:23 +0200 Subject: [PATCH 04/19] autodiff : Added Iris classification example --- dsppp/Examples/LICENSE-Iris.txt | 16 ++ dsppp/Examples/README.md | 52 +++++ dsppp/Examples/autodiff_iris.cpp | 207 ++++++++++++++++++ dsppp/Examples/iris_data.hpp | 181 +++++++++++++++ dsppp/Include/dsppp/autodiff/README.md | 11 + .../autodiff/operators/cross_entropy.hpp | 127 +++++++++++ dsppp/example.cproject.yml | 3 +- dsppp/tests/autodiff_test.cpp | 23 ++ 8 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 dsppp/Examples/LICENSE-Iris.txt create mode 100644 dsppp/Examples/README.md create mode 100644 dsppp/Examples/autodiff_iris.cpp create mode 100644 dsppp/Examples/iris_data.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp diff --git a/dsppp/Examples/LICENSE-Iris.txt b/dsppp/Examples/LICENSE-Iris.txt new file mode 100644 index 000000000..c5765b81a --- /dev/null +++ b/dsppp/Examples/LICENSE-Iris.txt @@ -0,0 +1,16 @@ +Iris dataset license and attribution +==================================== + +The data in iris_data.hpp is derived from the Iris dataset distributed by the +UCI Machine Learning Repository. The dataset is licensed under the Creative +Commons Attribution 4.0 International license (CC BY 4.0): + +https://creativecommons.org/licenses/by/4.0/legalcode + +Attribution: + +Fisher, R. A. (1936). Iris [Dataset]. UCI Machine Learning Repository. +https://doi.org/10.24432/C56C76 + +The CMSIS-DSP source code around the data remains covered by the CMSIS-DSP +project license. diff --git a/dsppp/Examples/README.md b/dsppp/Examples/README.md new file mode 100644 index 000000000..781691833 --- /dev/null +++ b/dsppp/Examples/README.md @@ -0,0 +1,52 @@ +# CMSIS-DSP C++ examples + +The examples demonstrate the CMSIS-DSP C++ API and its autodiff extension. +Each source file defines its own `main`, so enable only one example at a time +in `dsppp/example.cproject.yml` by commenting and uncommenting its `file` line. + +## Expression examples + +- `dot_product.cpp` evaluates a fused vector expression ending in a dot + product. +- `vector_op.cpp` demonstrates floating-point and fixed-point vector + expressions, vector views, and strided views. +- `matrix_op.cpp` demonstrates matrix expressions and row and column views. + +## Autodiff examples + +- `autodiff_example.cpp` is a minimal fully connected layer followed by ReLU. + It runs one forward and backward pass and prints the resulting gradients. +- `autodiff_regression.cpp` trains a cubic polynomial to approximate a sine + wave. It demonstrates RMSProp, reusable graphs, parameter freezing, and + saving model parameters. +- `autodiff_iris.cpp` trains a small classifier on the Iris flower dataset. + It uses Adam and reserves 30 of the 150 samples for a final test that is not + used during training. + +### Iris classifier + +The Iris classifier contains two fully connected layers. Cross entropy is the +training loss and is not part of the network used for inference. + +```mermaid +flowchart LR + input["Input
4 flower measurements"] --> fc1["Fully connected
4 to 8"] + fc1 --> relu["ReLU
8 values"] + relu --> fc2["Fully connected
8 to 3"] + fc2 --> softmax["Softmax
3 class probabilities"] + softmax --> loss["Cross entropy loss"] + target["One-hot target"] --> loss +``` + +The dataset is stored as floating-point measurements in `iris_data.hpp` and +normalized when each sample is loaded. Its license and attribution are in +`LICENSE-Iris.txt`. + +## Building an example + +After selecting one source in `example.cproject.yml`, the Cortex-M55 virtual +target can be built with: + +```text +cbuild test.csolution.yml --context example.Release+VHT-Corstone-300 +``` diff --git a/dsppp/Examples/autodiff_iris.cpp b/dsppp/Examples/autodiff_iris.cpp new file mode 100644 index 000000000..fad86e5e3 --- /dev/null +++ b/dsppp/Examples/autodiff_iris.cpp @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include +#include + +#include "iris_data.hpp" + +#include + +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +namespace { + +constexpr std::size_t input_size = 4U; +constexpr std::size_t hidden_size = 8U; +constexpr std::size_t class_count = 3U; +constexpr std::size_t training_count = 120U; +constexpr std::size_t epoch_count = 120U; + +struct Model +{ + float hidden_weight[hidden_size][input_size]; + float hidden_bias[hidden_size]; + float output_weight[class_count][hidden_size]; + float output_bias[class_count]; +}; + +struct TrainingState +{ + Model model{}; + Arena<2048> arena{}; + Adam<67U, 4U> optimizer{1.0e-2F}; + float input[input_size]{}; + float hidden_linear[hidden_size]{}; + float hidden[hidden_size]{}; + float logits[class_count]{}; + float probability[class_count]{}; + float target[class_count]{}; + float loss{}; + std::uint8_t training_index[training_count]{}; +}; + +static std::uint32_t random_state = 0x12345678U; + +static std::uint32_t random_u32() noexcept +{ + random_state = random_state * 1664525U + 1013904223U; + return random_state; +} + +static float random_weight() noexcept +{ + const float unit = static_cast((random_u32() >> 8) & 0xffffU) / + 65535.0F; + return (unit - 0.5F) * 0.5F; +} + +static void initialize(Model &model) noexcept +{ + for (std::size_t row = 0; row < hidden_size; ++row) + { + model.hidden_bias[row] = 0.0F; + for (std::size_t column = 0; column < input_size; ++column) + model.hidden_weight[row][column] = random_weight(); + } + for (std::size_t row = 0; row < class_count; ++row) + { + model.output_bias[row] = 0.0F; + for (std::size_t column = 0; column < hidden_size; ++column) + model.output_weight[row][column] = random_weight(); + } +} + +static bool is_test_sample(std::size_t index) noexcept +{ + // Every fifth member of each class is held out: 10 per class. + return (index % 50U) % 5U == 0U; +} + +static std::uint32_t predicted_class( + const float (&probability)[class_count]) noexcept +{ + float maximum; + std::uint32_t index; + arm_max_f32(probability, class_count, &maximum, &index); + return index; +} + +} // namespace + +int main() +{ + std::printf("Iris classification with CMSIS-DSP autodiff\n"); + // Keep the training buffers and optimizer state off the limited stack. + TrainingState *state = new TrainingState; + initialize(state->model); + + Tape &tape = state->arena.tape(); + tape.register_operator(); + tape.register_operator(); + tape.register_operator(); + tape.register_operator(); + + BufferView input = tape.input(state->input); + MatrixView hidden_weight = tape.parameter(state->model.hidden_weight); + BufferView hidden_bias = tape.parameter(state->model.hidden_bias); + MatrixView output_weight = tape.parameter(state->model.output_weight); + BufferView output_bias = tape.parameter(state->model.output_bias); + BufferView hidden_linear = tape.output(state->hidden_linear); + BufferView hidden = tape.output(state->hidden); + BufferView logits = tape.output(state->logits); + BufferView probability = tape.output(state->probability); + BufferView target = tape.input(state->target); + BufferView loss = tape.output(state->loss); + + if (!state->optimizer.add(hidden_weight) || + !state->optimizer.add(hidden_bias) || + !state->optimizer.add(output_weight) || + !state->optimizer.add(output_bias)) + { + delete state; + return 1; + } + + // Keep 30 samples for testing. They are never used during training. + std::size_t training_position = 0U; + for (std::size_t sample = 0; sample < iris_data::sample_count; ++sample) + if (!is_test_sample(sample)) + state->training_index[training_position++] = + static_cast(sample); + + tape.begin_graph(); + for (std::size_t epoch = 0; epoch < epoch_count; ++epoch) + { + // Shuffle the training patterns before each epoch. + for (std::size_t i = training_count - 1U; i > 0U; --i) + { + const std::size_t other = random_u32() % (i + 1U); + const std::uint8_t temporary = state->training_index[i]; + state->training_index[i] = state->training_index[other]; + state->training_index[other] = temporary; + } + + float epoch_loss = 0.0F; + for (std::size_t position = 0; position < training_count; ++position) + { + const std::size_t sample = state->training_index[position]; + iris_data::normalized_features(sample, state->input); + for (std::size_t i = 0; i < class_count; ++i) + state->target[i] = i == iris_data::samples[sample].label + ? 1.0F : 0.0F; + + if (!tape.rewind_graph()) + { + delete state; + return 1; + } + hidden_linear = fully_connected(input, hidden_weight, hidden_bias); + hidden = relu(hidden_linear); + logits = fully_connected(hidden, output_weight, output_bias); + probability = softmax(logits); + loss = cross_entropy(probability, target); + + state->optimizer.zero_grad(); + if (!tape.backward(loss) || !state->optimizer.step()) + { + delete state; + return 1; + } + epoch_loss += state->loss; + } + + if ((epoch + 1U) % 20U == 0U) + std::printf("epoch %u: mean loss=%g\n", + static_cast(epoch + 1U), + static_cast(epoch_loss / training_count)); + } + + // Final check on the 30 samples that were kept out of training. + unsigned correct = 0U; + { + RecordingScope inference(tape, false); + for (std::size_t sample = 0; sample < iris_data::sample_count; + ++sample) + { + if (!is_test_sample(sample)) continue; + iris_data::normalized_features(sample, state->input); + hidden_linear = fully_connected(input, hidden_weight, hidden_bias); + hidden = relu(hidden_linear); + logits = fully_connected(hidden, output_weight, output_bias); + probability = softmax(logits); + if (predicted_class(state->probability) == + iris_data::samples[sample].label) + ++correct; + } + } + std::printf("final test accuracy=%u/30\n", correct); + + delete state; + return 0; +} diff --git a/dsppp/Examples/iris_data.hpp b/dsppp/Examples/iris_data.hpp new file mode 100644 index 000000000..9398ba3d1 --- /dev/null +++ b/dsppp/Examples/iris_data.hpp @@ -0,0 +1,181 @@ +#pragma once + +#include + +namespace iris_data { + +struct Sample +{ + // Measurements in centimetres, followed by the class index. + float feature[4]; + unsigned char label; +}; + +constexpr std::size_t sample_count = 150U; +constexpr Sample samples[sample_count] = { + {{5.1F, 3.5F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.0F, 1.4F, 0.2F}, 0}, + {{4.7F, 3.2F, 1.3F, 0.2F}, 0}, + {{4.6F, 3.1F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.6F, 1.4F, 0.2F}, 0}, + {{5.4F, 3.9F, 1.7F, 0.4F}, 0}, + {{4.6F, 3.4F, 1.4F, 0.3F}, 0}, + {{5.0F, 3.4F, 1.5F, 0.2F}, 0}, + {{4.4F, 2.9F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.1F, 1.5F, 0.1F}, 0}, + {{5.4F, 3.7F, 1.5F, 0.2F}, 0}, + {{4.8F, 3.4F, 1.6F, 0.2F}, 0}, + {{4.8F, 3.0F, 1.4F, 0.1F}, 0}, + {{4.3F, 3.0F, 1.1F, 0.1F}, 0}, + {{5.8F, 4.0F, 1.2F, 0.2F}, 0}, + {{5.7F, 4.4F, 1.5F, 0.4F}, 0}, + {{5.4F, 3.9F, 1.3F, 0.4F}, 0}, + {{5.1F, 3.5F, 1.4F, 0.3F}, 0}, + {{5.7F, 3.8F, 1.7F, 0.3F}, 0}, + {{5.1F, 3.8F, 1.5F, 0.3F}, 0}, + {{5.4F, 3.4F, 1.7F, 0.2F}, 0}, + {{5.1F, 3.7F, 1.5F, 0.4F}, 0}, + {{4.6F, 3.6F, 1.0F, 0.2F}, 0}, + {{5.1F, 3.3F, 1.7F, 0.5F}, 0}, + {{4.8F, 3.4F, 1.9F, 0.2F}, 0}, + {{5.0F, 3.0F, 1.6F, 0.2F}, 0}, + {{5.0F, 3.4F, 1.6F, 0.4F}, 0}, + {{5.2F, 3.5F, 1.5F, 0.2F}, 0}, + {{5.2F, 3.4F, 1.4F, 0.2F}, 0}, + {{4.7F, 3.2F, 1.6F, 0.2F}, 0}, + {{4.8F, 3.1F, 1.6F, 0.2F}, 0}, + {{5.4F, 3.4F, 1.5F, 0.4F}, 0}, + {{5.2F, 4.1F, 1.5F, 0.1F}, 0}, + {{5.5F, 4.2F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.1F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.2F, 1.2F, 0.2F}, 0}, + {{5.5F, 3.5F, 1.3F, 0.2F}, 0}, + {{4.9F, 3.6F, 1.4F, 0.1F}, 0}, + {{4.4F, 3.0F, 1.3F, 0.2F}, 0}, + {{5.1F, 3.4F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.5F, 1.3F, 0.3F}, 0}, + {{4.5F, 2.3F, 1.3F, 0.3F}, 0}, + {{4.4F, 3.2F, 1.3F, 0.2F}, 0}, + {{5.0F, 3.5F, 1.6F, 0.6F}, 0}, + {{5.1F, 3.8F, 1.9F, 0.4F}, 0}, + {{4.8F, 3.0F, 1.4F, 0.3F}, 0}, + {{5.1F, 3.8F, 1.6F, 0.2F}, 0}, + {{4.6F, 3.2F, 1.4F, 0.2F}, 0}, + {{5.3F, 3.7F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.3F, 1.4F, 0.2F}, 0}, + {{7.0F, 3.2F, 4.7F, 1.4F}, 1}, + {{6.4F, 3.2F, 4.5F, 1.5F}, 1}, + {{6.9F, 3.1F, 4.9F, 1.5F}, 1}, + {{5.5F, 2.3F, 4.0F, 1.3F}, 1}, + {{6.5F, 2.8F, 4.6F, 1.5F}, 1}, + {{5.7F, 2.8F, 4.5F, 1.3F}, 1}, + {{6.3F, 3.3F, 4.7F, 1.6F}, 1}, + {{4.9F, 2.4F, 3.3F, 1.0F}, 1}, + {{6.6F, 2.9F, 4.6F, 1.3F}, 1}, + {{5.2F, 2.7F, 3.9F, 1.4F}, 1}, + {{5.0F, 2.0F, 3.5F, 1.0F}, 1}, + {{5.9F, 3.0F, 4.2F, 1.5F}, 1}, + {{6.0F, 2.2F, 4.0F, 1.0F}, 1}, + {{6.1F, 2.9F, 4.7F, 1.4F}, 1}, + {{5.6F, 2.9F, 3.6F, 1.3F}, 1}, + {{6.7F, 3.1F, 4.4F, 1.4F}, 1}, + {{5.6F, 3.0F, 4.5F, 1.5F}, 1}, + {{5.8F, 2.7F, 4.1F, 1.0F}, 1}, + {{6.2F, 2.2F, 4.5F, 1.5F}, 1}, + {{5.6F, 2.5F, 3.9F, 1.1F}, 1}, + {{5.9F, 3.2F, 4.8F, 1.8F}, 1}, + {{6.1F, 2.8F, 4.0F, 1.3F}, 1}, + {{6.3F, 2.5F, 4.9F, 1.5F}, 1}, + {{6.1F, 2.8F, 4.7F, 1.2F}, 1}, + {{6.4F, 2.9F, 4.3F, 1.3F}, 1}, + {{6.6F, 3.0F, 4.4F, 1.4F}, 1}, + {{6.8F, 2.8F, 4.8F, 1.4F}, 1}, + {{6.7F, 3.0F, 5.0F, 1.7F}, 1}, + {{6.0F, 2.9F, 4.5F, 1.5F}, 1}, + {{5.7F, 2.6F, 3.5F, 1.0F}, 1}, + {{5.5F, 2.4F, 3.8F, 1.1F}, 1}, + {{5.5F, 2.4F, 3.7F, 1.0F}, 1}, + {{5.8F, 2.7F, 3.9F, 1.2F}, 1}, + {{6.0F, 2.7F, 5.1F, 1.6F}, 1}, + {{5.4F, 3.0F, 4.5F, 1.5F}, 1}, + {{6.0F, 3.4F, 4.5F, 1.6F}, 1}, + {{6.7F, 3.1F, 4.7F, 1.5F}, 1}, + {{6.3F, 2.3F, 4.4F, 1.3F}, 1}, + {{5.6F, 3.0F, 4.1F, 1.3F}, 1}, + {{5.5F, 2.5F, 4.0F, 1.3F}, 1}, + {{5.5F, 2.6F, 4.4F, 1.2F}, 1}, + {{6.1F, 3.0F, 4.6F, 1.4F}, 1}, + {{5.8F, 2.6F, 4.0F, 1.2F}, 1}, + {{5.0F, 2.3F, 3.3F, 1.0F}, 1}, + {{5.6F, 2.7F, 4.2F, 1.3F}, 1}, + {{5.7F, 3.0F, 4.2F, 1.2F}, 1}, + {{5.7F, 2.9F, 4.2F, 1.3F}, 1}, + {{6.2F, 2.9F, 4.3F, 1.3F}, 1}, + {{5.1F, 2.5F, 3.0F, 1.1F}, 1}, + {{5.7F, 2.8F, 4.1F, 1.3F}, 1}, + {{6.3F, 3.3F, 6.0F, 2.5F}, 2}, + {{5.8F, 2.7F, 5.1F, 1.9F}, 2}, + {{7.1F, 3.0F, 5.9F, 2.1F}, 2}, + {{6.3F, 2.9F, 5.6F, 1.8F}, 2}, + {{6.5F, 3.0F, 5.8F, 2.2F}, 2}, + {{7.6F, 3.0F, 6.6F, 2.1F}, 2}, + {{4.9F, 2.5F, 4.5F, 1.7F}, 2}, + {{7.3F, 2.9F, 6.3F, 1.8F}, 2}, + {{6.7F, 2.5F, 5.8F, 1.8F}, 2}, + {{7.2F, 3.6F, 6.1F, 2.5F}, 2}, + {{6.5F, 3.2F, 5.1F, 2.0F}, 2}, + {{6.4F, 2.7F, 5.3F, 1.9F}, 2}, + {{6.8F, 3.0F, 5.5F, 2.1F}, 2}, + {{5.7F, 2.5F, 5.0F, 2.0F}, 2}, + {{5.8F, 2.8F, 5.1F, 2.4F}, 2}, + {{6.4F, 3.2F, 5.3F, 2.3F}, 2}, + {{6.5F, 3.0F, 5.5F, 1.8F}, 2}, + {{7.7F, 3.8F, 6.7F, 2.2F}, 2}, + {{7.7F, 2.6F, 6.9F, 2.3F}, 2}, + {{6.0F, 2.2F, 5.0F, 1.5F}, 2}, + {{6.9F, 3.2F, 5.7F, 2.3F}, 2}, + {{5.6F, 2.8F, 4.9F, 2.0F}, 2}, + {{7.7F, 2.8F, 6.7F, 2.0F}, 2}, + {{6.3F, 2.7F, 4.9F, 1.8F}, 2}, + {{6.7F, 3.3F, 5.7F, 2.1F}, 2}, + {{7.2F, 3.2F, 6.0F, 1.8F}, 2}, + {{6.2F, 2.8F, 4.8F, 1.8F}, 2}, + {{6.1F, 3.0F, 4.9F, 1.8F}, 2}, + {{6.4F, 2.8F, 5.6F, 2.1F}, 2}, + {{7.2F, 3.0F, 5.8F, 1.6F}, 2}, + {{7.4F, 2.8F, 6.1F, 1.9F}, 2}, + {{7.9F, 3.8F, 6.4F, 2.0F}, 2}, + {{6.4F, 2.8F, 5.6F, 2.2F}, 2}, + {{6.3F, 2.8F, 5.1F, 1.5F}, 2}, + {{6.1F, 2.6F, 5.6F, 1.4F}, 2}, + {{7.7F, 3.0F, 6.1F, 2.3F}, 2}, + {{6.3F, 3.4F, 5.6F, 2.4F}, 2}, + {{6.4F, 3.1F, 5.5F, 1.8F}, 2}, + {{6.0F, 3.0F, 4.8F, 1.8F}, 2}, + {{6.9F, 3.1F, 5.4F, 2.1F}, 2}, + {{6.7F, 3.1F, 5.6F, 2.4F}, 2}, + {{6.9F, 3.1F, 5.1F, 2.3F}, 2}, + {{5.8F, 2.7F, 5.1F, 1.9F}, 2}, + {{6.8F, 3.2F, 5.9F, 2.3F}, 2}, + {{6.7F, 3.3F, 5.7F, 2.5F}, 2}, + {{6.7F, 3.0F, 5.2F, 2.3F}, 2}, + {{6.3F, 2.5F, 5.0F, 1.9F}, 2}, + {{6.5F, 3.0F, 5.2F, 2.0F}, 2}, + {{6.2F, 3.4F, 5.4F, 2.3F}, 2}, + {{5.9F, 3.0F, 5.1F, 1.8F}, 2}, +}; + +// Population statistics of all 150 samples. +constexpr float mean[4] = {5.843333F, 3.057333F, 3.758000F, 1.199333F}; +constexpr float inverse_standard_deviation[4] = { + 1.211678F, 2.301971F, 0.568374F, 1.316322F}; + +inline void normalized_features(std::size_t index, float (&output)[4]) noexcept +{ + for (std::size_t feature = 0; feature < 4U; ++feature) + output[feature] = + (samples[index].feature[feature] - mean[feature]) * + inverse_standard_deviation[feature]; +} + +} // namespace iris_data diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 88ae8fe30..e832729c2 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -89,6 +89,7 @@ record, gradient reset, backward rule, and expression adapter: | `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | | `operators/relu.hpp` | `ReluOperator` | `relu(x)` | | `operators/softmax.hpp` | `SoftmaxOperator` | `softmax(x)` | +| `operators/cross_entropy.hpp` | `CrossEntropyOperator` | `cross_entropy(probability, target)` | | `operators/quadratic_error.hpp` | `QuadraticErrorOperator` | `quadratic_error(prediction, target)` | An application includes and registers only the operators it uses. An operator @@ -417,6 +418,16 @@ d(loss)/d(prediction[i]) = 2 * (prediction[i] - target[i]) The target must be an input view; gradients are retained only for the prediction path and ultimately for its parameters. +Categorical cross entropy also returns a scalar sum: + +```text +loss = -sum(target[i] * log(max(probability[i], 1e-7))) +d(loss)/d(probability[i]) = -target[i] / max(probability[i], 1e-7) +``` + +It is intended for a probability vector produced by `softmax` and a one-hot +target input. The probability floor keeps both the loss and gradient finite. + ### Polynomial sinusoid regression `dsppp/Examples/autodiff_regression.cpp` follows the polynomial PyTorch example without a diff --git a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp new file mode 100644 index 000000000..3dd8c5ebc --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp @@ -0,0 +1,127 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Categorical cross entropy: -sum(target[i] * log(probability[i])). */ +class CrossEntropyOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *probability_value; + float *probability_gradient; + const float *target_value; + std::size_t length; + }; + + static constexpr float minimum_probability = 1.0e-7F; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = 0.0F; + arm_fill_f32(0.0F, record.probability_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const float seed = record.output_gradient[0]; + if (seed == 0.0F) return; + arm_clip_f32(record.probability_value, record.probability_gradient, + minimum_probability, + std::numeric_limits::max(), record.length); + for (std::size_t i = 0; i < record.length; ++i) + { + record.probability_gradient[i] = + -seed * record.target_value[i] / + record.probability_gradient[i]; + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &probability, + const BufferView &target) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, probability, target) || + OperatorAccess::gradients(probability) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + const std::size_t length = OperatorAccess::length(probability); + float result = 0.0F; + if (length != 0U) + { + // Reuse the probability gradient as forward scratch. backward() + // resets it before accumulating any gradient. + float *scratch = OperatorAccess::gradients(probability); + arm_clip_f32(OperatorAccess::values(probability), scratch, + minimum_probability, + std::numeric_limits::max(), length); + arm_vlog_f32(scratch, scratch, length); + arm_dot_prod_f32(OperatorAccess::values(target), scratch, length, + &result); + } + OperatorAccess::values(output)[0] = -result; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->probability_value = OperatorAccess::values(probability); + record->probability_gradient = OperatorAccess::gradients(probability); + record->target_value = OperatorAccess::values(target); + record->length = length; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class CrossEntropyExpression +{ +public: + CrossEntropyExpression(const BufferView &probability, + const BufferView &target) noexcept + : probability_(probability), target_(target) {} + + void evaluate(BufferView &output) const noexcept + { + CrossEntropyOperator::evaluate(output, probability_, target_); + } + +private: + BufferView probability_; + BufferView target_; +}; + +inline CrossEntropyExpression cross_entropy( + const BufferView &probability, const BufferView &target) noexcept +{ + return CrossEntropyExpression(probability, target); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/example.cproject.yml b/dsppp/example.cproject.yml index b9f96bbe7..7c36cd1c2 100644 --- a/dsppp/example.cproject.yml +++ b/dsppp/example.cproject.yml @@ -5,7 +5,8 @@ project: #- file: Examples/dot_product.cpp #- file: Examples/vector_op.cpp #- file: Examples/matrix_op.cpp - - file: Examples/autodiff_regression.cpp + #- file: Examples/autodiff_regression.cpp + - file: Examples/autodiff_iris.cpp - file: clang_sse300.c for-context: - +MPS3-Corstone-300 diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index 5d87a1c04..5dcabc6f9 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -8,6 +8,7 @@ extern "C" { #include #include +#include #include #include #include @@ -534,6 +535,27 @@ void test11() assert(input.gradient(column) == expected[column]); } +void test12() +{ + // Categorical cross entropy consumes probabilities and a one-hot target. + Arena<512> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + float probability_value[] = {0.1F, 0.7F, 0.2F}; + float target_value[] = {0.0F, 1.0F, 0.0F}; + float loss_value = 0.0F; + BufferView probability = tape.parameter(probability_value); + BufferView target = tape.input(target_value); + BufferView loss = tape.output(loss_value); + loss = cross_entropy(probability, target); + assert(loss_value > 0.3566F && loss_value < 0.3568F); + assert(tape.backward(loss)); + assert(probability.gradient(0) == 0.0F); + assert(probability.gradient(1) < -1.4285F && + probability.gradient(1) > -1.4287F); + assert(probability.gradient(2) == 0.0F); +} + static void run_autodiff_tests() { test1(); @@ -547,6 +569,7 @@ static void run_autodiff_tests() test9(); test10(); test11(); + test12(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; From bbcf4531e534c8fe3ffdf899bf42207b6f097869 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 10:31:25 +0200 Subject: [PATCH 05/19] autodiff : Added a dropout operator --- dsppp/Include/dsppp/autodiff/README.md | 15 ++ .../dsppp/autodiff/operators/dropout.hpp | 178 ++++++++++++++++++ dsppp/tests/autodiff_test.cpp | 40 ++++ 3 files changed, 233 insertions(+) create mode 100644 dsppp/Include/dsppp/autodiff/operators/dropout.hpp diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index e832729c2..90210742b 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -84,6 +84,7 @@ record, gradient reset, backward rule, and expression adapter: | `operators/sub.hpp` | `SubOperator` | `a - b` | | `operators/multiply.hpp` | `MultiplyOperator` | `a * b` | | `operators/dot.hpp` | `DotOperator` | `dot(a, b)` | +| `operators/dropout.hpp` | `DropoutOperator` | `dropout(x, generator, probability)` | | `operators/scale.hpp` | `ScaleOperator` | `scale(x, constant)` | | `operators/offset.hpp` | `OffsetOperator` | `offset(x, constant)` | | `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | @@ -418,6 +419,20 @@ d(loss)/d(prediction[i]) = 2 * (prediction[i] - target[i]) The target must be an input view; gradients are retained only for the prediction path and ultimately for its parameters. +### Dropout + +Dropout uses caller-owned random state and inverted scaling during training: + +```cpp +DropoutGenerator generator(1234U); +tape.register_operator(); +hidden = dropout(hidden_linear, generator, 0.2F); +``` + +When recording is disabled with `RecordingScope`, dropout copies its input +unchanged for inference. The random state saved in each tape record lets the +backward pass regenerate the training mask without storing a mask vector. + Categorical cross entropy also returns a scalar sum: ```text diff --git a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp new file mode 100644 index 000000000..dd81f105a --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp @@ -0,0 +1,178 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Caller-owned deterministic random state used by dropout. */ +class DropoutGenerator +{ +public: + explicit DropoutGenerator(std::uint32_t seed = 0x6D2B79F5U) noexcept + : state_(seed == 0U ? 0x6D2B79F5U : seed) {} + + void seed(std::uint32_t value) noexcept + { + state_ = value == 0U ? 0x6D2B79F5U : value; + } + +private: + std::uint32_t state_; + friend class DropoutOperator; +}; + +/** Inverted dropout during recording, identity when recording is disabled. */ +class DropoutOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + float *input_gradient; + std::size_t length; + std::uint32_t random_state; + float drop_probability; + float scale; + }; + + static std::uint32_t next(std::uint32_t &state) noexcept + { + state ^= state << 13U; + state ^= state >> 17U; + state ^= state << 5U; + return state; + } + + static bool keep(std::uint32_t &state, float drop_probability) noexcept + { + constexpr float inverse_24_bit_range = 1.0F / 16777216.0F; + const float uniform = + static_cast(next(state) >> 8U) * inverse_24_bit_range; + return uniform >= drop_probability; + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, record.length); + if (record.input_gradient != nullptr) + arm_fill_f32(0.0F, record.input_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + + if (record.drop_probability == 0.0F) + { + arm_add_f32(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); + return; + } + + std::uint32_t state = record.random_state; + for (std::size_t i = 0; i < record.length; ++i) + if (keep(state, record.drop_probability)) + record.input_gradient[i] += + record.output_gradient[i] * record.scale; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + DropoutGenerator &generator, + float drop_probability) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::require(*tape)) + return false; + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || + !(drop_probability >= 0.0F && drop_probability < 1.0F)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + const std::size_t length = OperatorAccess::length(output); + if (!OperatorAccess::recording(*tape)) + { + arm_copy_f32(OperatorAccess::values(input), + OperatorAccess::values(output), length); + return OperatorAccess::status(*tape) == Status::ok; + } + if (length == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + const float scale = 1.0F / (1.0F - drop_probability); + const std::uint32_t initial_state = generator.state_; + if (drop_probability == 0.0F) + { + arm_copy_f32(OperatorAccess::values(input), + OperatorAccess::values(output), length); + } + else + { + for (std::size_t i = 0; i < length; ++i) + OperatorAccess::values(output)[i] = + keep(generator.state_, drop_probability) + ? OperatorAccess::values(input)[i] * scale + : 0.0F; + } + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) + { + generator.state_ = initial_state; + return false; + } + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); + record->length = length; + record->random_state = initial_state; + record->drop_probability = drop_probability; + record->scale = scale; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class DropoutExpression +{ +public: + DropoutExpression(const BufferView &input, DropoutGenerator &generator, + float drop_probability) noexcept + : input_(input), generator_(generator), + drop_probability_(drop_probability) {} + + void evaluate(BufferView &output) const noexcept + { + DropoutOperator::evaluate(output, input_, generator_, + drop_probability_); + } + +private: + BufferView input_; + DropoutGenerator &generator_; + float drop_probability_; +}; + +inline DropoutExpression dropout(const BufferView &input, + DropoutGenerator &generator, + float drop_probability) noexcept +{ + return DropoutExpression(input, generator, drop_probability); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index 5dcabc6f9..bfe5404cd 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -10,6 +10,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -556,6 +557,44 @@ void test12() assert(probability.gradient(2) == 0.0F); } +void test13() +{ + // Training applies inverted dropout and backward regenerates the same + // mask. Disabling recording makes dropout an identity for inference. + Arena<1024> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + float input_value[16]; + float output_value[16] = {}; + for (std::size_t i = 0; i < 16U; ++i) input_value[i] = 1.0F; + BufferView input = tape.parameter(input_value); + BufferView output = tape.output(output_value); + DropoutGenerator generator(1234U); + output = dropout(input, generator, 0.5F); + + unsigned dropped = 0U; + unsigned kept = 0U; + for (std::size_t i = 0; i < 16U; ++i) + { + assert(output_value[i] == 0.0F || output_value[i] == 2.0F); + output_value[i] == 0.0F ? ++dropped : ++kept; + } + assert(dropped != 0U && kept != 0U); + + float seed[16]; + for (std::size_t i = 0; i < 16U; ++i) seed[i] = 1.0F; + assert(tape.backward(output, seed, 16U)); + for (std::size_t i = 0; i < 16U; ++i) + assert(input.gradient(i) == output_value[i]); + + { + RecordingScope inference(tape, false); + output = dropout(input, generator, 0.5F); + for (std::size_t i = 0; i < 16U; ++i) + assert(output_value[i] == input_value[i]); + } +} + static void run_autodiff_tests() { test1(); @@ -570,6 +609,7 @@ static void run_autodiff_tests() test10(); test11(); test12(); + test13(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; From a4591dd6f789c1356fc8a2c0b66f3413fe58e392 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 11:37:42 +0200 Subject: [PATCH 06/19] autodiff : added matrix multiply --- dsppp/Include/dsppp/autodiff/README.md | 15 ++ .../autodiff/operators/matrix_multiply.hpp | 157 ++++++++++++++++++ dsppp/tests/autodiff_test.cpp | 34 ++++ 3 files changed, 206 insertions(+) create mode 100644 dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 90210742b..63d760603 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -88,6 +88,7 @@ record, gradient reset, backward rule, and expression adapter: | `operators/scale.hpp` | `ScaleOperator` | `scale(x, constant)` | | `operators/offset.hpp` | `OffsetOperator` | `offset(x, constant)` | | `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | +| `operators/matrix_multiply.hpp` | `MatrixMultiplyOperator` | `matrix_multiply(x, w)` | | `operators/relu.hpp` | `ReluOperator` | `relu(x)` | | `operators/softmax.hpp` | `SoftmaxOperator` | `softmax(x)` | | `operators/cross_entropy.hpp` | `CrossEntropyOperator` | `cross_entropy(probability, target)` | @@ -472,6 +473,20 @@ persisting RMSProp or Adam state; inference does not need that state. ### Fully connected and ReLU +The matrix-multiply operator computes `Y = W X`, where `W` is a row-major +parameter matrix and `X` is a row-major input matrix stored in a `BufferView`. +The number of rows of `X` is inferred from the number of columns of `W`; its +number of columns is inferred from the input buffer length. Only `W` receives +a gradient: + +```text +dW = dY X^T +``` + +The forward pass uses `arm_mat_mult_f32`. The backward pass computes every +element of `dW` with `arm_dot_prod_f32`, using the contiguous rows of `dY` and +`X` without materializing `X^T`. + A fully connected node computes `y = m * x + b`. `m` is a row-major matrix parameter, `b` is a vector parameter, and the number of matrix columns must match the input length. The number of rows must match both the bias and output diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp new file mode 100644 index 000000000..6e68590c3 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Matrix product Y = W X, differentiating only the parameter matrix W. */ +class MatrixMultiplyOperator +{ + struct Record + { + detail::Node node; + float *output_gradient; + const float *input_value; + float *weight_gradient; + std::size_t rows; + std::size_t inner; + std::size_t columns; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + arm_fill_f32(0.0F, record.output_gradient, + record.rows * record.columns); + arm_fill_f32(0.0F, record.weight_gradient, + record.rows * record.inner); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + + // dW = dY X^T. X and dY are row-major, so every term needed for one + // element of dW is read from one row of each matrix. + for (std::size_t row = 0; row < record.rows; ++row) + { + const float *output_gradient = + record.output_gradient + row * record.columns; + // Unrolling could improve the performances but there are not yet any abstraction + // in C++ API to do it. + // We do not want to write a custom not generic implementation + // using low level intrinscis. + for (std::size_t inner = 0; inner < record.inner; ++inner) + { + const float *input = + record.input_value + inner * record.columns; + float sum = 0.0F; + arm_dot_prod_f32(output_gradient, input, record.columns, + &sum); + record.weight_gradient[row * record.inner + inner] += sum; + } + } + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::require(*tape)) + return false; + + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t inner = OperatorAccess::columns(weights); + const std::size_t input_length = OperatorAccess::length(input); + const std::size_t columns = inner == 0U ? 0U : input_length / inner; + if (!OperatorAccess::valid(*tape, output) || + !OperatorAccess::valid(*tape, input) || + !OperatorAccess::valid(*tape, weight_buffer) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::role(input) != BufferRole::input || + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::gradients(weight_buffer) == nullptr || + rows == 0U || inner == 0U || columns == 0U || + input_length % inner != 0U || + rows > std::numeric_limits::max() || + inner > std::numeric_limits::max() || + columns > std::numeric_limits::max() || + OperatorAccess::length(output) != rows * columns) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + arm_matrix_instance_f32 weight_matrix; + arm_matrix_instance_f32 input_matrix; + arm_matrix_instance_f32 output_matrix; + arm_mat_init_f32(&weight_matrix, static_cast(rows), + static_cast(inner), + const_cast(OperatorAccess::values( + weight_buffer))); + arm_mat_init_f32(&input_matrix, static_cast(inner), + static_cast(columns), + const_cast(OperatorAccess::values(input))); + arm_mat_init_f32(&output_matrix, static_cast(rows), + static_cast(columns), + OperatorAccess::values(output)); + if (arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix) != + ARM_MATH_SUCCESS) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->rows = rows; + record->inner = inner; + record->columns = columns; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +class MatrixMultiplyExpression +{ +public: + MatrixMultiplyExpression(const BufferView &input, + const MatrixView &weights) noexcept + : input_(input), weights_(weights) {} + + void evaluate(BufferView &output) const noexcept + { + MatrixMultiplyOperator::evaluate(output, input_, weights_); + } + +private: + BufferView input_; + MatrixView weights_; +}; + +inline MatrixMultiplyExpression matrix_multiply( + const BufferView &input, const MatrixView &weights) noexcept +{ + return MatrixMultiplyExpression(input, weights); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index bfe5404cd..f668546ca 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -12,6 +12,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -595,6 +596,38 @@ void test13() } } +void test14() +{ + // Y = W X uses CMSIS-DSP matrix multiplication in the forward pass and + // computes only dW = dY X^T in the backward pass. + Arena<1024> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + float weight_value[2][3] = { + {1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}}; + float input_value[3][2] = { + {1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; + float output_value[2][2] = {}; + MatrixView weights = tape.parameter(weight_value); + BufferView input = tape.input(&input_value[0][0], 6U); + BufferView output = tape.output(&output_value[0][0], 4U); + output = matrix_multiply(input, weights); + assert(output_value[0][0] == 22.0F); + assert(output_value[0][1] == 28.0F); + assert(output_value[1][0] == 49.0F); + assert(output_value[1][1] == 64.0F); + + const float seed[] = {1.0F, 2.0F, 3.0F, 4.0F}; + assert(tape.backward(output, seed, 4U)); + const float expected_gradient[2][3] = { + {5.0F, 11.0F, 17.0F}, {11.0F, 25.0F, 39.0F}}; + for (std::size_t row = 0; row < 2U; ++row) + for (std::size_t column = 0; column < 3U; ++column) + assert(weights.gradient(row, column) == + expected_gradient[row][column]); + assert(!input.has_gradient()); +} + static void run_autodiff_tests() { test1(); @@ -610,6 +643,7 @@ static void run_autodiff_tests() test11(); test12(); test13(); + test14(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; From 4e491023f58e4c3fe593970775a60264e706f653 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 14:06:07 +0200 Subject: [PATCH 07/19] dsppp : Add a new lazy matvec operator to enable fusion on mat x vector Use this operator in backward pass of matrix multiply in autodiff --- Documentation/Doxygen/src/matrix.md | 30 +++- dsppp/Include/dsppp/DSP/matrix_multiply.hpp | 75 ++++++++++ .../Include/dsppp/Helium/matrix_multiply.hpp | 101 ++++++++++++++ .../Include/dsppp/Scalar/matrix_multiply.hpp | 63 +++++++++ .../autodiff/operators/matrix_multiply.hpp | 40 +++--- dsppp/Include/dsppp/matrix.hpp | 132 ++++++++++++++++++ dsppp/tests/matrix_test.cpp | 15 +- 7 files changed, 435 insertions(+), 21 deletions(-) diff --git a/Documentation/Doxygen/src/matrix.md b/Documentation/Doxygen/src/matrix.md index d07cfaca7..75f915ff6 100644 --- a/Documentation/Doxygen/src/matrix.md +++ b/Documentation/Doxygen/src/matrix.md @@ -41,9 +41,23 @@ Once you have initialized matrixes, you can operate on them: Matrix result = a * a + b; ``` -The operators `+` and `*` are merged into the loop. `*` is the element-wise multiply. For the vector / matrix products you should use the operator `dot`. +The operators `+` and `*` are merged into the loop. `*` is the element-wise multiply. For eagerly evaluated vector / matrix products you should use the operator `dot`. -Note that fusion of operators will not work with `dot(Matrix, Matrix`). It is only supported with vectors : `dot(Vector,Vector)` or `dot(Matrix,Vector)`. +Note that fusion of operators will not work with `dot(Matrix, Matrix)` or +`dot(Matrix, Vector)`, since both operations return an owning, eagerly +evaluated result. Vector dot products return a scalar. + +For a matrix times vector product that must be fused with vector operations, +use the lazy `matvec` operator: + +```cpp +result += matvec(matrix, vector); +``` + +`matvec` returns a non-owning expression. In this example, the matrix-vector +product is accumulated directly into `result` without allocating a temporary +vector. The operands must remain alive until the complete expression is +evaluated. Use `dot(matrix, vector)` when an owning result is required. ## VectorView @@ -136,6 +150,18 @@ The compiler may use the move semantic to copy the temporary result of the `dot` In this case, no copy would occur and `result` after the assignment would be a vector allocated by `dot` so using the `TMP_ALLOC` . +### matvec + +```cpp +result = bias + matvec(matrix, vector); +``` + +Unlike `dot(matrix, vector)`, `matvec(matrix, vector)` is lazy. It participates +in the normal one-dimensional fusion loop, so the matrix-vector result can be +combined with additions, multiplications, or accumulation without first being +stored in a temporary vector. Matrix-matrix products remain eager and continue +to use `dot`. + ### diagonal ```cpp diff --git a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp index 2b8c591ff..5d739acef 100644 --- a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp @@ -253,6 +253,81 @@ inline void _dot_m_v(RES &res, } } +namespace detail { + +// Compute one matrix-row/vector dot product for each of up to four consecutive +// rows. Each packed block from the vector is loaded once and reused by all dot +// product accumulators. +template +inline void matvec_dsp_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using TM = typename traits::Scalar; + using Acc = typename vector_traits::temp_accumulator; + constexpr int lanes = vector_traits::nb_lanes; + + Acc sums[ROWS] = {}; + const TM *rows[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + rows[row] = matrix.const_ptr() + + (first_row + row) * matrix.stride(); + + index_t column = 0; + for (; column + lanes <= matrix.columns(); column += lanes) + { + const auto vector_data = vector.vector_op(column); + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::vmacc( + sums[row], inner::vload1<1>(rows[row] + column), vector_data); + } + + for (; column < matrix.columns(); ++column) + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac( + sums[row], rows[row][column], vector[column]); + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same DSP kernel. +template() && !is_complex() && + !std::is_same::Scalar,Q31>::value && + number_traits::Scalar>::is_fixed, + bool>::type = true> +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const DSP* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_dsp_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_dsp_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_dsp_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_dsp_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + template +inline void matvec_helium_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using Acc = DotResult; + using Temp = typename vector_traits::temp_accumulator; + constexpr int lanes = vector_traits::nb_lanes; + + Temp vector_sums[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = vector_traits::temp_acc_zero(); + + index_t column = 0; + if constexpr (has_predicate()) + { + for (; column < matrix.columns(); column += lanes) + { + const vector_length_t remaining = matrix.columns() - column; + const auto predicate = inner::vctpq::mk(remaining); + const auto vector_data = + vector.vector_op_tail(column, remaining); + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = inner::vmacc( + vector_sums[row], + matrix.row(first_row + row).vector_op_tail( + column, remaining), + vector_data, predicate); + } + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator( + inner::vreduce(vector_sums[row])); + } + else + { + for (; column + lanes <= matrix.columns(); column += lanes) + { + const auto vector_data = vector.vector_op(column); + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = inner::vmacc( + vector_sums[row], + matrix.row(first_row + row).vector_op(column), + vector_data); + } + + Acc sums[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::vreduce(vector_sums[row]); + for (; column < matrix.columns(); ++column) + { + const auto value = vector[column]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac( + sums[row], matrix(first_row + row, column), value); + } + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); + } +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same Helium kernel. +template() && has_vector_inst() && + same_nb_lanes(), bool>::type = true> +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const Helium* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_helium_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_helium_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_helium_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_helium_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + #if defined(ARM_MATH_MVEI) || defined(ARM_MATH_MVEF) template +inline void matvec_scalar_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using Acc = typename number_traits::accumulator; + + Acc sums[ROWS] = {}; + const TM *rows[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + rows[row] = matrix.const_ptr() + + (first_row + row) * matrix.stride(); + + for (index_t column = 0; column < matrix.columns(); ++column) + { + const TV value = vector[column]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac(sums[row], rows[row][column], value); + } + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same scalar kernel. +template +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const Scalar* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_scalar_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_scalar_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_scalar_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_scalar_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + #include "matrix_multiply_fixed.hpp" #include "matrix_multiply_float.hpp" diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp index 6e68590c3..90e58897a 100644 --- a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -1,14 +1,18 @@ #pragma once #include +#include -#include #include #include #include #include +#define dbgInst(imm) __asm volatile("DBG %0\n\t" : :"Ir" ((imm)) ) +#define startSectionNB(num) dbgInst(((num) & 0x7) | 0x0) +#define stopSectionNB(num) dbgInst(((num) & 0x7) | 0x8) + namespace arm_cmsis_dsp { namespace autodiff { @@ -39,26 +43,26 @@ class MatrixMultiplyOperator { Record &record = reinterpret_cast(node); - // dW = dY X^T. X and dY are row-major, so every term needed for one - // element of dW is read from one row of each matrix. + ::arm_cmsis_dsp::MatrixView + input_value(const_cast(record.input_value), record.inner, + record.columns, record.columns); + + // dW = dY X^T. For each row of dY, this is X times that row. The lazy + // matvec expression fuses the product with gradient accumulation. + + startSectionNB(1); for (std::size_t row = 0; row < record.rows; ++row) { - const float *output_gradient = - record.output_gradient + row * record.columns; - // Unrolling could improve the performances but there are not yet any abstraction - // in C++ API to do it. - // We do not want to write a custom not generic implementation - // using low level intrinscis. - for (std::size_t inner = 0; inner < record.inner; ++inner) - { - const float *input = - record.input_value + inner * record.columns; - float sum = 0.0F; - arm_dot_prod_f32(output_gradient, input, record.columns, - &sum); - record.weight_gradient[row * record.inner + inner] += sum; - } + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + row * record.columns, 0, + record.columns); + ::arm_cmsis_dsp::VectorView weight_gradient( + record.weight_gradient + row * record.inner, 0, + record.inner); + weight_gradient += + ::arm_cmsis_dsp::matvec(input_value, output_gradient); } + stopSectionNB(1); } public: diff --git a/dsppp/Include/dsppp/matrix.hpp b/dsppp/Include/dsppp/matrix.hpp index 9b3e921f7..09217a3c3 100644 --- a/dsppp/Include/dsppp/matrix.hpp +++ b/dsppp/Include/dsppp/matrix.hpp @@ -554,6 +554,138 @@ struct VecRef,((R<0) || (C<0))> }; }; +/** Lazy matrix times vector expression. */ +template +struct _MatVec: _Expr<_MatVec> +{ + using MatrixScalar = typename traits::Scalar; + using VectorScalar = typename traits::Scalar; + using Scalar = typename MixedRes::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; + constexpr static int vector_lanes = vector_traits::nb_lanes; + constexpr static int block_size = vector_lanes > 4 ? vector_lanes : 4; +#else + constexpr static int block_size = 4; +#endif + + _MatVec(const M &matrix, const V &vector) + : matrix_(matrix), vector_(vector), cache_start_(matrix.rows()), + cache_count_(0) {} + + vector_length_t length() const { return matrix_.rows(); } + + Scalar operator[](const index_t row) const + { + load_block(row); + return cache_[row - cache_start_]; + } + +#if defined(HAS_VECTOR) + auto vector_op(const index_t row) const + { + load_block(row); + return inner::vload1<1>(cache_ + row - cache_start_); + } + + auto vector_op_tail(const index_t row, + const vector_length_t) const + { + return vector_op(row); + } +#endif + +private: + void load_block(const index_t row) const + { + if (row >= cache_start_ && row < cache_start_ + cache_count_) + return; + + cache_start_ = row - row % block_size; + const vector_length_t remaining = matrix_.rows() - cache_start_; + cache_count_ = remaining < block_size ? remaining : block_size; + for (index_t i = cache_count_; i < block_size; ++i) + cache_[i] = Scalar{}; + + _matvec_block(cache_, matrix_, vector_, cache_start_, cache_count_, + (::arm_cmsis_dsp::ARCH *)nullptr); + } + + M matrix_; + V vector_; + mutable Scalar cache_[block_size] = {}; + mutable index_t cache_start_; + mutable vector_length_t cache_count_; +}; + +template +struct traits<_MatVec> +{ + using Scalar = typename MixedRes::Scalar, + typename traits::Scalar>::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct ElementType<_MatVec> +{ + using type = typename MixedRes::Scalar, + typename traits::Scalar>::type; +}; + +template +struct IsVector<_MatVec> +{ + constexpr static bool value = true; +}; + +template +struct IsDynamic<_MatVec> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength<_MatVec> +{ + constexpr static vector_length_t value = NbRows::value; +}; + +template +struct Complexity<_MatVec> +{ + constexpr static int value = 1; +}; + +template +struct VecRef<_MatVec> +{ + using type = _MatVec; + static type ref(const type &expression) { return expression; } +}; + +/** + * @brief Create a lazy matrix times vector expression. + * + * Unlike dot(matrix, vector), matvec does not allocate or evaluate a result. + * It can therefore be fused with surrounding elementwise vector operations. + */ +template::value || + CompatibleDynamicMatVecProduct::value), + bool>::type = true> +inline auto matvec(const M &matrix, const V &vector) +{ + using MatrixRef = VecRef; + using VectorRef = VecRef; + return _MatVec( + MatrixRef::ref(matrix),VectorRef::ref(vector)); +} + /** Lazy transposed-matrix times vector expression. */ template struct _TransposedMatVec: _Expr<_TransposedMatVec> diff --git a/dsppp/tests/matrix_test.cpp b/dsppp/tests/matrix_test.cpp index a42e12de4..34d04a27c 100644 --- a/dsppp/tests/matrix_test.cpp +++ b/dsppp/tests/matrix_test.cpp @@ -893,9 +893,13 @@ void testmatvec() startSectionNB(1); #if defined(STATIC_TEST) PVector res = dot(m,a); + PVector fused; #else PVector res = dot(m,a); + PVector fused(R); #endif + fused = number_traits::one(); + fused += matvec(m,a); stopSectionNB(1); STOP_CYCLE_MEASUREMENT; @@ -926,6 +930,15 @@ void testmatvec() { printf("matrix times vector failed \r\n"); } + for (index_t row = 0; row < R; ++row) + { + const T expected = res[row] + number_traits::one(); + if (fused[row] != expected) + { + printf("lazy matrix times vector accumulation failed \r\n"); + break; + } + } std::cout << "=====\r\n"; } @@ -2355,4 +2368,4 @@ void matrix_test() #endif #endif #endif -} \ No newline at end of file +} From 3d99e99b87651d6abc356cc8e4932e5f24e750b6 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 15:25:21 +0200 Subject: [PATCH 08/19] autodiff: Remove debug code in mat mul operator --- dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp index 90e58897a..5bc4e9c71 100644 --- a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -9,10 +9,6 @@ #include #include -#define dbgInst(imm) __asm volatile("DBG %0\n\t" : :"Ir" ((imm)) ) -#define startSectionNB(num) dbgInst(((num) & 0x7) | 0x0) -#define stopSectionNB(num) dbgInst(((num) & 0x7) | 0x8) - namespace arm_cmsis_dsp { namespace autodiff { @@ -50,7 +46,6 @@ class MatrixMultiplyOperator // dW = dY X^T. For each row of dY, this is X times that row. The lazy // matvec expression fuses the product with gradient accumulation. - startSectionNB(1); for (std::size_t row = 0; row < record.rows; ++row) { ::arm_cmsis_dsp::VectorView output_gradient( @@ -62,7 +57,6 @@ class MatrixMultiplyOperator weight_gradient += ::arm_cmsis_dsp::matvec(input_value, output_gradient); } - stopSectionNB(1); } public: From 53ea969632f02195f1b349fdf8227bbb1132db1f Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Tue, 11 Aug 2026 15:48:11 +0200 Subject: [PATCH 09/19] Don't force cmsis toolbox version in dsppp csolution file --- dsppp/test.csolution.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dsppp/test.csolution.yml b/dsppp/test.csolution.yml index 4ed800a5b..39d342a51 100644 --- a/dsppp/test.csolution.yml +++ b/dsppp/test.csolution.yml @@ -124,4 +124,3 @@ solution: projects: - project: ./tests/test.cproject.yml - project: ./example.cproject.yml - created-for: CMSIS-Toolbox@2.14.1 From 48456adb91972fc4c6a7fead4e5a578aa5abb178 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Wed, 12 Aug 2026 09:07:08 +0200 Subject: [PATCH 10/19] Update autodiff documentation --- dsppp/Include/dsppp/autodiff/README.md | 793 +++--------------- dsppp/Include/dsppp/autodiff/doc/concepts.md | 133 +++ .../dsppp/autodiff/doc/implementation_flow.md | 281 +++++++ dsppp/Include/dsppp/autodiff/doc/operators.md | 196 +++++ .../Include/dsppp/autodiff/doc/optimizers.md | 174 ++++ .../dsppp/autodiff/doc/training_loop.md | 176 ++++ 6 files changed, 1059 insertions(+), 694 deletions(-) create mode 100644 dsppp/Include/dsppp/autodiff/doc/concepts.md create mode 100644 dsppp/Include/dsppp/autodiff/doc/implementation_flow.md create mode 100644 dsppp/Include/dsppp/autodiff/doc/operators.md create mode 100644 dsppp/Include/dsppp/autodiff/doc/optimizers.md create mode 100644 dsppp/Include/dsppp/autodiff/doc/training_loop.md diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 63d760603..587ec39c2 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -1,742 +1,147 @@ -# Reverse automatic differentiation reference +# Reverse automatic differentiation -`reverse.hpp` is a deliberately small reverse-mode automatic differentiation -(AD) implementation for embedded use. It currently handles `float` buffers, -vector arithmetic, learnable scalar scale and offset operations, vector dot -products, fully connected, ReLU, and softmax nodes, and a quadratic-error -loss. Fixed-storage Adam and RMSProp optimizers support user-written training -loops. The structure is intended to be extended with tensor operators. +Training an algorithm requires the derivative of a loss with respect to every +parameter being learned. Automatic differentiation (AD) computes these +gradients from the operations executed by the algorithm. + +Reverse-mode AD is efficient when a computation has more inputs than outputs. +Training normally has many parameter inputs and one scalar loss output, so one +forward evaluation followed by one backward pass computes the gradients for +all parameters. + +The purpose of adding reverse AD to CMSIS-DSP is to enable efficient on-device +learning on resource-constrained devices. It reuses accelerated CMSIS-DSP +kernels and the fusion capabilities of the CMSIS-DSP C++ extension, while +providing predictable fixed memory use, no heap allocation, and no exceptions. ## Why this implementation uses CMSIS-DSP Automatic differentiation needs high performance in both directions. The forward pass evaluates the model, while the backward pass propagates and accumulates gradients. This implementation uses CMSIS-DSP for both rather than -treating it only as a collection of forward inference kernels. +treating it only as a collection of forward-inference kernels. -Where an operation maps directly to an optimized CMSIS-DSP C kernel, the +When an operation maps directly to an optimized CMSIS-DSP C kernel, its forward pass uses that kernel. For example, dot products use -`arm_dot_prod_f32`, and fully connected matrix-vector products use -`arm_mat_vec_mult_f32`. These kernels provide implementations optimized for -the selected Arm target, including Helium implementations where available. +`arm_dot_prod_f32`, fully connected matrix-vector products use +`arm_mat_vec_mult_f32`, and matrix products use `arm_mat_mult_f32`. These +kernels provide implementations optimized for the selected Arm target, +including Helium implementations where available. -The CMSIS-DSP C++ extension is particularly valuable during the backward pass. -Reverse rules frequently combine several element-wise operations with an -accumulation. A typical example is: +The CMSIS-DSP C++ expression system is particularly useful in backward rules, +which often combine element-wise computation with accumulation. A typical +update is: ```text gradient += input * output_gradient ``` -Calling separate C kernels for the multiplication and addition would normally -require an intermediate buffer and two loops: one loop produces the products -and another accumulates them. The C++ expression system can fuse the complete -expression into one loop. That loop performs more useful computation for each -load and store, avoids the temporary buffer, reduces memory traffic, and gives -the compiler a larger loop body to vectorize effectively. +Calling separate multiplication and addition kernels would require a temporary +buffer and two loops. A C++ expression can fuse the complete update into one +loop, avoiding the temporary and reducing memory traffic while giving the +compiler a larger operation to optimize and vectorize. -The fully connected backward pass illustrates this approach: +For example, the fully connected backward pass computes: ```text bias_gradient += output_gradient weight_gradient += outer(output_gradient, input_value) -input_gradient += transpose(weight_value) * output_gradient +input_gradient += dot(transpose_view(weight_value), output_gradient) ``` -Bias and outer-product accumulation use fused CMSIS-DSP C++ expressions. The -input gradient uses optimized dot products over strided column views, avoiding -a materialized matrix transpose. The same principle applies to future vector, -matrix, and tensor reverse rules: express the whole local gradient update as a -fused accumulation whenever the C++ extension supports it. - -This fusion capability is an important distinction when selecting a Helium -math library for training. A library may provide individually optimized -Helium primitives but still require multiple loops and intermediate buffers -for a compound backward expression. CMSIS-DSP combines optimized kernels with -a C++ expression mechanism capable of fusing those loops, which is especially -important for backward passes because they contain more compound operations -and accumulations than typical forward inference code. - -## What are the arena and the tape? - -Reverse AD needs to remember the operations performed during the forward -calculation. It later visits those operations in the opposite order to -propagate derivatives from the output back to the inputs. This ordered record -of operations is traditionally called a **tape**, by analogy with recording a -sequence on magnetic tape and playing it backward. The name is standard AD -terminology; it is not related to a C++ container type. - -In this implementation, `Tape` does two jobs: - -1. During the forward calculation, it writes one small, fixed-size record for - each operation. A record contains non-owning buffer pointers and the - information needed by that operation's derivative rule. -2. `backward()` follows those records in reverse order and accumulates the - gradients. - -The tape core contains no numerical operators. Each operator is a separate, -self-contained class and header containing its validation, forward rule, tape -record, gradient reset, backward rule, and expression adapter: - -| Header | Operator class | Expression | -| --- | --- | --- | -| `operators/add.hpp` | `AddOperator` | `a + b` | -| `operators/sub.hpp` | `SubOperator` | `a - b` | -| `operators/multiply.hpp` | `MultiplyOperator` | `a * b` | -| `operators/dot.hpp` | `DotOperator` | `dot(a, b)` | -| `operators/dropout.hpp` | `DropoutOperator` | `dropout(x, generator, probability)` | -| `operators/scale.hpp` | `ScaleOperator` | `scale(x, constant)` | -| `operators/offset.hpp` | `OffsetOperator` | `offset(x, constant)` | -| `operators/fully_connected.hpp` | `FullyConnectedOperator` | `fully_connected(x, m, b)` | -| `operators/matrix_multiply.hpp` | `MatrixMultiplyOperator` | `matrix_multiply(x, w)` | -| `operators/relu.hpp` | `ReluOperator` | `relu(x)` | -| `operators/softmax.hpp` | `SoftmaxOperator` | `softmax(x)` | -| `operators/cross_entropy.hpp` | `CrossEntropyOperator` | `cross_entropy(probability, target)` | -| `operators/quadratic_error.hpp` | `QuadraticErrorOperator` | `quadratic_error(prediction, target)` | - -An application includes and registers only the operators it uses. An operator -header that is not included is not part of that translation unit. Registration -uses an allocation-free, open-addressed hash set of type tokens stored directly -in `Tape`; it does not instantiate or retain an operator object. Registration -and expression checks are expected O(1), with collision resolution by linear -probing and no deletion or tombstones. - -Expression evaluation checks registration before validation or forward -computation. Using an included but unregistered operator leaves the output -unchanged and sets the sticky status to `Status::operator_not_registered`. -The registry defaults to 16 distinct operator types. Define -`DSPPP_AUTODIFF_MAX_OPERATORS` to a larger power of two before including the -core header when an application needs more slots. Exceeding the configured -capacity sets `Status::operator_registry_full`. `Tape::reset()` preserves -registrations, so they normally need to be installed only once during -application setup. - -The tape needs memory for gradients and operation records. An `Arena` -owns exactly `Bytes` bytes of fixed storage and constructs a `Tape` that uses -that storage. It does -**not** allocate a `std::vector`, call the heap, or grow at runtime. For example, -`Arena<2048>` contains a 2048-byte array directly inside the `Arena` object. If -the object is a local variable, that storage is normally on the stack; if it is -static, the storage is static as well. +These are the expressions used by `FullyConnectedOperator::backward`. The +outer-product expression fuses multiplication with weight-gradient +accumulation. For the input gradient, `transpose_view` is a zero-copy view and +the `dot` overload returns a lazy transposed-matrix/vector expression, so its +result is accumulated without materializing either a transposed weight matrix +or a temporary result vector. Matrix multiplication similarly accumulates its +weight gradient through lazy `matvec` expressions. This combination of +optimized C kernels and fusible C++ expressions is especially important for +training, because backward passes contain more compound updates and +accumulations than typical forward inference code. -All value arrays, including intermediate and final outputs, are allocated by -the user. The way a buffer is registered tells the tape whether it needs a -gradient: +## How reverse differentiation works here -- `tape.input(values)` registers ordinary algorithm input. It allocates no - gradient because the application does not request derivatives for it. -- `tape.parameter(values)` registers trainable parameters and allocates their - gradients from the arena. -- `tape.output(values)` registers an intermediate or final output and allocates - its adjoint from the arena because it is needed during reverse propagation. +During the **forward pass**, each operator computes its output and, when +recording is enabled, adds a small record to a `Tape`. The record identifies +the values and gradients needed by that operator's derivative rule. The +ordered collection is called a tape because the operations are recorded going +forward and replayed in reverse. -These functions never copy or take ownership of values. This role-based rule -is used by vector and matrix operators and is intended for future tensors as -well. Operation -records contain only pointers, dimensions, and small operator-specific -metadata, so their cost does not grow with the amount of numerical data. -Gradient storage naturally requires one `float` per parameter or intermediate -element, but no storage is spent on input gradients. +The **backward pass** starts at the selected output, normally a scalar loss. It +sets the loss gradient to one and visits the recorded operations in reverse +order. Each operation applies its local derivative rule and accumulates its +contribution into the gradients of earlier intermediate results and learnable +parameters. This reverse application of the chain rule produces all parameter +gradients needed by an optimizer. -Applications that need exact placement of every buffer can use the overloads -`tape.parameter(values, gradients, length)` and -`tape.output(values, gradients, length)` with caller-owned gradient storage. -These advanced overloads perform no arena allocation for the view. +Numerical values remain in buffers owned by the application. An `Arena` +provides a fixed amount of memory for gradients and tape records, so memory use +cannot grow unexpectedly at runtime. Ordinary inputs registered with +`tape.input()` do not receive gradient storage. -Large buffers may live on the stack, in static memory, or in an -application-owned memory pool. They must remain alive until `backward()` -returns. Input values must not change between their use in the forward pass and -the backward pass, because derivative rules may read them. Output buffers must -be distinct from input buffers in this reference implementation. Recorded -computations use SSA-style storage: do not reuse or overwrite an earlier live -output buffer for another operation before `backward()` or `Tape::reset()`. +The implementation is modular: an application includes and registers only the +operator headers it uses. The core in `reverse.hpp` manages views, fixed arena +storage, operator registration, and reverse traversal; each operator header +contains its own forward computation and derivative rule. -## Basic use +## Minimal example ```cpp #include -#include -#include -#include #include using namespace arm_cmsis_dsp::autodiff; -Arena<2048> arena; // Gradient and operation-record storage. -Tape &tape = arena.tape(); -tape.register_operator(); -tape.register_operator(); -tape.register_operator(); -tape.register_operator(); - -float x_value[] = {1.0F, 2.0F}; -float alpha_value = 2.0F; -float beta_value = 3.0F; -float scaled_value[2] = {}; -float shifted_value[2] = {}; -float sum_value[2] = {}; -float result_value[1] = {}; - -BufferView x = tape.input(x_value); // No gradient for x. -BufferView alpha = tape.parameter(alpha_value); // Scalar parameter. -BufferView beta = tape.parameter(beta_value); // Scalar parameter. -BufferView scaled = tape.output(scaled_value); -BufferView shifted = tape.output(shifted_value); -BufferView sum = tape.output(sum_value); -BufferView result = tape.output(result_value); - -scaled = scale(x, alpha); // scaled = alpha * x -shifted = offset(scaled, beta); // shifted = scaled + beta -sum = shifted + x; -result = dot(sum, x); // result[0] == 24 - -if (tape.backward(result)) { - // x.has_gradient() == false - // alpha.gradient(0) == 5 - // beta.gradient(0) == 3 -} -``` - -## Training, optimizers, and frozen parameters - -`Adam` and -`RMSProp` keep all optimizer state in -fixed-size arrays inside the optimizer object. `MaximumElements` is the total -number of scalar parameter values and `MaximumParameters` is the maximum -number of separately registered parameter views (16 by default). Neither -optimizer allocates memory or throws exceptions. - -### RMSProp template and constructor arguments - -The two RMSProp template arguments are compile-time storage capacities, not -algorithm hyperparameters: - -```cpp -RMSProp optimizer; -``` - -- `MaximumElements` is the maximum total number of scalar values across all - parameter views added to the optimizer. It has no default. -- `MaximumParameters` is the maximum number of separately added parameter - views. It defaults to 16. - -For example, the polynomial regression has a three-element coefficient vector -and a separate scalar bias: - -```cpp -RMSProp<4, 2> optimizer; -optimizer.add(coefficients); // Three elements; first parameter view. -optimizer.add(bias); // One element; second parameter view. -``` - -This consumes all four element slots and both parameter-view slots. A matrix -counts as one parameter view, while its flattened `rows * columns` values all -count toward `MaximumElements`. Frozen parameters still occupy their original -slots. Adding the same value buffer again is idempotent and does not consume -another slot. - -These capacities determine the optimizer object's static memory footprint. -RMSProp contains one `float square_average_[MaximumElements]` array plus an -`Entry entries_[MaximumParameters]` metadata array. Each entry stores the -parameter value and gradient pointers, its length, its state-array offset, and -whether it is trainable. Parameter values and gradients are not copied into -the optimizer: values remain caller-owned and gradients remain in the tape -arena (or in caller storage when that overload is used). There is no heap -allocation and neither capacity can grow at runtime. - -When only the first argument is specified, the 16-view default applies: - -```cpp -RMSProp<100> optimizer; // Up to 100 scalar values in up to 16 views. -``` - -The constructor arguments configure the numerical RMSProp update: - -```cpp -RMSProp<4, 2> optimizer( - 1.0e-3F, // learning_rate - 0.99F, // alpha: squared-gradient moving-average decay - 1.0e-8F // epsilon: denominator stabilization -); -``` - -For each trainable scalar parameter, `step()` performs: - -```text -square_average = alpha * square_average - + (1 - alpha) * gradient^2 -parameter -= learning_rate * gradient / (sqrt(square_average) + epsilon) -``` - -If `add()` would exceed `MaximumParameters`, it returns `false` and sets -`OptimizerStatus::too_many_parameters`. If the total number of scalar values -would exceed `MaximumElements`, it sets `OptimizerStatus::too_many_elements`. -A non-parameter view, a missing gradient, or an unknown view passed to -`freeze()` sets `OptimizerStatus::invalid_parameter`. Optimizer errors are -sticky: after an error, `good()` is false, `status()` reports the first error, -and `step()` returns false. - -### Adam template and constructor arguments - -Adam uses the same two compile-time capacity arguments as RMSProp: - -```cpp -Adam optimizer; -``` - -- `MaximumElements` is the maximum total number of scalar parameter values. -- `MaximumParameters` is the maximum number of parameter views and defaults - to 16. - -Consequently, the same regression parameters fit in: - -```cpp -#include - -Adam<4, 2> optimizer; -optimizer.add(coefficients); // Three scalar values in one view. -optimizer.add(bias); // One scalar value in a second view. -``` - -Matrices, frozen parameters, and duplicate registrations are counted in the -same way as for RMSProp. `Adam<100>` means up to 100 scalar values distributed -across the default maximum of 16 parameter views. - -Adam keeps two state values per scalar parameter, so its principal state -storage is twice that of RMSProp: - -```text -float first_moment_[MaximumElements] -float second_moment_[MaximumElements] -Entry entries_[MaximumParameters] -``` - -It additionally stores scalar configuration values and the current powers of -`beta1` and `beta2` used for bias correction. Parameter values and gradients -are referenced through the metadata entries and are not copied. Adam performs -no heap allocation. - -The constructor arguments are: - -```cpp -Adam<4, 2> optimizer( - 1.0e-3F, // learning_rate - 0.9F, // beta1: first-moment decay - 0.999F, // beta2: second-moment decay - 1.0e-8F // epsilon: denominator stabilization -); -``` - -On successful optimizer step `t`, each trainable scalar is updated as follows: - -```text -first_moment = beta1 * first_moment - + (1 - beta1) * gradient -second_moment = beta2 * second_moment - + (1 - beta2) * gradient^2 - -corrected_first = first_moment / (1 - beta1^t) -corrected_second = second_moment / (1 - beta2^t) - -parameter -= learning_rate * corrected_first - / (sqrt(corrected_second) + epsilon) -``` - -The correction compensates for moments initialized to zero, particularly -during the first training steps. Adam's global step advances whenever -`step()` succeeds. Frozen entries are skipped: their parameter values and two -moment arrays remain unchanged, although they continue to occupy capacity. - -Adam reports the same sticky `OptimizerStatus` values as RMSProp: -`too_many_parameters`, `too_many_elements`, and `invalid_parameter`. -`add()`, `freeze()`, `good()`, `status()`, `zero_grad()`, and `step()` therefore -have the same usage pattern for both optimizer types. Switching optimizers in -a training loop normally requires only changing the included header, the -optimizer type, its capacities, and its numerical hyperparameters. - -The user owns the training loop. Register each parameter once, evaluate the -graph, run the reverse pass, and then update the parameters: - -```cpp -#include - -RMSProp<4, 2> optimizer(1.0e-3F); // Four scalar values in two views. -optimizer.add(coefficients); -optimizer.add(bias); - -tape.begin_graph(); // Everything allocated so far remains persistent. -for (std::size_t step = 0; step < number_of_steps; ++step) { - tape.rewind_graph(); // Reclaim records from the preceding iteration. - - for (std::size_t sample = 0; sample < sample_count; ++sample) { - // Scalar output views share the corresponding elements of the - // persistent vector gradient buffers. - BufferView x = tape.input(feature_value[sample]); - BufferView p = tape.output(&polynomial_value[sample], - &polynomial.gradients()[sample], 1); - BufferView y = tape.output(&prediction_value[sample], - &prediction.gradients()[sample], 1); - p = dot(x, coefficients); - y = p + bias; - } - - // prediction and target cover the complete training set. - loss = quadratic_error(prediction, target); - - optimizer.zero_grad(); - if (!tape.backward(loss) || !optimizer.step()) { - handle_error(); - } -} -``` - -`begin_graph()` places an arena mark after persistent gradient buffers. -`rewind_graph()` returns to that mark in constant time, preserving the views, -gradient buffers, parameter values, and operator registrations while releasing -the previous iteration's operation records. - -Parameters can be frozen without rebuilding the graph. A frozen parameter -still participates in forward and backward propagation, but `step()` leaves -its value and optimizer state unchanged: - -```cpp -freeze_parameters(optimizer, coefficients); // Bias-only fine tuning. -unfreeze_parameters(optimizer, coefficients); // Train it again later. -``` - -This API freezes parameter views rather than operator classes because an -operator is stateless and several invocations can use different parameters. -To freeze a layer, pass all parameter views owned by that layer. Both optimizer -implementations return `false` and set a sticky `OptimizerStatus` when an -operation exceeds capacity or receives an invalid parameter view. - -Quadratic error computes the sum, not the mean: - -```text -loss = sum((prediction[i] - target[i])^2) -d(loss)/d(prediction[i]) = 2 * (prediction[i] - target[i]) -``` - -The target must be an input view; gradients are retained only for the -prediction path and ultimately for its parameters. - -### Dropout - -Dropout uses caller-owned random state and inverted scaling during training: - -```cpp -DropoutGenerator generator(1234U); -tape.register_operator(); -hidden = dropout(hidden_linear, generator, 0.2F); -``` - -When recording is disabled with `RecordingScope`, dropout copies its input -unchanged for inference. The random state saved in each tape record lets the -backward pass regenerate the training mask without storing a mask vector. - -Categorical cross entropy also returns a scalar sum: - -```text -loss = -sum(target[i] * log(max(probability[i], 1e-7))) -d(loss)/d(probability[i]) = -target[i] / max(probability[i], 1e-7) -``` - -It is intended for a probability vector produced by `softmax` and a one-hot -target input. The probability floor keeps both the loss and gradient finite. - -### Polynomial sinusoid regression - -`dsppp/Examples/autodiff_regression.cpp` follows the polynomial PyTorch example without a -fully connected or ReLU node. For every sample it constructs the caller-owned -feature vector `{x, x^2, x^3}` and evaluates: - -```cpp -polynomial = dot(features, coefficients); -prediction = polynomial + bias; -loss = quadratic_error(prediction, target); -``` - -RMSProp learns `bias + c1*x + c2*x^2 + c3*x^3` from 100 uniformly spaced -points over `[-pi, pi]`. Each training step first computes all 100 predictions. -It then creates one quadratic-error node over both complete vectors, calls -`backward()` once, and calls the optimizer once. The optimized objective is -therefore the global sum -`sum((prediction[sample] - target[sample])^2)`, rather than 100 independent -online updates. Dividing the reported value by 100 gives its mean quadratic -error without changing the optimum. - -The example also demonstrates bias-only fine tuning and saving and restoring -values through the C-compatible `SinePolynomialParameters` struct. -The checkpoint deliberately contains parameter values only. Continuing -training with exactly the same optimizer trajectory would additionally require -persisting RMSProp or Adam state; inference does not need that state. - -### Fully connected and ReLU - -The matrix-multiply operator computes `Y = W X`, where `W` is a row-major -parameter matrix and `X` is a row-major input matrix stored in a `BufferView`. -The number of rows of `X` is inferred from the number of columns of `W`; its -number of columns is inferred from the input buffer length. Only `W` receives -a gradient: - -```text -dW = dY X^T -``` - -The forward pass uses `arm_mat_mult_f32`. The backward pass computes every -element of `dW` with `arm_dot_prod_f32`, using the contiguous rows of `dY` and -`X` without materializing `X^T`. - -A fully connected node computes `y = m * x + b`. `m` is a row-major matrix -parameter, `b` is a vector parameter, and the number of matrix columns must -match the input length. The number of rows must match both the bias and output -lengths: - -```cpp -#include -#include - -tape.register_operator(); -tape.register_operator(); -``` - -```cpp -float x_value[] = {2.0F, -1.0F}; -float m_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; -float b_value[] = {1.0F, 0.0F}; -float linear_value[2] = {}; -float activation_value[2] = {}; - -BufferView x = tape.input(x_value); -MatrixView m = tape.parameter(m_value); -BufferView b = tape.parameter(b_value); -BufferView linear = tape.output(linear_value); -BufferView activation = tape.output(activation_value); - -linear = fully_connected(x, m, b); // linear == {1, -7} -activation = relu(linear); // activation == {1, 0} - -const float seed[] = {1.0F, 1.0F}; -tape.backward(activation, seed, 2); -``` - -The fully connected backward rule is: - -```text -m_gradient[row, column] += y_gradient[row] * x_value[column] -b_gradient[row] += y_gradient[row] -x_gradient[column] += y_gradient[row] * m_value[row, column] -``` - -The forward matrix-vector product uses `arm_mat_vec_mult_f32`, followed by a -CMSIS-DSP C++ vector expression that adds the bias. The reverse pass uses fused -C++ expressions for bias accumulation and -`m_gradient += outer(y_gradient, x_value)`. When an input gradient is needed, -each row-major weight column is exposed as a strided view and accumulated with -the C++ dot implementation. This avoids a transposed matrix and all temporary -numerical buffers. Because the C matrix descriptor stores dimensions as -`uint16_t`, larger dimensions are rejected as a shape mismatch. - -The last line is skipped when `x` is an `input`. It is used when `x` is an -intermediate output, allowing several fully connected and activation nodes to -be chained while gradients are ultimately retained only for parameters. - -ReLU is element-wise. It propagates the output gradient when its input value is -strictly positive and propagates zero for negative values and at zero. - -### How buffer length is determined - -In the example, the vector buffers are actual fixed-size C arrays. The overload -below illustrates how each registration function receives an array by -reference, so the compiler deduces `Length` without storing runtime size -information in the array: - -```cpp -template -BufferView Tape::input(float (&values)[Length]); -``` - -This deduction only works while the expression still has an array type. Once -an array is converted to `float *`, its length is not available in C++ and -cannot be inferred safely. Buffers obtained dynamically, from a memory pool, or -through a pointer therefore use the explicit-length overload: - -```cpp -float *values = application_pool_allocate(number_of_elements); -BufferView dynamic_input = tape.input(values, number_of_elements); -``` - -The autodiff implementation does not allocate or free `values`; it only -allocates the corresponding gradient buffer in its fixed arena. The same rule -applies to matrix and future tensor views: dimensions can be deduced from true -array types when available, but pointer-based storage must supply its shape. - -For example, `tape.parameter(float_matrix)` deduces both dimensions from a true -`float[Rows][Columns]` array. A dynamically allocated row-major matrix uses -`tape.parameter(pointer, rows, columns)`. - -The same tape may instead use any caller-owned buffer: - -```cpp -alignas(std::max_align_t) unsigned char memory[2048]; -Tape tape(memory, sizeof(memory)); -``` - -`Tape::reset()` releases all arena-managed gradients and operation records in -constant time, without individual deallocation. It does not release or modify -caller-owned value buffers. All views become invalid after reset and must be -created again. - -## Value-only evaluation - -Recording can be disabled while an output is computed. Operators still write -the numerical result into the user-provided output buffer, but consume no -additional arena space for operation records and that output cannot be used as -the root of `backward()`: - -```cpp -const std::size_t before = tape.used(); +int main() { - RecordingScope no_gradient(tape, false); - scaled = scale(x, alpha); - shifted = offset(scaled, beta); - sum = shifted + x; - result = dot(sum, x); - use(result_value[0]); + Arena<512> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + + float x_value[] = {1.0F, 2.0F}; + float a_value = 3.0F; + float y_value[2] = {}; + + BufferView x = tape.input(x_value); + BufferView a = tape.parameter(a_value); + BufferView y = tape.output(y_value); + + // y = a * x + y = scale(x, a); // y = {3, 6}; records the operation + const float seed[] = {1.0F, 1.0F}; + tape.backward(y, seed, 2U); // da = 1*1 + 1*2 = 3 + return tape.good() ? 0 : 1; } -// tape.used() == before ``` -Calling `backward()` on a value-only output fails with -`Status::invalid_output`. The views and buffers may be reused later in a -recorded calculation. - -## Arena and failure model - -The implementation performs no `new`, `delete`, `malloc`, or standard-container -allocation. Placement `new` only starts the lifetime of records inside the -arena supplied by the caller. It does not request memory. - -There are no C++ exceptions. Errors are reported through `Tape::status()` and -the Boolean result of `backward()`. The first error is sticky: - -- `out_of_memory`: a gradient buffer or operation record did not fit in the - arena; -- `tape_mismatch`: views came from different tapes, dimensions or required - roles differ, or a required buffer is null; -- `invalid_output`: `backward()` received a value-only output or an invalid - seed; -- `operator_not_registered`: an expression used an operator type that was not - registered on this tape; -- `operator_registry_full`: the fixed registration list has no free slot. - -If an operation record exhausts the remaining arena, the operation still -computes its numeric value but the result is detached. A view whose gradient -allocation fails is invalid. Always check `tape.good()` or `backward()` before -consuming derivatives. `used()` can be measured on representative worst-case -graphs to select a static arena size. +## Documentation -## How the reverse pass works +- [Concepts and memory model](doc/concepts.md) explains the arena, tape, + buffer roles, recording, buffer lifetime, and errors. +- [Training loop](doc/training_loop.md) gives a complete, defined example and + explains setup, graph reuse, gradient clearing, reverse propagation, and the + optimizer update. +- [Operators](doc/operators.md) documents the current operator families, + formulas, shape rules, dropout behavior, and the CMSIS-DSP implementation + paths. +- [Optimizers](doc/optimizers.md) documents Adam and RMSProp capacities, + initialization, updates, freezing, and errors. +- [Worked implementation flow: `y = a * x`](doc/implementation_flow.md) follows + one expression through `reverse.hpp`, including its tape record, `producer`, + node links, gradient reset, seed, and backward rule. -Each recorded operation appends one fixed-size record. The common `Node` prefix -stores links and pointers to the operation's gradient reset and backward rules. -The rest of an operation record contains non-owning buffer pointers and -dimensions. -`backward()` first clears the associated gradient buffers, then walks the -linked tape in reverse creation order. +## Examples and tests -For vector add `z[i] = x[i] + y[i]`, the local rule is: +`dsppp/Examples/autodiff_regression.cpp` trains a polynomial approximation to +`sin(x)` with RMSProp. `dsppp/Examples/autodiff_iris.cpp` trains a small fully +connected classifier with Adam. -```text -x_gradient[i] += z_gradient[i] -y_gradient[i] += z_gradient[i] -``` - -For `z = dot(x, y)`, it is: - -```text -x_gradient[i] += z_gradient[0] * y_value[i] -y_gradient[i] += z_gradient[0] * x_value[i] -``` - -An addition or dot-product operand contributes to a gradient only when it is a -parameter or intermediate with gradient storage. An `input` has a null gradient -pointer, so the same backward rule simply skips that contribution. - -For vector scaling `z[i] = alpha[0] * x[i]`, `alpha` is a scalar parameter. -Its rule is: - -```text -alpha_gradient[0] += z_gradient[i] * x_value[i] (summed over i) -x_gradient[i] += alpha[0] * z_gradient[i] -``` - -For vector offset `z[i] = x[i] + beta[0]`, `beta` is also a scalar parameter: - -```text -beta_gradient[0] += z_gradient[i] (summed over i) -x_gradient[i] += z_gradient[i] -``` - -The input-gradient contribution is skipped when `x` was declared with -`tape.input(x)` and therefore has no gradient storage. - -All operators follow the same ownership rule: inputs and outputs stay in -caller storage, while their records retain non-owning pointers. Future matrix -and tensor operators should use views with shape and stride metadata rather -than copying numerical buffers into the tape. - -## Adding an operator - -The intended extension pattern is: - -1. Create one header in `operators/` and one uniquely named operator class. - That class identity is also its runtime registration token. -2. Define a trivially destructible record whose first member is `detail::Node`. - Store only non-owning buffer pointers and small shape/stride metadata. -3. Keep the forward evaluator, gradient reset, and backward rule in the - operator class. Every path must be `noexcept`. -4. At the start of evaluation, call - `OperatorAccess::require(tape)`. Use `OperatorAccess` for - validation, recording state, status reporting, and appending the record. -5. Add a small expression class with `evaluate(BufferView&)`; the generic - `BufferView::operator=` invokes it, so the core never needs modification. -6. Test registered execution, unregistered failure, value-only execution, - derivatives, tape exhaustion, shapes, aliases, and buffer lifetimes. - -Do not add an operator-specific method or record to `Tape`. Do not store -pointers to temporary caller data. If an operator needs a large forward -intermediate during its backward rule, make it an explicit caller-provided -workspace or output view rather than copying it into the tape arena. - -This implementation is intentionally contiguous and sequential. It is not -thread safe, does not manage buffer lifetimes, and does not yet support strides -or higher derivatives. A training graph is reevaluated on every iteration; -only its arena storage is reused. - -## Building and running the board test - -Autodiff is tested only with the existing dsppp board-test infrastructure; it -has no standalone host CMake project. `dsppp/tests/autodiff_test.cpp` is listed -in `dsppp/tests/test.cproject.yml`, and `AUTODIFF_TEST` is a test category in -`dsppp/run_all.py`. - -From the `dsppp` directory, select the category and its supported datatype: +Autodiff uses the existing dsppp board-test infrastructure. From `dsppp`, run: ```sh python run_all.py --test AUTODIFF_TEST --dt F32_DT ``` -`run_all.py` writes `test_config.h` and rebuilds when that generated -configuration changes. The test body is compiled and executed only when all -three generated selections are present: - -```cpp -#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) -``` - -Autodiff currently supports only `float` and the dynamic test mode. Other -datatype or static-mode configurations retain an empty `autodiff_test()` entry -point, so the shared project can still build without executing unsupported -autodiff cases. Board compiler options, CMSIS-DSP sources, linking, and runtime -selection continue to come from the existing solution and layer files. +It currently supports `float` and dynamic test mode. The test body is selected +when `AUTODIFF_TEST`, `F32_DT`, and `DYNAMIC_TEST` are defined. diff --git a/dsppp/Include/dsppp/autodiff/doc/concepts.md b/dsppp/Include/dsppp/autodiff/doc/concepts.md new file mode 100644 index 000000000..457d27626 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/concepts.md @@ -0,0 +1,133 @@ +# Concepts and memory model + +## Tape and arena + +Reverse-mode AD first computes values and records the operations that produced +them. It then visits those records in reverse order to propagate gradients. +This ordered record is conventionally called a **tape**: operations are +recorded during the forward pass and played backward during differentiation. + +`Tape` manages this record and the gradient buffers. `Arena` supplies +exactly `Bytes` bytes of storage to a `Tape`: + +```cpp +Arena<2048> arena; +Tape &tape = arena.tape(); +``` + +The capacity is fixed and no dynamic memory allocation is performed. If the +arena is too small for a gradient or operation record, the tape reports +`Status::out_of_memory` instead of trying to grow it. + +## Values, views, and gradients + +The application owns every value buffer, including intermediates and final +outputs. A `BufferView` is only a non-owning description containing value and +gradient pointers, a length, its tape, its role, and the operation that +produced it. + +- `tape.input(values)` creates an ordinary input. It has no gradient storage. +- `tape.parameter(values)` creates a trainable parameter and allocates its + gradient from the arena. +- `tape.output(values)` creates an intermediate or result and allocates its + gradient from the arena. + +Caller-owned gradients can be supplied with overloads such as +`tape.parameter(values, gradients, length)` and +`tape.output(values, gradients, length)`. Those views consume no arena space +for their gradient arrays. + +Values and caller-owned gradients must remain alive until `backward()` has +finished. Values read by a backward rule must not be changed after the forward +operation. Output storage must not alias an operator's inputs, and a live +intermediate must not be overwritten by a later operation before +`backward()`, `rewind_graph()`, or `reset()`. + +True C arrays preserve their extent, so `tape.input(array)` can deduce the +length. A pointer does not carry a length and requires an explicit overload: + +```cpp +BufferView input = tape.input(pointer, number_of_elements); +MatrixView weights = tape.parameter(pointer, rows, columns); +``` + +## Operator registration + +The core contains no numerical operators. Each operator is a separate header +and class. An application includes and registers only what it uses. This keeps +code size down: operators that are not referenced can be removed by the +compiler and linker. + +```cpp +#include + +tape.register_operator(); +``` + +Using an operator that has not been registered sets +`Status::operator_not_registered`. A tape can register 16 operator types by +default. Define `DSPPP_AUTODIFF_MAX_OPERATORS` to a larger power of two before +including `reverse.hpp` when more are needed. Registrations survive +`Tape::reset()`. + +See [Operators](operators.md) for the current operator API and formulas. + +## Recording and `RecordingScope` + +Normally an assignment such as `y = relu(x)` computes `y` and appends a record +for its backward rule. When recording is disabled, the numerical result is +still computed but no record is appended. This is useful for inference and +consumes no additional arena space. + +`RecordingScope` temporarily changes this tape setting for one block of code: + +```cpp +{ + RecordingScope inference(tape, false); + probability = softmax(logits); + // No operation record is added here. +} // The recording state that existed before the scope is restored. +``` + +The name means "the scope in which tape recording has this setting." Creating +the object saves the previous setting and applies the requested one. Leaving +the block restores the previous setting, including when the block exits early. + +A value computed without recording has no `producer`, so it cannot be passed +as the root of `backward()`. Doing so sets `Status::invalid_output`. + +## Reusing arena storage + +`Tape::reset()` releases all arena-managed gradients and records, invalidating +all views. Operator registrations are retained. + +Training normally needs to retain parameter and output gradient buffers while +rebuilding the operation records on every iteration: + +1. Create persistent views. +2. Call `begin_graph()`. This saves the current arena position. +3. At the start of each iteration, call `rewind_graph()`. It returns to that + saved position and clears the record list without invalidating the views. +4. Evaluate the forward graph again, which records fresh operations pointing + at the current values. + +Both marking and rewinding take constant time. + +## Failure model + +The first error is sticky until `clear_status()`, `reset()`, `begin_graph()`, or +`rewind_graph()` clears it as specified by the API. Check `tape.good()` or the +Boolean result of `backward()` before using gradients. + +- `out_of_memory`: a gradient or operation record did not fit in the arena. +- `tape_mismatch`: views, roles, shapes, pointers, or aliasing are invalid for + an operation. +- `invalid_output`: the backward root or seed is invalid, or graph rewinding + was requested without a mark. +- `operator_not_registered`: an expression's operator was not registered. +- `operator_registry_full`: no registration slot remains. + +If record allocation fails after a forward kernel runs, its numerical result +may already have been written, but the output is detached from the tape. +Measure `tape.used()` on representative worst-case graphs to choose an arena +capacity with suitable margin. diff --git a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md new file mode 100644 index 000000000..b6fbb3ee9 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md @@ -0,0 +1,281 @@ +# Worked implementation flow: `y = a * x` + +This example follows one vector scaling operation through the public API, +`ScaleOperator`, and the internals of `reverse.hpp`. It uses a trainable scalar +`a`, a constant input vector `x`, and an output vector `y`: + +```text +y[i] = a * x[i] +``` + +The scale operator is used instead of element-wise `MultiplyOperator` because +`a` has one element while `x` and `y` have several. + +## Complete example + +```cpp +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ + Arena<512> arena; + Tape &tape = arena.tape(); + tape.register_operator(); + + float a_value = 3.0F; + float x_value[] = {2.0F, -1.0F}; + float y_value[2] = {}; + + BufferView a = tape.parameter(a_value); + BufferView x = tape.input(x_value); + BufferView y = tape.output(y_value); + + y = scale(x, a); // Forward result: {6, -3}. + + const float seed[] = {1.0F, 1.0F}; + if (!tape.backward(y, seed, 2U)) + return 1; + + // a.gradient(0) == 1*2 + 1*(-1) == 1 + // x.has_gradient() == false + return 0; +} +``` + +The seed is the vector supplied as input to `backward()`. It is copied into +`y`'s gradient buffer before reverse traversal starts. Because `y` is a vector, +each output element has its own derivative with respect to `a`: + +```text +dy/da = {x[0], x[1]} = {2, -1} +``` + +The seed specifies how those output contributions are combined. With +`seed = {1, 1}`, the backward pass computes +`a.gradient(0) = 1*2 + 1*(-1) = 1`. A seed of `{1, 0}` would select only +`y[0]` and produce `a.gradient(0) = 2`. One call does not build the complete +derivative vector for `y`; it propagates the combination selected by the seed. +Training normally ends in a scalar loss, for which +`backward(scalar_output)` uses the default seed `1`. + +## Objects before the forward pass + +`Arena<512>` contains the storage array and constructs a `Tape` over it. The +tape initially has `used_ == 0`, `tail_ == nullptr`, `recording_ == true`, and +`status_ == Status::ok`. + +The three registrations create non-owning views: + +| View | `values_` points to | `gradients_` | `role_` | `producer_` | +| --- | --- | --- | --- | --- | +| `a` | `a_value` | one arena `float` | `parameter` | null | +| `x` | `x_value` | null | `input` | null | +| `y` | `y_value` | two arena `float`s | `intermediate` | null | + +`parameter()` and `output()` allocate and zero gradient arrays. `input()` does +not allocate one because the application did not request a derivative for +`x`. The value arrays themselves remain entirely caller-owned. + +`producer_` answers "which recorded node last produced this view?" Inputs and +parameters have no producer in this graph. It is initially null for `y` and is +set only after a record has been appended successfully. + +## Field map for `reverse.hpp` + +The core member names have these responsibilities: + +| Type | Member | Meaning | +| --- | --- | --- | +| `BufferView` | `values_` | Pointer to caller-owned forward values. | +| `BufferView` | `gradients_` | Pointer to its adjoint, or null for an input. | +| `BufferView` | `length_` | Number of scalar elements. | +| `BufferView` | `tape_` | Owning tape identity used for compatibility checks. | +| `BufferView` | `producer_` | Last successfully recorded node that wrote this view. | +| `BufferView` | `role_` | `input`, `parameter`, or `intermediate`. | +| `Tape` | `storage_`, `capacity_` | Start and byte size of caller-supplied arena storage. | +| `Tape` | `used_` | Current byte offset for the next aligned allocation. | +| `Tape` | `tail_` | Most recently appended node. | +| `Tape` | `recording_` | Whether forward operations append records. | +| `Tape` | `status_` | Sticky first-error status. | +| `Tape` | `graph_begin_`, `graph_marked_` | Saved allocation offset and validity flag used by graph rewinding. | +| `Tape` | `registered_operators_` | Fixed hash table of registered operator type tokens. | +| `detail::Node` | `previous` | Link to the preceding record. | +| `detail::Node` | `backward` | Function pointer for the record's local reverse rule. | +| `detail::Node` | `reset_gradient` | Function pointer that clears buffers used by that rule. | + +`OperatorAccess` is the narrow bridge used by separate operator headers to +read these private fields, validate views, report failure, append a record, and +set an output producer. It keeps operator-specific code out of `Tape` itself. + +## Forward assignment + +The statement + +```cpp +y = scale(x, a); +``` + +executes these steps: + +1. `scale(x, a)` returns a small `ScaleExpression` containing copies of the two + non-owning views. Only their pointers and metadata are copied; the `x` and + `a` value buffers are not copied. +2. `BufferView::operator=` calls `ScaleExpression::evaluate(y)`. +3. The expression calls `ScaleOperator::evaluate(y, x, a)`. +4. The operator clears `y.producer_`. This prevents an old record from being + mistaken for the producer if validation or allocation fails. +5. `OperatorAccess::require(tape)` checks the fixed registry. +6. Validation checks that all views belong to the same tape; `x` and `y` have + the same length; `a` is a one-element parameter with a gradient; and value + and gradient buffers do not alias illegally. +7. `arm_scale_f32` computes `{3*2, 3*(-1)}` into `y_value`, giving `{6, -3}`. +8. Because recording is enabled and the output is nonempty, the operator asks + `OperatorAccess::append` for a `ScaleOperator::Record` in the arena. + +The record contains a `detail::Node` as its first member, followed by only the +pointers and length required by the local derivative: + +```text +node +output_gradient -> y.gradients() +input_value -> x.values() +input_gradient -> null +scale_value -> a.values() +scale_gradient -> a.gradients() +length = 2 +``` + +`Tape::append` aligns arena storage, constructs the record in place, installs +the `backward` and `reset_gradient` function pointers, and links the new node: + +```cpp +record->node.previous = tail_; +tail_ = &record->node; +``` + +For this one-operation graph, `previous` is null. Finally, +`y.producer_ = &record->node` marks this record as the origin of `y`. + +```mermaid +flowchart LR + AV["a view
parameter
value 3, gradient da"] + XV["x view
input
values 2, -1
no gradient"] + E["ScaleOperator::evaluate"] + R["Scale record
Node + pointers + length"] + YV["y view
values 6, -3
gradient dy
producer"] + + AV --> E + XV --> E + E -->|"arm_scale_f32"| YV + E -->|"append when recording"| R + YV -. "producer_ points to" .-> R + R -->|"scale_gradient"| AV + R -->|"input_value"| XV +``` + +## What the node links mean + +Every record begins with the same three-field `detail::Node`: `previous`, +`backward`, and `reset_gradient`. `previous` links records in creation order. +If a later operation computes `z` from `y`, its node points to the scale node: + +```text +z.producer_ -> z record -> scale record -> null +``` + +The link is chronological, not a list of a node's operands. Operand +relationships are represented by the value and gradient pointers stored in +each operator-specific record. Starting at the requested output's `producer_` +prevents traversal into records created after that output. Earlier unrelated +records can be visited, but their output gradients are zero, so their local +rules make no contribution. + +## `backward(y, seed, 2)` + +`Tape::backward` performs three phases. + +### 1. Validate the root + +The tape must still be good; `y` must belong to this tape and have valid value +and gradient storage; `y.producer_` must be non-null; and the seed pointer and +length must match `y.length_`. A value-only result produced while recording was +disabled has no producer and is therefore rejected. + +### 2. Reset graph gradients, then install the seed + +Starting at `y.producer_`, the first reverse walk calls each node's +`reset_gradient` function. For the scale record this sets: + +```text +dy = {0, 0} +da = 0 +``` + +It would also clear `dx` if `x` had gradient storage. The tape then copies the +caller seed into the root output gradient: + +```text +dy = {1, 1} +``` + +Resetting before seeding is essential: otherwise resetting the root record +would erase the seed. Clearing all reachable buffers also makes repeated calls +to `backward()` deterministic instead of unintentionally accumulating an +earlier reverse pass. + +### 3. Traverse backward + +The second reverse walk calls the node's operator-specific `backward` function. +For `y[i] = a*x[i]`, the chain rule is: + +```text +da += sum(dy[i] * x[i]) +dx[i] += a * dy[i] +``` + +The implementation evaluates the first expression with the CMSIS-DSP C++ dot +operation: + +```text +da = 0 + dot({1, 1}, {2, -1}) = 1 +``` + +`input_gradient` is null because `x` came from `tape.input()`, so the `dx` +update is skipped. If `x` were an intermediate output, it would have gradient +storage and the same rule would accumulate `{3, 3}` into it; an earlier node +would then consume that gradient when reverse traversal reached it. + +```mermaid +sequenceDiagram + participant App + participant Tape + participant Node as Scale record + participant Grad as Gradient buffers + + App->>Tape: backward(y, seed, 2) + Tape->>Tape: validate y and y.producer_ + Tape->>Node: reset_gradient(node) + Node->>Grad: clear dy + Node->>Grad: clear da + Tape->>Grad: copy seed into dy + Tape->>Node: backward(node) + Node->>Grad: accumulate dot(dy, x) into da + Note over Node,Grad: dx is skipped because x has no gradient buffer + Tape-->>App: true +``` + +## Why records store pointers rather than views + +The operator record retains only the data needed later: raw non-owning +pointers, dimensions, small metadata, and the common node prefix. It does not +own or copy tensors and does not retain the temporary expression object. This +keeps record size independent of the numerical buffer length. Gradient storage +still costs one `float` per parameter or intermediate element. + +This is also why the caller must preserve values until the reverse pass. The +scale rule rereads `x_value` and `a_value`; changing either after the forward +pass would differentiate a different computation from the one whose `y_value` +was calculated. diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md new file mode 100644 index 000000000..54cc426fb --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -0,0 +1,196 @@ +# Operators + +Each operator header owns its validation, forward computation, fixed-size tape +record, gradient reset, backward rule, and expression adapter. Include the +header and register its operator class before evaluating the expression. + +In the formulas below, `g` is the gradient arriving from the operator's output. +Every input gradient is accumulated with `+=` when that input has gradient +storage. A view created by `tape.input()` has none, so its contribution is +skipped. + +## Element-wise arithmetic + +| Expression | Forward | Backward | +| --- | --- | --- | +| `z = x + y` | `z[i] = x[i] + y[i]` | `dx[i] += g[i]`; `dy[i] += g[i]` | +| `z = x - y` | `z[i] = x[i] - y[i]` | `dx[i] += g[i]`; `dy[i] -= g[i]` | +| `z = x * y` | `z[i] = x[i] * y[i]` | `dx[i] += g[i]*y[i]`; `dy[i] += g[i]*x[i]` | + +These expressions require equal-length views. The forward paths use +`arm_add_f32`, `arm_sub_f32`, and `arm_mult_f32`. Multiply's backward path uses +fused CMSIS-DSP C++ expressions so multiplication and accumulation need no +temporary product vector. + +## Dot, scalar scale, and scalar offset + +`dot(x, y)` returns one value: + +```text +z = sum(x[i] * y[i]) +dx[i] += g * y[i] +dy[i] += g * x[i] +``` + +Its forward pass uses `arm_dot_prod_f32`. + +`scale(x, a)` requires `a` to be a one-element parameter: + +```text +z[i] = a * x[i] +da += sum(g[i] * x[i]) +dx[i] += a * g[i] +``` + +The forward pass uses `arm_scale_f32`; the scalar gradient uses the C++ dot +expression. See the [worked implementation flow](implementation_flow.md) for a +line-by-line explanation. + +`offset(x, b)` likewise requires a one-element parameter: + +```text +z[i] = x[i] + b +db += sum(g[i]) +dx[i] += g[i] +``` + +Its forward pass uses `arm_offset_f32` and the bias gradient uses +`arm_accumulate_f32`. + +## ReLU and softmax + +ReLU computes `y[i] = max(0, x[i])` with `arm_clip_f32`. Its backward rule +passes `g[i]` only when the saved input value is strictly positive. The +derivative at zero is defined as zero. + +Softmax uses a log-sum-exp forward calculation for numerical stability: + +```text +y[i] = exp(x[i] - log(sum(exp(x)))) +projection = dot(g, y) +dx[i] += y[i] * (g[i] - projection) +``` + +The forward path uses `arm_logsumexp_f32`, `arm_offset_f32`, and `arm_vexp_f32`. + +## Losses + +Quadratic error returns a scalar sum, not a mean: + +```text +loss = sum((prediction[i] - target[i])^2) +d_prediction[i] += g * 2 * (prediction[i] - target[i]) +``` + +Categorical cross entropy also returns a scalar sum: + +```text +p_safe[i] = max(probability[i], 1e-7) +loss = -sum(target[i] * log(p_safe[i])) +d_probability[i] = -g * target[i] / p_safe[i] +``` + +For both losses, the target must be an input view. Cross entropy is intended +for a probability vector from softmax and a one-hot target. The probability +floor prevents non-finite loss and gradient values. Its forward pass reuses the +probability gradient buffer as scratch; `backward()` resets that buffer before +propagating gradients. + +## Fully connected + +`fully_connected(x, W, b)` computes `y = W*x + b`, where `W` is a row-major +matrix parameter and `b` is a parameter vector: + +```text +dW += outer(g, x) +db += g +dx += transpose(W) * g +``` + +The number of columns in `W` must equal the input length; its rows must equal +the bias and output lengths. Dimensions must fit the `uint16_t` CMSIS-DSP C +matrix descriptor. + +The forward matrix-vector product uses `arm_mat_vec_mult_f32`, followed by a +fused C++ bias accumulation. Backward bias and outer-product updates are fused +C++ expressions. If `x` needs a gradient, the current implementation evaluates +the lazy expression `dot(transpose_view(W), g)` and accumulates it into `dx`. +The transpose is a view: no transposed numerical matrix is allocated. + +## Matrix multiply + +`matrix_multiply(X, W)` computes `Y = W*X`. `W` is a row-major parameter matrix; +`X` is a row-major input matrix flattened into an input `BufferView`. If `W` +has shape `rows x inner`, the input length must be divisible by `inner`, and +the inferred input shape is `inner x columns`. The output length is +`rows*columns`. + +Only `W` is differentiated: + +```text +dW += dY * transpose(X) +``` + +The forward pass uses `arm_mat_mult_f32`. In the backward pass, each row of +`dW` is accumulated with the lazy C++ expression +`matvec(X, corresponding_row_of_dY)`. This fuses matrix-vector evaluation with +gradient accumulation; it does not call `arm_dot_prod_f32` once per weight and +does not materialize `transpose(X)`. + +## Dropout + +Dropout is a training regularizer. With drop probability `p`, each element is +kept independently with probability `1-p`. **Inverted dropout** scales kept +values during training: + +```text +mask[i] is 1 with probability (1-p), otherwise 0 +scale = 1 / (1-p) +y[i] = mask[i] * x[i] * scale +dx[i] += mask[i] * g[i] * scale +``` + +This is the standard inverted-dropout convention. Its expected training output +is `x`, so inference can be an identity operation without an extra scale: + +```text +E[y[i]] = (1-p) * x[i] / (1-p) = x[i] +``` + +The probability must satisfy `0 <= p < 1`. At `p == 0`, forward and backward +are identity operations. When tape recording is disabled, dropout copies its +input unchanged, which implements inference behavior. + +Random state is explicit and caller-owned: + +```cpp +DropoutGenerator generator(1234U); +tape.register_operator(); +hidden = dropout(hidden_linear, generator, 0.2F); +``` + +Initialize the generator once with a nonzero seed for a reproducible sequence. +A zero seed is replaced with the class's nonzero default because the xorshift +generator would otherwise remain zero. Each forward record saves the initial +random state, not an entire mask. Backward regenerates the identical mask from +that state. If record allocation fails, the generator is restored so a failed +operation does not consume random numbers. + +## Parameter initialization + +The AD layer does not initialize parameter values. The application must do so +before the first forward pass. Biases commonly start at zero. For neural-network +weight matrices, use a small random initialization appropriate to the +activation (for example Xavier/Glorot for many tanh or linear networks, or He +initialization for ReLU networks); identical zero weights can prevent hidden +units from learning distinct features. Optimizer moment/state arrays themselves +start at zero. + +## Common validation rules + +Views used by one expression must belong to the same tape and satisfy the +operator's role and shape requirements. Output and input value storage must be +distinct, and output gradient storage must not alias an input gradient. An +invalid combination sets the sticky `Status::tape_mismatch`. A successfully +computed value has a producer only when recording is enabled and its record was +appended successfully. diff --git a/dsppp/Include/dsppp/autodiff/doc/optimizers.md b/dsppp/Include/dsppp/autodiff/doc/optimizers.md new file mode 100644 index 000000000..4418a4a6c --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/optimizers.md @@ -0,0 +1,174 @@ +# Optimizers + +`Adam` and `RMSProp` update caller-owned parameter values from tape-managed or +caller-owned gradients. All optimizer metadata and numerical state are fixed +arrays inside the optimizer object; neither optimizer allocates memory. + +## Capacity arguments + +Both types have the same capacity template arguments: + +```cpp +Optimizer +``` + +- `MaximumElements` is the total number of scalar values across all added + parameter views. It has no default. +- `MaximumParameters` is the number of separately added views. It defaults to + 16. + +A single vector parameter of length 10 therefore needs 10 element slots but +only one parameter-view slot: + +```cpp +float value[10] = {}; +BufferView parameter = tape.parameter(value); +RMSProp<10, 1> optimizer; +optimizer.add(parameter); +``` + +A matrix also counts as one view, while all `rows*columns` entries count toward +`MaximumElements`. A three-element coefficient vector plus a separate scalar +bias fits exactly in either `RMSProp<4, 2>` or `Adam<4, 2>`. Writing +`Adam<100>` reserves 100 scalar state positions and the default 16 view slots. + +Adding the same value pointer twice is idempotent. A frozen parameter continues +to occupy both capacities. + +## RMSProp + +```cpp +RMSProp<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.99F, // alpha + 1.0e-8F // epsilon +); +``` + +For each trainable scalar, this implementation performs RMSProp without +momentum or centering: + +```text +square_average = alpha * square_average + + (1 - alpha) * gradient^2 +parameter -= learning_rate * gradient + / (sqrt(square_average) + epsilon) +``` + +`square_average` starts at zero. `alpha` controls how slowly squared-gradient +history changes; values near one produce longer memory. `epsilon` prevents a +zero or very small denominator. The defaults are conventional starting points, +but learning rate normally requires tuning for the model and loss scale. + +Storage consists principally of one +`float square_average_[MaximumElements]` plus +`Entry entries_[MaximumParameters]`. Each entry stores value and gradient +pointers, length, state offset, and whether the parameter is trainable. + +## Adam + +```cpp +Adam<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.9F, // beta1 + 0.999F, // beta2 + 1.0e-8F // epsilon +); +``` + +On successful step `t`: + +```text +first_moment = beta1 * first_moment + + (1 - beta1) * gradient +second_moment = beta2 * second_moment + + (1 - beta2) * gradient^2 + +corrected_first = first_moment / (1 - beta1^t) +corrected_second = second_moment / (1 - beta2^t) + +parameter -= learning_rate * corrected_first + / (sqrt(corrected_second) + epsilon) +``` + +Both moment arrays and their powers are initialized so the first successful +step applies the usual bias correction. `beta1` controls first-moment memory, +`beta2` controls squared-gradient memory, and `epsilon` stabilizes the +denominator. The defaults are standard initial choices. + +Adam stores +`first_moment_[MaximumElements]`, +`second_moment_[MaximumElements]`, and +`entries_[MaximumParameters]`, so its principal per-element state is twice +RMSProp's. Its global step advances each time `step()` succeeds. + +## Adding, clearing, and stepping + +Add each parameter once after creating its view: + +```cpp +float coefficient_value[3] = {}; +float bias_value = 0.0F; +BufferView coefficients = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); + +RMSProp<4, 2> optimizer; +optimizer.add(coefficients); +optimizer.add(bias); +``` + +Here `coefficients` is a three-element parameter `BufferView`, and `bias` is a +separate one-element parameter `BufferView`. + +The usual iteration order is: + +```cpp +evaluate_forward_graph(); +optimizer.zero_grad(); +if (!tape.backward(loss) || !optimizer.step()) + handle_error(); +``` + +`zero_grad()` clears all gradients for parameters known to the optimizer. +`backward()` then writes the gradients for the current graph, and `step()` +consumes them. See [Training loop](training_loop.md) for why the explicit clear +is useful even though backward resets gradients referenced by its records. + +## Freezing + +Freezing prevents updates without changing the recorded graph. In this +example, `coefficients` is a parameter `BufferView` created with +`tape.parameter()` and previously passed to `optimizer.add()`: + +```cpp +freeze_parameters(optimizer, coefficients); // Train other parameters. +unfreeze_parameters(optimizer, coefficients); // Resume updates later. +``` + +The same functions also accept parameter `MatrixView` objects. Multiple views +can be passed when a layer owns more than one parameter, such as weights and a +bias. + +The parameter still participates in forward and backward propagation, and its +gradient may be computed. `step()` skips its value and optimizer state, so both +remain unchanged while frozen. Operators are not frozen because they are +stateless; freeze every parameter view owned by the logical layer instead. + +## Status and errors + +Optimizer errors are sticky. After the first error, `good()` is false, +`status()` reports it, and `step()` returns false. + +- `too_many_parameters`: another distinct view would exceed + `MaximumParameters`. +- `too_many_elements`: its scalar length would exceed the remaining + `MaximumElements` capacity. +- `invalid_parameter`: `add()` received a non-parameter or a view without a + gradient, or `freeze()` received a view that had not been added. + +Parameter values and gradient arrays are referenced, not copied. They must +remain alive as long as the optimizer uses them. A checkpoint containing only +parameter values is sufficient for inference. Reproducing the exact continuation +of training also requires saving the optimizer's moment state and, for Adam, +its step-dependent powers; the current optimizer classes do not provide a +serialization API. diff --git a/dsppp/Include/dsppp/autodiff/doc/training_loop.md b/dsppp/Include/dsppp/autodiff/doc/training_loop.md new file mode 100644 index 000000000..0fe59b942 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/training_loop.md @@ -0,0 +1,176 @@ +# Training loop + +The application owns the model, data, outputs, and loop. The tape records one +evaluation of the model, computes gradients, and then an optimizer changes the +parameter values. + +## Complete linear-regression step + +This example fits `prediction = coefficient * feature + bias` to one sample. +Every variable used by the loop is declared here. + +```cpp +#include +#include +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ +Arena<1024> arena; +Tape &tape = arena.tape(); +tape.register_operator(); +tape.register_operator(); +tape.register_operator(); + +float coefficient_value[] = {0.0F}; +float bias_value[] = {0.0F}; +float feature_value[] = {2.0F}; +float target_value[] = {5.0F}; +float product_value[] = {0.0F}; +float prediction_value[] = {0.0F}; +float loss_value[] = {0.0F}; + +BufferView coefficient = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); +BufferView feature = tape.input(feature_value); +BufferView target = tape.input(target_value); +BufferView product = tape.output(product_value); +BufferView prediction = tape.output(prediction_value); +BufferView loss = tape.output(loss_value); + +RMSProp<2, 2> optimizer(1.0e-3F); +optimizer.add(coefficient); +optimizer.add(bias); + +tape.begin_graph(); +for (std::size_t step = 0; step < 100U; ++step) +{ + if (!tape.rewind_graph()) + break; + + product = coefficient * feature; + prediction = product + bias; + loss = quadratic_error(prediction, target); + + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) + break; +} +return tape.good() && optimizer.good() ? 0 : 1; +} +``` + +## The flow, in order + +### 1. Set up fixed state + +Register operator types once, create all persistent views, construct the +optimizer, and add each trainable parameter view. `RMSProp<2, 2>` reserves +state for two scalar values in at most two separately added views: the +one-element `coefficient` view and the one-element `bias` view. + +### 2. Mark persistent arena allocations + +`begin_graph()` records the arena position after the persistent gradient +buffers. It also starts a fresh record chain. Call it only after creating the +views that must survive between iterations. + +### 3. Rewind before an iteration + +`rewind_graph()` discards the previous iteration's operation records by moving +the arena position back to the mark. Parameter/output views and their gradient +arrays remain valid. It resets tape status, enables recording, and clears the +record-chain tail. + +### 4. Run and record the forward graph + +Each assignment computes caller-owned output values immediately and appends a +small record containing the pointers needed by its derivative rule. The graph +must be reevaluated every iteration because parameter values change after +`step()`. + +Here the forward values are: + +```text +product[0] = coefficient[0] * feature[0] +prediction[0] = product[0] + bias[0] +loss[0] = (prediction[0] - target[0])^2 +``` + +### 5. Clear optimizer-managed parameter gradients + +`optimizer.zero_grad()` clears every parameter gradient registered with the +optimizer. `Tape::backward()` also resets gradients referenced by the recorded +graph before installing its seed, so for a fixed, successful graph this call +is usually redundant. It is nevertheless useful explicit training-loop +hygiene: it also clears registered parameters omitted by a conditional graph, +and it prevents stale gradients from reaching `step()` if graph structure is +changed. Call it before `backward()`, not after, because after backward those +buffers contain the gradients that `step()` must consume. + +### 6. Run reverse propagation + +`backward(loss)` uses the default scalar seed `d(loss)/d(loss) = 1`. It first +resets graph gradient buffers, seeds the loss gradient, and visits operation +records in reverse creation order. Contributions use `+=`, so a parameter used +by several nodes receives their sum. + +For this graph: + +```text +d_loss/d_prediction = 2 * (prediction - target) +d_loss/d_coefficient = d_loss/d_prediction * feature +d_loss/d_bias = d_loss/d_prediction +``` + +Inputs have no gradient buffers, so derivatives for `feature` and `target` are +not retained. + +### 7. Update parameters + +`optimizer.step()` reads the gradients, updates its fixed-size state, and then +changes `coefficient_value` and `bias_value` in place. The next iteration's +forward pass therefore uses the new parameter values. + +```mermaid +flowchart TD + A["Register operators and create persistent views"] --> B["Add parameter views to optimizer"] + B --> C["begin_graph(): mark persistent arena state"] + C --> D["rewind_graph(): discard old records"] + D --> E["Forward assignments: compute values and append records"] + E --> F["optimizer.zero_grad(): clear registered parameter gradients"] + F --> G["tape.backward(): reset, seed, reverse traversal"] + G --> H["optimizer.step(): update state and parameter values"] + H --> D +``` + +## Batch accumulation + +To update once per batch, record every sample's prediction before constructing +one loss over the complete prediction and target buffers. Scalar per-sample +output views can share slices of persistent vector gradient buffers. The +regression example in `dsppp/Examples/autodiff_regression.cpp` demonstrates +this pattern with named `coefficients`, `bias`, `polynomial`, `prediction`, +`target`, and `loss` views. + +## Inference + +Inference still runs the operator forward kernels but does not need records: + +```cpp +{ + RecordingScope inference(tape, false); + product = coefficient * feature; + prediction = product + bias; + const float result = prediction_value[0]; + (void)result; +} +``` + +The previous recording state is restored at the closing brace. See +[Concepts and memory model](concepts.md#recording-and-recordingscope) for the +name and lifetime behavior. From f88cddb4ccf0ad4239b612d178e48876bf5ee25e Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Wed, 12 Aug 2026 09:56:30 +0200 Subject: [PATCH 11/19] Add more error handling to autodiff_iris.cpp --- dsppp/Examples/autodiff_iris.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dsppp/Examples/autodiff_iris.cpp b/dsppp/Examples/autodiff_iris.cpp index fad86e5e3..62d8d0a15 100644 --- a/dsppp/Examples/autodiff_iris.cpp +++ b/dsppp/Examples/autodiff_iris.cpp @@ -119,12 +119,21 @@ int main() BufferView target = tape.input(state->target); BufferView loss = tape.output(state->loss); + + if (!tape.good()) { + if (tape.status() == Status::out_of_memory) + std::printf("Autodiff arena is too small\n"); + return 1; + } + + if (!state->optimizer.add(hidden_weight) || !state->optimizer.add(hidden_bias) || !state->optimizer.add(output_weight) || !state->optimizer.add(output_bias)) { delete state; + std::printf("Failed to add parameters to optimizer\n"); return 1; } From f907d42611357f243ff85fe081a03bba4a5a68b3 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Wed, 12 Aug 2026 15:10:15 +0200 Subject: [PATCH 12/19] Added f16 implementation of autodiff and iris example now uses f16 --- dsppp/Examples/autodiff_example.cpp | 13 +- dsppp/Examples/autodiff_iris.cpp | 175 ++++++++++---- dsppp/Examples/autodiff_regression.cpp | 30 ++- dsppp/Examples/iris_data.hpp | 15 ++ dsppp/Include/dsppp/autodiff/README.md | 40 ++-- dsppp/Include/dsppp/autodiff/doc/concepts.md | 12 +- .../dsppp/autodiff/doc/implementation_flow.md | 22 +- dsppp/Include/dsppp/autodiff/doc/operators.md | 55 +++-- .../Include/dsppp/autodiff/doc/optimizers.md | 29 ++- .../dsppp/autodiff/doc/training_loop.md | 24 +- .../Include/dsppp/autodiff/operators/add.hpp | 117 ++++++---- .../autodiff/operators/cross_entropy.hpp | 204 +++++++++++----- .../Include/dsppp/autodiff/operators/dot.hpp | 133 +++++++---- .../dsppp/autodiff/operators/dropout.hpp | 144 ++++++++---- .../autodiff/operators/fully_connected.hpp | 181 +++++++++------ .../autodiff/operators/matrix_multiply.hpp | 141 ++++++----- .../dsppp/autodiff/operators/multiply.hpp | 138 ++++++----- .../dsppp/autodiff/operators/offset.hpp | 166 ++++++++----- .../autodiff/operators/quadratic_error.hpp | 119 +++++----- .../Include/dsppp/autodiff/operators/relu.hpp | 127 +++++++--- .../dsppp/autodiff/operators/scale.hpp | 147 +++++++----- .../dsppp/autodiff/operators/softmax.hpp | 154 +++++++++---- .../Include/dsppp/autodiff/operators/sub.hpp | 137 +++++++---- .../dsppp/autodiff/optimizers/adam.hpp | 127 +++++++--- .../dsppp/autodiff/optimizers/rmsprop.hpp | 95 ++++++-- dsppp/Include/dsppp/autodiff/reverse.hpp | 218 +++++++++--------- dsppp/test.cbuild-idx.yml | 2 +- dsppp/tests/autodiff_test.cpp | 143 +++++++++--- 28 files changed, 1888 insertions(+), 1020 deletions(-) diff --git a/dsppp/Examples/autodiff_example.cpp b/dsppp/Examples/autodiff_example.cpp index dd7ca32e8..d841b5130 100644 --- a/dsppp/Examples/autodiff_example.cpp +++ b/dsppp/Examples/autodiff_example.cpp @@ -9,9 +9,9 @@ using namespace arm_cmsis_dsp::autodiff; int main() { Arena<2048> arena; - Tape &tape = arena.tape(); - tape.register_operator(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); + tape.register_operator>(); float x_value[] = {2.0F, -1.0F}; float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; @@ -25,6 +25,13 @@ int main() BufferView linear = tape.output(linear_value); BufferView activation = tape.output(activation_value); + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + return 1; + } + linear = fully_connected(x, matrix, bias); activation = relu(linear); diff --git a/dsppp/Examples/autodiff_iris.cpp b/dsppp/Examples/autodiff_iris.cpp index 62d8d0a15..6fc22f79b 100644 --- a/dsppp/Examples/autodiff_iris.cpp +++ b/dsppp/Examples/autodiff_iris.cpp @@ -8,6 +8,7 @@ #include "iris_data.hpp" #include +#include #include #include @@ -15,8 +16,21 @@ using namespace arm_cmsis_dsp::autodiff; +/* Set to 0 to build the same example with float32 autodiff. It may also be + * overridden from the compiler command line with -D. */ +#ifndef DSPPP_AUTODIFF_IRIS_USE_FLOAT16 +#define DSPPP_AUTODIFF_IRIS_USE_FLOAT16 1 +#endif + +#if !DSPPP_AUTODIFF_IRIS_USE_FLOAT16 || defined(ARM_FLOAT16_SUPPORTED) namespace { +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 +using IrisType = float16_t; +#else +using IrisType = float; +#endif + constexpr std::size_t input_size = 4U; constexpr std::size_t hidden_size = 8U; constexpr std::size_t class_count = 3U; @@ -25,24 +39,31 @@ constexpr std::size_t epoch_count = 120U; struct Model { - float hidden_weight[hidden_size][input_size]; - float hidden_bias[hidden_size]; - float output_weight[class_count][hidden_size]; - float output_bias[class_count]; + IrisType hidden_weight[hidden_size][input_size]; + IrisType hidden_bias[hidden_size]; + IrisType output_weight[class_count][hidden_size]; + IrisType output_bias[class_count]; }; struct TrainingState { Model model{}; - Arena<2048> arena{}; - Adam<67U, 4U> optimizer{1.0e-2F}; - float input[input_size]{}; - float hidden_linear[hidden_size]{}; - float hidden[hidden_size]{}; - float logits[class_count]{}; - float probability[class_count]{}; - float target[class_count]{}; - float loss{}; + Arena<2048, IrisType> arena{}; + Adam<67U, 4U, IrisType> optimizer{static_cast(1.0e-2F)}; + IrisType input[input_size]{}; + IrisType hidden_linear[hidden_size]{}; + IrisType hidden[hidden_size]{}; + IrisType logits[class_count]{}; + IrisType probability[class_count]{}; + IrisType target[class_count]{}; + IrisType loss{}; + // Keep normalized float32 patterns for the original example path. The + // additional float16 array avoids converting samples inside the loop when + // the half-precision path is selected. + float f32_patterns[iris_data::sample_count][input_size]{}; +#if defined(ARM_FLOAT16_SUPPORTED) + float16_t f16_patterns[iris_data::sample_count][input_size]{}; +#endif std::uint8_t training_index[training_count]{}; }; @@ -54,11 +75,11 @@ static std::uint32_t random_u32() noexcept return random_state; } -static float random_weight() noexcept +static IrisType random_weight() noexcept { const float unit = static_cast((random_u32() >> 8) & 0xffffU) / 65535.0F; - return (unit - 0.5F) * 0.5F; + return static_cast((unit - 0.5F) * 0.5F); } static void initialize(Model &model) noexcept @@ -84,14 +105,25 @@ static bool is_test_sample(std::size_t index) noexcept } static std::uint32_t predicted_class( - const float (&probability)[class_count]) noexcept + const IrisType (&probability)[class_count]) noexcept { - float maximum; + IrisType maximum; std::uint32_t index; +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + arm_max_f16(probability, class_count, &maximum, &index); +#else arm_max_f32(probability, class_count, &maximum, &index); +#endif return index; } +static const char *label_name(unsigned label) noexcept +{ + static constexpr const char *names[class_count] = { + "Iris-setosa", "Iris-versicolor", "Iris-virginica"}; + return label < class_count ? names[label] : "unknown"; +} + } // namespace int main() @@ -100,32 +132,44 @@ int main() // Keep the training buffers and optimizer state off the limited stack. TrainingState *state = new TrainingState; initialize(state->model); - - Tape &tape = state->arena.tape(); - tape.register_operator(); - tape.register_operator(); - tape.register_operator(); - tape.register_operator(); - - BufferView input = tape.input(state->input); - MatrixView hidden_weight = tape.parameter(state->model.hidden_weight); - BufferView hidden_bias = tape.parameter(state->model.hidden_bias); - MatrixView output_weight = tape.parameter(state->model.output_weight); - BufferView output_bias = tape.parameter(state->model.output_bias); - BufferView hidden_linear = tape.output(state->hidden_linear); - BufferView hidden = tape.output(state->hidden); - BufferView logits = tape.output(state->logits); - BufferView probability = tape.output(state->probability); - BufferView target = tape.input(state->target); - BufferView loss = tape.output(state->loss); - - - if (!tape.good()) { - if (tape.status() == Status::out_of_memory) - std::printf("Autodiff arena is too small\n"); - return 1; + for (std::size_t sample = 0; sample < iris_data::sample_count; ++sample) + { + iris_data::normalized_features(sample, state->f32_patterns[sample]); +#if defined(ARM_FLOAT16_SUPPORTED) + for (std::size_t feature = 0; feature < input_size; ++feature) + state->f16_patterns[sample][feature] = static_cast( + state->f32_patterns[sample][feature]); +#endif } + Tape &tape = state->arena.tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + + BufferView input = tape.input(state->input); + MatrixView hidden_weight = tape.parameter(state->model.hidden_weight); + BufferView hidden_bias = tape.parameter(state->model.hidden_bias); + MatrixView output_weight = tape.parameter(state->model.output_weight); + BufferView output_bias = tape.parameter(state->model.output_bias); + BufferView hidden_linear = tape.output(state->hidden_linear); + BufferView hidden = tape.output(state->hidden); + BufferView logits = tape.output(state->logits); + BufferView probability = tape.output(state->probability); + BufferView target = tape.input(state->target); + BufferView loss = tape.output(state->loss); + + if (!tape.good()) + { + if (tape.status() == Status::out_of_memory) + std::printf("Autodiff arena is too small\n"); + else + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + delete state; + return 1; + } if (!state->optimizer.add(hidden_weight) || !state->optimizer.add(hidden_bias) || @@ -160,10 +204,17 @@ int main() for (std::size_t position = 0; position < training_count; ++position) { const std::size_t sample = state->training_index[position]; - iris_data::normalized_features(sample, state->input); +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f16_patterns[sample][feature]; +#else + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f32_patterns[sample][feature]; +#endif for (std::size_t i = 0; i < class_count; ++i) state->target[i] = i == iris_data::samples[sample].label - ? 1.0F : 0.0F; + ? static_cast(1.0F) + : static_cast(0.0F); if (!tape.rewind_graph()) { @@ -182,7 +233,7 @@ int main() delete state; return 1; } - epoch_loss += state->loss; + epoch_loss += static_cast(state->loss); } if ((epoch + 1U) % 20U == 0U) @@ -193,24 +244,50 @@ int main() // Final check on the 30 samples that were kept out of training. unsigned correct = 0U; + unsigned test_number = 0U; { - RecordingScope inference(tape, false); + RecordingScope inference(tape, false); for (std::size_t sample = 0; sample < iris_data::sample_count; ++sample) { if (!is_test_sample(sample)) continue; - iris_data::normalized_features(sample, state->input); +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f16_patterns[sample][feature]; +#else + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f32_patterns[sample][feature]; +#endif hidden_linear = fully_connected(input, hidden_weight, hidden_bias); hidden = relu(hidden_linear); logits = fully_connected(hidden, output_weight, output_bias); probability = softmax(logits); - if (predicted_class(state->probability) == - iris_data::samples[sample].label) + const unsigned expected = iris_data::samples[sample].label; + const unsigned detected = predicted_class(state->probability); + const bool match = detected == expected; + if (!match) + std::printf("\033[31m"); + std::printf("Test %u:\n Expected \"%s\"\n Detected \"%s\"\n", + ++test_number, label_name(expected), + label_name(detected)); + if (!match) + std::printf("\033[0m"); + if (match) ++correct; } } - std::printf("final test accuracy=%u/30\n", correct); + std::printf("final test accuracy=%u/30 tests\n", correct); delete state; return 0; } + +#else + +int main() +{ + std::printf("Iris float16 example requires ARM_FLOAT16_SUPPORTED\n"); + return 0; +} + +#endif diff --git a/dsppp/Examples/autodiff_regression.cpp b/dsppp/Examples/autodiff_regression.cpp index 7bbd2b857..4824e7d0e 100644 --- a/dsppp/Examples/autodiff_regression.cpp +++ b/dsppp/Examples/autodiff_regression.cpp @@ -54,10 +54,10 @@ int main() // global loss record. Values stay in caller storage; this arena contains // gradients and operation records only. Arena<32768> *arena = new Arena<32768>(); - Tape &tape = arena->tape(); - tape.register_operator(); - tape.register_operator(); - tape.register_operator(); + Tape &tape = arena->tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); float feature_value[sample_count][3] = {}; float polynomial_value[sample_count] = {}; @@ -85,11 +85,23 @@ int main() BufferView target = tape.input(target_value); BufferView loss = tape.output(loss_value); + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + delete arena; + return 1; + } + // Match the optimizer chosen by the PyTorch example. Adam can be used // here instead by including adam.hpp and changing only this type. RMSProp<4, 2> optimizer(1.0e-3F); - optimizer.add(coefficients); - optimizer.add(bias); + if (!optimizer.add(coefficients) || !optimizer.add(bias)) + { + std::printf("Failed to add parameters to optimizer\n"); + delete arena; + return 1; + } /* Set this to true for bias-only fine tuning. The same mechanism freezes * all parameters belonging to any selected layer/operator. @@ -104,7 +116,11 @@ int main() tape.begin_graph(); for (std::size_t step = 0; step < training_steps; ++step) { - tape.rewind_graph(); + if (!tape.rewind_graph()) + { + delete arena; + return 1; + } // Build all 100 predictions before constructing the loss. The scalar // views below share slices of the two arena-managed vector gradients; diff --git a/dsppp/Examples/iris_data.hpp b/dsppp/Examples/iris_data.hpp index 9398ba3d1..a37f0f38d 100644 --- a/dsppp/Examples/iris_data.hpp +++ b/dsppp/Examples/iris_data.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace iris_data { @@ -178,4 +179,18 @@ inline void normalized_features(std::size_t index, float (&output)[4]) noexcept inverse_standard_deviation[feature]; } +// The source pattern remains float32 so it can be shared by the original +// example. This overload provides a separate float16 pattern buffer for the +// half-precision autodiff example. +#if defined(ARM_FLOAT16_SUPPORTED) +inline void normalized_features(std::size_t index, + float16_t (&output)[4]) noexcept +{ + for (std::size_t feature = 0; feature < 4U; ++feature) + output[feature] = static_cast( + (samples[index].feature[feature] - mean[feature]) * + inverse_standard_deviation[feature]); +} +#endif + } // namespace iris_data diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 587ec39c2..02829fc7e 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -22,11 +22,11 @@ accumulates gradients. This implementation uses CMSIS-DSP for both rather than treating it only as a collection of forward-inference kernels. When an operation maps directly to an optimized CMSIS-DSP C kernel, its -forward pass uses that kernel. For example, dot products use -`arm_dot_prod_f32`, fully connected matrix-vector products use -`arm_mat_vec_mult_f32`, and matrix products use `arm_mat_mult_f32`. These -kernels provide implementations optimized for the selected Arm target, -including Helium implementations where available. +forward pass uses the kernel matching `T`. For example, float32 dot products +use `arm_dot_prod_f32` and float16 dot products use `arm_dot_prod_f16`; the +corresponding fully connected and matrix products use the f32 or f16 matrix +kernels. These kernels provide implementations optimized for the selected Arm +target, including Helium implementations where available. The CMSIS-DSP C++ expression system is particularly useful in backward rules, which often combine element-wise computation with accumulation. A typical @@ -75,11 +75,17 @@ contribution into the gradients of earlier intermediate results and learnable parameters. This reverse application of the chain rule produces all parameter gradients needed by an optimizer. -Numerical values remain in buffers owned by the application. An `Arena` -provides a fixed amount of memory for gradients and tape records, so memory use -cannot grow unexpectedly at runtime. Ordinary inputs registered with +Numerical values remain in buffers owned by the application. `BufferView` and +`Tape` support `float` and `float16_t`; the default `T` is `float`. An +`Arena` provides a fixed amount of memory for gradients and tape +records, so memory use cannot grow unexpectedly at runtime. Ordinary inputs +registered with `tape.input()` do not receive gradient storage. +On a target with CMSIS-DSP float16 support, select the half-precision path by +using `Arena`, `Tape`, and matching operator and +optimizer specializations. + The implementation is modular: an application includes and registers only the operator headers it uses. The core in `reverse.hpp` manages views, fixed arena storage, operator registration, and reverse traversal; each operator header @@ -95,17 +101,17 @@ using namespace arm_cmsis_dsp::autodiff; int main() { - Arena<512> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Arena<512, float> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); float x_value[] = {1.0F, 2.0F}; float a_value = 3.0F; float y_value[2] = {}; - BufferView x = tape.input(x_value); - BufferView a = tape.parameter(a_value); - BufferView y = tape.output(y_value); + BufferView x = tape.input(x_value); + BufferView a = tape.parameter(a_value); + BufferView y = tape.output(y_value); // y = a * x y = scale(x, a); // y = {3, 6}; records the operation @@ -143,5 +149,7 @@ Autodiff uses the existing dsppp board-test infrastructure. From `dsppp`, run: python run_all.py --test AUTODIFF_TEST --dt F32_DT ``` -It currently supports `float` and dynamic test mode. The test body is selected -when `AUTODIFF_TEST`, `F32_DT`, and `DYNAMIC_TEST` are defined. +The API supports `float` and, on targets defining `ARM_FLOAT16_SUPPORTED`, +`float16_t`. The board test currently selects the float32 path when +`AUTODIFF_TEST`, `F32_DT`, and `DYNAMIC_TEST` are defined; the Iris example +instantiates the float16 path. diff --git a/dsppp/Include/dsppp/autodiff/doc/concepts.md b/dsppp/Include/dsppp/autodiff/doc/concepts.md index 457d27626..cac8147af 100644 --- a/dsppp/Include/dsppp/autodiff/doc/concepts.md +++ b/dsppp/Include/dsppp/autodiff/doc/concepts.md @@ -7,12 +7,12 @@ them. It then visits those records in reverse order to propagate gradients. This ordered record is conventionally called a **tape**: operations are recorded during the forward pass and played backward during differentiation. -`Tape` manages this record and the gradient buffers. `Arena` supplies +`Tape` manages this record and the gradient buffers. `Arena` supplies exactly `Bytes` bytes of storage to a `Tape`: ```cpp -Arena<2048> arena; -Tape &tape = arena.tape(); +Arena<2048, float> arena; +Tape &tape = arena.tape(); ``` The capacity is fixed and no dynamic memory allocation is performed. If the @@ -47,8 +47,8 @@ True C arrays preserve their extent, so `tape.input(array)` can deduce the length. A pointer does not carry a length and requires an explicit overload: ```cpp -BufferView input = tape.input(pointer, number_of_elements); -MatrixView weights = tape.parameter(pointer, rows, columns); +BufferView input = tape.input(pointer, number_of_elements); +MatrixView weights = tape.parameter(pointer, rows, columns); ``` ## Operator registration @@ -61,7 +61,7 @@ compiler and linker. ```cpp #include -tape.register_operator(); +tape.register_operator>(); ``` Using an operator that has not been registered sets diff --git a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md index b6fbb3ee9..6c553c4dc 100644 --- a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md +++ b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md @@ -21,17 +21,17 @@ using namespace arm_cmsis_dsp::autodiff; int main() { - Arena<512> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Arena<512, float> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); float a_value = 3.0F; float x_value[] = {2.0F, -1.0F}; float y_value[2] = {}; - BufferView a = tape.parameter(a_value); - BufferView x = tape.input(x_value); - BufferView y = tape.output(y_value); + BufferView a = tape.parameter(a_value); + BufferView x = tape.input(x_value); + BufferView y = tape.output(y_value); y = scale(x, a); // Forward result: {6, -3}. @@ -57,13 +57,14 @@ The seed specifies how those output contributions are combined. With `seed = {1, 1}`, the backward pass computes `a.gradient(0) = 1*2 + 1*(-1) = 1`. A seed of `{1, 0}` would select only `y[0]` and produce `a.gradient(0) = 2`. One call does not build the complete -derivative vector for `y`; it propagates the combination selected by the seed. -Training normally ends in a scalar loss, for which +derivative vector for `y`; it propagates the selected scalar projection of the +vector output. No loss function is present in this example. Training normally +ends in a scalar loss, for which `backward(scalar_output)` uses the default seed `1`. ## Objects before the forward pass -`Arena<512>` contains the storage array and constructs a `Tape` over it. The +`Arena<512, float>` contains the storage array and constructs a `Tape` over it. The tape initially has `used_ == 0`, `tail_ == nullptr`, `recording_ == true`, and `status_ == Status::ok`. @@ -273,7 +274,8 @@ The operator record retains only the data needed later: raw non-owning pointers, dimensions, small metadata, and the common node prefix. It does not own or copy tensors and does not retain the temporary expression object. This keeps record size independent of the numerical buffer length. Gradient storage -still costs one `float` per parameter or intermediate element. +still costs one scalar of the selected type (`T`) per parameter or intermediate +element. This is also why the caller must preserve values until the reverse pass. The scale rule rereads `x_value` and `a_value`; changing either after the forward diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md index 54cc426fb..c64a8eba6 100644 --- a/dsppp/Include/dsppp/autodiff/doc/operators.md +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -1,8 +1,10 @@ # Operators Each operator header owns its validation, forward computation, fixed-size tape -record, gradient reset, backward rule, and expression adapter. Include the -header and register its operator class before evaluating the expression. +record, gradient reset, backward rule, and expression adapter. Operator classes +are templated on the scalar type (`float` by default and `float16_t` when +enabled). Include the header and register the matching specialization before +evaluating the expression. In the formulas below, `g` is the gradient arriving from the operator's output. Every input gradient is accumulated with `+=` when that input has gradient @@ -17,10 +19,10 @@ skipped. | `z = x - y` | `z[i] = x[i] - y[i]` | `dx[i] += g[i]`; `dy[i] -= g[i]` | | `z = x * y` | `z[i] = x[i] * y[i]` | `dx[i] += g[i]*y[i]`; `dy[i] += g[i]*x[i]` | -These expressions require equal-length views. The forward paths use -`arm_add_f32`, `arm_sub_f32`, and `arm_mult_f32`. Multiply's backward path uses -fused CMSIS-DSP C++ expressions so multiplication and accumulation need no -temporary product vector. +These expressions require equal-length views. The forward paths dispatch to +the matching f32 or f16 CMSIS-DSP kernels. Multiply's backward path uses fused +CMSIS-DSP C++ expressions so multiplication and accumulation need no temporary +product vector. ## Dot, scalar scale, and scalar offset @@ -32,7 +34,7 @@ dx[i] += g * y[i] dy[i] += g * x[i] ``` -Its forward pass uses `arm_dot_prod_f32`. +Its forward pass dispatches to `arm_dot_prod_f32` or `arm_dot_prod_f16`. `scale(x, a)` requires `a` to be a one-element parameter: @@ -42,9 +44,9 @@ da += sum(g[i] * x[i]) dx[i] += a * g[i] ``` -The forward pass uses `arm_scale_f32`; the scalar gradient uses the C++ dot -expression. See the [worked implementation flow](implementation_flow.md) for a -line-by-line explanation. +The forward pass dispatches to `arm_scale_f32` or `arm_scale_f16`; the scalar +gradient uses the C++ dot expression. See the [worked implementation +flow](implementation_flow.md) for a line-by-line explanation. `offset(x, b)` likewise requires a one-element parameter: @@ -54,14 +56,14 @@ db += sum(g[i]) dx[i] += g[i] ``` -Its forward pass uses `arm_offset_f32` and the bias gradient uses -`arm_accumulate_f32`. +Its forward pass dispatches to `arm_offset_f32` or `arm_offset_f16`; the bias +gradient uses the matching accumulate kernel. ## ReLU and softmax -ReLU computes `y[i] = max(0, x[i])` with `arm_clip_f32`. Its backward rule -passes `g[i]` only when the saved input value is strictly positive. The -derivative at zero is defined as zero. +ReLU computes `y[i] = max(0, x[i])` with the matching f32 or f16 clip kernel. +Its backward rule passes `g[i]` only when the saved input value is strictly +positive. The derivative at zero is defined as zero. Softmax uses a log-sum-exp forward calculation for numerical stability: @@ -71,7 +73,8 @@ projection = dot(g, y) dx[i] += y[i] * (g[i] - projection) ``` -The forward path uses `arm_logsumexp_f32`, `arm_offset_f32`, and `arm_vexp_f32`. +The forward path dispatches to the matching f32 or f16 log-sum-exp, offset, and +vector-exponential kernels. ## Losses @@ -82,10 +85,12 @@ loss = sum((prediction[i] - target[i])^2) d_prediction[i] += g * 2 * (prediction[i] - target[i]) ``` -Categorical cross entropy also returns a scalar sum: +Categorical cross entropy also returns a scalar sum. The probability floor is +`1e-7` for float32 and `1e-4` for float16; the larger half-precision floor keeps +the reciprocal used by the derivative finite: ```text -p_safe[i] = max(probability[i], 1e-7) +p_safe[i] = max(probability[i], floor) loss = -sum(target[i] * log(p_safe[i])) d_probability[i] = -g * target[i] / p_safe[i] ``` @@ -111,9 +116,9 @@ The number of columns in `W` must equal the input length; its rows must equal the bias and output lengths. Dimensions must fit the `uint16_t` CMSIS-DSP C matrix descriptor. -The forward matrix-vector product uses `arm_mat_vec_mult_f32`, followed by a -fused C++ bias accumulation. Backward bias and outer-product updates are fused -C++ expressions. If `x` needs a gradient, the current implementation evaluates +The forward matrix-vector product dispatches to `arm_mat_vec_mult_f32` or +`arm_mat_vec_mult_f16`, followed by a fused C++ bias accumulation. Backward bias +and outer-product updates are fused C++ expressions. If `x` needs a gradient, the current implementation evaluates the lazy expression `dot(transpose_view(W), g)` and accumulates it into `dx`. The transpose is a view: no transposed numerical matrix is allocated. @@ -131,10 +136,10 @@ Only `W` is differentiated: dW += dY * transpose(X) ``` -The forward pass uses `arm_mat_mult_f32`. In the backward pass, each row of -`dW` is accumulated with the lazy C++ expression +The forward pass dispatches to `arm_mat_mult_f32` or `arm_mat_mult_f16`. In the +backward pass, each row of `dW` is accumulated with the lazy C++ expression `matvec(X, corresponding_row_of_dY)`. This fuses matrix-vector evaluation with -gradient accumulation; it does not call `arm_dot_prod_f32` once per weight and +gradient accumulation; it does not call a scalar dot kernel once per weight and does not materialize `transpose(X)`. ## Dropout @@ -165,7 +170,7 @@ Random state is explicit and caller-owned: ```cpp DropoutGenerator generator(1234U); -tape.register_operator(); +tape.register_operator>(); hidden = dropout(hidden_linear, generator, 0.2F); ``` diff --git a/dsppp/Include/dsppp/autodiff/doc/optimizers.md b/dsppp/Include/dsppp/autodiff/doc/optimizers.md index 4418a4a6c..c3e62423e 100644 --- a/dsppp/Include/dsppp/autodiff/doc/optimizers.md +++ b/dsppp/Include/dsppp/autodiff/doc/optimizers.md @@ -1,8 +1,10 @@ # Optimizers `Adam` and `RMSProp` update caller-owned parameter values from tape-managed or -caller-owned gradients. All optimizer metadata and numerical state are fixed -arrays inside the optimizer object; neither optimizer allocates memory. +caller-owned gradients. Their third template argument selects the scalar type +(`float` by default, or `float16_t`), matching the tape views. All optimizer +metadata and numerical state are fixed arrays inside the optimizer object; +neither optimizer allocates memory. ## Capacity arguments @@ -22,7 +24,7 @@ only one parameter-view slot: ```cpp float value[10] = {}; -BufferView parameter = tape.parameter(value); +BufferView parameter = tape.parameter(value); RMSProp<10, 1> optimizer; optimizer.add(parameter); ``` @@ -31,6 +33,7 @@ A matrix also counts as one view, while all `rows*columns` entries count toward `MaximumElements`. A three-element coefficient vector plus a separate scalar bias fits exactly in either `RMSProp<4, 2>` or `Adam<4, 2>`. Writing `Adam<100>` reserves 100 scalar state positions and the default 16 view slots. +For half precision, use for example `Adam<100, 16, float16_t>`. Adding the same value pointer twice is idempotent. A frozen parameter continues to occupy both capacities. @@ -57,11 +60,13 @@ parameter -= learning_rate * gradient `square_average` starts at zero. `alpha` controls how slowly squared-gradient history changes; values near one produce longer memory. `epsilon` prevents a -zero or very small denominator. The defaults are conventional starting points, -but learning rate normally requires tuning for the model and loss scale. +zero or very small denominator. The default is `1e-8` for float32 and `1e-4` +for float16, where `1e-8` would round to zero. These are conventional starting +points, but learning rate normally requires tuning for the model and loss +scale. Storage consists principally of one -`float square_average_[MaximumElements]` plus +`T square_average_[MaximumElements]` plus `Entry entries_[MaximumParameters]`. Each entry stores value and gradient pointers, length, state offset, and whether the parameter is trainable. @@ -94,11 +99,13 @@ parameter -= learning_rate * corrected_first Both moment arrays and their powers are initialized so the first successful step applies the usual bias correction. `beta1` controls first-moment memory, `beta2` controls squared-gradient memory, and `epsilon` stabilizes the -denominator. The defaults are standard initial choices. +denominator. The default epsilon is `1e-8` for float32 and `1e-4` for +float16, where `1e-8` would round to zero. The other defaults are standard +initial choices. Adam stores -`first_moment_[MaximumElements]`, -`second_moment_[MaximumElements]`, and +`T first_moment_[MaximumElements]`, +`T second_moment_[MaximumElements]`, and `entries_[MaximumParameters]`, so its principal per-element state is twice RMSProp's. Its global step advances each time `step()` succeeds. @@ -109,8 +116,8 @@ Add each parameter once after creating its view: ```cpp float coefficient_value[3] = {}; float bias_value = 0.0F; -BufferView coefficients = tape.parameter(coefficient_value); -BufferView bias = tape.parameter(bias_value); +BufferView coefficients = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); RMSProp<4, 2> optimizer; optimizer.add(coefficients); diff --git a/dsppp/Include/dsppp/autodiff/doc/training_loop.md b/dsppp/Include/dsppp/autodiff/doc/training_loop.md index 0fe59b942..a73d7c5b7 100644 --- a/dsppp/Include/dsppp/autodiff/doc/training_loop.md +++ b/dsppp/Include/dsppp/autodiff/doc/training_loop.md @@ -20,11 +20,11 @@ using namespace arm_cmsis_dsp::autodiff; int main() { -Arena<1024> arena; -Tape &tape = arena.tape(); -tape.register_operator(); -tape.register_operator(); -tape.register_operator(); +Arena<1024, float> arena; +Tape &tape = arena.tape(); +tape.register_operator>(); +tape.register_operator>(); +tape.register_operator>(); float coefficient_value[] = {0.0F}; float bias_value[] = {0.0F}; @@ -34,13 +34,13 @@ float product_value[] = {0.0F}; float prediction_value[] = {0.0F}; float loss_value[] = {0.0F}; -BufferView coefficient = tape.parameter(coefficient_value); -BufferView bias = tape.parameter(bias_value); -BufferView feature = tape.input(feature_value); -BufferView target = tape.input(target_value); -BufferView product = tape.output(product_value); -BufferView prediction = tape.output(prediction_value); -BufferView loss = tape.output(loss_value); +BufferView coefficient = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); +BufferView feature = tape.input(feature_value); +BufferView target = tape.input(target_value); +BufferView product = tape.output(product_value); +BufferView prediction = tape.output(prediction_value); +BufferView loss = tape.output(loss_value); RMSProp<2, 2> optimizer(1.0e-3F); optimizer.add(coefficient); diff --git a/dsppp/Include/dsppp/autodiff/operators/add.hpp b/dsppp/Include/dsppp/autodiff/operators/add.hpp index 4cb1339dd..c0d3f9d3c 100644 --- a/dsppp/Include/dsppp/autodiff/operators/add.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/add.hpp @@ -3,97 +3,126 @@ #include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class AddOperator +template class AddOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - float *left_gradient; - float *right_gradient; + T *output_gradient; + T *left_gradient; + T *right_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.left_gradient != nullptr) - arm_fill_f32(0.0F, record.left_gradient, record.length); + fill(record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_fill_f32(0.0F, record.right_gradient, record.length); + fill(record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); if (record.left_gradient != nullptr) - arm_add_f32(record.left_gradient, record.output_gradient, - record.left_gradient, record.length); + add(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_add_f32(record.right_gradient, record.output_gradient, - record.right_gradient, record.length); + add(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); } public: - static bool evaluate(BufferView &output, const BufferView &left, - const BufferView &right) noexcept + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right) || - OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || - OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_add_f32(OperatorAccess::values(left), OperatorAccess::values(right), - OperatorAccess::values(output), - OperatorAccess::length(output)); - if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + add(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->left_gradient = OperatorAccess::gradients(left); - record->right_gradient = OperatorAccess::gradients(right); - record->length = OperatorAccess::length(output); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class AddExpression +template class AddExpression { public: - AddExpression(const BufferView &left, const BufferView &right) noexcept + AddExpression(const BufferView &left, const BufferView &right) noexcept : left_(left), right_(right) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - AddOperator::evaluate(output, left_, right_); + AddOperator::evaluate(output, left_, right_); } private: - BufferView left_; - BufferView right_; + BufferView left_; + BufferView right_; }; -inline AddExpression operator+(const BufferView &left, - const BufferView &right) noexcept +template +inline AddExpression operator+(const BufferView &left, + const BufferView &right) noexcept { - return AddExpression(left, right); + return AddExpression(left, right); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp index 3dd8c5ebc..fb15e7d59 100644 --- a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp @@ -2,125 +2,217 @@ #include +#include #include +#include #include +#include #include +#include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { /** Categorical cross entropy: -sum(target[i] * log(probability[i])). */ -class CrossEntropyOperator +template class CrossEntropyOperator { + static constexpr T maximum_value() noexcept + { + if constexpr (std::is_same::value) + return std::numeric_limits::max(); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return F16_MAX; +#else + else + return T{}; +#endif + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void clip(const T *input, T *output, T low, T high, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_clip_f32(input, output, low, high, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_clip_f16(input, output, low, high, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + output[i] = input[i] < low ? low : (input[i] > high ? high : input[i]); +#endif + } + + static void vlog(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_vlog_f32(data, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_vlog_f16(data, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + data[i] = static_cast(std::log(static_cast(data[i]))); +#endif + } + + static void dot(const T *left, const T *right, std::size_t length, + T *result) noexcept + { + if constexpr (std::is_same::value) + arm_dot_prod_f32(left, right, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_dot_prod_f16(left, right, static_cast(length), result); +#else + else + { + *result = T{}; + for (std::size_t i = 0; i < length; ++i) *result += left[i] * right[i]; + } +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *probability_value; - float *probability_gradient; - const float *target_value; + T *output_gradient; + const T *probability_value; + T *probability_gradient; + const T *target_value; std::size_t length; }; - static constexpr float minimum_probability = 1.0e-7F; + static constexpr T minimum_probability() noexcept + { + // 1e-7 is useful for float32, but its reciprocal is too large for a + // finite float16 gradient. Keep the half-precision derivative inside + // its representable range before the softmax reverse rule consumes it. + if constexpr (std::is_same::value) + return T{1.0e-7F}; + return T{1.0e-4F}; + } static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - record.output_gradient[0] = 0.0F; - arm_fill_f32(0.0F, record.probability_gradient, record.length); + record.output_gradient[0] = T{}; + fill(record.probability_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - const float seed = record.output_gradient[0]; - if (seed == 0.0F) return; - arm_clip_f32(record.probability_value, record.probability_gradient, - minimum_probability, - std::numeric_limits::max(), record.length); + const T seed = record.output_gradient[0]; + if (static_cast(seed) == 0.0F) return; + clip(record.probability_value, record.probability_gradient, + minimum_probability(), maximum_value(), record.length); for (std::size_t i = 0; i < record.length; ++i) { - record.probability_gradient[i] = - -seed * record.target_value[i] / - record.probability_gradient[i]; + const float denominator = static_cast(record.probability_gradient[i]); + float gradient = -static_cast(seed) * + static_cast(record.target_value[i]) / + denominator; + if constexpr (!std::is_same::value) + { + const float limit = static_cast(maximum_value()); + if (gradient > limit) gradient = limit; + if (gradient < -limit) gradient = -limit; + } + record.probability_gradient[i] = static_cast(gradient); } } public: - static bool evaluate(BufferView &output, const BufferView &probability, - const BufferView &target) noexcept + static bool evaluate(BufferView &output, const BufferView &probability, + const BufferView &target) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || - !OperatorAccess::require(*tape)) + !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - OperatorAccess::length(output) != 1U || - OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, probability, target) || - OperatorAccess::gradients(probability) == nullptr || - OperatorAccess::role(target) != BufferRole::input) + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, probability, target) || + OperatorAccess::gradients(probability) == nullptr || + OperatorAccess::role(target) != BufferRole::input) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const std::size_t length = OperatorAccess::length(probability); - float result = 0.0F; + const std::size_t length = OperatorAccess::length(probability); + T result = T{}; if (length != 0U) { // Reuse the probability gradient as forward scratch. backward() // resets it before accumulating any gradient. - float *scratch = OperatorAccess::gradients(probability); - arm_clip_f32(OperatorAccess::values(probability), scratch, - minimum_probability, - std::numeric_limits::max(), length); - arm_vlog_f32(scratch, scratch, length); - arm_dot_prod_f32(OperatorAccess::values(target), scratch, length, - &result); + T *scratch = OperatorAccess::gradients(probability); + clip(OperatorAccess::values(probability), scratch, minimum_probability(), + maximum_value(), length); + vlog(scratch, length); + dot(OperatorAccess::values(target), scratch, length, &result); } - OperatorAccess::values(output)[0] = -result; - if (!OperatorAccess::recording(*tape)) - return OperatorAccess::status(*tape) == Status::ok; + OperatorAccess::values(output)[0] = static_cast(-static_cast(result)); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->probability_value = OperatorAccess::values(probability); - record->probability_gradient = OperatorAccess::gradients(probability); - record->target_value = OperatorAccess::values(target); + record->output_gradient = OperatorAccess::gradients(output); + record->probability_value = OperatorAccess::values(probability); + record->probability_gradient = OperatorAccess::gradients(probability); + record->target_value = OperatorAccess::values(target); record->length = length; - OperatorAccess::set_producer(output, &record->node); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class CrossEntropyExpression +template class CrossEntropyExpression { public: - CrossEntropyExpression(const BufferView &probability, - const BufferView &target) noexcept + CrossEntropyExpression(const BufferView &probability, + const BufferView &target) noexcept : probability_(probability), target_(target) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - CrossEntropyOperator::evaluate(output, probability_, target_); + CrossEntropyOperator::evaluate(output, probability_, target_); } private: - BufferView probability_; - BufferView target_; + BufferView probability_; + BufferView target_; }; -inline CrossEntropyExpression cross_entropy( - const BufferView &probability, const BufferView &target) noexcept +template +inline CrossEntropyExpression cross_entropy( + const BufferView &probability, const BufferView &target) noexcept { - return CrossEntropyExpression(probability, target); + return CrossEntropyExpression(probability, target); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp index 2fe2c4404..abf6ea8e1 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dot.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -7,111 +7,144 @@ #include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class DotOperator +template class DotOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void dot(const T *left, const T *right, std::size_t length, + T *result) noexcept + { + if constexpr (std::is_same::value) + arm_dot_prod_f32(left, right, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_dot_prod_f16(left, right, static_cast(length), result); +#else + else + { + *result = T{}; + for (std::size_t i = 0; i < length; ++i) + *result += left[i] * right[i]; + } +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *left_value; - float *left_gradient; - const float *right_value; - float *right_gradient; + T *output_gradient; + const T *left_value; + T *left_gradient; + const T *right_value; + T *right_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - record.output_gradient[0] = 0.0F; + record.output_gradient[0] = T{}; if (record.left_gradient != nullptr) - arm_fill_f32(0.0F, record.left_gradient, record.length); + fill(record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_fill_f32(0.0F, record.right_gradient, record.length); + fill(record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - const float gradient = record.output_gradient[0]; - if (gradient == 0.0F) return; + const T gradient = record.output_gradient[0]; + if (static_cast(gradient) == 0.0F) return; if (record.left_gradient != nullptr) { - VectorView left_grad(record.left_gradient, 0, record.length); - VectorView right_val(const_cast(record.right_value), 0, record.length); + VectorView left_grad(record.left_gradient, 0, record.length); + VectorView right_val(const_cast(record.right_value), 0, record.length); left_grad += right_val * gradient; } if (record.right_gradient != nullptr) { - VectorView right_grad(record.right_gradient, 0, record.length); - VectorView left_val(const_cast(record.left_value), 0, record.length); + VectorView right_grad(record.right_gradient, 0, record.length); + VectorView left_val(const_cast(record.left_value), 0, record.length); right_grad += left_val * gradient; } } public: - static bool evaluate(BufferView &output, const BufferView &left, - const BufferView &right) noexcept + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - OperatorAccess::length(output) != 1U || - OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, left, right) || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right)) + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, left, right) || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - float value = 0.0F; - arm_dot_prod_f32(OperatorAccess::values(left), - OperatorAccess::values(right), - OperatorAccess::length(left), &value); - OperatorAccess::values(output)[0] = value; - if (!OperatorAccess::recording(*tape)) - return OperatorAccess::status(*tape) == Status::ok; + T value = T{}; + dot(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::length(left), &value); + OperatorAccess::values(output)[0] = value; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->left_value = OperatorAccess::values(left); - record->left_gradient = OperatorAccess::gradients(left); - record->right_value = OperatorAccess::values(right); - record->right_gradient = OperatorAccess::gradients(right); - record->length = OperatorAccess::length(left); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(left); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class DotExpression +template class DotExpression { public: - DotExpression(const BufferView &left, const BufferView &right) noexcept + DotExpression(const BufferView &left, const BufferView &right) noexcept : left_(left), right_(right) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - DotOperator::evaluate(output, left_, right_); + DotOperator::evaluate(output, left_, right_); } private: - BufferView left_; - BufferView right_; + BufferView left_; + BufferView right_; }; -inline DotExpression dot(const BufferView &left, const BufferView &right) noexcept +template +inline DotExpression dot(const BufferView &left, const BufferView &right) noexcept { - return DotExpression(left, right); + return DotExpression(left, right); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp index dd81f105a..4d97f4b5d 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include @@ -24,21 +26,61 @@ class DropoutGenerator private: std::uint32_t state_; - friend class DropoutOperator; + template friend class DropoutOperator; }; /** Inverted dropout during recording, identity when recording is disabled. */ -class DropoutOperator +template class DropoutOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void copy(const T *input, T *output, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_copy_f32(input, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_copy_f16(input, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i]; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - float *input_gradient; + T *output_gradient; + T *input_gradient; std::size_t length; std::uint32_t random_state; float drop_probability; - float scale; + T scale; }; static std::uint32_t next(std::uint32_t &state) noexcept @@ -60,9 +102,9 @@ class DropoutOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.input_gradient != nullptr) - arm_fill_f32(0.0F, record.input_gradient, record.length); + fill(record.input_gradient, record.length); } static void backward(detail::Node &node) noexcept @@ -72,106 +114,114 @@ class DropoutOperator if (record.drop_probability == 0.0F) { - arm_add_f32(record.input_gradient, record.output_gradient, - record.input_gradient, record.length); + add(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); return; } std::uint32_t state = record.random_state; for (std::size_t i = 0; i < record.length; ++i) + { if (keep(state, record.drop_probability)) - record.input_gradient[i] += - record.output_gradient[i] * record.scale; + { + if constexpr (std::is_same::value) + record.input_gradient[i] += record.output_gradient[i] * record.scale; + else + record.input_gradient[i] = static_cast( + static_cast(record.input_gradient[i]) + + static_cast(record.output_gradient[i]) * + static_cast(record.scale)); + } + } } public: - static bool evaluate(BufferView &output, const BufferView &input, + static bool evaluate(BufferView &output, const BufferView &input, DropoutGenerator &generator, float drop_probability) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || !(drop_probability >= 0.0F && drop_probability < 1.0F)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const std::size_t length = OperatorAccess::length(output); - if (!OperatorAccess::recording(*tape)) + const std::size_t length = OperatorAccess::length(output); + if (!OperatorAccess::recording(*tape)) { - arm_copy_f32(OperatorAccess::values(input), - OperatorAccess::values(output), length); - return OperatorAccess::status(*tape) == Status::ok; + copy(OperatorAccess::values(input), OperatorAccess::values(output), length); + return OperatorAccess::status(*tape) == Status::ok; } if (length == 0U) - return OperatorAccess::status(*tape) == Status::ok; + return OperatorAccess::status(*tape) == Status::ok; const float scale = 1.0F / (1.0F - drop_probability); const std::uint32_t initial_state = generator.state_; if (drop_probability == 0.0F) { - arm_copy_f32(OperatorAccess::values(input), - OperatorAccess::values(output), length); + copy(OperatorAccess::values(input), OperatorAccess::values(output), length); } else { for (std::size_t i = 0; i < length; ++i) - OperatorAccess::values(output)[i] = + OperatorAccess::values(output)[i] = keep(generator.state_, drop_probability) - ? OperatorAccess::values(input)[i] * scale - : 0.0F; + ? OperatorAccess::values(input)[i] * static_cast(scale) + : T{}; } - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) { generator.state_ = initial_state; return false; } - record->output_gradient = OperatorAccess::gradients(output); - record->input_gradient = OperatorAccess::gradients(input); + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); record->length = length; record->random_state = initial_state; record->drop_probability = drop_probability; - record->scale = scale; - OperatorAccess::set_producer(output, &record->node); + record->scale = static_cast(scale); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class DropoutExpression +template class DropoutExpression { public: - DropoutExpression(const BufferView &input, DropoutGenerator &generator, + DropoutExpression(const BufferView &input, DropoutGenerator &generator, float drop_probability) noexcept : input_(input), generator_(generator), drop_probability_(drop_probability) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - DropoutOperator::evaluate(output, input_, generator_, - drop_probability_); + DropoutOperator::evaluate(output, input_, generator_, + drop_probability_); } private: - BufferView input_; + BufferView input_; DropoutGenerator &generator_; float drop_probability_; }; -inline DropoutExpression dropout(const BufferView &input, - DropoutGenerator &generator, - float drop_probability) noexcept +template +inline DropoutExpression dropout(const BufferView &input, + DropoutGenerator &generator, + float drop_probability) noexcept { - return DropoutExpression(input, generator, drop_probability); + return DropoutExpression(input, generator, drop_probability); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp index cc37e11d5..b180c3ea7 100644 --- a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -7,24 +7,39 @@ #include #include +#include #include +#include #include namespace arm_cmsis_dsp { namespace autodiff { -class FullyConnectedOperator +template class FullyConnectedOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *input_value; - float *input_gradient; - const float *weight_value; - float *weight_gradient; - float *bias_gradient; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *weight_value; + T *weight_gradient; + T *bias_gradient; std::size_t rows; std::size_t columns; }; @@ -32,12 +47,11 @@ class FullyConnectedOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.rows); - arm_fill_f32(0.0F, record.bias_gradient, record.rows); - arm_fill_f32(0.0F, record.weight_gradient, - record.rows * record.columns); + fill(record.output_gradient, record.rows); + fill(record.bias_gradient, record.rows); + fill(record.weight_gradient, record.rows * record.columns); if (record.input_gradient != nullptr && record.columns != 0U) - arm_fill_f32(0.0F, record.input_gradient, record.columns); + fill(record.input_gradient, record.columns); } static void backward(detail::Node &node) noexcept @@ -45,9 +59,9 @@ class FullyConnectedOperator Record &record = reinterpret_cast(node); if (record.rows == 0U) return; - ::arm_cmsis_dsp::VectorView output_gradient( + ::arm_cmsis_dsp::VectorView output_gradient( record.output_gradient, 0, record.rows); - ::arm_cmsis_dsp::VectorView bias_gradient( + ::arm_cmsis_dsp::VectorView bias_gradient( record.bias_gradient, 0, record.rows); // Bias is shared by every input sample, so its gradient must be @@ -56,9 +70,9 @@ class FullyConnectedOperator if (record.columns != 0U) { - ::arm_cmsis_dsp::VectorView input_value( - const_cast(record.input_value), 0, record.columns); - ::arm_cmsis_dsp::MatrixView input_value( + const_cast(record.input_value), 0, record.columns); + ::arm_cmsis_dsp::MatrixView weight_gradient(record.weight_gradient, record.rows, record.columns, record.columns); @@ -70,11 +84,11 @@ class FullyConnectedOperator if (record.input_gradient == nullptr) return; - ::arm_cmsis_dsp::VectorView input_gradient( + ::arm_cmsis_dsp::VectorView input_gradient( record.input_gradient, 0, record.columns); - ::arm_cmsis_dsp::MatrixView - weight_value(const_cast(record.weight_value), + weight_value(const_cast(record.weight_value), record.rows, record.columns, record.columns); input_gradient += ::arm_cmsis_dsp::dot( ::arm_cmsis_dsp::transpose_view(weight_value), @@ -83,54 +97,68 @@ class FullyConnectedOperator } public: - static bool evaluate(BufferView &output, const BufferView &input, - const MatrixView &weights, - const BufferView &bias) noexcept + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights, + const BufferView &bias) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || - !OperatorAccess::require(*tape)) + !OperatorAccess::template require>(*tape)) return false; - const BufferView &weight_buffer = OperatorAccess::buffer(weights); - if (!OperatorAccess::valid(*tape, output) || - !OperatorAccess::valid(*tape, input) || - !OperatorAccess::valid(*tape, weight_buffer) || - !OperatorAccess::valid(*tape, bias) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::length(output) != OperatorAccess::rows(weights) || - OperatorAccess::length(input) != OperatorAccess::columns(weights) || - OperatorAccess::length(bias) != OperatorAccess::rows(weights) || - OperatorAccess::rows(weights) > + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + if (!OperatorAccess::valid(*tape, output) || + !OperatorAccess::valid(*tape, input) || + !OperatorAccess::valid(*tape, weight_buffer) || + !OperatorAccess::valid(*tape, bias) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::length(output) != OperatorAccess::rows(weights) || + OperatorAccess::length(input) != OperatorAccess::columns(weights) || + OperatorAccess::length(bias) != OperatorAccess::rows(weights) || + OperatorAccess::rows(weights) > std::numeric_limits::max() || - OperatorAccess::columns(weights) > + OperatorAccess::columns(weights) > std::numeric_limits::max() || - OperatorAccess::role(weight_buffer) != BufferRole::parameter || - OperatorAccess::role(bias) != BufferRole::parameter) + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::role(bias) != BufferRole::parameter) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const std::size_t rows = OperatorAccess::rows(weights); - const std::size_t columns = OperatorAccess::columns(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t columns = OperatorAccess::columns(weights); if (rows != 0U) { - ::arm_cmsis_dsp::VectorView output_value( - OperatorAccess::values(output), 0, rows); - ::arm_cmsis_dsp::VectorView bias_value( - const_cast(OperatorAccess::values(bias)), 0, rows); + ::arm_cmsis_dsp::VectorView output_value( + OperatorAccess::values(output), 0, rows); + ::arm_cmsis_dsp::VectorView bias_value( + const_cast(OperatorAccess::values(bias)), 0, rows); if (columns != 0U) { + if constexpr (std::is_same::value) + { arm_matrix_instance_f32 weight_matrix; arm_mat_init_f32( &weight_matrix, static_cast(rows), static_cast(columns), - const_cast(OperatorAccess::values(weight_buffer))); + const_cast(OperatorAccess::values(weight_buffer))); arm_mat_vec_mult_f32(&weight_matrix, - OperatorAccess::values(input), - OperatorAccess::values(output)); + OperatorAccess::values(input), + OperatorAccess::values(output)); + } + else + { + arm_matrix_instance_f16 weight_matrix; + arm_mat_init_f16( + &weight_matrix, static_cast(rows), + static_cast(columns), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_vec_mult_f16(&weight_matrix, + OperatorAccess::values(input), + OperatorAccess::values(output)); + } output_value += bias_value; } else @@ -138,50 +166,51 @@ class FullyConnectedOperator // VectorView deliberately deletes copy assignment. This // dimension-zero edge has no matrix product to optimize. for (std::size_t row = 0; row < rows; ++row) - OperatorAccess::values(output)[row] = - OperatorAccess::values(bias)[row]; + OperatorAccess::values(output)[row] = + OperatorAccess::values(bias)[row]; } } - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->input_value = OperatorAccess::values(input); - record->input_gradient = OperatorAccess::gradients(input); - record->weight_value = OperatorAccess::values(weight_buffer); - record->weight_gradient = OperatorAccess::gradients(weight_buffer); - record->bias_gradient = OperatorAccess::gradients(bias); - record->rows = OperatorAccess::rows(weights); - record->columns = OperatorAccess::columns(weights); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->weight_value = OperatorAccess::values(weight_buffer); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->bias_gradient = OperatorAccess::gradients(bias); + record->rows = OperatorAccess::rows(weights); + record->columns = OperatorAccess::columns(weights); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class FullyConnectedExpression +template class FullyConnectedExpression { public: - FullyConnectedExpression(const BufferView &input, const MatrixView &weights, - const BufferView &bias) noexcept + FullyConnectedExpression(const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept : input_(input), weights_(weights), bias_(bias) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - FullyConnectedOperator::evaluate(output, input_, weights_, bias_); + FullyConnectedOperator::evaluate(output, input_, weights_, bias_); } private: - BufferView input_; - MatrixView weights_; - BufferView bias_; + BufferView input_; + MatrixView weights_; + BufferView bias_; }; -inline FullyConnectedExpression fully_connected( - const BufferView &input, const MatrixView &weights, - const BufferView &bias) noexcept +template +inline FullyConnectedExpression fully_connected( + const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept { - return FullyConnectedExpression(input, weights, bias); + return FullyConnectedExpression(input, weights, bias); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp index 5bc4e9c71..76ee9c7fb 100644 --- a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include @@ -13,14 +15,27 @@ namespace arm_cmsis_dsp { namespace autodiff { /** Matrix product Y = W X, differentiating only the parameter matrix W. */ -class MatrixMultiplyOperator +template class MatrixMultiplyOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *input_value; - float *weight_gradient; + T *output_gradient; + const T *input_value; + T *weight_gradient; std::size_t rows; std::size_t inner; std::size_t columns; @@ -29,18 +44,16 @@ class MatrixMultiplyOperator static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, - record.rows * record.columns); - arm_fill_f32(0.0F, record.weight_gradient, - record.rows * record.inner); + fill(record.output_gradient, record.rows * record.columns); + fill(record.weight_gradient, record.rows * record.inner); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - ::arm_cmsis_dsp::MatrixView - input_value(const_cast(record.input_value), record.inner, + ::arm_cmsis_dsp::MatrixView + input_value(const_cast(record.input_value), record.inner, record.columns, record.columns); // dW = dY X^T. For each row of dY, this is X times that row. The lazy @@ -48,10 +61,10 @@ class MatrixMultiplyOperator for (std::size_t row = 0; row < record.rows; ++row) { - ::arm_cmsis_dsp::VectorView output_gradient( + ::arm_cmsis_dsp::VectorView output_gradient( record.output_gradient + row * record.columns, 0, record.columns); - ::arm_cmsis_dsp::VectorView weight_gradient( + ::arm_cmsis_dsp::VectorView weight_gradient( record.weight_gradient + row * record.inner, 0, record.inner); weight_gradient += @@ -60,95 +73,115 @@ class MatrixMultiplyOperator } public: - static bool evaluate(BufferView &output, const BufferView &input, - const MatrixView &weights) noexcept + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || - !OperatorAccess::require(*tape)) + !OperatorAccess::template require>(*tape)) return false; - const BufferView &weight_buffer = OperatorAccess::buffer(weights); - const std::size_t rows = OperatorAccess::rows(weights); - const std::size_t inner = OperatorAccess::columns(weights); - const std::size_t input_length = OperatorAccess::length(input); + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t inner = OperatorAccess::columns(weights); + const std::size_t input_length = OperatorAccess::length(input); const std::size_t columns = inner == 0U ? 0U : input_length / inner; - if (!OperatorAccess::valid(*tape, output) || - !OperatorAccess::valid(*tape, input) || - !OperatorAccess::valid(*tape, weight_buffer) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::role(input) != BufferRole::input || - OperatorAccess::role(weight_buffer) != BufferRole::parameter || - OperatorAccess::gradients(weight_buffer) == nullptr || + if (!OperatorAccess::valid(*tape, output) || + !OperatorAccess::valid(*tape, input) || + !OperatorAccess::valid(*tape, weight_buffer) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::role(input) != BufferRole::input || + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::gradients(weight_buffer) == nullptr || rows == 0U || inner == 0U || columns == 0U || input_length % inner != 0U || rows > std::numeric_limits::max() || inner > std::numeric_limits::max() || columns > std::numeric_limits::max() || - OperatorAccess::length(output) != rows * columns) + OperatorAccess::length(output) != rows * columns) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } + arm_status matrix_status; + if constexpr (std::is_same::value) + { arm_matrix_instance_f32 weight_matrix; arm_matrix_instance_f32 input_matrix; arm_matrix_instance_f32 output_matrix; arm_mat_init_f32(&weight_matrix, static_cast(rows), static_cast(inner), - const_cast(OperatorAccess::values( - weight_buffer))); + const_cast(OperatorAccess::values(weight_buffer))); arm_mat_init_f32(&input_matrix, static_cast(inner), static_cast(columns), - const_cast(OperatorAccess::values(input))); + const_cast(OperatorAccess::values(input))); arm_mat_init_f32(&output_matrix, static_cast(rows), static_cast(columns), - OperatorAccess::values(output)); - if (arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix) != - ARM_MATH_SUCCESS) + OperatorAccess::values(output)); + matrix_status = arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix); + } + else + { + arm_matrix_instance_f16 weight_matrix; + arm_matrix_instance_f16 input_matrix; + arm_matrix_instance_f16 output_matrix; + arm_mat_init_f16(&weight_matrix, static_cast(rows), + static_cast(inner), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_init_f16(&input_matrix, static_cast(inner), + static_cast(columns), + const_cast(OperatorAccess::values(input))); + arm_mat_init_f16(&output_matrix, static_cast(rows), + static_cast(columns), + OperatorAccess::values(output)); + matrix_status = arm_mat_mult_f16(&weight_matrix, &input_matrix, &output_matrix); + } + if (matrix_status != ARM_MATH_SUCCESS) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - if (!OperatorAccess::recording(*tape)) - return OperatorAccess::status(*tape) == Status::ok; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->input_value = OperatorAccess::values(input); - record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); record->rows = rows; record->inner = inner; record->columns = columns; - OperatorAccess::set_producer(output, &record->node); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class MatrixMultiplyExpression +template class MatrixMultiplyExpression { public: - MatrixMultiplyExpression(const BufferView &input, - const MatrixView &weights) noexcept + MatrixMultiplyExpression(const BufferView &input, + const MatrixView &weights) noexcept : input_(input), weights_(weights) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - MatrixMultiplyOperator::evaluate(output, input_, weights_); + MatrixMultiplyOperator::evaluate(output, input_, weights_); } private: - BufferView input_; - MatrixView weights_; + BufferView input_; + MatrixView weights_; }; -inline MatrixMultiplyExpression matrix_multiply( - const BufferView &input, const MatrixView &weights) noexcept +template +inline MatrixMultiplyExpression matrix_multiply( + const BufferView &input, const MatrixView &weights) noexcept { - return MatrixMultiplyExpression(input, weights); + return MatrixMultiplyExpression(input, weights); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp index 1370b96e6..f6f1b0bf9 100644 --- a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp @@ -5,117 +5,145 @@ #include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class MultiplyOperator +template class MultiplyOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void multiply(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_mult_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_mult_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] * right[i]; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *left_value; - float *left_gradient; - const float *right_value; - float *right_gradient; + T *output_gradient; + const T *left_value; + T *left_gradient; + const T *right_value; + T *right_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.left_gradient != nullptr) - arm_fill_f32(0.0F, record.left_gradient, record.length); + fill(record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_fill_f32(0.0F, record.right_gradient, record.length); + fill(record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - ::arm_cmsis_dsp::VectorView output_gradient( + ::arm_cmsis_dsp::VectorView output_gradient( record.output_gradient, 0, record.length); if (record.left_gradient != nullptr) { - ::arm_cmsis_dsp::VectorView left_gradient( + ::arm_cmsis_dsp::VectorView left_gradient( record.left_gradient, 0, record.length); - ::arm_cmsis_dsp::VectorView right_value( - const_cast(record.right_value), 0, record.length); + ::arm_cmsis_dsp::VectorView right_value( + const_cast(record.right_value), 0, record.length); left_gradient += output_gradient * right_value; } if (record.right_gradient != nullptr) { - ::arm_cmsis_dsp::VectorView right_gradient( + ::arm_cmsis_dsp::VectorView right_gradient( record.right_gradient, 0, record.length); - ::arm_cmsis_dsp::VectorView left_value( - const_cast(record.left_value), 0, record.length); + ::arm_cmsis_dsp::VectorView left_value( + const_cast(record.left_value), 0, record.length); right_gradient += output_gradient * left_value; } } public: - static bool evaluate(BufferView &output, const BufferView &left, - const BufferView &right) noexcept + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(left) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(right)) + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_mult_f32(OperatorAccess::values(left), - OperatorAccess::values(right), - OperatorAccess::values(output), - OperatorAccess::length(output)); - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + multiply(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->left_value = OperatorAccess::values(left); - record->left_gradient = OperatorAccess::gradients(left); - record->right_value = OperatorAccess::values(right); - record->right_gradient = OperatorAccess::gradients(right); - record->length = OperatorAccess::length(output); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class MultiplyExpression +template class MultiplyExpression { public: - MultiplyExpression(const BufferView &left, const BufferView &right) noexcept + MultiplyExpression(const BufferView &left, const BufferView &right) noexcept : left_(left), right_(right) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - MultiplyOperator::evaluate(output, left_, right_); + MultiplyOperator::evaluate(output, left_, right_); } private: - BufferView left_; - BufferView right_; + BufferView left_; + BufferView right_; }; -inline MultiplyExpression operator*(const BufferView &left, - const BufferView &right) noexcept +template +inline MultiplyExpression operator*(const BufferView &left, + const BufferView &right) noexcept { - return MultiplyExpression(left, right); + return MultiplyExpression(left, right); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/offset.hpp b/dsppp/Include/dsppp/autodiff/operators/offset.hpp index 12f4c5cbc..99f1ff40a 100644 --- a/dsppp/Include/dsppp/autodiff/operators/offset.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/offset.hpp @@ -3,105 +3,165 @@ #include #include +#include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class OffsetOperator +template class OffsetOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void apply_offset(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_offset_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_offset_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] + value; +#endif + } + + static void accumulate(const T *input, std::size_t length, T *result) noexcept + { + if constexpr (std::is_same::value) + arm_accumulate_f32(input, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_accumulate_f16(input, static_cast(length), result); +#else + else + for (std::size_t i = 0; i < length; ++i) *result += input[i]; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - float *input_gradient; - float *offset_gradient; + T *output_gradient; + T *input_gradient; + T *offset_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.input_gradient != nullptr) - arm_fill_f32(0.0F, record.input_gradient, record.length); - record.offset_gradient[0] = 0.0F; + fill(record.input_gradient, record.length); + record.offset_gradient[0] = T{}; } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); if (record.input_gradient != nullptr) - arm_add_f32(record.input_gradient, record.output_gradient, - record.input_gradient, record.length); - float gradient_sum = 0.0F; - arm_accumulate_f32(record.output_gradient, record.length, - &gradient_sum); - record.offset_gradient[0] += gradient_sum; + add(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); + T gradient_sum = T{}; + accumulate(record.output_gradient, record.length, &gradient_sum); + if constexpr (std::is_same::value) + record.offset_gradient[0] += gradient_sum; + else + record.offset_gradient[0] = static_cast( + static_cast(record.offset_gradient[0]) + + static_cast(gradient_sum)); } public: - static bool evaluate(BufferView &output, const BufferView &input, - const BufferView &offset) noexcept + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &offset) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - !OperatorAccess::valid(*tape, offset) || - OperatorAccess::length(offset) != 1U || - OperatorAccess::role(offset) != BufferRole::parameter || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::gradients(offset) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::values(output) == OperatorAccess::values(offset) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(offset)) + if (!OperatorAccess::compatible(*tape, output, input) || + !OperatorAccess::valid(*tape, offset) || + OperatorAccess::length(offset) != 1U || + OperatorAccess::role(offset) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(offset) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(offset) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(offset)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_offset_f32(OperatorAccess::values(input), - OperatorAccess::values(offset)[0], - OperatorAccess::values(output), - OperatorAccess::length(output)); - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + apply_offset(OperatorAccess::values(input), OperatorAccess::values(offset)[0], + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->input_gradient = OperatorAccess::gradients(input); - record->offset_gradient = OperatorAccess::gradients(offset); - record->length = OperatorAccess::length(output); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); + record->offset_gradient = OperatorAccess::gradients(offset); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class OffsetExpression +template class OffsetExpression { public: - OffsetExpression(const BufferView &input, const BufferView &offset) noexcept + OffsetExpression(const BufferView &input, const BufferView &offset) noexcept : input_(input), offset_(offset) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - OffsetOperator::evaluate(output, input_, offset_); + OffsetOperator::evaluate(output, input_, offset_); } private: - BufferView input_; - BufferView offset_; + BufferView input_; + BufferView offset_; }; -inline OffsetExpression offset(const BufferView &input, - const BufferView &constant) noexcept +template +inline OffsetExpression offset(const BufferView &input, + const BufferView &constant) noexcept { - return OffsetExpression(input, constant); + return OffsetExpression(input, constant); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp index d77a11512..699986cc2 100644 --- a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp @@ -5,108 +5,123 @@ #include #include +#include namespace arm_cmsis_dsp { namespace autodiff { /** Sum-of-squared-errors loss: sum((prediction - target)^2). */ -class QuadraticErrorOperator +template class QuadraticErrorOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *prediction_value; - float *prediction_gradient; - const float *target_value; + T *output_gradient; + const T *prediction_value; + T *prediction_gradient; + const T *target_value; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - record.output_gradient[0] = 0.0F; - arm_fill_f32(0.0F, record.prediction_gradient, record.length); + record.output_gradient[0] = T{}; + fill(record.prediction_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - const float seed = record.output_gradient[0]; - if (seed == 0.0F) return; - ::arm_cmsis_dsp::VectorView prediction_gradient( + const T seed = record.output_gradient[0]; + if (static_cast(seed) == 0.0F) return; + ::arm_cmsis_dsp::VectorView prediction_gradient( record.prediction_gradient, 0, record.length); - ::arm_cmsis_dsp::VectorView prediction_value( - const_cast(record.prediction_value), 0, record.length); - ::arm_cmsis_dsp::VectorView target_value( - const_cast(record.target_value), 0, record.length); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(record.prediction_value), 0, record.length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(record.target_value), 0, record.length); prediction_gradient += - (prediction_value - target_value) * (2.0F * seed); + (prediction_value - target_value) * static_cast(2.0F * static_cast(seed)); } public: - static bool evaluate(BufferView &output, const BufferView &prediction, - const BufferView &target) noexcept + static bool evaluate(BufferView &output, const BufferView &prediction, + const BufferView &target) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || - !OperatorAccess::require(*tape)) + !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - OperatorAccess::length(output) != 1U || - OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, prediction, target) || - OperatorAccess::gradients(prediction) == nullptr || - OperatorAccess::role(target) != BufferRole::input) + if (!OperatorAccess::valid(*tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(*tape, prediction, target) || + OperatorAccess::gradients(prediction) == nullptr || + OperatorAccess::role(target) != BufferRole::input) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const std::size_t length = OperatorAccess::length(prediction); - ::arm_cmsis_dsp::VectorView prediction_value( - const_cast(OperatorAccess::values(prediction)), 0, + const std::size_t length = OperatorAccess::length(prediction); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(OperatorAccess::values(prediction)), 0, length); - ::arm_cmsis_dsp::VectorView target_value( - const_cast(OperatorAccess::values(target)), 0, length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(OperatorAccess::values(target)), 0, length); const auto error = prediction_value - target_value; - OperatorAccess::values(output)[0] = + OperatorAccess::values(output)[0] = ::arm_cmsis_dsp::dot(error, error); - if (!OperatorAccess::recording(*tape)) - return OperatorAccess::status(*tape) == Status::ok; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->prediction_value = OperatorAccess::values(prediction); - record->prediction_gradient = OperatorAccess::gradients(prediction); - record->target_value = OperatorAccess::values(target); - record->length = OperatorAccess::length(prediction); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->prediction_value = OperatorAccess::values(prediction); + record->prediction_gradient = OperatorAccess::gradients(prediction); + record->target_value = OperatorAccess::values(target); + record->length = OperatorAccess::length(prediction); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class QuadraticErrorExpression +template class QuadraticErrorExpression { public: - QuadraticErrorExpression(const BufferView &prediction, - const BufferView &target) noexcept + QuadraticErrorExpression(const BufferView &prediction, + const BufferView &target) noexcept : prediction_(prediction), target_(target) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - QuadraticErrorOperator::evaluate(output, prediction_, target_); + QuadraticErrorOperator::evaluate(output, prediction_, target_); } private: - BufferView prediction_; - BufferView target_; + BufferView prediction_; + BufferView target_; }; -inline QuadraticErrorExpression quadratic_error( - const BufferView &prediction, const BufferView &target) noexcept +template +inline QuadraticErrorExpression quadratic_error( + const BufferView &prediction, const BufferView &target) noexcept { - return QuadraticErrorExpression(prediction, target); + return QuadraticErrorExpression(prediction, target); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/relu.hpp b/dsppp/Include/dsppp/autodiff/operators/relu.hpp index 41121e422..304c05d19 100644 --- a/dsppp/Include/dsppp/autodiff/operators/relu.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/relu.hpp @@ -2,90 +2,143 @@ #include +#include #include +#include #include +#include #include namespace arm_cmsis_dsp { namespace autodiff { -class ReluOperator +template class ReluOperator { + static constexpr T maximum_value() noexcept + { + if constexpr (std::is_same::value) + return std::numeric_limits::max(); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return F16_MAX; +#else + else + return T{}; +#endif + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void clip(const T *input, T *output, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_clip_f32(input, output, T{}, maximum_value(), + static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_clip_f16(input, output, T{}, maximum_value(), + static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + output[i] = input[i] > T{} ? input[i] : T{}; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *input_value; - float *input_gradient; + T *output_gradient; + const T *input_value; + T *input_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.input_gradient != nullptr) - arm_fill_f32(0.0F, record.input_gradient, record.length); + fill(record.input_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); if (record.input_gradient == nullptr) return; for (std::size_t i = 0; i < record.length; ++i) - if (record.input_value[i] > 0.0F) - record.input_gradient[i] += record.output_gradient[i]; + { + if (static_cast(record.input_value[i]) > 0.0F) + { + if constexpr (std::is_same::value) + record.input_gradient[i] += record.output_gradient[i]; + else + record.input_gradient[i] = static_cast( + static_cast(record.input_gradient[i]) + + static_cast(record.output_gradient[i])); + } + } } public: - static bool evaluate(BufferView &output, const BufferView &input) noexcept + static bool evaluate(BufferView &output, const BufferView &input) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_clip_f32(OperatorAccess::values(input), - OperatorAccess::values(output), 0.0F, - std::numeric_limits::max(), - OperatorAccess::length(input)); - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + clip(OperatorAccess::values(input), OperatorAccess::values(output), + OperatorAccess::length(input)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->input_value = OperatorAccess::values(input); - record->input_gradient = OperatorAccess::gradients(input); - record->length = OperatorAccess::length(input); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->length = OperatorAccess::length(input); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class ReluExpression +template class ReluExpression { public: - explicit ReluExpression(const BufferView &input) noexcept : input_(input) {} - void evaluate(BufferView &output) const noexcept + explicit ReluExpression(const BufferView &input) noexcept : input_(input) {} + void evaluate(BufferView &output) const noexcept { - ReluOperator::evaluate(output, input_); + ReluOperator::evaluate(output, input_); } private: - BufferView input_; + BufferView input_; }; -inline ReluExpression relu(const BufferView &input) noexcept +template +inline ReluExpression relu(const BufferView &input) noexcept { - return ReluExpression(input); + return ReluExpression(input); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/scale.hpp b/dsppp/Include/dsppp/autodiff/operators/scale.hpp index c13c6e734..14669d12c 100644 --- a/dsppp/Include/dsppp/autodiff/operators/scale.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/scale.hpp @@ -5,113 +5,146 @@ #include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class ScaleOperator +template class ScaleOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void apply_scale(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_scale_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_scale_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] * value; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *input_value; - float *input_gradient; - const float *scale_value; - float *scale_gradient; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + T *scale_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.input_gradient != nullptr) - arm_fill_f32(0.0F, record.input_gradient, record.length); - record.scale_gradient[0] = 0.0F; + fill(record.input_gradient, record.length); + record.scale_gradient[0] = T{}; } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - ::arm_cmsis_dsp::VectorView output_gradient( + ::arm_cmsis_dsp::VectorView output_gradient( record.output_gradient, 0, record.length); - ::arm_cmsis_dsp::VectorView input_value( - const_cast(record.input_value), 0, record.length); - record.scale_gradient[0] += - ::arm_cmsis_dsp::dot(output_gradient, input_value); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value), 0, record.length); + const T contribution = static_cast(::arm_cmsis_dsp::dot(output_gradient, input_value)); + if constexpr (std::is_same::value) + record.scale_gradient[0] += contribution; + else + record.scale_gradient[0] = static_cast( + static_cast(record.scale_gradient[0]) + + static_cast(contribution)); if (record.input_gradient != nullptr) { - ::arm_cmsis_dsp::VectorView input_gradient( + ::arm_cmsis_dsp::VectorView input_gradient( record.input_gradient, 0, record.length); input_gradient += output_gradient * record.scale_value[0]; } } public: - static bool evaluate(BufferView &output, const BufferView &input, - const BufferView &scale) noexcept + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - !OperatorAccess::valid(*tape, scale) || - OperatorAccess::length(scale) != 1U || - OperatorAccess::role(scale) != BufferRole::parameter || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::gradients(scale) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::values(output) == OperatorAccess::values(scale) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(scale)) + if (!OperatorAccess::compatible(*tape, output, input) || + !OperatorAccess::valid(*tape, scale) || + OperatorAccess::length(scale) != 1U || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(scale) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(scale)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_scale_f32(OperatorAccess::values(input), - OperatorAccess::values(scale)[0], - OperatorAccess::values(output), - OperatorAccess::length(output)); - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + apply_scale(OperatorAccess::values(input), OperatorAccess::values(scale)[0], + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->input_value = OperatorAccess::values(input); - record->input_gradient = OperatorAccess::gradients(input); - record->scale_value = OperatorAccess::values(scale); - record->scale_gradient = OperatorAccess::gradients(scale); - record->length = OperatorAccess::length(output); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->scale_gradient = OperatorAccess::gradients(scale); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class ScaleExpression +template class ScaleExpression { public: - ScaleExpression(const BufferView &input, const BufferView &scale) noexcept + ScaleExpression(const BufferView &input, const BufferView &scale) noexcept : input_(input), scale_(scale) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - ScaleOperator::evaluate(output, input_, scale_); + ScaleOperator::evaluate(output, input_, scale_); } private: - BufferView input_; - BufferView scale_; + BufferView input_; + BufferView scale_; }; -inline ScaleExpression scale(const BufferView &input, - const BufferView &constant) noexcept +template +inline ScaleExpression scale(const BufferView &input, + const BufferView &constant) noexcept { - return ScaleExpression(input, constant); + return ScaleExpression(input, constant); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp index a5dcb7690..6c4db270c 100644 --- a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp @@ -5,30 +5,95 @@ #include #include +#include #include +#include #include +#include #include +#include + +#include namespace arm_cmsis_dsp { namespace autodiff { -class SoftmaxOperator +template class SoftmaxOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static T logsumexp(const T *input, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + return arm_logsumexp_f32(input, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return arm_logsumexp_f16(input, static_cast(length)); +#else + else + { + float sum = 0.0F; + for (std::size_t i = 0; i < length; ++i) + sum += std::exp(static_cast(input[i])); + return static_cast(std::log(sum)); + } +#endif + } + + static void offset(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_offset_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_offset_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] + value; +#endif + } + + static void exp(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_vexp_f32(data, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_vexp_f16(data, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + data[i] = static_cast(std::exp(static_cast(data[i]))); +#endif + } + struct Record { detail::Node node; - float *output_gradient; - const float *output_value; - float *input_gradient; + T *output_gradient; + const T *output_value; + T *input_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.input_gradient != nullptr) - arm_fill_f32(0.0F, record.input_gradient, record.length); + fill(record.input_gradient, record.length); } static void backward(detail::Node &node) noexcept @@ -36,74 +101,73 @@ class SoftmaxOperator Record &record = reinterpret_cast(node); if (record.input_gradient == nullptr) return; - ::arm_cmsis_dsp::VectorView output_gradient( + ::arm_cmsis_dsp::VectorView output_gradient( record.output_gradient, 0, record.length); - ::arm_cmsis_dsp::VectorView output_value( - const_cast(record.output_value), 0, record.length); - ::arm_cmsis_dsp::VectorView input_gradient( + ::arm_cmsis_dsp::VectorView output_value( + const_cast(record.output_value), 0, record.length); + ::arm_cmsis_dsp::VectorView input_gradient( record.input_gradient, 0, record.length); - const float projection = + const T projection = ::arm_cmsis_dsp::dot(output_gradient, output_value); input_gradient += output_value * (output_gradient - projection); } public: - static bool evaluate(BufferView &output, const BufferView &input) noexcept + static bool evaluate(BufferView &output, const BufferView &input) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input)) + if (!OperatorAccess::compatible(*tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(input)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - const std::size_t length = OperatorAccess::length(input); + const std::size_t length = OperatorAccess::length(input); if (length == 0U) - return OperatorAccess::status(*tape) == Status::ok; - - const float log_sum = arm_logsumexp_f32( - OperatorAccess::values(input), length); - arm_offset_f32(OperatorAccess::values(input), -log_sum, - OperatorAccess::values(output), length); - arm_vexp_f32(OperatorAccess::values(output), - OperatorAccess::values(output), length); - if (!OperatorAccess::recording(*tape)) - return OperatorAccess::status(*tape) == Status::ok; - - Record *record = OperatorAccess::append(*tape, backward, reset); + return OperatorAccess::status(*tape) == Status::ok; + + const T log_sum = logsumexp(OperatorAccess::values(input), length); + offset(OperatorAccess::values(input), static_cast(-static_cast(log_sum)), + OperatorAccess::values(output), length); + exp(OperatorAccess::values(output), length); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->output_value = OperatorAccess::values(output); - record->input_gradient = OperatorAccess::gradients(input); + record->output_gradient = OperatorAccess::gradients(output); + record->output_value = OperatorAccess::values(output); + record->input_gradient = OperatorAccess::gradients(input); record->length = length; - OperatorAccess::set_producer(output, &record->node); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class SoftmaxExpression +template class SoftmaxExpression { public: - explicit SoftmaxExpression(const BufferView &input) noexcept + explicit SoftmaxExpression(const BufferView &input) noexcept : input_(input) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - SoftmaxOperator::evaluate(output, input_); + SoftmaxOperator::evaluate(output, input_); } private: - BufferView input_; + BufferView input_; }; -inline SoftmaxExpression softmax(const BufferView &input) noexcept +template +inline SoftmaxExpression softmax(const BufferView &input) noexcept { - return SoftmaxExpression(input); + return SoftmaxExpression(input); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/operators/sub.hpp b/dsppp/Include/dsppp/autodiff/operators/sub.hpp index 6b637262e..724e409e6 100644 --- a/dsppp/Include/dsppp/autodiff/operators/sub.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/sub.hpp @@ -3,100 +3,143 @@ #include #include +#include #include +#include namespace arm_cmsis_dsp { namespace autodiff { -class SubOperator +template class SubOperator { + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void sub(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_sub_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_sub_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] - right[i]; +#endif + } + struct Record { detail::Node node; - float *output_gradient; - float *left_gradient; - float *right_gradient; + T *output_gradient; + T *left_gradient; + T *right_gradient; std::size_t length; }; static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - arm_fill_f32(0.0F, record.output_gradient, record.length); + fill(record.output_gradient, record.length); if (record.left_gradient != nullptr) - arm_fill_f32(0.0F, record.left_gradient, record.length); + fill(record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_fill_f32(0.0F, record.right_gradient, record.length); + fill(record.right_gradient, record.length); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); if (record.left_gradient != nullptr) - arm_add_f32(record.left_gradient, record.output_gradient, - record.left_gradient, record.length); + add(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); if (record.right_gradient != nullptr) - arm_sub_f32(record.right_gradient, record.output_gradient, - record.right_gradient, record.length); + sub(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); } public: - static bool evaluate(BufferView &output, const BufferView &left, - const BufferView &right) noexcept + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::require(*tape)) + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(left) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(right)) + if (!OperatorAccess::compatible(*tape, output, left) || + !OperatorAccess::compatible(*tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == + OperatorAccess::gradients(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } - arm_sub_f32(OperatorAccess::values(left), OperatorAccess::values(right), - OperatorAccess::values(output), - OperatorAccess::length(output)); - if (!OperatorAccess::recording(*tape) || - OperatorAccess::length(output) == 0U) - return OperatorAccess::status(*tape) == Status::ok; + sub(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; - Record *record = OperatorAccess::append(*tape, backward, reset); + Record *record = OperatorAccess::template append(*tape, backward, reset); if (record == nullptr) return false; - record->output_gradient = OperatorAccess::gradients(output); - record->left_gradient = OperatorAccess::gradients(left); - record->right_gradient = OperatorAccess::gradients(right); - record->length = OperatorAccess::length(output); - OperatorAccess::set_producer(output, &record->node); + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); return true; } }; -class SubExpression +template class SubExpression { public: - SubExpression(const BufferView &left, const BufferView &right) noexcept + SubExpression(const BufferView &left, const BufferView &right) noexcept : left_(left), right_(right) {} - void evaluate(BufferView &output) const noexcept + void evaluate(BufferView &output) const noexcept { - SubOperator::evaluate(output, left_, right_); + SubOperator::evaluate(output, left_, right_); } private: - BufferView left_; - BufferView right_; + BufferView left_; + BufferView right_; }; -inline SubExpression operator-(const BufferView &left, - const BufferView &right) noexcept +template +inline SubExpression operator-(const BufferView &left, + const BufferView &right) noexcept { - return SubExpression(left, right); + return SubExpression(left, right); } } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp index ee48103ac..0bc6ff557 100644 --- a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp +++ b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp @@ -6,37 +6,82 @@ #include #include +#include namespace arm_cmsis_dsp { namespace autodiff { /** Fixed-storage Adam optimizer for use in a user-written training loop. */ template + std::size_t MaximumParameters = 16U, + typename T = float> class Adam { static_assert(MaximumElements > 0U, "Adam needs state storage"); static_assert(MaximumParameters > 0U, "Adam needs parameter slots"); + + static constexpr T default_epsilon() noexcept + { + // 1e-8 is below the useful range of half precision and rounds to zero. + if constexpr (std::is_same::value) + return T{1.0e-8F}; + return T{1.0e-4F}; + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static T add(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left + right; + return static_cast(static_cast(left) + static_cast(right)); + } + static T sub(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left - right; + return static_cast(static_cast(left) - static_cast(right)); + } + static T mul(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left * right; + return static_cast(static_cast(left) * static_cast(right)); + } + static T div(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left / right; + return static_cast(static_cast(left) / static_cast(right)); + } + struct Entry { - float *values; - float *gradients; + T *values; + T *gradients; std::size_t length; std::size_t offset; bool trainable; }; public: - explicit Adam(float learning_rate = 1.0e-3F, float beta1 = 0.9F, - float beta2 = 0.999F, float epsilon = 1.0e-8F) noexcept + explicit Adam(T learning_rate = T{1.0e-3F}, T beta1 = T{0.9F}, + T beta2 = T{0.999F}, T epsilon = default_epsilon()) noexcept : learning_rate_(learning_rate), beta1_(beta1), beta2_(beta2), - epsilon_(epsilon), beta1_power_(1.0F), beta2_power_(1.0F), + epsilon_(epsilon), beta1_power_(T{1}), beta2_power_(T{1}), parameter_count_(0U), element_count_(0U), status_(OptimizerStatus::ok), entries_{}, first_moment_{}, second_moment_{} { } - bool add(BufferView parameter) noexcept + bool add(BufferView parameter) noexcept { if (parameter.role() != BufferRole::parameter || !parameter.has_gradient()) @@ -45,17 +90,17 @@ class Adam parameter.length()); } - bool add(MatrixView parameter) noexcept + bool add(MatrixView parameter) noexcept { return add_impl(parameter.values(), parameter.gradients(), parameter.length()); } - bool freeze(BufferView parameter, bool frozen = true) noexcept + bool freeze(BufferView parameter, bool frozen = true) noexcept { return set_trainable(parameter.values(), !frozen); } - bool freeze(MatrixView parameter, bool frozen = true) noexcept + bool freeze(MatrixView parameter, bool frozen = true) noexcept { return set_trainable(parameter.values(), !frozen); } @@ -63,40 +108,46 @@ class Adam void zero_grad() noexcept { for (std::size_t p = 0; p < parameter_count_; ++p) - arm_fill_f32(0.0F, entries_[p].gradients, - entries_[p].length); + fill(entries_[p].gradients, entries_[p].length); } bool step() noexcept { if (status_ != OptimizerStatus::ok) return false; - beta1_power_ *= beta1_; - beta2_power_ *= beta2_; - const float first_correction = 1.0F - beta1_power_; - const float second_correction = 1.0F - beta2_power_; + beta1_power_ = mul(beta1_power_, beta1_); + beta2_power_ = mul(beta2_power_, beta2_); + const T first_correction = + static_cast(1.0F - static_cast(beta1_power_)); + const T second_correction = + static_cast(1.0F - static_cast(beta2_power_)); + const T one_minus_beta1 = + static_cast(1.0F - static_cast(beta1_)); + const T one_minus_beta2 = + static_cast(1.0F - static_cast(beta2_)); for (std::size_t p = 0; p < parameter_count_; ++p) { Entry &entry = entries_[p]; if (!entry.trainable) continue; - ::arm_cmsis_dsp::VectorView gradients( + ::arm_cmsis_dsp::VectorView gradients( entry.gradients, 0, entry.length); - ::arm_cmsis_dsp::VectorView first_moment( + ::arm_cmsis_dsp::VectorView first_moment( first_moment_ + entry.offset, 0, entry.length); - ::arm_cmsis_dsp::VectorView second_moment( + ::arm_cmsis_dsp::VectorView second_moment( second_moment_ + entry.offset, 0, entry.length); - first_moment = first_moment * beta1_ + - gradients * (1.0F - beta1_); + first_moment = first_moment * beta1_ + gradients * one_minus_beta1; second_moment = second_moment * beta2_ + - gradients * gradients * (1.0F - beta2_); + gradients * gradients * one_minus_beta2; for (std::size_t i = 0; i < entry.length; ++i) { const std::size_t state = entry.offset + i; - const float corrected_first = - first_moment_[state] / first_correction; - const float corrected_second = - second_moment_[state] / second_correction; - entry.values[i] -= learning_rate_ * corrected_first / - (std::sqrt(corrected_second) + epsilon_); + const T corrected_first = div(first_moment_[state], first_correction); + const T corrected_second = div(second_moment_[state], second_correction); + const T denominator = add( + static_cast(std::sqrt(static_cast(corrected_second))), + epsilon_); + entry.values[i] = sub( + entry.values[i], + div(mul(learning_rate_, corrected_first), denominator)); } } return true; @@ -106,7 +157,7 @@ class Adam bool good() const noexcept { return status_ == OptimizerStatus::ok; } private: - bool add_impl(float *values, float *gradients, std::size_t length) noexcept + bool add_impl(T *values, T *gradients, std::size_t length) noexcept { if (values == nullptr || gradients == nullptr) return fail(OptimizerStatus::invalid_parameter); @@ -122,7 +173,7 @@ class Adam return true; } - bool set_trainable(const float *values, bool trainable) noexcept + bool set_trainable(const T *values, bool trainable) noexcept { for (std::size_t i = 0; i < parameter_count_; ++i) if (entries_[i].values == values) @@ -139,18 +190,18 @@ class Adam return false; } - float learning_rate_; - float beta1_; - float beta2_; - float epsilon_; - float beta1_power_; - float beta2_power_; + T learning_rate_; + T beta1_; + T beta2_; + T epsilon_; + T beta1_power_; + T beta2_power_; std::size_t parameter_count_; std::size_t element_count_; OptimizerStatus status_; Entry entries_[MaximumParameters]; - float first_moment_[MaximumElements]; - float second_moment_[MaximumElements]; + T first_moment_[MaximumElements]; + T second_moment_[MaximumElements]; }; } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp index 7d5a95fe4..f03ff10c6 100644 --- a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp +++ b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp @@ -6,37 +6,80 @@ #include #include +#include namespace arm_cmsis_dsp { namespace autodiff { /** Fixed-storage RMSProp optimizer without momentum or centering. */ template + std::size_t MaximumParameters = 16U, + typename T = float> class RMSProp { static_assert(MaximumElements > 0U, "RMSProp needs state storage"); static_assert(MaximumParameters > 0U, "RMSProp needs parameter slots"); + static constexpr T default_epsilon() noexcept + { + // 1e-8 is below the useful range of half precision and rounds to zero. + if constexpr (std::is_same::value) + return T{1.0e-8F}; + return T{1.0e-4F}; + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + static T add(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left + right; + return static_cast(static_cast(left) + static_cast(right)); + } + static T sub(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left - right; + return static_cast(static_cast(left) - static_cast(right)); + } + static T mul(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left * right; + return static_cast(static_cast(left) * static_cast(right)); + } + static T div(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left / right; + return static_cast(static_cast(left) / static_cast(right)); + } + struct Entry { - float *values; - float *gradients; + T *values; + T *gradients; std::size_t length; std::size_t offset; bool trainable; }; public: - explicit RMSProp(float learning_rate = 1.0e-3F, float alpha = 0.99F, - float epsilon = 1.0e-8F) noexcept + explicit RMSProp(T learning_rate = T{1.0e-3F}, T alpha = T{0.99F}, + T epsilon = default_epsilon()) noexcept : learning_rate_(learning_rate), alpha_(alpha), epsilon_(epsilon), parameter_count_(0U), element_count_(0U), status_(OptimizerStatus::ok), entries_{}, square_average_{} { } - bool add(BufferView parameter) noexcept + bool add(BufferView parameter) noexcept { if (parameter.role() != BufferRole::parameter || !parameter.has_gradient()) @@ -44,16 +87,16 @@ class RMSProp return add_impl(parameter.values(), parameter.gradients(), parameter.length()); } - bool add(MatrixView parameter) noexcept + bool add(MatrixView parameter) noexcept { return add_impl(parameter.values(), parameter.gradients(), parameter.length()); } - bool freeze(BufferView parameter, bool frozen = true) noexcept + bool freeze(BufferView parameter, bool frozen = true) noexcept { return set_trainable(parameter.values(), !frozen); } - bool freeze(MatrixView parameter, bool frozen = true) noexcept + bool freeze(MatrixView parameter, bool frozen = true) noexcept { return set_trainable(parameter.values(), !frozen); } @@ -61,8 +104,7 @@ class RMSProp void zero_grad() noexcept { for (std::size_t p = 0; p < parameter_count_; ++p) - arm_fill_f32(0.0F, entries_[p].gradients, - entries_[p].length); + fill(entries_[p].gradients, entries_[p].length); } bool step() noexcept @@ -72,18 +114,25 @@ class RMSProp { Entry &entry = entries_[p]; if (!entry.trainable) continue; - ::arm_cmsis_dsp::VectorView gradients( + ::arm_cmsis_dsp::VectorView gradients( entry.gradients, 0, entry.length); - ::arm_cmsis_dsp::VectorView square_average( + ::arm_cmsis_dsp::VectorView square_average( square_average_ + entry.offset, 0, entry.length); + const T one_minus_alpha = + static_cast(1.0F - static_cast(alpha_)); square_average = square_average * alpha_ + - gradients * gradients * (1.0F - alpha_); + gradients * gradients * one_minus_alpha; for (std::size_t i = 0; i < entry.length; ++i) { const std::size_t state = entry.offset + i; - const float gradient = entry.gradients[i]; - entry.values[i] -= learning_rate_ * gradient / - (std::sqrt(square_average_[state]) + epsilon_); + const T gradient = entry.gradients[i]; + const T denominator = add( + static_cast(std::sqrt( + static_cast(square_average_[state]))), + epsilon_); + entry.values[i] = sub( + entry.values[i], + div(mul(learning_rate_, gradient), denominator)); } } return true; @@ -93,7 +142,7 @@ class RMSProp bool good() const noexcept { return status_ == OptimizerStatus::ok; } private: - bool add_impl(float *values, float *gradients, std::size_t length) noexcept + bool add_impl(T *values, T *gradients, std::size_t length) noexcept { if (values == nullptr || gradients == nullptr) return fail(OptimizerStatus::invalid_parameter); @@ -108,7 +157,7 @@ class RMSProp element_count_ += length; return true; } - bool set_trainable(const float *values, bool trainable) noexcept + bool set_trainable(const T *values, bool trainable) noexcept { for (std::size_t i = 0; i < parameter_count_; ++i) if (entries_[i].values == values) @@ -124,14 +173,14 @@ class RMSProp return false; } - float learning_rate_; - float alpha_; - float epsilon_; + T learning_rate_; + T alpha_; + T epsilon_; std::size_t parameter_count_; std::size_t element_count_; OptimizerStatus status_; Entry entries_[MaximumParameters]; - float square_average_[MaximumElements]; + T square_average_[MaximumElements]; }; } // namespace autodiff diff --git a/dsppp/Include/dsppp/autodiff/reverse.hpp b/dsppp/Include/dsppp/autodiff/reverse.hpp index a42ac6247..4560593a2 100644 --- a/dsppp/Include/dsppp/autodiff/reverse.hpp +++ b/dsppp/Include/dsppp/autodiff/reverse.hpp @@ -16,8 +16,8 @@ namespace autodiff { #define DSPPP_AUTODIFF_MAX_OPERATORS 16 #endif -class Tape; -class OperatorAccess; +template class Tape; +template class OperatorAccess; namespace detail { @@ -55,7 +55,7 @@ enum class BufferRole }; /** Non-owning view of caller values and an associated gradient buffer. */ -class BufferView +template class BufferView { public: BufferView() noexcept @@ -64,22 +64,22 @@ class BufferView { } - float *values() noexcept { return values_; } - const float *values() const noexcept { return values_; } - float *gradients() noexcept { return gradients_; } - const float *gradients() const noexcept { return gradients_; } + T *values() noexcept { return values_; } + const T *values() const noexcept { return values_; } + T *gradients() noexcept { return gradients_; } + const T *gradients() const noexcept { return gradients_; } std::size_t length() const noexcept { return length_; } BufferRole role() const noexcept { return role_; } bool has_gradient() const noexcept { return gradients_ != nullptr; } - float &operator[](std::size_t index) noexcept { return values_[index]; } - const float &operator[](std::size_t index) const noexcept + T &operator[](std::size_t index) noexcept { return values_[index]; } + const T &operator[](std::size_t index) const noexcept { return values_[index]; } - float gradient(std::size_t index) const noexcept + T gradient(std::size_t index) const noexcept { - return gradients_ == nullptr ? 0.0F : gradients_[index]; + return gradients_ == nullptr ? T{} : gradients_[index]; } /** Evaluate any expression supplied by a separately included operator. */ @@ -91,68 +91,68 @@ class BufferView } private: - BufferView(float *values, float *gradients, std::size_t length, - Tape *tape, BufferRole role) noexcept + BufferView(T *values, T *gradients, std::size_t length, + Tape *tape, BufferRole role) noexcept : values_(values), gradients_(gradients), length_(length), tape_(tape), producer_(nullptr), role_(role) { } - float *values_; - float *gradients_; + T *values_; + T *gradients_; std::size_t length_; - Tape *tape_; + Tape *tape_; detail::Node *producer_; BufferRole role_; - friend class Tape; - friend class OperatorAccess; + friend class Tape; + friend class OperatorAccess; }; /** Non-owning row-major matrix parameter view. */ -class MatrixView +template class MatrixView { public: MatrixView() noexcept : buffer_(), rows_(0U), columns_(0U) {} std::size_t rows() const noexcept { return rows_; } std::size_t columns() const noexcept { return columns_; } - float *values() noexcept { return buffer_.values(); } - const float *values() const noexcept { return buffer_.values(); } - float *gradients() noexcept { return buffer_.gradients(); } - const float *gradients() const noexcept { return buffer_.gradients(); } + T *values() noexcept { return buffer_.values(); } + const T *values() const noexcept { return buffer_.values(); } + T *gradients() noexcept { return buffer_.gradients(); } + const T *gradients() const noexcept { return buffer_.gradients(); } std::size_t length() const noexcept { return rows_ * columns_; } - float &operator()(std::size_t row, std::size_t column) noexcept + T &operator()(std::size_t row, std::size_t column) noexcept { return buffer_.values()[row * columns_ + column]; } - const float &operator()(std::size_t row, - std::size_t column) const noexcept + const T &operator()(std::size_t row, + std::size_t column) const noexcept { return buffer_.values()[row * columns_ + column]; } - float gradient(std::size_t row, std::size_t column) const noexcept + T gradient(std::size_t row, std::size_t column) const noexcept { return buffer_.gradient(row * columns_ + column); } private: - MatrixView(const BufferView &buffer, std::size_t rows, + MatrixView(const BufferView &buffer, std::size_t rows, std::size_t columns) noexcept : buffer_(buffer), rows_(rows), columns_(columns) { } - BufferView buffer_; + BufferView buffer_; std::size_t rows_; std::size_t columns_; - friend class Tape; - friend class OperatorAccess; + friend class Tape; + friend class OperatorAccess; }; /** Reverse-mode tape using caller-supplied storage and a fixed operator list. */ -class Tape +template class Tape { public: static constexpr std::size_t maximum_registered_operators = @@ -267,185 +267,185 @@ class Tape void set_recording(bool enabled) noexcept { recording_ = enabled; } /** Generic active view; output() is clearer for application code. */ - BufferView view(float *values, std::size_t length) noexcept + BufferView view(T *values, std::size_t length) noexcept { if (length != 0U && values == nullptr) { set_error(Status::tape_mismatch); - return BufferView(values, nullptr, length, this, + return BufferView(values, nullptr, length, this, BufferRole::intermediate); } - if (length > static_cast(-1) / sizeof(float)) + if (length > static_cast(-1) / sizeof(T)) { set_error(Status::out_of_memory); - return BufferView(values, nullptr, length, this, + return BufferView(values, nullptr, length, this, BufferRole::intermediate); } - float *gradients = nullptr; + T *gradients = nullptr; if (length != 0U) { - gradients = static_cast( - allocate(length * sizeof(float), alignof(float))); + gradients = static_cast( + allocate(length * sizeof(T), alignof(T))); if (gradients != nullptr) { for (std::size_t i = 0; i < length; ++i) { - gradients[i] = 0.0F; + gradients[i] = T{}; } } } - return BufferView(values, gradients, length, this, + return BufferView(values, gradients, length, this, BufferRole::intermediate); } template - BufferView view(float (&values)[Length]) noexcept + BufferView view(T (&values)[Length]) noexcept { return view(values, Length); } - BufferView view(float *values, float *gradients, + BufferView view(T *values, T *gradients, std::size_t length) noexcept { if (length != 0U && (values == nullptr || gradients == nullptr)) { set_error(Status::tape_mismatch); } - return BufferView(values, gradients, length, this, + return BufferView(values, gradients, length, this, BufferRole::intermediate); } template - BufferView view(float (&values)[Length], - float (&gradients)[Length]) noexcept + BufferView view(T (&values)[Length], + T (&gradients)[Length]) noexcept { return view(values, gradients, Length); } - BufferView input(float *values, std::size_t length) noexcept + BufferView input(T *values, std::size_t length) noexcept { if (length != 0U && values == nullptr) { set_error(Status::tape_mismatch); } - return BufferView(values, nullptr, length, this, BufferRole::input); + return BufferView(values, nullptr, length, this, BufferRole::input); } template - BufferView input(float (&values)[Length]) noexcept + BufferView input(T (&values)[Length]) noexcept { return input(values, Length); } - BufferView input(float &value) noexcept { return input(&value, 1U); } + BufferView input(T &value) noexcept { return input(&value, 1U); } - BufferView parameter(float *values, std::size_t length) noexcept + BufferView parameter(T *values, std::size_t length) noexcept { - BufferView result = view(values, length); + BufferView result = view(values, length); result.role_ = BufferRole::parameter; return result; } template - BufferView parameter(float (&values)[Length]) noexcept + BufferView parameter(T (&values)[Length]) noexcept { return parameter(values, Length); } - BufferView parameter(float &value) noexcept + BufferView parameter(T &value) noexcept { return parameter(&value, 1U); } - BufferView parameter(float *values, float *gradients, + BufferView parameter(T *values, T *gradients, std::size_t length) noexcept { - BufferView result = view(values, gradients, length); + BufferView result = view(values, gradients, length); result.role_ = BufferRole::parameter; return result; } template - BufferView parameter(float (&values)[Length], - float (&gradients)[Length]) noexcept + BufferView parameter(T (&values)[Length], + T (&gradients)[Length]) noexcept { return parameter(values, gradients, Length); } - BufferView parameter(float &value, float &gradient) noexcept + BufferView parameter(T &value, T &gradient) noexcept { return parameter(&value, &gradient, 1U); } - MatrixView parameter(float *values, std::size_t rows, + MatrixView parameter(T *values, std::size_t rows, std::size_t columns) noexcept { if (columns != 0U && rows > static_cast(-1) / columns) { set_error(Status::out_of_memory); - return MatrixView(); + return MatrixView(); } - return MatrixView(parameter(values, rows * columns), rows, columns); + return MatrixView(parameter(values, rows * columns), rows, columns); } template - MatrixView parameter(float (&values)[Rows][Columns]) noexcept + MatrixView parameter(T (&values)[Rows][Columns]) noexcept { return parameter(&values[0][0], Rows, Columns); } - MatrixView parameter(float *values, float *gradients, std::size_t rows, + MatrixView parameter(T *values, T *gradients, std::size_t rows, std::size_t columns) noexcept { if (columns != 0U && rows > static_cast(-1) / columns) { set_error(Status::out_of_memory); - return MatrixView(); + return MatrixView(); } - return MatrixView(parameter(values, gradients, rows * columns), rows, - columns); + return MatrixView(parameter(values, gradients, rows * columns), rows, + columns); } template - MatrixView parameter(float (&values)[Rows][Columns], - float (&gradients)[Rows][Columns]) noexcept + MatrixView parameter(T (&values)[Rows][Columns], + T (&gradients)[Rows][Columns]) noexcept { return parameter(&values[0][0], &gradients[0][0], Rows, Columns); } - BufferView output(float *values, std::size_t length) noexcept + BufferView output(T *values, std::size_t length) noexcept { return view(values, length); } template - BufferView output(float (&values)[Length]) noexcept + BufferView output(T (&values)[Length]) noexcept { return output(values, Length); } - BufferView output(float &value) noexcept { return output(&value, 1U); } + BufferView output(T &value) noexcept { return output(&value, 1U); } - BufferView output(float *values, float *gradients, + BufferView output(T *values, T *gradients, std::size_t length) noexcept { return view(values, gradients, length); } template - BufferView output(float (&values)[Length], - float (&gradients)[Length]) noexcept + BufferView output(T (&values)[Length], + T (&gradients)[Length]) noexcept { return output(values, gradients, Length); } - BufferView output(float &value, float &gradient) noexcept + BufferView output(T &value, T &gradient) noexcept { return output(&value, &gradient, 1U); } - bool backward(const BufferView &output, float seed = 1.0F) noexcept + bool backward(const BufferView &output, T seed = T{1}) noexcept { if (output.length_ != 1U) { @@ -455,7 +455,7 @@ class Tape return backward(output, &seed, 1U); } - bool backward(const BufferView &output, const float *seed, + bool backward(const BufferView &output, const T *seed, std::size_t seed_length) noexcept { if (status_ != Status::ok) @@ -497,7 +497,7 @@ class Tape (maximum_registered_operators - 1U); } - bool valid(const BufferView &view) const noexcept + bool valid(const BufferView &view) const noexcept { return view.tape_ == this && (view.length_ == 0U || @@ -572,66 +572,66 @@ class Tape std::size_t registered_count_; const void *registered_operators_[maximum_registered_operators]; - friend class OperatorAccess; + friend class OperatorAccess; }; /** Narrow internal interface used by independently defined operators. */ -class OperatorAccess +template class OperatorAccess { public: - static Tape *tape(const BufferView &view) noexcept { return view.tape_; } - static float *values(BufferView &view) noexcept { return view.values_; } - static const float *values(const BufferView &view) noexcept + static Tape *tape(const BufferView &view) noexcept { return view.tape_; } + static T *values(BufferView &view) noexcept { return view.values_; } + static const T *values(const BufferView &view) noexcept { return view.values_; } - static float *gradients(const BufferView &view) noexcept + static T *gradients(const BufferView &view) noexcept { return view.gradients_; } - static std::size_t length(const BufferView &view) noexcept + static std::size_t length(const BufferView &view) noexcept { return view.length_; } - static BufferRole role(const BufferView &view) noexcept { return view.role_; } - static detail::Node *producer(const BufferView &view) noexcept + static BufferRole role(const BufferView &view) noexcept { return view.role_; } + static detail::Node *producer(const BufferView &view) noexcept { return view.producer_; } - static void set_producer(BufferView &view, detail::Node *node) noexcept + static void set_producer(BufferView &view, detail::Node *node) noexcept { view.producer_ = node; } - static bool valid(const Tape &tape, const BufferView &view) noexcept + static bool valid(const Tape &tape, const BufferView &view) noexcept { return tape.valid(view); } - static bool compatible(const Tape &tape, const BufferView &left, - const BufferView &right) noexcept + static bool compatible(const Tape &tape, const BufferView &left, + const BufferView &right) noexcept { return valid(tape, left) && valid(tape, right) && length(left) == length(right); } - static const BufferView &buffer(const MatrixView &matrix) noexcept + static const BufferView &buffer(const MatrixView &matrix) noexcept { return matrix.buffer_; } - static std::size_t rows(const MatrixView &matrix) noexcept + static std::size_t rows(const MatrixView &matrix) noexcept { return matrix.rows_; } - static std::size_t columns(const MatrixView &matrix) noexcept + static std::size_t columns(const MatrixView &matrix) noexcept { return matrix.columns_; } - static bool recording(const Tape &tape) noexcept { return tape.recording_; } - static Status status(const Tape &tape) noexcept { return tape.status_; } - static void fail(Tape &tape, Status status) noexcept { tape.set_error(status); } + static bool recording(const Tape &tape) noexcept { return tape.recording_; } + static Status status(const Tape &tape) noexcept { return tape.status_; } + static void fail(Tape &tape, Status status) noexcept { tape.set_error(status); } template - static bool require(Tape &tape) noexcept + static bool require(Tape &tape) noexcept { - if (!tape.is_operator_registered()) + if (!tape.template is_operator_registered()) { tape.set_error(Status::operator_not_registered); return false; @@ -640,17 +640,17 @@ class OperatorAccess } template - static Record *append(Tape &tape, detail::BackwardFunction backward, + static Record *append(Tape &tape, detail::BackwardFunction backward, detail::ResetFunction reset_gradient) noexcept { - return tape.append(backward, reset_gradient); + return tape.template append(backward, reset_gradient); } }; -class RecordingScope +template class RecordingScope { public: - RecordingScope(Tape &tape, bool enabled) noexcept + RecordingScope(Tape &tape, bool enabled) noexcept : tape_(tape), previous_(tape.recording()) { tape_.set_recording(enabled); @@ -660,11 +660,11 @@ class RecordingScope RecordingScope &operator=(const RecordingScope &) = delete; private: - Tape &tape_; + Tape &tape_; bool previous_; }; -template +template class Arena { public: @@ -672,12 +672,12 @@ class Arena Arena() noexcept : storage_{}, tape_(storage_, Bytes) {} Arena(const Arena &) = delete; Arena &operator=(const Arena &) = delete; - Tape &tape() noexcept { return tape_; } - const Tape &tape() const noexcept { return tape_; } + Tape &tape() noexcept { return tape_; } + const Tape &tape() const noexcept { return tape_; } private: alignas(std::max_align_t) unsigned char storage_[Bytes]; - Tape tape_; + Tape tape_; }; } // namespace autodiff diff --git a/dsppp/test.cbuild-idx.yml b/dsppp/test.cbuild-idx.yml index 211b64606..c3fb5d96d 100644 --- a/dsppp/test.cbuild-idx.yml +++ b/dsppp/test.cbuild-idx.yml @@ -17,7 +17,7 @@ build-idx: info: - test.cbuild-pack.yml - file is already up-to-date - test+VHT-Corstone-300.cbuild-run.yml - file is already up-to-date - - example.Release+VHT-Corstone-300.cbuild.yml - file is already up-to-date + - example.Release+VHT-Corstone-300.cbuild.yml - file generated successfully packs-unused: - pack: ARM::CMSIS-Compiler@2.2.0 - pack: ARM::Cortex_DFP@1.2.0 diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index f668546ca..89f4d67d4 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -50,9 +50,9 @@ static void test1() // Vector add followed by dot. Values and outputs belong to the caller; // gradients and two fixed-size operation records use the tape arena. Arena<2048> *buffer_arena = new Arena<2048>(); - Tape &buffer_tape = buffer_arena->tape(); - buffer_tape.register_operator(); - buffer_tape.register_operator(); + Tape &buffer_tape = buffer_arena->tape(); + buffer_tape.register_operator>(); + buffer_tape.register_operator>(); float x_value[] = {1.0F, 2.0F, 3.0F}; float w_value[] = {4.0F, 5.0F, 6.0F}; float sum_value[3] = {}; @@ -107,10 +107,10 @@ void test2() // Only parameters receive final gradients. The x input has no gradient // allocation, while scale and offset are trainable scalar parameters. Arena<2048> *parameter_arena = new Arena<2048>(); - Tape ¶meter_tape = parameter_arena->tape(); - parameter_tape.register_operator(); - parameter_tape.register_operator(); - parameter_tape.register_operator(); + Tape ¶meter_tape = parameter_arena->tape(); + parameter_tape.register_operator>(); + parameter_tape.register_operator>(); + parameter_tape.register_operator>(); float input_value[] = {1.0F, 2.0F, 3.0F}; float alpha_value = 2.0F; float beta_value = 10.0F; @@ -150,9 +150,9 @@ void test3() // Fully connected followed by ReLU. Only the positive first neuron // contributes to the matrix and bias parameter gradients. Arena<2048> *network_arena = new Arena<2048>(); - Tape &network_tape = network_arena->tape(); - network_tape.register_operator(); - network_tape.register_operator(); + Tape &network_tape = network_arena->tape(); + network_tape.register_operator>(); + network_tape.register_operator>(); float network_input_value[] = {2.0F, -1.0F}; float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; float bias_value[] = {1.0F, 0.0F}; @@ -190,8 +190,8 @@ void test4() { // ReLU uses a zero derivative at exactly zero. Arena<512> *relu_arena = new Arena<512>(); - Tape &relu_tape = relu_arena->tape(); - relu_tape.register_operator(); + Tape &relu_tape = relu_arena->tape(); + relu_tape.register_operator>(); float relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; float relu_output_value[3] = {}; BufferView relu_parameter = relu_tape.parameter(relu_parameter_value); @@ -211,8 +211,8 @@ void test5() // Softmax is normalized and its vector-Jacobian product has zero sum. Arena<512> *softmax_arena = new Arena<512>(); - Tape &softmax_tape = softmax_arena->tape(); - softmax_tape.register_operator(); + Tape &softmax_tape = softmax_arena->tape(); + softmax_tape.register_operator>(); float logits_value[] = {0.0F, 0.0F, 0.0F}; float probability_value[3] = {}; BufferView logits = softmax_tape.parameter(logits_value); @@ -241,7 +241,7 @@ void test6() // Including an operator does not enable it. Evaluation fails until that // operator type is explicitly registered on this tape. Arena<256> *registry_arena = new Arena<256>(); - Tape ®istry_tape = registry_arena->tape(); + Tape ®istry_tape = registry_arena->tape(); float registry_left_value[] = {1.0F}; float registry_right_value[] = {2.0F}; float registry_output_value[] = {0.0F}; @@ -252,7 +252,7 @@ void test6() assert(registry_tape.status() == Status::operator_not_registered); assert(registry_output_value[0] == 0.0F); registry_tape.clear_status(); - assert(registry_tape.register_operator()); + assert(registry_tape.register_operator>()); registry_output = registry_left + registry_right; assert(registry_tape.good()); assert(registry_output_value[0] == 3.0F); @@ -264,10 +264,10 @@ void test7() // Quadratic loss, reusable graph records, Adam, and selective freezing. Arena<1024> *training_arena = new Arena<1024>(); - Tape &training_tape = training_arena->tape(); - training_tape.register_operator(); - training_tape.register_operator(); - training_tape.register_operator(); + Tape &training_tape = training_arena->tape(); + training_tape.register_operator>(); + training_tape.register_operator>(); + training_tape.register_operator>(); float feature_value[] = {2.0F}; float coefficient_value[] = {3.0F}; float bias_parameter_value[] = {1.0F}; @@ -311,10 +311,10 @@ void test8() // A single vector loss accumulates contributions from every sample into // shared polynomial parameters before an optimizer step. Arena<2048> *batch_arena = new Arena<2048>(); - Tape &batch_tape = batch_arena->tape(); - batch_tape.register_operator(); - batch_tape.register_operator(); - batch_tape.register_operator(); + Tape &batch_tape = batch_arena->tape(); + batch_tape.register_operator>(); + batch_tape.register_operator>(); + batch_tape.register_operator>(); float batch_feature_value[2][1] = {{1.0F}, {2.0F}}; float batch_coefficient_value = 3.0F; float batch_bias_value = 1.0F; @@ -356,7 +356,7 @@ void test9() { // RMSProp uses the same parameter registration and freezing API. Arena<128> *rms_arena = new Arena<128>(); - Tape &rms_tape = rms_arena->tape(); + Tape &rms_tape = rms_arena->tape(); float rms_value = 1.0F; BufferView rms_parameter = rms_tape.parameter(rms_value); RMSProp<1> rmsprop(1.0e-2F); @@ -371,9 +371,9 @@ void test10() { // Subtraction and elementwise multiplication have data operands only. Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); + tape.register_operator>(); float left_value[] = {2.0F, 4.0F, 6.0F}; float left_gradient[3] = {}; float right_value[] = {1.0F, 2.0F, 3.0F}; @@ -517,8 +517,8 @@ void test11() // Exercise the same kernel through the fully connected backward pass. Arena<4096> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); float input_value[5] = {}; float input_gradient[5] = {}; float weight_gradient[3][5] = {}; @@ -541,8 +541,8 @@ void test12() { // Categorical cross entropy consumes probabilities and a one-hot target. Arena<512> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); float probability_value[] = {0.1F, 0.7F, 0.2F}; float target_value[] = {0.0F, 1.0F, 0.0F}; float loss_value = 0.0F; @@ -563,8 +563,8 @@ void test13() // Training applies inverted dropout and backward regenerates the same // mask. Disabling recording makes dropout an identity for inference. Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); float input_value[16]; float output_value[16] = {}; for (std::size_t i = 0; i < 16U; ++i) input_value[i] = 1.0F; @@ -601,8 +601,8 @@ void test14() // Y = W X uses CMSIS-DSP matrix multiplication in the forward pass and // computes only dW = dY X^T in the backward pass. Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator(); + Tape &tape = arena.tape(); + tape.register_operator>(); float weight_value[2][3] = { {1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}}; float input_value[3][2] = { @@ -648,7 +648,7 @@ static void run_autodiff_tests() // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; Tape tiny(tiny_memory, sizeof(tiny_memory)); - tiny.register_operator(); + tiny.register_operator>(); float tiny_input_value[1] = {2.0F}; float tiny_input_gradient[1] = {}; float tiny_output_value[1] = {}; @@ -662,6 +662,75 @@ static void run_autodiff_tests() assert(tiny.status() == Status::out_of_memory); assert(!tiny.backward(tiny_output)); +#if defined(ARM_FLOAT16_SUPPORTED) + // The same graph can be instantiated with CMSIS-DSP float16 kernels. + Arena<512, float16_t> half_arena; + Tape &half_tape = half_arena.tape(); + half_tape.register_operator>(); + half_tape.register_operator>(); + float16_t half_input_value[2] = {static_cast(1.0F), + static_cast(2.0F)}; + float16_t half_scale_value = static_cast(3.0F); + float16_t half_scaled_value[2] = {}; + float16_t half_loss_value = {}; + BufferView half_input = half_tape.input(half_input_value); + BufferView half_scale = half_tape.parameter(half_scale_value); + BufferView half_scaled = half_tape.output(half_scaled_value); + BufferView half_loss = half_tape.output(half_loss_value); + half_scaled = scale(half_input, half_scale); + half_loss = dot(half_scaled, half_input); + assert(half_tape.backward(half_loss)); + assert(static_cast(half_loss_value) > 14.9F && + static_cast(half_loss_value) < 15.1F); + assert(static_cast(half_scale.gradient(0)) > 4.9F && + static_cast(half_scale.gradient(0)) < 5.1F); + + // ReLU and categorical cross entropy use finite float16 clip bounds. + // In particular, numeric_limits<__fp16>::max() is not specialized by all + // embedded C++ libraries and can otherwise evaluate to zero. + Arena<1024, float16_t> half_classification_arena; + Tape &half_classification_tape = + half_classification_arena.tape(); + half_classification_tape.register_operator>(); + half_classification_tape.register_operator>(); + half_classification_tape.register_operator>(); + float16_t half_relu_input_value[3] = { + static_cast(-1.0F), static_cast(0.5F), + static_cast(2.0F)}; + float16_t half_relu_output_value[3] = {}; + float16_t half_logits_value[3] = { + static_cast(0.2F), static_cast(-0.1F), + static_cast(0.3F)}; + float16_t half_probability_value[3] = {}; + float16_t half_target_value[3] = { + static_cast(0.0F), static_cast(1.0F), + static_cast(0.0F)}; + float16_t half_classification_loss_value = {}; + BufferView half_relu_input = + half_classification_tape.input(half_relu_input_value); + BufferView half_relu_output = + half_classification_tape.output(half_relu_output_value); + BufferView half_logits = + half_classification_tape.parameter(half_logits_value); + BufferView half_probability = + half_classification_tape.output(half_probability_value); + BufferView half_target = + half_classification_tape.input(half_target_value); + BufferView half_classification_loss = + half_classification_tape.output(half_classification_loss_value); + half_relu_output = relu(half_relu_input); + assert(static_cast(half_relu_output_value[1]) > 0.49F && + static_cast(half_relu_output_value[2]) > 1.9F); + half_probability = softmax(half_logits); + half_classification_loss = cross_entropy(half_probability, half_target); + assert(static_cast(half_classification_loss_value) > 0.9F && + static_cast(half_classification_loss_value) < 1.3F); + assert(half_classification_tape.backward(half_classification_loss)); + assert(static_cast(half_logits.gradient(0)) < 0.5F && + static_cast(half_logits.gradient(1)) < -0.2F && + static_cast(half_logits.gradient(2)) > 0.2F); +#endif + } #undef assert From a0800c570b7a6bed43a5d7096ce5042c18d86743 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Thu, 13 Aug 2026 07:22:53 +0200 Subject: [PATCH 13/19] Refactor validation in autodiff operators --- dsppp/Include/dsppp/autodiff/doc/concepts.md | 4 +- .../dsppp/autodiff/doc/implementation_flow.md | 7 +-- dsppp/Include/dsppp/autodiff/doc/operators.md | 22 +++++--- .../Include/dsppp/autodiff/operators/add.hpp | 29 ++++++---- .../autodiff/operators/cross_entropy.hpp | 29 ++++++---- .../Include/dsppp/autodiff/operators/dot.hpp | 29 ++++++---- .../dsppp/autodiff/operators/dropout.hpp | 28 ++++++---- .../autodiff/operators/fully_connected.hpp | 36 ++++++++----- .../autodiff/operators/matrix_multiply.hpp | 53 ++++++++++++++----- .../dsppp/autodiff/operators/multiply.hpp | 33 +++++++----- .../dsppp/autodiff/operators/offset.hpp | 35 +++++++----- .../autodiff/operators/quadratic_error.hpp | 29 ++++++---- .../Include/dsppp/autodiff/operators/relu.hpp | 24 ++++++--- .../dsppp/autodiff/operators/scale.hpp | 35 +++++++----- .../dsppp/autodiff/operators/softmax.hpp | 25 ++++++--- .../Include/dsppp/autodiff/operators/sub.hpp | 33 +++++++----- dsppp/Include/dsppp/autodiff/reverse.hpp | 4 ++ dsppp/test.cbuild-idx.yml | 2 +- 18 files changed, 309 insertions(+), 148 deletions(-) diff --git a/dsppp/Include/dsppp/autodiff/doc/concepts.md b/dsppp/Include/dsppp/autodiff/doc/concepts.md index cac8147af..db0b8df92 100644 --- a/dsppp/Include/dsppp/autodiff/doc/concepts.md +++ b/dsppp/Include/dsppp/autodiff/doc/concepts.md @@ -120,8 +120,8 @@ The first error is sticky until `clear_status()`, `reset()`, `begin_graph()`, or Boolean result of `backward()` before using gradients. - `out_of_memory`: a gradient or operation record did not fit in the arena. -- `tape_mismatch`: views, roles, shapes, pointers, or aliasing are invalid for - an operation. +- `tape_mismatch`: with `DSPPP_AUTODIFF_ENABLE_VALIDATION=1`, views, roles, + shapes, pointers, or aliasing are invalid for an operation. - `invalid_output`: the backward root or seed is invalid, or graph rewinding was requested without a mark. - `operator_not_registered`: an expression's operator was not registered. diff --git a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md index 6c553c4dc..45df59e16 100644 --- a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md +++ b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md @@ -129,9 +129,10 @@ executes these steps: 4. The operator clears `y.producer_`. This prevents an old record from being mistaken for the producer if validation or allocation fails. 5. `OperatorAccess::require(tape)` checks the fixed registry. -6. Validation checks that all views belong to the same tape; `x` and `y` have - the same length; `a` is a one-element parameter with a gradient; and value - and gradient buffers do not alias illegally. +6. When `DSPPP_AUTODIFF_ENABLE_VALIDATION=1`, validation checks that all views + belong to the same tape; `x` and `y` have the same length; `a` is a + one-element parameter with a gradient; and value and gradient buffers do + not alias illegally. These checks are compiled out by default. 7. `arm_scale_f32` computes `{3*2, 3*(-1)}` into `y_value`, giving `{6, -3}`. 8. Because recording is enabled and the output is nonempty, the operator asks `OperatorAccess::append` for a `ScaleOperator::Record` in the arena. diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md index c64a8eba6..190f834e1 100644 --- a/dsppp/Include/dsppp/autodiff/doc/operators.md +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -193,9 +193,19 @@ start at zero. ## Common validation rules -Views used by one expression must belong to the same tape and satisfy the -operator's role and shape requirements. Output and input value storage must be -distinct, and output gradient storage must not alias an input gradient. An -invalid combination sets the sticky `Status::tape_mismatch`. A successfully -computed value has a producer only when recording is enabled and its record was -appended successfully. +Operator argument validation is disabled by default so it adds no overhead to +the evaluation path. Define `DSPPP_AUTODIFF_ENABLE_VALIDATION` to `1` before +including the autodiff headers, or define it consistently for the complete +build, to enable these checks. + +When enabled, views used by one expression must belong to the same tape and +satisfy the operator's role and shape requirements. Output and input value +storage must be distinct, and output gradient storage must not alias an input +gradient. An invalid combination sets the sticky +`Status::tape_mismatch`. When validation is disabled, the application is +responsible for meeting these preconditions; invalid arguments may cause +out-of-bounds access or otherwise undefined results. + +A successfully computed value has a producer only when recording is enabled +and its record was appended successfully. Operator registration and arena +allocation failures are checked in both validation configurations. diff --git a/dsppp/Include/dsppp/autodiff/operators/add.hpp b/dsppp/Include/dsppp/autodiff/operators/add.hpp index c0d3f9d3c..e07bc835a 100644 --- a/dsppp/Include/dsppp/autodiff/operators/add.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/add.hpp @@ -69,25 +69,36 @@ template class AddOperator record.right_gradient, record.length); } -public: - static bool evaluate(BufferView &output, const BufferView &left, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::template require>(*tape)) - return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || OperatorAccess::gradients(output) == nullptr || OperatorAccess::values(output) == OperatorAccess::values(left) || OperatorAccess::values(output) == OperatorAccess::values(right) || OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif add(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::values(output), OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) diff --git a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp index fb15e7d59..db9551691 100644 --- a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp @@ -143,6 +143,23 @@ template class CrossEntropyOperator } } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &probability, + const BufferView &target) noexcept + { + if (!OperatorAccess::valid(tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(tape, probability, target) || + OperatorAccess::gradients(probability) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &probability, const BufferView &target) noexcept @@ -152,16 +169,10 @@ template class CrossEntropyOperator if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - OperatorAccess::length(output) != 1U || - OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, probability, target) || - OperatorAccess::gradients(probability) == nullptr || - OperatorAccess::role(target) != BufferRole::input) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, probability, target)) return false; - } +#endif const std::size_t length = OperatorAccess::length(probability); T result = T{}; diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp index abf6ea8e1..6a8ce61ff 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dot.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -89,24 +89,35 @@ template class DotOperator } } -public: - static bool evaluate(BufferView &output, const BufferView &left, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, const BufferView &right) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::template require>(*tape)) - return false; - if (!OperatorAccess::valid(*tape, output) || + if (!OperatorAccess::valid(tape, output) || OperatorAccess::length(output) != 1U || OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, left, right) || + !OperatorAccess::compatible(tape, left, right) || OperatorAccess::values(output) == OperatorAccess::values(left) || OperatorAccess::values(output) == OperatorAccess::values(right)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif T value = T{}; dot(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::length(left), &value); diff --git a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp index 4d97f4b5d..fc36a4d16 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp @@ -135,6 +135,22 @@ template class DropoutOperator } } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + float drop_probability) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + !(drop_probability >= 0.0F && drop_probability < 1.0F)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &input, DropoutGenerator &generator, @@ -144,16 +160,10 @@ template class DropoutOperator OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || - !(drop_probability >= 0.0F && drop_probability < 1.0F)) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, drop_probability)) return false; - } +#endif const std::size_t length = OperatorAccess::length(output); if (!OperatorAccess::recording(*tape)) diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp index b180c3ea7..feeb43282 100644 --- a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -96,21 +96,16 @@ template class FullyConnectedOperator } } -public: - static bool evaluate(BufferView &output, const BufferView &input, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, const MatrixView &weights, const BufferView &bias) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || - !OperatorAccess::template require>(*tape)) - return false; const BufferView &weight_buffer = OperatorAccess::buffer(weights); - if (!OperatorAccess::valid(*tape, output) || - !OperatorAccess::valid(*tape, input) || - !OperatorAccess::valid(*tape, weight_buffer) || - !OperatorAccess::valid(*tape, bias) || + if (!OperatorAccess::valid(tape, output) || + !OperatorAccess::valid(tape, input) || + !OperatorAccess::valid(tape, weight_buffer) || + !OperatorAccess::valid(tape, bias) || OperatorAccess::gradients(output) == nullptr || OperatorAccess::length(output) != OperatorAccess::rows(weights) || OperatorAccess::length(input) != OperatorAccess::columns(weights) || @@ -122,10 +117,27 @@ template class FullyConnectedOperator OperatorAccess::role(weight_buffer) != BufferRole::parameter || OperatorAccess::role(bias) != BufferRole::parameter) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights, + const BufferView &bias) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, weights, bias)) + return false; +#endif + const BufferView &weight_buffer = OperatorAccess::buffer(weights); const std::size_t rows = OperatorAccess::rows(weights); const std::size_t columns = OperatorAccess::columns(weights); if (rows != 0U) diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp index 76ee9c7fb..ef2f2ca9b 100644 --- a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -72,24 +72,18 @@ template class MatrixMultiplyOperator } } -public: - static bool evaluate(BufferView &output, const BufferView &input, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, const MatrixView &weights) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || - !OperatorAccess::template require>(*tape)) - return false; - const BufferView &weight_buffer = OperatorAccess::buffer(weights); const std::size_t rows = OperatorAccess::rows(weights); const std::size_t inner = OperatorAccess::columns(weights); const std::size_t input_length = OperatorAccess::length(input); const std::size_t columns = inner == 0U ? 0U : input_length / inner; - if (!OperatorAccess::valid(*tape, output) || - !OperatorAccess::valid(*tape, input) || - !OperatorAccess::valid(*tape, weight_buffer) || + if (!OperatorAccess::valid(tape, output) || + !OperatorAccess::valid(tape, input) || + !OperatorAccess::valid(tape, weight_buffer) || OperatorAccess::gradients(output) == nullptr || OperatorAccess::role(input) != BufferRole::input || OperatorAccess::role(weight_buffer) != BufferRole::parameter || @@ -101,11 +95,34 @@ template class MatrixMultiplyOperator columns > std::numeric_limits::max() || OperatorAccess::length(output) != rows * columns) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, weights)) + return false; +#endif + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t inner = OperatorAccess::columns(weights); + const std::size_t input_length = OperatorAccess::length(input); + const std::size_t columns = inner == 0U ? 0U : input_length / inner; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION arm_status matrix_status; +#endif if constexpr (std::is_same::value) { arm_matrix_instance_f32 weight_matrix; @@ -120,7 +137,10 @@ template class MatrixMultiplyOperator arm_mat_init_f32(&output_matrix, static_cast(rows), static_cast(columns), OperatorAccess::values(output)); - matrix_status = arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + matrix_status = +#endif + arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix); } else { @@ -136,13 +156,18 @@ template class MatrixMultiplyOperator arm_mat_init_f16(&output_matrix, static_cast(rows), static_cast(columns), OperatorAccess::values(output)); - matrix_status = arm_mat_mult_f16(&weight_matrix, &input_matrix, &output_matrix); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + matrix_status = +#endif + arm_mat_mult_f16(&weight_matrix, &input_matrix, &output_matrix); } +#if DSPPP_AUTODIFF_ENABLE_VALIDATION if (matrix_status != ARM_MATH_SUCCESS) { OperatorAccess::fail(*tape, Status::tape_mismatch); return false; } +#endif if (!OperatorAccess::recording(*tape)) return OperatorAccess::status(*tape) == Status::ok; diff --git a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp index f6f1b0bf9..70947c76b 100644 --- a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp @@ -85,6 +85,24 @@ template class MultiplyOperator } } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &left, const BufferView &right) noexcept @@ -93,19 +111,10 @@ template class MultiplyOperator OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(left) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(right)) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) return false; - } +#endif multiply(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::values(output), OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || diff --git a/dsppp/Include/dsppp/autodiff/operators/offset.hpp b/dsppp/Include/dsppp/autodiff/operators/offset.hpp index 99f1ff40a..e13eff0b9 100644 --- a/dsppp/Include/dsppp/autodiff/operators/offset.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/offset.hpp @@ -102,30 +102,39 @@ template class OffsetOperator static_cast(gradient_sum)); } -public: - static bool evaluate(BufferView &output, const BufferView &input, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, const BufferView &offset) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::template require>(*tape)) - return false; - if (!OperatorAccess::compatible(*tape, output, input) || - !OperatorAccess::valid(*tape, offset) || + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, offset) || OperatorAccess::length(offset) != 1U || OperatorAccess::role(offset) != BufferRole::parameter || OperatorAccess::gradients(output) == nullptr || OperatorAccess::gradients(offset) == nullptr || OperatorAccess::values(output) == OperatorAccess::values(input) || OperatorAccess::values(output) == OperatorAccess::values(offset) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(offset)) + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(offset)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &offset) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, offset)) + return false; +#endif apply_offset(OperatorAccess::values(input), OperatorAccess::values(offset)[0], OperatorAccess::values(output), OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || diff --git a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp index 699986cc2..4c1feb260 100644 --- a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp @@ -58,6 +58,23 @@ template class QuadraticErrorOperator (prediction_value - target_value) * static_cast(2.0F * static_cast(seed)); } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &prediction, + const BufferView &target) noexcept + { + if (!OperatorAccess::valid(tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(tape, prediction, target) || + OperatorAccess::gradients(prediction) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &prediction, const BufferView &target) noexcept @@ -67,16 +84,10 @@ template class QuadraticErrorOperator if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::valid(*tape, output) || - OperatorAccess::length(output) != 1U || - OperatorAccess::gradients(output) == nullptr || - !OperatorAccess::compatible(*tape, prediction, target) || - OperatorAccess::gradients(prediction) == nullptr || - OperatorAccess::role(target) != BufferRole::input) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, prediction, target)) return false; - } +#endif const std::size_t length = OperatorAccess::length(prediction); ::arm_cmsis_dsp::VectorView prediction_value( diff --git a/dsppp/Include/dsppp/autodiff/operators/relu.hpp b/dsppp/Include/dsppp/autodiff/operators/relu.hpp index 304c05d19..9f661eeb3 100644 --- a/dsppp/Include/dsppp/autodiff/operators/relu.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/relu.hpp @@ -91,6 +91,20 @@ template class ReluOperator } } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &input) noexcept { @@ -98,14 +112,10 @@ template class ReluOperator OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input)) return false; - } +#endif clip(OperatorAccess::values(input), OperatorAccess::values(output), OperatorAccess::length(input)); if (!OperatorAccess::recording(*tape) || diff --git a/dsppp/Include/dsppp/autodiff/operators/scale.hpp b/dsppp/Include/dsppp/autodiff/operators/scale.hpp index 14669d12c..c177a565f 100644 --- a/dsppp/Include/dsppp/autodiff/operators/scale.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/scale.hpp @@ -83,30 +83,39 @@ template class ScaleOperator } } -public: - static bool evaluate(BufferView &output, const BufferView &input, + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, const BufferView &scale) noexcept { - Tape *tape = OperatorAccess::tape(output); - OperatorAccess::set_producer(output, nullptr); - if (tape == nullptr || !OperatorAccess::template require>(*tape)) - return false; - if (!OperatorAccess::compatible(*tape, output, input) || - !OperatorAccess::valid(*tape, scale) || + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || OperatorAccess::length(scale) != 1U || OperatorAccess::role(scale) != BufferRole::parameter || OperatorAccess::gradients(output) == nullptr || OperatorAccess::gradients(scale) == nullptr || OperatorAccess::values(output) == OperatorAccess::values(input) || OperatorAccess::values(output) == OperatorAccess::values(scale) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(scale)) + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(scale)) { - OperatorAccess::fail(*tape, Status::tape_mismatch); + OperatorAccess::fail(tape, Status::tape_mismatch); return false; } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale)) + return false; +#endif apply_scale(OperatorAccess::values(input), OperatorAccess::values(scale)[0], OperatorAccess::values(output), OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || diff --git a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp index 6c4db270c..4689d3b59 100644 --- a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp @@ -112,6 +112,20 @@ template class SoftmaxOperator input_gradient += output_value * (output_gradient - projection); } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &input) noexcept { @@ -119,15 +133,10 @@ template class SoftmaxOperator OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, input) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(input) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(input)) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input)) return false; - } +#endif const std::size_t length = OperatorAccess::length(input); if (length == 0U) diff --git a/dsppp/Include/dsppp/autodiff/operators/sub.hpp b/dsppp/Include/dsppp/autodiff/operators/sub.hpp index 724e409e6..6c3c5abce 100644 --- a/dsppp/Include/dsppp/autodiff/operators/sub.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/sub.hpp @@ -83,6 +83,24 @@ template class SubOperator record.right_gradient, record.length); } + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + public: static bool evaluate(BufferView &output, const BufferView &left, const BufferView &right) noexcept @@ -91,19 +109,10 @@ template class SubOperator OperatorAccess::set_producer(output, nullptr); if (tape == nullptr || !OperatorAccess::template require>(*tape)) return false; - if (!OperatorAccess::compatible(*tape, output, left) || - !OperatorAccess::compatible(*tape, output, right) || - OperatorAccess::gradients(output) == nullptr || - OperatorAccess::values(output) == OperatorAccess::values(left) || - OperatorAccess::values(output) == OperatorAccess::values(right) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(left) || - OperatorAccess::gradients(output) == - OperatorAccess::gradients(right)) - { - OperatorAccess::fail(*tape, Status::tape_mismatch); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) return false; - } +#endif sub(OperatorAccess::values(left), OperatorAccess::values(right), OperatorAccess::values(output), OperatorAccess::length(output)); if (!OperatorAccess::recording(*tape) || diff --git a/dsppp/Include/dsppp/autodiff/reverse.hpp b/dsppp/Include/dsppp/autodiff/reverse.hpp index 4560593a2..4f01f4c76 100644 --- a/dsppp/Include/dsppp/autodiff/reverse.hpp +++ b/dsppp/Include/dsppp/autodiff/reverse.hpp @@ -16,6 +16,10 @@ namespace autodiff { #define DSPPP_AUTODIFF_MAX_OPERATORS 16 #endif +#ifndef DSPPP_AUTODIFF_ENABLE_VALIDATION +#define DSPPP_AUTODIFF_ENABLE_VALIDATION 0 +#endif + template class Tape; template class OperatorAccess; diff --git a/dsppp/test.cbuild-idx.yml b/dsppp/test.cbuild-idx.yml index c3fb5d96d..211b64606 100644 --- a/dsppp/test.cbuild-idx.yml +++ b/dsppp/test.cbuild-idx.yml @@ -17,7 +17,7 @@ build-idx: info: - test.cbuild-pack.yml - file is already up-to-date - test+VHT-Corstone-300.cbuild-run.yml - file is already up-to-date - - example.Release+VHT-Corstone-300.cbuild.yml - file generated successfully + - example.Release+VHT-Corstone-300.cbuild.yml - file is already up-to-date packs-unused: - pack: ARM::CMSIS-Compiler@2.2.0 - pack: ARM::Cortex_DFP@1.2.0 From ab59af93eab8f5215973eadde51b754caa29784d Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Thu, 13 Aug 2026 07:47:02 +0200 Subject: [PATCH 14/19] autodiff : Add SGD optimizer and LMS filter example --- dsppp/Examples/README.md | 4 + dsppp/Examples/autodiff_lms.cpp | 124 +++++++++++++++ dsppp/Include/dsppp/autodiff/README.md | 7 +- .../Include/dsppp/autodiff/doc/optimizers.md | 42 +++-- .../Include/dsppp/autodiff/operators/dot.hpp | 9 +- .../Include/dsppp/autodiff/optimizers/sgd.hpp | 143 ++++++++++++++++++ dsppp/example.cproject.yml | 1 + dsppp/tests/autodiff_test.cpp | 37 +++++ 8 files changed, 351 insertions(+), 16 deletions(-) create mode 100644 dsppp/Examples/autodiff_lms.cpp create mode 100644 dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp diff --git a/dsppp/Examples/README.md b/dsppp/Examples/README.md index 781691833..af85ea7cc 100644 --- a/dsppp/Examples/README.md +++ b/dsppp/Examples/README.md @@ -19,6 +19,10 @@ in `dsppp/example.cproject.yml` by commenting and uncommenting its `file` line. - `autodiff_regression.cpp` trains a cubic polynomial to approximate a sine wave. It demonstrates RMSProp, reusable graphs, parameter freezing, and saving model parameters. +- `autodiff_lms.cpp` identifies an unknown FIR filter with a per-sample LMS + update expressed as quadratic error, reverse differentiation, and SGD. It is + an educational demonstration; the specialized CMSIS-DSP LMS functions are + more efficient for production filtering. - `autodiff_iris.cpp` trains a small classifier on the Iris flower dataset. It uses Adam and reserves 30 of the 150 samples for a final test that is not used during training. diff --git a/dsppp/Examples/autodiff_lms.cpp b/dsppp/Examples/autodiff_lms.cpp new file mode 100644 index 000000000..1c79cd375 --- /dev/null +++ b/dsppp/Examples/autodiff_lms.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include + +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +/* + * Educational example only. + * + * This example deliberately expresses a least mean square (LMS) adaptive + * filter with the generic autodiff API to show how per-sample reverse-mode + * differentiation becomes online stochastic gradient descent. For a real LMS + * filter, a specialized implementation such as arm_lms_f32 is more efficient: + * it combines FIR state handling, output calculation, error calculation, and + * coefficient updates directly, without tape records or a separate backward + * pass. arm_lms_f32 accepts blocks but updates its coefficients after every + * sample inside the block. + */ + +static float next_excitation(std::uint32_t &state) noexcept +{ + state = state * 1664525U + 1013904223U; + const std::uint32_t value = state >> 8; + return static_cast(value) * (2.0F / 16777215.0F) - 1.0F; +} + +int main() +{ + constexpr std::size_t tap_count = 4U; + constexpr std::size_t sample_count = 2000U; + constexpr std::size_t measurement_count = 100U; + constexpr float mu = 0.05F; + + // Unknown FIR system that the adaptive filter will identify. Element zero + // multiplies the newest sample, element one the preceding sample, and so on. + const float reference_coefficients[tap_count] = { + 0.6F, -0.3F, 0.2F, 0.1F}; + float coefficients_value[tap_count] = {}; + float input_state_value[tap_count] = {}; + float desired_value[1] = {}; + float output_value[1] = {}; + float loss_value[1] = {}; + + Arena<512> arena; + Tape &tape = arena.tape(); + if (!tape.register_operator>() || + !tape.register_operator>()) + { + std::printf("Failed to register autodiff operators\n"); + return 1; + } + + BufferView coefficients = tape.parameter(coefficients_value); + BufferView input_state = tape.input(input_state_value); + BufferView desired = tape.input(desired_value); + BufferView output = tape.output(output_value); + BufferView loss = tape.output(loss_value); + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + return 1; + } + + // quadratic_error computes (output - desired)^2, whose coefficient + // gradient contains a factor of two. learning_rate = mu/2 therefore gives + // the classical LMS update: coefficient += mu * error * input_state. + SGD optimizer(mu * 0.5F); + if (!optimizer.add(coefficients)) + { + std::printf("Failed to add LMS coefficients to SGD\n"); + return 1; + } + + tape.begin_graph(); + std::uint32_t random_state = 1U; + float final_error_sum = 0.0F; + + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + // Maintain the FIR delay line in caller-owned memory. + for (std::size_t tap = tap_count - 1U; tap > 0U; --tap) + input_state_value[tap] = input_state_value[tap - 1U]; + input_state_value[0] = next_excitation(random_state); + + desired_value[0] = 0.0F; + for (std::size_t tap = 0; tap < tap_count; ++tap) + desired_value[0] += + reference_coefficients[tap] * input_state_value[tap]; + + // Each sample creates a fresh two-node graph. Rewinding releases the + // preceding sample's records while retaining all persistent views and + // gradient buffers. + if (!tape.rewind_graph()) return 1; + output = dot(input_state, coefficients); + loss = quadratic_error(output, desired); + + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if (sample >= sample_count - measurement_count) + final_error_sum += loss_value[0]; + } + + std::printf("reference coefficients: {%g, %g, %g, %g}\n", + static_cast(reference_coefficients[0]), + static_cast(reference_coefficients[1]), + static_cast(reference_coefficients[2]), + static_cast(reference_coefficients[3])); + std::printf("learned coefficients: {%g, %g, %g, %g}\n", + static_cast(coefficients_value[0]), + static_cast(coefficients_value[1]), + static_cast(coefficients_value[2]), + static_cast(coefficients_value[3])); + std::printf("mean squared error over final %u samples: %g\n", + static_cast(measurement_count), + static_cast(final_error_sum / measurement_count)); + + return tape.good() && optimizer.good() ? 0 : 1; +} diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 02829fc7e..5af47793d 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -131,7 +131,7 @@ int main() - [Operators](doc/operators.md) documents the current operator families, formulas, shape rules, dropout behavior, and the CMSIS-DSP implementation paths. -- [Optimizers](doc/optimizers.md) documents Adam and RMSProp capacities, +- [Optimizers](doc/optimizers.md) documents SGD, Adam, and RMSProp capacities, initialization, updates, freezing, and errors. - [Worked implementation flow: `y = a * x`](doc/implementation_flow.md) follows one expression through `reverse.hpp`, including its tape record, `producer`, @@ -140,8 +140,9 @@ int main() ## Examples and tests `dsppp/Examples/autodiff_regression.cpp` trains a polynomial approximation to -`sin(x)` with RMSProp. `dsppp/Examples/autodiff_iris.cpp` trains a small fully -connected classifier with Adam. +`sin(x)` with RMSProp. `dsppp/Examples/autodiff_lms.cpp` demonstrates +per-sample LMS adaptation with SGD. `dsppp/Examples/autodiff_iris.cpp` trains a +small fully connected classifier with Adam. Autodiff uses the existing dsppp board-test infrastructure. From `dsppp`, run: diff --git a/dsppp/Include/dsppp/autodiff/doc/optimizers.md b/dsppp/Include/dsppp/autodiff/doc/optimizers.md index c3e62423e..95ae6139b 100644 --- a/dsppp/Include/dsppp/autodiff/doc/optimizers.md +++ b/dsppp/Include/dsppp/autodiff/doc/optimizers.md @@ -1,14 +1,14 @@ # Optimizers -`Adam` and `RMSProp` update caller-owned parameter values from tape-managed or -caller-owned gradients. Their third template argument selects the scalar type -(`float` by default, or `float16_t`), matching the tape views. All optimizer -metadata and numerical state are fixed arrays inside the optimizer object; -neither optimizer allocates memory. +`SGD`, `Adam`, and `RMSProp` update caller-owned parameter values from +tape-managed or caller-owned gradients. Their third template argument selects +the scalar type (`float` by default, or `float16_t`), matching the tape views. +All optimizer metadata and numerical state are fixed inside the optimizer +object; none of the optimizers allocates memory. ## Capacity arguments -Both types have the same capacity template arguments: +All three types have the same capacity template arguments: ```cpp Optimizer @@ -31,13 +31,37 @@ optimizer.add(parameter); A matrix also counts as one view, while all `rows*columns` entries count toward `MaximumElements`. A three-element coefficient vector plus a separate scalar -bias fits exactly in either `RMSProp<4, 2>` or `Adam<4, 2>`. Writing +bias fits exactly in `SGD<4, 2>`, `RMSProp<4, 2>`, or `Adam<4, 2>`. Writing `Adam<100>` reserves 100 scalar state positions and the default 16 view slots. For half precision, use for example `Adam<100, 16, float16_t>`. Adding the same value pointer twice is idempotent. A frozen parameter continues to occupy both capacities. +## SGD + +```cpp +SGD<4, 2> optimizer(1.0e-3F); // learning_rate +``` + +SGD applies plain stochastic gradient descent without momentum: + +```text +parameter -= learning_rate * gradient +``` + +The implementation uses a CMSIS-DSP C++ vector expression that fuses scaling +and subtraction into one loop. It therefore needs no intermediate vector and +has no per-element optimizer state. `MaximumElements` still limits the total +number of registered scalar parameters, while +`Entry entries_[MaximumParameters]` holds their non-owning pointers, lengths, +and frozen state. + +For online learning, such as LMS adaptation, one `step()` after each sample is +stochastic gradient descent because the current sample gradient estimates the +gradient of the expected loss. With this library's unscaled quadratic error, +an LMS step size `mu` corresponds to an SGD learning rate of `mu / 2`. + ## RMSProp ```cpp @@ -176,6 +200,6 @@ Optimizer errors are sticky. After the first error, `good()` is false, Parameter values and gradient arrays are referenced, not copied. They must remain alive as long as the optimizer uses them. A checkpoint containing only parameter values is sufficient for inference. Reproducing the exact continuation -of training also requires saving the optimizer's moment state and, for Adam, +of training also requires saving RMSProp or Adam moment state and, for Adam, its step-dependent powers; the current optimizer classes do not provide a -serialization API. +serialization API. Plain SGD has no additional numerical state to save. diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp index 6a8ce61ff..02535aada 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dot.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -2,15 +2,16 @@ #include -#include -#include -#include - +#include #include #include #include #include +#include +#include +#include + namespace arm_cmsis_dsp { namespace autodiff { diff --git a/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp b/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp new file mode 100644 index 000000000..287750a73 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage stochastic gradient descent optimizer without momentum. */ +template +class SGD +{ + static_assert(MaximumElements > 0U, "SGD needs element capacity"); + static_assert(MaximumParameters > 0U, "SGD needs parameter slots"); + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + struct Entry + { + T *values; + T *gradients; + std::size_t length; + bool trainable; + }; + +public: + explicit SGD(T learning_rate = T{1.0e-3F}) noexcept + : learning_rate_(learning_rate), parameter_count_(0U), + element_count_(0U), status_(OptimizerStatus::ok), entries_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + fill(entries_[p].gradients, entries_[p].length); + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView values( + entry.values, 0, entry.length); + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + // The C++ expression fuses scaling and subtraction into one loop, + // avoiding the temporary buffer required by separate C kernels. + values -= gradients * learning_rate_; + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(T *values, T *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = Entry{values, gradients, length, true}; + element_count_ += length; + return true; + } + + bool set_trainable(const T *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + T learning_rate_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/example.cproject.yml b/dsppp/example.cproject.yml index 7c36cd1c2..54bdfc41d 100644 --- a/dsppp/example.cproject.yml +++ b/dsppp/example.cproject.yml @@ -6,6 +6,7 @@ project: #- file: Examples/vector_op.cpp #- file: Examples/matrix_op.cpp #- file: Examples/autodiff_regression.cpp + #- file: Examples/autodiff_lms.cpp - file: Examples/autodiff_iris.cpp - file: clang_sse300.c for-context: diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index 89f4d67d4..42ae36feb 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -22,6 +22,7 @@ extern "C" { #include #include #include +#include #include @@ -628,6 +629,29 @@ void test14() assert(!input.has_gradient()); } +void test15() +{ + // SGD performs one fused parameter -= learning_rate * gradient update. + Arena<128> arena; + Tape &tape = arena.tape(); + float value[] = {1.0F, -2.0F}; + BufferView parameter = tape.parameter(value); + SGD<2, 1> optimizer(0.25F); + assert(optimizer.add(parameter)); + parameter.gradients()[0] = 2.0F; + parameter.gradients()[1] = -4.0F; + assert(optimizer.step()); + assert(value[0] == 0.5F); + assert(value[1] == -1.0F); + optimizer.zero_grad(); + assert(parameter.gradient(0) == 0.0F); + assert(parameter.gradient(1) == 0.0F); + assert(freeze_parameters(optimizer, parameter)); + parameter.gradients()[0] = 1.0F; + assert(optimizer.step()); + assert(value[0] == 0.5F); +} + static void run_autodiff_tests() { test1(); @@ -644,6 +668,7 @@ static void run_autodiff_tests() test12(); test13(); test14(); + test15(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; @@ -685,6 +710,18 @@ static void run_autodiff_tests() assert(static_cast(half_scale.gradient(0)) > 4.9F && static_cast(half_scale.gradient(0)) < 5.1F); + float16_t half_sgd_value[] = {static_cast(1.0F), + static_cast(-2.0F)}; + BufferView half_sgd_parameter = + half_tape.parameter(half_sgd_value); + SGD<2, 1, float16_t> half_sgd(static_cast(0.25F)); + assert(half_sgd.add(half_sgd_parameter)); + half_sgd_parameter.gradients()[0] = static_cast(2.0F); + half_sgd_parameter.gradients()[1] = static_cast(-4.0F); + assert(half_sgd.step()); + assert(static_cast(half_sgd_value[0]) == 0.5F); + assert(static_cast(half_sgd_value[1]) == -1.0F); + // ReLU and categorical cross entropy use finite float16 clip bounds. // In particular, numeric_limits<__fp16>::max() is not specialized by all // embedded C++ libraries and can otherwise evaluate to zero. From 17a041d86f3622b8b048a046b713125e43f040e5 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Thu, 13 Aug 2026 08:06:06 +0200 Subject: [PATCH 15/19] Added f16 tests for autodiff --- dsppp/Include/dsppp/autodiff/README.md | 43 +- dsppp/main.c | 3 +- dsppp/tests/autodiff_test.cpp | 540 ++++++++++--------------- 3 files changed, 254 insertions(+), 332 deletions(-) diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 5af47793d..65901cd2a 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -18,11 +18,10 @@ providing predictable fixed memory use, no heap allocation, and no exceptions. Automatic differentiation needs high performance in both directions. The forward pass evaluates the model, while the backward pass propagates and -accumulates gradients. This implementation uses CMSIS-DSP for both rather than -treating it only as a collection of forward-inference kernels. +accumulates gradients. This implementation uses CMSIS-DSP for both. When an operation maps directly to an optimized CMSIS-DSP C kernel, its -forward pass uses the kernel matching `T`. For example, float32 dot products +forward pass uses the CMSIS-DSP kernel. For example, float32 dot products use `arm_dot_prod_f32` and float16 dot products use `arm_dot_prod_f16`; the corresponding fully connected and matrix products use the f32 or f16 matrix kernels. These kernels provide implementations optimized for the selected Arm @@ -60,6 +59,27 @@ optimized C kernels and fusible C++ expressions is especially important for training, because backward passes contain more compound updates and accumulations than typical forward inference code. +## Examples + +### Polynomial regression + +`dsppp/Examples/autodiff_regression.cpp` trains a cubic polynomial to +approximate `sin(x)`. It demonstrates a batch loss, RMSProp, graph reuse, +parameter freezing, and saving learned values. + +### LMS adaptive filter + +`dsppp/Examples/autodiff_lms.cpp` identifies an unknown FIR filter with a +per-sample quadratic loss and SGD update. It is an educational demonstration; +the specialized CMSIS-DSP LMS implementation is more efficient for production +filtering. + +### Iris classifier + +`dsppp/Examples/autodiff_iris.cpp` trains a small two-layer classifier with +Adam and tests it on 30 patterns excluded from training. A single macro selects +the float32 or float16 implementation. + ## How reverse differentiation works here During the **forward pass**, each operator computes its output and, when @@ -137,20 +157,15 @@ int main() one expression through `reverse.hpp`, including its tape record, `producer`, node links, gradient reset, seed, and backward rule. -## Examples and tests - -`dsppp/Examples/autodiff_regression.cpp` trains a polynomial approximation to -`sin(x)` with RMSProp. `dsppp/Examples/autodiff_lms.cpp` demonstrates -per-sample LMS adaptation with SGD. `dsppp/Examples/autodiff_iris.cpp` trains a -small fully connected classifier with Adam. +## Tests -Autodiff uses the existing dsppp board-test infrastructure. From `dsppp`, run: +Autodiff is tested through the usual dsppp C++ board-test framework. The same +type-generic suite is instantiated for the datatype selected by `F32_DT` or +`F16_DT`. From `dsppp`, run for example: ```sh python run_all.py --test AUTODIFF_TEST --dt F32_DT ``` -The API supports `float` and, on targets defining `ARM_FLOAT16_SUPPORTED`, -`float16_t`. The board test currently selects the float32 path when -`AUTODIFF_TEST`, `F32_DT`, and `DYNAMIC_TEST` are defined; the Iris example -instantiates the float16 path. +Use `--dt F16_DT` to run the float16 instantiation on a target defining +`ARM_FLOAT16_SUPPORTED`. diff --git a/dsppp/main.c b/dsppp/main.c index e0e47cb2e..a2d5a2320 100644 --- a/dsppp/main.c +++ b/dsppp/main.c @@ -58,7 +58,8 @@ int main(void) #if defined(DOT_TEST) dot_test(); #endif - #if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) + #if defined(AUTODIFF_TEST) && \ + (defined(F32_DT) || defined(F16_DT)) && defined(DYNAMIC_TEST) autodiff_test(); #endif #if defined(VECTOR_TEST) diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index 42ae36feb..bbbf12493 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -4,7 +4,8 @@ extern "C" { extern void autodiff_test(); } -#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) +#if defined(AUTODIFF_TEST) && defined(DYNAMIC_TEST) && \ + (defined(F32_DT) || defined(F16_DT)) #include #include @@ -46,25 +47,33 @@ using namespace arm_cmsis_dsp::autodiff; #endif #define assert(condition) AUTODIFF_CHECK(condition) +template +static bool close_to(T actual, float expected, float tolerance) noexcept +{ + const float difference = static_cast(actual) - expected; + return difference > -tolerance && difference < tolerance; +} + +template static void test1() { // Vector add followed by dot. Values and outputs belong to the caller; // gradients and two fixed-size operation records use the tape arena. - Arena<2048> *buffer_arena = new Arena<2048>(); - Tape &buffer_tape = buffer_arena->tape(); - buffer_tape.register_operator>(); - buffer_tape.register_operator>(); - float x_value[] = {1.0F, 2.0F, 3.0F}; - float w_value[] = {4.0F, 5.0F, 6.0F}; - float sum_value[3] = {}; - float result_value[1] = {}; + Arena<2048, T> *buffer_arena = new Arena<2048, T>(); + Tape &buffer_tape = buffer_arena->tape(); + buffer_tape.template register_operator>(); + buffer_tape.template register_operator>(); + T x_value[] = {1.0F, 2.0F, 3.0F}; + T w_value[] = {4.0F, 5.0F, 6.0F}; + T sum_value[3] = {}; + T result_value[1] = {}; BufferView x_view = buffer_tape.view(x_value); BufferView w_view = buffer_tape.view(w_value); BufferView sum_view = buffer_tape.view(sum_value); BufferView result_view = buffer_tape.view(result_value); const std::size_t gradients_end = buffer_tape.used(); - assert(gradients_end >= 10U * sizeof(float)); + assert(gradients_end >= 10U * sizeof(T)); @@ -90,7 +99,7 @@ static void test1() // A vector output accepts a caller-provided vector-Jacobian seed. - const float sum_seed[] = {1.0F, 2.0F, 3.0F}; + const T sum_seed[] = {1.0F, 2.0F, 3.0F}; assert(buffer_tape.backward(sum_view, sum_seed, 3)); for (std::size_t i = 0; i < 3; ++i) { @@ -101,23 +110,24 @@ static void test1() delete buffer_arena; } -void test2() +template +static void test2() { // Only parameters receive final gradients. The x input has no gradient // allocation, while scale and offset are trainable scalar parameters. - Arena<2048> *parameter_arena = new Arena<2048>(); - Tape ¶meter_tape = parameter_arena->tape(); - parameter_tape.register_operator>(); - parameter_tape.register_operator>(); - parameter_tape.register_operator>(); - float input_value[] = {1.0F, 2.0F, 3.0F}; - float alpha_value = 2.0F; - float beta_value = 10.0F; - float scaled_value[3] = {}; - float added_value[3] = {}; - float loss_value[1] = {}; + Arena<2048, T> *parameter_arena = new Arena<2048, T>(); + Tape ¶meter_tape = parameter_arena->tape(); + parameter_tape.template register_operator>(); + parameter_tape.template register_operator>(); + parameter_tape.template register_operator>(); + T input_value[] = {1.0F, 2.0F, 3.0F}; + T alpha_value = 2.0F; + T beta_value = 10.0F; + T scaled_value[3] = {}; + T added_value[3] = {}; + T loss_value[1] = {}; BufferView input_view = parameter_tape.input(input_value); const std::size_t after_input = parameter_tape.used(); @@ -145,20 +155,21 @@ void test2() delete parameter_arena; } -void test3() +template +static void test3() { // Fully connected followed by ReLU. Only the positive first neuron // contributes to the matrix and bias parameter gradients. - Arena<2048> *network_arena = new Arena<2048>(); - Tape &network_tape = network_arena->tape(); - network_tape.register_operator>(); - network_tape.register_operator>(); - float network_input_value[] = {2.0F, -1.0F}; - float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; - float bias_value[] = {1.0F, 0.0F}; - float linear_value[2] = {}; - float activation_value[2] = {}; + Arena<2048, T> *network_arena = new Arena<2048, T>(); + Tape &network_tape = network_arena->tape(); + network_tape.template register_operator>(); + network_tape.template register_operator>(); + T network_input_value[] = {2.0F, -1.0F}; + T matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; + T bias_value[] = {1.0F, 0.0F}; + T linear_value[2] = {}; + T activation_value[2] = {}; BufferView network_input = network_tape.input(network_input_value); MatrixView matrix = network_tape.parameter(matrix_value); @@ -173,7 +184,7 @@ void test3() assert(activation_value[0] == 1.0F); assert(activation_value[1] == 0.0F); - const float activation_seed[] = {1.0F, 1.0F}; + const T activation_seed[] = {1.0F, 1.0F}; assert(network_tape.backward(activation, activation_seed, 2)); assert(matrix.gradient(0, 0) == 2.0F); assert(matrix.gradient(0, 1) == -1.0F); @@ -187,18 +198,19 @@ void test3() } -void test4() +template +static void test4() { // ReLU uses a zero derivative at exactly zero. - Arena<512> *relu_arena = new Arena<512>(); - Tape &relu_tape = relu_arena->tape(); - relu_tape.register_operator>(); - float relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; - float relu_output_value[3] = {}; + Arena<512, T> *relu_arena = new Arena<512, T>(); + Tape &relu_tape = relu_arena->tape(); + relu_tape.template register_operator>(); + T relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; + T relu_output_value[3] = {}; BufferView relu_parameter = relu_tape.parameter(relu_parameter_value); BufferView relu_output = relu_tape.output(relu_output_value); relu_output = relu(relu_parameter); - const float relu_seed[] = {1.0F, 1.0F, 1.0F}; + const T relu_seed[] = {1.0F, 1.0F, 1.0F}; assert(relu_tape.backward(relu_output, relu_seed, 3)); assert(relu_parameter.gradient(0) == 0.0F); assert(relu_parameter.gradient(1) == 0.0F); @@ -207,45 +219,50 @@ void test4() delete relu_arena; } -void test5() +template +static void test5() { // Softmax is normalized and its vector-Jacobian product has zero sum. - Arena<512> *softmax_arena = new Arena<512>(); - Tape &softmax_tape = softmax_arena->tape(); - softmax_tape.register_operator>(); - float logits_value[] = {0.0F, 0.0F, 0.0F}; - float probability_value[3] = {}; + Arena<512, T> *softmax_arena = new Arena<512, T>(); + Tape &softmax_tape = softmax_arena->tape(); + softmax_tape.template register_operator>(); + T logits_value[] = {0.0F, 0.0F, 0.0F}; + T probability_value[3] = {}; BufferView logits = softmax_tape.parameter(logits_value); BufferView probability = softmax_tape.output(probability_value); probability = softmax(logits); - const float probability_sum = probability_value[0] + - probability_value[1] + probability_value[2]; - assert(probability_sum > 0.9999F && probability_sum < 1.0001F); + const T probability_sum = static_cast( + static_cast(probability_value[0]) + + static_cast(probability_value[1]) + + static_cast(probability_value[2])); + const float probability_tolerance = + std::is_same::value ? 2.0e-4F : 2.0e-3F; + assert(close_to(probability_sum, 1.0F, probability_tolerance)); for (std::size_t i = 0; i < 3U; ++i) - assert(probability_value[i] > 0.3332F && - probability_value[i] < 0.3335F); - const float softmax_seed[] = {1.0F, 2.0F, 3.0F}; + assert(close_to(probability_value[i], 1.0F / 3.0F, + probability_tolerance)); + const T softmax_seed[] = {1.0F, 2.0F, 3.0F}; assert(softmax_tape.backward(probability, softmax_seed, 3U)); - assert(logits.gradient(0) < -0.3332F && - logits.gradient(0) > -0.3335F); - assert(logits.gradient(1) > -1.0e-6F && - logits.gradient(1) < 1.0e-6F); - assert(logits.gradient(2) > 0.3332F && - logits.gradient(2) < 0.3335F); + assert(close_to(logits.gradient(0), -1.0F / 3.0F, + probability_tolerance)); + assert(close_to(logits.gradient(1), 0.0F, probability_tolerance)); + assert(close_to(logits.gradient(2), 1.0F / 3.0F, + probability_tolerance)); delete softmax_arena; } -void test6() +template +static void test6() { // Including an operator does not enable it. Evaluation fails until that // operator type is explicitly registered on this tape. - Arena<256> *registry_arena = new Arena<256>(); - Tape ®istry_tape = registry_arena->tape(); - float registry_left_value[] = {1.0F}; - float registry_right_value[] = {2.0F}; - float registry_output_value[] = {0.0F}; + Arena<256, T> *registry_arena = new Arena<256, T>(); + Tape ®istry_tape = registry_arena->tape(); + T registry_left_value[] = {1.0F}; + T registry_right_value[] = {2.0F}; + T registry_output_value[] = {0.0F}; BufferView registry_left = registry_tape.input(registry_left_value); BufferView registry_right = registry_tape.input(registry_right_value); BufferView registry_output = registry_tape.output(registry_output_value); @@ -253,29 +270,30 @@ void test6() assert(registry_tape.status() == Status::operator_not_registered); assert(registry_output_value[0] == 0.0F); registry_tape.clear_status(); - assert(registry_tape.register_operator>()); + assert(registry_tape.template register_operator>()); registry_output = registry_left + registry_right; assert(registry_tape.good()); assert(registry_output_value[0] == 3.0F); delete registry_arena; } -void test7() +template +static void test7() { // Quadratic loss, reusable graph records, Adam, and selective freezing. - Arena<1024> *training_arena = new Arena<1024>(); - Tape &training_tape = training_arena->tape(); - training_tape.register_operator>(); - training_tape.register_operator>(); - training_tape.register_operator>(); - float feature_value[] = {2.0F}; - float coefficient_value[] = {3.0F}; - float bias_parameter_value[] = {1.0F}; - float dot_value[1] = {}; - float prediction_value[1] = {}; - float target_value[] = {0.0F}; - float training_loss_value[1] = {}; + Arena<1024, T> *training_arena = new Arena<1024, T>(); + Tape &training_tape = training_arena->tape(); + training_tape.template register_operator>(); + training_tape.template register_operator>(); + training_tape.template register_operator>(); + T feature_value[] = {2.0F}; + T coefficient_value[] = {3.0F}; + T bias_parameter_value[] = {1.0F}; + T dot_value[1] = {}; + T prediction_value[1] = {}; + T target_value[] = {0.0F}; + T training_loss_value[1] = {}; BufferView feature = training_tape.input(feature_value); BufferView coefficient = training_tape.parameter(coefficient_value); BufferView bias_parameter = training_tape.parameter(bias_parameter_value); @@ -283,7 +301,7 @@ void test7() BufferView prediction = training_tape.output(prediction_value); BufferView target = training_tape.input(target_value); BufferView loss = training_tape.output(training_loss_value); - Adam<2> adam(1.0e-2F); + Adam<2, 16U, T> adam(1.0e-2F); assert(adam.add(coefficient)); assert(adam.add(bias_parameter)); assert(freeze_parameters(adam, coefficient)); @@ -306,23 +324,24 @@ void test7() delete training_arena; } -void test8() +template +static void test8() { // A single vector loss accumulates contributions from every sample into // shared polynomial parameters before an optimizer step. - Arena<2048> *batch_arena = new Arena<2048>(); - Tape &batch_tape = batch_arena->tape(); - batch_tape.register_operator>(); - batch_tape.register_operator>(); - batch_tape.register_operator>(); - float batch_feature_value[2][1] = {{1.0F}, {2.0F}}; - float batch_coefficient_value = 3.0F; - float batch_bias_value = 1.0F; - float batch_polynomial_value[2] = {}; - float batch_prediction_value[2] = {}; - float batch_target_value[2] = {}; - float batch_loss_value = 0.0F; + Arena<2048, T> *batch_arena = new Arena<2048, T>(); + Tape &batch_tape = batch_arena->tape(); + batch_tape.template register_operator>(); + batch_tape.template register_operator>(); + batch_tape.template register_operator>(); + T batch_feature_value[2][1] = {{1.0F}, {2.0F}}; + T batch_coefficient_value = 3.0F; + T batch_bias_value = 1.0F; + T batch_polynomial_value[2] = {}; + T batch_prediction_value[2] = {}; + T batch_target_value[2] = {}; + T batch_loss_value = 0.0F; BufferView batch_coefficient = batch_tape.parameter(batch_coefficient_value); BufferView batch_bias = batch_tape.parameter(batch_bias_value); @@ -353,14 +372,15 @@ void test8() delete batch_arena; } -void test9() +template +static void test9() { // RMSProp uses the same parameter registration and freezing API. - Arena<128> *rms_arena = new Arena<128>(); - Tape &rms_tape = rms_arena->tape(); - float rms_value = 1.0F; + Arena<128, T> *rms_arena = new Arena<128, T>(); + Tape &rms_tape = rms_arena->tape(); + T rms_value = 1.0F; BufferView rms_parameter = rms_tape.parameter(rms_value); - RMSProp<1> rmsprop(1.0e-2F); + RMSProp<1, 16U, T> rmsprop(1.0e-2F); assert(rmsprop.add(rms_parameter)); rms_parameter.gradients()[0] = 2.0F; assert(rmsprop.step()); @@ -368,19 +388,20 @@ void test9() delete rms_arena; } -void test10() +template +static void test10() { // Subtraction and elementwise multiplication have data operands only. - Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator>(); - tape.register_operator>(); - float left_value[] = {2.0F, 4.0F, 6.0F}; - float left_gradient[3] = {}; - float right_value[] = {1.0F, 2.0F, 3.0F}; - float right_gradient[3] = {}; - float difference_value[3] = {}; - float product_value[3] = {}; + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + tape.template register_operator>(); + T left_value[] = {2.0F, 4.0F, 6.0F}; + T left_gradient[3] = {}; + T right_value[] = {1.0F, 2.0F, 3.0F}; + T right_gradient[3] = {}; + T difference_value[3] = {}; + T product_value[3] = {}; BufferView left = tape.view(left_value, left_gradient, 3U); BufferView right = tape.view(right_value, right_gradient, 3U); BufferView difference = tape.output(difference_value); @@ -390,7 +411,7 @@ void test10() assert(product_value[0] == 1.0F); assert(product_value[1] == 4.0F); assert(product_value[2] == 9.0F); - const float seed[] = {1.0F, 1.0F, 1.0F}; + const T seed[] = {1.0F, 1.0F, 1.0F}; assert(tape.backward(product, seed, 3U)); for (std::size_t i = 0; i < 3U; ++i) { @@ -428,46 +449,21 @@ static void test_transposed_dot_type() } } -template -static void test_transposed_dot_mixed_type() -{ - using Result = typename ::arm_cmsis_dsp::MixedRes::type; - const TM matrix_one = ::arm_cmsis_dsp::number_traits::one(); - const TV vector_one = ::arm_cmsis_dsp::number_traits::one(); - TM matrix_value[2][5] = {}; - for (std::size_t column = 0; column < 5; ++column) - matrix_value[0][column] = matrix_one; - TV vector_value[2] = {vector_one,TV{}}; - - ::arm_cmsis_dsp::MatrixView matrix( - &matrix_value[0][0], 2, 5, 5); - ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 2); - ::arm_cmsis_dsp::Matrix - materialized_transpose(5,2); - ::arm_cmsis_dsp::transposeTo(materialized_transpose,matrix); - ::arm_cmsis_dsp::Vector reference = - ::arm_cmsis_dsp::dot(materialized_transpose,vector); - ::arm_cmsis_dsp::Vector result = - ::arm_cmsis_dsp::dot(::arm_cmsis_dsp::transpose_view(matrix),vector); - for (std::size_t column = 0; column < 5; ++column) - assert(result[column] == reference[column]); -} - -void test11() +template +static void test11() { // A transposed view selects the fused matrix-vector kernel without // materializing the transpose. Five columns exercise the MVE tail. - float matrix_value[3][5] = { + T matrix_value[3][5] = { {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}, {6.0F, 7.0F, 8.0F, 9.0F, 10.0F}, {11.0F, 12.0F, 13.0F, 14.0F, 15.0F}}; - float vector_value[] = {2.0F, -1.0F, 0.5F}; - const float expected[] = {1.5F, 3.0F, 4.5F, 6.0F, 7.5F}; + T vector_value[] = {2.0F, -1.0F, 0.5F}; + const T expected[] = {1.5F, 3.0F, 4.5F, 6.0F, 7.5F}; - ::arm_cmsis_dsp::MatrixView matrix( + ::arm_cmsis_dsp::MatrixView matrix( &matrix_value[0][0], 3, 5, 5); - ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 3); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 3); const auto transposed = ::arm_cmsis_dsp::transpose_view(matrix); assert(transposed.rows() == 5); assert(transposed.columns() == 3); @@ -477,56 +473,31 @@ void test11() for (std::size_t column = 0; column < 5; ++column) assert(result[column] == expected[column]); - ::arm_cmsis_dsp::Matrix static_matrix; - ::arm_cmsis_dsp::Vector static_vector; + ::arm_cmsis_dsp::Matrix static_matrix; + ::arm_cmsis_dsp::Vector static_vector; for (std::size_t row = 0; row < 2; ++row) for (std::size_t column = 0; column < 5; ++column) static_matrix(row,column) = 1.0F; - static_vector = 1.0F; - ::arm_cmsis_dsp::Vector static_result = + static_vector = static_cast(1.0F); + ::arm_cmsis_dsp::Vector static_result = ::arm_cmsis_dsp::dot( ::arm_cmsis_dsp::transpose_view(static_matrix),static_vector); for (std::size_t column = 0; column < 5; ++column) assert(static_result[column] == 2.0F); - test_transposed_dot_type(); - test_transposed_dot_type(); - test_transposed_dot_type>(); -#if !defined(DISABLEFLOAT16) - test_transposed_dot_type(); - test_transposed_dot_type>(); -#endif - test_transposed_dot_type<::arm_cmsis_dsp::Q31>(); - test_transposed_dot_type>(); - test_transposed_dot_type<::arm_cmsis_dsp::Q15>(); - test_transposed_dot_type>(); - test_transposed_dot_type<::arm_cmsis_dsp::Q7>(); - test_transposed_dot_mixed_type,float>(); - test_transposed_dot_mixed_type>(); -#if !defined(DISABLEFLOAT16) - test_transposed_dot_mixed_type,float16_t>(); - test_transposed_dot_mixed_type>(); -#endif - test_transposed_dot_mixed_type< - std::complex<::arm_cmsis_dsp::Q31>,::arm_cmsis_dsp::Q31>(); - test_transposed_dot_mixed_type< - ::arm_cmsis_dsp::Q31,std::complex<::arm_cmsis_dsp::Q31>>(); - test_transposed_dot_mixed_type< - std::complex<::arm_cmsis_dsp::Q15>,::arm_cmsis_dsp::Q15>(); - test_transposed_dot_mixed_type< - ::arm_cmsis_dsp::Q15,std::complex<::arm_cmsis_dsp::Q15>>(); + test_transposed_dot_type(); // Exercise the same kernel through the fully connected backward pass. - Arena<4096> arena; - Tape &tape = arena.tape(); - tape.register_operator>(); - float input_value[5] = {}; - float input_gradient[5] = {}; - float weight_gradient[3][5] = {}; - float bias_value[3] = {}; - float bias_gradient[3] = {}; - float output_value[3] = {}; - float output_gradient[3] = {}; + Arena<4096, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T input_value[5] = {}; + T input_gradient[5] = {}; + T weight_gradient[3][5] = {}; + T bias_value[3] = {}; + T bias_gradient[3] = {}; + T output_value[3] = {}; + T output_gradient[3] = {}; BufferView input = tape.view(input_value, input_gradient, 5); MatrixView weights = tape.parameter( &matrix_value[0][0], &weight_gradient[0][0], 3, 5); @@ -538,36 +509,42 @@ void test11() assert(input.gradient(column) == expected[column]); } -void test12() +template +static void test12() { // Categorical cross entropy consumes probabilities and a one-hot target. - Arena<512> arena; - Tape &tape = arena.tape(); - tape.register_operator>(); - float probability_value[] = {0.1F, 0.7F, 0.2F}; - float target_value[] = {0.0F, 1.0F, 0.0F}; - float loss_value = 0.0F; + Arena<512, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T probability_value[] = {0.1F, 0.7F, 0.2F}; + T target_value[] = {0.0F, 1.0F, 0.0F}; + T loss_value = 0.0F; BufferView probability = tape.parameter(probability_value); BufferView target = tape.input(target_value); BufferView loss = tape.output(loss_value); loss = cross_entropy(probability, target); - assert(loss_value > 0.3566F && loss_value < 0.3568F); + const float loss_tolerance = + std::is_same::value ? 2.0e-4F : 3.0e-3F; + const float gradient_tolerance = + std::is_same::value ? 2.0e-4F : 1.0e-2F; + assert(close_to(loss_value, 0.35667494F, loss_tolerance)); assert(tape.backward(loss)); assert(probability.gradient(0) == 0.0F); - assert(probability.gradient(1) < -1.4285F && - probability.gradient(1) > -1.4287F); + assert(close_to(probability.gradient(1), -1.42857143F, + gradient_tolerance)); assert(probability.gradient(2) == 0.0F); } -void test13() +template +static void test13() { // Training applies inverted dropout and backward regenerates the same // mask. Disabling recording makes dropout an identity for inference. - Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator>(); - float input_value[16]; - float output_value[16] = {}; + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T input_value[16]; + T output_value[16] = {}; for (std::size_t i = 0; i < 16U; ++i) input_value[i] = 1.0F; BufferView input = tape.parameter(input_value); BufferView output = tape.output(output_value); @@ -583,7 +560,7 @@ void test13() } assert(dropped != 0U && kept != 0U); - float seed[16]; + T seed[16]; for (std::size_t i = 0; i < 16U; ++i) seed[i] = 1.0F; assert(tape.backward(output, seed, 16U)); for (std::size_t i = 0; i < 16U; ++i) @@ -597,18 +574,19 @@ void test13() } } -void test14() +template +static void test14() { // Y = W X uses CMSIS-DSP matrix multiplication in the forward pass and // computes only dW = dY X^T in the backward pass. - Arena<1024> arena; - Tape &tape = arena.tape(); - tape.register_operator>(); - float weight_value[2][3] = { + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T weight_value[2][3] = { {1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}}; - float input_value[3][2] = { + T input_value[3][2] = { {1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; - float output_value[2][2] = {}; + T output_value[2][2] = {}; MatrixView weights = tape.parameter(weight_value); BufferView input = tape.input(&input_value[0][0], 6U); BufferView output = tape.output(&output_value[0][0], 4U); @@ -618,9 +596,9 @@ void test14() assert(output_value[1][0] == 49.0F); assert(output_value[1][1] == 64.0F); - const float seed[] = {1.0F, 2.0F, 3.0F, 4.0F}; + const T seed[] = {1.0F, 2.0F, 3.0F, 4.0F}; assert(tape.backward(output, seed, 4U)); - const float expected_gradient[2][3] = { + const T expected_gradient[2][3] = { {5.0F, 11.0F, 17.0F}, {11.0F, 25.0F, 39.0F}}; for (std::size_t row = 0; row < 2U; ++row) for (std::size_t column = 0; column < 3U; ++column) @@ -629,14 +607,15 @@ void test14() assert(!input.has_gradient()); } -void test15() +template +static void test15() { // SGD performs one fused parameter -= learning_rate * gradient update. - Arena<128> arena; - Tape &tape = arena.tape(); - float value[] = {1.0F, -2.0F}; + Arena<128, T> arena; + Tape &tape = arena.tape(); + T value[] = {1.0F, -2.0F}; BufferView parameter = tape.parameter(value); - SGD<2, 1> optimizer(0.25F); + SGD<2, 1, T> optimizer(0.25F); assert(optimizer.add(parameter)); parameter.gradients()[0] = 2.0F; parameter.gradients()[1] = -4.0F; @@ -652,32 +631,33 @@ void test15() assert(value[0] == 0.5F); } +template static void run_autodiff_tests() { - test1(); - test2(); - test3(); - test4(); - test5(); - test6(); - test7(); - test8(); - test9(); - test10(); - test11(); - test12(); - test13(); - test14(); - test15(); + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + test7(); + test8(); + test9(); + test10(); + test11(); + test12(); + test13(); + test14(); + test15(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; - Tape tiny(tiny_memory, sizeof(tiny_memory)); - tiny.register_operator>(); - float tiny_input_value[1] = {2.0F}; - float tiny_input_gradient[1] = {}; - float tiny_output_value[1] = {}; - float tiny_output_gradient[1] = {}; + Tape tiny(tiny_memory, sizeof(tiny_memory)); + tiny.template register_operator>(); + T tiny_input_value[1] = {2.0F}; + T tiny_input_gradient[1] = {}; + T tiny_output_value[1] = {}; + T tiny_output_gradient[1] = {}; BufferView tiny_input = tiny.view(tiny_input_value, tiny_input_gradient, 1); BufferView tiny_output = @@ -687,86 +667,7 @@ static void run_autodiff_tests() assert(tiny.status() == Status::out_of_memory); assert(!tiny.backward(tiny_output)); -#if defined(ARM_FLOAT16_SUPPORTED) - // The same graph can be instantiated with CMSIS-DSP float16 kernels. - Arena<512, float16_t> half_arena; - Tape &half_tape = half_arena.tape(); - half_tape.register_operator>(); - half_tape.register_operator>(); - float16_t half_input_value[2] = {static_cast(1.0F), - static_cast(2.0F)}; - float16_t half_scale_value = static_cast(3.0F); - float16_t half_scaled_value[2] = {}; - float16_t half_loss_value = {}; - BufferView half_input = half_tape.input(half_input_value); - BufferView half_scale = half_tape.parameter(half_scale_value); - BufferView half_scaled = half_tape.output(half_scaled_value); - BufferView half_loss = half_tape.output(half_loss_value); - half_scaled = scale(half_input, half_scale); - half_loss = dot(half_scaled, half_input); - assert(half_tape.backward(half_loss)); - assert(static_cast(half_loss_value) > 14.9F && - static_cast(half_loss_value) < 15.1F); - assert(static_cast(half_scale.gradient(0)) > 4.9F && - static_cast(half_scale.gradient(0)) < 5.1F); - - float16_t half_sgd_value[] = {static_cast(1.0F), - static_cast(-2.0F)}; - BufferView half_sgd_parameter = - half_tape.parameter(half_sgd_value); - SGD<2, 1, float16_t> half_sgd(static_cast(0.25F)); - assert(half_sgd.add(half_sgd_parameter)); - half_sgd_parameter.gradients()[0] = static_cast(2.0F); - half_sgd_parameter.gradients()[1] = static_cast(-4.0F); - assert(half_sgd.step()); - assert(static_cast(half_sgd_value[0]) == 0.5F); - assert(static_cast(half_sgd_value[1]) == -1.0F); - - // ReLU and categorical cross entropy use finite float16 clip bounds. - // In particular, numeric_limits<__fp16>::max() is not specialized by all - // embedded C++ libraries and can otherwise evaluate to zero. - Arena<1024, float16_t> half_classification_arena; - Tape &half_classification_tape = - half_classification_arena.tape(); - half_classification_tape.register_operator>(); - half_classification_tape.register_operator>(); - half_classification_tape.register_operator>(); - float16_t half_relu_input_value[3] = { - static_cast(-1.0F), static_cast(0.5F), - static_cast(2.0F)}; - float16_t half_relu_output_value[3] = {}; - float16_t half_logits_value[3] = { - static_cast(0.2F), static_cast(-0.1F), - static_cast(0.3F)}; - float16_t half_probability_value[3] = {}; - float16_t half_target_value[3] = { - static_cast(0.0F), static_cast(1.0F), - static_cast(0.0F)}; - float16_t half_classification_loss_value = {}; - BufferView half_relu_input = - half_classification_tape.input(half_relu_input_value); - BufferView half_relu_output = - half_classification_tape.output(half_relu_output_value); - BufferView half_logits = - half_classification_tape.parameter(half_logits_value); - BufferView half_probability = - half_classification_tape.output(half_probability_value); - BufferView half_target = - half_classification_tape.input(half_target_value); - BufferView half_classification_loss = - half_classification_tape.output(half_classification_loss_value); - half_relu_output = relu(half_relu_input); - assert(static_cast(half_relu_output_value[1]) > 0.49F && - static_cast(half_relu_output_value[2]) > 1.9F); - half_probability = softmax(half_logits); - half_classification_loss = cross_entropy(half_probability, half_target); - assert(static_cast(half_classification_loss_value) > 0.9F && - static_cast(half_classification_loss_value) < 1.3F); - assert(half_classification_tape.backward(half_classification_loss)); - assert(static_cast(half_logits.gradient(0)) < 0.5F && - static_cast(half_logits.gradient(1)) < -0.2F && - static_cast(half_logits.gradient(2)) > 0.2F); -#endif + } @@ -777,8 +678,13 @@ static void run_autodiff_tests() void autodiff_test() { -#if defined(AUTODIFF_TEST) && defined(F32_DT) && defined(DYNAMIC_TEST) - printf("Running autodiff tests...\r\n"); - run_autodiff_tests(); +#if defined(AUTODIFF_TEST) && defined(DYNAMIC_TEST) +#if defined(F32_DT) + printf("Running float32 autodiff tests...\r\n"); + run_autodiff_tests(); +#elif defined(F16_DT) && defined(ARM_FLOAT16_SUPPORTED) + printf("Running float16 autodiff tests...\r\n"); + run_autodiff_tests(); +#endif #endif } From f398285f621227bb5cc3361bc90c4459c547e936 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Thu, 13 Aug 2026 11:18:19 +0200 Subject: [PATCH 16/19] autodiff : Added quantize / dequantize operators Added an example of quantization aware training for a fully connected layer. --- AGENTS.md | 8 + dsppp/Examples/README.md | 2 + .../Examples/autodiff_fully_connected_qat.cpp | 200 ++++++++++++++++ dsppp/Include/dsppp/autodiff/doc/operators.md | 50 ++++ .../dsppp/autodiff/operators/dequantize.hpp | 192 +++++++++++++++ .../operators/quantization_support.hpp | 182 ++++++++++++++ .../dsppp/autodiff/operators/quantize.hpp | 223 ++++++++++++++++++ dsppp/example.cproject.yml | 3 +- dsppp/tests/autodiff_test.cpp | 94 ++++++++ 9 files changed, 953 insertions(+), 1 deletion(-) create mode 100644 dsppp/Examples/autodiff_fully_connected_qat.cpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/dequantize.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp create mode 100644 dsppp/Include/dsppp/autodiff/operators/quantize.hpp diff --git a/AGENTS.md b/AGENTS.md index 2603cf348..6a3ad53ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,3 +20,11 @@ - For each of the generated pickle file, generate missing C files with : `python processTests.py -gen . -p Patterns -d Parameters -f -e` - Switch to `cmsis_build` working directory - Run python script `python runall.py` + +## Testing instructions for dsppp autodiff +- Do not use or install `pyocd` for autodiff tests. +- Change working directory to the `dsppp` folder. +- Set `AVH_FVP_PLUGINS` to an empty string before launching the FVP. +- Prepend the CMSIS Toolbox `bin` directory and the Python directory containing `python3.dll` to `PATH`. +- Build the generated project with `cbuild -O cprj test.csolution.yml --toolchain AC6 -c test.Release+VHT-Corstone-300`. +- Run `FVP_Corstone_SSE-300_Ethos-U55.exe` directly with `-f fvp_configs\VHT-Corstone-300.txt -a cpu0=cprj\out\test\VHT-Corstone-300\Release\test.axf`. diff --git a/dsppp/Examples/README.md b/dsppp/Examples/README.md index af85ea7cc..be652b707 100644 --- a/dsppp/Examples/README.md +++ b/dsppp/Examples/README.md @@ -26,6 +26,8 @@ in `dsppp/example.cproject.yml` by commenting and uncommenting its `file` line. - `autodiff_iris.cpp` trains a small classifier on the Iris flower dataset. It uses Adam and reserves 30 of the 150 samples for a final test that is not used during training. +- `autodiff_fully_connected_qat.cpp` demonstrates quantization-aware training + of a fully connected layer for later deployment with CMSIS-NN or Ethos-U. ### Iris classifier diff --git a/dsppp/Examples/autodiff_fully_connected_qat.cpp b/dsppp/Examples/autodiff_fully_connected_qat.cpp new file mode 100644 index 000000000..365cca400 --- /dev/null +++ b/dsppp/Examples/autodiff_fully_connected_qat.cpp @@ -0,0 +1,200 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +namespace { + +constexpr std::size_t input_count = 3U; +constexpr std::size_t output_count = 2U; +constexpr std::size_t sample_count = 4U; + +} // namespace + +int main() +{ + constexpr std::size_t training_steps = 400U; + constexpr Int8Quantization activation_quantization = + Int8Quantization::activation(); + constexpr Int8Quantization weight_quantization = + Int8Quantization::weights(output_count, input_count); + + // A small fixed training set and the outputs of a reference 3-to-2 + // affine layer. The network below learns these targets while every input, + // weight, and output forward pass is subjected to signed int8 Q/DQ. + float input_value[sample_count][input_count] = { + {-1.0F, -0.5F, 0.25F}, + {0.5F, 1.0F, -1.0F}, + {1.0F, 0.25F, 0.5F}, + {-0.25F, -1.0F, 1.0F}}; + float target_value[sample_count][output_count] = { + {0.1F, 0.425F}, + {-1.275F, -1.2F}, + {0.7875F, -0.1375F}, + {1.6625F, 0.675F}}; + + float weight_value[output_count][input_count] = { + {0.2F, -0.1F, 0.3F}, {-0.2F, 0.1F, 0.2F}}; + float weight_gradient[output_count][input_count] = {}; + float bias_value[output_count] = {}; + + // LiteRT/CMSIS-NN/Ethos-U weight quantization is symmetric and uses one + // scale per output channel. Its zero-point is fixed at zero. + float weight_scale_value[output_count] = {0.01F, 0.01F}; + float weight_zero_point_value[output_count] = {}; + + // Activations are asymmetric per-tensor. These are optimizer parameters; + // Q/DQ automatically keeps them in the deployable signed-int8 domain. + float input_scale_value = 0.01F; + float input_zero_point_value = 0.0F; + float output_scale_value = 0.01F; + float output_zero_point_value = 0.0F; + + // Quantize produces the integer int8 values, represented as float so the + // autodiff graph remains floating point. Dequantize reconstructs the + // floating values that the layer would observe during int8 inference. + float quantized_weight_value[output_count][input_count] = {}; + float dequantized_weight_value[output_count][input_count] = {}; + float dequantized_weight_gradient[output_count][input_count] = {}; + float quantized_input_value[input_count] = {}; + float dequantized_input_value[input_count] = {}; + float linear_value[output_count] = {}; + float quantized_output_value[output_count] = {}; + float prediction_value[output_count] = {}; + float loss_value = 0.0F; + + // Keep the arena out of the small embedded process stack. Static storage + // also makes the fixed memory cost explicit in the linker map. + static Arena<8192> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + + BufferView weights = tape.parameter( + &weight_value[0][0], &weight_gradient[0][0], + output_count * input_count); + BufferView bias = tape.parameter(bias_value); + BufferView weight_scale = tape.parameter(weight_scale_value); + BufferView weight_zero_point = tape.input(weight_zero_point_value); + BufferView input_scale = tape.parameter(input_scale_value); + BufferView input_zero_point = tape.parameter(input_zero_point_value); + BufferView output_scale = tape.parameter(output_scale_value); + BufferView output_zero_point = tape.parameter(output_zero_point_value); + + BufferView quantized_weight = tape.output( + &quantized_weight_value[0][0], output_count * input_count); + BufferView dequantized_weight = tape.output( + &dequantized_weight_value[0][0], &dequantized_weight_gradient[0][0], + output_count * input_count); + + // The FC operator consumes a MatrixView. This matrix aliases the Q/DQ + // output above, so its gradients continue through dequantize and quantize + // to the original floating weights and their per-channel scales. + MatrixView dequantized_weight_matrix = tape.parameter( + &dequantized_weight_value[0][0], &dequantized_weight_gradient[0][0], + output_count, input_count); + + BufferView quantized_input = tape.output(quantized_input_value); + BufferView dequantized_input = tape.output(dequantized_input_value); + BufferView linear = tape.output(linear_value); + BufferView quantized_output = tape.output(quantized_output_value); + BufferView prediction = tape.output(prediction_value); + BufferView loss = tape.output(loss_value); + + Adam<14U, 8U> optimizer(2.0e-2F); + if (!optimizer.add(weights) || !optimizer.add(bias) || + !optimizer.add(weight_scale) || !optimizer.add(input_scale) || + !optimizer.add(input_zero_point) || !optimizer.add(output_scale) || + !optimizer.add(output_zero_point) || !tape.good()) + { + std::printf("QAT setup failed\n"); + return 1; + } + + tape.begin_graph(); + std::printf("Fully connected signed-int8 QAT\n"); + for (std::size_t step = 0; step < training_steps; ++step) + { + if (!tape.rewind_graph()) return 1; + const std::size_t sample = step % sample_count; + BufferView input = tape.input(input_value[sample]); + BufferView target = tape.input(target_value[sample]); + + quantized_weight = quantize(weights, weight_scale, + weight_zero_point, weight_quantization); + dequantized_weight = dequantize(quantized_weight, weight_scale, + weight_zero_point, + weight_quantization); + quantized_input = quantize(input, input_scale, input_zero_point, + activation_quantization); + dequantized_input = dequantize(quantized_input, input_scale, + input_zero_point, + activation_quantization); + linear = fully_connected(dequantized_input, + dequantized_weight_matrix, bias); + quantized_output = quantize(linear, output_scale, output_zero_point, + activation_quantization); + prediction = dequantize(quantized_output, output_scale, + output_zero_point, + activation_quantization); + + loss = quadratic_error(prediction, target); + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if ((step + 1U) % 100U == 0U) + std::printf("step %u, sample quadratic error = %g\n", + static_cast(step + 1U), + static_cast(loss_value)); + } + + // Refresh the integer-valued floating weights after the final optimizer + // update. Casting these values to int8_t is exact. + { + RecordingScope inference(tape, false); + quantized_weight = quantize(weights, weight_scale, + weight_zero_point, weight_quantization); + } + std::int8_t exported_weight[output_count][input_count] = {}; + for (std::size_t row = 0; row < output_count; ++row) + for (std::size_t column = 0; column < input_count; ++column) + exported_weight[row][column] = static_cast( + std::nearbyint(quantized_weight_value[row][column])); + + std::int32_t exported_bias[output_count] = {}; + for (std::size_t row = 0; row < output_count; ++row) + exported_bias[row] = static_cast(std::nearbyint( + bias_value[row] / + (input_scale_value * weight_scale_value[row]))); + + std::printf("input: scale=%g, zero_point=%g, CMSIS-NN offset=%d\n", + static_cast(input_scale_value), + static_cast(std::nearbyint(input_zero_point_value)), + static_cast(cmsis_nn_offset(input_zero_point_value))); + std::printf("output: scale=%g, zero_point=%g, CMSIS-NN offset=%d\n", + static_cast(output_scale_value), + static_cast(std::nearbyint(output_zero_point_value)), + static_cast(cmsis_nn_offset(output_zero_point_value))); + for (std::size_t row = 0; row < output_count; ++row) + { + std::printf("channel %u: weight_scale=%g, weights={%d, %d, %d}, " + "bias=%ld\n", + static_cast(row), + static_cast(weight_scale_value[row]), + static_cast(exported_weight[row][0]), + static_cast(exported_weight[row][1]), + static_cast(exported_weight[row][2]), + static_cast(exported_bias[row])); + } + return 0; +} diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md index 190f834e1..143e491eb 100644 --- a/dsppp/Include/dsppp/autodiff/doc/operators.md +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -76,6 +76,56 @@ dx[i] += y[i] * (g[i] - projection) The forward path dispatches to the matching f32 or f16 log-sum-exp, offset, and vector-exponential kernels. +## Signed int8 quantize/dequantize + +`quantize` and `dequantize` implement the signed int8 affine scheme used by +LiteRT, CMSIS-NN, and Ethos-U while keeping every graph buffer in `float` or +`float16_t`. The quantized codes are integer-valued floating-point numbers; +no `int8_t` participates in training. + +For scale `s`, zero-point `z`, and integer limits `qmin` and `qmax`: + +```text +q[i] = clamp(nearbyint(x[i] / s) + nearbyint(z), qmin, qmax) +y[i] = (q[i] - nearbyint(z)) * s +``` + +Use `Int8Quantization::activation()` for asymmetric per-tensor activations. +It uses the backend-required `[-128, 127]` range and a learnable scale and +zero-point. Use `Int8Quantization::weights(axis_size, inner_size)` for weights. +It uses symmetric `[-127, 127]` codes, forces the effective zero-point to zero, +and supports one learnable scale per quantized axis. `inner_size` is the +product of the row-major dimensions after that axis. Examples are: + +```cpp +auto activation_q = Int8Quantization::activation(); +auto fully_connected_w = Int8Quantization::weights(outputs, inputs); +auto conv2d_ohwi_w = Int8Quantization::weights(outputs, height * width * inputs); +auto depthwise_hwc_w = Int8Quantization::weights(channels, 1U); +``` + +For symmetric weights, pass a zero-valued input view as `zero_point`; it is +fixed and receives no gradient. For activations, both scale and zero-point are +parameter views. Scale values must remain strictly positive. An optimizer can +update both activation parameters and the per-axis weight scales. Q/DQ projects +these parameters before each forward calculation: scale receives a small +positive numerical floor, asymmetric zero-points are clamped to the int8 +domain, and symmetric weight zero-points are forced to zero. Training loops do +not need to enforce these constraints themselves. + +The Q/DQ pair uses a straight-through estimator. Inside the int8 range its +combined input derivative is one; saturated inputs receive zero. The local +rules also retain the scale derivative caused by quantization error and a +zero-point derivative at saturation, allowing the representable range to be +learned. The forward path is the same affine quantize/dequantize calculation +used when exporting the final int8 values. + +CMSIS-NN calls the negated zero-point an `offset`. Use +`cmsis_nn_offset(zero_point)` when filling APIs such as `input_offset` or +`output_offset`. Biases are not processed by this Q/DQ pair: CMSIS-NN/LiteRT +requires int32 bias with zero-point zero and scale +`input_scale * weight_scale[channel]`. + ## Losses Quadratic error returns a scalar sum, not a mean: diff --git a/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp new file mode 100644 index 000000000..d1d97e429 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp @@ -0,0 +1,192 @@ +#pragma once + +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Dequantize floating-point int8 codes back to the tape's floating type. */ +template class DequantizeOperator +{ + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + const T *zero_point_value; + T *scale_gradient; + T *zero_point_gradient; + std::size_t length; + Int8Quantization quantization; + }; + + static float rounded(float value) noexcept { return std::nearbyint(value); } + + static void add(T &destination, float contribution) noexcept + { + destination = static_cast(static_cast(destination) + + contribution); + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + record.output_gradient[i] = T{}; + if (record.input_gradient != nullptr) + record.input_gradient[i] = T{}; + } + for (std::size_t i = 0; i < record.quantization.parameter_count(); ++i) + { + record.scale_gradient[i] = T{}; + if (record.zero_point_gradient != nullptr) + record.zero_point_gradient[i] = T{}; + } + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + const std::size_t p = record.quantization.parameter_index(i); + const float gradient = static_cast(record.output_gradient[i]); + const float scale = static_cast(record.scale_value[p]); + const float zero = record.quantization.asymmetric() + ? rounded(static_cast( + record.zero_point_value[p])) + : 0.0F; + if (record.input_gradient != nullptr) + add(record.input_gradient[i], gradient * scale); + add(record.scale_gradient[p], + gradient * (static_cast(record.input_value[i]) - zero)); + if (record.zero_point_gradient != nullptr && + record.quantization.asymmetric()) + add(record.zero_point_gradient[p], -gradient * scale); + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + const Int8Quantization &quantization) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || + !OperatorAccess::valid(tape, zero_point) || + OperatorAccess::length(scale) != quantization.parameter_count() || + OperatorAccess::length(zero_point) != quantization.parameter_count() || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + !quantization.valid_for(OperatorAccess::length(input)) || + OperatorAccess::values(output) == OperatorAccess::values(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + if (quantization.asymmetric() && + (OperatorAccess::role(zero_point) != BufferRole::parameter || + OperatorAccess::gradients(zero_point) == nullptr)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; + if (!quantization.valid_for(OperatorAccess::length(input))) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + quantization.constrain_parameters( + const_cast(OperatorAccess::values(scale)), + const_cast(OperatorAccess::values(zero_point))); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale, zero_point, quantization)) + return false; +#endif + for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + { + const std::size_t p = quantization.parameter_index(i); + const float zero = quantization.asymmetric() + ? rounded(static_cast( + OperatorAccess::values(zero_point)[p])) + : 0.0F; + OperatorAccess::values(output)[i] = static_cast( + (static_cast(OperatorAccess::values(input)[i]) - zero) * + static_cast(OperatorAccess::values(scale)[p])); + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append( + *tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->zero_point_value = OperatorAccess::values(zero_point); + record->scale_gradient = OperatorAccess::gradients(scale); + record->zero_point_gradient = quantization.asymmetric() + ? OperatorAccess::gradients(zero_point) + : nullptr; + record->length = OperatorAccess::length(output); + record->quantization = quantization; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class DequantizeExpression +{ +public: + DequantizeExpression(const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + : input_(input), scale_(scale), zero_point_(zero_point), + quantization_(quantization) {} + void evaluate(BufferView &output) const noexcept + { + DequantizeOperator::evaluate(output, input_, scale_, zero_point_, + quantization_); + } +private: + BufferView input_; + BufferView scale_; + BufferView zero_point_; + Int8Quantization quantization_; +}; + +template +inline DequantizeExpression dequantize( + const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization = Int8Quantization::activation()) noexcept +{ + return DequantizeExpression(input, scale, zero_point, quantization); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp b/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp new file mode 100644 index 000000000..3fbbd2412 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** LiteRT/CMSIS-NN/Ethos-U signed int8 quantization layout. */ +class Int8Quantization +{ +public: + constexpr Int8Quantization() noexcept + : quant_min_(-128), quant_max_(127), parameter_count_(1U), + inner_size_(1U), asymmetric_(true) + { + } + + /** Per-tensor asymmetric activation quantization. */ + static constexpr Int8Quantization activation() noexcept + { + return Int8Quantization(-128, 127, 1U, 1U, true); + } + + /** + * Symmetric per-axis weight quantization. + * + * axis_size is the number of scales and inner_size is the product of the + * dimensions following the quantized axis in a row-major tensor. Thus a + * fully-connected [output, input] matrix uses (output, input), Conv2D + * OHWI uses (output, H*W*I), and depthwise Conv2D uses (channels, 1). + */ + static constexpr Int8Quantization weights(std::size_t axis_size, + std::size_t inner_size) noexcept + { + return Int8Quantization(-127, 127, axis_size, inner_size, false); + } + + constexpr int quant_min() const noexcept { return quant_min_; } + constexpr int quant_max() const noexcept { return quant_max_; } + constexpr std::size_t parameter_count() const noexcept + { + return parameter_count_; + } + constexpr std::size_t inner_size() const noexcept { return inner_size_; } + constexpr bool asymmetric() const noexcept { return asymmetric_; } + + constexpr std::size_t parameter_index(std::size_t element) const noexcept + { + return parameter_count_ == 1U + ? 0U + : (element / inner_size_) % parameter_count_; + } + + constexpr bool valid_for(std::size_t length) const noexcept + { + return parameter_count_ != 0U && inner_size_ != 0U && + (parameter_count_ == 1U || + ((length % parameter_count_) == 0U && + (length / parameter_count_) % inner_size_ == 0U)); + } + + /** Keep learned scales in the numerically safe, strictly-positive range. */ + template + void constrain_scale(T &scale) const noexcept + { + if (static_cast(scale) < minimum_scale()) + scale = static_cast(minimum_scale()); + } + + template + void constrain_scales(T (&scales)[Length]) const noexcept + { + for (std::size_t i = 0; i < Length; ++i) + constrain_scale(scales[i]); + } + + /** + * Keep learned zero-points in this scheme's integer domain. Symmetric + * weight quantization always forces them to zero. + */ + template + void constrain_zero_point(T &zero_point) const noexcept + { + if (!asymmetric_) + { + zero_point = T{}; + return; + } + if (static_cast(zero_point) < static_cast(quant_min_)) + zero_point = static_cast(quant_min_); + if (static_cast(zero_point) > static_cast(quant_max_)) + zero_point = static_cast(quant_max_); + } + + template + void constrain_zero_points(T (&zero_points)[Length]) const noexcept + { + for (std::size_t i = 0; i < Length; ++i) + constrain_zero_point(zero_points[i]); + } + + /** Project all learned parameters into this backend quantization scheme. */ + template + void constrain_parameters(T *scales, T *zero_points) const noexcept + { + if constexpr (std::is_same::value) + { + arm_clip_f32(scales, scales, minimum_scale(), + std::numeric_limits::max(), + static_cast(parameter_count_)); + if (asymmetric_) + arm_clip_f32(zero_points, zero_points, + static_cast(quant_min_), + static_cast(quant_max_), + static_cast(parameter_count_)); + else + arm_fill_f32(0.0F, zero_points, + static_cast(parameter_count_)); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + arm_clip_f16(scales, scales, static_cast(minimum_scale()), + F16_MAX, static_cast(parameter_count_)); + if (asymmetric_) + arm_clip_f16(zero_points, zero_points, + static_cast(quant_min_), + static_cast(quant_max_), + static_cast(parameter_count_)); + else + arm_fill_f16(static_cast(0.0F), zero_points, + static_cast(parameter_count_)); + } +#endif + else + for (std::size_t i = 0; i < parameter_count_; ++i) + { + constrain_scale(scales[i]); + constrain_zero_point(zero_points[i]); + } + } + +private: + // The backend only requires scale > 0. This small floor additionally + // prevents division by zero after a finite-precision optimizer update. + static constexpr float minimum_scale() noexcept { return 1.0e-4F; } + + constexpr Int8Quantization(int quant_min, int quant_max, + std::size_t parameter_count, + std::size_t inner_size, + bool asymmetric) noexcept + : quant_min_(quant_min), quant_max_(quant_max), + parameter_count_(parameter_count), inner_size_(inner_size), + asymmetric_(asymmetric) + { + } + + int quant_min_; + int quant_max_; + std::size_t parameter_count_; + std::size_t inner_size_; + bool asymmetric_; +}; + +/** CMSIS-NN APIs use offset = -zero_point. */ +inline int32_t cmsis_nn_offset(float zero_point) noexcept +{ + return -static_cast(std::nearbyint(zero_point)); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quantize.hpp b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp new file mode 100644 index 000000000..8bc9e71a3 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp @@ -0,0 +1,223 @@ +#pragma once + +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Quantize to signed int8 codes represented in the tape's floating type. */ +template class QuantizeOperator +{ + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + const T *zero_point_value; + T *scale_gradient; + T *zero_point_gradient; + std::size_t length; + Int8Quantization quantization; + }; + + static float rounded(float value) noexcept { return std::nearbyint(value); } + + static void add(T &destination, float contribution) noexcept + { + destination = static_cast(static_cast(destination) + + contribution); + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + record.output_gradient[i] = T{}; + if (record.input_gradient != nullptr) + record.input_gradient[i] = T{}; + } + for (std::size_t i = 0; i < record.quantization.parameter_count(); ++i) + { + record.scale_gradient[i] = T{}; + if (record.zero_point_gradient != nullptr) + record.zero_point_gradient[i] = T{}; + } + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t i = 0; i < record.length; ++i) + { + const std::size_t p = record.quantization.parameter_index(i); + const float scale = static_cast(record.scale_value[p]); + const float zero_point = record.quantization.asymmetric() + ? rounded(static_cast( + record.zero_point_value[p])) + : 0.0F; + const float input = static_cast(record.input_value[i]); + const float raw_code = rounded(input / scale) + zero_point; + if (raw_code < static_cast(record.quantization.quant_min()) || + raw_code > static_cast(record.quantization.quant_max())) + continue; + + const float gradient = static_cast(record.output_gradient[i]); + if (record.input_gradient != nullptr) + add(record.input_gradient[i], gradient / scale); + add(record.scale_gradient[p], + -gradient * input / (scale * scale)); + if (record.zero_point_gradient != nullptr && + record.quantization.asymmetric()) + add(record.zero_point_gradient[p], gradient); + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + const Int8Quantization &quantization) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || + !OperatorAccess::valid(tape, zero_point) || + OperatorAccess::length(scale) != quantization.parameter_count() || + OperatorAccess::length(zero_point) != quantization.parameter_count() || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + !quantization.valid_for(OperatorAccess::length(input)) || + OperatorAccess::values(output) == OperatorAccess::values(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + if (quantization.asymmetric()) + { + if (OperatorAccess::role(zero_point) != BufferRole::parameter || + OperatorAccess::gradients(zero_point) == nullptr) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + } + else + { + for (std::size_t i = 0; i < quantization.parameter_count(); ++i) + if (rounded(static_cast(OperatorAccess::values(zero_point)[i])) != 0.0F) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; + if (!quantization.valid_for(OperatorAccess::length(input))) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + quantization.constrain_parameters( + const_cast(OperatorAccess::values(scale)), + const_cast(OperatorAccess::values(zero_point))); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale, zero_point, quantization)) + return false; +#endif + for (std::size_t p = 0; p < quantization.parameter_count(); ++p) + if (!(static_cast(OperatorAccess::values(scale)[p]) > 0.0F)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + { + const std::size_t p = quantization.parameter_index(i); + const float scale_value = + static_cast(OperatorAccess::values(scale)[p]); + const float zero = quantization.asymmetric() + ? rounded(static_cast( + OperatorAccess::values(zero_point)[p])) + : 0.0F; + float code = rounded(static_cast( + OperatorAccess::values(input)[i]) / + scale_value) + zero; + if (code < static_cast(quantization.quant_min())) + code = static_cast(quantization.quant_min()); + if (code > static_cast(quantization.quant_max())) + code = static_cast(quantization.quant_max()); + OperatorAccess::values(output)[i] = static_cast(code); + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append( + *tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->zero_point_value = OperatorAccess::values(zero_point); + record->scale_gradient = OperatorAccess::gradients(scale); + record->zero_point_gradient = quantization.asymmetric() + ? OperatorAccess::gradients(zero_point) + : nullptr; + record->length = OperatorAccess::length(output); + record->quantization = quantization; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class QuantizeExpression +{ +public: + QuantizeExpression(const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + : input_(input), scale_(scale), zero_point_(zero_point), + quantization_(quantization) {} + void evaluate(BufferView &output) const noexcept + { + QuantizeOperator::evaluate(output, input_, scale_, zero_point_, + quantization_); + } +private: + BufferView input_; + BufferView scale_; + BufferView zero_point_; + Int8Quantization quantization_; +}; + +template +inline QuantizeExpression quantize( + const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization = Int8Quantization::activation()) noexcept +{ + return QuantizeExpression(input, scale, zero_point, quantization); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/example.cproject.yml b/dsppp/example.cproject.yml index 54bdfc41d..3018c1184 100644 --- a/dsppp/example.cproject.yml +++ b/dsppp/example.cproject.yml @@ -7,7 +7,8 @@ project: #- file: Examples/matrix_op.cpp #- file: Examples/autodiff_regression.cpp #- file: Examples/autodiff_lms.cpp - - file: Examples/autodiff_iris.cpp + - file: Examples/autodiff_fully_connected_qat.cpp + #- file: Examples/autodiff_iris.cpp - file: clang_sse300.c for-context: - +MPS3-Corstone-300 diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index bbbf12493..08155b639 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -12,6 +12,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -631,6 +633,97 @@ static void test15() assert(value[0] == 0.5F); } +template +static void test16() +{ + // Q/DQ keeps float storage while reproducing LiteRT/CMSIS-NN signed int8 + // activation codes. Its combined backward pass is an STE in range and + // also exposes gradients for scale and zero-point learning. + Arena<2048, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + tape.template register_operator>(); + T input_value[] = {-20.0F, -0.26F, 0.24F, 30.0F}; + T input_gradient[4] = {}; + T scale_value = 0.1F; + T zero_point_value = -3.0F; + T code_value[4] = {}; + T output_value[4] = {}; + BufferView input = tape.view(input_value, input_gradient, 4U); + BufferView scale_parameter = tape.parameter(scale_value); + BufferView zero_point_parameter = tape.parameter(zero_point_value); + BufferView code = tape.output(code_value); + BufferView output = tape.output(output_value); + code = quantize(input, scale_parameter, zero_point_parameter); + output = dequantize(code, scale_parameter, zero_point_parameter); + + const float expected_code[] = {-128.0F, -6.0F, -1.0F, 127.0F}; + const float expected_output[] = {-12.5F, -0.3F, 0.2F, 13.0F}; + const float tolerance = std::is_same::value ? 1.0e-5F : 2.0e-2F; + for (std::size_t i = 0; i < 4U; ++i) + { + assert(close_to(code_value[i], expected_code[i], tolerance)); + assert(close_to(output_value[i], expected_output[i], tolerance)); + } + + const T seed[] = {1.0F, 1.0F, 1.0F, 1.0F}; + assert(tape.backward(output, seed, 4U)); + assert(close_to(input.gradient(0), 0.0F, tolerance)); + assert(close_to(input.gradient(1), 1.0F, tolerance)); + assert(close_to(input.gradient(2), 1.0F, tolerance)); + assert(close_to(input.gradient(3), 0.0F, tolerance)); + assert(close_to(scale_parameter.gradient(0), 4.2F, + std::is_same::value ? 2.0e-4F : 8.0e-2F)); + assert(close_to(zero_point_parameter.gradient(0), -0.2F, + std::is_same::value ? 2.0e-4F : 8.0e-3F)); + + // Weight quantization follows the backend's symmetric per-output-axis + // contract: [-127, 127], zero-point zero, one scale per row here. + Arena<1536, T> weight_arena; + Tape &weight_tape = weight_arena.tape(); + weight_tape.template register_operator>(); + weight_tape.template register_operator>(); + T weight_value[] = {-1.2F, -0.2F, 0.6F, -2.0F, 0.8F, 2.6F}; + T weight_scale_value[] = {0.1F, 0.2F}; + T weight_zero_value[] = {4.0F, -4.0F}; + T weight_code_value[6] = {}; + T weight_output_value[6] = {}; + BufferView weight = weight_tape.input(weight_value); + BufferView weight_scale = weight_tape.parameter(weight_scale_value); + BufferView weight_zero = weight_tape.input(weight_zero_value); + BufferView weight_code = weight_tape.output(weight_code_value); + BufferView weight_output = weight_tape.output(weight_output_value); + const Int8Quantization weight_quantization = + Int8Quantization::weights(2U, 3U); + weight_code = quantize(weight, weight_scale, weight_zero, + weight_quantization); + weight_output = dequantize(weight_code, weight_scale, weight_zero, + weight_quantization); + const float expected_weight_code[] = {-12.0F, -2.0F, 6.0F, + -10.0F, 4.0F, 13.0F}; + for (std::size_t i = 0; i < 6U; ++i) + { + assert(close_to(weight_code_value[i], expected_weight_code[i], + tolerance)); + assert(close_to(weight_output_value[i], + static_cast(weight_value[i]), tolerance)); + } + assert(cmsis_nn_offset(-3.0F) == 3); + assert(static_cast(weight_zero_value[0]) == 0.0F); + assert(static_cast(weight_zero_value[1]) == 0.0F); + + // Q/DQ owns parameter projection; training loops do not need to clamp + // values after an optimizer step. + scale_value = -1.0F; + zero_point_value = 200.0F; + { + RecordingScope inference(tape, false); + code = quantize(input, scale_parameter, zero_point_parameter); + } + assert(static_cast(scale_value) > 0.0F); + assert(static_cast(zero_point_value) == 127.0F); +} + template static void run_autodiff_tests() { @@ -649,6 +742,7 @@ static void run_autodiff_tests() test13(); test14(); test15(); + test16(); // Arena exhaustion is explicit and backward cannot return partial results. alignas(std::max_align_t) unsigned char tiny_memory[1]; From f6fbd1147c1afed02a9c24a18a5c456c31fdad50 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Thu, 13 Aug 2026 11:36:04 +0200 Subject: [PATCH 17/19] autodiff : Update README with new example --- dsppp/Include/dsppp/autodiff/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 65901cd2a..9a1e4b34a 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -80,6 +80,12 @@ filtering. Adam and tests it on 30 patterns excluded from training. A single macro selects the float32 or float16 implementation. +### Quantization-aware training + +`dsppp/Examples/autodiff_fully_connected_qat.cpp` demonstrates +quantization-aware training of a fully connected layer for later deployment +with CMSIS-NN or Ethos-U. + ## How reverse differentiation works here During the **forward pass**, each operator computes its output and, when From b42339d2009d9cfb38763a121053e0d913a4bd09 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Fri, 14 Aug 2026 10:08:06 +0200 Subject: [PATCH 18/19] autodiff : Added optimization for quantize / dequantize Added python script to convert a pytorch model to autodiff C++ version (Only support the operators in autodiff) --- dsppp/Include/dsppp/Helium/basic.hpp | 185 +++++ dsppp/Include/dsppp/Scalar/basic.hpp | 90 +++ dsppp/Include/dsppp/algorithms.hpp | 112 +++ dsppp/Include/dsppp/autodiff/README.md | 14 + dsppp/Include/dsppp/autodiff/doc/operators.md | 8 + .../dsppp/autodiff/doc/pytorch_conversion.md | 123 +++ .../dsppp/autodiff/operators/dequantize.hpp | 127 ++- .../dsppp/autodiff/operators/quantize.hpp | 114 ++- .../Include/dsppp/autodiff/scripts/README.md | 5 + .../dsppp/autodiff/scripts/iris_pytorch.py | 147 ++++ .../autodiff/scripts/pytorch_to_autodiff.py | 752 ++++++++++++++++++ .../scripts/test_pytorch_to_autodiff.py | 106 +++ dsppp/Include/dsppp/forward.hpp | 3 + dsppp/Include/dsppp/vec.hpp | 5 - dsppp/tests/autodiff_test.cpp | 31 + 15 files changed, 1750 insertions(+), 72 deletions(-) create mode 100644 dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md create mode 100644 dsppp/Include/dsppp/autodiff/scripts/README.md create mode 100644 dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py create mode 100644 dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py create mode 100644 dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py diff --git a/dsppp/Include/dsppp/Helium/basic.hpp b/dsppp/Include/dsppp/Helium/basic.hpp index 079bd5c97..1266da250 100644 --- a/dsppp/Include/dsppp/Helium/basic.hpp +++ b/dsppp/Include/dsppp/Helium/basic.hpp @@ -18,6 +18,191 @@ */ #if defined(ARM_MATH_MVEI) || defined(ARM_MATH_MVEF) + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_to_nearest(DST& destination, const SRC& source, + const vector_length_t length, + const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + value = vrndnq_f32(value); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + value = vrndnq_f16(value); +#endif + destination.vector_store_tail(i, length - i, value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + const auto minimum_vector = inner::vconst(minimum); + const auto maximum_vector = inner::vconst(maximum); + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + value = vrndnq_f32(value); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + value = vrndnq_f16(value); +#endif + value = inner::vadd(value, offset); + value = vmaxnmq(value, minimum_vector); + value = vminnmq(value, maximum_vector); + destination.vector_store_tail(i, length - i, value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + { + value = vrndnq_f32(vmulq_n_f32(value, multiplier)); + value = vaddq_n_f32(value, offset); + value = vmaxnmq(value, inner::vconst(minimum)); + value = vminnmq(value, inner::vconst(maximum)); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + auto bottom = vrndnq_f32( + vmulq_n_f32(vcvtbq_f32_f16(value), multiplier)); + auto top = vrndnq_f32( + vmulq_n_f32(vcvttq_f32_f16(value), multiplier)); + const float offset_f32 = static_cast(offset); + const auto minimum_f32 = vdupq_n_f32(static_cast(minimum)); + const auto maximum_f32 = vdupq_n_f32(static_cast(maximum)); + bottom = vmaxnmq(vaddq_n_f32(bottom, offset_f32), minimum_f32); + top = vmaxnmq(vaddq_n_f32(top, offset_f32), minimum_f32); + bottom = vminnmq(bottom, maximum_f32); + top = vminnmq(top, maximum_f32); + value = vcvtbq_f16_f32(value, bottom); + value = vcvttq_f16_f32(value, top); + } +#endif + destination.vector_store_tail(i, length - i, value); + } +} + +template +inline mve_pred16_t _nearest_even_range_predicate( + const MASK& mask, index_t i, vector_length_t remaining, + const Helium* = nullptr) +{ + using T = typename MASK::Scalar; + const mve_pred16_t tail = inner::vctpq::mk(remaining); + auto value = mask.values().vector_op_tail(i, remaining); + if constexpr (std::is_same::value) + { + value = vrndnq_f32(vmulq_n_f32(value, mask.multiplier())); + value = inner::vadd(value, mask.offset(), tail); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + auto bottom = vrndnq_f32( + vmulq_n_f32(vcvtbq_f32_f16(value), mask.multiplier())); + auto top = vrndnq_f32( + vmulq_n_f32(vcvttq_f32_f16(value), mask.multiplier())); + const float offset = static_cast(mask.offset()); + bottom = vaddq_n_f32(bottom, offset); + top = vaddq_n_f32(top, offset); + value = vcvtbq_f16_f32(value, bottom); + value = vcvttq_f16_f32(value, top); + } +#endif + mve_pred16_t selected = vcmpgeq_m(value, mask.minimum(), tail); + return vcmpleq_m(value, mask.maximum(), selected); +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale, + const vector_length_t length, + const Helium* architecture = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + const mve_pred16_t selected = + _nearest_even_range_predicate(mask, i, length - i, architecture); + auto destination_value = destination.vector_op_tail(i, length - i); + destination_value = vfmaq_m( + destination_value, source.vector_op_tail(i, length - i), + scale, selected); + destination.vector_store_tail(i, length - i, destination_value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline auto _masked_dot_sum(const A& a, const B& b, const MASK& mask, + const vector_length_t length, + const Helium* architecture = nullptr) +{ + using T = typename traits::Scalar; + using Vector = typename vector_traits::vector; + constexpr int lanes = vector_traits::nb_lanes; + Vector dot = vector_traits::temp_acc_zero(); + Vector sum = vector_traits::temp_acc_zero(); + for (index_t i = 0; i < length; i += lanes) + { + const mve_pred16_t selected = + _nearest_even_range_predicate(mask, i, length - i, architecture); + const auto first = a.vector_op_tail(i, length - i); + dot = inner::vmacc(dot, first, b.vector_op_tail(i, length - i), + selected); + sum = vaddq_m(sum, sum, first, selected); + } + return MaskedDotSum{inner::vreduce(dot), inner::vreduce(sum)}; +} + /** * @brief Fill evaluator for Helium * diff --git a/dsppp/Include/dsppp/Scalar/basic.hpp b/dsppp/Include/dsppp/Scalar/basic.hpp index 528b76458..2f5b51a0b 100644 --- a/dsppp/Include/dsppp/Scalar/basic.hpp +++ b/dsppp/Include/dsppp/Scalar/basic.hpp @@ -12,6 +12,96 @@ #define SCALAR_UNROLL 2 +template +inline void _round_to_nearest(DST& destination, const SRC& source, + const vector_length_t length, + const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + destination[i] = static_cast( + std::nearbyint(static_cast(source[i]))); +} + +template +inline void _round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + { + float value = std::nearbyint(static_cast(source[i])) + + static_cast(offset); + if (value < static_cast(minimum)) + value = static_cast(minimum); + if (value > static_cast(maximum)) + value = static_cast(maximum); + destination[i] = static_cast(value); + } +} + +template +inline void _round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + { + float value = std::nearbyint( + static_cast(source[i]) * multiplier) + + static_cast(offset); + if (value < static_cast(minimum)) + value = static_cast(minimum); + if (value > static_cast(maximum)) + value = static_cast(maximum); + destination[i] = static_cast(value); + } +} + +template +inline bool _nearest_even_range_predicate( + const MASK& mask, index_t i, vector_length_t, + const Scalar* = nullptr) +{ + return mask[i]; +} + +template +inline void _masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale, + const vector_length_t length, + const Scalar* architecture = nullptr) +{ + for (index_t i = 0; i < length; ++i) + if (_nearest_even_range_predicate(mask, i, 1, architecture)) + destination[i] += source[i] * scale; +} + +template +inline auto _masked_dot_sum(const A& a, const B& b, const MASK& mask, + const vector_length_t length, + const Scalar* architecture = nullptr) +{ + using T = typename traits::Scalar; + MaskedDotSum result{T{}, T{}}; + for (index_t i = 0; i < length; ++i) + if (_nearest_even_range_predicate(mask, i, 1, architecture)) + { + result.dot += a[i] * b[i]; + result.sum += a[i]; + } + return result; +} + /** * @brief Fill evaluator for scalar architecture * diff --git a/dsppp/Include/dsppp/algorithms.hpp b/dsppp/Include/dsppp/algorithms.hpp index 921b89681..6f3309620 100644 --- a/dsppp/Include/dsppp/algorithms.hpp +++ b/dsppp/Include/dsppp/algorithms.hpp @@ -2,6 +2,8 @@ /** @file */ #pragma once +#include + /** \defgroup DSPPP C++ extension * C++ template extension to CMSIS-DSP. It is not yet part of * the pack but the headers can be found on the @@ -23,6 +25,116 @@ namespace arm_cmsis_dsp { * Algorithms written in an architecture independent way */ +/** Result of a masked dot product and sum computed in one traversal. */ +template +struct MaskedDotSum +{ + T dot; + T sum; +}; + +/** + * Mask selecting values whose scaled nearest-even integer lies in a range. + * + * The test is inclusive and evaluates + * `minimum <= nearbyint(value * multiplier) + offset <= maximum`. + */ +template +class NearestEvenRangeMask +{ +public: + using Scalar = typename traits::Scalar; + using Stored = typename VecRef::type; + + NearestEvenRangeMask(const V& values, float multiplier, Scalar offset, + Scalar minimum, Scalar maximum) + : values_(VecRef::ref(values)), multiplier_(multiplier), + offset_(offset), minimum_(minimum), maximum_(maximum) {} + + bool operator[](index_t i) const + { + const float rounded = std::nearbyint( + static_cast(values_[i]) * multiplier_); + const float selected = rounded + static_cast(offset_); + return selected >= static_cast(minimum_) && + selected <= static_cast(maximum_); + } + + vector_length_t length() const { return values_.length(); } + + const Stored& values() const { return values_; } + float multiplier() const { return multiplier_; } + Scalar offset() const { return offset_; } + Scalar minimum() const { return minimum_; } + Scalar maximum() const { return maximum_; } + +private: + Stored values_; + float multiplier_; + Scalar offset_; + Scalar minimum_; + Scalar maximum_; +}; + +template +inline auto nearest_even_range_mask( + const V& values, float multiplier, + typename traits::Scalar offset, typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + return NearestEvenRangeMask(values, multiplier, offset, + minimum, maximum); +} + +/** Scale in float, round to nearest-even, offset, and clip. */ +template +inline void round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + _round_scaled_to_nearest_clipped( + destination, source, multiplier, offset, minimum, maximum, + destination.length(), CURRENT_ARCH); +} + +/** Element-wise nearest-even rounding into an existing vector. */ +template +inline void round_to_nearest(DST& destination, const SRC& source) +{ + _round_to_nearest(destination, source, destination.length(), CURRENT_ARCH); +} + +/** Nearest-even rounding followed by an offset and inclusive clipping. */ +template +inline void round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + _round_to_nearest_clipped(destination, source, offset, minimum, maximum, + destination.length(), CURRENT_ARCH); +} + +/** Accumulate `source * scale` into destination where mask is true. */ +template +inline void masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale) +{ + _masked_scale_add(destination, source, mask, scale, + destination.length(), CURRENT_ARCH); +} + +/** Compute a masked dot product and masked sum of the first operand. */ +template +inline auto masked_dot_sum(const A& a, const B& b, const MASK& mask) +{ + return _masked_dot_sum(a, b, mask, a.length(), CURRENT_ARCH); +} + /* Matrix transpose diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 9a1e4b34a..5fa510396 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -162,6 +162,20 @@ int main() - [Worked implementation flow: `y = a * x`](doc/implementation_flow.md) follows one expression through `reverse.hpp`, including its tape record, `producer`, node links, gradient reset, seed, and backward rule. +- [Converting PyTorch modules](doc/pytorch_conversion.md) documents supported + mappings, batch and shape constraints, generated code, and the Iris + comparison script. + +## PyTorch conversion + +[`scripts/pytorch_to_autodiff.py`](scripts/pytorch_to_autodiff.py) converts a +supported batch-1 PyTorch `nn.Module` to a reusable CMSIS-DSP autodiff C++ +class. It rejects unsupported operators and values that cannot be represented +as scalars, vectors, or matrices. The accompanying +[`scripts/iris_pytorch.py`](scripts/iris_pytorch.py) provides a PyTorch version +of the Iris network, dataset preparation, training loop, and test split. +See [Converting PyTorch modules](doc/pytorch_conversion.md) for supported +mappings, constraints, generated code, and usage. ## Tests diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md index 143e491eb..d3017c425 100644 --- a/dsppp/Include/dsppp/autodiff/doc/operators.md +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -120,6 +120,14 @@ zero-point derivative at saturation, allowing the representable range to be learned. The forward path is the same affine quantize/dequantize calculation used when exporting the final int8 values. +Dequantization is affine, so its forward calculation and backward input +gradient use fused CMSIS-DSP C++ vector expressions. Its parameter gradients +use vector dot-product and accumulation kernels. Quantization uses the C++ +extension's nearest-even rounding, range-mask, masked accumulation, and masked +dot/sum algorithms. Their Helium implementations use MVE rounding and +predication directly without allocating a mask buffer. Other architectures +use portable scalar implementations based on `std::nearbyint`. + CMSIS-NN calls the negated zero-point an `offset`. Use `cmsis_nn_offset(zero_point)` when filling APIs such as `input_offset` or `output_offset`. Biases are not processed by this Q/DQ pair: CMSIS-NN/LiteRT diff --git a/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md b/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md new file mode 100644 index 000000000..c49171a54 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md @@ -0,0 +1,123 @@ +# Converting PyTorch modules + +`scripts/pytorch_to_autodiff.py` converts a PyTorch `nn.Module` into a +reusable C++ class using the CMSIS-DSP autodiff API. The generated class owns +the trained parameter values, tape views, intermediate buffers, operator +registration, `forward()`, and an `add_parameters()` optimizer helper. + +## Model constraints + +The input has an implicit batch dimension of one. Pass only the embedded shape +to the converter. For example, a PyTorch input shaped `(1, 4)` uses +`--input-shape 4`. Every traced value is checked after removing that batch +dimension. Scalars, vectors, and matrices are accepted; higher-rank tensors or +a batch other than one stop conversion with an error. + +The current mappings are: + +| PyTorch | CMSIS-DSP autodiff | +| --- | --- | +| `nn.Linear` | `fully_connected` | +| `nn.ReLU`, `torch.relu`, functional ReLU | `relu` | +| vector `nn.Softmax`, `torch.softmax`, functional softmax | `softmax` | +| `nn.Dropout`, functional dropout | `dropout` | +| `nn.Identity` | no operation | +| equal-shaped `+`, `-`, `*` | elementwise add, subtract, multiply | +| tensor and scalar-parameter `+`, `*` | `offset`, `scale` | +| vector `matmul`, `inner`, `dot` | `dot` | + +In-place operations, multiple inputs or outputs, unsupported ATen operators, +and incompatible shapes are rejected. A +bias-free `nn.Linear` receives a fixed zero bias because the autodiff fully +connected operator requires a bias view; that synthetic bias is not added to +the optimizer. + +## Running the converter + +From `dsppp`, explicitly select `dsppp/.venv`. The `--no-project` option is +important because otherwise uv finds the repository-level `pyproject.toml` +and may run with the repository-level `.venv` instead: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --input-shape 4 --factory create_model -o generated_iris.hpp +``` + +Install or inspect packages in that same environment by specifying its Python +interpreter as well: + +```text +uv pip install --python .venv\Scripts\python.exe torch numpy +uv pip list --python .venv\Scripts\python.exe +``` + +Alternatively, activate `dsppp/.venv` and pass `--active` to `uv run`. A plain +`uv run` is intentionally not shown because project discovery can select a +different environment even when `dsppp/.venv` exists. + +The model file must provide a zero-argument factory returning an `nn.Module`: + +```python +def create_model() -> torch.nn.Module: + return MyNetwork() +``` + +Conversion uses `torch.export.export` with one zero-valued batch-1 input. The +result is a functional ATen graph with model parameters lifted into explicit +graph inputs. The converter reads their values and trainable state through the +exported graph signature and state dictionary, then accepts only its documented +ATen allowlist. It does not train the module. Dropout is always emitted with +its probability, independently of the PyTorch module's current training state. +On the C++ side it is active while the autodiff tape records training and is an +identity when recording is disabled for inference. +Use `--dtype float16` to emit a `float16_t` network instead of the default +float32 network. `--arena-bytes` selects the generated autodiff arena capacity. + +The converter supports two explicit export modes. The default +`--export-mode trained` embeds the module's current parameters and preserves a +trained module exactly. For a network that will be trained with autodiff, use +`--export-mode empty`. Empty mode emits zero-initialized parameter storage and a compact C++ +`initialize_parameters()` loop: fully connected weights use uniform +random initialization and biases start at zero. + +## Using generated code for training + +The generated constructor creates network buffers and registers network +operators, but does not call `tape.begin_graph()` or `tape.rewind_graph()`. +The application can therefore create target and loss buffers on the same tape, +register a loss operator, add parameters to an optimizer, and then begin and +reuse the graph in the normal training loop. + +The input values are written to the public `input_value` array. `forward()` +evaluates the converted network, and `output()` returns its final +`BufferView`. `add_parameters(optimizer)` adds every trainable PyTorch +parameter while omitting frozen and synthetic parameters. + +## Iris comparison + +`scripts/iris_pytorch.py` implements the same `4 -> 8 -> 3` classifier, +normalization, per-sample Adam updates, 120/30 training/test split, and +120-epoch default used by `dsppp/Examples/autodiff_iris.cpp`. It reads the +exact dataset from `dsppp/Examples/iris_data.hpp`. + +Export an empty network without running the Python training loop with: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --export generated_iris.hpp +``` + +This export uses generated C++ random initialization rather than embedding the +random values assigned by the newly constructed PyTorch module. + +With no arguments, the script runs the 120-epoch PyTorch comparison without +exporting. To train and then export the trained parameters, request the number +of epochs explicitly: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --epochs 120 --export generated_trained_iris.hpp +``` diff --git a/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp index d1d97e429..dc7b899f2 100644 --- a/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp @@ -3,6 +3,13 @@ #include #include +#include + +#include +#include +#include +#include + #include namespace arm_cmsis_dsp { @@ -33,42 +40,87 @@ template class DequantizeOperator contribution); } + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(0.0F, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_fill_f16(static_cast(0.0F), data, + static_cast(length)); +#endif + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; + } + + static T sum(const T *data, std::size_t length) noexcept + { + T result = T{}; + if constexpr (std::is_same::value) + arm_accumulate_f32(data, static_cast(length), &result); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_accumulate_f16(data, static_cast(length), &result); +#endif + else + for (std::size_t i = 0; i < length; ++i) result += data[i]; + return result; + } + + static std::size_t block_length(const Int8Quantization &quantization, + std::size_t remaining) noexcept + { + if (quantization.parameter_count() == 1U) + return remaining; + return remaining < quantization.inner_size() + ? remaining + : quantization.inner_size(); + } + static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) - { - record.output_gradient[i] = T{}; - if (record.input_gradient != nullptr) - record.input_gradient[i] = T{}; - } - for (std::size_t i = 0; i < record.quantization.parameter_count(); ++i) - { - record.scale_gradient[i] = T{}; - if (record.zero_point_gradient != nullptr) - record.zero_point_gradient[i] = T{}; - } + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + fill(record.scale_gradient, record.quantization.parameter_count()); + if (record.zero_point_gradient != nullptr) + fill(record.zero_point_gradient, + record.quantization.parameter_count()); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) + for (std::size_t offset = 0; offset < record.length;) { - const std::size_t p = record.quantization.parameter_index(i); - const float gradient = static_cast(record.output_gradient[i]); - const float scale = static_cast(record.scale_value[p]); - const float zero = record.quantization.asymmetric() - ? rounded(static_cast( - record.zero_point_value[p])) - : 0.0F; + const std::size_t p = record.quantization.parameter_index(offset); + const std::size_t length = block_length(record.quantization, + record.length - offset); + const T scale = record.scale_value[p]; + const T zero = record.quantization.asymmetric() + ? static_cast(rounded(static_cast( + record.zero_point_value[p]))) + : T{}; + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + offset, 0, length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value) + offset, 0, length); if (record.input_gradient != nullptr) - add(record.input_gradient[i], gradient * scale); - add(record.scale_gradient[p], - gradient * (static_cast(record.input_value[i]) - zero)); + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient + offset, 0, length); + input_gradient += output_gradient * scale; + } + add(record.scale_gradient[p], static_cast( + ::arm_cmsis_dsp::dot(output_gradient, input_value - zero))); if (record.zero_point_gradient != nullptr && record.quantization.asymmetric()) - add(record.zero_point_gradient[p], -gradient * scale); + add(record.zero_point_gradient[p], + -static_cast(scale) * + static_cast(sum(record.output_gradient + offset, + length))); + offset += length; } } @@ -125,16 +177,25 @@ template class DequantizeOperator if (!validate(*tape, output, input, scale, zero_point, quantization)) return false; #endif - for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + const T *input_value = OperatorAccess::values(input); + T *output_value = OperatorAccess::values(output); + const std::size_t output_length = OperatorAccess::length(output); + for (std::size_t offset = 0; offset < output_length;) { - const std::size_t p = quantization.parameter_index(i); - const float zero = quantization.asymmetric() - ? rounded(static_cast( - OperatorAccess::values(zero_point)[p])) - : 0.0F; - OperatorAccess::values(output)[i] = static_cast( - (static_cast(OperatorAccess::values(input)[i]) - zero) * - static_cast(OperatorAccess::values(scale)[p])); + const std::size_t p = quantization.parameter_index(offset); + const std::size_t length = block_length( + quantization, output_length - offset); + const T zero = quantization.asymmetric() + ? static_cast(rounded(static_cast( + OperatorAccess::values(zero_point)[p]))) + : T{}; + ::arm_cmsis_dsp::VectorView input_block( + const_cast(input_value) + offset, 0, length); + ::arm_cmsis_dsp::VectorView output_block( + output_value + offset, 0, length); + output_block = (input_block - zero) * + OperatorAccess::values(scale)[p]; + offset += length; } if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) diff --git a/dsppp/Include/dsppp/autodiff/operators/quantize.hpp b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp index 8bc9e71a3..98d89aa44 100644 --- a/dsppp/Include/dsppp/autodiff/operators/quantize.hpp +++ b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp @@ -3,6 +3,11 @@ #include #include +#include + +#include +#include + #include namespace arm_cmsis_dsp { @@ -33,48 +38,83 @@ template class QuantizeOperator contribution); } + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(0.0F, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_fill_f16(static_cast(0.0F), data, + static_cast(length)); +#endif + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; + } + + static std::size_t block_length(const Int8Quantization &quantization, + std::size_t remaining) noexcept + { + if (quantization.parameter_count() == 1U) + return remaining; + return remaining < quantization.inner_size() + ? remaining + : quantization.inner_size(); + } + static void reset(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) - { - record.output_gradient[i] = T{}; - if (record.input_gradient != nullptr) - record.input_gradient[i] = T{}; - } - for (std::size_t i = 0; i < record.quantization.parameter_count(); ++i) - { - record.scale_gradient[i] = T{}; - if (record.zero_point_gradient != nullptr) - record.zero_point_gradient[i] = T{}; - } + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + fill(record.scale_gradient, record.quantization.parameter_count()); + if (record.zero_point_gradient != nullptr) + fill(record.zero_point_gradient, + record.quantization.parameter_count()); } static void backward(detail::Node &node) noexcept { Record &record = reinterpret_cast(node); - for (std::size_t i = 0; i < record.length; ++i) + for (std::size_t offset = 0; offset < record.length;) { - const std::size_t p = record.quantization.parameter_index(i); + const std::size_t p = record.quantization.parameter_index(offset); + const std::size_t length = block_length(record.quantization, + record.length - offset); const float scale = static_cast(record.scale_value[p]); + const float inverse_scale = 1.0F / scale; + const float inverse_scale_squared = inverse_scale * inverse_scale; const float zero_point = record.quantization.asymmetric() ? rounded(static_cast( record.zero_point_value[p])) : 0.0F; - const float input = static_cast(record.input_value[i]); - const float raw_code = rounded(input / scale) + zero_point; - if (raw_code < static_cast(record.quantization.quant_min()) || - raw_code > static_cast(record.quantization.quant_max())) - continue; - - const float gradient = static_cast(record.output_gradient[i]); + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + offset, 0, length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value) + offset, 0, length); + const auto selected = ::arm_cmsis_dsp::nearest_even_range_mask( + input_value, inverse_scale, + static_cast(zero_point), + static_cast(record.quantization.quant_min()), + static_cast(record.quantization.quant_max())); if (record.input_gradient != nullptr) - add(record.input_gradient[i], gradient / scale); + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient + offset, 0, length); + ::arm_cmsis_dsp::masked_scale_add( + input_gradient, output_gradient, selected, + static_cast(inverse_scale)); + } + const auto reductions = ::arm_cmsis_dsp::masked_dot_sum( + output_gradient, input_value, selected); add(record.scale_gradient[p], - -gradient * input / (scale * scale)); + -static_cast(reductions.dot) * + inverse_scale_squared); if (record.zero_point_gradient != nullptr && record.quantization.asymmetric()) - add(record.zero_point_gradient[p], gradient); + add(record.zero_point_gradient[p], + static_cast(reductions.sum)); + offset += length; } } @@ -149,23 +189,29 @@ template class QuantizeOperator return false; } - for (std::size_t i = 0; i < OperatorAccess::length(output); ++i) + const std::size_t output_length = OperatorAccess::length(output); + for (std::size_t offset = 0; offset < output_length;) { - const std::size_t p = quantization.parameter_index(i); + const std::size_t p = quantization.parameter_index(offset); + const std::size_t length = block_length( + quantization, output_length - offset); const float scale_value = static_cast(OperatorAccess::values(scale)[p]); + const float inverse_scale = 1.0F / scale_value; const float zero = quantization.asymmetric() ? rounded(static_cast( OperatorAccess::values(zero_point)[p])) : 0.0F; - float code = rounded(static_cast( - OperatorAccess::values(input)[i]) / - scale_value) + zero; - if (code < static_cast(quantization.quant_min())) - code = static_cast(quantization.quant_min()); - if (code > static_cast(quantization.quant_max())) - code = static_cast(quantization.quant_max()); - OperatorAccess::values(output)[i] = static_cast(code); + ::arm_cmsis_dsp::VectorView input_block( + const_cast(OperatorAccess::values(input)) + offset, + 0, length); + ::arm_cmsis_dsp::VectorView output_block( + OperatorAccess::values(output) + offset, 0, length); + ::arm_cmsis_dsp::round_scaled_to_nearest_clipped( + output_block, input_block, inverse_scale, static_cast(zero), + static_cast(quantization.quant_min()), + static_cast(quantization.quant_max())); + offset += length; } if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) diff --git a/dsppp/Include/dsppp/autodiff/scripts/README.md b/dsppp/Include/dsppp/autodiff/scripts/README.md new file mode 100644 index 000000000..1ac3d7461 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/README.md @@ -0,0 +1,5 @@ +# PyTorch conversion scripts + +See [Converting PyTorch modules](../doc/pytorch_conversion.md) for supported +operators, model constraints, converter usage, generated code integration, and +the PyTorch Iris comparison. diff --git a/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py b/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py new file mode 100644 index 000000000..75c7feab0 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""PyTorch counterpart of dsppp/Examples/autodiff_iris.cpp.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import numpy as np +import torch +from torch import nn + +from pytorch_to_autodiff import convert_module + + +INPUT_SIZE = 4 +HIDDEN_SIZE = 8 +CLASS_COUNT = 3 +TRAINING_COUNT = 120 + + +class IrisNetwork(nn.Module): + def __init__(self) -> None: + super().__init__() + self.hidden = nn.Linear(INPUT_SIZE, HIDDEN_SIZE) + self.relu = nn.ReLU() + self.output = nn.Linear(HIDDEN_SIZE, CLASS_COUNT) + self.softmax = nn.Softmax(dim=-1) + + def logits(self, value: torch.Tensor) -> torch.Tensor: + return self.output(self.relu(self.hidden(value))) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return self.softmax(self.logits(value)) + + +def create_model() -> nn.Module: + """Factory used by pytorch_to_autodiff.py.""" + return IrisNetwork() + + +def load_dataset() -> tuple[torch.Tensor, torch.Tensor]: + """Load the exact samples and normalization used by the C++ example.""" + header = Path(__file__).resolve().parents[4] / "Examples" / "iris_data.hpp" + text = header.read_text(encoding="utf-8") + pattern = re.compile( + r"\{\{\s*([-+0-9.]+)F,\s*([-+0-9.]+)F,\s*" + r"([-+0-9.]+)F,\s*([-+0-9.]+)F\s*\},\s*([0-2])\s*\}" + ) + rows = pattern.findall(text) + if len(rows) != 150: + raise RuntimeError(f"expected 150 Iris samples in {header}, found {len(rows)}") + features = np.asarray([[float(value) for value in row[:4]] for row in rows], dtype=np.float32) + labels = np.asarray([int(row[4]) for row in rows], dtype=np.int64) + mean = np.asarray([5.843333, 3.057333, 3.758000, 1.199333], dtype=np.float32) + inverse_std = np.asarray([1.211678, 2.301971, 0.568374, 1.316322], dtype=np.float32) + features = (features - mean) * inverse_std + return torch.from_numpy(features), torch.from_numpy(labels) + + +def is_test_sample(index: int) -> bool: + return (index % 50) % 5 == 0 + + +def initialize_like_cpp(model: IrisNetwork, generator: torch.Generator) -> None: + with torch.no_grad(): + for layer in (model.hidden, model.output): + layer.weight.uniform_(-0.25, 0.25, generator=generator) + layer.bias.zero_() + + +def train(model: IrisNetwork, epochs: int, seed: int) -> tuple[int, int]: + features, labels = load_dataset() + training_indices = torch.tensor( + [index for index in range(len(labels)) if not is_test_sample(index)], dtype=torch.long + ) + test_indices = torch.tensor( + [index for index in range(len(labels)) if is_test_sample(index)], dtype=torch.long + ) + generator = torch.Generator().manual_seed(seed) + initialize_like_cpp(model, generator) + optimizer = torch.optim.Adam(model.parameters(), lr=1.0e-2) + loss_function = nn.CrossEntropyLoss(reduction="sum") + + model.train() + for epoch in range(epochs): + order = training_indices[torch.randperm(TRAINING_COUNT, generator=generator)] + epoch_loss = 0.0 + for sample in order.tolist(): + optimizer.zero_grad() + logits = model.logits(features[sample : sample + 1]) + loss = loss_function(logits, labels[sample : sample + 1]) + loss.backward() + optimizer.step() + epoch_loss += float(loss.detach()) + if (epoch + 1) % 20 == 0: + print(f"epoch {epoch + 1}: mean loss={epoch_loss / TRAINING_COUNT:g}") + + model.eval() + with torch.no_grad(): + detected = model(features[test_indices]).argmax(dim=1) + correct = int((detected == labels[test_indices]).sum()) + print(f"final test accuracy={correct}/30 tests") + return correct, len(test_indices) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--epochs", + type=int, + help="train for this many epochs; defaults to 120 when not exporting", + ) + parser.add_argument("--seed", type=int, default=0x12345678) + parser.add_argument("--export", type=Path, help="export trained model to an autodiff header") + parser.add_argument("--dtype", choices=("float32", "float16"), default="float32") + parser.add_argument("--arena-bytes", type=int, default=2048) + arguments = parser.parse_args() + if arguments.epochs is not None and arguments.epochs < 0: + parser.error("--epochs must be nonnegative") + + torch.manual_seed(arguments.seed) + model = IrisNetwork() + trained = arguments.epochs is not None or arguments.export is None + if trained: + train(model, arguments.epochs if arguments.epochs is not None else 120, + arguments.seed) + else: + model.eval() + if arguments.export is not None: + generated = convert_module( + model, + (INPUT_SIZE,), + class_name="IrisNetwork", + namespace="generated_iris", + dtype=arguments.dtype, + arena_bytes=arguments.arena_bytes, + export_mode="trained" if trained else "empty", + ) + arguments.export.write_text(generated, encoding="utf-8") + print(f"wrote {arguments.export}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py b/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py new file mode 100644 index 000000000..4bfd6f16c --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py @@ -0,0 +1,752 @@ +#!/usr/bin/env python3 +"""Convert a supported PyTorch nn.Module to CMSIS-DSP autodiff C++ code.""" + +from __future__ import annotations + +import argparse +import importlib.util +import math +import re +import sys +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any, Sequence + +import torch +from torch import fx, nn + + +class ConversionError(RuntimeError): + """Raised when a module cannot be represented by the autodiff API.""" + + +@dataclass(frozen=True) +class Value: + name: str + shape: tuple[int, ...] + + @property + def length(self) -> int: + return math.prod(self.shape) if self.shape else 1 + + +@dataclass(frozen=True) +class Parameter: + name: str + shape: tuple[int, ...] + values: tuple[float, ...] + trainable: bool + matrix_view: bool = False + initialization: str = "preserve" + + +@dataclass(frozen=True) +class Operation: + kind: str + output: Value + inputs: tuple[str, ...] + attributes: tuple[Any, ...] = () + + +@dataclass(frozen=True) +class ConvertedModule: + input: Value + output_name: str + values: tuple[Value, ...] + parameters: tuple[Parameter, ...] + operations: tuple[Operation, ...] + + +def _identifier(name: str) -> str: + result = re.sub(r"[^A-Za-z0-9_]", "_", name) + if not result or result[0].isdigit(): + result = "value_" + result + return result + + +def _shape_text(shape: Sequence[int]) -> str: + return "scalar" if not shape else " x ".join(str(v) for v in shape) + + +def _normalized_shape(node: fx.Node) -> tuple[int, ...]: + value = node.meta.get("val") + if not isinstance(value, torch.Tensor): + raise ConversionError( + f"node '{node.name}' does not produce a tensor with a known shape" + ) + shape = tuple(int(dimension) for dimension in value.shape) + if not shape or shape[0] != 1: + raise ConversionError( + f"node '{node.name}' has shape {shape}; the leading batch dimension " + "must exist and equal 1" + ) + embedded_shape = shape[1:] + if len(embedded_shape) > 2: + raise ConversionError( + f"node '{node.name}' becomes {_shape_text(embedded_shape)} after " + "removing batch 1; autodiff supports only scalars, vectors, and matrices" + ) + return embedded_shape + + +def _tensor_values(tensor: torch.Tensor) -> tuple[float, ...]: + flat = tensor.detach().cpu().reshape(-1).tolist() + result = tuple(float(value) for value in flat) + if not all(math.isfinite(value) for value in result): + raise ConversionError("model parameters must contain only finite values") + return result + + +def _node_argument(argument: Any, aliases: dict[fx.Node, str]) -> str: + if not isinstance(argument, fx.Node) or argument not in aliases: + raise ConversionError("operator arguments must be traced tensor values") + return aliases[argument] + + +def _validate_vector_softmax(node: fx.Node, shape: tuple[int, ...], dim: int) -> None: + if len(shape) != 1 or dim not in (-1, 1): + raise ConversionError( + f"softmax node '{node.name}' must operate on the feature dimension " + "of a batch-1 vector; got embedded shape " + _shape_text(shape) + ) + + +def inspect_module(module: nn.Module, input_shape: Sequence[int]) -> ConvertedModule: + """Export and validate a module, returning an architecture-neutral model.""" + shape = tuple(int(value) for value in input_shape) + if any(value <= 0 for value in shape): + raise ConversionError("all input dimensions must be positive") + if len(shape) > 2: + raise ConversionError( + "input_shape excludes batch and may describe only a scalar, vector, or matrix" + ) + + model_parameters = dict(module.named_parameters()) + first_parameter = next(iter(model_parameters.values()), None) + dtype = first_parameter.dtype if first_parameter is not None else torch.float32 + device = first_parameter.device if first_parameter is not None else torch.device("cpu") + example = torch.zeros((1, *shape), dtype=dtype, device=device) + try: + exported = torch.export.export(module, (example,)) + except Exception as error: + raise ConversionError(f"PyTorch ATen export failed: {error}") from error + + graph = exported.graph_module.graph + specs = {spec.arg.name: spec for spec in exported.graph_signature.input_specs} + aliases: dict[fx.Node, str] = {} + shapes: dict[str, tuple[int, ...]] = {} + values: list[Value] = [] + parameters: list[Parameter] = [] + parameter_indices: dict[str, int] = {} + operations: list[Operation] = [] + input_value: Value | None = None + output_name: str | None = None + + def tensor_for_spec(spec: Any) -> torch.Tensor: + target = str(spec.target) + kind = spec.kind.name + if kind in ("PARAMETER", "BUFFER"): + tensor = exported.state_dict[target] + elif kind == "CONSTANT_TENSOR": + tensor = exported.constants[target] + else: + raise ConversionError(f"unsupported lifted input kind {kind}") + if not isinstance(tensor, torch.Tensor): + raise ConversionError(f"lifted input '{target}' is not a tensor") + return tensor + + def unique_parameter_name(target: str) -> str: + base = _identifier(target) + name = base + suffix = 2 + while name in parameter_indices: + name = f"{base}_{suffix}" + suffix += 1 + return name + + for node in graph.nodes: + if node.op != "placeholder": + continue + spec = specs.get(str(node.target)) + if spec is None: + raise ConversionError(f"placeholder '{node.name}' has no graph signature entry") + if spec.kind.name == "USER_INPUT": + if input_value is not None: + raise ConversionError("exactly one tensor input is supported") + input_value = Value("input", _normalized_shape(node)) + aliases[node] = input_value.name + shapes[input_value.name] = input_value.shape + continue + tensor = tensor_for_spec(spec) + parameter_shape = tuple(int(value) for value in tensor.shape) + if len(parameter_shape) > 2: + raise ConversionError( + f"parameter '{spec.target}' has unsupported shape {parameter_shape}" + ) + name = unique_parameter_name(str(spec.target)) + source_parameter = model_parameters.get(str(spec.target)) + parameter = Parameter( + name, + parameter_shape, + _tensor_values(tensor), + bool(source_parameter is not None and source_parameter.requires_grad), + ) + parameter_indices[name] = len(parameters) + parameters.append(parameter) + aliases[node] = name + shapes[name] = parameter_shape + + if input_value is None: + raise ConversionError("exactly one tensor input is supported") + + def mark_parameter(name: str, *, matrix_view: bool = False, + initialization: str = "preserve") -> None: + index = parameter_indices.get(name) + if index is None: + raise ConversionError(f"'{name}' is not a lifted model parameter") + current = parameters[index] + parameters[index] = replace( + current, + matrix_view=current.matrix_view or matrix_view, + initialization=( + initialization + if current.initialization == "preserve" + else current.initialization + ), + ) + + def new_output(node: fx.Node) -> Value: + value = Value(_identifier(node.name), _normalized_shape(node)) + values.append(value) + aliases[node] = value.name + shapes[value.name] = value.shape + return value + + def add_scalar_constant(value: Any, node_name: str) -> str: + try: + scalar = float(value) + except (TypeError, ValueError) as error: + raise ConversionError( + f"ATen node '{node_name}' has a non-numeric scalar operand" + ) from error + if not math.isfinite(scalar): + raise ConversionError(f"ATen node '{node_name}' has a non-finite scalar operand") + name = unique_parameter_name(node_name + "_constant") + parameter_indices[name] = len(parameters) + parameters.append(Parameter(name, (), (scalar,), False)) + shapes[name] = () + return name + + def operand(argument: Any, node_name: str) -> tuple[str, tuple[int, ...], bool]: + if isinstance(argument, fx.Node): + if argument not in aliases: + raise ConversionError(f"ATen node '{node_name}' uses an unsupported value") + name = aliases[argument] + return name, shapes[name], name in parameter_indices + name = add_scalar_constant(argument, node_name) + return name, (), True + + def add_arithmetic(node: fx.Node, kind: str) -> None: + if len(node.args) < 2: + raise ConversionError(f"ATen arithmetic node '{node.name}' needs two operands") + if float(node.args[2] if len(node.args) > 2 else node.kwargs.get("alpha", 1.0)) != 1.0: + raise ConversionError( + f"ATen arithmetic node '{node.name}' uses unsupported alpha" + ) + left_name, left_shape, left_parameter = operand(node.args[0], node.name) + right_name, right_shape, right_parameter = operand(node.args[1], node.name) + output = new_output(node) + if left_shape == right_shape: + if output.shape != left_shape: + raise ConversionError(f"ATen arithmetic node '{node.name}' changed shape") + operations.append(Operation(kind, output, (left_name, right_name))) + return + if kind in ("add", "multiply"): + if not left_shape and left_parameter and right_shape == output.shape: + scalar_name, value_name = left_name, right_name + elif not right_shape and right_parameter and left_shape == output.shape: + scalar_name, value_name = right_name, left_name + else: + raise ConversionError( + f"ATen node '{node.name}' uses broadcasting not supported by autodiff" + ) + operations.append(Operation( + "offset" if kind == "add" else "scale", + output, + (value_name, scalar_name), + )) + return + raise ConversionError( + f"ATen subtraction node '{node.name}' requires equal-shaped operands" + ) + + def add_dot(node: fx.Node) -> None: + left_name, left_shape, _ = operand(node.args[0], node.name) + right_name, right_shape, _ = operand(node.args[1], node.name) + output = new_output(node) + if len(left_shape) != 1 or left_shape != right_shape or output.shape: + raise ConversionError( + f"ATen node '{node.name}' is not a vector dot product" + ) + operations.append(Operation("dot", output, (left_name, right_name))) + + aten = torch.ops.aten + arithmetic = { + aten.add.Tensor: "add", + aten.add.Scalar: "add", + aten.sub.Tensor: "sub", + aten.sub.Scalar: "sub", + aten.mul.Tensor: "multiply", + aten.mul.Scalar: "multiply", + } + dot_targets = {aten.matmul.default, aten.inner.default, aten.dot.default} + + for node in graph.nodes: + if node.op == "placeholder": + continue + if node.op == "output": + returned = node.args[0] + if not isinstance(returned, (tuple, list)) or len(returned) != 1: + raise ConversionError("the module must return one tensor") + output_name = _node_argument(returned[0], aliases) + continue + if node.op != "call_function": + raise ConversionError( + f"unsupported exported graph node '{node.name}' of kind '{node.op}'" + ) + + target = node.target + if target == aten.linear.default: + source = _node_argument(node.args[0], aliases) + weight = _node_argument(node.args[1], aliases) + source_shape = shapes[source] + weight_shape = shapes[weight] + if len(source_shape) != 1 or len(weight_shape) != 2: + raise ConversionError(f"ATen linear node '{node.name}' requires vector input") + if source_shape[0] != weight_shape[1]: + raise ConversionError(f"ATen linear node '{node.name}' has incompatible dimensions") + mark_parameter(weight, matrix_view=True, initialization="xavier") + if len(node.args) < 3 or node.args[2] is None: + bias = unique_parameter_name(node.name + "_bias") + parameter_indices[bias] = len(parameters) + parameters.append( + Parameter(bias, (weight_shape[0],), (0.0,) * weight_shape[0], False) + ) + shapes[bias] = (weight_shape[0],) + else: + bias = _node_argument(node.args[2], aliases) + if shapes[bias] != (weight_shape[0],): + raise ConversionError(f"ATen linear node '{node.name}' has invalid bias") + mark_parameter(bias, initialization="zero") + output = new_output(node) + if output.shape != (weight_shape[0],): + raise ConversionError(f"ATen linear node '{node.name}' has invalid output") + operations.append(Operation("fully_connected", output, (source, weight, bias))) + elif target == aten.relu.default: + output = new_output(node) + operations.append(Operation("relu", output, (_node_argument(node.args[0], aliases),))) + elif target in (aten.softmax.int, aten._softmax.default): + output = new_output(node) + dimension = int(node.args[1]) + _validate_vector_softmax(node, output.shape, dimension) + operations.append(Operation("softmax", output, (_node_argument(node.args[0], aliases),))) + elif target == aten.dropout.default: + source = _node_argument(node.args[0], aliases) + probability = float(node.args[1]) + if not 0.0 <= probability < 1.0: + raise ConversionError(f"ATen dropout node '{node.name}' has invalid probability") + output = new_output(node) + operations.append(Operation("dropout", output, (source,), (probability,))) + elif target in arithmetic: + add_arithmetic(node, arithmetic[target]) + elif target in dot_targets: + add_dot(node) + else: + raise ConversionError( + f"unsupported ATen operator at node '{node.name}': {target}" + ) + + if output_name is None: + raise ConversionError("the exported module has no tensor output") + return ConvertedModule( + input_value, + output_name, + tuple(values), + tuple(parameters), + tuple(operations), + ) + + +def _float_literal(value: float) -> str: + text = format(value, ".9g") + if "e" not in text.lower() and "." not in text: + text += ".0" + return f"static_cast({text}F)" + + +def _initializer(parameter: Parameter) -> str: + if len(parameter.shape) == 2: + rows, columns = parameter.shape + groups = [] + for row in range(rows): + start = row * columns + groups.append( + "{" + ", ".join(_float_literal(v) for v in parameter.values[start:start + columns]) + "}" + ) + return "{" + ",\n ".join(groups) + "}" + return "{" + ", ".join(_float_literal(v) for v in parameter.values) + "}" + + +def _array_suffix(shape: tuple[int, ...]) -> str: + dimensions = shape if shape else (1,) + return "".join(f"[{dimension}]" for dimension in dimensions) + + +def generate_cpp( + converted: ConvertedModule, + *, + class_name: str = "GeneratedNetwork", + namespace: str = "generated_autodiff", + dtype: str = "float32", + arena_bytes: int = 4096, + export_mode: str = "trained", +) -> str: + """Generate a reusable C++ class backed by CMSIS-DSP autodiff.""" + if dtype not in ("float32", "float16"): + raise ConversionError("dtype must be float32 or float16") + if arena_bytes <= 0: + raise ConversionError("arena_bytes must be positive") + if export_mode not in ("trained", "empty"): + raise ConversionError("export_mode must be trained or empty") + class_name = _identifier(class_name) + namespace = _identifier(namespace) + + parameter_name_set = {parameter.name for parameter in converted.parameters} + value_name_set = {value.name for value in converted.values} + + def view_name(name: str) -> str: + if name == converted.input.name: + return "input" + if name in parameter_name_set: + return name + if name in value_name_set: + return name + "_buffer" + raise AssertionError(name) + + kinds = {operation.kind for operation in converted.operations} + header_for = { + "add": "add", + "sub": "sub", + "multiply": "multiply", + "scale": "scale", + "offset": "offset", + "dot": "dot", + "fully_connected": "fully_connected", + "relu": "relu", + "softmax": "softmax", + "dropout": "dropout", + } + operator_for = { + "add": "AddOperator", + "sub": "SubOperator", + "multiply": "MultiplyOperator", + "scale": "ScaleOperator", + "offset": "OffsetOperator", + "dot": "DotOperator", + "fully_connected": "FullyConnectedOperator", + "relu": "ReluOperator", + "softmax": "SoftmaxOperator", + "dropout": "DropoutOperator", + } + includes = [] + if "softmax" in kinds: + # Include these at global scope before the C++ expression headers. + # Some architecture headers include the same C headers from their + # namespace; the include guards must already have established the C + # declarations globally. + includes.extend( + [ + "#include ", + "#include ", + ] + ) + includes.append("#include ") + includes.extend( + f"#include " + for kind in sorted(kinds) + ) + includes.extend(["#include ", "#include "]) + + scalar = "float" if dtype == "float32" else "float16_t" + lines = [ + "// Generated by pytorch_to_autodiff.py. Do not edit parameter data by hand.", + f"// Export mode: {export_mode}.", + "#pragma once", + *includes, + "", + f"namespace {namespace} {{", + "using namespace arm_cmsis_dsp::autodiff;", + "", + f"class {class_name}", + "{", + "public:", + f" using Scalar = {scalar};", + f" static constexpr std::size_t arena_bytes = {arena_bytes}U;", + f" static constexpr std::size_t input_length = {converted.input.length}U;", + "", + f" Scalar input_value[{converted.input.length}]{{}};", + ] + for parameter in converted.parameters: + if export_mode == "empty" and parameter.trainable: + lines.append( + f" Scalar {parameter.name}_value{_array_suffix(parameter.shape)}{{}};" + ) + else: + lines.extend( + [ + f" Scalar {parameter.name}_value{_array_suffix(parameter.shape)} =", + f" {_initializer(parameter)};", + ] + ) + for value in converted.values: + lines.append(f" Scalar {value.name}_value[{value.length}]{{}};") + if "dropout" in kinds: + lines.append(" DropoutGenerator dropout_generator{0x6D2B79F5U};") + lines.extend( + [ + "", + " Arena arena{};", + " Tape &tape;", + " BufferView input;", + ] + ) + for parameter in converted.parameters: + view_type = "MatrixView" if parameter.matrix_view else "BufferView" + lines.append(f" {view_type} {parameter.name};") + for value in converted.values: + lines.append(f" BufferView {view_name(value.name)};") + + initializers = ["tape(arena.tape())", "input(tape.input(input_value))"] + for parameter in converted.parameters: + if len(parameter.shape) == 2 and not parameter.matrix_view: + length = math.prod(parameter.shape) + initializers.append( + f"{parameter.name}(tape.parameter(&{parameter.name}_value[0][0], " + f"{length}U))" + ) + else: + initializers.append( + f"{parameter.name}(tape.parameter({parameter.name}_value))" + ) + initializers.extend( + f"{view_name(value.name)}(tape.output({value.name}_value))" + for value in converted.values + ) + lines.extend(["", f" {class_name}()", " : " + ",\n ".join(initializers), " {"]) + for kind in sorted(kinds): + lines.append(f" tape.register_operator<{operator_for[kind]}>();") + if export_mode == "empty" and any( + parameter.trainable for parameter in converted.parameters + ): + lines.append(" initialize_parameters();") + lines.append(" }") + + if export_mode == "empty": + lines.extend( + [ + "", + " void initialize_parameters(std::uint32_t seed = 0x12345678U)", + " {", + " std::uint32_t state = seed;", + ] + ) + for parameter in converted.parameters: + if not parameter.trainable: + continue + if len(parameter.shape) == 2: + rows, columns = parameter.shape + bound = math.sqrt(6.0 / (rows + columns)) + lines.extend( + [ + f" for (std::size_t row = 0; row < {rows}U; ++row)", + f" for (std::size_t column = 0; column < {columns}U; ++column)", + " {", + " state = state * 1664525U + 1013904223U;", + " const float unit = static_cast(", + " (state >> 8U) & 0xffffU) / 65535.0F;", + f" {parameter.name}_value[row][column] = static_cast(", + f" (unit * 2.0F - 1.0F) * {format(bound, '.9g')}F);", + " }", + ] + ) + else: + length = math.prod(parameter.shape) if parameter.shape else 1 + lines.extend( + [ + f" for (std::size_t i = 0; i < {length}U; ++i)", + f" {parameter.name}_value[i] = Scalar{{}};", + ] + ) + lines.append(" }") + + lines.extend(["", " bool forward()", " {"]) + for operation in converted.operations: + output = view_name(operation.output.name) + if operation.kind == "fully_connected": + source, weight, bias = operation.inputs + expression = ( + f"::arm_cmsis_dsp::autodiff::fully_connected(" + f"{view_name(source)}, {view_name(weight)}, {view_name(bias)})" + ) + elif operation.kind in ("relu", "softmax"): + expression = ( + f"::arm_cmsis_dsp::autodiff::{operation.kind}(" + f"{view_name(operation.inputs[0])})" + ) + elif operation.kind == "add": + expression = ( + f"{view_name(operation.inputs[0])} + " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind == "sub": + expression = ( + f"{view_name(operation.inputs[0])} - " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind == "multiply": + expression = ( + f"{view_name(operation.inputs[0])} * " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind in ("scale", "offset"): + expression = ( + f"::arm_cmsis_dsp::autodiff::{operation.kind}(" + f"{view_name(operation.inputs[0])}, " + f"{view_name(operation.inputs[1])})" + ) + elif operation.kind == "dot": + expression = ( + f"::arm_cmsis_dsp::autodiff::dot(" + f"{view_name(operation.inputs[0])}, " + f"{view_name(operation.inputs[1])})" + ) + elif operation.kind == "dropout": + probability = format(operation.attributes[0], ".9g") + expression = ( + f"::arm_cmsis_dsp::autodiff::dropout(" + f"{view_name(operation.inputs[0])}, dropout_generator, " + f"{probability}F)" + ) + else: + raise AssertionError(operation.kind) + lines.append(f" {output} = {expression};") + lines.extend( + [ + " return tape.good();", + " }", + "", + " BufferView output() const", + " {", + f" return {view_name(converted.output_name)};", + " }", + "", + " template ", + " bool add_parameters(Optimizer &optimizer)", + " {", + " bool result = true;", + ] + ) + for parameter in converted.parameters: + if parameter.trainable: + lines.append(f" result = optimizer.add({parameter.name}) && result;") + lines.extend( + [ + " return result;", + " }", + "};", + "", + f"}} // namespace {namespace}", + "", + ] + ) + return "\n".join(lines) + + +def convert_module(module: nn.Module, input_shape: Sequence[int], **options: Any) -> str: + """Validate ``module`` and return generated CMSIS-DSP autodiff C++.""" + return generate_cpp(inspect_module(module, input_shape), **options) + + +def _parse_shape(text: str) -> tuple[int, ...]: + if text.strip().lower() in ("", "scalar"): + return () + try: + return tuple(int(value) for value in re.split(r"[x,]", text)) + except ValueError as error: + raise argparse.ArgumentTypeError("shape must look like 4 or 3,4") from error + + +def _load_factory(path: Path, factory_name: str) -> nn.Module: + path = path.resolve() + spec = importlib.util.spec_from_file_location("pytorch_autodiff_input", path) + if spec is None or spec.loader is None: + raise ConversionError(f"cannot import model file '{path}'") + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(path.parent)) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + factory = getattr(module, factory_name, None) + if not callable(factory): + raise ConversionError(f"'{path}' has no callable '{factory_name}'") + model = factory() + if not isinstance(model, nn.Module): + raise ConversionError(f"'{factory_name}' must return torch.nn.Module") + return model + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", type=Path, help="Python file containing a model factory") + parser.add_argument("--factory", default="create_model", help="zero-argument model factory") + parser.add_argument( + "--input-shape", + required=True, + type=_parse_shape, + help="shape excluding the mandatory batch dimension, for example 4 or 3,4", + ) + parser.add_argument("-o", "--output", type=Path, help="output header (stdout if omitted)") + parser.add_argument("--class-name", default="GeneratedNetwork") + parser.add_argument("--namespace", default="generated_autodiff") + parser.add_argument("--dtype", choices=("float32", "float16"), default="float32") + parser.add_argument("--arena-bytes", type=int, default=4096) + parser.add_argument( + "--export-mode", + choices=("trained", "empty"), + default="trained", + help="embed trained values, or emit an empty randomly initialized network", + ) + arguments = parser.parse_args(argv) + try: + model = _load_factory(arguments.model, arguments.factory) + generated = convert_module( + model, + arguments.input_shape, + class_name=arguments.class_name, + namespace=arguments.namespace, + dtype=arguments.dtype, + arena_bytes=arguments.arena_bytes, + export_mode=arguments.export_mode, + ) + if arguments.output is None: + sys.stdout.write(generated) + else: + arguments.output.write_text(generated, encoding="utf-8") + except (ConversionError, OSError) as error: + parser.exit(1, f"error: {error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py b/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py new file mode 100644 index 000000000..85ee0f063 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py @@ -0,0 +1,106 @@ +import unittest + +import torch +from torch import nn + +from pytorch_to_autodiff import ConversionError, convert_module + + +class SupportedModel(nn.Module): + def __init__(self): + super().__init__() + self.first = nn.Linear(4, 8) + self.relu = nn.ReLU() + self.second = nn.Linear(8, 3) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, value): + return self.softmax(self.second(self.relu(self.first(value)))) + + +class ArithmeticModel(nn.Module): + def __init__(self): + super().__init__() + self.scale = nn.Parameter(torch.tensor(2.0)) + self.offset = nn.Parameter(torch.tensor(-0.5)) + self.dot_weight = nn.Parameter(torch.ones(4)) + + def forward(self, value): + original = value + value = value + value + value = value - original + value = value * value + value = value * self.scale + value = value + self.offset + return torch.matmul(value, self.dot_weight) + + +class ScalarConstantModel(nn.Module): + def forward(self, value): + return value * 2.0 + 1.0 + + +class ConverterTest(unittest.TestCase): + def test_supported_network(self): + generated = convert_module(SupportedModel(), (4,), arena_bytes=2048) + self.assertIn("Export mode: trained", generated) + self.assertIn("fully_connected(input, first_weight, first_bias)", generated) + self.assertIn("softmax(linear_1_buffer)", generated) + self.assertIn("Arena", generated) + + def test_empty_export_does_not_embed_trainable_values(self): + model = nn.Linear(4, 3) + with torch.no_grad(): + model.weight.fill_(0.123456789) + model.bias.fill_(0.234567891) + generated = convert_module( + model, (4,), export_mode="empty" + ) + self.assertIn("weight_value[3][4]{}", generated) + self.assertIn("initialize_parameters();", generated) + self.assertIn("unit * 2.0F - 1.0F", generated) + self.assertNotIn("0.123456", generated) + self.assertNotIn("0.234567", generated) + + def test_arithmetic_scale_offset_and_dot(self): + generated = convert_module(ArithmeticModel(), (4,)) + self.assertIn("AddOperator", generated) + self.assertIn("SubOperator", generated) + self.assertIn("MultiplyOperator", generated) + self.assertIn("ScaleOperator", generated) + self.assertIn("OffsetOperator", generated) + self.assertIn("DotOperator", generated) + self.assertIn("dot(", generated) + + def test_scalar_constants_use_scale_and_offset(self): + generated = convert_module(ScalarConstantModel(), (4,)) + self.assertIn("ScaleOperator", generated) + self.assertIn("OffsetOperator", generated) + self.assertIn("static_cast(2.0F)", generated) + self.assertIn("static_cast(1.0F)", generated) + + def test_dropout_training_state(self): + training = nn.Dropout(0.2).train() + inference = nn.Dropout(0.2).eval() + self.assertIn("DropoutOperator", convert_module(training, (4,))) + self.assertIn("DropoutOperator", convert_module(inference, (4,))) + self.assertIn("dropout(input, dropout_generator, 0.2F)", + convert_module(inference, (4,))) + self.assertIn("DropoutOperator", + convert_module(nn.Dropout(0.0).eval(), (4,))) + + def test_unsupported_operator_stops_conversion(self): + with self.assertRaisesRegex(ConversionError, "unsupported ATen operator"): + convert_module(nn.Sequential(nn.Sigmoid()), (4,)) + + def test_higher_rank_input_stops_conversion(self): + with self.assertRaisesRegex(ConversionError, "only a scalar, vector, or matrix"): + convert_module(SupportedModel(), (2, 1, 4)) + + def test_linear_requires_vector(self): + with self.assertRaisesRegex(ConversionError, "requires vector input"): + convert_module(nn.Linear(4, 3), (2, 4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/dsppp/Include/dsppp/forward.hpp b/dsppp/Include/dsppp/forward.hpp index 22e807388..f6f9c4b4b 100644 --- a/dsppp/Include/dsppp/forward.hpp +++ b/dsppp/Include/dsppp/forward.hpp @@ -10,6 +10,9 @@ struct Vector_Base; template struct VectorView; +template +struct VecRef; + template typename Allocator> struct Vector; diff --git a/dsppp/Include/dsppp/vec.hpp b/dsppp/Include/dsppp/vec.hpp index fbe79faeb..94b65be69 100644 --- a/dsppp/Include/dsppp/vec.hpp +++ b/dsppp/Include/dsppp/vec.hpp @@ -23,11 +23,6 @@ namespace arm_cmsis_dsp { * @{ */ -template -struct VecRef; - - - template struct VecRef> { diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp index 08155b639..001a8e331 100644 --- a/dsppp/tests/autodiff_test.cpp +++ b/dsppp/tests/autodiff_test.cpp @@ -636,6 +636,37 @@ static void test15() template static void test16() { + // C++ extension primitives used by QAT: nearest-even rounding and + // range-masked accumulation/reduction. Half-way cases verify ties-to-even. + T rounding_input[] = {-2.5F, -1.5F, -0.5F, 0.5F, 1.5F, 2.5F}; + T rounding_output[6] = {}; + T rounding_accumulator[6] = {}; + T rounding_one[] = {1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 1.0F}; + ::arm_cmsis_dsp::VectorView rounding_input_view(rounding_input, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_output_view(rounding_output, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_accumulator_view( + rounding_accumulator, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_one_view(rounding_one, 0, 6U); + ::arm_cmsis_dsp::round_to_nearest(rounding_output_view, + rounding_input_view); + const float expected_rounding[] = {-2.0F, -2.0F, 0.0F, + 0.0F, 2.0F, 2.0F}; + for (std::size_t i = 0; i < 6U; ++i) + assert(static_cast(rounding_output[i]) == expected_rounding[i]); + const auto central = ::arm_cmsis_dsp::nearest_even_range_mask( + rounding_input_view, 1.0F, static_cast(0.0F), + static_cast(-1.0F), static_cast(1.0F)); + ::arm_cmsis_dsp::masked_scale_add( + rounding_accumulator_view, rounding_one_view, central, + static_cast(2.0F)); + const auto central_reduction = ::arm_cmsis_dsp::masked_dot_sum( + rounding_one_view, rounding_input_view, central); + for (std::size_t i = 0; i < 6U; ++i) + assert(static_cast(rounding_accumulator[i]) == + (i == 2U || i == 3U ? 2.0F : 0.0F)); + assert(static_cast(central_reduction.dot) == 0.0F); + assert(static_cast(central_reduction.sum) == 2.0F); + // Q/DQ keeps float storage while reproducing LiteRT/CMSIS-NN signed int8 // activation codes. Its combined backward pass is an STE in range and // also exposes gradients for scale and zero-point learning. From 36f82cfeae4f93af0101eda6054aae095716b9c4 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Fri, 14 Aug 2026 12:34:19 +0200 Subject: [PATCH 19/19] autodiff : Improve documentation for quantization aware training and Ethos --- dsppp/Include/dsppp/autodiff/README.md | 10 +- .../doc/ethos_cmsis_nn_fine_tuning.md | 136 ++++++++++++++++++ dsppp/Include/dsppp/autodiff/doc/operators.md | 6 + 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md index 5fa510396..1888260f2 100644 --- a/dsppp/Include/dsppp/autodiff/README.md +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -84,7 +84,11 @@ the float32 or float16 implementation. `dsppp/Examples/autodiff_fully_connected_qat.cpp` demonstrates quantization-aware training of a fully connected layer for later deployment -with CMSIS-NN or Ethos-U. +with CMSIS-NN or Ethos-U. See +[Fine-tuning an output layer after Ethos-U](doc/ethos_cmsis_nn_fine_tuning.md) +for the main deployment use case: keeping the Ethos-U output and trained-layer +input quantization parameters identical so their int8 tensors connect without +requantization. ## How reverse differentiation works here @@ -157,6 +161,10 @@ int main() - [Operators](doc/operators.md) documents the current operator families, formulas, shape rules, dropout behavior, and the CMSIS-DSP implementation paths. +- [Fine-tuning an output layer after Ethos-U](doc/ethos_cmsis_nn_fine_tuning.md) + explains how to fix the QAT input quantization parameters to the Ethos-U + output interface and deploy the trained layer without an intermediate + conversion. - [Optimizers](doc/optimizers.md) documents SGD, Adam, and RMSProp capacities, initialization, updates, freezing, and errors. - [Worked implementation flow: `y = a * x`](doc/implementation_flow.md) follows diff --git a/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md b/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md new file mode 100644 index 000000000..64ad6bfba --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md @@ -0,0 +1,136 @@ +# Fine-tuning an output layer after Ethos-U + +A common on-device training arrangement is to run a fixed feature extractor on +Ethos-U and train only its final layer with CMSIS-DSP autodiff. During +training, Ethos-U produces a signed-int8 feature vector. The application +dequantizes that vector because the autodiff graph uses floating-point +buffers. Quantization-aware training (QAT) then makes the trainable layer +behave like the int8 layer that will eventually be deployed with Ethos-U or +CMSIS-NN. + +The important interface contract is that the Ethos-U output and the trained +layer input use exactly the same quantization parameters. If the Ethos-U +output has scale `s_interface` and zero-point `z_interface`, the input Q/DQ +pair of the autodiff network must use those same fixed values: + +```text +Ethos-U output during training Autodiff layer + +q_ethos (int8) -- dequantize(s_interface, z_interface) --> x (float) + -- Q(s_interface, z_interface) + -- DQ(s_interface, z_interface) --> trainable layer +``` + +For an int8 value `q_ethos`, the floating-point training input is: + +```text +x = (q_ethos - z_interface) * s_interface +``` + +Applying Q/DQ with the same parameters maps this value back to `q_ethos` and +then reconstructs the same floating-point value, apart from floating-point +rounding. The pair records the deployed input quantization in the training +graph and also handles values from other floating-point training sources with +the same clipping and rounding that the int8 layer will see. + +## Keeping the interface parameters fixed + +The interface scale and zero-point are metadata exported with the Ethos-U +network. They are a deployment constraint, not values for the output-layer +optimizer to learn. Copy them into persistent values and use them for both +operators: + +```cpp +constexpr Int8Quantization activation_quantization = + Int8Quantization::activation(); + +// Exact quantization metadata of the Ethos-U output tensor. +float interface_scale_value = ethos_output_scale; +float interface_zero_point_value = ethos_output_zero_point; + +// Q/DQ currently requires parameter views so that its backward records have +// gradient storage. These views remain fixed because they are not added to +// the optimizer. +BufferView interface_scale = tape.parameter(interface_scale_value); +BufferView interface_zero_point = tape.parameter(interface_zero_point_value); + +quantized_input = quantize(input, interface_scale, interface_zero_point, + activation_quantization); +dequantized_input = dequantize(quantized_input, interface_scale, + interface_zero_point, + activation_quantization); +``` + +Do **not** call `optimizer.add(interface_scale)` or +`optimizer.add(interface_zero_point)`. Add only the values that should be +learned, for example the new layer's weights, bias, weight scales, output +scale, and output zero-point: + +```cpp +optimizer.add(weights); +optimizer.add(bias); +optimizer.add(weight_scale); +optimizer.add(output_scale); +optimizer.add(output_zero_point); +``` + +If the interface views were already added to an optimizer, they can instead +be kept unchanged with `freeze_parameters(optimizer, interface_scale, +interface_zero_point)`. Not adding them is simpler when they must remain fixed +for the entire training run. + +Although `tape.input()` normally expresses a non-trainable value, it must not +currently be used for these two Q/DQ arguments. The quantize and dequantize +operators require parameter views for scale and, for asymmetric activation +quantization, zero-point. Whether a parameter is updated is controlled by the +optimizer, not by the presence of a Q/DQ node. + +The copied scale must be positive and the zero-point must be the integer int8 +zero-point from the Ethos-U tensor metadata. Q/DQ applies the +`Int8Quantization::activation()` constraints, but it must not be relied upon +to invent or recalibrate this fixed interface. + +## Training and deployment + +During training: + +1. Run the fixed prefix on Ethos-U and obtain its int8 output. +2. Dequantize it with the Ethos-U output scale and zero-point. +3. Feed the resulting float vector to autodiff. +4. Apply input Q/DQ using the same fixed interface parameters. +5. Train the new layer through its weight Q/DQ and output Q/DQ pairs. + +The new layer's weight quantization parameters are independent of the +interface. Its output scale and zero-point may also be learned unless the next +deployed operator imposes another fixed quantized interface. Export the +trained floating-point weights to symmetric per-output-channel int8 weights, +and export the bias as int32 with scale: + +```text +bias_scale[channel] = s_interface * weight_scale[channel] +``` + +For inference, remove the training-only Q/DQ simulation and connect the int8 +tensors directly: + +```text +Ethos-U prefix -- q_ethos (int8) --> int8 trained layer +``` + +No data conversion is needed when all of the following match: + +- signed-int8 element type; +- tensor shape, ordering, and memory layout; +- input scale equals the Ethos-U output scale; +- input zero-point equals the Ethos-U output zero-point. + +For CMSIS-NN, configure the trained layer with +`input_offset = cmsis_nn_offset(interface_zero_point_value)`. Its output +requantization multiplier and shift are derived in the usual way from the +interface scale, per-channel weight scale, and layer output scale. + +If either interface quantization parameter differs, direct connection is not +equivalent. An int8-to-int8 requantization step is then required; dequantizing +to float and quantizing again is another, usually less efficient, option. The +purpose of fixing the input Q/DQ parameters during fine-tuning is to avoid +both conversions in the deployed network. diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md index d3017c425..6f0401e54 100644 --- a/dsppp/Include/dsppp/autodiff/doc/operators.md +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -134,6 +134,12 @@ CMSIS-NN calls the negated zero-point an `offset`. Use requires int32 bias with zero-point zero and scale `input_scale * weight_scale[channel]`. +When an autodiff layer is trained after a fixed Ethos-U network, its input +scale and zero-point must normally remain equal to the Ethos-U output +parameters. See [Fine-tuning an output layer after +Ethos-U](ethos_cmsis_nn_fine_tuning.md) for the fixed-parameter setup and the +conditions for connecting the two int8 tensors without requantization. + ## Losses Quadratic error returns a scalar sum, not a mean: