diff --git a/.gitignore b/.gitignore index 4661b4bbb..42aeb53a0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ build-* .venv/ uv.lock *.cbuild-pack.yml +disasm_* \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 2603cf348..6a3ad53ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,3 +20,11 @@ - For each of the generated pickle file, generate missing C files with : `python processTests.py -gen . -p Patterns -d Parameters -f -e` - Switch to `cmsis_build` working directory - Run python script `python runall.py` + +## Testing instructions for dsppp autodiff +- Do not use or install `pyocd` for autodiff tests. +- Change working directory to the `dsppp` folder. +- Set `AVH_FVP_PLUGINS` to an empty string before launching the FVP. +- Prepend the CMSIS Toolbox `bin` directory and the Python directory containing `python3.dll` to `PATH`. +- Build the generated project with `cbuild -O cprj test.csolution.yml --toolchain AC6 -c test.Release+VHT-Corstone-300`. +- Run `FVP_Corstone_SSE-300_Ethos-U55.exe` directly with `-f fvp_configs\VHT-Corstone-300.txt -a cpu0=cprj\out\test\VHT-Corstone-300\Release\test.axf`. diff --git a/Documentation/Doxygen/src/matrix.md b/Documentation/Doxygen/src/matrix.md index d07cfaca7..75f915ff6 100644 --- a/Documentation/Doxygen/src/matrix.md +++ b/Documentation/Doxygen/src/matrix.md @@ -41,9 +41,23 @@ Once you have initialized matrixes, you can operate on them: Matrix result = a * a + b; ``` -The operators `+` and `*` are merged into the loop. `*` is the element-wise multiply. For the vector / matrix products you should use the operator `dot`. +The operators `+` and `*` are merged into the loop. `*` is the element-wise multiply. For eagerly evaluated vector / matrix products you should use the operator `dot`. -Note that fusion of operators will not work with `dot(Matrix, Matrix`). It is only supported with vectors : `dot(Vector,Vector)` or `dot(Matrix,Vector)`. +Note that fusion of operators will not work with `dot(Matrix, Matrix)` or +`dot(Matrix, Vector)`, since both operations return an owning, eagerly +evaluated result. Vector dot products return a scalar. + +For a matrix times vector product that must be fused with vector operations, +use the lazy `matvec` operator: + +```cpp +result += matvec(matrix, vector); +``` + +`matvec` returns a non-owning expression. In this example, the matrix-vector +product is accumulated directly into `result` without allocating a temporary +vector. The operands must remain alive until the complete expression is +evaluated. Use `dot(matrix, vector)` when an owning result is required. ## VectorView @@ -136,6 +150,18 @@ The compiler may use the move semantic to copy the temporary result of the `dot` In this case, no copy would occur and `result` after the assignment would be a vector allocated by `dot` so using the `TMP_ALLOC` . +### matvec + +```cpp +result = bias + matvec(matrix, vector); +``` + +Unlike `dot(matrix, vector)`, `matvec(matrix, vector)` is lazy. It participates +in the normal one-dimensional fusion loop, so the matrix-vector result can be +combined with additions, multiplications, or accumulation without first being +stored in a temporary vector. Matrix-matrix products remain eager and continue +to use `dot`. + ### diagonal ```cpp diff --git a/dsppp/.clangd b/dsppp/.clangd new file mode 100644 index 000000000..9d672aec9 --- /dev/null +++ b/dsppp/.clangd @@ -0,0 +1,18 @@ +CompileFlags: + CompilationDatabase: c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release + +--- +If: + PathMatch: .*\.(c|C|h)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release\compile_macros_c.h + +--- +If: + PathMatch: .*\.(cpp|c\+\+|C\+\+|cxx|cc|CC|hpp)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\example\VHT-Corstone-300\Release\compile_macros_cxx.h diff --git a/dsppp/Examples/LICENSE-Iris.txt b/dsppp/Examples/LICENSE-Iris.txt new file mode 100644 index 000000000..c5765b81a --- /dev/null +++ b/dsppp/Examples/LICENSE-Iris.txt @@ -0,0 +1,16 @@ +Iris dataset license and attribution +==================================== + +The data in iris_data.hpp is derived from the Iris dataset distributed by the +UCI Machine Learning Repository. The dataset is licensed under the Creative +Commons Attribution 4.0 International license (CC BY 4.0): + +https://creativecommons.org/licenses/by/4.0/legalcode + +Attribution: + +Fisher, R. A. (1936). Iris [Dataset]. UCI Machine Learning Repository. +https://doi.org/10.24432/C56C76 + +The CMSIS-DSP source code around the data remains covered by the CMSIS-DSP +project license. diff --git a/dsppp/Examples/README.md b/dsppp/Examples/README.md new file mode 100644 index 000000000..be652b707 --- /dev/null +++ b/dsppp/Examples/README.md @@ -0,0 +1,58 @@ +# CMSIS-DSP C++ examples + +The examples demonstrate the CMSIS-DSP C++ API and its autodiff extension. +Each source file defines its own `main`, so enable only one example at a time +in `dsppp/example.cproject.yml` by commenting and uncommenting its `file` line. + +## Expression examples + +- `dot_product.cpp` evaluates a fused vector expression ending in a dot + product. +- `vector_op.cpp` demonstrates floating-point and fixed-point vector + expressions, vector views, and strided views. +- `matrix_op.cpp` demonstrates matrix expressions and row and column views. + +## Autodiff examples + +- `autodiff_example.cpp` is a minimal fully connected layer followed by ReLU. + It runs one forward and backward pass and prints the resulting gradients. +- `autodiff_regression.cpp` trains a cubic polynomial to approximate a sine + wave. It demonstrates RMSProp, reusable graphs, parameter freezing, and + saving model parameters. +- `autodiff_lms.cpp` identifies an unknown FIR filter with a per-sample LMS + update expressed as quadratic error, reverse differentiation, and SGD. It is + an educational demonstration; the specialized CMSIS-DSP LMS functions are + more efficient for production filtering. +- `autodiff_iris.cpp` trains a small classifier on the Iris flower dataset. + It uses Adam and reserves 30 of the 150 samples for a final test that is not + used during training. +- `autodiff_fully_connected_qat.cpp` demonstrates quantization-aware training + of a fully connected layer for later deployment with CMSIS-NN or Ethos-U. + +### Iris classifier + +The Iris classifier contains two fully connected layers. Cross entropy is the +training loss and is not part of the network used for inference. + +```mermaid +flowchart LR + input["Input
4 flower measurements"] --> fc1["Fully connected
4 to 8"] + fc1 --> relu["ReLU
8 values"] + relu --> fc2["Fully connected
8 to 3"] + fc2 --> softmax["Softmax
3 class probabilities"] + softmax --> loss["Cross entropy loss"] + target["One-hot target"] --> loss +``` + +The dataset is stored as floating-point measurements in `iris_data.hpp` and +normalized when each sample is loaded. Its license and attribution are in +`LICENSE-Iris.txt`. + +## Building an example + +After selecting one source in `example.cproject.yml`, the Cortex-M55 virtual +target can be built with: + +```text +cbuild test.csolution.yml --context example.Release+VHT-Corstone-300 +``` diff --git a/dsppp/Examples/autodiff_example.cpp b/dsppp/Examples/autodiff_example.cpp new file mode 100644 index 000000000..d841b5130 --- /dev/null +++ b/dsppp/Examples/autodiff_example.cpp @@ -0,0 +1,62 @@ +#include +#include +#include + +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ + Arena<2048> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); + tape.register_operator>(); + + float x_value[] = {2.0F, -1.0F}; + float matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; + float bias_value[] = {1.0F, 0.0F}; + float linear_value[2] = {}; + float activation_value[2] = {}; + + BufferView x = tape.input(x_value); + MatrixView matrix = tape.parameter(matrix_value); + BufferView bias = tape.parameter(bias_value); + BufferView linear = tape.output(linear_value); + BufferView activation = tape.output(activation_value); + + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + return 1; + } + + linear = fully_connected(x, matrix, bias); + activation = relu(linear); + + // Seed both ReLU outputs. The second neuron is negative, so ReLU blocks + // its gradient during the backward pass. + const float seed[] = {1.0F, 1.0F}; + if (!tape.backward(activation, seed, 2)) + { + return 1; + } + + std::printf("linear = {%g, %g}\n", static_cast(linear[0]), + static_cast(linear[1])); + std::printf("relu = {%g, %g}\n", static_cast(activation[0]), + static_cast(activation[1])); + std::printf("db = {%g, %g}\n", static_cast(bias.gradient(0)), + static_cast(bias.gradient(1))); + for (std::size_t row = 0; row < matrix.rows(); ++row) + { + for (std::size_t column = 0; column < matrix.columns(); ++column) + { + std::printf("dm[%u][%u] = %g\n", static_cast(row), + static_cast(column), + static_cast(matrix.gradient(row, column))); + } + } + return 0; +} diff --git a/dsppp/Examples/autodiff_fully_connected_qat.cpp b/dsppp/Examples/autodiff_fully_connected_qat.cpp new file mode 100644 index 000000000..365cca400 --- /dev/null +++ b/dsppp/Examples/autodiff_fully_connected_qat.cpp @@ -0,0 +1,200 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +namespace { + +constexpr std::size_t input_count = 3U; +constexpr std::size_t output_count = 2U; +constexpr std::size_t sample_count = 4U; + +} // namespace + +int main() +{ + constexpr std::size_t training_steps = 400U; + constexpr Int8Quantization activation_quantization = + Int8Quantization::activation(); + constexpr Int8Quantization weight_quantization = + Int8Quantization::weights(output_count, input_count); + + // A small fixed training set and the outputs of a reference 3-to-2 + // affine layer. The network below learns these targets while every input, + // weight, and output forward pass is subjected to signed int8 Q/DQ. + float input_value[sample_count][input_count] = { + {-1.0F, -0.5F, 0.25F}, + {0.5F, 1.0F, -1.0F}, + {1.0F, 0.25F, 0.5F}, + {-0.25F, -1.0F, 1.0F}}; + float target_value[sample_count][output_count] = { + {0.1F, 0.425F}, + {-1.275F, -1.2F}, + {0.7875F, -0.1375F}, + {1.6625F, 0.675F}}; + + float weight_value[output_count][input_count] = { + {0.2F, -0.1F, 0.3F}, {-0.2F, 0.1F, 0.2F}}; + float weight_gradient[output_count][input_count] = {}; + float bias_value[output_count] = {}; + + // LiteRT/CMSIS-NN/Ethos-U weight quantization is symmetric and uses one + // scale per output channel. Its zero-point is fixed at zero. + float weight_scale_value[output_count] = {0.01F, 0.01F}; + float weight_zero_point_value[output_count] = {}; + + // Activations are asymmetric per-tensor. These are optimizer parameters; + // Q/DQ automatically keeps them in the deployable signed-int8 domain. + float input_scale_value = 0.01F; + float input_zero_point_value = 0.0F; + float output_scale_value = 0.01F; + float output_zero_point_value = 0.0F; + + // Quantize produces the integer int8 values, represented as float so the + // autodiff graph remains floating point. Dequantize reconstructs the + // floating values that the layer would observe during int8 inference. + float quantized_weight_value[output_count][input_count] = {}; + float dequantized_weight_value[output_count][input_count] = {}; + float dequantized_weight_gradient[output_count][input_count] = {}; + float quantized_input_value[input_count] = {}; + float dequantized_input_value[input_count] = {}; + float linear_value[output_count] = {}; + float quantized_output_value[output_count] = {}; + float prediction_value[output_count] = {}; + float loss_value = 0.0F; + + // Keep the arena out of the small embedded process stack. Static storage + // also makes the fixed memory cost explicit in the linker map. + static Arena<8192> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + + BufferView weights = tape.parameter( + &weight_value[0][0], &weight_gradient[0][0], + output_count * input_count); + BufferView bias = tape.parameter(bias_value); + BufferView weight_scale = tape.parameter(weight_scale_value); + BufferView weight_zero_point = tape.input(weight_zero_point_value); + BufferView input_scale = tape.parameter(input_scale_value); + BufferView input_zero_point = tape.parameter(input_zero_point_value); + BufferView output_scale = tape.parameter(output_scale_value); + BufferView output_zero_point = tape.parameter(output_zero_point_value); + + BufferView quantized_weight = tape.output( + &quantized_weight_value[0][0], output_count * input_count); + BufferView dequantized_weight = tape.output( + &dequantized_weight_value[0][0], &dequantized_weight_gradient[0][0], + output_count * input_count); + + // The FC operator consumes a MatrixView. This matrix aliases the Q/DQ + // output above, so its gradients continue through dequantize and quantize + // to the original floating weights and their per-channel scales. + MatrixView dequantized_weight_matrix = tape.parameter( + &dequantized_weight_value[0][0], &dequantized_weight_gradient[0][0], + output_count, input_count); + + BufferView quantized_input = tape.output(quantized_input_value); + BufferView dequantized_input = tape.output(dequantized_input_value); + BufferView linear = tape.output(linear_value); + BufferView quantized_output = tape.output(quantized_output_value); + BufferView prediction = tape.output(prediction_value); + BufferView loss = tape.output(loss_value); + + Adam<14U, 8U> optimizer(2.0e-2F); + if (!optimizer.add(weights) || !optimizer.add(bias) || + !optimizer.add(weight_scale) || !optimizer.add(input_scale) || + !optimizer.add(input_zero_point) || !optimizer.add(output_scale) || + !optimizer.add(output_zero_point) || !tape.good()) + { + std::printf("QAT setup failed\n"); + return 1; + } + + tape.begin_graph(); + std::printf("Fully connected signed-int8 QAT\n"); + for (std::size_t step = 0; step < training_steps; ++step) + { + if (!tape.rewind_graph()) return 1; + const std::size_t sample = step % sample_count; + BufferView input = tape.input(input_value[sample]); + BufferView target = tape.input(target_value[sample]); + + quantized_weight = quantize(weights, weight_scale, + weight_zero_point, weight_quantization); + dequantized_weight = dequantize(quantized_weight, weight_scale, + weight_zero_point, + weight_quantization); + quantized_input = quantize(input, input_scale, input_zero_point, + activation_quantization); + dequantized_input = dequantize(quantized_input, input_scale, + input_zero_point, + activation_quantization); + linear = fully_connected(dequantized_input, + dequantized_weight_matrix, bias); + quantized_output = quantize(linear, output_scale, output_zero_point, + activation_quantization); + prediction = dequantize(quantized_output, output_scale, + output_zero_point, + activation_quantization); + + loss = quadratic_error(prediction, target); + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if ((step + 1U) % 100U == 0U) + std::printf("step %u, sample quadratic error = %g\n", + static_cast(step + 1U), + static_cast(loss_value)); + } + + // Refresh the integer-valued floating weights after the final optimizer + // update. Casting these values to int8_t is exact. + { + RecordingScope inference(tape, false); + quantized_weight = quantize(weights, weight_scale, + weight_zero_point, weight_quantization); + } + std::int8_t exported_weight[output_count][input_count] = {}; + for (std::size_t row = 0; row < output_count; ++row) + for (std::size_t column = 0; column < input_count; ++column) + exported_weight[row][column] = static_cast( + std::nearbyint(quantized_weight_value[row][column])); + + std::int32_t exported_bias[output_count] = {}; + for (std::size_t row = 0; row < output_count; ++row) + exported_bias[row] = static_cast(std::nearbyint( + bias_value[row] / + (input_scale_value * weight_scale_value[row]))); + + std::printf("input: scale=%g, zero_point=%g, CMSIS-NN offset=%d\n", + static_cast(input_scale_value), + static_cast(std::nearbyint(input_zero_point_value)), + static_cast(cmsis_nn_offset(input_zero_point_value))); + std::printf("output: scale=%g, zero_point=%g, CMSIS-NN offset=%d\n", + static_cast(output_scale_value), + static_cast(std::nearbyint(output_zero_point_value)), + static_cast(cmsis_nn_offset(output_zero_point_value))); + for (std::size_t row = 0; row < output_count; ++row) + { + std::printf("channel %u: weight_scale=%g, weights={%d, %d, %d}, " + "bias=%ld\n", + static_cast(row), + static_cast(weight_scale_value[row]), + static_cast(exported_weight[row][0]), + static_cast(exported_weight[row][1]), + static_cast(exported_weight[row][2]), + static_cast(exported_bias[row])); + } + return 0; +} diff --git a/dsppp/Examples/autodiff_iris.cpp b/dsppp/Examples/autodiff_iris.cpp new file mode 100644 index 000000000..6fc22f79b --- /dev/null +++ b/dsppp/Examples/autodiff_iris.cpp @@ -0,0 +1,293 @@ +#include +#include +#include +#include +#include +#include + +#include "iris_data.hpp" + +#include +#include + +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +/* Set to 0 to build the same example with float32 autodiff. It may also be + * overridden from the compiler command line with -D. */ +#ifndef DSPPP_AUTODIFF_IRIS_USE_FLOAT16 +#define DSPPP_AUTODIFF_IRIS_USE_FLOAT16 1 +#endif + +#if !DSPPP_AUTODIFF_IRIS_USE_FLOAT16 || defined(ARM_FLOAT16_SUPPORTED) +namespace { + +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 +using IrisType = float16_t; +#else +using IrisType = float; +#endif + +constexpr std::size_t input_size = 4U; +constexpr std::size_t hidden_size = 8U; +constexpr std::size_t class_count = 3U; +constexpr std::size_t training_count = 120U; +constexpr std::size_t epoch_count = 120U; + +struct Model +{ + IrisType hidden_weight[hidden_size][input_size]; + IrisType hidden_bias[hidden_size]; + IrisType output_weight[class_count][hidden_size]; + IrisType output_bias[class_count]; +}; + +struct TrainingState +{ + Model model{}; + Arena<2048, IrisType> arena{}; + Adam<67U, 4U, IrisType> optimizer{static_cast(1.0e-2F)}; + IrisType input[input_size]{}; + IrisType hidden_linear[hidden_size]{}; + IrisType hidden[hidden_size]{}; + IrisType logits[class_count]{}; + IrisType probability[class_count]{}; + IrisType target[class_count]{}; + IrisType loss{}; + // Keep normalized float32 patterns for the original example path. The + // additional float16 array avoids converting samples inside the loop when + // the half-precision path is selected. + float f32_patterns[iris_data::sample_count][input_size]{}; +#if defined(ARM_FLOAT16_SUPPORTED) + float16_t f16_patterns[iris_data::sample_count][input_size]{}; +#endif + std::uint8_t training_index[training_count]{}; +}; + +static std::uint32_t random_state = 0x12345678U; + +static std::uint32_t random_u32() noexcept +{ + random_state = random_state * 1664525U + 1013904223U; + return random_state; +} + +static IrisType random_weight() noexcept +{ + const float unit = static_cast((random_u32() >> 8) & 0xffffU) / + 65535.0F; + return static_cast((unit - 0.5F) * 0.5F); +} + +static void initialize(Model &model) noexcept +{ + for (std::size_t row = 0; row < hidden_size; ++row) + { + model.hidden_bias[row] = 0.0F; + for (std::size_t column = 0; column < input_size; ++column) + model.hidden_weight[row][column] = random_weight(); + } + for (std::size_t row = 0; row < class_count; ++row) + { + model.output_bias[row] = 0.0F; + for (std::size_t column = 0; column < hidden_size; ++column) + model.output_weight[row][column] = random_weight(); + } +} + +static bool is_test_sample(std::size_t index) noexcept +{ + // Every fifth member of each class is held out: 10 per class. + return (index % 50U) % 5U == 0U; +} + +static std::uint32_t predicted_class( + const IrisType (&probability)[class_count]) noexcept +{ + IrisType maximum; + std::uint32_t index; +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + arm_max_f16(probability, class_count, &maximum, &index); +#else + arm_max_f32(probability, class_count, &maximum, &index); +#endif + return index; +} + +static const char *label_name(unsigned label) noexcept +{ + static constexpr const char *names[class_count] = { + "Iris-setosa", "Iris-versicolor", "Iris-virginica"}; + return label < class_count ? names[label] : "unknown"; +} + +} // namespace + +int main() +{ + std::printf("Iris classification with CMSIS-DSP autodiff\n"); + // Keep the training buffers and optimizer state off the limited stack. + TrainingState *state = new TrainingState; + initialize(state->model); + for (std::size_t sample = 0; sample < iris_data::sample_count; ++sample) + { + iris_data::normalized_features(sample, state->f32_patterns[sample]); +#if defined(ARM_FLOAT16_SUPPORTED) + for (std::size_t feature = 0; feature < input_size; ++feature) + state->f16_patterns[sample][feature] = static_cast( + state->f32_patterns[sample][feature]); +#endif + } + + Tape &tape = state->arena.tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + + BufferView input = tape.input(state->input); + MatrixView hidden_weight = tape.parameter(state->model.hidden_weight); + BufferView hidden_bias = tape.parameter(state->model.hidden_bias); + MatrixView output_weight = tape.parameter(state->model.output_weight); + BufferView output_bias = tape.parameter(state->model.output_bias); + BufferView hidden_linear = tape.output(state->hidden_linear); + BufferView hidden = tape.output(state->hidden); + BufferView logits = tape.output(state->logits); + BufferView probability = tape.output(state->probability); + BufferView target = tape.input(state->target); + BufferView loss = tape.output(state->loss); + + if (!tape.good()) + { + if (tape.status() == Status::out_of_memory) + std::printf("Autodiff arena is too small\n"); + else + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + delete state; + return 1; + } + + if (!state->optimizer.add(hidden_weight) || + !state->optimizer.add(hidden_bias) || + !state->optimizer.add(output_weight) || + !state->optimizer.add(output_bias)) + { + delete state; + std::printf("Failed to add parameters to optimizer\n"); + return 1; + } + + // Keep 30 samples for testing. They are never used during training. + std::size_t training_position = 0U; + for (std::size_t sample = 0; sample < iris_data::sample_count; ++sample) + if (!is_test_sample(sample)) + state->training_index[training_position++] = + static_cast(sample); + + tape.begin_graph(); + for (std::size_t epoch = 0; epoch < epoch_count; ++epoch) + { + // Shuffle the training patterns before each epoch. + for (std::size_t i = training_count - 1U; i > 0U; --i) + { + const std::size_t other = random_u32() % (i + 1U); + const std::uint8_t temporary = state->training_index[i]; + state->training_index[i] = state->training_index[other]; + state->training_index[other] = temporary; + } + + float epoch_loss = 0.0F; + for (std::size_t position = 0; position < training_count; ++position) + { + const std::size_t sample = state->training_index[position]; +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f16_patterns[sample][feature]; +#else + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f32_patterns[sample][feature]; +#endif + for (std::size_t i = 0; i < class_count; ++i) + state->target[i] = i == iris_data::samples[sample].label + ? static_cast(1.0F) + : static_cast(0.0F); + + if (!tape.rewind_graph()) + { + delete state; + return 1; + } + hidden_linear = fully_connected(input, hidden_weight, hidden_bias); + hidden = relu(hidden_linear); + logits = fully_connected(hidden, output_weight, output_bias); + probability = softmax(logits); + loss = cross_entropy(probability, target); + + state->optimizer.zero_grad(); + if (!tape.backward(loss) || !state->optimizer.step()) + { + delete state; + return 1; + } + epoch_loss += static_cast(state->loss); + } + + if ((epoch + 1U) % 20U == 0U) + std::printf("epoch %u: mean loss=%g\n", + static_cast(epoch + 1U), + static_cast(epoch_loss / training_count)); + } + + // Final check on the 30 samples that were kept out of training. + unsigned correct = 0U; + unsigned test_number = 0U; + { + RecordingScope inference(tape, false); + for (std::size_t sample = 0; sample < iris_data::sample_count; + ++sample) + { + if (!is_test_sample(sample)) continue; +#if DSPPP_AUTODIFF_IRIS_USE_FLOAT16 + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f16_patterns[sample][feature]; +#else + for (std::size_t feature = 0; feature < input_size; ++feature) + state->input[feature] = state->f32_patterns[sample][feature]; +#endif + hidden_linear = fully_connected(input, hidden_weight, hidden_bias); + hidden = relu(hidden_linear); + logits = fully_connected(hidden, output_weight, output_bias); + probability = softmax(logits); + const unsigned expected = iris_data::samples[sample].label; + const unsigned detected = predicted_class(state->probability); + const bool match = detected == expected; + if (!match) + std::printf("\033[31m"); + std::printf("Test %u:\n Expected \"%s\"\n Detected \"%s\"\n", + ++test_number, label_name(expected), + label_name(detected)); + if (!match) + std::printf("\033[0m"); + if (match) + ++correct; + } + } + std::printf("final test accuracy=%u/30 tests\n", correct); + + delete state; + return 0; +} + +#else + +int main() +{ + std::printf("Iris float16 example requires ARM_FLOAT16_SUPPORTED\n"); + return 0; +} + +#endif diff --git a/dsppp/Examples/autodiff_lms.cpp b/dsppp/Examples/autodiff_lms.cpp new file mode 100644 index 000000000..1c79cd375 --- /dev/null +++ b/dsppp/Examples/autodiff_lms.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include + +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +/* + * Educational example only. + * + * This example deliberately expresses a least mean square (LMS) adaptive + * filter with the generic autodiff API to show how per-sample reverse-mode + * differentiation becomes online stochastic gradient descent. For a real LMS + * filter, a specialized implementation such as arm_lms_f32 is more efficient: + * it combines FIR state handling, output calculation, error calculation, and + * coefficient updates directly, without tape records or a separate backward + * pass. arm_lms_f32 accepts blocks but updates its coefficients after every + * sample inside the block. + */ + +static float next_excitation(std::uint32_t &state) noexcept +{ + state = state * 1664525U + 1013904223U; + const std::uint32_t value = state >> 8; + return static_cast(value) * (2.0F / 16777215.0F) - 1.0F; +} + +int main() +{ + constexpr std::size_t tap_count = 4U; + constexpr std::size_t sample_count = 2000U; + constexpr std::size_t measurement_count = 100U; + constexpr float mu = 0.05F; + + // Unknown FIR system that the adaptive filter will identify. Element zero + // multiplies the newest sample, element one the preceding sample, and so on. + const float reference_coefficients[tap_count] = { + 0.6F, -0.3F, 0.2F, 0.1F}; + float coefficients_value[tap_count] = {}; + float input_state_value[tap_count] = {}; + float desired_value[1] = {}; + float output_value[1] = {}; + float loss_value[1] = {}; + + Arena<512> arena; + Tape &tape = arena.tape(); + if (!tape.register_operator>() || + !tape.register_operator>()) + { + std::printf("Failed to register autodiff operators\n"); + return 1; + } + + BufferView coefficients = tape.parameter(coefficients_value); + BufferView input_state = tape.input(input_state_value); + BufferView desired = tape.input(desired_value); + BufferView output = tape.output(output_value); + BufferView loss = tape.output(loss_value); + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + return 1; + } + + // quadratic_error computes (output - desired)^2, whose coefficient + // gradient contains a factor of two. learning_rate = mu/2 therefore gives + // the classical LMS update: coefficient += mu * error * input_state. + SGD optimizer(mu * 0.5F); + if (!optimizer.add(coefficients)) + { + std::printf("Failed to add LMS coefficients to SGD\n"); + return 1; + } + + tape.begin_graph(); + std::uint32_t random_state = 1U; + float final_error_sum = 0.0F; + + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + // Maintain the FIR delay line in caller-owned memory. + for (std::size_t tap = tap_count - 1U; tap > 0U; --tap) + input_state_value[tap] = input_state_value[tap - 1U]; + input_state_value[0] = next_excitation(random_state); + + desired_value[0] = 0.0F; + for (std::size_t tap = 0; tap < tap_count; ++tap) + desired_value[0] += + reference_coefficients[tap] * input_state_value[tap]; + + // Each sample creates a fresh two-node graph. Rewinding releases the + // preceding sample's records while retaining all persistent views and + // gradient buffers. + if (!tape.rewind_graph()) return 1; + output = dot(input_state, coefficients); + loss = quadratic_error(output, desired); + + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if (sample >= sample_count - measurement_count) + final_error_sum += loss_value[0]; + } + + std::printf("reference coefficients: {%g, %g, %g, %g}\n", + static_cast(reference_coefficients[0]), + static_cast(reference_coefficients[1]), + static_cast(reference_coefficients[2]), + static_cast(reference_coefficients[3])); + std::printf("learned coefficients: {%g, %g, %g, %g}\n", + static_cast(coefficients_value[0]), + static_cast(coefficients_value[1]), + static_cast(coefficients_value[2]), + static_cast(coefficients_value[3])); + std::printf("mean squared error over final %u samples: %g\n", + static_cast(measurement_count), + static_cast(final_error_sum / measurement_count)); + + return tape.good() && optimizer.good() ? 0 : 1; +} diff --git a/dsppp/Examples/autodiff_regression.cpp b/dsppp/Examples/autodiff_regression.cpp new file mode 100644 index 000000000..4824e7d0e --- /dev/null +++ b/dsppp/Examples/autodiff_regression.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include + +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +/* C-compatible representation suitable for RAM, flash, or application-defined + * nonvolatile storage. It contains values only; optimizer state is not needed + * for inference on a later run. + */ +struct SinePolynomialParameters +{ + float coefficients[3]; // x, x^2, x^3 + float bias; +}; + +static void save_parameters(SinePolynomialParameters &destination, + const SinePolynomialParameters &source) noexcept +{ + for (std::size_t i = 0; i < 3; ++i) + destination.coefficients[i] = source.coefficients[i]; + destination.bias = source.bias; +} + +static void restore_parameters(SinePolynomialParameters &destination, + const SinePolynomialParameters &source) noexcept +{ + save_parameters(destination, source); +} + +static float infer(const SinePolynomialParameters ¶meters, + float x) noexcept +{ + const float x2 = x * x; + return parameters.bias + parameters.coefficients[0] * x + + parameters.coefficients[1] * x2 + + parameters.coefficients[2] * x2 * x; +} + +int main() +{ + + constexpr std::size_t sample_count = 100U; + constexpr std::size_t training_steps = 4000U; + constexpr float pi = 3.14159265358979323846F; + + SinePolynomialParameters model = {{0.0F, 0.0F, 0.0F}, 0.0F}; + // One training graph contains two scalar operations per sample and one + // global loss record. Values stay in caller storage; this arena contains + // gradients and operation records only. + Arena<32768> *arena = new Arena<32768>(); + Tape &tape = arena->tape(); + tape.register_operator>(); + tape.register_operator>(); + tape.register_operator>(); + + float feature_value[sample_count][3] = {}; + float polynomial_value[sample_count] = {}; + float prediction_value[sample_count] = {}; + float target_value[sample_count] = {}; + float loss_value[1] = {}; + + // Construct the complete fixed training set once. No sample storage is + // allocated by the autodiff implementation. + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + const float x = -pi + 2.0F * pi * static_cast(sample) / + static_cast(sample_count - 1U); + const float x2 = x * x; + feature_value[sample][0] = x; + feature_value[sample][1] = x2; + feature_value[sample][2] = x2 * x; + target_value[sample] = std::sin(x); + } + + BufferView coefficients = tape.parameter(model.coefficients); + BufferView bias = tape.parameter(model.bias); + BufferView polynomial = tape.output(polynomial_value); + BufferView prediction = tape.output(prediction_value); + BufferView target = tape.input(target_value); + BufferView loss = tape.output(loss_value); + + if (!tape.good()) + { + std::printf("Autodiff setup failed (status=%u)\n", + static_cast(tape.status())); + delete arena; + return 1; + } + + // Match the optimizer chosen by the PyTorch example. Adam can be used + // here instead by including adam.hpp and changing only this type. + RMSProp<4, 2> optimizer(1.0e-3F); + if (!optimizer.add(coefficients) || !optimizer.add(bias)) + { + std::printf("Failed to add parameters to optimizer\n"); + delete arena; + return 1; + } + + /* Set this to true for bias-only fine tuning. The same mechanism freezes + * all parameters belonging to any selected layer/operator. + */ + constexpr bool fine_tune_bias_only = false; + if (fine_tune_bias_only) + freeze_parameters(optimizer, coefficients); + + /* Views and gradient buffers are persistent. Only operation records after + * this point are rewound for every complete batch. + */ + tape.begin_graph(); + for (std::size_t step = 0; step < training_steps; ++step) + { + if (!tape.rewind_graph()) + { + delete arena; + return 1; + } + + // Build all 100 predictions before constructing the loss. The scalar + // views below share slices of the two arena-managed vector gradients; + // creating them allocates no additional gradient buffers. + for (std::size_t sample = 0; sample < sample_count; ++sample) + { + BufferView features = tape.input(feature_value[sample]); + BufferView polynomial_element = tape.output( + &polynomial_value[sample], &polynomial.gradients()[sample], 1U); + BufferView prediction_element = tape.output( + &prediction_value[sample], &prediction.gradients()[sample], 1U); + + polynomial_element = dot(features, coefficients); + prediction_element = polynomial_element + bias; + } + + // One scalar loss represents the whole sampled period. Consequently + // backward() accumulates parameter gradients from every sample and + // the optimizer performs exactly one global update per step. + loss = quadratic_error(prediction, target); + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) return 1; + + if ((step + 1U) % 100U == 0U) + std::printf("step %u, global quadratic error = %g, mean = %g\n", + static_cast(step + 1U), + static_cast(loss_value[0]), + static_cast(loss_value[0] / sample_count)); + } + + SinePolynomialParameters checkpoint{}; + save_parameters(checkpoint, model); + std::printf("learned: y = %g + %g*x + %g*x^2 + %g*x^3\n", + static_cast(checkpoint.bias), + static_cast(checkpoint.coefficients[0]), + static_cast(checkpoint.coefficients[1]), + static_cast(checkpoint.coefficients[2])); + + /* Simulate a later program run. In a real target, checkpoint would first + * be persisted and later loaded from platform-specific nonvolatile memory. + */ + SinePolynomialParameters restored{}; + restore_parameters(restored, checkpoint); + std::printf("restored inference at pi/2: predicted=%g, reference=%g\n", + static_cast(infer(restored, pi * 0.5F)), + static_cast(std::sin(pi * 0.5F))); + + delete arena; + return 0; +} diff --git a/dsppp/Examples/iris_data.hpp b/dsppp/Examples/iris_data.hpp new file mode 100644 index 000000000..a37f0f38d --- /dev/null +++ b/dsppp/Examples/iris_data.hpp @@ -0,0 +1,196 @@ +#pragma once + +#include +#include + +namespace iris_data { + +struct Sample +{ + // Measurements in centimetres, followed by the class index. + float feature[4]; + unsigned char label; +}; + +constexpr std::size_t sample_count = 150U; +constexpr Sample samples[sample_count] = { + {{5.1F, 3.5F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.0F, 1.4F, 0.2F}, 0}, + {{4.7F, 3.2F, 1.3F, 0.2F}, 0}, + {{4.6F, 3.1F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.6F, 1.4F, 0.2F}, 0}, + {{5.4F, 3.9F, 1.7F, 0.4F}, 0}, + {{4.6F, 3.4F, 1.4F, 0.3F}, 0}, + {{5.0F, 3.4F, 1.5F, 0.2F}, 0}, + {{4.4F, 2.9F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.1F, 1.5F, 0.1F}, 0}, + {{5.4F, 3.7F, 1.5F, 0.2F}, 0}, + {{4.8F, 3.4F, 1.6F, 0.2F}, 0}, + {{4.8F, 3.0F, 1.4F, 0.1F}, 0}, + {{4.3F, 3.0F, 1.1F, 0.1F}, 0}, + {{5.8F, 4.0F, 1.2F, 0.2F}, 0}, + {{5.7F, 4.4F, 1.5F, 0.4F}, 0}, + {{5.4F, 3.9F, 1.3F, 0.4F}, 0}, + {{5.1F, 3.5F, 1.4F, 0.3F}, 0}, + {{5.7F, 3.8F, 1.7F, 0.3F}, 0}, + {{5.1F, 3.8F, 1.5F, 0.3F}, 0}, + {{5.4F, 3.4F, 1.7F, 0.2F}, 0}, + {{5.1F, 3.7F, 1.5F, 0.4F}, 0}, + {{4.6F, 3.6F, 1.0F, 0.2F}, 0}, + {{5.1F, 3.3F, 1.7F, 0.5F}, 0}, + {{4.8F, 3.4F, 1.9F, 0.2F}, 0}, + {{5.0F, 3.0F, 1.6F, 0.2F}, 0}, + {{5.0F, 3.4F, 1.6F, 0.4F}, 0}, + {{5.2F, 3.5F, 1.5F, 0.2F}, 0}, + {{5.2F, 3.4F, 1.4F, 0.2F}, 0}, + {{4.7F, 3.2F, 1.6F, 0.2F}, 0}, + {{4.8F, 3.1F, 1.6F, 0.2F}, 0}, + {{5.4F, 3.4F, 1.5F, 0.4F}, 0}, + {{5.2F, 4.1F, 1.5F, 0.1F}, 0}, + {{5.5F, 4.2F, 1.4F, 0.2F}, 0}, + {{4.9F, 3.1F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.2F, 1.2F, 0.2F}, 0}, + {{5.5F, 3.5F, 1.3F, 0.2F}, 0}, + {{4.9F, 3.6F, 1.4F, 0.1F}, 0}, + {{4.4F, 3.0F, 1.3F, 0.2F}, 0}, + {{5.1F, 3.4F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.5F, 1.3F, 0.3F}, 0}, + {{4.5F, 2.3F, 1.3F, 0.3F}, 0}, + {{4.4F, 3.2F, 1.3F, 0.2F}, 0}, + {{5.0F, 3.5F, 1.6F, 0.6F}, 0}, + {{5.1F, 3.8F, 1.9F, 0.4F}, 0}, + {{4.8F, 3.0F, 1.4F, 0.3F}, 0}, + {{5.1F, 3.8F, 1.6F, 0.2F}, 0}, + {{4.6F, 3.2F, 1.4F, 0.2F}, 0}, + {{5.3F, 3.7F, 1.5F, 0.2F}, 0}, + {{5.0F, 3.3F, 1.4F, 0.2F}, 0}, + {{7.0F, 3.2F, 4.7F, 1.4F}, 1}, + {{6.4F, 3.2F, 4.5F, 1.5F}, 1}, + {{6.9F, 3.1F, 4.9F, 1.5F}, 1}, + {{5.5F, 2.3F, 4.0F, 1.3F}, 1}, + {{6.5F, 2.8F, 4.6F, 1.5F}, 1}, + {{5.7F, 2.8F, 4.5F, 1.3F}, 1}, + {{6.3F, 3.3F, 4.7F, 1.6F}, 1}, + {{4.9F, 2.4F, 3.3F, 1.0F}, 1}, + {{6.6F, 2.9F, 4.6F, 1.3F}, 1}, + {{5.2F, 2.7F, 3.9F, 1.4F}, 1}, + {{5.0F, 2.0F, 3.5F, 1.0F}, 1}, + {{5.9F, 3.0F, 4.2F, 1.5F}, 1}, + {{6.0F, 2.2F, 4.0F, 1.0F}, 1}, + {{6.1F, 2.9F, 4.7F, 1.4F}, 1}, + {{5.6F, 2.9F, 3.6F, 1.3F}, 1}, + {{6.7F, 3.1F, 4.4F, 1.4F}, 1}, + {{5.6F, 3.0F, 4.5F, 1.5F}, 1}, + {{5.8F, 2.7F, 4.1F, 1.0F}, 1}, + {{6.2F, 2.2F, 4.5F, 1.5F}, 1}, + {{5.6F, 2.5F, 3.9F, 1.1F}, 1}, + {{5.9F, 3.2F, 4.8F, 1.8F}, 1}, + {{6.1F, 2.8F, 4.0F, 1.3F}, 1}, + {{6.3F, 2.5F, 4.9F, 1.5F}, 1}, + {{6.1F, 2.8F, 4.7F, 1.2F}, 1}, + {{6.4F, 2.9F, 4.3F, 1.3F}, 1}, + {{6.6F, 3.0F, 4.4F, 1.4F}, 1}, + {{6.8F, 2.8F, 4.8F, 1.4F}, 1}, + {{6.7F, 3.0F, 5.0F, 1.7F}, 1}, + {{6.0F, 2.9F, 4.5F, 1.5F}, 1}, + {{5.7F, 2.6F, 3.5F, 1.0F}, 1}, + {{5.5F, 2.4F, 3.8F, 1.1F}, 1}, + {{5.5F, 2.4F, 3.7F, 1.0F}, 1}, + {{5.8F, 2.7F, 3.9F, 1.2F}, 1}, + {{6.0F, 2.7F, 5.1F, 1.6F}, 1}, + {{5.4F, 3.0F, 4.5F, 1.5F}, 1}, + {{6.0F, 3.4F, 4.5F, 1.6F}, 1}, + {{6.7F, 3.1F, 4.7F, 1.5F}, 1}, + {{6.3F, 2.3F, 4.4F, 1.3F}, 1}, + {{5.6F, 3.0F, 4.1F, 1.3F}, 1}, + {{5.5F, 2.5F, 4.0F, 1.3F}, 1}, + {{5.5F, 2.6F, 4.4F, 1.2F}, 1}, + {{6.1F, 3.0F, 4.6F, 1.4F}, 1}, + {{5.8F, 2.6F, 4.0F, 1.2F}, 1}, + {{5.0F, 2.3F, 3.3F, 1.0F}, 1}, + {{5.6F, 2.7F, 4.2F, 1.3F}, 1}, + {{5.7F, 3.0F, 4.2F, 1.2F}, 1}, + {{5.7F, 2.9F, 4.2F, 1.3F}, 1}, + {{6.2F, 2.9F, 4.3F, 1.3F}, 1}, + {{5.1F, 2.5F, 3.0F, 1.1F}, 1}, + {{5.7F, 2.8F, 4.1F, 1.3F}, 1}, + {{6.3F, 3.3F, 6.0F, 2.5F}, 2}, + {{5.8F, 2.7F, 5.1F, 1.9F}, 2}, + {{7.1F, 3.0F, 5.9F, 2.1F}, 2}, + {{6.3F, 2.9F, 5.6F, 1.8F}, 2}, + {{6.5F, 3.0F, 5.8F, 2.2F}, 2}, + {{7.6F, 3.0F, 6.6F, 2.1F}, 2}, + {{4.9F, 2.5F, 4.5F, 1.7F}, 2}, + {{7.3F, 2.9F, 6.3F, 1.8F}, 2}, + {{6.7F, 2.5F, 5.8F, 1.8F}, 2}, + {{7.2F, 3.6F, 6.1F, 2.5F}, 2}, + {{6.5F, 3.2F, 5.1F, 2.0F}, 2}, + {{6.4F, 2.7F, 5.3F, 1.9F}, 2}, + {{6.8F, 3.0F, 5.5F, 2.1F}, 2}, + {{5.7F, 2.5F, 5.0F, 2.0F}, 2}, + {{5.8F, 2.8F, 5.1F, 2.4F}, 2}, + {{6.4F, 3.2F, 5.3F, 2.3F}, 2}, + {{6.5F, 3.0F, 5.5F, 1.8F}, 2}, + {{7.7F, 3.8F, 6.7F, 2.2F}, 2}, + {{7.7F, 2.6F, 6.9F, 2.3F}, 2}, + {{6.0F, 2.2F, 5.0F, 1.5F}, 2}, + {{6.9F, 3.2F, 5.7F, 2.3F}, 2}, + {{5.6F, 2.8F, 4.9F, 2.0F}, 2}, + {{7.7F, 2.8F, 6.7F, 2.0F}, 2}, + {{6.3F, 2.7F, 4.9F, 1.8F}, 2}, + {{6.7F, 3.3F, 5.7F, 2.1F}, 2}, + {{7.2F, 3.2F, 6.0F, 1.8F}, 2}, + {{6.2F, 2.8F, 4.8F, 1.8F}, 2}, + {{6.1F, 3.0F, 4.9F, 1.8F}, 2}, + {{6.4F, 2.8F, 5.6F, 2.1F}, 2}, + {{7.2F, 3.0F, 5.8F, 1.6F}, 2}, + {{7.4F, 2.8F, 6.1F, 1.9F}, 2}, + {{7.9F, 3.8F, 6.4F, 2.0F}, 2}, + {{6.4F, 2.8F, 5.6F, 2.2F}, 2}, + {{6.3F, 2.8F, 5.1F, 1.5F}, 2}, + {{6.1F, 2.6F, 5.6F, 1.4F}, 2}, + {{7.7F, 3.0F, 6.1F, 2.3F}, 2}, + {{6.3F, 3.4F, 5.6F, 2.4F}, 2}, + {{6.4F, 3.1F, 5.5F, 1.8F}, 2}, + {{6.0F, 3.0F, 4.8F, 1.8F}, 2}, + {{6.9F, 3.1F, 5.4F, 2.1F}, 2}, + {{6.7F, 3.1F, 5.6F, 2.4F}, 2}, + {{6.9F, 3.1F, 5.1F, 2.3F}, 2}, + {{5.8F, 2.7F, 5.1F, 1.9F}, 2}, + {{6.8F, 3.2F, 5.9F, 2.3F}, 2}, + {{6.7F, 3.3F, 5.7F, 2.5F}, 2}, + {{6.7F, 3.0F, 5.2F, 2.3F}, 2}, + {{6.3F, 2.5F, 5.0F, 1.9F}, 2}, + {{6.5F, 3.0F, 5.2F, 2.0F}, 2}, + {{6.2F, 3.4F, 5.4F, 2.3F}, 2}, + {{5.9F, 3.0F, 5.1F, 1.8F}, 2}, +}; + +// Population statistics of all 150 samples. +constexpr float mean[4] = {5.843333F, 3.057333F, 3.758000F, 1.199333F}; +constexpr float inverse_standard_deviation[4] = { + 1.211678F, 2.301971F, 0.568374F, 1.316322F}; + +inline void normalized_features(std::size_t index, float (&output)[4]) noexcept +{ + for (std::size_t feature = 0; feature < 4U; ++feature) + output[feature] = + (samples[index].feature[feature] - mean[feature]) * + inverse_standard_deviation[feature]; +} + +// The source pattern remains float32 so it can be shared by the original +// example. This overload provides a separate float16 pattern buffer for the +// half-precision autodiff example. +#if defined(ARM_FLOAT16_SUPPORTED) +inline void normalized_features(std::size_t index, + float16_t (&output)[4]) noexcept +{ + for (std::size_t feature = 0; feature < 4U; ++feature) + output[feature] = static_cast( + (samples[index].feature[feature] - mean[feature]) * + inverse_standard_deviation[feature]); +} +#endif + +} // namespace iris_data diff --git a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp index ace09a188..5d739acef 100644 --- a/dsppp/Include/dsppp/DSP/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/DSP/matrix_multiply.hpp @@ -253,6 +253,98 @@ inline void _dot_m_v(RES &res, } } +namespace detail { + +// Compute one matrix-row/vector dot product for each of up to four consecutive +// rows. Each packed block from the vector is loaded once and reused by all dot +// product accumulators. +template +inline void matvec_dsp_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using TM = typename traits::Scalar; + using Acc = typename vector_traits::temp_accumulator; + constexpr int lanes = vector_traits::nb_lanes; + + Acc sums[ROWS] = {}; + const TM *rows[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + rows[row] = matrix.const_ptr() + + (first_row + row) * matrix.stride(); + + index_t column = 0; + for (; column + lanes <= matrix.columns(); column += lanes) + { + const auto vector_data = vector.vector_op(column); + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::vmacc( + sums[row], inner::vload1<1>(rows[row] + column), vector_data); + } + + for (; column < matrix.columns(); ++column) + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac( + sums[row], rows[row][column], vector[column]); + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same DSP kernel. +template() && !is_complex() && + !std::is_same::Scalar,Q31>::value && + number_traits::Scalar>::is_fixed, + bool>::type = true> +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const DSP* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_dsp_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_dsp_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_dsp_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_dsp_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + +template() && + !is_complex() && + !is_complex() && + !std::is_same::Scalar,Q31>::value && + number_traits::Scalar>::is_fixed,bool>::type = true> +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const DSP* = nullptr) +{ + detail::dot_transposed_unrolled(res,m,v); +} + template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_to_nearest(DST& destination, const SRC& source, + const vector_length_t length, + const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + value = vrndnq_f32(value); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + value = vrndnq_f16(value); +#endif + destination.vector_store_tail(i, length - i, value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + const auto minimum_vector = inner::vconst(minimum); + const auto maximum_vector = inner::vconst(maximum); + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + value = vrndnq_f32(value); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + value = vrndnq_f16(value); +#endif + value = inner::vadd(value, offset); + value = vmaxnmq(value, minimum_vector); + value = vminnmq(value, maximum_vector); + destination.vector_store_tail(i, length - i, value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Helium* = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + auto value = source.vector_op_tail(i, length - i); + if constexpr (std::is_same::value) + { + value = vrndnq_f32(vmulq_n_f32(value, multiplier)); + value = vaddq_n_f32(value, offset); + value = vmaxnmq(value, inner::vconst(minimum)); + value = vminnmq(value, inner::vconst(maximum)); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + auto bottom = vrndnq_f32( + vmulq_n_f32(vcvtbq_f32_f16(value), multiplier)); + auto top = vrndnq_f32( + vmulq_n_f32(vcvttq_f32_f16(value), multiplier)); + const float offset_f32 = static_cast(offset); + const auto minimum_f32 = vdupq_n_f32(static_cast(minimum)); + const auto maximum_f32 = vdupq_n_f32(static_cast(maximum)); + bottom = vmaxnmq(vaddq_n_f32(bottom, offset_f32), minimum_f32); + top = vmaxnmq(vaddq_n_f32(top, offset_f32), minimum_f32); + bottom = vminnmq(bottom, maximum_f32); + top = vminnmq(top, maximum_f32); + value = vcvtbq_f16_f32(value, bottom); + value = vcvttq_f16_f32(value, top); + } +#endif + destination.vector_store_tail(i, length - i, value); + } +} + +template +inline mve_pred16_t _nearest_even_range_predicate( + const MASK& mask, index_t i, vector_length_t remaining, + const Helium* = nullptr) +{ + using T = typename MASK::Scalar; + const mve_pred16_t tail = inner::vctpq::mk(remaining); + auto value = mask.values().vector_op_tail(i, remaining); + if constexpr (std::is_same::value) + { + value = vrndnq_f32(vmulq_n_f32(value, mask.multiplier())); + value = inner::vadd(value, mask.offset(), tail); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + auto bottom = vrndnq_f32( + vmulq_n_f32(vcvtbq_f32_f16(value), mask.multiplier())); + auto top = vrndnq_f32( + vmulq_n_f32(vcvttq_f32_f16(value), mask.multiplier())); + const float offset = static_cast(mask.offset()); + bottom = vaddq_n_f32(bottom, offset); + top = vaddq_n_f32(top, offset); + value = vcvtbq_f16_f32(value, bottom); + value = vcvttq_f16_f32(value, top); + } +#endif + mve_pred16_t selected = vcmpgeq_m(value, mask.minimum(), tail); + return vcmpleq_m(value, mask.maximum(), selected); +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline void _masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale, + const vector_length_t length, + const Helium* architecture = nullptr) +{ + using T = typename traits::Scalar; + constexpr int lanes = vector_traits::nb_lanes; + for (index_t i = 0; i < length; i += lanes) + { + const mve_pred16_t selected = + _nearest_even_range_predicate(mask, i, length - i, architecture); + auto destination_value = destination.vector_op_tail(i, length - i); + destination_value = vfmaq_m( + destination_value, source.vector_op_tail(i, length - i), + scale, selected); + destination.vector_store_tail(i, length - i, destination_value); + } +} + +template() && + has_vector_inst() && + vector_idx_pair() && + has_predicate(),bool>::type = true> +inline auto _masked_dot_sum(const A& a, const B& b, const MASK& mask, + const vector_length_t length, + const Helium* architecture = nullptr) +{ + using T = typename traits::Scalar; + using Vector = typename vector_traits::vector; + constexpr int lanes = vector_traits::nb_lanes; + Vector dot = vector_traits::temp_acc_zero(); + Vector sum = vector_traits::temp_acc_zero(); + for (index_t i = 0; i < length; i += lanes) + { + const mve_pred16_t selected = + _nearest_even_range_predicate(mask, i, length - i, architecture); + const auto first = a.vector_op_tail(i, length - i); + dot = inner::vmacc(dot, first, b.vector_op_tail(i, length - i), + selected); + sum = vaddq_m(sum, sum, first, selected); + } + return MaskedDotSum{inner::vreduce(dot), inner::vreduce(sum)}; +} + /** * @brief Fill evaluator for Helium * diff --git a/dsppp/Include/dsppp/Helium/matrix_multiply.hpp b/dsppp/Include/dsppp/Helium/matrix_multiply.hpp index ccd7ef080..65abd7c4e 100644 --- a/dsppp/Include/dsppp/Helium/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/Helium/matrix_multiply.hpp @@ -64,6 +64,166 @@ inline void _dot_m_v(RES &res, } +namespace detail { + +// Compute one matrix-row/vector dot product for each of up to four consecutive +// rows. Each vector block is loaded once and reused by all dot product +// accumulators. +template +inline void matvec_helium_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using Acc = DotResult; + using Temp = typename vector_traits::temp_accumulator; + constexpr int lanes = vector_traits::nb_lanes; + + Temp vector_sums[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = vector_traits::temp_acc_zero(); + + index_t column = 0; + if constexpr (has_predicate()) + { + for (; column < matrix.columns(); column += lanes) + { + const vector_length_t remaining = matrix.columns() - column; + const auto predicate = inner::vctpq::mk(remaining); + const auto vector_data = + vector.vector_op_tail(column, remaining); + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = inner::vmacc( + vector_sums[row], + matrix.row(first_row + row).vector_op_tail( + column, remaining), + vector_data, predicate); + } + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator( + inner::vreduce(vector_sums[row])); + } + else + { + for (; column + lanes <= matrix.columns(); column += lanes) + { + const auto vector_data = vector.vector_op(column); + for (index_t row = 0; row < ROWS; ++row) + vector_sums[row] = inner::vmacc( + vector_sums[row], + matrix.row(first_row + row).vector_op(column), + vector_data); + } + + Acc sums[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::vreduce(vector_sums[row]); + for (; column < matrix.columns(); ++column) + { + const auto value = vector[column]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac( + sums[row], matrix(first_row + row, column), value); + } + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); + } +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same Helium kernel. +template() && has_vector_inst() && + same_nb_lanes(), bool>::type = true> +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const Helium* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_helium_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_helium_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_helium_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_helium_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + +#if defined(ARM_MATH_MVEI) || defined(ARM_MATH_MVEF) + +template() && + has_vector_inst() && + same_nb_lanes() && + same_nb_lanes(),bool>::type = true> +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const Helium* = nullptr) +{ + if constexpr (is_float()) + { + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using T = typename MixedRes::type; + using Acc = typename vector_traits::temp_accumulator; + constexpr int nb_lanes = vector_traits::nb_lanes; + + const auto &original = m.original(); + const vector_length_t rows = original.rows(); + const vector_length_t columns = original.columns(); + vector_length_t column = 0; + + for (; column <= columns - nb_lanes; column += nb_lanes) + { + Acc sum = vector_traits::temp_acc_zero(); + for (index_t row = 0; row < rows; ++row) + sum = inner::vmacc(sum, + original.row(row).vector_op(column), + v[row]); + inner::vstore1<1>(res.ptr() + column,sum); + } + + const vector_length_t remaining = columns - column; + if (remaining > 0) + { + const mve_pred16_t predicate = inner::vctpq::mk(remaining); + Acc sum = vector_traits::temp_acc_zero(); + for (index_t row = 0; row < rows; ++row) + sum = inner::vmacc( + sum, + original.row(row).vector_op_tail(column,remaining), + v[row]); + inner::vstore1_z<1>(res.ptr() + column,sum,remaining,predicate); + } + } + else + { + detail::dot_transposed_unrolled(res,m,v); + } +} + +#endif + #define MATRIX_DIM2 2 #define MATRIX_DIM3 3 #define MATRIX_DIM4 4 @@ -404,4 +564,4 @@ __STATIC_INLINE void _dot_m_m(const MA& pSrcA, #endif -/*! @} */ \ No newline at end of file +/*! @} */ diff --git a/dsppp/Include/dsppp/Scalar/basic.hpp b/dsppp/Include/dsppp/Scalar/basic.hpp index 528b76458..2f5b51a0b 100644 --- a/dsppp/Include/dsppp/Scalar/basic.hpp +++ b/dsppp/Include/dsppp/Scalar/basic.hpp @@ -12,6 +12,96 @@ #define SCALAR_UNROLL 2 +template +inline void _round_to_nearest(DST& destination, const SRC& source, + const vector_length_t length, + const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + destination[i] = static_cast( + std::nearbyint(static_cast(source[i]))); +} + +template +inline void _round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + { + float value = std::nearbyint(static_cast(source[i])) + + static_cast(offset); + if (value < static_cast(minimum)) + value = static_cast(minimum); + if (value > static_cast(maximum)) + value = static_cast(maximum); + destination[i] = static_cast(value); + } +} + +template +inline void _round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum, + const vector_length_t length, const Scalar* = nullptr) +{ + using T = typename traits::Scalar; + for (index_t i = 0; i < length; ++i) + { + float value = std::nearbyint( + static_cast(source[i]) * multiplier) + + static_cast(offset); + if (value < static_cast(minimum)) + value = static_cast(minimum); + if (value > static_cast(maximum)) + value = static_cast(maximum); + destination[i] = static_cast(value); + } +} + +template +inline bool _nearest_even_range_predicate( + const MASK& mask, index_t i, vector_length_t, + const Scalar* = nullptr) +{ + return mask[i]; +} + +template +inline void _masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale, + const vector_length_t length, + const Scalar* architecture = nullptr) +{ + for (index_t i = 0; i < length; ++i) + if (_nearest_even_range_predicate(mask, i, 1, architecture)) + destination[i] += source[i] * scale; +} + +template +inline auto _masked_dot_sum(const A& a, const B& b, const MASK& mask, + const vector_length_t length, + const Scalar* architecture = nullptr) +{ + using T = typename traits::Scalar; + MaskedDotSum result{T{}, T{}}; + for (index_t i = 0; i < length; ++i) + if (_nearest_even_range_predicate(mask, i, 1, architecture)) + { + result.dot += a[i] * b[i]; + result.sum += a[i]; + } + return result; +} + /** * @brief Fill evaluator for scalar architecture * diff --git a/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp b/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp index 3bbfdb714..5fb49abb7 100644 --- a/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp +++ b/dsppp/Include/dsppp/Scalar/matrix_multiply.hpp @@ -41,6 +41,17 @@ __STATIC_INLINE void _arm_mat_trans( * @tparam V Vector datatype * @tparam RES Result datatype */ +template +inline void _dot_m_v(RES &res, + const TransposeView &m, + const V &v, + const Scalar* = nullptr) +{ + detail::dot_transposed_unrolled(res,m,v); +} + template @@ -151,7 +162,70 @@ inline void _dot_m_v(RES &res, } } +namespace detail { + +// Compute one matrix-row/vector dot product for each of up to four consecutive +// rows. Each scalar from the vector is loaded once and reused by all dot +// product accumulators. +template +inline void matvec_scalar_rows(T *output, const M &matrix, const V &vector, + const index_t first_row) +{ + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using Acc = typename number_traits::accumulator; + + Acc sums[ROWS] = {}; + const TM *rows[ROWS]; + for (index_t row = 0; row < ROWS; ++row) + rows[row] = matrix.const_ptr() + + (first_row + row) * matrix.stride(); + + for (index_t column = 0; column < matrix.columns(); ++column) + { + const TV value = vector[column]; + for (index_t row = 0; row < ROWS; ++row) + sums[row] = inner::mac(sums[row], rows[row][column], value); + } + + for (index_t row = 0; row < ROWS; ++row) + output[row] = inner::from_accumulator(sums[row]); +} + +} // namespace detail + +// Fill the lazy matvec cache in groups of four rows, then handle the final +// one to three rows with the same scalar kernel. +template +inline void _matvec_block(T *output, const M &matrix, const V &vector, + index_t first_row, vector_length_t row_count, + const Scalar* = nullptr) +{ + while (row_count >= 4) + { + detail::matvec_scalar_rows<4>(output, matrix, vector, first_row); + output += 4; + first_row += 4; + row_count -= 4; + } + + switch (row_count) + { + case 3: + detail::matvec_scalar_rows<3>(output, matrix, vector, first_row); + break; + case 2: + detail::matvec_scalar_rows<2>(output, matrix, vector, first_row); + break; + case 1: + detail::matvec_scalar_rows<1>(output, matrix, vector, first_row); + break; + default: + break; + } +} + #include "matrix_multiply_fixed.hpp" #include "matrix_multiply_float.hpp" -/*! @} */ \ No newline at end of file +/*! @} */ diff --git a/dsppp/Include/dsppp/algorithms.hpp b/dsppp/Include/dsppp/algorithms.hpp index 921b89681..6f3309620 100644 --- a/dsppp/Include/dsppp/algorithms.hpp +++ b/dsppp/Include/dsppp/algorithms.hpp @@ -2,6 +2,8 @@ /** @file */ #pragma once +#include + /** \defgroup DSPPP C++ extension * C++ template extension to CMSIS-DSP. It is not yet part of * the pack but the headers can be found on the @@ -23,6 +25,116 @@ namespace arm_cmsis_dsp { * Algorithms written in an architecture independent way */ +/** Result of a masked dot product and sum computed in one traversal. */ +template +struct MaskedDotSum +{ + T dot; + T sum; +}; + +/** + * Mask selecting values whose scaled nearest-even integer lies in a range. + * + * The test is inclusive and evaluates + * `minimum <= nearbyint(value * multiplier) + offset <= maximum`. + */ +template +class NearestEvenRangeMask +{ +public: + using Scalar = typename traits::Scalar; + using Stored = typename VecRef::type; + + NearestEvenRangeMask(const V& values, float multiplier, Scalar offset, + Scalar minimum, Scalar maximum) + : values_(VecRef::ref(values)), multiplier_(multiplier), + offset_(offset), minimum_(minimum), maximum_(maximum) {} + + bool operator[](index_t i) const + { + const float rounded = std::nearbyint( + static_cast(values_[i]) * multiplier_); + const float selected = rounded + static_cast(offset_); + return selected >= static_cast(minimum_) && + selected <= static_cast(maximum_); + } + + vector_length_t length() const { return values_.length(); } + + const Stored& values() const { return values_; } + float multiplier() const { return multiplier_; } + Scalar offset() const { return offset_; } + Scalar minimum() const { return minimum_; } + Scalar maximum() const { return maximum_; } + +private: + Stored values_; + float multiplier_; + Scalar offset_; + Scalar minimum_; + Scalar maximum_; +}; + +template +inline auto nearest_even_range_mask( + const V& values, float multiplier, + typename traits::Scalar offset, typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + return NearestEvenRangeMask(values, multiplier, offset, + minimum, maximum); +} + +/** Scale in float, round to nearest-even, offset, and clip. */ +template +inline void round_scaled_to_nearest_clipped( + DST& destination, const SRC& source, float multiplier, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + _round_scaled_to_nearest_clipped( + destination, source, multiplier, offset, minimum, maximum, + destination.length(), CURRENT_ARCH); +} + +/** Element-wise nearest-even rounding into an existing vector. */ +template +inline void round_to_nearest(DST& destination, const SRC& source) +{ + _round_to_nearest(destination, source, destination.length(), CURRENT_ARCH); +} + +/** Nearest-even rounding followed by an offset and inclusive clipping. */ +template +inline void round_to_nearest_clipped( + DST& destination, const SRC& source, + typename traits::Scalar offset, + typename traits::Scalar minimum, + typename traits::Scalar maximum) +{ + _round_to_nearest_clipped(destination, source, offset, minimum, maximum, + destination.length(), CURRENT_ARCH); +} + +/** Accumulate `source * scale` into destination where mask is true. */ +template +inline void masked_scale_add(DST& destination, const SRC& source, + const MASK& mask, + typename traits::Scalar scale) +{ + _masked_scale_add(destination, source, mask, scale, + destination.length(), CURRENT_ARCH); +} + +/** Compute a masked dot product and masked sum of the first operand. */ +template +inline auto masked_dot_sum(const A& a, const B& b, const MASK& mask) +{ + return _masked_dot_sum(a, b, mask, a.length(), CURRENT_ARCH); +} + /* Matrix transpose diff --git a/dsppp/Include/dsppp/autodiff/README.md b/dsppp/Include/dsppp/autodiff/README.md new file mode 100644 index 000000000..1888260f2 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/README.md @@ -0,0 +1,199 @@ +# Reverse automatic differentiation + +Training an algorithm requires the derivative of a loss with respect to every +parameter being learned. Automatic differentiation (AD) computes these +gradients from the operations executed by the algorithm. + +Reverse-mode AD is efficient when a computation has more inputs than outputs. +Training normally has many parameter inputs and one scalar loss output, so one +forward evaluation followed by one backward pass computes the gradients for +all parameters. + +The purpose of adding reverse AD to CMSIS-DSP is to enable efficient on-device +learning on resource-constrained devices. It reuses accelerated CMSIS-DSP +kernels and the fusion capabilities of the CMSIS-DSP C++ extension, while +providing predictable fixed memory use, no heap allocation, and no exceptions. + +## Why this implementation uses CMSIS-DSP + +Automatic differentiation needs high performance in both directions. The +forward pass evaluates the model, while the backward pass propagates and +accumulates gradients. This implementation uses CMSIS-DSP for both. + +When an operation maps directly to an optimized CMSIS-DSP C kernel, its +forward pass uses the CMSIS-DSP kernel. For example, float32 dot products +use `arm_dot_prod_f32` and float16 dot products use `arm_dot_prod_f16`; the +corresponding fully connected and matrix products use the f32 or f16 matrix +kernels. These kernels provide implementations optimized for the selected Arm +target, including Helium implementations where available. + +The CMSIS-DSP C++ expression system is particularly useful in backward rules, +which often combine element-wise computation with accumulation. A typical +update is: + +```text +gradient += input * output_gradient +``` + +Calling separate multiplication and addition kernels would require a temporary +buffer and two loops. A C++ expression can fuse the complete update into one +loop, avoiding the temporary and reducing memory traffic while giving the +compiler a larger operation to optimize and vectorize. + +For example, the fully connected backward pass computes: + +```text +bias_gradient += output_gradient +weight_gradient += outer(output_gradient, input_value) +input_gradient += dot(transpose_view(weight_value), output_gradient) +``` + +These are the expressions used by `FullyConnectedOperator::backward`. The +outer-product expression fuses multiplication with weight-gradient +accumulation. For the input gradient, `transpose_view` is a zero-copy view and +the `dot` overload returns a lazy transposed-matrix/vector expression, so its +result is accumulated without materializing either a transposed weight matrix +or a temporary result vector. Matrix multiplication similarly accumulates its +weight gradient through lazy `matvec` expressions. This combination of +optimized C kernels and fusible C++ expressions is especially important for +training, because backward passes contain more compound updates and +accumulations than typical forward inference code. + +## Examples + +### Polynomial regression + +`dsppp/Examples/autodiff_regression.cpp` trains a cubic polynomial to +approximate `sin(x)`. It demonstrates a batch loss, RMSProp, graph reuse, +parameter freezing, and saving learned values. + +### LMS adaptive filter + +`dsppp/Examples/autodiff_lms.cpp` identifies an unknown FIR filter with a +per-sample quadratic loss and SGD update. It is an educational demonstration; +the specialized CMSIS-DSP LMS implementation is more efficient for production +filtering. + +### Iris classifier + +`dsppp/Examples/autodiff_iris.cpp` trains a small two-layer classifier with +Adam and tests it on 30 patterns excluded from training. A single macro selects +the float32 or float16 implementation. + +### Quantization-aware training + +`dsppp/Examples/autodiff_fully_connected_qat.cpp` demonstrates +quantization-aware training of a fully connected layer for later deployment +with CMSIS-NN or Ethos-U. See +[Fine-tuning an output layer after Ethos-U](doc/ethos_cmsis_nn_fine_tuning.md) +for the main deployment use case: keeping the Ethos-U output and trained-layer +input quantization parameters identical so their int8 tensors connect without +requantization. + +## How reverse differentiation works here + +During the **forward pass**, each operator computes its output and, when +recording is enabled, adds a small record to a `Tape`. The record identifies +the values and gradients needed by that operator's derivative rule. The +ordered collection is called a tape because the operations are recorded going +forward and replayed in reverse. + +The **backward pass** starts at the selected output, normally a scalar loss. It +sets the loss gradient to one and visits the recorded operations in reverse +order. Each operation applies its local derivative rule and accumulates its +contribution into the gradients of earlier intermediate results and learnable +parameters. This reverse application of the chain rule produces all parameter +gradients needed by an optimizer. + +Numerical values remain in buffers owned by the application. `BufferView` and +`Tape` support `float` and `float16_t`; the default `T` is `float`. An +`Arena` provides a fixed amount of memory for gradients and tape +records, so memory use cannot grow unexpectedly at runtime. Ordinary inputs +registered with +`tape.input()` do not receive gradient storage. + +On a target with CMSIS-DSP float16 support, select the half-precision path by +using `Arena`, `Tape`, and matching operator and +optimizer specializations. + +The implementation is modular: an application includes and registers only the +operator headers it uses. The core in `reverse.hpp` manages views, fixed arena +storage, operator registration, and reverse traversal; each operator header +contains its own forward computation and derivative rule. + +## Minimal example + +```cpp +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ + Arena<512, float> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); + + float x_value[] = {1.0F, 2.0F}; + float a_value = 3.0F; + float y_value[2] = {}; + + BufferView x = tape.input(x_value); + BufferView a = tape.parameter(a_value); + BufferView y = tape.output(y_value); + + // y = a * x + y = scale(x, a); // y = {3, 6}; records the operation + const float seed[] = {1.0F, 1.0F}; + tape.backward(y, seed, 2U); // da = 1*1 + 1*2 = 3 + return tape.good() ? 0 : 1; +} +``` + +## Documentation + +- [Concepts and memory model](doc/concepts.md) explains the arena, tape, + buffer roles, recording, buffer lifetime, and errors. +- [Training loop](doc/training_loop.md) gives a complete, defined example and + explains setup, graph reuse, gradient clearing, reverse propagation, and the + optimizer update. +- [Operators](doc/operators.md) documents the current operator families, + formulas, shape rules, dropout behavior, and the CMSIS-DSP implementation + paths. +- [Fine-tuning an output layer after Ethos-U](doc/ethos_cmsis_nn_fine_tuning.md) + explains how to fix the QAT input quantization parameters to the Ethos-U + output interface and deploy the trained layer without an intermediate + conversion. +- [Optimizers](doc/optimizers.md) documents SGD, Adam, and RMSProp capacities, + initialization, updates, freezing, and errors. +- [Worked implementation flow: `y = a * x`](doc/implementation_flow.md) follows + one expression through `reverse.hpp`, including its tape record, `producer`, + node links, gradient reset, seed, and backward rule. +- [Converting PyTorch modules](doc/pytorch_conversion.md) documents supported + mappings, batch and shape constraints, generated code, and the Iris + comparison script. + +## PyTorch conversion + +[`scripts/pytorch_to_autodiff.py`](scripts/pytorch_to_autodiff.py) converts a +supported batch-1 PyTorch `nn.Module` to a reusable CMSIS-DSP autodiff C++ +class. It rejects unsupported operators and values that cannot be represented +as scalars, vectors, or matrices. The accompanying +[`scripts/iris_pytorch.py`](scripts/iris_pytorch.py) provides a PyTorch version +of the Iris network, dataset preparation, training loop, and test split. +See [Converting PyTorch modules](doc/pytorch_conversion.md) for supported +mappings, constraints, generated code, and usage. + +## Tests + +Autodiff is tested through the usual dsppp C++ board-test framework. The same +type-generic suite is instantiated for the datatype selected by `F32_DT` or +`F16_DT`. From `dsppp`, run for example: + +```sh +python run_all.py --test AUTODIFF_TEST --dt F32_DT +``` + +Use `--dt F16_DT` to run the float16 instantiation on a target defining +`ARM_FLOAT16_SUPPORTED`. diff --git a/dsppp/Include/dsppp/autodiff/doc/concepts.md b/dsppp/Include/dsppp/autodiff/doc/concepts.md new file mode 100644 index 000000000..db0b8df92 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/concepts.md @@ -0,0 +1,133 @@ +# Concepts and memory model + +## Tape and arena + +Reverse-mode AD first computes values and records the operations that produced +them. It then visits those records in reverse order to propagate gradients. +This ordered record is conventionally called a **tape**: operations are +recorded during the forward pass and played backward during differentiation. + +`Tape` manages this record and the gradient buffers. `Arena` supplies +exactly `Bytes` bytes of storage to a `Tape`: + +```cpp +Arena<2048, float> arena; +Tape &tape = arena.tape(); +``` + +The capacity is fixed and no dynamic memory allocation is performed. If the +arena is too small for a gradient or operation record, the tape reports +`Status::out_of_memory` instead of trying to grow it. + +## Values, views, and gradients + +The application owns every value buffer, including intermediates and final +outputs. A `BufferView` is only a non-owning description containing value and +gradient pointers, a length, its tape, its role, and the operation that +produced it. + +- `tape.input(values)` creates an ordinary input. It has no gradient storage. +- `tape.parameter(values)` creates a trainable parameter and allocates its + gradient from the arena. +- `tape.output(values)` creates an intermediate or result and allocates its + gradient from the arena. + +Caller-owned gradients can be supplied with overloads such as +`tape.parameter(values, gradients, length)` and +`tape.output(values, gradients, length)`. Those views consume no arena space +for their gradient arrays. + +Values and caller-owned gradients must remain alive until `backward()` has +finished. Values read by a backward rule must not be changed after the forward +operation. Output storage must not alias an operator's inputs, and a live +intermediate must not be overwritten by a later operation before +`backward()`, `rewind_graph()`, or `reset()`. + +True C arrays preserve their extent, so `tape.input(array)` can deduce the +length. A pointer does not carry a length and requires an explicit overload: + +```cpp +BufferView input = tape.input(pointer, number_of_elements); +MatrixView weights = tape.parameter(pointer, rows, columns); +``` + +## Operator registration + +The core contains no numerical operators. Each operator is a separate header +and class. An application includes and registers only what it uses. This keeps +code size down: operators that are not referenced can be removed by the +compiler and linker. + +```cpp +#include + +tape.register_operator>(); +``` + +Using an operator that has not been registered sets +`Status::operator_not_registered`. A tape can register 16 operator types by +default. Define `DSPPP_AUTODIFF_MAX_OPERATORS` to a larger power of two before +including `reverse.hpp` when more are needed. Registrations survive +`Tape::reset()`. + +See [Operators](operators.md) for the current operator API and formulas. + +## Recording and `RecordingScope` + +Normally an assignment such as `y = relu(x)` computes `y` and appends a record +for its backward rule. When recording is disabled, the numerical result is +still computed but no record is appended. This is useful for inference and +consumes no additional arena space. + +`RecordingScope` temporarily changes this tape setting for one block of code: + +```cpp +{ + RecordingScope inference(tape, false); + probability = softmax(logits); + // No operation record is added here. +} // The recording state that existed before the scope is restored. +``` + +The name means "the scope in which tape recording has this setting." Creating +the object saves the previous setting and applies the requested one. Leaving +the block restores the previous setting, including when the block exits early. + +A value computed without recording has no `producer`, so it cannot be passed +as the root of `backward()`. Doing so sets `Status::invalid_output`. + +## Reusing arena storage + +`Tape::reset()` releases all arena-managed gradients and records, invalidating +all views. Operator registrations are retained. + +Training normally needs to retain parameter and output gradient buffers while +rebuilding the operation records on every iteration: + +1. Create persistent views. +2. Call `begin_graph()`. This saves the current arena position. +3. At the start of each iteration, call `rewind_graph()`. It returns to that + saved position and clears the record list without invalidating the views. +4. Evaluate the forward graph again, which records fresh operations pointing + at the current values. + +Both marking and rewinding take constant time. + +## Failure model + +The first error is sticky until `clear_status()`, `reset()`, `begin_graph()`, or +`rewind_graph()` clears it as specified by the API. Check `tape.good()` or the +Boolean result of `backward()` before using gradients. + +- `out_of_memory`: a gradient or operation record did not fit in the arena. +- `tape_mismatch`: with `DSPPP_AUTODIFF_ENABLE_VALIDATION=1`, views, roles, + shapes, pointers, or aliasing are invalid for an operation. +- `invalid_output`: the backward root or seed is invalid, or graph rewinding + was requested without a mark. +- `operator_not_registered`: an expression's operator was not registered. +- `operator_registry_full`: no registration slot remains. + +If record allocation fails after a forward kernel runs, its numerical result +may already have been written, but the output is detached from the tape. +Measure `tape.used()` on representative worst-case graphs to choose an arena +capacity with suitable margin. diff --git a/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md b/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md new file mode 100644 index 000000000..64ad6bfba --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/ethos_cmsis_nn_fine_tuning.md @@ -0,0 +1,136 @@ +# Fine-tuning an output layer after Ethos-U + +A common on-device training arrangement is to run a fixed feature extractor on +Ethos-U and train only its final layer with CMSIS-DSP autodiff. During +training, Ethos-U produces a signed-int8 feature vector. The application +dequantizes that vector because the autodiff graph uses floating-point +buffers. Quantization-aware training (QAT) then makes the trainable layer +behave like the int8 layer that will eventually be deployed with Ethos-U or +CMSIS-NN. + +The important interface contract is that the Ethos-U output and the trained +layer input use exactly the same quantization parameters. If the Ethos-U +output has scale `s_interface` and zero-point `z_interface`, the input Q/DQ +pair of the autodiff network must use those same fixed values: + +```text +Ethos-U output during training Autodiff layer + +q_ethos (int8) -- dequantize(s_interface, z_interface) --> x (float) + -- Q(s_interface, z_interface) + -- DQ(s_interface, z_interface) --> trainable layer +``` + +For an int8 value `q_ethos`, the floating-point training input is: + +```text +x = (q_ethos - z_interface) * s_interface +``` + +Applying Q/DQ with the same parameters maps this value back to `q_ethos` and +then reconstructs the same floating-point value, apart from floating-point +rounding. The pair records the deployed input quantization in the training +graph and also handles values from other floating-point training sources with +the same clipping and rounding that the int8 layer will see. + +## Keeping the interface parameters fixed + +The interface scale and zero-point are metadata exported with the Ethos-U +network. They are a deployment constraint, not values for the output-layer +optimizer to learn. Copy them into persistent values and use them for both +operators: + +```cpp +constexpr Int8Quantization activation_quantization = + Int8Quantization::activation(); + +// Exact quantization metadata of the Ethos-U output tensor. +float interface_scale_value = ethos_output_scale; +float interface_zero_point_value = ethos_output_zero_point; + +// Q/DQ currently requires parameter views so that its backward records have +// gradient storage. These views remain fixed because they are not added to +// the optimizer. +BufferView interface_scale = tape.parameter(interface_scale_value); +BufferView interface_zero_point = tape.parameter(interface_zero_point_value); + +quantized_input = quantize(input, interface_scale, interface_zero_point, + activation_quantization); +dequantized_input = dequantize(quantized_input, interface_scale, + interface_zero_point, + activation_quantization); +``` + +Do **not** call `optimizer.add(interface_scale)` or +`optimizer.add(interface_zero_point)`. Add only the values that should be +learned, for example the new layer's weights, bias, weight scales, output +scale, and output zero-point: + +```cpp +optimizer.add(weights); +optimizer.add(bias); +optimizer.add(weight_scale); +optimizer.add(output_scale); +optimizer.add(output_zero_point); +``` + +If the interface views were already added to an optimizer, they can instead +be kept unchanged with `freeze_parameters(optimizer, interface_scale, +interface_zero_point)`. Not adding them is simpler when they must remain fixed +for the entire training run. + +Although `tape.input()` normally expresses a non-trainable value, it must not +currently be used for these two Q/DQ arguments. The quantize and dequantize +operators require parameter views for scale and, for asymmetric activation +quantization, zero-point. Whether a parameter is updated is controlled by the +optimizer, not by the presence of a Q/DQ node. + +The copied scale must be positive and the zero-point must be the integer int8 +zero-point from the Ethos-U tensor metadata. Q/DQ applies the +`Int8Quantization::activation()` constraints, but it must not be relied upon +to invent or recalibrate this fixed interface. + +## Training and deployment + +During training: + +1. Run the fixed prefix on Ethos-U and obtain its int8 output. +2. Dequantize it with the Ethos-U output scale and zero-point. +3. Feed the resulting float vector to autodiff. +4. Apply input Q/DQ using the same fixed interface parameters. +5. Train the new layer through its weight Q/DQ and output Q/DQ pairs. + +The new layer's weight quantization parameters are independent of the +interface. Its output scale and zero-point may also be learned unless the next +deployed operator imposes another fixed quantized interface. Export the +trained floating-point weights to symmetric per-output-channel int8 weights, +and export the bias as int32 with scale: + +```text +bias_scale[channel] = s_interface * weight_scale[channel] +``` + +For inference, remove the training-only Q/DQ simulation and connect the int8 +tensors directly: + +```text +Ethos-U prefix -- q_ethos (int8) --> int8 trained layer +``` + +No data conversion is needed when all of the following match: + +- signed-int8 element type; +- tensor shape, ordering, and memory layout; +- input scale equals the Ethos-U output scale; +- input zero-point equals the Ethos-U output zero-point. + +For CMSIS-NN, configure the trained layer with +`input_offset = cmsis_nn_offset(interface_zero_point_value)`. Its output +requantization multiplier and shift are derived in the usual way from the +interface scale, per-channel weight scale, and layer output scale. + +If either interface quantization parameter differs, direct connection is not +equivalent. An int8-to-int8 requantization step is then required; dequantizing +to float and quantizing again is another, usually less efficient, option. The +purpose of fixing the input Q/DQ parameters during fine-tuning is to avoid +both conversions in the deployed network. diff --git a/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md new file mode 100644 index 000000000..45df59e16 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/implementation_flow.md @@ -0,0 +1,284 @@ +# Worked implementation flow: `y = a * x` + +This example follows one vector scaling operation through the public API, +`ScaleOperator`, and the internals of `reverse.hpp`. It uses a trainable scalar +`a`, a constant input vector `x`, and an output vector `y`: + +```text +y[i] = a * x[i] +``` + +The scale operator is used instead of element-wise `MultiplyOperator` because +`a` has one element while `x` and `y` have several. + +## Complete example + +```cpp +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ + Arena<512, float> arena; + Tape &tape = arena.tape(); + tape.register_operator>(); + + float a_value = 3.0F; + float x_value[] = {2.0F, -1.0F}; + float y_value[2] = {}; + + BufferView a = tape.parameter(a_value); + BufferView x = tape.input(x_value); + BufferView y = tape.output(y_value); + + y = scale(x, a); // Forward result: {6, -3}. + + const float seed[] = {1.0F, 1.0F}; + if (!tape.backward(y, seed, 2U)) + return 1; + + // a.gradient(0) == 1*2 + 1*(-1) == 1 + // x.has_gradient() == false + return 0; +} +``` + +The seed is the vector supplied as input to `backward()`. It is copied into +`y`'s gradient buffer before reverse traversal starts. Because `y` is a vector, +each output element has its own derivative with respect to `a`: + +```text +dy/da = {x[0], x[1]} = {2, -1} +``` + +The seed specifies how those output contributions are combined. With +`seed = {1, 1}`, the backward pass computes +`a.gradient(0) = 1*2 + 1*(-1) = 1`. A seed of `{1, 0}` would select only +`y[0]` and produce `a.gradient(0) = 2`. One call does not build the complete +derivative vector for `y`; it propagates the selected scalar projection of the +vector output. No loss function is present in this example. Training normally +ends in a scalar loss, for which +`backward(scalar_output)` uses the default seed `1`. + +## Objects before the forward pass + +`Arena<512, float>` contains the storage array and constructs a `Tape` over it. The +tape initially has `used_ == 0`, `tail_ == nullptr`, `recording_ == true`, and +`status_ == Status::ok`. + +The three registrations create non-owning views: + +| View | `values_` points to | `gradients_` | `role_` | `producer_` | +| --- | --- | --- | --- | --- | +| `a` | `a_value` | one arena `float` | `parameter` | null | +| `x` | `x_value` | null | `input` | null | +| `y` | `y_value` | two arena `float`s | `intermediate` | null | + +`parameter()` and `output()` allocate and zero gradient arrays. `input()` does +not allocate one because the application did not request a derivative for +`x`. The value arrays themselves remain entirely caller-owned. + +`producer_` answers "which recorded node last produced this view?" Inputs and +parameters have no producer in this graph. It is initially null for `y` and is +set only after a record has been appended successfully. + +## Field map for `reverse.hpp` + +The core member names have these responsibilities: + +| Type | Member | Meaning | +| --- | --- | --- | +| `BufferView` | `values_` | Pointer to caller-owned forward values. | +| `BufferView` | `gradients_` | Pointer to its adjoint, or null for an input. | +| `BufferView` | `length_` | Number of scalar elements. | +| `BufferView` | `tape_` | Owning tape identity used for compatibility checks. | +| `BufferView` | `producer_` | Last successfully recorded node that wrote this view. | +| `BufferView` | `role_` | `input`, `parameter`, or `intermediate`. | +| `Tape` | `storage_`, `capacity_` | Start and byte size of caller-supplied arena storage. | +| `Tape` | `used_` | Current byte offset for the next aligned allocation. | +| `Tape` | `tail_` | Most recently appended node. | +| `Tape` | `recording_` | Whether forward operations append records. | +| `Tape` | `status_` | Sticky first-error status. | +| `Tape` | `graph_begin_`, `graph_marked_` | Saved allocation offset and validity flag used by graph rewinding. | +| `Tape` | `registered_operators_` | Fixed hash table of registered operator type tokens. | +| `detail::Node` | `previous` | Link to the preceding record. | +| `detail::Node` | `backward` | Function pointer for the record's local reverse rule. | +| `detail::Node` | `reset_gradient` | Function pointer that clears buffers used by that rule. | + +`OperatorAccess` is the narrow bridge used by separate operator headers to +read these private fields, validate views, report failure, append a record, and +set an output producer. It keeps operator-specific code out of `Tape` itself. + +## Forward assignment + +The statement + +```cpp +y = scale(x, a); +``` + +executes these steps: + +1. `scale(x, a)` returns a small `ScaleExpression` containing copies of the two + non-owning views. Only their pointers and metadata are copied; the `x` and + `a` value buffers are not copied. +2. `BufferView::operator=` calls `ScaleExpression::evaluate(y)`. +3. The expression calls `ScaleOperator::evaluate(y, x, a)`. +4. The operator clears `y.producer_`. This prevents an old record from being + mistaken for the producer if validation or allocation fails. +5. `OperatorAccess::require(tape)` checks the fixed registry. +6. When `DSPPP_AUTODIFF_ENABLE_VALIDATION=1`, validation checks that all views + belong to the same tape; `x` and `y` have the same length; `a` is a + one-element parameter with a gradient; and value and gradient buffers do + not alias illegally. These checks are compiled out by default. +7. `arm_scale_f32` computes `{3*2, 3*(-1)}` into `y_value`, giving `{6, -3}`. +8. Because recording is enabled and the output is nonempty, the operator asks + `OperatorAccess::append` for a `ScaleOperator::Record` in the arena. + +The record contains a `detail::Node` as its first member, followed by only the +pointers and length required by the local derivative: + +```text +node +output_gradient -> y.gradients() +input_value -> x.values() +input_gradient -> null +scale_value -> a.values() +scale_gradient -> a.gradients() +length = 2 +``` + +`Tape::append` aligns arena storage, constructs the record in place, installs +the `backward` and `reset_gradient` function pointers, and links the new node: + +```cpp +record->node.previous = tail_; +tail_ = &record->node; +``` + +For this one-operation graph, `previous` is null. Finally, +`y.producer_ = &record->node` marks this record as the origin of `y`. + +```mermaid +flowchart LR + AV["a view
parameter
value 3, gradient da"] + XV["x view
input
values 2, -1
no gradient"] + E["ScaleOperator::evaluate"] + R["Scale record
Node + pointers + length"] + YV["y view
values 6, -3
gradient dy
producer"] + + AV --> E + XV --> E + E -->|"arm_scale_f32"| YV + E -->|"append when recording"| R + YV -. "producer_ points to" .-> R + R -->|"scale_gradient"| AV + R -->|"input_value"| XV +``` + +## What the node links mean + +Every record begins with the same three-field `detail::Node`: `previous`, +`backward`, and `reset_gradient`. `previous` links records in creation order. +If a later operation computes `z` from `y`, its node points to the scale node: + +```text +z.producer_ -> z record -> scale record -> null +``` + +The link is chronological, not a list of a node's operands. Operand +relationships are represented by the value and gradient pointers stored in +each operator-specific record. Starting at the requested output's `producer_` +prevents traversal into records created after that output. Earlier unrelated +records can be visited, but their output gradients are zero, so their local +rules make no contribution. + +## `backward(y, seed, 2)` + +`Tape::backward` performs three phases. + +### 1. Validate the root + +The tape must still be good; `y` must belong to this tape and have valid value +and gradient storage; `y.producer_` must be non-null; and the seed pointer and +length must match `y.length_`. A value-only result produced while recording was +disabled has no producer and is therefore rejected. + +### 2. Reset graph gradients, then install the seed + +Starting at `y.producer_`, the first reverse walk calls each node's +`reset_gradient` function. For the scale record this sets: + +```text +dy = {0, 0} +da = 0 +``` + +It would also clear `dx` if `x` had gradient storage. The tape then copies the +caller seed into the root output gradient: + +```text +dy = {1, 1} +``` + +Resetting before seeding is essential: otherwise resetting the root record +would erase the seed. Clearing all reachable buffers also makes repeated calls +to `backward()` deterministic instead of unintentionally accumulating an +earlier reverse pass. + +### 3. Traverse backward + +The second reverse walk calls the node's operator-specific `backward` function. +For `y[i] = a*x[i]`, the chain rule is: + +```text +da += sum(dy[i] * x[i]) +dx[i] += a * dy[i] +``` + +The implementation evaluates the first expression with the CMSIS-DSP C++ dot +operation: + +```text +da = 0 + dot({1, 1}, {2, -1}) = 1 +``` + +`input_gradient` is null because `x` came from `tape.input()`, so the `dx` +update is skipped. If `x` were an intermediate output, it would have gradient +storage and the same rule would accumulate `{3, 3}` into it; an earlier node +would then consume that gradient when reverse traversal reached it. + +```mermaid +sequenceDiagram + participant App + participant Tape + participant Node as Scale record + participant Grad as Gradient buffers + + App->>Tape: backward(y, seed, 2) + Tape->>Tape: validate y and y.producer_ + Tape->>Node: reset_gradient(node) + Node->>Grad: clear dy + Node->>Grad: clear da + Tape->>Grad: copy seed into dy + Tape->>Node: backward(node) + Node->>Grad: accumulate dot(dy, x) into da + Note over Node,Grad: dx is skipped because x has no gradient buffer + Tape-->>App: true +``` + +## Why records store pointers rather than views + +The operator record retains only the data needed later: raw non-owning +pointers, dimensions, small metadata, and the common node prefix. It does not +own or copy tensors and does not retain the temporary expression object. This +keeps record size independent of the numerical buffer length. Gradient storage +still costs one scalar of the selected type (`T`) per parameter or intermediate +element. + +This is also why the caller must preserve values until the reverse pass. The +scale rule rereads `x_value` and `a_value`; changing either after the forward +pass would differentiate a different computation from the one whose `y_value` +was calculated. diff --git a/dsppp/Include/dsppp/autodiff/doc/operators.md b/dsppp/Include/dsppp/autodiff/doc/operators.md new file mode 100644 index 000000000..6f0401e54 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/operators.md @@ -0,0 +1,275 @@ +# Operators + +Each operator header owns its validation, forward computation, fixed-size tape +record, gradient reset, backward rule, and expression adapter. Operator classes +are templated on the scalar type (`float` by default and `float16_t` when +enabled). Include the header and register the matching specialization before +evaluating the expression. + +In the formulas below, `g` is the gradient arriving from the operator's output. +Every input gradient is accumulated with `+=` when that input has gradient +storage. A view created by `tape.input()` has none, so its contribution is +skipped. + +## Element-wise arithmetic + +| Expression | Forward | Backward | +| --- | --- | --- | +| `z = x + y` | `z[i] = x[i] + y[i]` | `dx[i] += g[i]`; `dy[i] += g[i]` | +| `z = x - y` | `z[i] = x[i] - y[i]` | `dx[i] += g[i]`; `dy[i] -= g[i]` | +| `z = x * y` | `z[i] = x[i] * y[i]` | `dx[i] += g[i]*y[i]`; `dy[i] += g[i]*x[i]` | + +These expressions require equal-length views. The forward paths dispatch to +the matching f32 or f16 CMSIS-DSP kernels. Multiply's backward path uses fused +CMSIS-DSP C++ expressions so multiplication and accumulation need no temporary +product vector. + +## Dot, scalar scale, and scalar offset + +`dot(x, y)` returns one value: + +```text +z = sum(x[i] * y[i]) +dx[i] += g * y[i] +dy[i] += g * x[i] +``` + +Its forward pass dispatches to `arm_dot_prod_f32` or `arm_dot_prod_f16`. + +`scale(x, a)` requires `a` to be a one-element parameter: + +```text +z[i] = a * x[i] +da += sum(g[i] * x[i]) +dx[i] += a * g[i] +``` + +The forward pass dispatches to `arm_scale_f32` or `arm_scale_f16`; the scalar +gradient uses the C++ dot expression. See the [worked implementation +flow](implementation_flow.md) for a line-by-line explanation. + +`offset(x, b)` likewise requires a one-element parameter: + +```text +z[i] = x[i] + b +db += sum(g[i]) +dx[i] += g[i] +``` + +Its forward pass dispatches to `arm_offset_f32` or `arm_offset_f16`; the bias +gradient uses the matching accumulate kernel. + +## ReLU and softmax + +ReLU computes `y[i] = max(0, x[i])` with the matching f32 or f16 clip kernel. +Its backward rule passes `g[i]` only when the saved input value is strictly +positive. The derivative at zero is defined as zero. + +Softmax uses a log-sum-exp forward calculation for numerical stability: + +```text +y[i] = exp(x[i] - log(sum(exp(x)))) +projection = dot(g, y) +dx[i] += y[i] * (g[i] - projection) +``` + +The forward path dispatches to the matching f32 or f16 log-sum-exp, offset, and +vector-exponential kernels. + +## Signed int8 quantize/dequantize + +`quantize` and `dequantize` implement the signed int8 affine scheme used by +LiteRT, CMSIS-NN, and Ethos-U while keeping every graph buffer in `float` or +`float16_t`. The quantized codes are integer-valued floating-point numbers; +no `int8_t` participates in training. + +For scale `s`, zero-point `z`, and integer limits `qmin` and `qmax`: + +```text +q[i] = clamp(nearbyint(x[i] / s) + nearbyint(z), qmin, qmax) +y[i] = (q[i] - nearbyint(z)) * s +``` + +Use `Int8Quantization::activation()` for asymmetric per-tensor activations. +It uses the backend-required `[-128, 127]` range and a learnable scale and +zero-point. Use `Int8Quantization::weights(axis_size, inner_size)` for weights. +It uses symmetric `[-127, 127]` codes, forces the effective zero-point to zero, +and supports one learnable scale per quantized axis. `inner_size` is the +product of the row-major dimensions after that axis. Examples are: + +```cpp +auto activation_q = Int8Quantization::activation(); +auto fully_connected_w = Int8Quantization::weights(outputs, inputs); +auto conv2d_ohwi_w = Int8Quantization::weights(outputs, height * width * inputs); +auto depthwise_hwc_w = Int8Quantization::weights(channels, 1U); +``` + +For symmetric weights, pass a zero-valued input view as `zero_point`; it is +fixed and receives no gradient. For activations, both scale and zero-point are +parameter views. Scale values must remain strictly positive. An optimizer can +update both activation parameters and the per-axis weight scales. Q/DQ projects +these parameters before each forward calculation: scale receives a small +positive numerical floor, asymmetric zero-points are clamped to the int8 +domain, and symmetric weight zero-points are forced to zero. Training loops do +not need to enforce these constraints themselves. + +The Q/DQ pair uses a straight-through estimator. Inside the int8 range its +combined input derivative is one; saturated inputs receive zero. The local +rules also retain the scale derivative caused by quantization error and a +zero-point derivative at saturation, allowing the representable range to be +learned. The forward path is the same affine quantize/dequantize calculation +used when exporting the final int8 values. + +Dequantization is affine, so its forward calculation and backward input +gradient use fused CMSIS-DSP C++ vector expressions. Its parameter gradients +use vector dot-product and accumulation kernels. Quantization uses the C++ +extension's nearest-even rounding, range-mask, masked accumulation, and masked +dot/sum algorithms. Their Helium implementations use MVE rounding and +predication directly without allocating a mask buffer. Other architectures +use portable scalar implementations based on `std::nearbyint`. + +CMSIS-NN calls the negated zero-point an `offset`. Use +`cmsis_nn_offset(zero_point)` when filling APIs such as `input_offset` or +`output_offset`. Biases are not processed by this Q/DQ pair: CMSIS-NN/LiteRT +requires int32 bias with zero-point zero and scale +`input_scale * weight_scale[channel]`. + +When an autodiff layer is trained after a fixed Ethos-U network, its input +scale and zero-point must normally remain equal to the Ethos-U output +parameters. See [Fine-tuning an output layer after +Ethos-U](ethos_cmsis_nn_fine_tuning.md) for the fixed-parameter setup and the +conditions for connecting the two int8 tensors without requantization. + +## Losses + +Quadratic error returns a scalar sum, not a mean: + +```text +loss = sum((prediction[i] - target[i])^2) +d_prediction[i] += g * 2 * (prediction[i] - target[i]) +``` + +Categorical cross entropy also returns a scalar sum. The probability floor is +`1e-7` for float32 and `1e-4` for float16; the larger half-precision floor keeps +the reciprocal used by the derivative finite: + +```text +p_safe[i] = max(probability[i], floor) +loss = -sum(target[i] * log(p_safe[i])) +d_probability[i] = -g * target[i] / p_safe[i] +``` + +For both losses, the target must be an input view. Cross entropy is intended +for a probability vector from softmax and a one-hot target. The probability +floor prevents non-finite loss and gradient values. Its forward pass reuses the +probability gradient buffer as scratch; `backward()` resets that buffer before +propagating gradients. + +## Fully connected + +`fully_connected(x, W, b)` computes `y = W*x + b`, where `W` is a row-major +matrix parameter and `b` is a parameter vector: + +```text +dW += outer(g, x) +db += g +dx += transpose(W) * g +``` + +The number of columns in `W` must equal the input length; its rows must equal +the bias and output lengths. Dimensions must fit the `uint16_t` CMSIS-DSP C +matrix descriptor. + +The forward matrix-vector product dispatches to `arm_mat_vec_mult_f32` or +`arm_mat_vec_mult_f16`, followed by a fused C++ bias accumulation. Backward bias +and outer-product updates are fused C++ expressions. If `x` needs a gradient, the current implementation evaluates +the lazy expression `dot(transpose_view(W), g)` and accumulates it into `dx`. +The transpose is a view: no transposed numerical matrix is allocated. + +## Matrix multiply + +`matrix_multiply(X, W)` computes `Y = W*X`. `W` is a row-major parameter matrix; +`X` is a row-major input matrix flattened into an input `BufferView`. If `W` +has shape `rows x inner`, the input length must be divisible by `inner`, and +the inferred input shape is `inner x columns`. The output length is +`rows*columns`. + +Only `W` is differentiated: + +```text +dW += dY * transpose(X) +``` + +The forward pass dispatches to `arm_mat_mult_f32` or `arm_mat_mult_f16`. In the +backward pass, each row of `dW` is accumulated with the lazy C++ expression +`matvec(X, corresponding_row_of_dY)`. This fuses matrix-vector evaluation with +gradient accumulation; it does not call a scalar dot kernel once per weight and +does not materialize `transpose(X)`. + +## Dropout + +Dropout is a training regularizer. With drop probability `p`, each element is +kept independently with probability `1-p`. **Inverted dropout** scales kept +values during training: + +```text +mask[i] is 1 with probability (1-p), otherwise 0 +scale = 1 / (1-p) +y[i] = mask[i] * x[i] * scale +dx[i] += mask[i] * g[i] * scale +``` + +This is the standard inverted-dropout convention. Its expected training output +is `x`, so inference can be an identity operation without an extra scale: + +```text +E[y[i]] = (1-p) * x[i] / (1-p) = x[i] +``` + +The probability must satisfy `0 <= p < 1`. At `p == 0`, forward and backward +are identity operations. When tape recording is disabled, dropout copies its +input unchanged, which implements inference behavior. + +Random state is explicit and caller-owned: + +```cpp +DropoutGenerator generator(1234U); +tape.register_operator>(); +hidden = dropout(hidden_linear, generator, 0.2F); +``` + +Initialize the generator once with a nonzero seed for a reproducible sequence. +A zero seed is replaced with the class's nonzero default because the xorshift +generator would otherwise remain zero. Each forward record saves the initial +random state, not an entire mask. Backward regenerates the identical mask from +that state. If record allocation fails, the generator is restored so a failed +operation does not consume random numbers. + +## Parameter initialization + +The AD layer does not initialize parameter values. The application must do so +before the first forward pass. Biases commonly start at zero. For neural-network +weight matrices, use a small random initialization appropriate to the +activation (for example Xavier/Glorot for many tanh or linear networks, or He +initialization for ReLU networks); identical zero weights can prevent hidden +units from learning distinct features. Optimizer moment/state arrays themselves +start at zero. + +## Common validation rules + +Operator argument validation is disabled by default so it adds no overhead to +the evaluation path. Define `DSPPP_AUTODIFF_ENABLE_VALIDATION` to `1` before +including the autodiff headers, or define it consistently for the complete +build, to enable these checks. + +When enabled, views used by one expression must belong to the same tape and +satisfy the operator's role and shape requirements. Output and input value +storage must be distinct, and output gradient storage must not alias an input +gradient. An invalid combination sets the sticky +`Status::tape_mismatch`. When validation is disabled, the application is +responsible for meeting these preconditions; invalid arguments may cause +out-of-bounds access or otherwise undefined results. + +A successfully computed value has a producer only when recording is enabled +and its record was appended successfully. Operator registration and arena +allocation failures are checked in both validation configurations. diff --git a/dsppp/Include/dsppp/autodiff/doc/optimizers.md b/dsppp/Include/dsppp/autodiff/doc/optimizers.md new file mode 100644 index 000000000..95ae6139b --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/optimizers.md @@ -0,0 +1,205 @@ +# Optimizers + +`SGD`, `Adam`, and `RMSProp` update caller-owned parameter values from +tape-managed or caller-owned gradients. Their third template argument selects +the scalar type (`float` by default, or `float16_t`), matching the tape views. +All optimizer metadata and numerical state are fixed inside the optimizer +object; none of the optimizers allocates memory. + +## Capacity arguments + +All three types have the same capacity template arguments: + +```cpp +Optimizer +``` + +- `MaximumElements` is the total number of scalar values across all added + parameter views. It has no default. +- `MaximumParameters` is the number of separately added views. It defaults to + 16. + +A single vector parameter of length 10 therefore needs 10 element slots but +only one parameter-view slot: + +```cpp +float value[10] = {}; +BufferView parameter = tape.parameter(value); +RMSProp<10, 1> optimizer; +optimizer.add(parameter); +``` + +A matrix also counts as one view, while all `rows*columns` entries count toward +`MaximumElements`. A three-element coefficient vector plus a separate scalar +bias fits exactly in `SGD<4, 2>`, `RMSProp<4, 2>`, or `Adam<4, 2>`. Writing +`Adam<100>` reserves 100 scalar state positions and the default 16 view slots. +For half precision, use for example `Adam<100, 16, float16_t>`. + +Adding the same value pointer twice is idempotent. A frozen parameter continues +to occupy both capacities. + +## SGD + +```cpp +SGD<4, 2> optimizer(1.0e-3F); // learning_rate +``` + +SGD applies plain stochastic gradient descent without momentum: + +```text +parameter -= learning_rate * gradient +``` + +The implementation uses a CMSIS-DSP C++ vector expression that fuses scaling +and subtraction into one loop. It therefore needs no intermediate vector and +has no per-element optimizer state. `MaximumElements` still limits the total +number of registered scalar parameters, while +`Entry entries_[MaximumParameters]` holds their non-owning pointers, lengths, +and frozen state. + +For online learning, such as LMS adaptation, one `step()` after each sample is +stochastic gradient descent because the current sample gradient estimates the +gradient of the expected loss. With this library's unscaled quadratic error, +an LMS step size `mu` corresponds to an SGD learning rate of `mu / 2`. + +## RMSProp + +```cpp +RMSProp<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.99F, // alpha + 1.0e-8F // epsilon +); +``` + +For each trainable scalar, this implementation performs RMSProp without +momentum or centering: + +```text +square_average = alpha * square_average + + (1 - alpha) * gradient^2 +parameter -= learning_rate * gradient + / (sqrt(square_average) + epsilon) +``` + +`square_average` starts at zero. `alpha` controls how slowly squared-gradient +history changes; values near one produce longer memory. `epsilon` prevents a +zero or very small denominator. The default is `1e-8` for float32 and `1e-4` +for float16, where `1e-8` would round to zero. These are conventional starting +points, but learning rate normally requires tuning for the model and loss +scale. + +Storage consists principally of one +`T square_average_[MaximumElements]` plus +`Entry entries_[MaximumParameters]`. Each entry stores value and gradient +pointers, length, state offset, and whether the parameter is trainable. + +## Adam + +```cpp +Adam<4, 2> optimizer( + 1.0e-3F, // learning_rate + 0.9F, // beta1 + 0.999F, // beta2 + 1.0e-8F // epsilon +); +``` + +On successful step `t`: + +```text +first_moment = beta1 * first_moment + + (1 - beta1) * gradient +second_moment = beta2 * second_moment + + (1 - beta2) * gradient^2 + +corrected_first = first_moment / (1 - beta1^t) +corrected_second = second_moment / (1 - beta2^t) + +parameter -= learning_rate * corrected_first + / (sqrt(corrected_second) + epsilon) +``` + +Both moment arrays and their powers are initialized so the first successful +step applies the usual bias correction. `beta1` controls first-moment memory, +`beta2` controls squared-gradient memory, and `epsilon` stabilizes the +denominator. The default epsilon is `1e-8` for float32 and `1e-4` for +float16, where `1e-8` would round to zero. The other defaults are standard +initial choices. + +Adam stores +`T first_moment_[MaximumElements]`, +`T second_moment_[MaximumElements]`, and +`entries_[MaximumParameters]`, so its principal per-element state is twice +RMSProp's. Its global step advances each time `step()` succeeds. + +## Adding, clearing, and stepping + +Add each parameter once after creating its view: + +```cpp +float coefficient_value[3] = {}; +float bias_value = 0.0F; +BufferView coefficients = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); + +RMSProp<4, 2> optimizer; +optimizer.add(coefficients); +optimizer.add(bias); +``` + +Here `coefficients` is a three-element parameter `BufferView`, and `bias` is a +separate one-element parameter `BufferView`. + +The usual iteration order is: + +```cpp +evaluate_forward_graph(); +optimizer.zero_grad(); +if (!tape.backward(loss) || !optimizer.step()) + handle_error(); +``` + +`zero_grad()` clears all gradients for parameters known to the optimizer. +`backward()` then writes the gradients for the current graph, and `step()` +consumes them. See [Training loop](training_loop.md) for why the explicit clear +is useful even though backward resets gradients referenced by its records. + +## Freezing + +Freezing prevents updates without changing the recorded graph. In this +example, `coefficients` is a parameter `BufferView` created with +`tape.parameter()` and previously passed to `optimizer.add()`: + +```cpp +freeze_parameters(optimizer, coefficients); // Train other parameters. +unfreeze_parameters(optimizer, coefficients); // Resume updates later. +``` + +The same functions also accept parameter `MatrixView` objects. Multiple views +can be passed when a layer owns more than one parameter, such as weights and a +bias. + +The parameter still participates in forward and backward propagation, and its +gradient may be computed. `step()` skips its value and optimizer state, so both +remain unchanged while frozen. Operators are not frozen because they are +stateless; freeze every parameter view owned by the logical layer instead. + +## Status and errors + +Optimizer errors are sticky. After the first error, `good()` is false, +`status()` reports it, and `step()` returns false. + +- `too_many_parameters`: another distinct view would exceed + `MaximumParameters`. +- `too_many_elements`: its scalar length would exceed the remaining + `MaximumElements` capacity. +- `invalid_parameter`: `add()` received a non-parameter or a view without a + gradient, or `freeze()` received a view that had not been added. + +Parameter values and gradient arrays are referenced, not copied. They must +remain alive as long as the optimizer uses them. A checkpoint containing only +parameter values is sufficient for inference. Reproducing the exact continuation +of training also requires saving RMSProp or Adam moment state and, for Adam, +its step-dependent powers; the current optimizer classes do not provide a +serialization API. Plain SGD has no additional numerical state to save. diff --git a/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md b/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md new file mode 100644 index 000000000..c49171a54 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/pytorch_conversion.md @@ -0,0 +1,123 @@ +# Converting PyTorch modules + +`scripts/pytorch_to_autodiff.py` converts a PyTorch `nn.Module` into a +reusable C++ class using the CMSIS-DSP autodiff API. The generated class owns +the trained parameter values, tape views, intermediate buffers, operator +registration, `forward()`, and an `add_parameters()` optimizer helper. + +## Model constraints + +The input has an implicit batch dimension of one. Pass only the embedded shape +to the converter. For example, a PyTorch input shaped `(1, 4)` uses +`--input-shape 4`. Every traced value is checked after removing that batch +dimension. Scalars, vectors, and matrices are accepted; higher-rank tensors or +a batch other than one stop conversion with an error. + +The current mappings are: + +| PyTorch | CMSIS-DSP autodiff | +| --- | --- | +| `nn.Linear` | `fully_connected` | +| `nn.ReLU`, `torch.relu`, functional ReLU | `relu` | +| vector `nn.Softmax`, `torch.softmax`, functional softmax | `softmax` | +| `nn.Dropout`, functional dropout | `dropout` | +| `nn.Identity` | no operation | +| equal-shaped `+`, `-`, `*` | elementwise add, subtract, multiply | +| tensor and scalar-parameter `+`, `*` | `offset`, `scale` | +| vector `matmul`, `inner`, `dot` | `dot` | + +In-place operations, multiple inputs or outputs, unsupported ATen operators, +and incompatible shapes are rejected. A +bias-free `nn.Linear` receives a fixed zero bias because the autodiff fully +connected operator requires a bias view; that synthetic bias is not added to +the optimizer. + +## Running the converter + +From `dsppp`, explicitly select `dsppp/.venv`. The `--no-project` option is +important because otherwise uv finds the repository-level `pyproject.toml` +and may run with the repository-level `.venv` instead: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --input-shape 4 --factory create_model -o generated_iris.hpp +``` + +Install or inspect packages in that same environment by specifying its Python +interpreter as well: + +```text +uv pip install --python .venv\Scripts\python.exe torch numpy +uv pip list --python .venv\Scripts\python.exe +``` + +Alternatively, activate `dsppp/.venv` and pass `--active` to `uv run`. A plain +`uv run` is intentionally not shown because project discovery can select a +different environment even when `dsppp/.venv` exists. + +The model file must provide a zero-argument factory returning an `nn.Module`: + +```python +def create_model() -> torch.nn.Module: + return MyNetwork() +``` + +Conversion uses `torch.export.export` with one zero-valued batch-1 input. The +result is a functional ATen graph with model parameters lifted into explicit +graph inputs. The converter reads their values and trainable state through the +exported graph signature and state dictionary, then accepts only its documented +ATen allowlist. It does not train the module. Dropout is always emitted with +its probability, independently of the PyTorch module's current training state. +On the C++ side it is active while the autodiff tape records training and is an +identity when recording is disabled for inference. +Use `--dtype float16` to emit a `float16_t` network instead of the default +float32 network. `--arena-bytes` selects the generated autodiff arena capacity. + +The converter supports two explicit export modes. The default +`--export-mode trained` embeds the module's current parameters and preserves a +trained module exactly. For a network that will be trained with autodiff, use +`--export-mode empty`. Empty mode emits zero-initialized parameter storage and a compact C++ +`initialize_parameters()` loop: fully connected weights use uniform +random initialization and biases start at zero. + +## Using generated code for training + +The generated constructor creates network buffers and registers network +operators, but does not call `tape.begin_graph()` or `tape.rewind_graph()`. +The application can therefore create target and loss buffers on the same tape, +register a loss operator, add parameters to an optimizer, and then begin and +reuse the graph in the normal training loop. + +The input values are written to the public `input_value` array. `forward()` +evaluates the converted network, and `output()` returns its final +`BufferView`. `add_parameters(optimizer)` adds every trainable PyTorch +parameter while omitting frozen and synthetic parameters. + +## Iris comparison + +`scripts/iris_pytorch.py` implements the same `4 -> 8 -> 3` classifier, +normalization, per-sample Adam updates, 120/30 training/test split, and +120-epoch default used by `dsppp/Examples/autodiff_iris.cpp`. It reads the +exact dataset from `dsppp/Examples/iris_data.hpp`. + +Export an empty network without running the Python training loop with: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --export generated_iris.hpp +``` + +This export uses generated C++ random initialization rather than embedding the +random values assigned by the newly constructed PyTorch module. + +With no arguments, the script runs the 120-epoch PyTorch comparison without +exporting. To train and then export the trained parameters, request the number +of epochs explicitly: + +```text +uv run --no-project --python .venv/Scripts/python.exe \ + Include/dsppp/autodiff/scripts/iris_pytorch.py \ + --epochs 120 --export generated_trained_iris.hpp +``` diff --git a/dsppp/Include/dsppp/autodiff/doc/training_loop.md b/dsppp/Include/dsppp/autodiff/doc/training_loop.md new file mode 100644 index 000000000..a73d7c5b7 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/doc/training_loop.md @@ -0,0 +1,176 @@ +# Training loop + +The application owns the model, data, outputs, and loop. The tape records one +evaluation of the model, computes gradients, and then an optimizer changes the +parameter values. + +## Complete linear-regression step + +This example fits `prediction = coefficient * feature + bias` to one sample. +Every variable used by the loop is declared here. + +```cpp +#include +#include +#include +#include +#include + +using namespace arm_cmsis_dsp::autodiff; + +int main() +{ +Arena<1024, float> arena; +Tape &tape = arena.tape(); +tape.register_operator>(); +tape.register_operator>(); +tape.register_operator>(); + +float coefficient_value[] = {0.0F}; +float bias_value[] = {0.0F}; +float feature_value[] = {2.0F}; +float target_value[] = {5.0F}; +float product_value[] = {0.0F}; +float prediction_value[] = {0.0F}; +float loss_value[] = {0.0F}; + +BufferView coefficient = tape.parameter(coefficient_value); +BufferView bias = tape.parameter(bias_value); +BufferView feature = tape.input(feature_value); +BufferView target = tape.input(target_value); +BufferView product = tape.output(product_value); +BufferView prediction = tape.output(prediction_value); +BufferView loss = tape.output(loss_value); + +RMSProp<2, 2> optimizer(1.0e-3F); +optimizer.add(coefficient); +optimizer.add(bias); + +tape.begin_graph(); +for (std::size_t step = 0; step < 100U; ++step) +{ + if (!tape.rewind_graph()) + break; + + product = coefficient * feature; + prediction = product + bias; + loss = quadratic_error(prediction, target); + + optimizer.zero_grad(); + if (!tape.backward(loss) || !optimizer.step()) + break; +} +return tape.good() && optimizer.good() ? 0 : 1; +} +``` + +## The flow, in order + +### 1. Set up fixed state + +Register operator types once, create all persistent views, construct the +optimizer, and add each trainable parameter view. `RMSProp<2, 2>` reserves +state for two scalar values in at most two separately added views: the +one-element `coefficient` view and the one-element `bias` view. + +### 2. Mark persistent arena allocations + +`begin_graph()` records the arena position after the persistent gradient +buffers. It also starts a fresh record chain. Call it only after creating the +views that must survive between iterations. + +### 3. Rewind before an iteration + +`rewind_graph()` discards the previous iteration's operation records by moving +the arena position back to the mark. Parameter/output views and their gradient +arrays remain valid. It resets tape status, enables recording, and clears the +record-chain tail. + +### 4. Run and record the forward graph + +Each assignment computes caller-owned output values immediately and appends a +small record containing the pointers needed by its derivative rule. The graph +must be reevaluated every iteration because parameter values change after +`step()`. + +Here the forward values are: + +```text +product[0] = coefficient[0] * feature[0] +prediction[0] = product[0] + bias[0] +loss[0] = (prediction[0] - target[0])^2 +``` + +### 5. Clear optimizer-managed parameter gradients + +`optimizer.zero_grad()` clears every parameter gradient registered with the +optimizer. `Tape::backward()` also resets gradients referenced by the recorded +graph before installing its seed, so for a fixed, successful graph this call +is usually redundant. It is nevertheless useful explicit training-loop +hygiene: it also clears registered parameters omitted by a conditional graph, +and it prevents stale gradients from reaching `step()` if graph structure is +changed. Call it before `backward()`, not after, because after backward those +buffers contain the gradients that `step()` must consume. + +### 6. Run reverse propagation + +`backward(loss)` uses the default scalar seed `d(loss)/d(loss) = 1`. It first +resets graph gradient buffers, seeds the loss gradient, and visits operation +records in reverse creation order. Contributions use `+=`, so a parameter used +by several nodes receives their sum. + +For this graph: + +```text +d_loss/d_prediction = 2 * (prediction - target) +d_loss/d_coefficient = d_loss/d_prediction * feature +d_loss/d_bias = d_loss/d_prediction +``` + +Inputs have no gradient buffers, so derivatives for `feature` and `target` are +not retained. + +### 7. Update parameters + +`optimizer.step()` reads the gradients, updates its fixed-size state, and then +changes `coefficient_value` and `bias_value` in place. The next iteration's +forward pass therefore uses the new parameter values. + +```mermaid +flowchart TD + A["Register operators and create persistent views"] --> B["Add parameter views to optimizer"] + B --> C["begin_graph(): mark persistent arena state"] + C --> D["rewind_graph(): discard old records"] + D --> E["Forward assignments: compute values and append records"] + E --> F["optimizer.zero_grad(): clear registered parameter gradients"] + F --> G["tape.backward(): reset, seed, reverse traversal"] + G --> H["optimizer.step(): update state and parameter values"] + H --> D +``` + +## Batch accumulation + +To update once per batch, record every sample's prediction before constructing +one loss over the complete prediction and target buffers. Scalar per-sample +output views can share slices of persistent vector gradient buffers. The +regression example in `dsppp/Examples/autodiff_regression.cpp` demonstrates +this pattern with named `coefficients`, `bias`, `polynomial`, `prediction`, +`target`, and `loss` views. + +## Inference + +Inference still runs the operator forward kernels but does not need records: + +```cpp +{ + RecordingScope inference(tape, false); + product = coefficient * feature; + prediction = product + bias; + const float result = prediction_value[0]; + (void)result; +} +``` + +The previous recording state is restored at the closing brace. See +[Concepts and memory model](concepts.md#recording-and-recordingscope) for the +name and lifetime behavior. diff --git a/dsppp/Include/dsppp/autodiff/operators/add.hpp b/dsppp/Include/dsppp/autodiff/operators/add.hpp new file mode 100644 index 000000000..e07bc835a --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/add.hpp @@ -0,0 +1,140 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class AddOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + T *left_gradient; + T *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.left_gradient != nullptr) + fill(record.left_gradient, record.length); + if (record.right_gradient != nullptr) + fill(record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.left_gradient != nullptr) + add(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); + if (record.right_gradient != nullptr) + add(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif + add(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class AddExpression +{ +public: + AddExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + AddOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +template +inline AddExpression operator+(const BufferView &left, + const BufferView &right) noexcept +{ + return AddExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp new file mode 100644 index 000000000..db9551691 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/cross_entropy.hpp @@ -0,0 +1,230 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Categorical cross entropy: -sum(target[i] * log(probability[i])). */ +template class CrossEntropyOperator +{ + static constexpr T maximum_value() noexcept + { + if constexpr (std::is_same::value) + return std::numeric_limits::max(); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return F16_MAX; +#else + else + return T{}; +#endif + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void clip(const T *input, T *output, T low, T high, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_clip_f32(input, output, low, high, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_clip_f16(input, output, low, high, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + output[i] = input[i] < low ? low : (input[i] > high ? high : input[i]); +#endif + } + + static void vlog(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_vlog_f32(data, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_vlog_f16(data, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + data[i] = static_cast(std::log(static_cast(data[i]))); +#endif + } + + static void dot(const T *left, const T *right, std::size_t length, + T *result) noexcept + { + if constexpr (std::is_same::value) + arm_dot_prod_f32(left, right, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_dot_prod_f16(left, right, static_cast(length), result); +#else + else + { + *result = T{}; + for (std::size_t i = 0; i < length; ++i) *result += left[i] * right[i]; + } +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *probability_value; + T *probability_gradient; + const T *target_value; + std::size_t length; + }; + + static constexpr T minimum_probability() noexcept + { + // 1e-7 is useful for float32, but its reciprocal is too large for a + // finite float16 gradient. Keep the half-precision derivative inside + // its representable range before the softmax reverse rule consumes it. + if constexpr (std::is_same::value) + return T{1.0e-7F}; + return T{1.0e-4F}; + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = T{}; + fill(record.probability_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const T seed = record.output_gradient[0]; + if (static_cast(seed) == 0.0F) return; + clip(record.probability_value, record.probability_gradient, + minimum_probability(), maximum_value(), record.length); + for (std::size_t i = 0; i < record.length; ++i) + { + const float denominator = static_cast(record.probability_gradient[i]); + float gradient = -static_cast(seed) * + static_cast(record.target_value[i]) / + denominator; + if constexpr (!std::is_same::value) + { + const float limit = static_cast(maximum_value()); + if (gradient > limit) gradient = limit; + if (gradient < -limit) gradient = -limit; + } + record.probability_gradient[i] = static_cast(gradient); + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &probability, + const BufferView &target) noexcept + { + if (!OperatorAccess::valid(tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(tape, probability, target) || + OperatorAccess::gradients(probability) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &probability, + const BufferView &target) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, probability, target)) + return false; +#endif + + const std::size_t length = OperatorAccess::length(probability); + T result = T{}; + if (length != 0U) + { + // Reuse the probability gradient as forward scratch. backward() + // resets it before accumulating any gradient. + T *scratch = OperatorAccess::gradients(probability); + clip(OperatorAccess::values(probability), scratch, minimum_probability(), + maximum_value(), length); + vlog(scratch, length); + dot(OperatorAccess::values(target), scratch, length, &result); + } + OperatorAccess::values(output)[0] = static_cast(-static_cast(result)); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->probability_value = OperatorAccess::values(probability); + record->probability_gradient = OperatorAccess::gradients(probability); + record->target_value = OperatorAccess::values(target); + record->length = length; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class CrossEntropyExpression +{ +public: + CrossEntropyExpression(const BufferView &probability, + const BufferView &target) noexcept + : probability_(probability), target_(target) {} + + void evaluate(BufferView &output) const noexcept + { + CrossEntropyOperator::evaluate(output, probability_, target_); + } + +private: + BufferView probability_; + BufferView target_; +}; + +template +inline CrossEntropyExpression cross_entropy( + const BufferView &probability, const BufferView &target) noexcept +{ + return CrossEntropyExpression(probability, target); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp new file mode 100644 index 000000000..dc7b899f2 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dequantize.hpp @@ -0,0 +1,253 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Dequantize floating-point int8 codes back to the tape's floating type. */ +template class DequantizeOperator +{ + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + const T *zero_point_value; + T *scale_gradient; + T *zero_point_gradient; + std::size_t length; + Int8Quantization quantization; + }; + + static float rounded(float value) noexcept { return std::nearbyint(value); } + + static void add(T &destination, float contribution) noexcept + { + destination = static_cast(static_cast(destination) + + contribution); + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(0.0F, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_fill_f16(static_cast(0.0F), data, + static_cast(length)); +#endif + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; + } + + static T sum(const T *data, std::size_t length) noexcept + { + T result = T{}; + if constexpr (std::is_same::value) + arm_accumulate_f32(data, static_cast(length), &result); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_accumulate_f16(data, static_cast(length), &result); +#endif + else + for (std::size_t i = 0; i < length; ++i) result += data[i]; + return result; + } + + static std::size_t block_length(const Int8Quantization &quantization, + std::size_t remaining) noexcept + { + if (quantization.parameter_count() == 1U) + return remaining; + return remaining < quantization.inner_size() + ? remaining + : quantization.inner_size(); + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + fill(record.scale_gradient, record.quantization.parameter_count()); + if (record.zero_point_gradient != nullptr) + fill(record.zero_point_gradient, + record.quantization.parameter_count()); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t offset = 0; offset < record.length;) + { + const std::size_t p = record.quantization.parameter_index(offset); + const std::size_t length = block_length(record.quantization, + record.length - offset); + const T scale = record.scale_value[p]; + const T zero = record.quantization.asymmetric() + ? static_cast(rounded(static_cast( + record.zero_point_value[p]))) + : T{}; + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + offset, 0, length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value) + offset, 0, length); + if (record.input_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient + offset, 0, length); + input_gradient += output_gradient * scale; + } + add(record.scale_gradient[p], static_cast( + ::arm_cmsis_dsp::dot(output_gradient, input_value - zero))); + if (record.zero_point_gradient != nullptr && + record.quantization.asymmetric()) + add(record.zero_point_gradient[p], + -static_cast(scale) * + static_cast(sum(record.output_gradient + offset, + length))); + offset += length; + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + const Int8Quantization &quantization) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || + !OperatorAccess::valid(tape, zero_point) || + OperatorAccess::length(scale) != quantization.parameter_count() || + OperatorAccess::length(zero_point) != quantization.parameter_count() || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + !quantization.valid_for(OperatorAccess::length(input)) || + OperatorAccess::values(output) == OperatorAccess::values(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + if (quantization.asymmetric() && + (OperatorAccess::role(zero_point) != BufferRole::parameter || + OperatorAccess::gradients(zero_point) == nullptr)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; + if (!quantization.valid_for(OperatorAccess::length(input))) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + quantization.constrain_parameters( + const_cast(OperatorAccess::values(scale)), + const_cast(OperatorAccess::values(zero_point))); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale, zero_point, quantization)) + return false; +#endif + const T *input_value = OperatorAccess::values(input); + T *output_value = OperatorAccess::values(output); + const std::size_t output_length = OperatorAccess::length(output); + for (std::size_t offset = 0; offset < output_length;) + { + const std::size_t p = quantization.parameter_index(offset); + const std::size_t length = block_length( + quantization, output_length - offset); + const T zero = quantization.asymmetric() + ? static_cast(rounded(static_cast( + OperatorAccess::values(zero_point)[p]))) + : T{}; + ::arm_cmsis_dsp::VectorView input_block( + const_cast(input_value) + offset, 0, length); + ::arm_cmsis_dsp::VectorView output_block( + output_value + offset, 0, length); + output_block = (input_block - zero) * + OperatorAccess::values(scale)[p]; + offset += length; + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append( + *tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->zero_point_value = OperatorAccess::values(zero_point); + record->scale_gradient = OperatorAccess::gradients(scale); + record->zero_point_gradient = quantization.asymmetric() + ? OperatorAccess::gradients(zero_point) + : nullptr; + record->length = OperatorAccess::length(output); + record->quantization = quantization; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class DequantizeExpression +{ +public: + DequantizeExpression(const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + : input_(input), scale_(scale), zero_point_(zero_point), + quantization_(quantization) {} + void evaluate(BufferView &output) const noexcept + { + DequantizeOperator::evaluate(output, input_, scale_, zero_point_, + quantization_); + } +private: + BufferView input_; + BufferView scale_; + BufferView zero_point_; + Int8Quantization quantization_; +}; + +template +inline DequantizeExpression dequantize( + const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization = Int8Quantization::activation()) noexcept +{ + return DequantizeExpression(input, scale, zero_point, quantization); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/dot.hpp b/dsppp/Include/dsppp/autodiff/operators/dot.hpp new file mode 100644 index 000000000..02535aada --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dot.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class DotOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void dot(const T *left, const T *right, std::size_t length, + T *result) noexcept + { + if constexpr (std::is_same::value) + arm_dot_prod_f32(left, right, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_dot_prod_f16(left, right, static_cast(length), result); +#else + else + { + *result = T{}; + for (std::size_t i = 0; i < length; ++i) + *result += left[i] * right[i]; + } +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *left_value; + T *left_gradient; + const T *right_value; + T *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = T{}; + if (record.left_gradient != nullptr) + fill(record.left_gradient, record.length); + if (record.right_gradient != nullptr) + fill(record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const T gradient = record.output_gradient[0]; + if (static_cast(gradient) == 0.0F) return; + + if (record.left_gradient != nullptr) + { + VectorView left_grad(record.left_gradient, 0, record.length); + VectorView right_val(const_cast(record.right_value), 0, record.length); + left_grad += right_val * gradient; + } + if (record.right_gradient != nullptr) + { + VectorView right_grad(record.right_gradient, 0, record.length); + VectorView left_val(const_cast(record.left_value), 0, record.length); + right_grad += left_val * gradient; + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::valid(tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(tape, left, right) || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif + T value = T{}; + dot(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::length(left), &value); + OperatorAccess::values(output)[0] = value; + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(left); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class DotExpression +{ +public: + DotExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + DotOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +template +inline DotExpression dot(const BufferView &left, const BufferView &right) noexcept +{ + return DotExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/dropout.hpp b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp new file mode 100644 index 000000000..fc36a4d16 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/dropout.hpp @@ -0,0 +1,238 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Caller-owned deterministic random state used by dropout. */ +class DropoutGenerator +{ +public: + explicit DropoutGenerator(std::uint32_t seed = 0x6D2B79F5U) noexcept + : state_(seed == 0U ? 0x6D2B79F5U : seed) {} + + void seed(std::uint32_t value) noexcept + { + state_ = value == 0U ? 0x6D2B79F5U : value; + } + +private: + std::uint32_t state_; + template friend class DropoutOperator; +}; + +/** Inverted dropout during recording, identity when recording is disabled. */ +template class DropoutOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void copy(const T *input, T *output, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_copy_f32(input, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_copy_f16(input, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i]; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + T *input_gradient; + std::size_t length; + std::uint32_t random_state; + float drop_probability; + T scale; + }; + + static std::uint32_t next(std::uint32_t &state) noexcept + { + state ^= state << 13U; + state ^= state >> 17U; + state ^= state << 5U; + return state; + } + + static bool keep(std::uint32_t &state, float drop_probability) noexcept + { + constexpr float inverse_24_bit_range = 1.0F / 16777216.0F; + const float uniform = + static_cast(next(state) >> 8U) * inverse_24_bit_range; + return uniform >= drop_probability; + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + + if (record.drop_probability == 0.0F) + { + add(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); + return; + } + + std::uint32_t state = record.random_state; + for (std::size_t i = 0; i < record.length; ++i) + { + if (keep(state, record.drop_probability)) + { + if constexpr (std::is_same::value) + record.input_gradient[i] += record.output_gradient[i] * record.scale; + else + record.input_gradient[i] = static_cast( + static_cast(record.input_gradient[i]) + + static_cast(record.output_gradient[i]) * + static_cast(record.scale)); + } + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + float drop_probability) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + !(drop_probability >= 0.0F && drop_probability < 1.0F)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + DropoutGenerator &generator, + float drop_probability) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, drop_probability)) + return false; +#endif + + const std::size_t length = OperatorAccess::length(output); + if (!OperatorAccess::recording(*tape)) + { + copy(OperatorAccess::values(input), OperatorAccess::values(output), length); + return OperatorAccess::status(*tape) == Status::ok; + } + if (length == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + const float scale = 1.0F / (1.0F - drop_probability); + const std::uint32_t initial_state = generator.state_; + if (drop_probability == 0.0F) + { + copy(OperatorAccess::values(input), OperatorAccess::values(output), length); + } + else + { + for (std::size_t i = 0; i < length; ++i) + OperatorAccess::values(output)[i] = + keep(generator.state_, drop_probability) + ? OperatorAccess::values(input)[i] * static_cast(scale) + : T{}; + } + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) + { + generator.state_ = initial_state; + return false; + } + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); + record->length = length; + record->random_state = initial_state; + record->drop_probability = drop_probability; + record->scale = static_cast(scale); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class DropoutExpression +{ +public: + DropoutExpression(const BufferView &input, DropoutGenerator &generator, + float drop_probability) noexcept + : input_(input), generator_(generator), + drop_probability_(drop_probability) {} + + void evaluate(BufferView &output) const noexcept + { + DropoutOperator::evaluate(output, input_, generator_, + drop_probability_); + } + +private: + BufferView input_; + DropoutGenerator &generator_; + float drop_probability_; +}; + +template +inline DropoutExpression dropout(const BufferView &input, + DropoutGenerator &generator, + float drop_probability) noexcept +{ + return DropoutExpression(input, generator, drop_probability); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp new file mode 100644 index 000000000..feeb43282 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/fully_connected.hpp @@ -0,0 +1,229 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class FullyConnectedOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *weight_value; + T *weight_gradient; + T *bias_gradient; + std::size_t rows; + std::size_t columns; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.rows); + fill(record.bias_gradient, record.rows); + fill(record.weight_gradient, record.rows * record.columns); + if (record.input_gradient != nullptr && record.columns != 0U) + fill(record.input_gradient, record.columns); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.rows == 0U) return; + + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.rows); + ::arm_cmsis_dsp::VectorView bias_gradient( + record.bias_gradient, 0, record.rows); + + // Bias is shared by every input sample, so its gradient must be + // accumulated instead of overwritten. + bias_gradient += output_gradient; + + if (record.columns != 0U) + { + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value), 0, record.columns); + ::arm_cmsis_dsp::MatrixView + weight_gradient(record.weight_gradient, record.rows, + record.columns, record.columns); + + // dW = dy outer x. The C++ expression engine fuses the multiply + // and accumulation without constructing an outer-product buffer. + weight_gradient += + ::arm_cmsis_dsp::outer(output_gradient, input_value); + + if (record.input_gradient == nullptr) return; + + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.columns); + ::arm_cmsis_dsp::MatrixView + weight_value(const_cast(record.weight_value), + record.rows, record.columns, record.columns); + input_gradient += ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(weight_value), + output_gradient); + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const MatrixView &weights, + const BufferView &bias) noexcept + { + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + if (!OperatorAccess::valid(tape, output) || + !OperatorAccess::valid(tape, input) || + !OperatorAccess::valid(tape, weight_buffer) || + !OperatorAccess::valid(tape, bias) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::length(output) != OperatorAccess::rows(weights) || + OperatorAccess::length(input) != OperatorAccess::columns(weights) || + OperatorAccess::length(bias) != OperatorAccess::rows(weights) || + OperatorAccess::rows(weights) > + std::numeric_limits::max() || + OperatorAccess::columns(weights) > + std::numeric_limits::max() || + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::role(bias) != BufferRole::parameter) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights, + const BufferView &bias) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, weights, bias)) + return false; +#endif + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t columns = OperatorAccess::columns(weights); + if (rows != 0U) + { + ::arm_cmsis_dsp::VectorView output_value( + OperatorAccess::values(output), 0, rows); + ::arm_cmsis_dsp::VectorView bias_value( + const_cast(OperatorAccess::values(bias)), 0, rows); + + if (columns != 0U) + { + if constexpr (std::is_same::value) + { + arm_matrix_instance_f32 weight_matrix; + arm_mat_init_f32( + &weight_matrix, static_cast(rows), + static_cast(columns), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_vec_mult_f32(&weight_matrix, + OperatorAccess::values(input), + OperatorAccess::values(output)); + } + else + { + arm_matrix_instance_f16 weight_matrix; + arm_mat_init_f16( + &weight_matrix, static_cast(rows), + static_cast(columns), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_vec_mult_f16(&weight_matrix, + OperatorAccess::values(input), + OperatorAccess::values(output)); + } + output_value += bias_value; + } + else + { + // VectorView deliberately deletes copy assignment. This + // dimension-zero edge has no matrix product to optimize. + for (std::size_t row = 0; row < rows; ++row) + OperatorAccess::values(output)[row] = + OperatorAccess::values(bias)[row]; + } + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->weight_value = OperatorAccess::values(weight_buffer); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->bias_gradient = OperatorAccess::gradients(bias); + record->rows = OperatorAccess::rows(weights); + record->columns = OperatorAccess::columns(weights); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class FullyConnectedExpression +{ +public: + FullyConnectedExpression(const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept + : input_(input), weights_(weights), bias_(bias) {} + void evaluate(BufferView &output) const noexcept + { + FullyConnectedOperator::evaluate(output, input_, weights_, bias_); + } +private: + BufferView input_; + MatrixView weights_; + BufferView bias_; +}; + +template +inline FullyConnectedExpression fully_connected( + const BufferView &input, const MatrixView &weights, + const BufferView &bias) noexcept +{ + return FullyConnectedExpression(input, weights, bias); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp new file mode 100644 index 000000000..ef2f2ca9b --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/matrix_multiply.hpp @@ -0,0 +1,213 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Matrix product Y = W X, differentiating only the parameter matrix W. */ +template class MatrixMultiplyOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *weight_gradient; + std::size_t rows; + std::size_t inner; + std::size_t columns; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.rows * record.columns); + fill(record.weight_gradient, record.rows * record.inner); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + + ::arm_cmsis_dsp::MatrixView + input_value(const_cast(record.input_value), record.inner, + record.columns, record.columns); + + // dW = dY X^T. For each row of dY, this is X times that row. The lazy + // matvec expression fuses the product with gradient accumulation. + + for (std::size_t row = 0; row < record.rows; ++row) + { + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + row * record.columns, 0, + record.columns); + ::arm_cmsis_dsp::VectorView weight_gradient( + record.weight_gradient + row * record.inner, 0, + record.inner); + weight_gradient += + ::arm_cmsis_dsp::matvec(input_value, output_gradient); + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const MatrixView &weights) noexcept + { + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t inner = OperatorAccess::columns(weights); + const std::size_t input_length = OperatorAccess::length(input); + const std::size_t columns = inner == 0U ? 0U : input_length / inner; + if (!OperatorAccess::valid(tape, output) || + !OperatorAccess::valid(tape, input) || + !OperatorAccess::valid(tape, weight_buffer) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::role(input) != BufferRole::input || + OperatorAccess::role(weight_buffer) != BufferRole::parameter || + OperatorAccess::gradients(weight_buffer) == nullptr || + rows == 0U || inner == 0U || columns == 0U || + input_length % inner != 0U || + rows > std::numeric_limits::max() || + inner > std::numeric_limits::max() || + columns > std::numeric_limits::max() || + OperatorAccess::length(output) != rows * columns) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const MatrixView &weights) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, weights)) + return false; +#endif + + const BufferView &weight_buffer = OperatorAccess::buffer(weights); + const std::size_t rows = OperatorAccess::rows(weights); + const std::size_t inner = OperatorAccess::columns(weights); + const std::size_t input_length = OperatorAccess::length(input); + const std::size_t columns = inner == 0U ? 0U : input_length / inner; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + arm_status matrix_status; +#endif + if constexpr (std::is_same::value) + { + arm_matrix_instance_f32 weight_matrix; + arm_matrix_instance_f32 input_matrix; + arm_matrix_instance_f32 output_matrix; + arm_mat_init_f32(&weight_matrix, static_cast(rows), + static_cast(inner), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_init_f32(&input_matrix, static_cast(inner), + static_cast(columns), + const_cast(OperatorAccess::values(input))); + arm_mat_init_f32(&output_matrix, static_cast(rows), + static_cast(columns), + OperatorAccess::values(output)); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + matrix_status = +#endif + arm_mat_mult_f32(&weight_matrix, &input_matrix, &output_matrix); + } + else + { + arm_matrix_instance_f16 weight_matrix; + arm_matrix_instance_f16 input_matrix; + arm_matrix_instance_f16 output_matrix; + arm_mat_init_f16(&weight_matrix, static_cast(rows), + static_cast(inner), + const_cast(OperatorAccess::values(weight_buffer))); + arm_mat_init_f16(&input_matrix, static_cast(inner), + static_cast(columns), + const_cast(OperatorAccess::values(input))); + arm_mat_init_f16(&output_matrix, static_cast(rows), + static_cast(columns), + OperatorAccess::values(output)); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + matrix_status = +#endif + arm_mat_mult_f16(&weight_matrix, &input_matrix, &output_matrix); + } +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (matrix_status != ARM_MATH_SUCCESS) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } +#endif + + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->weight_gradient = OperatorAccess::gradients(weight_buffer); + record->rows = rows; + record->inner = inner; + record->columns = columns; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class MatrixMultiplyExpression +{ +public: + MatrixMultiplyExpression(const BufferView &input, + const MatrixView &weights) noexcept + : input_(input), weights_(weights) {} + + void evaluate(BufferView &output) const noexcept + { + MatrixMultiplyOperator::evaluate(output, input_, weights_); + } + +private: + BufferView input_; + MatrixView weights_; +}; + +template +inline MatrixMultiplyExpression matrix_multiply( + const BufferView &input, const MatrixView &weights) noexcept +{ + return MatrixMultiplyExpression(input, weights); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/multiply.hpp b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp new file mode 100644 index 000000000..70947c76b --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/multiply.hpp @@ -0,0 +1,159 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class MultiplyOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void multiply(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_mult_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_mult_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] * right[i]; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *left_value; + T *left_gradient; + const T *right_value; + T *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.left_gradient != nullptr) + fill(record.left_gradient, record.length); + if (record.right_gradient != nullptr) + fill(record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + if (record.left_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView left_gradient( + record.left_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView right_value( + const_cast(record.right_value), 0, record.length); + left_gradient += output_gradient * right_value; + } + if (record.right_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView right_gradient( + record.right_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView left_value( + const_cast(record.left_value), 0, record.length); + right_gradient += output_gradient * left_value; + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif + multiply(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_value = OperatorAccess::values(left); + record->left_gradient = OperatorAccess::gradients(left); + record->right_value = OperatorAccess::values(right); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class MultiplyExpression +{ +public: + MultiplyExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + MultiplyOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +template +inline MultiplyExpression operator*(const BufferView &left, + const BufferView &right) noexcept +{ + return MultiplyExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/offset.hpp b/dsppp/Include/dsppp/autodiff/operators/offset.hpp new file mode 100644 index 000000000..e13eff0b9 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/offset.hpp @@ -0,0 +1,177 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class OffsetOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void apply_offset(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_offset_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_offset_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] + value; +#endif + } + + static void accumulate(const T *input, std::size_t length, T *result) noexcept + { + if constexpr (std::is_same::value) + arm_accumulate_f32(input, static_cast(length), result); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_accumulate_f16(input, static_cast(length), result); +#else + else + for (std::size_t i = 0; i < length; ++i) *result += input[i]; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + T *input_gradient; + T *offset_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + record.offset_gradient[0] = T{}; + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient != nullptr) + add(record.input_gradient, record.output_gradient, + record.input_gradient, record.length); + T gradient_sum = T{}; + accumulate(record.output_gradient, record.length, &gradient_sum); + if constexpr (std::is_same::value) + record.offset_gradient[0] += gradient_sum; + else + record.offset_gradient[0] = static_cast( + static_cast(record.offset_gradient[0]) + + static_cast(gradient_sum)); + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &offset) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, offset) || + OperatorAccess::length(offset) != 1U || + OperatorAccess::role(offset) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(offset) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(offset) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(offset)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &offset) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, offset)) + return false; +#endif + apply_offset(OperatorAccess::values(input), OperatorAccess::values(offset)[0], + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_gradient = OperatorAccess::gradients(input); + record->offset_gradient = OperatorAccess::gradients(offset); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class OffsetExpression +{ +public: + OffsetExpression(const BufferView &input, const BufferView &offset) noexcept + : input_(input), offset_(offset) {} + void evaluate(BufferView &output) const noexcept + { + OffsetOperator::evaluate(output, input_, offset_); + } +private: + BufferView input_; + BufferView offset_; +}; + +template +inline OffsetExpression offset(const BufferView &input, + const BufferView &constant) noexcept +{ + return OffsetExpression(input, constant); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp new file mode 100644 index 000000000..4c1feb260 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quadratic_error.hpp @@ -0,0 +1,139 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Sum-of-squared-errors loss: sum((prediction - target)^2). */ +template class QuadraticErrorOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *prediction_value; + T *prediction_gradient; + const T *target_value; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + record.output_gradient[0] = T{}; + fill(record.prediction_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + const T seed = record.output_gradient[0]; + if (static_cast(seed) == 0.0F) return; + ::arm_cmsis_dsp::VectorView prediction_gradient( + record.prediction_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(record.prediction_value), 0, record.length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(record.target_value), 0, record.length); + prediction_gradient += + (prediction_value - target_value) * static_cast(2.0F * static_cast(seed)); + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &prediction, + const BufferView &target) noexcept + { + if (!OperatorAccess::valid(tape, output) || + OperatorAccess::length(output) != 1U || + OperatorAccess::gradients(output) == nullptr || + !OperatorAccess::compatible(tape, prediction, target) || + OperatorAccess::gradients(prediction) == nullptr || + OperatorAccess::role(target) != BufferRole::input) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &prediction, + const BufferView &target) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, prediction, target)) + return false; +#endif + + const std::size_t length = OperatorAccess::length(prediction); + ::arm_cmsis_dsp::VectorView prediction_value( + const_cast(OperatorAccess::values(prediction)), 0, + length); + ::arm_cmsis_dsp::VectorView target_value( + const_cast(OperatorAccess::values(target)), 0, length); + const auto error = prediction_value - target_value; + OperatorAccess::values(output)[0] = + ::arm_cmsis_dsp::dot(error, error); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->prediction_value = OperatorAccess::values(prediction); + record->prediction_gradient = OperatorAccess::gradients(prediction); + record->target_value = OperatorAccess::values(target); + record->length = OperatorAccess::length(prediction); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class QuadraticErrorExpression +{ +public: + QuadraticErrorExpression(const BufferView &prediction, + const BufferView &target) noexcept + : prediction_(prediction), target_(target) {} + void evaluate(BufferView &output) const noexcept + { + QuadraticErrorOperator::evaluate(output, prediction_, target_); + } +private: + BufferView prediction_; + BufferView target_; +}; + +template +inline QuadraticErrorExpression quadratic_error( + const BufferView &prediction, const BufferView &target) noexcept +{ + return QuadraticErrorExpression(prediction, target); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp b/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp new file mode 100644 index 000000000..3fbbd2412 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quantization_support.hpp @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** LiteRT/CMSIS-NN/Ethos-U signed int8 quantization layout. */ +class Int8Quantization +{ +public: + constexpr Int8Quantization() noexcept + : quant_min_(-128), quant_max_(127), parameter_count_(1U), + inner_size_(1U), asymmetric_(true) + { + } + + /** Per-tensor asymmetric activation quantization. */ + static constexpr Int8Quantization activation() noexcept + { + return Int8Quantization(-128, 127, 1U, 1U, true); + } + + /** + * Symmetric per-axis weight quantization. + * + * axis_size is the number of scales and inner_size is the product of the + * dimensions following the quantized axis in a row-major tensor. Thus a + * fully-connected [output, input] matrix uses (output, input), Conv2D + * OHWI uses (output, H*W*I), and depthwise Conv2D uses (channels, 1). + */ + static constexpr Int8Quantization weights(std::size_t axis_size, + std::size_t inner_size) noexcept + { + return Int8Quantization(-127, 127, axis_size, inner_size, false); + } + + constexpr int quant_min() const noexcept { return quant_min_; } + constexpr int quant_max() const noexcept { return quant_max_; } + constexpr std::size_t parameter_count() const noexcept + { + return parameter_count_; + } + constexpr std::size_t inner_size() const noexcept { return inner_size_; } + constexpr bool asymmetric() const noexcept { return asymmetric_; } + + constexpr std::size_t parameter_index(std::size_t element) const noexcept + { + return parameter_count_ == 1U + ? 0U + : (element / inner_size_) % parameter_count_; + } + + constexpr bool valid_for(std::size_t length) const noexcept + { + return parameter_count_ != 0U && inner_size_ != 0U && + (parameter_count_ == 1U || + ((length % parameter_count_) == 0U && + (length / parameter_count_) % inner_size_ == 0U)); + } + + /** Keep learned scales in the numerically safe, strictly-positive range. */ + template + void constrain_scale(T &scale) const noexcept + { + if (static_cast(scale) < minimum_scale()) + scale = static_cast(minimum_scale()); + } + + template + void constrain_scales(T (&scales)[Length]) const noexcept + { + for (std::size_t i = 0; i < Length; ++i) + constrain_scale(scales[i]); + } + + /** + * Keep learned zero-points in this scheme's integer domain. Symmetric + * weight quantization always forces them to zero. + */ + template + void constrain_zero_point(T &zero_point) const noexcept + { + if (!asymmetric_) + { + zero_point = T{}; + return; + } + if (static_cast(zero_point) < static_cast(quant_min_)) + zero_point = static_cast(quant_min_); + if (static_cast(zero_point) > static_cast(quant_max_)) + zero_point = static_cast(quant_max_); + } + + template + void constrain_zero_points(T (&zero_points)[Length]) const noexcept + { + for (std::size_t i = 0; i < Length; ++i) + constrain_zero_point(zero_points[i]); + } + + /** Project all learned parameters into this backend quantization scheme. */ + template + void constrain_parameters(T *scales, T *zero_points) const noexcept + { + if constexpr (std::is_same::value) + { + arm_clip_f32(scales, scales, minimum_scale(), + std::numeric_limits::max(), + static_cast(parameter_count_)); + if (asymmetric_) + arm_clip_f32(zero_points, zero_points, + static_cast(quant_min_), + static_cast(quant_max_), + static_cast(parameter_count_)); + else + arm_fill_f32(0.0F, zero_points, + static_cast(parameter_count_)); + } +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + { + arm_clip_f16(scales, scales, static_cast(minimum_scale()), + F16_MAX, static_cast(parameter_count_)); + if (asymmetric_) + arm_clip_f16(zero_points, zero_points, + static_cast(quant_min_), + static_cast(quant_max_), + static_cast(parameter_count_)); + else + arm_fill_f16(static_cast(0.0F), zero_points, + static_cast(parameter_count_)); + } +#endif + else + for (std::size_t i = 0; i < parameter_count_; ++i) + { + constrain_scale(scales[i]); + constrain_zero_point(zero_points[i]); + } + } + +private: + // The backend only requires scale > 0. This small floor additionally + // prevents division by zero after a finite-precision optimizer update. + static constexpr float minimum_scale() noexcept { return 1.0e-4F; } + + constexpr Int8Quantization(int quant_min, int quant_max, + std::size_t parameter_count, + std::size_t inner_size, + bool asymmetric) noexcept + : quant_min_(quant_min), quant_max_(quant_max), + parameter_count_(parameter_count), inner_size_(inner_size), + asymmetric_(asymmetric) + { + } + + int quant_min_; + int quant_max_; + std::size_t parameter_count_; + std::size_t inner_size_; + bool asymmetric_; +}; + +/** CMSIS-NN APIs use offset = -zero_point. */ +inline int32_t cmsis_nn_offset(float zero_point) noexcept +{ + return -static_cast(std::nearbyint(zero_point)); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/quantize.hpp b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp new file mode 100644 index 000000000..98d89aa44 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/quantize.hpp @@ -0,0 +1,269 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Quantize to signed int8 codes represented in the tape's floating type. */ +template class QuantizeOperator +{ + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + const T *zero_point_value; + T *scale_gradient; + T *zero_point_gradient; + std::size_t length; + Int8Quantization quantization; + }; + + static float rounded(float value) noexcept { return std::nearbyint(value); } + + static void add(T &destination, float contribution) noexcept + { + destination = static_cast(static_cast(destination) + + contribution); + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(0.0F, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else if constexpr (std::is_same::value) + arm_fill_f16(static_cast(0.0F), data, + static_cast(length)); +#endif + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; + } + + static std::size_t block_length(const Int8Quantization &quantization, + std::size_t remaining) noexcept + { + if (quantization.parameter_count() == 1U) + return remaining; + return remaining < quantization.inner_size() + ? remaining + : quantization.inner_size(); + } + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + fill(record.scale_gradient, record.quantization.parameter_count()); + if (record.zero_point_gradient != nullptr) + fill(record.zero_point_gradient, + record.quantization.parameter_count()); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + for (std::size_t offset = 0; offset < record.length;) + { + const std::size_t p = record.quantization.parameter_index(offset); + const std::size_t length = block_length(record.quantization, + record.length - offset); + const float scale = static_cast(record.scale_value[p]); + const float inverse_scale = 1.0F / scale; + const float inverse_scale_squared = inverse_scale * inverse_scale; + const float zero_point = record.quantization.asymmetric() + ? rounded(static_cast( + record.zero_point_value[p])) + : 0.0F; + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient + offset, 0, length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value) + offset, 0, length); + const auto selected = ::arm_cmsis_dsp::nearest_even_range_mask( + input_value, inverse_scale, + static_cast(zero_point), + static_cast(record.quantization.quant_min()), + static_cast(record.quantization.quant_max())); + if (record.input_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient + offset, 0, length); + ::arm_cmsis_dsp::masked_scale_add( + input_gradient, output_gradient, selected, + static_cast(inverse_scale)); + } + const auto reductions = ::arm_cmsis_dsp::masked_dot_sum( + output_gradient, input_value, selected); + add(record.scale_gradient[p], + -static_cast(reductions.dot) * + inverse_scale_squared); + if (record.zero_point_gradient != nullptr && + record.quantization.asymmetric()) + add(record.zero_point_gradient[p], + static_cast(reductions.sum)); + offset += length; + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + const Int8Quantization &quantization) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || + !OperatorAccess::valid(tape, zero_point) || + OperatorAccess::length(scale) != quantization.parameter_count() || + OperatorAccess::length(zero_point) != quantization.parameter_count() || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + !quantization.valid_for(OperatorAccess::length(input)) || + OperatorAccess::values(output) == OperatorAccess::values(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + if (quantization.asymmetric()) + { + if (OperatorAccess::role(zero_point) != BufferRole::parameter || + OperatorAccess::gradients(zero_point) == nullptr) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + } + else + { + for (std::size_t i = 0; i < quantization.parameter_count(); ++i) + if (rounded(static_cast(OperatorAccess::values(zero_point)[i])) != 0.0F) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || + !OperatorAccess::template require>(*tape)) + return false; + if (!quantization.valid_for(OperatorAccess::length(input))) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + quantization.constrain_parameters( + const_cast(OperatorAccess::values(scale)), + const_cast(OperatorAccess::values(zero_point))); +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale, zero_point, quantization)) + return false; +#endif + for (std::size_t p = 0; p < quantization.parameter_count(); ++p) + if (!(static_cast(OperatorAccess::values(scale)[p]) > 0.0F)) + { + OperatorAccess::fail(*tape, Status::tape_mismatch); + return false; + } + + const std::size_t output_length = OperatorAccess::length(output); + for (std::size_t offset = 0; offset < output_length;) + { + const std::size_t p = quantization.parameter_index(offset); + const std::size_t length = block_length( + quantization, output_length - offset); + const float scale_value = + static_cast(OperatorAccess::values(scale)[p]); + const float inverse_scale = 1.0F / scale_value; + const float zero = quantization.asymmetric() + ? rounded(static_cast( + OperatorAccess::values(zero_point)[p])) + : 0.0F; + ::arm_cmsis_dsp::VectorView input_block( + const_cast(OperatorAccess::values(input)) + offset, + 0, length); + ::arm_cmsis_dsp::VectorView output_block( + OperatorAccess::values(output) + offset, 0, length); + ::arm_cmsis_dsp::round_scaled_to_nearest_clipped( + output_block, input_block, inverse_scale, static_cast(zero), + static_cast(quantization.quant_min()), + static_cast(quantization.quant_max())); + offset += length; + } + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append( + *tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->zero_point_value = OperatorAccess::values(zero_point); + record->scale_gradient = OperatorAccess::gradients(scale); + record->zero_point_gradient = quantization.asymmetric() + ? OperatorAccess::gradients(zero_point) + : nullptr; + record->length = OperatorAccess::length(output); + record->quantization = quantization; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class QuantizeExpression +{ +public: + QuantizeExpression(const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization) noexcept + : input_(input), scale_(scale), zero_point_(zero_point), + quantization_(quantization) {} + void evaluate(BufferView &output) const noexcept + { + QuantizeOperator::evaluate(output, input_, scale_, zero_point_, + quantization_); + } +private: + BufferView input_; + BufferView scale_; + BufferView zero_point_; + Int8Quantization quantization_; +}; + +template +inline QuantizeExpression quantize( + const BufferView &input, const BufferView &scale, + const BufferView &zero_point, + Int8Quantization quantization = Int8Quantization::activation()) noexcept +{ + return QuantizeExpression(input, scale, zero_point, quantization); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/relu.hpp b/dsppp/Include/dsppp/autodiff/operators/relu.hpp new file mode 100644 index 000000000..9f661eeb3 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/relu.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class ReluOperator +{ + static constexpr T maximum_value() noexcept + { + if constexpr (std::is_same::value) + return std::numeric_limits::max(); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return F16_MAX; +#else + else + return T{}; +#endif + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void clip(const T *input, T *output, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_clip_f32(input, output, T{}, maximum_value(), + static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_clip_f16(input, output, T{}, maximum_value(), + static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + output[i] = input[i] > T{} ? input[i] : T{}; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + } + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + for (std::size_t i = 0; i < record.length; ++i) + { + if (static_cast(record.input_value[i]) > 0.0F) + { + if constexpr (std::is_same::value) + record.input_gradient[i] += record.output_gradient[i]; + else + record.input_gradient[i] = static_cast( + static_cast(record.input_gradient[i]) + + static_cast(record.output_gradient[i])); + } + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input)) + return false; +#endif + clip(OperatorAccess::values(input), OperatorAccess::values(output), + OperatorAccess::length(input)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->length = OperatorAccess::length(input); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class ReluExpression +{ +public: + explicit ReluExpression(const BufferView &input) noexcept : input_(input) {} + void evaluate(BufferView &output) const noexcept + { + ReluOperator::evaluate(output, input_); + } +private: + BufferView input_; +}; + +template +inline ReluExpression relu(const BufferView &input) noexcept +{ + return ReluExpression(input); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/scale.hpp b/dsppp/Include/dsppp/autodiff/operators/scale.hpp new file mode 100644 index 000000000..c177a565f --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/scale.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class ScaleOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void apply_scale(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_scale_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_scale_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] * value; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *input_value; + T *input_gradient; + const T *scale_value; + T *scale_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + record.scale_gradient[0] = T{}; + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView input_value( + const_cast(record.input_value), 0, record.length); + const T contribution = static_cast(::arm_cmsis_dsp::dot(output_gradient, input_value)); + if constexpr (std::is_same::value) + record.scale_gradient[0] += contribution; + else + record.scale_gradient[0] = static_cast( + static_cast(record.scale_gradient[0]) + + static_cast(contribution)); + if (record.input_gradient != nullptr) + { + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.length); + input_gradient += output_gradient * record.scale_value[0]; + } + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input, + const BufferView &scale) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + !OperatorAccess::valid(tape, scale) || + OperatorAccess::length(scale) != 1U || + OperatorAccess::role(scale) != BufferRole::parameter || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::gradients(scale) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::values(output) == OperatorAccess::values(scale) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(scale)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input, + const BufferView &scale) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input, scale)) + return false; +#endif + apply_scale(OperatorAccess::values(input), OperatorAccess::values(scale)[0], + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->input_value = OperatorAccess::values(input); + record->input_gradient = OperatorAccess::gradients(input); + record->scale_value = OperatorAccess::values(scale); + record->scale_gradient = OperatorAccess::gradients(scale); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class ScaleExpression +{ +public: + ScaleExpression(const BufferView &input, const BufferView &scale) noexcept + : input_(input), scale_(scale) {} + void evaluate(BufferView &output) const noexcept + { + ScaleOperator::evaluate(output, input_, scale_); + } +private: + BufferView input_; + BufferView scale_; +}; + +template +inline ScaleExpression scale(const BufferView &input, + const BufferView &constant) noexcept +{ + return ScaleExpression(input, constant); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/softmax.hpp b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp new file mode 100644 index 000000000..4689d3b59 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/softmax.hpp @@ -0,0 +1,183 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class SoftmaxOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static T logsumexp(const T *input, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + return arm_logsumexp_f32(input, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + return arm_logsumexp_f16(input, static_cast(length)); +#else + else + { + float sum = 0.0F; + for (std::size_t i = 0; i < length; ++i) + sum += std::exp(static_cast(input[i])); + return static_cast(std::log(sum)); + } +#endif + } + + static void offset(const T *input, T value, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_offset_f32(input, value, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_offset_f16(input, value, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = input[i] + value; +#endif + } + + static void exp(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_vexp_f32(data, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_vexp_f16(data, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) + data[i] = static_cast(std::exp(static_cast(data[i]))); +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + const T *output_value; + T *input_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.input_gradient != nullptr) + fill(record.input_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.input_gradient == nullptr) return; + + ::arm_cmsis_dsp::VectorView output_gradient( + record.output_gradient, 0, record.length); + ::arm_cmsis_dsp::VectorView output_value( + const_cast(record.output_value), 0, record.length); + ::arm_cmsis_dsp::VectorView input_gradient( + record.input_gradient, 0, record.length); + const T projection = + ::arm_cmsis_dsp::dot(output_gradient, output_value); + input_gradient += output_value * (output_gradient - projection); + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &input) noexcept + { + if (!OperatorAccess::compatible(tape, output, input) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(input) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(input)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &input) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, input)) + return false; +#endif + + const std::size_t length = OperatorAccess::length(input); + if (length == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + const T log_sum = logsumexp(OperatorAccess::values(input), length); + offset(OperatorAccess::values(input), static_cast(-static_cast(log_sum)), + OperatorAccess::values(output), length); + exp(OperatorAccess::values(output), length); + if (!OperatorAccess::recording(*tape)) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->output_value = OperatorAccess::values(output); + record->input_gradient = OperatorAccess::gradients(input); + record->length = length; + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class SoftmaxExpression +{ +public: + explicit SoftmaxExpression(const BufferView &input) noexcept + : input_(input) {} + void evaluate(BufferView &output) const noexcept + { + SoftmaxOperator::evaluate(output, input_); + } +private: + BufferView input_; +}; + +template +inline SoftmaxExpression softmax(const BufferView &input) noexcept +{ + return SoftmaxExpression(input); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/operators/sub.hpp b/dsppp/Include/dsppp/autodiff/operators/sub.hpp new file mode 100644 index 000000000..6c3c5abce --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/operators/sub.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +template class SubOperator +{ + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static void add(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_add_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_add_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] + right[i]; +#endif + } + + static void sub(const T *left, const T *right, T *output, + std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_sub_f32(left, right, output, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_sub_f16(left, right, output, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) output[i] = left[i] - right[i]; +#endif + } + + struct Record + { + detail::Node node; + T *output_gradient; + T *left_gradient; + T *right_gradient; + std::size_t length; + }; + + static void reset(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + fill(record.output_gradient, record.length); + if (record.left_gradient != nullptr) + fill(record.left_gradient, record.length); + if (record.right_gradient != nullptr) + fill(record.right_gradient, record.length); + } + + static void backward(detail::Node &node) noexcept + { + Record &record = reinterpret_cast(node); + if (record.left_gradient != nullptr) + add(record.left_gradient, record.output_gradient, + record.left_gradient, record.length); + if (record.right_gradient != nullptr) + sub(record.right_gradient, record.output_gradient, + record.right_gradient, record.length); + } + + static bool validate(Tape &tape, const BufferView &output, + const BufferView &left, + const BufferView &right) noexcept + { + if (!OperatorAccess::compatible(tape, output, left) || + !OperatorAccess::compatible(tape, output, right) || + OperatorAccess::gradients(output) == nullptr || + OperatorAccess::values(output) == OperatorAccess::values(left) || + OperatorAccess::values(output) == OperatorAccess::values(right) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(left) || + OperatorAccess::gradients(output) == OperatorAccess::gradients(right)) + { + OperatorAccess::fail(tape, Status::tape_mismatch); + return false; + } + return true; + } + +public: + static bool evaluate(BufferView &output, const BufferView &left, + const BufferView &right) noexcept + { + Tape *tape = OperatorAccess::tape(output); + OperatorAccess::set_producer(output, nullptr); + if (tape == nullptr || !OperatorAccess::template require>(*tape)) + return false; +#if DSPPP_AUTODIFF_ENABLE_VALIDATION + if (!validate(*tape, output, left, right)) + return false; +#endif + sub(OperatorAccess::values(left), OperatorAccess::values(right), + OperatorAccess::values(output), OperatorAccess::length(output)); + if (!OperatorAccess::recording(*tape) || + OperatorAccess::length(output) == 0U) + return OperatorAccess::status(*tape) == Status::ok; + + Record *record = OperatorAccess::template append(*tape, backward, reset); + if (record == nullptr) return false; + record->output_gradient = OperatorAccess::gradients(output); + record->left_gradient = OperatorAccess::gradients(left); + record->right_gradient = OperatorAccess::gradients(right); + record->length = OperatorAccess::length(output); + OperatorAccess::set_producer(output, &record->node); + return true; + } +}; + +template class SubExpression +{ +public: + SubExpression(const BufferView &left, const BufferView &right) noexcept + : left_(left), right_(right) {} + void evaluate(BufferView &output) const noexcept + { + SubOperator::evaluate(output, left_, right_); + } +private: + BufferView left_; + BufferView right_; +}; + +template +inline SubExpression operator-(const BufferView &left, + const BufferView &right) noexcept +{ + return SubExpression(left, right); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp new file mode 100644 index 000000000..0bc6ff557 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/adam.hpp @@ -0,0 +1,208 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage Adam optimizer for use in a user-written training loop. */ +template +class Adam +{ + static_assert(MaximumElements > 0U, "Adam needs state storage"); + static_assert(MaximumParameters > 0U, "Adam needs parameter slots"); + + static constexpr T default_epsilon() noexcept + { + // 1e-8 is below the useful range of half precision and rounds to zero. + if constexpr (std::is_same::value) + return T{1.0e-8F}; + return T{1.0e-4F}; + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + static T add(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left + right; + return static_cast(static_cast(left) + static_cast(right)); + } + static T sub(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left - right; + return static_cast(static_cast(left) - static_cast(right)); + } + static T mul(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left * right; + return static_cast(static_cast(left) * static_cast(right)); + } + static T div(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left / right; + return static_cast(static_cast(left) / static_cast(right)); + } + + struct Entry + { + T *values; + T *gradients; + std::size_t length; + std::size_t offset; + bool trainable; + }; + +public: + explicit Adam(T learning_rate = T{1.0e-3F}, T beta1 = T{0.9F}, + T beta2 = T{0.999F}, T epsilon = default_epsilon()) noexcept + : learning_rate_(learning_rate), beta1_(beta1), beta2_(beta2), + epsilon_(epsilon), beta1_power_(T{1}), beta2_power_(T{1}), + parameter_count_(0U), element_count_(0U), status_(OptimizerStatus::ok), + entries_{}, first_moment_{}, second_moment_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + fill(entries_[p].gradients, entries_[p].length); + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + beta1_power_ = mul(beta1_power_, beta1_); + beta2_power_ = mul(beta2_power_, beta2_); + const T first_correction = + static_cast(1.0F - static_cast(beta1_power_)); + const T second_correction = + static_cast(1.0F - static_cast(beta2_power_)); + const T one_minus_beta1 = + static_cast(1.0F - static_cast(beta1_)); + const T one_minus_beta2 = + static_cast(1.0F - static_cast(beta2_)); + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + ::arm_cmsis_dsp::VectorView first_moment( + first_moment_ + entry.offset, 0, entry.length); + ::arm_cmsis_dsp::VectorView second_moment( + second_moment_ + entry.offset, 0, entry.length); + first_moment = first_moment * beta1_ + gradients * one_minus_beta1; + second_moment = second_moment * beta2_ + + gradients * gradients * one_minus_beta2; + for (std::size_t i = 0; i < entry.length; ++i) + { + const std::size_t state = entry.offset + i; + const T corrected_first = div(first_moment_[state], first_correction); + const T corrected_second = div(second_moment_[state], second_correction); + const T denominator = add( + static_cast(std::sqrt(static_cast(corrected_second))), + epsilon_); + entry.values[i] = sub( + entry.values[i], + div(mul(learning_rate_, corrected_first), denominator)); + } + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(T *values, T *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = + Entry{values, gradients, length, element_count_, true}; + element_count_ += length; + return true; + } + + bool set_trainable(const T *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + T learning_rate_; + T beta1_; + T beta2_; + T epsilon_; + T beta1_power_; + T beta2_power_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; + T first_moment_[MaximumElements]; + T second_moment_[MaximumElements]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/common.hpp b/dsppp/Include/dsppp/autodiff/optimizers/common.hpp new file mode 100644 index 000000000..3f04db4cf --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/common.hpp @@ -0,0 +1,30 @@ +#pragma once + +namespace arm_cmsis_dsp { +namespace autodiff { + +enum class OptimizerStatus +{ + ok, + too_many_parameters, + too_many_elements, + invalid_parameter +}; + +/** Freeze all parameter views belonging to one logical operator/layer. */ +template +bool freeze_parameters(Optimizer &optimizer, Parameters... parameters) noexcept +{ + return (optimizer.freeze(parameters, true) && ...); +} + +/** Re-enable updates for a previously frozen operator/layer. */ +template +bool unfreeze_parameters(Optimizer &optimizer, + Parameters... parameters) noexcept +{ + return (optimizer.freeze(parameters, false) && ...); +} + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp new file mode 100644 index 000000000..f03ff10c6 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/rmsprop.hpp @@ -0,0 +1,187 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage RMSProp optimizer without momentum or centering. */ +template +class RMSProp +{ + static_assert(MaximumElements > 0U, "RMSProp needs state storage"); + static_assert(MaximumParameters > 0U, "RMSProp needs parameter slots"); + + static constexpr T default_epsilon() noexcept + { + // 1e-8 is below the useful range of half precision and rounds to zero. + if constexpr (std::is_same::value) + return T{1.0e-8F}; + return T{1.0e-4F}; + } + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + static T add(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left + right; + return static_cast(static_cast(left) + static_cast(right)); + } + static T sub(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left - right; + return static_cast(static_cast(left) - static_cast(right)); + } + static T mul(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left * right; + return static_cast(static_cast(left) * static_cast(right)); + } + static T div(T left, T right) noexcept + { + if constexpr (std::is_same::value) return left / right; + return static_cast(static_cast(left) / static_cast(right)); + } + + struct Entry + { + T *values; + T *gradients; + std::size_t length; + std::size_t offset; + bool trainable; + }; + +public: + explicit RMSProp(T learning_rate = T{1.0e-3F}, T alpha = T{0.99F}, + T epsilon = default_epsilon()) noexcept + : learning_rate_(learning_rate), alpha_(alpha), epsilon_(epsilon), + parameter_count_(0U), element_count_(0U), + status_(OptimizerStatus::ok), entries_{}, square_average_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + fill(entries_[p].gradients, entries_[p].length); + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + ::arm_cmsis_dsp::VectorView square_average( + square_average_ + entry.offset, 0, entry.length); + const T one_minus_alpha = + static_cast(1.0F - static_cast(alpha_)); + square_average = square_average * alpha_ + + gradients * gradients * one_minus_alpha; + for (std::size_t i = 0; i < entry.length; ++i) + { + const std::size_t state = entry.offset + i; + const T gradient = entry.gradients[i]; + const T denominator = add( + static_cast(std::sqrt( + static_cast(square_average_[state]))), + epsilon_); + entry.values[i] = sub( + entry.values[i], + div(mul(learning_rate_, gradient), denominator)); + } + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(T *values, T *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = + Entry{values, gradients, length, element_count_, true}; + element_count_ += length; + return true; + } + bool set_trainable(const T *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + T learning_rate_; + T alpha_; + T epsilon_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; + T square_average_[MaximumElements]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp b/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp new file mode 100644 index 000000000..287750a73 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/optimizers/sgd.hpp @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +/** Fixed-storage stochastic gradient descent optimizer without momentum. */ +template +class SGD +{ + static_assert(MaximumElements > 0U, "SGD needs element capacity"); + static_assert(MaximumParameters > 0U, "SGD needs parameter slots"); + + static void fill(T *data, std::size_t length) noexcept + { + if constexpr (std::is_same::value) + arm_fill_f32(T{}, data, static_cast(length)); +#if defined(ARM_FLOAT16_SUPPORTED) + else + arm_fill_f16(T{}, data, static_cast(length)); +#else + else + for (std::size_t i = 0; i < length; ++i) data[i] = T{}; +#endif + } + + struct Entry + { + T *values; + T *gradients; + std::size_t length; + bool trainable; + }; + +public: + explicit SGD(T learning_rate = T{1.0e-3F}) noexcept + : learning_rate_(learning_rate), parameter_count_(0U), + element_count_(0U), status_(OptimizerStatus::ok), entries_{} + { + } + + bool add(BufferView parameter) noexcept + { + if (parameter.role() != BufferRole::parameter || + !parameter.has_gradient()) + return fail(OptimizerStatus::invalid_parameter); + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool add(MatrixView parameter) noexcept + { + return add_impl(parameter.values(), parameter.gradients(), + parameter.length()); + } + + bool freeze(BufferView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + bool freeze(MatrixView parameter, bool frozen = true) noexcept + { + return set_trainable(parameter.values(), !frozen); + } + + void zero_grad() noexcept + { + for (std::size_t p = 0; p < parameter_count_; ++p) + fill(entries_[p].gradients, entries_[p].length); + } + + bool step() noexcept + { + if (status_ != OptimizerStatus::ok) return false; + for (std::size_t p = 0; p < parameter_count_; ++p) + { + Entry &entry = entries_[p]; + if (!entry.trainable) continue; + ::arm_cmsis_dsp::VectorView values( + entry.values, 0, entry.length); + ::arm_cmsis_dsp::VectorView gradients( + entry.gradients, 0, entry.length); + // The C++ expression fuses scaling and subtraction into one loop, + // avoiding the temporary buffer required by separate C kernels. + values -= gradients * learning_rate_; + } + return true; + } + + OptimizerStatus status() const noexcept { return status_; } + bool good() const noexcept { return status_ == OptimizerStatus::ok; } + +private: + bool add_impl(T *values, T *gradients, std::size_t length) noexcept + { + if (values == nullptr || gradients == nullptr) + return fail(OptimizerStatus::invalid_parameter); + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) return true; + if (parameter_count_ == MaximumParameters) + return fail(OptimizerStatus::too_many_parameters); + if (length > MaximumElements - element_count_) + return fail(OptimizerStatus::too_many_elements); + entries_[parameter_count_++] = Entry{values, gradients, length, true}; + element_count_ += length; + return true; + } + + bool set_trainable(const T *values, bool trainable) noexcept + { + for (std::size_t i = 0; i < parameter_count_; ++i) + if (entries_[i].values == values) + { + entries_[i].trainable = trainable; + return true; + } + return fail(OptimizerStatus::invalid_parameter); + } + + bool fail(OptimizerStatus status) noexcept + { + if (status_ == OptimizerStatus::ok) status_ = status; + return false; + } + + T learning_rate_; + std::size_t parameter_count_; + std::size_t element_count_; + OptimizerStatus status_; + Entry entries_[MaximumParameters]; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/reverse.hpp b/dsppp/Include/dsppp/autodiff/reverse.hpp new file mode 100644 index 000000000..4f01f4c76 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/reverse.hpp @@ -0,0 +1,688 @@ +// -*- C++ -*- +/** @file + * @brief Allocation-free reverse-mode AD core and operator registry. + */ +#pragma once + +#include +#include +#include +#include + +namespace arm_cmsis_dsp { +namespace autodiff { + +#ifndef DSPPP_AUTODIFF_MAX_OPERATORS +#define DSPPP_AUTODIFF_MAX_OPERATORS 16 +#endif + +#ifndef DSPPP_AUTODIFF_ENABLE_VALIDATION +#define DSPPP_AUTODIFF_ENABLE_VALIDATION 0 +#endif + +template class Tape; +template class OperatorAccess; + +namespace detail { + +struct Node; +using BackwardFunction = void (*)(Node &) noexcept; +using ResetFunction = void (*)(Node &) noexcept; + +struct Node +{ + Node *previous; + BackwardFunction backward; + ResetFunction reset_gradient; +}; + +template +inline constexpr unsigned char operator_token = 0U; + +} // namespace detail + +enum class Status +{ + ok, + out_of_memory, + tape_mismatch, + invalid_output, + operator_not_registered, + operator_registry_full +}; + +enum class BufferRole +{ + input, + parameter, + intermediate +}; + +/** Non-owning view of caller values and an associated gradient buffer. */ +template class BufferView +{ +public: + BufferView() noexcept + : values_(nullptr), gradients_(nullptr), length_(0U), tape_(nullptr), + producer_(nullptr), role_(BufferRole::input) + { + } + + T *values() noexcept { return values_; } + const T *values() const noexcept { return values_; } + T *gradients() noexcept { return gradients_; } + const T *gradients() const noexcept { return gradients_; } + std::size_t length() const noexcept { return length_; } + BufferRole role() const noexcept { return role_; } + bool has_gradient() const noexcept { return gradients_ != nullptr; } + + T &operator[](std::size_t index) noexcept { return values_[index]; } + const T &operator[](std::size_t index) const noexcept + { + return values_[index]; + } + T gradient(std::size_t index) const noexcept + { + return gradients_ == nullptr ? T{} : gradients_[index]; + } + + /** Evaluate any expression supplied by a separately included operator. */ + template + BufferView &operator=(const Expression &expression) noexcept + { + expression.evaluate(*this); + return *this; + } + +private: + BufferView(T *values, T *gradients, std::size_t length, + Tape *tape, BufferRole role) noexcept + : values_(values), gradients_(gradients), length_(length), tape_(tape), + producer_(nullptr), role_(role) + { + } + + T *values_; + T *gradients_; + std::size_t length_; + Tape *tape_; + detail::Node *producer_; + BufferRole role_; + + friend class Tape; + friend class OperatorAccess; +}; + +/** Non-owning row-major matrix parameter view. */ +template class MatrixView +{ +public: + MatrixView() noexcept : buffer_(), rows_(0U), columns_(0U) {} + + std::size_t rows() const noexcept { return rows_; } + std::size_t columns() const noexcept { return columns_; } + T *values() noexcept { return buffer_.values(); } + const T *values() const noexcept { return buffer_.values(); } + T *gradients() noexcept { return buffer_.gradients(); } + const T *gradients() const noexcept { return buffer_.gradients(); } + std::size_t length() const noexcept { return rows_ * columns_; } + T &operator()(std::size_t row, std::size_t column) noexcept + { + return buffer_.values()[row * columns_ + column]; + } + const T &operator()(std::size_t row, + std::size_t column) const noexcept + { + return buffer_.values()[row * columns_ + column]; + } + T gradient(std::size_t row, std::size_t column) const noexcept + { + return buffer_.gradient(row * columns_ + column); + } + +private: + MatrixView(const BufferView &buffer, std::size_t rows, + std::size_t columns) noexcept + : buffer_(buffer), rows_(rows), columns_(columns) + { + } + + BufferView buffer_; + std::size_t rows_; + std::size_t columns_; + + friend class Tape; + friend class OperatorAccess; +}; + +/** Reverse-mode tape using caller-supplied storage and a fixed operator list. */ +template class Tape +{ +public: + static constexpr std::size_t maximum_registered_operators = + DSPPP_AUTODIFF_MAX_OPERATORS; + static_assert(maximum_registered_operators > 0U, + "The operator registry must contain at least one slot"); + static_assert((maximum_registered_operators & + (maximum_registered_operators - 1U)) == 0U, + "The operator registry capacity must be a power of two"); + + Tape(void *storage, std::size_t bytes) noexcept + : storage_(static_cast(storage)), capacity_(bytes), + used_(0U), tail_(nullptr), recording_(true), status_(Status::ok), + graph_begin_(0U), graph_marked_(false), registered_count_(0U), + registered_operators_{} + { + } + + Tape(const Tape &) = delete; + Tape &operator=(const Tape &) = delete; + + /** Register an operator type once. Registration uses no arena storage. */ + template + bool register_operator() noexcept + { + const void *token = &detail::operator_token; + const std::size_t first = operator_hash(token); + for (std::size_t probe = 0U; + probe < maximum_registered_operators; ++probe) + { + const std::size_t slot = + (first + probe) & (maximum_registered_operators - 1U); + if (registered_operators_[slot] == token) + { + return true; + } + if (registered_operators_[slot] == nullptr) + { + registered_operators_[slot] = token; + ++registered_count_; + return true; + } + } + set_error(Status::operator_registry_full); + return false; + } + + template + bool is_operator_registered() const noexcept + { + const void *token = &detail::operator_token; + const std::size_t first = operator_hash(token); + for (std::size_t probe = 0U; + probe < maximum_registered_operators; ++probe) + { + const std::size_t slot = + (first + probe) & (maximum_registered_operators - 1U); + if (registered_operators_[slot] == token) + { + return true; + } + if (registered_operators_[slot] == nullptr) + { + return false; + } + } + return false; + } + + /** Release gradients and records. Operator registrations are preserved. */ + void reset() noexcept + { + used_ = 0U; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + graph_begin_ = 0U; + graph_marked_ = false; + } + + /** Mark all current arena allocations as persistent training state. */ + void begin_graph() noexcept + { + graph_begin_ = used_; + graph_marked_ = true; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + } + + /** Reclaim only operation records, preserving views and their gradients. */ + bool rewind_graph() noexcept + { + if (!graph_marked_) + { + set_error(Status::invalid_output); + return false; + } + used_ = graph_begin_; + tail_ = nullptr; + recording_ = true; + status_ = Status::ok; + return true; + } + + std::size_t used() const noexcept { return used_; } + std::size_t capacity() const noexcept { return capacity_; } + Status status() const noexcept { return status_; } + bool good() const noexcept { return status_ == Status::ok; } + void clear_status() noexcept { status_ = Status::ok; } + bool recording() const noexcept { return recording_; } + void set_recording(bool enabled) noexcept { recording_ = enabled; } + + /** Generic active view; output() is clearer for application code. */ + BufferView view(T *values, std::size_t length) noexcept + { + if (length != 0U && values == nullptr) + { + set_error(Status::tape_mismatch); + return BufferView(values, nullptr, length, this, + BufferRole::intermediate); + } + if (length > static_cast(-1) / sizeof(T)) + { + set_error(Status::out_of_memory); + return BufferView(values, nullptr, length, this, + BufferRole::intermediate); + } + + T *gradients = nullptr; + if (length != 0U) + { + gradients = static_cast( + allocate(length * sizeof(T), alignof(T))); + if (gradients != nullptr) + { + for (std::size_t i = 0; i < length; ++i) + { + gradients[i] = T{}; + } + } + } + return BufferView(values, gradients, length, this, + BufferRole::intermediate); + } + + template + BufferView view(T (&values)[Length]) noexcept + { + return view(values, Length); + } + + BufferView view(T *values, T *gradients, + std::size_t length) noexcept + { + if (length != 0U && (values == nullptr || gradients == nullptr)) + { + set_error(Status::tape_mismatch); + } + return BufferView(values, gradients, length, this, + BufferRole::intermediate); + } + + template + BufferView view(T (&values)[Length], + T (&gradients)[Length]) noexcept + { + return view(values, gradients, Length); + } + + BufferView input(T *values, std::size_t length) noexcept + { + if (length != 0U && values == nullptr) + { + set_error(Status::tape_mismatch); + } + return BufferView(values, nullptr, length, this, BufferRole::input); + } + + template + BufferView input(T (&values)[Length]) noexcept + { + return input(values, Length); + } + + BufferView input(T &value) noexcept { return input(&value, 1U); } + + BufferView parameter(T *values, std::size_t length) noexcept + { + BufferView result = view(values, length); + result.role_ = BufferRole::parameter; + return result; + } + + template + BufferView parameter(T (&values)[Length]) noexcept + { + return parameter(values, Length); + } + + BufferView parameter(T &value) noexcept + { + return parameter(&value, 1U); + } + + BufferView parameter(T *values, T *gradients, + std::size_t length) noexcept + { + BufferView result = view(values, gradients, length); + result.role_ = BufferRole::parameter; + return result; + } + + template + BufferView parameter(T (&values)[Length], + T (&gradients)[Length]) noexcept + { + return parameter(values, gradients, Length); + } + + BufferView parameter(T &value, T &gradient) noexcept + { + return parameter(&value, &gradient, 1U); + } + + MatrixView parameter(T *values, std::size_t rows, + std::size_t columns) noexcept + { + if (columns != 0U && rows > static_cast(-1) / columns) + { + set_error(Status::out_of_memory); + return MatrixView(); + } + return MatrixView(parameter(values, rows * columns), rows, columns); + } + + template + MatrixView parameter(T (&values)[Rows][Columns]) noexcept + { + return parameter(&values[0][0], Rows, Columns); + } + + MatrixView parameter(T *values, T *gradients, std::size_t rows, + std::size_t columns) noexcept + { + if (columns != 0U && rows > static_cast(-1) / columns) + { + set_error(Status::out_of_memory); + return MatrixView(); + } + return MatrixView(parameter(values, gradients, rows * columns), rows, + columns); + } + + template + MatrixView parameter(T (&values)[Rows][Columns], + T (&gradients)[Rows][Columns]) noexcept + { + return parameter(&values[0][0], &gradients[0][0], Rows, Columns); + } + + BufferView output(T *values, std::size_t length) noexcept + { + return view(values, length); + } + + template + BufferView output(T (&values)[Length]) noexcept + { + return output(values, Length); + } + + BufferView output(T &value) noexcept { return output(&value, 1U); } + + BufferView output(T *values, T *gradients, + std::size_t length) noexcept + { + return view(values, gradients, length); + } + + template + BufferView output(T (&values)[Length], + T (&gradients)[Length]) noexcept + { + return output(values, gradients, Length); + } + + BufferView output(T &value, T &gradient) noexcept + { + return output(&value, &gradient, 1U); + } + + bool backward(const BufferView &output, T seed = T{1}) noexcept + { + if (output.length_ != 1U) + { + set_error(Status::invalid_output); + return false; + } + return backward(output, &seed, 1U); + } + + bool backward(const BufferView &output, const T *seed, + std::size_t seed_length) noexcept + { + if (status_ != Status::ok) + { + return false; + } + if (!valid(output) || output.producer_ == nullptr || seed == nullptr || + seed_length != output.length_) + { + set_error(Status::invalid_output); + return false; + } + + for (detail::Node *node = output.producer_; node != nullptr; + node = node->previous) + { + node->reset_gradient(*node); + } + for (std::size_t i = 0; i < output.length_; ++i) + { + output.gradients_[i] = seed[i]; + } + for (detail::Node *node = output.producer_; node != nullptr; + node = node->previous) + { + node->backward(*node); + } + return true; + } + +private: + static std::size_t operator_hash(const void *token) noexcept + { + std::uintptr_t value = reinterpret_cast(token); + value ^= value >> 4U; + value *= static_cast(0x9E3779B1U); + value ^= value >> (sizeof(std::uintptr_t) * 4U); + return static_cast(value) & + (maximum_registered_operators - 1U); + } + + bool valid(const BufferView &view) const noexcept + { + return view.tape_ == this && + (view.length_ == 0U || + (view.values_ != nullptr && + (view.role_ == BufferRole::input || + view.gradients_ != nullptr))); + } + + void set_error(Status error) noexcept + { + if (status_ == Status::ok) + { + status_ = error; + } + } + + void *allocate(std::size_t bytes, std::size_t alignment) noexcept + { + if (storage_ == nullptr || alignment == 0U) + { + set_error(Status::out_of_memory); + return nullptr; + } + const std::uintptr_t base = reinterpret_cast(storage_); + const std::uintptr_t current = base + used_; + const std::size_t padding = static_cast( + (alignment - (current % alignment)) % alignment); + if (used_ > capacity_) + { + set_error(Status::out_of_memory); + return nullptr; + } + const std::size_t remaining = capacity_ - used_; + if (padding > remaining || bytes > remaining - padding) + { + set_error(Status::out_of_memory); + return nullptr; + } + used_ += padding; + void *result = storage_ + used_; + used_ += bytes; + return result; + } + + template + Record *append(detail::BackwardFunction backward, + detail::ResetFunction reset_gradient) noexcept + { + static_assert(std::is_trivially_destructible::value, + "Tape records are discarded without destructors"); + void *memory = allocate(sizeof(Record), alignof(Record)); + if (memory == nullptr) + { + return nullptr; + } + Record *record = ::new (memory) Record(); + record->node.previous = tail_; + record->node.backward = backward; + record->node.reset_gradient = reset_gradient; + tail_ = &record->node; + return record; + } + + unsigned char *storage_; + std::size_t capacity_; + std::size_t used_; + detail::Node *tail_; + bool recording_; + Status status_; + std::size_t graph_begin_; + bool graph_marked_; + std::size_t registered_count_; + const void *registered_operators_[maximum_registered_operators]; + + friend class OperatorAccess; +}; + +/** Narrow internal interface used by independently defined operators. */ +template class OperatorAccess +{ +public: + static Tape *tape(const BufferView &view) noexcept { return view.tape_; } + static T *values(BufferView &view) noexcept { return view.values_; } + static const T *values(const BufferView &view) noexcept + { + return view.values_; + } + static T *gradients(const BufferView &view) noexcept + { + return view.gradients_; + } + static std::size_t length(const BufferView &view) noexcept + { + return view.length_; + } + static BufferRole role(const BufferView &view) noexcept { return view.role_; } + static detail::Node *producer(const BufferView &view) noexcept + { + return view.producer_; + } + static void set_producer(BufferView &view, detail::Node *node) noexcept + { + view.producer_ = node; + } + static bool valid(const Tape &tape, const BufferView &view) noexcept + { + return tape.valid(view); + } + static bool compatible(const Tape &tape, const BufferView &left, + const BufferView &right) noexcept + { + return valid(tape, left) && valid(tape, right) && + length(left) == length(right); + } + static const BufferView &buffer(const MatrixView &matrix) noexcept + { + return matrix.buffer_; + } + static std::size_t rows(const MatrixView &matrix) noexcept + { + return matrix.rows_; + } + static std::size_t columns(const MatrixView &matrix) noexcept + { + return matrix.columns_; + } + static bool recording(const Tape &tape) noexcept { return tape.recording_; } + static Status status(const Tape &tape) noexcept { return tape.status_; } + static void fail(Tape &tape, Status status) noexcept { tape.set_error(status); } + + template + static bool require(Tape &tape) noexcept + { + if (!tape.template is_operator_registered()) + { + tape.set_error(Status::operator_not_registered); + return false; + } + return true; + } + + template + static Record *append(Tape &tape, detail::BackwardFunction backward, + detail::ResetFunction reset_gradient) noexcept + { + return tape.template append(backward, reset_gradient); + } +}; + +template class RecordingScope +{ +public: + RecordingScope(Tape &tape, bool enabled) noexcept + : tape_(tape), previous_(tape.recording()) + { + tape_.set_recording(enabled); + } + ~RecordingScope() noexcept { tape_.set_recording(previous_); } + RecordingScope(const RecordingScope &) = delete; + RecordingScope &operator=(const RecordingScope &) = delete; + +private: + Tape &tape_; + bool previous_; +}; + +template +class Arena +{ +public: + static_assert(Bytes > 0U, "An autodiff arena must contain storage"); + Arena() noexcept : storage_{}, tape_(storage_, Bytes) {} + Arena(const Arena &) = delete; + Arena &operator=(const Arena &) = delete; + Tape &tape() noexcept { return tape_; } + const Tape &tape() const noexcept { return tape_; } + +private: + alignas(std::max_align_t) unsigned char storage_[Bytes]; + Tape tape_; +}; + +} // namespace autodiff +} // namespace arm_cmsis_dsp diff --git a/dsppp/Include/dsppp/autodiff/scripts/README.md b/dsppp/Include/dsppp/autodiff/scripts/README.md new file mode 100644 index 000000000..1ac3d7461 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/README.md @@ -0,0 +1,5 @@ +# PyTorch conversion scripts + +See [Converting PyTorch modules](../doc/pytorch_conversion.md) for supported +operators, model constraints, converter usage, generated code integration, and +the PyTorch Iris comparison. diff --git a/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py b/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py new file mode 100644 index 000000000..75c7feab0 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/iris_pytorch.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""PyTorch counterpart of dsppp/Examples/autodiff_iris.cpp.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import numpy as np +import torch +from torch import nn + +from pytorch_to_autodiff import convert_module + + +INPUT_SIZE = 4 +HIDDEN_SIZE = 8 +CLASS_COUNT = 3 +TRAINING_COUNT = 120 + + +class IrisNetwork(nn.Module): + def __init__(self) -> None: + super().__init__() + self.hidden = nn.Linear(INPUT_SIZE, HIDDEN_SIZE) + self.relu = nn.ReLU() + self.output = nn.Linear(HIDDEN_SIZE, CLASS_COUNT) + self.softmax = nn.Softmax(dim=-1) + + def logits(self, value: torch.Tensor) -> torch.Tensor: + return self.output(self.relu(self.hidden(value))) + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return self.softmax(self.logits(value)) + + +def create_model() -> nn.Module: + """Factory used by pytorch_to_autodiff.py.""" + return IrisNetwork() + + +def load_dataset() -> tuple[torch.Tensor, torch.Tensor]: + """Load the exact samples and normalization used by the C++ example.""" + header = Path(__file__).resolve().parents[4] / "Examples" / "iris_data.hpp" + text = header.read_text(encoding="utf-8") + pattern = re.compile( + r"\{\{\s*([-+0-9.]+)F,\s*([-+0-9.]+)F,\s*" + r"([-+0-9.]+)F,\s*([-+0-9.]+)F\s*\},\s*([0-2])\s*\}" + ) + rows = pattern.findall(text) + if len(rows) != 150: + raise RuntimeError(f"expected 150 Iris samples in {header}, found {len(rows)}") + features = np.asarray([[float(value) for value in row[:4]] for row in rows], dtype=np.float32) + labels = np.asarray([int(row[4]) for row in rows], dtype=np.int64) + mean = np.asarray([5.843333, 3.057333, 3.758000, 1.199333], dtype=np.float32) + inverse_std = np.asarray([1.211678, 2.301971, 0.568374, 1.316322], dtype=np.float32) + features = (features - mean) * inverse_std + return torch.from_numpy(features), torch.from_numpy(labels) + + +def is_test_sample(index: int) -> bool: + return (index % 50) % 5 == 0 + + +def initialize_like_cpp(model: IrisNetwork, generator: torch.Generator) -> None: + with torch.no_grad(): + for layer in (model.hidden, model.output): + layer.weight.uniform_(-0.25, 0.25, generator=generator) + layer.bias.zero_() + + +def train(model: IrisNetwork, epochs: int, seed: int) -> tuple[int, int]: + features, labels = load_dataset() + training_indices = torch.tensor( + [index for index in range(len(labels)) if not is_test_sample(index)], dtype=torch.long + ) + test_indices = torch.tensor( + [index for index in range(len(labels)) if is_test_sample(index)], dtype=torch.long + ) + generator = torch.Generator().manual_seed(seed) + initialize_like_cpp(model, generator) + optimizer = torch.optim.Adam(model.parameters(), lr=1.0e-2) + loss_function = nn.CrossEntropyLoss(reduction="sum") + + model.train() + for epoch in range(epochs): + order = training_indices[torch.randperm(TRAINING_COUNT, generator=generator)] + epoch_loss = 0.0 + for sample in order.tolist(): + optimizer.zero_grad() + logits = model.logits(features[sample : sample + 1]) + loss = loss_function(logits, labels[sample : sample + 1]) + loss.backward() + optimizer.step() + epoch_loss += float(loss.detach()) + if (epoch + 1) % 20 == 0: + print(f"epoch {epoch + 1}: mean loss={epoch_loss / TRAINING_COUNT:g}") + + model.eval() + with torch.no_grad(): + detected = model(features[test_indices]).argmax(dim=1) + correct = int((detected == labels[test_indices]).sum()) + print(f"final test accuracy={correct}/30 tests") + return correct, len(test_indices) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--epochs", + type=int, + help="train for this many epochs; defaults to 120 when not exporting", + ) + parser.add_argument("--seed", type=int, default=0x12345678) + parser.add_argument("--export", type=Path, help="export trained model to an autodiff header") + parser.add_argument("--dtype", choices=("float32", "float16"), default="float32") + parser.add_argument("--arena-bytes", type=int, default=2048) + arguments = parser.parse_args() + if arguments.epochs is not None and arguments.epochs < 0: + parser.error("--epochs must be nonnegative") + + torch.manual_seed(arguments.seed) + model = IrisNetwork() + trained = arguments.epochs is not None or arguments.export is None + if trained: + train(model, arguments.epochs if arguments.epochs is not None else 120, + arguments.seed) + else: + model.eval() + if arguments.export is not None: + generated = convert_module( + model, + (INPUT_SIZE,), + class_name="IrisNetwork", + namespace="generated_iris", + dtype=arguments.dtype, + arena_bytes=arguments.arena_bytes, + export_mode="trained" if trained else "empty", + ) + arguments.export.write_text(generated, encoding="utf-8") + print(f"wrote {arguments.export}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py b/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py new file mode 100644 index 000000000..4bfd6f16c --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/pytorch_to_autodiff.py @@ -0,0 +1,752 @@ +#!/usr/bin/env python3 +"""Convert a supported PyTorch nn.Module to CMSIS-DSP autodiff C++ code.""" + +from __future__ import annotations + +import argparse +import importlib.util +import math +import re +import sys +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any, Sequence + +import torch +from torch import fx, nn + + +class ConversionError(RuntimeError): + """Raised when a module cannot be represented by the autodiff API.""" + + +@dataclass(frozen=True) +class Value: + name: str + shape: tuple[int, ...] + + @property + def length(self) -> int: + return math.prod(self.shape) if self.shape else 1 + + +@dataclass(frozen=True) +class Parameter: + name: str + shape: tuple[int, ...] + values: tuple[float, ...] + trainable: bool + matrix_view: bool = False + initialization: str = "preserve" + + +@dataclass(frozen=True) +class Operation: + kind: str + output: Value + inputs: tuple[str, ...] + attributes: tuple[Any, ...] = () + + +@dataclass(frozen=True) +class ConvertedModule: + input: Value + output_name: str + values: tuple[Value, ...] + parameters: tuple[Parameter, ...] + operations: tuple[Operation, ...] + + +def _identifier(name: str) -> str: + result = re.sub(r"[^A-Za-z0-9_]", "_", name) + if not result or result[0].isdigit(): + result = "value_" + result + return result + + +def _shape_text(shape: Sequence[int]) -> str: + return "scalar" if not shape else " x ".join(str(v) for v in shape) + + +def _normalized_shape(node: fx.Node) -> tuple[int, ...]: + value = node.meta.get("val") + if not isinstance(value, torch.Tensor): + raise ConversionError( + f"node '{node.name}' does not produce a tensor with a known shape" + ) + shape = tuple(int(dimension) for dimension in value.shape) + if not shape or shape[0] != 1: + raise ConversionError( + f"node '{node.name}' has shape {shape}; the leading batch dimension " + "must exist and equal 1" + ) + embedded_shape = shape[1:] + if len(embedded_shape) > 2: + raise ConversionError( + f"node '{node.name}' becomes {_shape_text(embedded_shape)} after " + "removing batch 1; autodiff supports only scalars, vectors, and matrices" + ) + return embedded_shape + + +def _tensor_values(tensor: torch.Tensor) -> tuple[float, ...]: + flat = tensor.detach().cpu().reshape(-1).tolist() + result = tuple(float(value) for value in flat) + if not all(math.isfinite(value) for value in result): + raise ConversionError("model parameters must contain only finite values") + return result + + +def _node_argument(argument: Any, aliases: dict[fx.Node, str]) -> str: + if not isinstance(argument, fx.Node) or argument not in aliases: + raise ConversionError("operator arguments must be traced tensor values") + return aliases[argument] + + +def _validate_vector_softmax(node: fx.Node, shape: tuple[int, ...], dim: int) -> None: + if len(shape) != 1 or dim not in (-1, 1): + raise ConversionError( + f"softmax node '{node.name}' must operate on the feature dimension " + "of a batch-1 vector; got embedded shape " + _shape_text(shape) + ) + + +def inspect_module(module: nn.Module, input_shape: Sequence[int]) -> ConvertedModule: + """Export and validate a module, returning an architecture-neutral model.""" + shape = tuple(int(value) for value in input_shape) + if any(value <= 0 for value in shape): + raise ConversionError("all input dimensions must be positive") + if len(shape) > 2: + raise ConversionError( + "input_shape excludes batch and may describe only a scalar, vector, or matrix" + ) + + model_parameters = dict(module.named_parameters()) + first_parameter = next(iter(model_parameters.values()), None) + dtype = first_parameter.dtype if first_parameter is not None else torch.float32 + device = first_parameter.device if first_parameter is not None else torch.device("cpu") + example = torch.zeros((1, *shape), dtype=dtype, device=device) + try: + exported = torch.export.export(module, (example,)) + except Exception as error: + raise ConversionError(f"PyTorch ATen export failed: {error}") from error + + graph = exported.graph_module.graph + specs = {spec.arg.name: spec for spec in exported.graph_signature.input_specs} + aliases: dict[fx.Node, str] = {} + shapes: dict[str, tuple[int, ...]] = {} + values: list[Value] = [] + parameters: list[Parameter] = [] + parameter_indices: dict[str, int] = {} + operations: list[Operation] = [] + input_value: Value | None = None + output_name: str | None = None + + def tensor_for_spec(spec: Any) -> torch.Tensor: + target = str(spec.target) + kind = spec.kind.name + if kind in ("PARAMETER", "BUFFER"): + tensor = exported.state_dict[target] + elif kind == "CONSTANT_TENSOR": + tensor = exported.constants[target] + else: + raise ConversionError(f"unsupported lifted input kind {kind}") + if not isinstance(tensor, torch.Tensor): + raise ConversionError(f"lifted input '{target}' is not a tensor") + return tensor + + def unique_parameter_name(target: str) -> str: + base = _identifier(target) + name = base + suffix = 2 + while name in parameter_indices: + name = f"{base}_{suffix}" + suffix += 1 + return name + + for node in graph.nodes: + if node.op != "placeholder": + continue + spec = specs.get(str(node.target)) + if spec is None: + raise ConversionError(f"placeholder '{node.name}' has no graph signature entry") + if spec.kind.name == "USER_INPUT": + if input_value is not None: + raise ConversionError("exactly one tensor input is supported") + input_value = Value("input", _normalized_shape(node)) + aliases[node] = input_value.name + shapes[input_value.name] = input_value.shape + continue + tensor = tensor_for_spec(spec) + parameter_shape = tuple(int(value) for value in tensor.shape) + if len(parameter_shape) > 2: + raise ConversionError( + f"parameter '{spec.target}' has unsupported shape {parameter_shape}" + ) + name = unique_parameter_name(str(spec.target)) + source_parameter = model_parameters.get(str(spec.target)) + parameter = Parameter( + name, + parameter_shape, + _tensor_values(tensor), + bool(source_parameter is not None and source_parameter.requires_grad), + ) + parameter_indices[name] = len(parameters) + parameters.append(parameter) + aliases[node] = name + shapes[name] = parameter_shape + + if input_value is None: + raise ConversionError("exactly one tensor input is supported") + + def mark_parameter(name: str, *, matrix_view: bool = False, + initialization: str = "preserve") -> None: + index = parameter_indices.get(name) + if index is None: + raise ConversionError(f"'{name}' is not a lifted model parameter") + current = parameters[index] + parameters[index] = replace( + current, + matrix_view=current.matrix_view or matrix_view, + initialization=( + initialization + if current.initialization == "preserve" + else current.initialization + ), + ) + + def new_output(node: fx.Node) -> Value: + value = Value(_identifier(node.name), _normalized_shape(node)) + values.append(value) + aliases[node] = value.name + shapes[value.name] = value.shape + return value + + def add_scalar_constant(value: Any, node_name: str) -> str: + try: + scalar = float(value) + except (TypeError, ValueError) as error: + raise ConversionError( + f"ATen node '{node_name}' has a non-numeric scalar operand" + ) from error + if not math.isfinite(scalar): + raise ConversionError(f"ATen node '{node_name}' has a non-finite scalar operand") + name = unique_parameter_name(node_name + "_constant") + parameter_indices[name] = len(parameters) + parameters.append(Parameter(name, (), (scalar,), False)) + shapes[name] = () + return name + + def operand(argument: Any, node_name: str) -> tuple[str, tuple[int, ...], bool]: + if isinstance(argument, fx.Node): + if argument not in aliases: + raise ConversionError(f"ATen node '{node_name}' uses an unsupported value") + name = aliases[argument] + return name, shapes[name], name in parameter_indices + name = add_scalar_constant(argument, node_name) + return name, (), True + + def add_arithmetic(node: fx.Node, kind: str) -> None: + if len(node.args) < 2: + raise ConversionError(f"ATen arithmetic node '{node.name}' needs two operands") + if float(node.args[2] if len(node.args) > 2 else node.kwargs.get("alpha", 1.0)) != 1.0: + raise ConversionError( + f"ATen arithmetic node '{node.name}' uses unsupported alpha" + ) + left_name, left_shape, left_parameter = operand(node.args[0], node.name) + right_name, right_shape, right_parameter = operand(node.args[1], node.name) + output = new_output(node) + if left_shape == right_shape: + if output.shape != left_shape: + raise ConversionError(f"ATen arithmetic node '{node.name}' changed shape") + operations.append(Operation(kind, output, (left_name, right_name))) + return + if kind in ("add", "multiply"): + if not left_shape and left_parameter and right_shape == output.shape: + scalar_name, value_name = left_name, right_name + elif not right_shape and right_parameter and left_shape == output.shape: + scalar_name, value_name = right_name, left_name + else: + raise ConversionError( + f"ATen node '{node.name}' uses broadcasting not supported by autodiff" + ) + operations.append(Operation( + "offset" if kind == "add" else "scale", + output, + (value_name, scalar_name), + )) + return + raise ConversionError( + f"ATen subtraction node '{node.name}' requires equal-shaped operands" + ) + + def add_dot(node: fx.Node) -> None: + left_name, left_shape, _ = operand(node.args[0], node.name) + right_name, right_shape, _ = operand(node.args[1], node.name) + output = new_output(node) + if len(left_shape) != 1 or left_shape != right_shape or output.shape: + raise ConversionError( + f"ATen node '{node.name}' is not a vector dot product" + ) + operations.append(Operation("dot", output, (left_name, right_name))) + + aten = torch.ops.aten + arithmetic = { + aten.add.Tensor: "add", + aten.add.Scalar: "add", + aten.sub.Tensor: "sub", + aten.sub.Scalar: "sub", + aten.mul.Tensor: "multiply", + aten.mul.Scalar: "multiply", + } + dot_targets = {aten.matmul.default, aten.inner.default, aten.dot.default} + + for node in graph.nodes: + if node.op == "placeholder": + continue + if node.op == "output": + returned = node.args[0] + if not isinstance(returned, (tuple, list)) or len(returned) != 1: + raise ConversionError("the module must return one tensor") + output_name = _node_argument(returned[0], aliases) + continue + if node.op != "call_function": + raise ConversionError( + f"unsupported exported graph node '{node.name}' of kind '{node.op}'" + ) + + target = node.target + if target == aten.linear.default: + source = _node_argument(node.args[0], aliases) + weight = _node_argument(node.args[1], aliases) + source_shape = shapes[source] + weight_shape = shapes[weight] + if len(source_shape) != 1 or len(weight_shape) != 2: + raise ConversionError(f"ATen linear node '{node.name}' requires vector input") + if source_shape[0] != weight_shape[1]: + raise ConversionError(f"ATen linear node '{node.name}' has incompatible dimensions") + mark_parameter(weight, matrix_view=True, initialization="xavier") + if len(node.args) < 3 or node.args[2] is None: + bias = unique_parameter_name(node.name + "_bias") + parameter_indices[bias] = len(parameters) + parameters.append( + Parameter(bias, (weight_shape[0],), (0.0,) * weight_shape[0], False) + ) + shapes[bias] = (weight_shape[0],) + else: + bias = _node_argument(node.args[2], aliases) + if shapes[bias] != (weight_shape[0],): + raise ConversionError(f"ATen linear node '{node.name}' has invalid bias") + mark_parameter(bias, initialization="zero") + output = new_output(node) + if output.shape != (weight_shape[0],): + raise ConversionError(f"ATen linear node '{node.name}' has invalid output") + operations.append(Operation("fully_connected", output, (source, weight, bias))) + elif target == aten.relu.default: + output = new_output(node) + operations.append(Operation("relu", output, (_node_argument(node.args[0], aliases),))) + elif target in (aten.softmax.int, aten._softmax.default): + output = new_output(node) + dimension = int(node.args[1]) + _validate_vector_softmax(node, output.shape, dimension) + operations.append(Operation("softmax", output, (_node_argument(node.args[0], aliases),))) + elif target == aten.dropout.default: + source = _node_argument(node.args[0], aliases) + probability = float(node.args[1]) + if not 0.0 <= probability < 1.0: + raise ConversionError(f"ATen dropout node '{node.name}' has invalid probability") + output = new_output(node) + operations.append(Operation("dropout", output, (source,), (probability,))) + elif target in arithmetic: + add_arithmetic(node, arithmetic[target]) + elif target in dot_targets: + add_dot(node) + else: + raise ConversionError( + f"unsupported ATen operator at node '{node.name}': {target}" + ) + + if output_name is None: + raise ConversionError("the exported module has no tensor output") + return ConvertedModule( + input_value, + output_name, + tuple(values), + tuple(parameters), + tuple(operations), + ) + + +def _float_literal(value: float) -> str: + text = format(value, ".9g") + if "e" not in text.lower() and "." not in text: + text += ".0" + return f"static_cast({text}F)" + + +def _initializer(parameter: Parameter) -> str: + if len(parameter.shape) == 2: + rows, columns = parameter.shape + groups = [] + for row in range(rows): + start = row * columns + groups.append( + "{" + ", ".join(_float_literal(v) for v in parameter.values[start:start + columns]) + "}" + ) + return "{" + ",\n ".join(groups) + "}" + return "{" + ", ".join(_float_literal(v) for v in parameter.values) + "}" + + +def _array_suffix(shape: tuple[int, ...]) -> str: + dimensions = shape if shape else (1,) + return "".join(f"[{dimension}]" for dimension in dimensions) + + +def generate_cpp( + converted: ConvertedModule, + *, + class_name: str = "GeneratedNetwork", + namespace: str = "generated_autodiff", + dtype: str = "float32", + arena_bytes: int = 4096, + export_mode: str = "trained", +) -> str: + """Generate a reusable C++ class backed by CMSIS-DSP autodiff.""" + if dtype not in ("float32", "float16"): + raise ConversionError("dtype must be float32 or float16") + if arena_bytes <= 0: + raise ConversionError("arena_bytes must be positive") + if export_mode not in ("trained", "empty"): + raise ConversionError("export_mode must be trained or empty") + class_name = _identifier(class_name) + namespace = _identifier(namespace) + + parameter_name_set = {parameter.name for parameter in converted.parameters} + value_name_set = {value.name for value in converted.values} + + def view_name(name: str) -> str: + if name == converted.input.name: + return "input" + if name in parameter_name_set: + return name + if name in value_name_set: + return name + "_buffer" + raise AssertionError(name) + + kinds = {operation.kind for operation in converted.operations} + header_for = { + "add": "add", + "sub": "sub", + "multiply": "multiply", + "scale": "scale", + "offset": "offset", + "dot": "dot", + "fully_connected": "fully_connected", + "relu": "relu", + "softmax": "softmax", + "dropout": "dropout", + } + operator_for = { + "add": "AddOperator", + "sub": "SubOperator", + "multiply": "MultiplyOperator", + "scale": "ScaleOperator", + "offset": "OffsetOperator", + "dot": "DotOperator", + "fully_connected": "FullyConnectedOperator", + "relu": "ReluOperator", + "softmax": "SoftmaxOperator", + "dropout": "DropoutOperator", + } + includes = [] + if "softmax" in kinds: + # Include these at global scope before the C++ expression headers. + # Some architecture headers include the same C headers from their + # namespace; the include guards must already have established the C + # declarations globally. + includes.extend( + [ + "#include ", + "#include ", + ] + ) + includes.append("#include ") + includes.extend( + f"#include " + for kind in sorted(kinds) + ) + includes.extend(["#include ", "#include "]) + + scalar = "float" if dtype == "float32" else "float16_t" + lines = [ + "// Generated by pytorch_to_autodiff.py. Do not edit parameter data by hand.", + f"// Export mode: {export_mode}.", + "#pragma once", + *includes, + "", + f"namespace {namespace} {{", + "using namespace arm_cmsis_dsp::autodiff;", + "", + f"class {class_name}", + "{", + "public:", + f" using Scalar = {scalar};", + f" static constexpr std::size_t arena_bytes = {arena_bytes}U;", + f" static constexpr std::size_t input_length = {converted.input.length}U;", + "", + f" Scalar input_value[{converted.input.length}]{{}};", + ] + for parameter in converted.parameters: + if export_mode == "empty" and parameter.trainable: + lines.append( + f" Scalar {parameter.name}_value{_array_suffix(parameter.shape)}{{}};" + ) + else: + lines.extend( + [ + f" Scalar {parameter.name}_value{_array_suffix(parameter.shape)} =", + f" {_initializer(parameter)};", + ] + ) + for value in converted.values: + lines.append(f" Scalar {value.name}_value[{value.length}]{{}};") + if "dropout" in kinds: + lines.append(" DropoutGenerator dropout_generator{0x6D2B79F5U};") + lines.extend( + [ + "", + " Arena arena{};", + " Tape &tape;", + " BufferView input;", + ] + ) + for parameter in converted.parameters: + view_type = "MatrixView" if parameter.matrix_view else "BufferView" + lines.append(f" {view_type} {parameter.name};") + for value in converted.values: + lines.append(f" BufferView {view_name(value.name)};") + + initializers = ["tape(arena.tape())", "input(tape.input(input_value))"] + for parameter in converted.parameters: + if len(parameter.shape) == 2 and not parameter.matrix_view: + length = math.prod(parameter.shape) + initializers.append( + f"{parameter.name}(tape.parameter(&{parameter.name}_value[0][0], " + f"{length}U))" + ) + else: + initializers.append( + f"{parameter.name}(tape.parameter({parameter.name}_value))" + ) + initializers.extend( + f"{view_name(value.name)}(tape.output({value.name}_value))" + for value in converted.values + ) + lines.extend(["", f" {class_name}()", " : " + ",\n ".join(initializers), " {"]) + for kind in sorted(kinds): + lines.append(f" tape.register_operator<{operator_for[kind]}>();") + if export_mode == "empty" and any( + parameter.trainable for parameter in converted.parameters + ): + lines.append(" initialize_parameters();") + lines.append(" }") + + if export_mode == "empty": + lines.extend( + [ + "", + " void initialize_parameters(std::uint32_t seed = 0x12345678U)", + " {", + " std::uint32_t state = seed;", + ] + ) + for parameter in converted.parameters: + if not parameter.trainable: + continue + if len(parameter.shape) == 2: + rows, columns = parameter.shape + bound = math.sqrt(6.0 / (rows + columns)) + lines.extend( + [ + f" for (std::size_t row = 0; row < {rows}U; ++row)", + f" for (std::size_t column = 0; column < {columns}U; ++column)", + " {", + " state = state * 1664525U + 1013904223U;", + " const float unit = static_cast(", + " (state >> 8U) & 0xffffU) / 65535.0F;", + f" {parameter.name}_value[row][column] = static_cast(", + f" (unit * 2.0F - 1.0F) * {format(bound, '.9g')}F);", + " }", + ] + ) + else: + length = math.prod(parameter.shape) if parameter.shape else 1 + lines.extend( + [ + f" for (std::size_t i = 0; i < {length}U; ++i)", + f" {parameter.name}_value[i] = Scalar{{}};", + ] + ) + lines.append(" }") + + lines.extend(["", " bool forward()", " {"]) + for operation in converted.operations: + output = view_name(operation.output.name) + if operation.kind == "fully_connected": + source, weight, bias = operation.inputs + expression = ( + f"::arm_cmsis_dsp::autodiff::fully_connected(" + f"{view_name(source)}, {view_name(weight)}, {view_name(bias)})" + ) + elif operation.kind in ("relu", "softmax"): + expression = ( + f"::arm_cmsis_dsp::autodiff::{operation.kind}(" + f"{view_name(operation.inputs[0])})" + ) + elif operation.kind == "add": + expression = ( + f"{view_name(operation.inputs[0])} + " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind == "sub": + expression = ( + f"{view_name(operation.inputs[0])} - " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind == "multiply": + expression = ( + f"{view_name(operation.inputs[0])} * " + f"{view_name(operation.inputs[1])}" + ) + elif operation.kind in ("scale", "offset"): + expression = ( + f"::arm_cmsis_dsp::autodiff::{operation.kind}(" + f"{view_name(operation.inputs[0])}, " + f"{view_name(operation.inputs[1])})" + ) + elif operation.kind == "dot": + expression = ( + f"::arm_cmsis_dsp::autodiff::dot(" + f"{view_name(operation.inputs[0])}, " + f"{view_name(operation.inputs[1])})" + ) + elif operation.kind == "dropout": + probability = format(operation.attributes[0], ".9g") + expression = ( + f"::arm_cmsis_dsp::autodiff::dropout(" + f"{view_name(operation.inputs[0])}, dropout_generator, " + f"{probability}F)" + ) + else: + raise AssertionError(operation.kind) + lines.append(f" {output} = {expression};") + lines.extend( + [ + " return tape.good();", + " }", + "", + " BufferView output() const", + " {", + f" return {view_name(converted.output_name)};", + " }", + "", + " template ", + " bool add_parameters(Optimizer &optimizer)", + " {", + " bool result = true;", + ] + ) + for parameter in converted.parameters: + if parameter.trainable: + lines.append(f" result = optimizer.add({parameter.name}) && result;") + lines.extend( + [ + " return result;", + " }", + "};", + "", + f"}} // namespace {namespace}", + "", + ] + ) + return "\n".join(lines) + + +def convert_module(module: nn.Module, input_shape: Sequence[int], **options: Any) -> str: + """Validate ``module`` and return generated CMSIS-DSP autodiff C++.""" + return generate_cpp(inspect_module(module, input_shape), **options) + + +def _parse_shape(text: str) -> tuple[int, ...]: + if text.strip().lower() in ("", "scalar"): + return () + try: + return tuple(int(value) for value in re.split(r"[x,]", text)) + except ValueError as error: + raise argparse.ArgumentTypeError("shape must look like 4 or 3,4") from error + + +def _load_factory(path: Path, factory_name: str) -> nn.Module: + path = path.resolve() + spec = importlib.util.spec_from_file_location("pytorch_autodiff_input", path) + if spec is None or spec.loader is None: + raise ConversionError(f"cannot import model file '{path}'") + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(path.parent)) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + factory = getattr(module, factory_name, None) + if not callable(factory): + raise ConversionError(f"'{path}' has no callable '{factory_name}'") + model = factory() + if not isinstance(model, nn.Module): + raise ConversionError(f"'{factory_name}' must return torch.nn.Module") + return model + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", type=Path, help="Python file containing a model factory") + parser.add_argument("--factory", default="create_model", help="zero-argument model factory") + parser.add_argument( + "--input-shape", + required=True, + type=_parse_shape, + help="shape excluding the mandatory batch dimension, for example 4 or 3,4", + ) + parser.add_argument("-o", "--output", type=Path, help="output header (stdout if omitted)") + parser.add_argument("--class-name", default="GeneratedNetwork") + parser.add_argument("--namespace", default="generated_autodiff") + parser.add_argument("--dtype", choices=("float32", "float16"), default="float32") + parser.add_argument("--arena-bytes", type=int, default=4096) + parser.add_argument( + "--export-mode", + choices=("trained", "empty"), + default="trained", + help="embed trained values, or emit an empty randomly initialized network", + ) + arguments = parser.parse_args(argv) + try: + model = _load_factory(arguments.model, arguments.factory) + generated = convert_module( + model, + arguments.input_shape, + class_name=arguments.class_name, + namespace=arguments.namespace, + dtype=arguments.dtype, + arena_bytes=arguments.arena_bytes, + export_mode=arguments.export_mode, + ) + if arguments.output is None: + sys.stdout.write(generated) + else: + arguments.output.write_text(generated, encoding="utf-8") + except (ConversionError, OSError) as error: + parser.exit(1, f"error: {error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py b/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py new file mode 100644 index 000000000..85ee0f063 --- /dev/null +++ b/dsppp/Include/dsppp/autodiff/scripts/test_pytorch_to_autodiff.py @@ -0,0 +1,106 @@ +import unittest + +import torch +from torch import nn + +from pytorch_to_autodiff import ConversionError, convert_module + + +class SupportedModel(nn.Module): + def __init__(self): + super().__init__() + self.first = nn.Linear(4, 8) + self.relu = nn.ReLU() + self.second = nn.Linear(8, 3) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, value): + return self.softmax(self.second(self.relu(self.first(value)))) + + +class ArithmeticModel(nn.Module): + def __init__(self): + super().__init__() + self.scale = nn.Parameter(torch.tensor(2.0)) + self.offset = nn.Parameter(torch.tensor(-0.5)) + self.dot_weight = nn.Parameter(torch.ones(4)) + + def forward(self, value): + original = value + value = value + value + value = value - original + value = value * value + value = value * self.scale + value = value + self.offset + return torch.matmul(value, self.dot_weight) + + +class ScalarConstantModel(nn.Module): + def forward(self, value): + return value * 2.0 + 1.0 + + +class ConverterTest(unittest.TestCase): + def test_supported_network(self): + generated = convert_module(SupportedModel(), (4,), arena_bytes=2048) + self.assertIn("Export mode: trained", generated) + self.assertIn("fully_connected(input, first_weight, first_bias)", generated) + self.assertIn("softmax(linear_1_buffer)", generated) + self.assertIn("Arena", generated) + + def test_empty_export_does_not_embed_trainable_values(self): + model = nn.Linear(4, 3) + with torch.no_grad(): + model.weight.fill_(0.123456789) + model.bias.fill_(0.234567891) + generated = convert_module( + model, (4,), export_mode="empty" + ) + self.assertIn("weight_value[3][4]{}", generated) + self.assertIn("initialize_parameters();", generated) + self.assertIn("unit * 2.0F - 1.0F", generated) + self.assertNotIn("0.123456", generated) + self.assertNotIn("0.234567", generated) + + def test_arithmetic_scale_offset_and_dot(self): + generated = convert_module(ArithmeticModel(), (4,)) + self.assertIn("AddOperator", generated) + self.assertIn("SubOperator", generated) + self.assertIn("MultiplyOperator", generated) + self.assertIn("ScaleOperator", generated) + self.assertIn("OffsetOperator", generated) + self.assertIn("DotOperator", generated) + self.assertIn("dot(", generated) + + def test_scalar_constants_use_scale_and_offset(self): + generated = convert_module(ScalarConstantModel(), (4,)) + self.assertIn("ScaleOperator", generated) + self.assertIn("OffsetOperator", generated) + self.assertIn("static_cast(2.0F)", generated) + self.assertIn("static_cast(1.0F)", generated) + + def test_dropout_training_state(self): + training = nn.Dropout(0.2).train() + inference = nn.Dropout(0.2).eval() + self.assertIn("DropoutOperator", convert_module(training, (4,))) + self.assertIn("DropoutOperator", convert_module(inference, (4,))) + self.assertIn("dropout(input, dropout_generator, 0.2F)", + convert_module(inference, (4,))) + self.assertIn("DropoutOperator", + convert_module(nn.Dropout(0.0).eval(), (4,))) + + def test_unsupported_operator_stops_conversion(self): + with self.assertRaisesRegex(ConversionError, "unsupported ATen operator"): + convert_module(nn.Sequential(nn.Sigmoid()), (4,)) + + def test_higher_rank_input_stops_conversion(self): + with self.assertRaisesRegex(ConversionError, "only a scalar, vector, or matrix"): + convert_module(SupportedModel(), (2, 1, 4)) + + def test_linear_requires_vector(self): + with self.assertRaisesRegex(ConversionError, "requires vector input"): + convert_module(nn.Linear(4, 3), (2, 4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/dsppp/Include/dsppp/forward.hpp b/dsppp/Include/dsppp/forward.hpp index e9a17c2ec..f6f9c4b4b 100644 --- a/dsppp/Include/dsppp/forward.hpp +++ b/dsppp/Include/dsppp/forward.hpp @@ -10,6 +10,9 @@ struct Vector_Base; template struct VectorView; +template +struct VecRef; + template typename Allocator> struct Vector; @@ -21,6 +24,9 @@ struct Matrix; template struct MatrixView; +template +struct TransposeView; + template struct NbRows; diff --git a/dsppp/Include/dsppp/matrix.hpp b/dsppp/Include/dsppp/matrix.hpp index e299c3733..09217a3c3 100644 --- a/dsppp/Include/dsppp/matrix.hpp +++ b/dsppp/Include/dsppp/matrix.hpp @@ -25,6 +25,105 @@ namespace arm_cmsis_dsp { * @{ */ +/** + * @brief Zero-copy transposed view of a matrix. + * + * The view only swaps matrix indexing and dimensions. Algorithms can + * recognize this type and select a kernel which consumes the original + * row-major storage without materializing a transposed matrix. + * + * @tparam M Matrix datatype + */ +template +struct TransposeView +{ + using Scalar = typename traits::Scalar; + using Storage = typename VecRef::type; + + explicit TransposeView(const M &matrix) : matrix_(VecRef::ref(matrix)) {} + + vector_length_t rows() const { return matrix_.columns(); } + vector_length_t columns() const { return matrix_.rows(); } + + Scalar operator()(const index_t row, const index_t column) const + { + return matrix_(column, row); + } + + auto row(const index_t row) const { return matrix_.col(row); } + auto col(const index_t column) const { return matrix_.row(column); } + + const Storage &original() const { return matrix_; } + +private: + Storage matrix_; +}; + +/** + * @brief Create a zero-copy transposed matrix view. + * + * @tparam M Matrix datatype + * @param matrix Matrix whose dimensions and indexing are to be transposed + * @return A non-owning transposed view + */ +template::value, bool>::type = true> +inline TransposeView transpose_view(const M &matrix) +{ + return TransposeView(matrix); +} + +template +struct traits> +{ + using Scalar = typename traits::Scalar; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct HasMatrixIndexing> +{ + constexpr static bool value = true; +}; + +template +struct ElementType> +{ + using type = typename ElementType::type; +}; + +template +struct IsDynamic> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength> +{ + constexpr static vector_length_t value = StaticLength::value; +}; + +template +struct NbRows> +{ + constexpr static vector_length_t value = NbCols::value; +}; + +template +struct NbCols> +{ + constexpr static vector_length_t value = NbRows::value; +}; + +template +struct OutputVectorDim> +{ + constexpr static vector_length_t value = NbCols::value; +}; + template typename A> struct traits> @@ -455,6 +554,306 @@ struct VecRef,((R<0) || (C<0))> }; }; +/** Lazy matrix times vector expression. */ +template +struct _MatVec: _Expr<_MatVec> +{ + using MatrixScalar = typename traits::Scalar; + using VectorScalar = typename traits::Scalar; + using Scalar = typename MixedRes::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; + constexpr static int vector_lanes = vector_traits::nb_lanes; + constexpr static int block_size = vector_lanes > 4 ? vector_lanes : 4; +#else + constexpr static int block_size = 4; +#endif + + _MatVec(const M &matrix, const V &vector) + : matrix_(matrix), vector_(vector), cache_start_(matrix.rows()), + cache_count_(0) {} + + vector_length_t length() const { return matrix_.rows(); } + + Scalar operator[](const index_t row) const + { + load_block(row); + return cache_[row - cache_start_]; + } + +#if defined(HAS_VECTOR) + auto vector_op(const index_t row) const + { + load_block(row); + return inner::vload1<1>(cache_ + row - cache_start_); + } + + auto vector_op_tail(const index_t row, + const vector_length_t) const + { + return vector_op(row); + } +#endif + +private: + void load_block(const index_t row) const + { + if (row >= cache_start_ && row < cache_start_ + cache_count_) + return; + + cache_start_ = row - row % block_size; + const vector_length_t remaining = matrix_.rows() - cache_start_; + cache_count_ = remaining < block_size ? remaining : block_size; + for (index_t i = cache_count_; i < block_size; ++i) + cache_[i] = Scalar{}; + + _matvec_block(cache_, matrix_, vector_, cache_start_, cache_count_, + (::arm_cmsis_dsp::ARCH *)nullptr); + } + + M matrix_; + V vector_; + mutable Scalar cache_[block_size] = {}; + mutable index_t cache_start_; + mutable vector_length_t cache_count_; +}; + +template +struct traits<_MatVec> +{ + using Scalar = typename MixedRes::Scalar, + typename traits::Scalar>::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct ElementType<_MatVec> +{ + using type = typename MixedRes::Scalar, + typename traits::Scalar>::type; +}; + +template +struct IsVector<_MatVec> +{ + constexpr static bool value = true; +}; + +template +struct IsDynamic<_MatVec> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength<_MatVec> +{ + constexpr static vector_length_t value = NbRows::value; +}; + +template +struct Complexity<_MatVec> +{ + constexpr static int value = 1; +}; + +template +struct VecRef<_MatVec> +{ + using type = _MatVec; + static type ref(const type &expression) { return expression; } +}; + +/** + * @brief Create a lazy matrix times vector expression. + * + * Unlike dot(matrix, vector), matvec does not allocate or evaluate a result. + * It can therefore be fused with surrounding elementwise vector operations. + */ +template::value || + CompatibleDynamicMatVecProduct::value), + bool>::type = true> +inline auto matvec(const M &matrix, const V &vector) +{ + using MatrixRef = VecRef; + using VectorRef = VecRef; + return _MatVec( + MatrixRef::ref(matrix),VectorRef::ref(vector)); +} + +/** Lazy transposed-matrix times vector expression. */ +template +struct _TransposedMatVec: _Expr<_TransposedMatVec> +{ + using MatrixScalar = typename traits::Scalar; + using VectorScalar = typename traits::Scalar; + using Scalar = typename MixedRes::type; + using Accumulator = typename number_traits::accumulator; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif + + _TransposedMatVec(const TransposeView &matrix, const V &vector) + : matrix_(matrix), vector_(vector) {} + + vector_length_t length() const { return matrix_.rows(); } + + Scalar operator[](const index_t column) const + { + Accumulator sum{}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::mac(sum,original(row,column),vector_[row]); + return inner::from_accumulator(sum); + } + +#if defined(HAS_VECTOR) + auto vector_op(const index_t column) const + { + if constexpr (has_vector_inst() && has_vector_inst() && + same_nb_lanes() && is_float()) + { + using VectorAccumulator = + typename vector_traits::temp_accumulator; + VectorAccumulator sum = vector_traits::temp_acc_zero(); + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::vmacc(sum, + original.row(row).vector_op(column), + vector_[row]); + return sum; + } + else + { + constexpr int lanes = vector_traits::nb_lanes; + Accumulator sums[lanes] = {}; + Scalar values[lanes] = {}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + { + const VectorScalar value = vector_[row]; + for (index_t lane = 0; lane < lanes; ++lane) + sums[lane] = inner::mac( + sums[lane],original(row,column + lane),value); + } + for (index_t lane = 0; lane < lanes; ++lane) + values[lane] = inner::from_accumulator(sums[lane]); + return inner::vload1<1>(values); + } + } + + auto vector_op_tail(const index_t column, + const vector_length_t remaining) const + { + if constexpr (has_vector_inst() && has_vector_inst() && + same_nb_lanes() && is_float()) + { + using VectorAccumulator = + typename vector_traits::temp_accumulator; + VectorAccumulator sum = vector_traits::temp_acc_zero(); + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + sum = inner::vmacc( + sum, + original.row(row).vector_op_tail(column,remaining), + vector_[row]); + return sum; + } + else + { + constexpr int lanes = vector_traits::nb_lanes; + Accumulator sums[lanes] = {}; + Scalar values[lanes] = {}; + const auto &original = matrix_.original(); + for (index_t row = 0; row < original.rows(); ++row) + { + const VectorScalar value = vector_[row]; + for (index_t lane = 0; lane < remaining; ++lane) + sums[lane] = inner::mac( + sums[lane],original(row,column + lane),value); + } + for (index_t lane = 0; lane < remaining; ++lane) + values[lane] = inner::from_accumulator(sums[lane]); + return inner::vload1<1>(values); + } + } +#endif + +private: + TransposeView matrix_; + V vector_; +}; + +template +struct traits<_TransposedMatVec> +{ + using Scalar = typename MixedRes::Scalar, + typename traits::Scalar>::type; +#if defined(HAS_VECTOR) + using Vector = typename traits::Vector; +#endif +}; + +template +struct ElementType<_TransposedMatVec> +{ + using type = typename MixedRes::Scalar, + typename traits::Scalar>::type; +}; + +template +struct IsVector<_TransposedMatVec> +{ + constexpr static bool value = true; +}; + +template +struct IsDynamic<_TransposedMatVec> +{ + constexpr static bool value = IsDynamic::value; +}; + +template +struct StaticLength<_TransposedMatVec> +{ + constexpr static vector_length_t value = NbCols::value; +}; + +template +struct Complexity<_TransposedMatVec> +{ + constexpr static int value = 1; +}; + +template +struct VecRef<_TransposedMatVec> +{ + using type = _TransposedMatVec; + static type ref(const type &expression) { return expression; } +}; + +#if !defined(ARM_MATH_NEON) + +template,V>::value || + CompatibleDynamicMatVecProduct,V>::value), + bool>::type = true> +inline auto dot(const TransposeView &matrix, const V &vector) +{ + using VectorRef = VecRef; + return _TransposedMatVec( + matrix,VectorRef::ref(vector)); +} + +#endif + /***************** * diff --git a/dsppp/Include/dsppp/vec.hpp b/dsppp/Include/dsppp/vec.hpp index dab176ac4..94b65be69 100644 --- a/dsppp/Include/dsppp/vec.hpp +++ b/dsppp/Include/dsppp/vec.hpp @@ -23,11 +23,6 @@ namespace arm_cmsis_dsp { * @{ */ -template -struct VecRef; - - - template struct VecRef> { @@ -586,6 +581,57 @@ Core algorithms that cannot be expressed only with high level abstractions and need intrinsics. */ +namespace detail { + +/** Four-column fallback for a transposed matrix times vector product. */ +template +inline void dot_transposed_unrolled(RES &res, + const TransposeView &m, + const V &v) +{ + using TM = typename traits::Scalar; + using TV = typename traits::Scalar; + using T = typename MixedRes::type; + using Acc = typename number_traits::accumulator; + + const auto &original = m.original(); + const vector_length_t rows = original.rows(); + const vector_length_t columns = original.columns(); + vector_length_t column = 0; + + for (; column <= columns - 4; column += 4) + { + Acc sum0{}; + Acc sum1{}; + Acc sum2{}; + Acc sum3{}; + + for (index_t row = 0; row < rows; ++row) + { + const TV value = v[row]; + sum0 = inner::mac(sum0, original(row,column), value); + sum1 = inner::mac(sum1, original(row,column + 1), value); + sum2 = inner::mac(sum2, original(row,column + 2), value); + sum3 = inner::mac(sum3, original(row,column + 3), value); + } + + res[column] = inner::from_accumulator(sum0); + res[column + 1] = inner::from_accumulator(sum1); + res[column + 2] = inner::from_accumulator(sum2); + res[column + 3] = inner::from_accumulator(sum3); + } + + for (; column < columns; ++column) + { + Acc sum{}; + for (index_t row = 0; row < rows; ++row) + sum = inner::mac(sum, original(row,column), v[row]); + res[column] = inner::from_accumulator(sum); + } +} + +} // namespace detail + #include "Helium/matrix_multiply.hpp" #include "DSP/matrix_multiply.hpp" #include "Scalar/matrix_multiply.hpp" diff --git a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h index 2ff3eaa77..0e9746a7e 100644 --- a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h +++ b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. All rights reserved. + * Copyright (c) 2020-2024 Arm Limited. All rights reserved. * * Licensed under the Apache License Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,65 +31,65 @@ /* ARM MPS3 IO SCC */ #define MPS3_IO_S -#define MPS3_IO_DEV MPS3_IO_DEV_S +#define MPS3_IO_DEV MPS3_IO_DEV_S /* I2C_SBCon */ #define I2C0_SBCON_S -#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S +#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S /* I2S */ #define MPS3_I2S_S -#define MPS3_I2S_DEV MPS3_I2S_DEV_S +#define MPS3_I2S_DEV MPS3_I2S_DEV_S /* ARM UART Controller PL011 */ #define UART0_CMSDK_S -#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S +#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S #define UART1_CMSDK_S -#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S +#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S -#define DEFAULT_UART_BAUDRATE 115200U +#define DEFAULT_UART_BAUDRATE 115200U /* To be used as CODE and DATA sram */ #define MPC_ISRAM0_S -#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S +#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S #define MPC_ISRAM1_S -#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S +#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S #define MPC_SRAM_S -#define MPC_SRAM_DEV MPC_SRAM_DEV_S +#define MPC_SRAM_DEV MPC_SRAM_DEV_S #define MPC_QSPI_S -#define MPC_QSPI_DEV MPC_QSPI_DEV_S +#define MPC_QSPI_DEV MPC_QSPI_DEV_S /** System Counter Armv8-M */ #define SYSCOUNTER_CNTRL_ARMV8_M_S -#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S +#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S #define SYSCOUNTER_READ_ARMV8_M_S -#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S +#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S /** * Arbitrary scaling values for test purposes */ -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u +#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u /* System timer */ #define SYSTIMER0_ARMV8_M_S -#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S +#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S #define SYSTIMER1_ARMV8_M_S -#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S +#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S #define SYSTIMER2_ARMV8_M_S -#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S +#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S #define SYSTIMER3_ARMV8_M_S -#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S +#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S -#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) +#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) /* CMSDK GPIO driver structures */ #define GPIO0_CMSDK_S @@ -141,9 +141,8 @@ /* ARM SPI PL022 */ /* Invalid device stubs are not defined */ -#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ +#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ #define SPI1_PL022_S #define SPI1_PL022_DEV SPI1_PL022_DEV_S - -#endif /* __DEVICE_CFG_H__ */ +#endif /* __DEVICE_CFG_H__ */ diff --git a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 b/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 deleted file mode 100644 index 2ff3eaa77..000000000 --- a/dsppp/RTE/Device/SSE-300-MPS3/device_cfg.h.base@1.1.3 +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2020-2022 Arm Limited. All rights reserved. - * - * Licensed under the Apache License Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing software - * distributed under the License is distributed on an "AS IS" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __DEVICE_CFG_H__ -#define __DEVICE_CFG_H__ - -/** - * \file device_cfg.h - * \brief Configuration file native driver re-targeting - * - * \details This file can be used to add native driver specific macro - * definitions to select which peripherals are available in the build. - * - * This is a default device configuration file with all peripherals enabled. - */ - -/* Secure only peripheral configuration */ - -/* ARM MPS3 IO SCC */ -#define MPS3_IO_S -#define MPS3_IO_DEV MPS3_IO_DEV_S - -/* I2C_SBCon */ -#define I2C0_SBCON_S -#define I2C0_SBCON_DEV I2C0_SBCON_DEV_S - -/* I2S */ -#define MPS3_I2S_S -#define MPS3_I2S_DEV MPS3_I2S_DEV_S - -/* ARM UART Controller PL011 */ -#define UART0_CMSDK_S -#define UART0_CMSDK_DEV UART0_CMSDK_DEV_S -#define UART1_CMSDK_S -#define UART1_CMSDK_DEV UART1_CMSDK_DEV_S - -#define DEFAULT_UART_BAUDRATE 115200U - -/* To be used as CODE and DATA sram */ -#define MPC_ISRAM0_S -#define MPC_ISRAM0_DEV MPC_ISRAM0_DEV_S - -#define MPC_ISRAM1_S -#define MPC_ISRAM1_DEV MPC_ISRAM0_DEV_S - -#define MPC_SRAM_S -#define MPC_SRAM_DEV MPC_SRAM_DEV_S - -#define MPC_QSPI_S -#define MPC_QSPI_DEV MPC_QSPI_DEV_S - -/** System Counter Armv8-M */ -#define SYSCOUNTER_CNTRL_ARMV8_M_S -#define SYSCOUNTER_CNTRL_ARMV8_M_DEV SYSCOUNTER_CNTRL_ARMV8_M_DEV_S - -#define SYSCOUNTER_READ_ARMV8_M_S -#define SYSCOUNTER_READ_ARMV8_M_DEV SYSCOUNTER_READ_ARMV8_M_DEV_S -/** - * Arbitrary scaling values for test purposes - */ -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE0_FRACT 0u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_INT 1u -#define SYSCOUNTER_ARMV8_M_DEFAULT_SCALE1_FRACT 0u - -/* System timer */ -#define SYSTIMER0_ARMV8_M_S -#define SYSTIMER0_ARMV8_M_DEV SYSTIMER0_ARMV8_M_DEV_S -#define SYSTIMER1_ARMV8_M_S -#define SYSTIMER1_ARMV8_M_DEV SYSTIMER1_ARMV8_M_DEV_S -#define SYSTIMER2_ARMV8_M_S -#define SYSTIMER2_ARMV8_M_DEV SYSTIMER2_ARMV8_M_DEV_S -#define SYSTIMER3_ARMV8_M_S -#define SYSTIMER3_ARMV8_M_DEV SYSTIMER3_ARMV8_M_DEV_S - -#define SYSTIMER0_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER1_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER2_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) -#define SYSTIMER3_ARMV8M_DEFAULT_FREQ_HZ (25000000ul) - -/* CMSDK GPIO driver structures */ -#define GPIO0_CMSDK_S -#define GPIO0_CMSDK_DEV GPIO0_CMSDK_DEV_S -#define GPIO1_CMSDK_S -#define GPIO1_CMSDK_DEV GPIO1_CMSDK_DEV_S -#define GPIO2_CMSDK_S -#define GPIO2_CMSDK_DEV GPIO2_CMSDK_DEV_S -#define GPIO3_CMSDK_S -#define GPIO3_CMSDK_DEV GPIO3_CMSDK_DEV_S - -/* System Watchdogs */ -#define SYSWDOG_ARMV8_M_S -#define SYSWDOG_ARMV8_M_DEV SYSWDOG_ARMV8_M_DEV_S - -/* ARM MPC SIE 300 driver structures */ -#define MPC_VM0_S -#define MPC_VM0_DEV MPC_VM0_DEV_S -#define MPC_VM1_S -#define MPC_VM1_DEV MPC_VM1_DEV_S -#define MPC_SSRAM2_S -#define MPC_SSRAM2_DEV MPC_SSRAM2_DEV_S -#define MPC_SSRAM3_S -#define MPC_SSRAM3_DEV MPC_SSRAM3_DEV_S - -/* ARM PPC driver structures */ -#define PPC_SSE300_MAIN0_S -#define PPC_SSE300_MAIN0_DEV PPC_SSE300_MAIN0_DEV_S -#define PPC_SSE300_MAIN_EXP0_S -#define PPC_SSE300_MAIN_EXP0_DEV PPC_SSE300_MAIN_EXP0_DEV_S -#define PPC_SSE300_MAIN_EXP1_S -#define PPC_SSE300_MAIN_EXP1_DEV PPC_SSE300_MAIN_EXP1_DEV_S -#define PPC_SSE300_MAIN_EXP2_S -#define PPC_SSE300_MAIN_EXP2_DEV PPC_SSE300_MAIN_EXP2_DEV_S -#define PPC_SSE300_MAIN_EXP3_S -#define PPC_SSE300_MAIN_EXP3_DEV PPC_SSE300_MAIN_EXP3_DEV_S -#define PPC_SSE300_PERIPH0_S -#define PPC_SSE300_PERIPH0_DEV PPC_SSE300_PERIPH0_DEV_S -#define PPC_SSE300_PERIPH1_S -#define PPC_SSE300_PERIPH1_DEV PPC_SSE300_PERIPH1_DEV_S -#define PPC_SSE300_PERIPH_EXP0_S -#define PPC_SSE300_PERIPH_EXP0_DEV PPC_SSE300_PERIPH_EXP0_DEV_S -#define PPC_SSE300_PERIPH_EXP1_S -#define PPC_SSE300_PERIPH_EXP1_DEV PPC_SSE300_PERIPH_EXP1_DEV_S -#define PPC_SSE300_PERIPH_EXP2_S -#define PPC_SSE300_PERIPH_EXP2_DEV PPC_SSE300_PERIPH_EXP2_DEV_S -#define PPC_SSE300_PERIPH_EXP3_S -#define PPC_SSE300_PERIPH_EXP3_DEV PPC_SSE300_PERIPH_EXP3_DEV_S - -/* ARM SPI PL022 */ -/* Invalid device stubs are not defined */ -#define DEFAULT_SPI_SPEED_HZ 4000000U /* 4MHz */ -#define SPI1_PL022_S -#define SPI1_PL022_DEV SPI1_PL022_DEV_S - - -#endif /* __DEVICE_CFG_H__ */ diff --git a/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h b/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h index 5613b81ae..0ce63c7b0 100644 --- a/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h +++ b/dsppp/RTE/_Release_VHT-Corstone-300/RTE_Components.h @@ -1,8 +1,8 @@ /* * CSOLUTION generated file: DO NOT EDIT! - * Generated by: csolution version 2.10.0 + * Generated by: csolution version 2.14.1+p38-gf512b381 * - * Project: 'test.Release+VHT-Corstone-300' + * Project: 'example.Release+VHT-Corstone-300' * Target: 'Release+VHT-Corstone-300' */ @@ -15,6 +15,10 @@ */ #define CMSIS_device_header "SSE300MPS3.h" +/* ARM::Device:Native Driver:SysCounter@1.1.0 */ +#define RTE_SYSCOUNTER 1 +/* ARM::Device:Native Driver:Timeout@1.0.0 */ +#define RTE_TIMEOUT 1 #endif /* RTE_COMPONENTS_H */ diff --git a/dsppp/example.cproject.yml b/dsppp/example.cproject.yml index 0e41fef75..3018c1184 100644 --- a/dsppp/example.cproject.yml +++ b/dsppp/example.cproject.yml @@ -4,7 +4,11 @@ project: files: #- file: Examples/dot_product.cpp #- file: Examples/vector_op.cpp - - file: Examples/matrix_op.cpp + #- file: Examples/matrix_op.cpp + #- file: Examples/autodiff_regression.cpp + #- file: Examples/autodiff_lms.cpp + - file: Examples/autodiff_fully_connected_qat.cpp + #- file: Examples/autodiff_iris.cpp - file: clang_sse300.c for-context: - +MPS3-Corstone-300 @@ -16,7 +20,7 @@ project: components: - component: ARM::CMSIS:CORE - - component: ARM::CMSIS:DSP@1.15.0 + - component: ARM::CMSIS:DSP@1.17.1 - component: ARM::Device:Startup&C Startup for-context: - +VHT-Corstone-300 diff --git a/dsppp/main.c b/dsppp/main.c index 3a7cdcefd..a2d5a2320 100644 --- a/dsppp/main.c +++ b/dsppp/main.c @@ -58,6 +58,10 @@ int main(void) #if defined(DOT_TEST) dot_test(); #endif + #if defined(AUTODIFF_TEST) && \ + (defined(F32_DT) || defined(F16_DT)) && defined(DYNAMIC_TEST) + autodiff_test(); + #endif #if defined(VECTOR_TEST) vector_test(); #endif diff --git a/dsppp/run_all.py b/dsppp/run_all.py index 3d1d3abf4..c57afd265 100644 --- a/dsppp/run_all.py +++ b/dsppp/run_all.py @@ -181,6 +181,7 @@ def cmd_args(): AVHROOT = args.avh ALL_TESTS=["DOT_TEST", + "AUTODIFF_TEST", "VECTOR_TEST", "ROW_TEST", "COL_TEST", diff --git a/dsppp/test.cbuild-idx.yml b/dsppp/test.cbuild-idx.yml new file mode 100644 index 000000000..211b64606 --- /dev/null +++ b/dsppp/test.cbuild-idx.yml @@ -0,0 +1,23 @@ +build-idx: + generated-by: csolution version 2.14.1+p38-gf512b381 + cdefault: cdefault.yml + csolution: test.csolution.yml + cbuild-run: out/test+VHT-Corstone-300.cbuild-run.yml + tmpdir: tmp/VHT-Corstone-300/default + cprojects: + - cproject: tests/test.cproject.yml + clayers: + - clayer: ../Testing/cmsis_build/dsp.clayer.yml + - cproject: example.cproject.yml + cbuilds: + - cbuild: out/example/VHT-Corstone-300/Release/example.Release+VHT-Corstone-300.cbuild.yml + project: example + configuration: .Release+VHT-Corstone-300 + messages: + info: + - test.cbuild-pack.yml - file is already up-to-date + - test+VHT-Corstone-300.cbuild-run.yml - file is already up-to-date + - example.Release+VHT-Corstone-300.cbuild.yml - file is already up-to-date + packs-unused: + - pack: ARM::CMSIS-Compiler@2.2.0 + - pack: ARM::Cortex_DFP@1.2.0 diff --git a/dsppp/test.cbuild-pack.yml b/dsppp/test.cbuild-pack.yml index 4ffc655f0..c22651802 100644 --- a/dsppp/test.cbuild-pack.yml +++ b/dsppp/test.cbuild-pack.yml @@ -3,15 +3,31 @@ cbuild-pack: - resolved-pack: ARM::CMSIS@6.0.0 selected-by-pack: - ARM::CMSIS@6.0.0 + - resolved-pack: ARM::CMSIS@6.3.0 + selected-by-pack: + - ARM::CMSIS@6.3.0 - resolved-pack: ARM::CMSIS-Compiler@2.0.0 selected-by-pack: - ARM::CMSIS-Compiler@2.0.0 + - resolved-pack: ARM::CMSIS-Compiler@2.2.0 + selected-by-pack: + - ARM::CMSIS-Compiler@2.2.0 - resolved-pack: ARM::CMSIS-DSP@1.15.0 selected-by-pack: + - ARM::CMSIS-DSP - ARM::CMSIS-DSP@1.15.0 + - resolved-pack: ARM::CMSIS-DSP@1.17.1 + selected-by-pack: + - ARM::CMSIS-DSP@1.17.1 - resolved-pack: ARM::Cortex_DFP@1.0.0 selected-by-pack: - ARM::Cortex_DFP@1.0.0 + - resolved-pack: ARM::Cortex_DFP@1.2.0 + selected-by-pack: + - ARM::Cortex_DFP@1.2.0 - resolved-pack: ARM::V2M_MPS3_SSE_300_BSP@1.4.0 selected-by-pack: - ARM::V2M_MPS3_SSE_300_BSP@1.4.0 + - resolved-pack: ARM::V2M_MPS3_SSE_300_BSP@1.5.0 + selected-by-pack: + - ARM::V2M_MPS3_SSE_300_BSP@1.5.0 diff --git a/dsppp/test.csolution.yml b/dsppp/test.csolution.yml index 73f4e5e70..39d342a51 100644 --- a/dsppp/test.csolution.yml +++ b/dsppp/test.csolution.yml @@ -6,10 +6,11 @@ solution: cdefault: packs: - - pack: ARM::CMSIS@6.0.0 - - pack: ARM::V2M_MPS3_SSE_300_BSP@1.4.0 - - pack: ARM::CMSIS-Compiler@2.0.0 - - pack: ARM::Cortex_DFP@1.0.0 + - pack: ARM::CMSIS@6.3.0 + - pack: ARM::V2M_MPS3_SSE_300_BSP@1.5.0 + - pack: ARM::CMSIS-Compiler@2.2.0 + - pack: ARM::Cortex_DFP@1.2.0 + - pack: ARM::CMSIS-DSP@1.17.1 target-types: - type: MPS3-Corstone-300 @@ -29,7 +30,7 @@ solution: - -Wno-sign-compare - -Wno-unused-parameter Link: - - --specs=nosys.specs + - --specs=nosys.specs - for-compiler: CLANG C: - -Wno-sign-compare @@ -39,6 +40,10 @@ solution: - -Wno-unused-parameter Link: - -lcrt0 + target-set: + - set: + images: + - project-context: test.Release - type: VHT-Corstone-300 device: ARM::SSE-300-MPS3 @@ -57,6 +62,14 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: example.Release + debugger: + name: Arm-FVP + model: FVP_Corstone_SSE-300_Ethos-U55 + config-file: fvp_configs/VHT-Corstone-300.txt - type: VHT-M0P device: ARMCM0P @@ -76,6 +89,10 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: test.Release - type: VHT-M4 device: ARMCM4 @@ -95,13 +112,15 @@ solution: Link: - -lcrt0-semihost - -lsemihost + target-set: + - set: + images: + - project-context: test.Release build-types: - type: Release debug: on - projects: - project: ./tests/test.cproject.yml - project: ./example.cproject.yml - \ No newline at end of file diff --git a/dsppp/test_config.h b/dsppp/test_config.h index 0d8d3e34b..27e35bb1a 100644 --- a/dsppp/test_config.h +++ b/dsppp/test_config.h @@ -8,9 +8,9 @@ //#define ONLY_BENCHMARKS -#define DOT_TEST -#define COMPLEX_F32_DT -#define STATIC_TEST +#define AUTODIFF_TEST +#define F32_DT +#define DYNAMIC_TEST #endif diff --git a/dsppp/tests/.clangd b/dsppp/tests/.clangd new file mode 100644 index 000000000..1d4dc2e8c --- /dev/null +++ b/dsppp/tests/.clangd @@ -0,0 +1,18 @@ +CompileFlags: + CompilationDatabase: c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release + +--- +If: + PathMatch: .*\.(c|C|h)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release\compile_macros_c.h + +--- +If: + PathMatch: .*\.(cpp|c\+\+|C\+\+|cxx|cc|CC|hpp)$ +CompileFlags: + Add: + - -include + - c:\Users\chrfav01\benchresults\cmsis\CMSIS-DSP\dsppp\out\test\MPS3-Corstone-300\Release\compile_macros_cxx.h diff --git a/dsppp/tests/autodiff_test.cpp b/dsppp/tests/autodiff_test.cpp new file mode 100644 index 000000000..001a8e331 --- /dev/null +++ b/dsppp/tests/autodiff_test.cpp @@ -0,0 +1,815 @@ +#include "test_config.h" + +extern "C" { + extern void autodiff_test(); +} + +#if defined(AUTODIFF_TEST) && defined(DYNAMIC_TEST) && \ + (defined(F32_DT) || defined(F16_DT)) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace arm_cmsis_dsp::autodiff; + +// Board Release builds define NDEBUG, so the standard assert macro cannot be +// used as a test oracle. Keep the existing concise checks but make failures +// visible to run_all.py, which treats output containing "Error" as a failure. +#define AUTODIFF_CHECK(condition) \ + do \ + { \ + if (!(condition)) \ + { \ + std::printf("Error: autodiff check failed at line %u\r\n", \ + static_cast(__LINE__)); \ + } \ + } while (false) + +#ifdef assert +#undef assert +#endif +#define assert(condition) AUTODIFF_CHECK(condition) + +template +static bool close_to(T actual, float expected, float tolerance) noexcept +{ + const float difference = static_cast(actual) - expected; + return difference > -tolerance && difference < tolerance; +} + +template +static void test1() +{ + // Vector add followed by dot. Values and outputs belong to the caller; + // gradients and two fixed-size operation records use the tape arena. + Arena<2048, T> *buffer_arena = new Arena<2048, T>(); + Tape &buffer_tape = buffer_arena->tape(); + buffer_tape.template register_operator>(); + buffer_tape.template register_operator>(); + T x_value[] = {1.0F, 2.0F, 3.0F}; + T w_value[] = {4.0F, 5.0F, 6.0F}; + T sum_value[3] = {}; + T result_value[1] = {}; + + BufferView x_view = buffer_tape.view(x_value); + BufferView w_view = buffer_tape.view(w_value); + BufferView sum_view = buffer_tape.view(sum_value); + BufferView result_view = buffer_tape.view(result_value); + const std::size_t gradients_end = buffer_tape.used(); + assert(gradients_end >= 10U * sizeof(T)); + + + + { + RecordingScope no_gradient(buffer_tape, false); + sum_view = x_view + w_view; + result_view = dot(sum_view, w_view); + assert(result_value[0] == 109.0F); + } + assert(buffer_tape.used() == gradients_end); + + + sum_view = x_view + w_view; + result_view = dot(sum_view, w_view); + assert(buffer_tape.good()); + assert(buffer_tape.backward(result_view)); + for (std::size_t i = 0; i < 3; ++i) + { + assert(x_view.gradient(i) == w_value[i]); + assert(w_view.gradient(i) == x_value[i] + 2.0F * w_value[i]); + } + + + + // A vector output accepts a caller-provided vector-Jacobian seed. + const T sum_seed[] = {1.0F, 2.0F, 3.0F}; + assert(buffer_tape.backward(sum_view, sum_seed, 3)); + for (std::size_t i = 0; i < 3; ++i) + { + assert(x_view.gradient(i) == sum_seed[i]); + assert(w_view.gradient(i) == sum_seed[i]); + } + + delete buffer_arena; +} + +template +static void test2() +{ + + + // Only parameters receive final gradients. The x input has no gradient + // allocation, while scale and offset are trainable scalar parameters. + Arena<2048, T> *parameter_arena = new Arena<2048, T>(); + Tape ¶meter_tape = parameter_arena->tape(); + parameter_tape.template register_operator>(); + parameter_tape.template register_operator>(); + parameter_tape.template register_operator>(); + T input_value[] = {1.0F, 2.0F, 3.0F}; + T alpha_value = 2.0F; + T beta_value = 10.0F; + T scaled_value[3] = {}; + T added_value[3] = {}; + T loss_value[1] = {}; + + BufferView input_view = parameter_tape.input(input_value); + const std::size_t after_input = parameter_tape.used(); + BufferView alpha_view = parameter_tape.parameter(alpha_value); + BufferView beta_view = parameter_tape.parameter(beta_value); + BufferView scaled_view = parameter_tape.output(scaled_value); + BufferView added_view = parameter_tape.output(added_value); + BufferView loss_view = parameter_tape.output(loss_value); + + assert(after_input == 0U); + assert(!input_view.has_gradient()); + assert(alpha_view.role() == BufferRole::parameter); + assert(beta_view.role() == BufferRole::parameter); + + scaled_view = scale(input_view, alpha_view); + added_view = offset(scaled_view, beta_view); + loss_view = dot(added_view, input_view); + assert(loss_value[0] == 88.0F); + assert(parameter_tape.backward(loss_view)); + assert(alpha_view.gradient(0) == 14.0F); + assert(beta_view.gradient(0) == 6.0F); + for (std::size_t i = 0; i < 3; ++i) + assert(input_view.gradient(i) == 0.0F); + + delete parameter_arena; +} + +template +static void test3() +{ + + // Fully connected followed by ReLU. Only the positive first neuron + // contributes to the matrix and bias parameter gradients. + Arena<2048, T> *network_arena = new Arena<2048, T>(); + Tape &network_tape = network_arena->tape(); + network_tape.template register_operator>(); + network_tape.template register_operator>(); + T network_input_value[] = {2.0F, -1.0F}; + T matrix_value[2][2] = {{1.0F, 2.0F}, {-3.0F, 1.0F}}; + T bias_value[] = {1.0F, 0.0F}; + T linear_value[2] = {}; + T activation_value[2] = {}; + + BufferView network_input = network_tape.input(network_input_value); + MatrixView matrix = network_tape.parameter(matrix_value); + BufferView bias = network_tape.parameter(bias_value); + BufferView linear = network_tape.output(linear_value); + BufferView activation = network_tape.output(activation_value); + + linear = fully_connected(network_input, matrix, bias); + activation = relu(linear); + assert(linear_value[0] == 1.0F); + assert(linear_value[1] == -7.0F); + assert(activation_value[0] == 1.0F); + assert(activation_value[1] == 0.0F); + + const T activation_seed[] = {1.0F, 1.0F}; + assert(network_tape.backward(activation, activation_seed, 2)); + assert(matrix.gradient(0, 0) == 2.0F); + assert(matrix.gradient(0, 1) == -1.0F); + assert(matrix.gradient(1, 0) == 0.0F); + assert(matrix.gradient(1, 1) == 0.0F); + assert(bias.gradient(0) == 1.0F); + assert(bias.gradient(1) == 0.0F); + assert(!network_input.has_gradient()); + delete network_arena; + + +} + +template +static void test4() +{ + // ReLU uses a zero derivative at exactly zero. + Arena<512, T> *relu_arena = new Arena<512, T>(); + Tape &relu_tape = relu_arena->tape(); + relu_tape.template register_operator>(); + T relu_parameter_value[] = {-1.0F, 0.0F, 2.0F}; + T relu_output_value[3] = {}; + BufferView relu_parameter = relu_tape.parameter(relu_parameter_value); + BufferView relu_output = relu_tape.output(relu_output_value); + relu_output = relu(relu_parameter); + const T relu_seed[] = {1.0F, 1.0F, 1.0F}; + assert(relu_tape.backward(relu_output, relu_seed, 3)); + assert(relu_parameter.gradient(0) == 0.0F); + assert(relu_parameter.gradient(1) == 0.0F); + assert(relu_parameter.gradient(2) == 1.0F); + + delete relu_arena; +} + +template +static void test5() +{ + + // Softmax is normalized and its vector-Jacobian product has zero sum. + Arena<512, T> *softmax_arena = new Arena<512, T>(); + Tape &softmax_tape = softmax_arena->tape(); + softmax_tape.template register_operator>(); + T logits_value[] = {0.0F, 0.0F, 0.0F}; + T probability_value[3] = {}; + BufferView logits = softmax_tape.parameter(logits_value); + BufferView probability = softmax_tape.output(probability_value); + probability = softmax(logits); + const T probability_sum = static_cast( + static_cast(probability_value[0]) + + static_cast(probability_value[1]) + + static_cast(probability_value[2])); + const float probability_tolerance = + std::is_same::value ? 2.0e-4F : 2.0e-3F; + assert(close_to(probability_sum, 1.0F, probability_tolerance)); + for (std::size_t i = 0; i < 3U; ++i) + assert(close_to(probability_value[i], 1.0F / 3.0F, + probability_tolerance)); + const T softmax_seed[] = {1.0F, 2.0F, 3.0F}; + assert(softmax_tape.backward(probability, softmax_seed, 3U)); + assert(close_to(logits.gradient(0), -1.0F / 3.0F, + probability_tolerance)); + assert(close_to(logits.gradient(1), 0.0F, probability_tolerance)); + assert(close_to(logits.gradient(2), 1.0F / 3.0F, + probability_tolerance)); + delete softmax_arena; +} + +template +static void test6() +{ + + // Including an operator does not enable it. Evaluation fails until that + // operator type is explicitly registered on this tape. + Arena<256, T> *registry_arena = new Arena<256, T>(); + Tape ®istry_tape = registry_arena->tape(); + T registry_left_value[] = {1.0F}; + T registry_right_value[] = {2.0F}; + T registry_output_value[] = {0.0F}; + BufferView registry_left = registry_tape.input(registry_left_value); + BufferView registry_right = registry_tape.input(registry_right_value); + BufferView registry_output = registry_tape.output(registry_output_value); + registry_output = registry_left + registry_right; + assert(registry_tape.status() == Status::operator_not_registered); + assert(registry_output_value[0] == 0.0F); + registry_tape.clear_status(); + assert(registry_tape.template register_operator>()); + registry_output = registry_left + registry_right; + assert(registry_tape.good()); + assert(registry_output_value[0] == 3.0F); + delete registry_arena; +} + +template +static void test7() +{ + + // Quadratic loss, reusable graph records, Adam, and selective freezing. + Arena<1024, T> *training_arena = new Arena<1024, T>(); + Tape &training_tape = training_arena->tape(); + training_tape.template register_operator>(); + training_tape.template register_operator>(); + training_tape.template register_operator>(); + T feature_value[] = {2.0F}; + T coefficient_value[] = {3.0F}; + T bias_parameter_value[] = {1.0F}; + T dot_value[1] = {}; + T prediction_value[1] = {}; + T target_value[] = {0.0F}; + T training_loss_value[1] = {}; + BufferView feature = training_tape.input(feature_value); + BufferView coefficient = training_tape.parameter(coefficient_value); + BufferView bias_parameter = training_tape.parameter(bias_parameter_value); + BufferView dot_output = training_tape.output(dot_value); + BufferView prediction = training_tape.output(prediction_value); + BufferView target = training_tape.input(target_value); + BufferView loss = training_tape.output(training_loss_value); + Adam<2, 16U, T> adam(1.0e-2F); + assert(adam.add(coefficient)); + assert(adam.add(bias_parameter)); + assert(freeze_parameters(adam, coefficient)); + training_tape.begin_graph(); + const std::size_t persistent_training_bytes = training_tape.used(); + dot_output = dot(feature, coefficient); + prediction = dot_output + bias_parameter; + loss = quadratic_error(prediction, target); + adam.zero_grad(); + assert(training_tape.backward(loss)); + assert(training_loss_value[0] == 49.0F); + assert(coefficient.gradient(0) == 28.0F); + assert(bias_parameter.gradient(0) == 14.0F); + assert(adam.step()); + assert(coefficient_value[0] == 3.0F); // Frozen. + assert(bias_parameter_value[0] < 1.0F); + assert(training_tape.rewind_graph()); + assert(training_tape.used() == persistent_training_bytes); + assert(unfreeze_parameters(adam, coefficient)); + delete training_arena; +} + +template +static void test8() +{ + + // A single vector loss accumulates contributions from every sample into + // shared polynomial parameters before an optimizer step. + Arena<2048, T> *batch_arena = new Arena<2048, T>(); + Tape &batch_tape = batch_arena->tape(); + batch_tape.template register_operator>(); + batch_tape.template register_operator>(); + batch_tape.template register_operator>(); + T batch_feature_value[2][1] = {{1.0F}, {2.0F}}; + T batch_coefficient_value = 3.0F; + T batch_bias_value = 1.0F; + T batch_polynomial_value[2] = {}; + T batch_prediction_value[2] = {}; + T batch_target_value[2] = {}; + T batch_loss_value = 0.0F; + BufferView batch_coefficient = + batch_tape.parameter(batch_coefficient_value); + BufferView batch_bias = batch_tape.parameter(batch_bias_value); + BufferView batch_polynomial = + batch_tape.output(batch_polynomial_value); + BufferView batch_prediction = + batch_tape.output(batch_prediction_value); + BufferView batch_target = batch_tape.input(batch_target_value); + BufferView batch_loss = batch_tape.output(batch_loss_value); + for (std::size_t sample = 0; sample < 2U; ++sample) + { + BufferView batch_feature = + batch_tape.input(batch_feature_value[sample]); + BufferView batch_polynomial_element = batch_tape.output( + &batch_polynomial_value[sample], + &batch_polynomial.gradients()[sample], 1U); + BufferView batch_prediction_element = batch_tape.output( + &batch_prediction_value[sample], + &batch_prediction.gradients()[sample], 1U); + batch_polynomial_element = dot(batch_feature, batch_coefficient); + batch_prediction_element = batch_polynomial_element + batch_bias; + } + batch_loss = quadratic_error(batch_prediction, batch_target); + assert(batch_tape.backward(batch_loss)); + assert(batch_loss_value == 65.0F); // 4^2 + 7^2. + assert(batch_coefficient.gradient(0) == 36.0F); + assert(batch_bias.gradient(0) == 22.0F); + delete batch_arena; +} + +template +static void test9() +{ + // RMSProp uses the same parameter registration and freezing API. + Arena<128, T> *rms_arena = new Arena<128, T>(); + Tape &rms_tape = rms_arena->tape(); + T rms_value = 1.0F; + BufferView rms_parameter = rms_tape.parameter(rms_value); + RMSProp<1, 16U, T> rmsprop(1.0e-2F); + assert(rmsprop.add(rms_parameter)); + rms_parameter.gradients()[0] = 2.0F; + assert(rmsprop.step()); + assert(rms_value < 1.0F); + delete rms_arena; +} + +template +static void test10() +{ + // Subtraction and elementwise multiplication have data operands only. + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + tape.template register_operator>(); + T left_value[] = {2.0F, 4.0F, 6.0F}; + T left_gradient[3] = {}; + T right_value[] = {1.0F, 2.0F, 3.0F}; + T right_gradient[3] = {}; + T difference_value[3] = {}; + T product_value[3] = {}; + BufferView left = tape.view(left_value, left_gradient, 3U); + BufferView right = tape.view(right_value, right_gradient, 3U); + BufferView difference = tape.output(difference_value); + BufferView product = tape.output(product_value); + difference = left - right; + product = difference * right; + assert(product_value[0] == 1.0F); + assert(product_value[1] == 4.0F); + assert(product_value[2] == 9.0F); + const T seed[] = {1.0F, 1.0F, 1.0F}; + assert(tape.backward(product, seed, 3U)); + for (std::size_t i = 0; i < 3U; ++i) + { + assert(left.gradient(i) == right_value[i]); + assert(right.gradient(i) == 0.0F); + } +} + +template +static void test_transposed_dot_type() +{ + const T one = ::arm_cmsis_dsp::number_traits::one(); + T matrix_value[2][5] = {}; + for (std::size_t column = 0; column < 5; ++column) + matrix_value[0][column] = one; + T vector_value[2] = {one,T{}}; + + ::arm_cmsis_dsp::MatrixView matrix( + &matrix_value[0][0], 2, 5, 5); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 2); + ::arm_cmsis_dsp::Matrix + materialized_transpose(5,2); + ::arm_cmsis_dsp::transposeTo(materialized_transpose,matrix); + auto reference = ::arm_cmsis_dsp::dot(materialized_transpose,vector); + auto result = ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(matrix),vector); + for (std::size_t column = 0; column < 5; ++column) + { + if constexpr (std::is_same::value) + assert(static_cast(result[column]) == + static_cast(reference[column])); + else + assert(result[column] == reference[column]); + } +} + +template +static void test11() +{ + // A transposed view selects the fused matrix-vector kernel without + // materializing the transpose. Five columns exercise the MVE tail. + T matrix_value[3][5] = { + {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}, + {6.0F, 7.0F, 8.0F, 9.0F, 10.0F}, + {11.0F, 12.0F, 13.0F, 14.0F, 15.0F}}; + T vector_value[] = {2.0F, -1.0F, 0.5F}; + const T expected[] = {1.5F, 3.0F, 4.5F, 6.0F, 7.5F}; + + ::arm_cmsis_dsp::MatrixView matrix( + &matrix_value[0][0], 3, 5, 5); + ::arm_cmsis_dsp::VectorView vector(vector_value, 0, 3); + const auto transposed = ::arm_cmsis_dsp::transpose_view(matrix); + assert(transposed.rows() == 5); + assert(transposed.columns() == 3); + assert(transposed(4, 2) == 15.0F); + + auto result = ::arm_cmsis_dsp::dot(transposed, vector); + for (std::size_t column = 0; column < 5; ++column) + assert(result[column] == expected[column]); + + ::arm_cmsis_dsp::Matrix static_matrix; + ::arm_cmsis_dsp::Vector static_vector; + for (std::size_t row = 0; row < 2; ++row) + for (std::size_t column = 0; column < 5; ++column) + static_matrix(row,column) = 1.0F; + static_vector = static_cast(1.0F); + ::arm_cmsis_dsp::Vector static_result = + ::arm_cmsis_dsp::dot( + ::arm_cmsis_dsp::transpose_view(static_matrix),static_vector); + for (std::size_t column = 0; column < 5; ++column) + assert(static_result[column] == 2.0F); + + test_transposed_dot_type(); + + // Exercise the same kernel through the fully connected backward pass. + Arena<4096, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T input_value[5] = {}; + T input_gradient[5] = {}; + T weight_gradient[3][5] = {}; + T bias_value[3] = {}; + T bias_gradient[3] = {}; + T output_value[3] = {}; + T output_gradient[3] = {}; + BufferView input = tape.view(input_value, input_gradient, 5); + MatrixView weights = tape.parameter( + &matrix_value[0][0], &weight_gradient[0][0], 3, 5); + BufferView bias = tape.parameter(bias_value, bias_gradient, 3); + BufferView output = tape.view(output_value, output_gradient, 3); + output = fully_connected(input, weights, bias); + assert(tape.backward(output, vector_value, 3)); + for (std::size_t column = 0; column < 5; ++column) + assert(input.gradient(column) == expected[column]); +} + +template +static void test12() +{ + // Categorical cross entropy consumes probabilities and a one-hot target. + Arena<512, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T probability_value[] = {0.1F, 0.7F, 0.2F}; + T target_value[] = {0.0F, 1.0F, 0.0F}; + T loss_value = 0.0F; + BufferView probability = tape.parameter(probability_value); + BufferView target = tape.input(target_value); + BufferView loss = tape.output(loss_value); + loss = cross_entropy(probability, target); + const float loss_tolerance = + std::is_same::value ? 2.0e-4F : 3.0e-3F; + const float gradient_tolerance = + std::is_same::value ? 2.0e-4F : 1.0e-2F; + assert(close_to(loss_value, 0.35667494F, loss_tolerance)); + assert(tape.backward(loss)); + assert(probability.gradient(0) == 0.0F); + assert(close_to(probability.gradient(1), -1.42857143F, + gradient_tolerance)); + assert(probability.gradient(2) == 0.0F); +} + +template +static void test13() +{ + // Training applies inverted dropout and backward regenerates the same + // mask. Disabling recording makes dropout an identity for inference. + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T input_value[16]; + T output_value[16] = {}; + for (std::size_t i = 0; i < 16U; ++i) input_value[i] = 1.0F; + BufferView input = tape.parameter(input_value); + BufferView output = tape.output(output_value); + DropoutGenerator generator(1234U); + output = dropout(input, generator, 0.5F); + + unsigned dropped = 0U; + unsigned kept = 0U; + for (std::size_t i = 0; i < 16U; ++i) + { + assert(output_value[i] == 0.0F || output_value[i] == 2.0F); + output_value[i] == 0.0F ? ++dropped : ++kept; + } + assert(dropped != 0U && kept != 0U); + + T seed[16]; + for (std::size_t i = 0; i < 16U; ++i) seed[i] = 1.0F; + assert(tape.backward(output, seed, 16U)); + for (std::size_t i = 0; i < 16U; ++i) + assert(input.gradient(i) == output_value[i]); + + { + RecordingScope inference(tape, false); + output = dropout(input, generator, 0.5F); + for (std::size_t i = 0; i < 16U; ++i) + assert(output_value[i] == input_value[i]); + } +} + +template +static void test14() +{ + // Y = W X uses CMSIS-DSP matrix multiplication in the forward pass and + // computes only dW = dY X^T in the backward pass. + Arena<1024, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + T weight_value[2][3] = { + {1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}}; + T input_value[3][2] = { + {1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; + T output_value[2][2] = {}; + MatrixView weights = tape.parameter(weight_value); + BufferView input = tape.input(&input_value[0][0], 6U); + BufferView output = tape.output(&output_value[0][0], 4U); + output = matrix_multiply(input, weights); + assert(output_value[0][0] == 22.0F); + assert(output_value[0][1] == 28.0F); + assert(output_value[1][0] == 49.0F); + assert(output_value[1][1] == 64.0F); + + const T seed[] = {1.0F, 2.0F, 3.0F, 4.0F}; + assert(tape.backward(output, seed, 4U)); + const T expected_gradient[2][3] = { + {5.0F, 11.0F, 17.0F}, {11.0F, 25.0F, 39.0F}}; + for (std::size_t row = 0; row < 2U; ++row) + for (std::size_t column = 0; column < 3U; ++column) + assert(weights.gradient(row, column) == + expected_gradient[row][column]); + assert(!input.has_gradient()); +} + +template +static void test15() +{ + // SGD performs one fused parameter -= learning_rate * gradient update. + Arena<128, T> arena; + Tape &tape = arena.tape(); + T value[] = {1.0F, -2.0F}; + BufferView parameter = tape.parameter(value); + SGD<2, 1, T> optimizer(0.25F); + assert(optimizer.add(parameter)); + parameter.gradients()[0] = 2.0F; + parameter.gradients()[1] = -4.0F; + assert(optimizer.step()); + assert(value[0] == 0.5F); + assert(value[1] == -1.0F); + optimizer.zero_grad(); + assert(parameter.gradient(0) == 0.0F); + assert(parameter.gradient(1) == 0.0F); + assert(freeze_parameters(optimizer, parameter)); + parameter.gradients()[0] = 1.0F; + assert(optimizer.step()); + assert(value[0] == 0.5F); +} + +template +static void test16() +{ + // C++ extension primitives used by QAT: nearest-even rounding and + // range-masked accumulation/reduction. Half-way cases verify ties-to-even. + T rounding_input[] = {-2.5F, -1.5F, -0.5F, 0.5F, 1.5F, 2.5F}; + T rounding_output[6] = {}; + T rounding_accumulator[6] = {}; + T rounding_one[] = {1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 1.0F}; + ::arm_cmsis_dsp::VectorView rounding_input_view(rounding_input, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_output_view(rounding_output, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_accumulator_view( + rounding_accumulator, 0, 6U); + ::arm_cmsis_dsp::VectorView rounding_one_view(rounding_one, 0, 6U); + ::arm_cmsis_dsp::round_to_nearest(rounding_output_view, + rounding_input_view); + const float expected_rounding[] = {-2.0F, -2.0F, 0.0F, + 0.0F, 2.0F, 2.0F}; + for (std::size_t i = 0; i < 6U; ++i) + assert(static_cast(rounding_output[i]) == expected_rounding[i]); + const auto central = ::arm_cmsis_dsp::nearest_even_range_mask( + rounding_input_view, 1.0F, static_cast(0.0F), + static_cast(-1.0F), static_cast(1.0F)); + ::arm_cmsis_dsp::masked_scale_add( + rounding_accumulator_view, rounding_one_view, central, + static_cast(2.0F)); + const auto central_reduction = ::arm_cmsis_dsp::masked_dot_sum( + rounding_one_view, rounding_input_view, central); + for (std::size_t i = 0; i < 6U; ++i) + assert(static_cast(rounding_accumulator[i]) == + (i == 2U || i == 3U ? 2.0F : 0.0F)); + assert(static_cast(central_reduction.dot) == 0.0F); + assert(static_cast(central_reduction.sum) == 2.0F); + + // Q/DQ keeps float storage while reproducing LiteRT/CMSIS-NN signed int8 + // activation codes. Its combined backward pass is an STE in range and + // also exposes gradients for scale and zero-point learning. + Arena<2048, T> arena; + Tape &tape = arena.tape(); + tape.template register_operator>(); + tape.template register_operator>(); + T input_value[] = {-20.0F, -0.26F, 0.24F, 30.0F}; + T input_gradient[4] = {}; + T scale_value = 0.1F; + T zero_point_value = -3.0F; + T code_value[4] = {}; + T output_value[4] = {}; + BufferView input = tape.view(input_value, input_gradient, 4U); + BufferView scale_parameter = tape.parameter(scale_value); + BufferView zero_point_parameter = tape.parameter(zero_point_value); + BufferView code = tape.output(code_value); + BufferView output = tape.output(output_value); + code = quantize(input, scale_parameter, zero_point_parameter); + output = dequantize(code, scale_parameter, zero_point_parameter); + + const float expected_code[] = {-128.0F, -6.0F, -1.0F, 127.0F}; + const float expected_output[] = {-12.5F, -0.3F, 0.2F, 13.0F}; + const float tolerance = std::is_same::value ? 1.0e-5F : 2.0e-2F; + for (std::size_t i = 0; i < 4U; ++i) + { + assert(close_to(code_value[i], expected_code[i], tolerance)); + assert(close_to(output_value[i], expected_output[i], tolerance)); + } + + const T seed[] = {1.0F, 1.0F, 1.0F, 1.0F}; + assert(tape.backward(output, seed, 4U)); + assert(close_to(input.gradient(0), 0.0F, tolerance)); + assert(close_to(input.gradient(1), 1.0F, tolerance)); + assert(close_to(input.gradient(2), 1.0F, tolerance)); + assert(close_to(input.gradient(3), 0.0F, tolerance)); + assert(close_to(scale_parameter.gradient(0), 4.2F, + std::is_same::value ? 2.0e-4F : 8.0e-2F)); + assert(close_to(zero_point_parameter.gradient(0), -0.2F, + std::is_same::value ? 2.0e-4F : 8.0e-3F)); + + // Weight quantization follows the backend's symmetric per-output-axis + // contract: [-127, 127], zero-point zero, one scale per row here. + Arena<1536, T> weight_arena; + Tape &weight_tape = weight_arena.tape(); + weight_tape.template register_operator>(); + weight_tape.template register_operator>(); + T weight_value[] = {-1.2F, -0.2F, 0.6F, -2.0F, 0.8F, 2.6F}; + T weight_scale_value[] = {0.1F, 0.2F}; + T weight_zero_value[] = {4.0F, -4.0F}; + T weight_code_value[6] = {}; + T weight_output_value[6] = {}; + BufferView weight = weight_tape.input(weight_value); + BufferView weight_scale = weight_tape.parameter(weight_scale_value); + BufferView weight_zero = weight_tape.input(weight_zero_value); + BufferView weight_code = weight_tape.output(weight_code_value); + BufferView weight_output = weight_tape.output(weight_output_value); + const Int8Quantization weight_quantization = + Int8Quantization::weights(2U, 3U); + weight_code = quantize(weight, weight_scale, weight_zero, + weight_quantization); + weight_output = dequantize(weight_code, weight_scale, weight_zero, + weight_quantization); + const float expected_weight_code[] = {-12.0F, -2.0F, 6.0F, + -10.0F, 4.0F, 13.0F}; + for (std::size_t i = 0; i < 6U; ++i) + { + assert(close_to(weight_code_value[i], expected_weight_code[i], + tolerance)); + assert(close_to(weight_output_value[i], + static_cast(weight_value[i]), tolerance)); + } + assert(cmsis_nn_offset(-3.0F) == 3); + assert(static_cast(weight_zero_value[0]) == 0.0F); + assert(static_cast(weight_zero_value[1]) == 0.0F); + + // Q/DQ owns parameter projection; training loops do not need to clamp + // values after an optimizer step. + scale_value = -1.0F; + zero_point_value = 200.0F; + { + RecordingScope inference(tape, false); + code = quantize(input, scale_parameter, zero_point_parameter); + } + assert(static_cast(scale_value) > 0.0F); + assert(static_cast(zero_point_value) == 127.0F); +} + +template +static void run_autodiff_tests() +{ + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + test7(); + test8(); + test9(); + test10(); + test11(); + test12(); + test13(); + test14(); + test15(); + test16(); + + // Arena exhaustion is explicit and backward cannot return partial results. + alignas(std::max_align_t) unsigned char tiny_memory[1]; + Tape tiny(tiny_memory, sizeof(tiny_memory)); + tiny.template register_operator>(); + T tiny_input_value[1] = {2.0F}; + T tiny_input_gradient[1] = {}; + T tiny_output_value[1] = {}; + T tiny_output_gradient[1] = {}; + BufferView tiny_input = + tiny.view(tiny_input_value, tiny_input_gradient, 1); + BufferView tiny_output = + tiny.view(tiny_output_value, tiny_output_gradient, 1); + tiny_output = tiny_input + tiny_input; + assert(tiny_output_value[0] == 4.0F); + assert(tiny.status() == Status::out_of_memory); + assert(!tiny.backward(tiny_output)); + + + +} + +#undef assert +#undef AUTODIFF_CHECK + +#endif + +void autodiff_test() +{ +#if defined(AUTODIFF_TEST) && defined(DYNAMIC_TEST) +#if defined(F32_DT) + printf("Running float32 autodiff tests...\r\n"); + run_autodiff_tests(); +#elif defined(F16_DT) && defined(ARM_FLOAT16_SUPPORTED) + printf("Running float16 autodiff tests...\r\n"); + run_autodiff_tests(); +#endif +#endif +} diff --git a/dsppp/tests/matrix_test.cpp b/dsppp/tests/matrix_test.cpp index a42e12de4..34d04a27c 100644 --- a/dsppp/tests/matrix_test.cpp +++ b/dsppp/tests/matrix_test.cpp @@ -893,9 +893,13 @@ void testmatvec() startSectionNB(1); #if defined(STATIC_TEST) PVector res = dot(m,a); + PVector fused; #else PVector res = dot(m,a); + PVector fused(R); #endif + fused = number_traits::one(); + fused += matvec(m,a); stopSectionNB(1); STOP_CYCLE_MEASUREMENT; @@ -926,6 +930,15 @@ void testmatvec() { printf("matrix times vector failed \r\n"); } + for (index_t row = 0; row < R; ++row) + { + const T expected = res[row] + number_traits::one(); + if (fused[row] != expected) + { + printf("lazy matrix times vector accumulation failed \r\n"); + break; + } + } std::cout << "=====\r\n"; } @@ -2355,4 +2368,4 @@ void matrix_test() #endif #endif #endif -} \ No newline at end of file +} diff --git a/dsppp/tests/test.cproject.yml b/dsppp/tests/test.cproject.yml index 432af5ab7..67e42ee99 100644 --- a/dsppp/tests/test.cproject.yml +++ b/dsppp/tests/test.cproject.yml @@ -4,6 +4,7 @@ project: files: - file: matrix_test.cpp - file: dot_test.cpp + - file: autodiff_test.cpp - file: vector_test.cpp - file: row_test.cpp - file: col_test.cpp diff --git a/dsppp/tests/test.h b/dsppp/tests/test.h index 9273f47f0..3fe33c015 100644 --- a/dsppp/tests/test.h +++ b/dsppp/tests/test.h @@ -6,6 +6,7 @@ extern void matrix_test(void); extern void dot_test(void); +extern void autodiff_test(void); extern void vector_test(void); extern void row_test(void); extern void col_test(void);