|
| 1 | +// Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. |
| 2 | +// Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. |
| 3 | +// All Rights reserved. See files LICENSE and NOTICE for details. |
| 4 | +// |
| 5 | +// This file is part of CEED, a collection of benchmarks, miniapps, software |
| 6 | +// libraries and APIs for efficient high-order finite element and spectral |
| 7 | +// element discretizations for exascale applications. For more information and |
| 8 | +// source code availability see http://github.com/ceed. |
| 9 | +// |
| 10 | +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, |
| 11 | +// a collaborative effort of two U.S. Department of Energy organizations (Office |
| 12 | +// of Science and the National Nuclear Security Administration) responsible for |
| 13 | +// the planning and preparation of a capable exascale ecosystem, including |
| 14 | +// software, applications, hardware, advanced system engineering and early |
| 15 | +// testbed platforms, in support of the nation's exascale computing imperative. |
| 16 | + |
| 17 | +#include <ceed/ceed.h> |
| 18 | +#include <ceed/backend.h> |
| 19 | +#ifdef __ARM_FEATURE_SVE |
| 20 | +#include <arm_sve.h> |
| 21 | +#endif |
| 22 | +#include <stdbool.h> |
| 23 | +#include "ceed-sve.h" |
| 24 | + |
| 25 | +//------------------------------------------------------------------------------ |
| 26 | +// Blocked Tensor Contract |
| 27 | +//------------------------------------------------------------------------------ |
| 28 | +static inline int CeedTensorContract_Sve_Blocked(CeedTensorContract contract, |
| 29 | + CeedInt A, CeedInt B, CeedInt C, CeedInt J, const float *restrict t, |
| 30 | + CeedTransposeMode t_mode, const CeedInt add, const float *restrict u, |
| 31 | + float *restrict v, const CeedInt JJ) { |
| 32 | + CeedInt t_stride_0 = B, t_stride_1 = 1; |
| 33 | + if (t_mode == CEED_TRANSPOSE) { |
| 34 | + t_stride_0 = 1; t_stride_1 = J; |
| 35 | + } |
| 36 | + |
| 37 | + for (CeedInt a=0; a<A; a++) |
| 38 | + for (CeedInt b=0; b<B; b++) |
| 39 | + // Blocks of JJ rows |
| 40 | + for (CeedInt j=0; j<(J/JJ)*JJ; j+=JJ) |
| 41 | + for (CeedInt jj=0; jj<JJ; jj++) // unroll |
| 42 | + // C vectorization by compiler |
| 43 | + for (int32_t c=0; c<C; c+=svcntd()) { |
| 44 | + svbool_t pg = svwhilelt_b32(c, C); |
| 45 | + // Load u, v into vectors |
| 46 | + svfloat32_t u_vec = svld1(pg, &u[(a*B+b)*C+c]); |
| 47 | + svfloat32_t v_vec = svld1(pg, &v[(a*J+j+jj)*C+c]); |
| 48 | + // Basis matrix value |
| 49 | + float tq = t[(j+jj)*t_stride_0 + b*t_stride_1]; |
| 50 | + // fmadd |
| 51 | + svst1(pg, &v[(a*J+j+jj)*C+c], svmla_x(pg, v_vec, u_vec, tq)); |
| 52 | + } |
| 53 | + |
| 54 | + // Remainder of rows |
| 55 | + CeedInt j=(J/JJ)*JJ; |
| 56 | + if (j < J) |
| 57 | + for (CeedInt a=0; a<A; a++) |
| 58 | + for (CeedInt b=0; b<B; b++) |
| 59 | + // Blocks of JJ rows |
| 60 | + for (CeedInt jj=0; jj<J-j; jj++) // not unrolled |
| 61 | + // C vectorization by compiler |
| 62 | + for (int32_t c=0; c<C; c+=svcntd()) { |
| 63 | + svbool_t pg = svwhilelt_b32(c, C); |
| 64 | + // Load u, v into vectors |
| 65 | + svfloat32_t u_vec = svld1(pg, &u[(a*B+b)*C+c]); |
| 66 | + svfloat32_t v_vec = svld1(pg, &v[(a*J+j+jj)*C+c]); |
| 67 | + // Basis matrix value |
| 68 | + float tq = t[(j+jj)*t_stride_0 + b*t_stride_1]; |
| 69 | + // fmadd |
| 70 | + svst1(pg, &v[(a*J+j+jj)*C+c], svmla_x(pg, v_vec, u_vec, tq)); |
| 71 | + } |
| 72 | + |
| 73 | + return CEED_ERROR_SUCCESS; |
| 74 | +} |
| 75 | + |
| 76 | +//------------------------------------------------------------------------------ |
| 77 | +// Blocked Tensor Contract |
| 78 | +//------------------------------------------------------------------------------ |
| 79 | +static inline int CeedTensorContract_Sve_Serial(CeedTensorContract contract, |
| 80 | + CeedInt A, CeedInt B, CeedInt C, CeedInt J, const float *restrict t, |
| 81 | + CeedTransposeMode t_mode, const CeedInt add, const float *restrict u, |
| 82 | + float *restrict v, const CeedInt JJ) { |
| 83 | + CeedInt t_stride_0 = B, t_stride_1 = 1; |
| 84 | + if (t_mode == CEED_TRANSPOSE) { |
| 85 | + t_stride_0 = 1; t_stride_1 = J; |
| 86 | + } |
| 87 | + |
| 88 | + for (CeedInt a=0; a<A; a++) |
| 89 | + for (CeedInt b=0; b<B; b++) |
| 90 | + for (CeedInt j=0; j<(J/JJ)*JJ; j+=JJ) |
| 91 | + for (CeedInt jj=0; jj<JJ; jj++) // unroll |
| 92 | + v[a*J+(j+jj)] += t[(j+jj)*t_stride_0 + b*t_stride_1] * u[a*B+b]; |
| 93 | + |
| 94 | + CeedInt j=(J/JJ)*JJ; |
| 95 | + if (j < J) |
| 96 | + for (CeedInt a=0; a<A; a++) |
| 97 | + for (CeedInt b=0; b<B; b++) |
| 98 | + for (CeedInt jj=0; jj<J-j; jj++) // not unrolled |
| 99 | + v[a*J+(j+jj)] += t[(j+jj)*t_stride_0 + b*t_stride_1] * u[a*B+b]; |
| 100 | + |
| 101 | + return CEED_ERROR_SUCCESS; |
| 102 | +} |
| 103 | + |
| 104 | +//------------------------------------------------------------------------------ |
| 105 | +// Tensor Contract - Common Sizes |
| 106 | +//------------------------------------------------------------------------------ |
| 107 | +static int CeedTensorContract_Sve_Blocked_8(CeedTensorContract contract, |
| 108 | + CeedInt A, CeedInt B, CeedInt C, CeedInt J, const float *restrict t, |
| 109 | + CeedTransposeMode t_mode, const CeedInt add, const float *restrict u, |
| 110 | + float *restrict v) { |
| 111 | + return CeedTensorContract_Sve_Blocked(contract, A, B, C, J, t, t_mode, add, u, |
| 112 | + v, 8); |
| 113 | +} |
| 114 | +static int CeedTensorContract_Sve_Serial_8(CeedTensorContract contract, |
| 115 | + CeedInt A, CeedInt B, CeedInt C, CeedInt J, const float *restrict t, |
| 116 | + CeedTransposeMode t_mode, const CeedInt add, const float *restrict u, |
| 117 | + float *restrict v) { |
| 118 | + return CeedTensorContract_Sve_Serial(contract, A, B, C, J, t, t_mode, add, u, v, |
| 119 | + 8); |
| 120 | +} |
| 121 | + |
| 122 | +//------------------------------------------------------------------------------ |
| 123 | +// Tensor Contract Apply |
| 124 | +//------------------------------------------------------------------------------ |
| 125 | +static int CeedTensorContractApply_Sve(CeedTensorContract contract, CeedInt A, |
| 126 | + CeedInt B, CeedInt C, CeedInt J, |
| 127 | + const float *restrict t, |
| 128 | + CeedTransposeMode t_mode, |
| 129 | + const CeedInt add, |
| 130 | + const float *restrict u, |
| 131 | + float *restrict v) { |
| 132 | + if (!add) |
| 133 | + for (CeedInt q=0; q<A*J*C; q++) |
| 134 | + v[q] = (float) 0.0; |
| 135 | + |
| 136 | + if (C == 1) |
| 137 | + CeedTensorContract_Sve_Serial_8(contract, A, B, C, J, t, t_mode, true, u, v); |
| 138 | + else |
| 139 | + CeedTensorContract_Sve_Blocked_8(contract, A, B, C, J, t, t_mode, true, u, v); |
| 140 | + |
| 141 | + return CEED_ERROR_SUCCESS; |
| 142 | +} |
| 143 | + |
| 144 | +//------------------------------------------------------------------------------ |
| 145 | +// Tensor Contract Create |
| 146 | +//------------------------------------------------------------------------------ |
| 147 | +int CeedTensorContractCreate_f32_Sve(CeedBasis basis, |
| 148 | + CeedTensorContract contract) { |
| 149 | + int ierr; |
| 150 | + Ceed ceed; |
| 151 | + ierr = CeedTensorContractGetCeed(contract, &ceed); CeedChkBackend(ierr); |
| 152 | + |
| 153 | + ierr = CeedSetBackendFunction(ceed, "TensorContract", contract, "Apply", |
| 154 | + CeedTensorContractApply_Sve); CeedChkBackend(ierr); |
| 155 | + |
| 156 | + return CEED_ERROR_SUCCESS; |
| 157 | +} |
| 158 | +//------------------------------------------------------------------------------ |
0 commit comments