Unroll NEON ApplyGateH output rows

Author: XIN XIE

lib/simulator_neon.h

@@ -346,27 +346,65 @@ class SimulatorNEON final : public SimulatorBase {
         is[k] = vld1q_f32(p0 + xss[k] + 4);
       }
 
-      uint64_t j = 0;
+      unsigned k = 0;
+
+      for (; k + 1 < hsize; k += 2) {
+        const fp_type* v0 = v + 2 * k * hsize;
+        const fp_type* v1 = v0 + 2 * hsize;
+
+        float32x4_t ru0 = vdupq_n_f32(v0[0]);
+        float32x4_t iu0 = vdupq_n_f32(v0[1]);
+        float32x4_t rn0 = vmulq_f32(rs[0], ru0);
+        float32x4_t in0 = vmulq_f32(rs[0], iu0);
+        rn0 = vfmsq_f32(rn0, is[0], iu0);
+        in0 = vfmaq_f32(in0, is[0], ru0);
+
+        float32x4_t ru1 = vdupq_n_f32(v1[0]);
+        float32x4_t iu1 = vdupq_n_f32(v1[1]);
+        float32x4_t rn1 = vmulq_f32(rs[0], ru1);
+        float32x4_t in1 = vmulq_f32(rs[0], iu1);
+        rn1 = vfmsq_f32(rn1, is[0], iu1);
+        in1 = vfmaq_f32(in1, is[0], ru1);
 
-      for (unsigned k = 0; k < hsize; ++k) {
-        float32x4_t ru = vdupq_n_f32(v[j]);
-        float32x4_t iu = vdupq_n_f32(v[j + 1]);
+        for (unsigned l = 1; l < hsize; ++l) {
+          ru0 = vdupq_n_f32(v0[2 * l]);
+          iu0 = vdupq_n_f32(v0[2 * l + 1]);
+          rn0 = vfmaq_f32(rn0, rs[l], ru0);
+          in0 = vfmaq_f32(in0, rs[l], iu0);
+          rn0 = vfmsq_f32(rn0, is[l], iu0);
+          in0 = vfmaq_f32(in0, is[l], ru0);
+
+          ru1 = vdupq_n_f32(v1[2 * l]);
+          iu1 = vdupq_n_f32(v1[2 * l + 1]);
+          rn1 = vfmaq_f32(rn1, rs[l], ru1);
+          in1 = vfmaq_f32(in1, rs[l], iu1);
+          rn1 = vfmsq_f32(rn1, is[l], iu1);
+          in1 = vfmaq_f32(in1, is[l], ru1);
+        }
+
+        vst1q_f32(p0 + xss[k], rn0);
+        vst1q_f32(p0 + xss[k] + 4, in0);
+        vst1q_f32(p0 + xss[k + 1], rn1);
+        vst1q_f32(p0 + xss[k + 1] + 4, in1);
+      }
+
+      for (; k < hsize; ++k) {
+        const fp_type* vk = v + 2 * k * hsize;
+
+        float32x4_t ru = vdupq_n_f32(vk[0]);
+        float32x4_t iu = vdupq_n_f32(vk[1]);
         float32x4_t rn = vmulq_f32(rs[0], ru);
         float32x4_t in = vmulq_f32(rs[0], iu);
         rn = vfmsq_f32(rn, is[0], iu);
         in = vfmaq_f32(in, is[0], ru);
 
-        j += 2;
-
         for (unsigned l = 1; l < hsize; ++l) {
-          ru = vdupq_n_f32(v[j]);
-          iu = vdupq_n_f32(v[j + 1]);
+          ru = vdupq_n_f32(vk[2 * l]);
+          iu = vdupq_n_f32(vk[2 * l + 1]);
           rn = vfmaq_f32(rn, rs[l], ru);
           in = vfmaq_f32(in, rs[l], iu);
           rn = vfmsq_f32(rn, is[l], iu);
           in = vfmaq_f32(in, is[l], ru);
-
-          j += 2;
         }
 
         vst1q_f32(p0 + xss[k], rn);