Skip to content

Commit dc3aa2c

Browse files
authored
Fix non-SVE ARM64 potentially using non-zeroed register in SDOT/DDOT accumulation (#5918)
* Zero d0 register as it may not be identical with OUT, amend clobber list * Add sdot/ddot reproducer from issue 5917 as ARM64-specific utest
1 parent b338322 commit dc3aa2c

2 files changed

Lines changed: 33 additions & 5 deletions

File tree

kernel/arm64/dot_kernel_asimd.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -262,16 +262,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
262262

263263
static RETURN_TYPE dot_kernel_asimd(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
264264
{
265-
#ifndef DOUBLE
266-
volatile
267-
#endif
265+
268266
RETURN_TYPE dot = 0.0;
269267
BLASLONG j = 0;
270268

271269
__asm__ __volatile__ (
272270
" fmov "OUT", "REG0" \n"
271+
" fmov d0, xzr \n"
273272
" fmov d1, xzr \n"
274-
" fmov d2, xzr \n"
273+
" fmov d2, xzr \n"
275274
" fmov d3, xzr \n"
276275
" fmov d4, xzr \n"
277276
" fmov d5, xzr \n"
@@ -342,7 +341,10 @@ static RETURN_TYPE dot_kernel_asimd(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT
342341
[J_] "r" (j)
343342
: "cc",
344343
"memory",
345-
"d1", "d2", "d3", "d4", "d5", "d6", "d7"
344+
"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
345+
"v16", "v17", "v18", "v19", "v20", "v21", "v22",
346+
"v23", "v24", "v25", "v26", "v27", "v28", "v29",
347+
"v30", "v31"
346348
);
347349

348350
return dot;

utest/test_dsdot.c

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,29 @@ CTEST(dsdot,dsdot_n_1)
4848

4949
}
5050
#endif
51+
#ifdef ARMV8
52+
#if defined(BUILD_SINGLE)
53+
CTEST(sdot,sdot_n_1)
54+
{
55+
static float x[64], y[64];
56+
for (int i = 0; i < 64; i++) { x[i] = 1.0f; y[i] = 1.0f; }
57+
blasint n = 64, inc = 1;
58+
float junk[4] = {1e6f, 1e6f, 1e6f, 1e6f};
59+
__asm__ volatile("ld1 {v0.4s}, [%0]" :: "r"(junk) : "v0");
60+
float r = BLASFUNC(sdot)(&n, x, &inc, y, &inc);
61+
ASSERT_DBL_NEAR_TOL(64.,r, DOUBLE_EPS);
62+
}
63+
#endif
64+
#if defined(BUILD_DOUBLE)
65+
CTEST(ddot,ddot_n_1)
66+
{
67+
static double x[64], y[64];
68+
for (int i = 0; i < 64; i++) { x[i] = 1.0f; y[i] = 1.0f; }
69+
blasint n = 64, inc = 1;
70+
double junk[4] = {1e6f, 1e6f, 1e6f, 1e6f};
71+
__asm__ volatile("ld1 {v0.4s}, [%0]" :: "r"(junk) : "v0");
72+
double r = BLASFUNC(ddot)(&n, x, &inc, y, &inc);
73+
ASSERT_DBL_NEAR_TOL(64.,r, DOUBLE_EPS);
74+
}
75+
#endif
76+
#endif

0 commit comments

Comments
 (0)