From: Wilco Dijkstra via Libc-alpha <libc-alpha@sourceware.org>
To: "naohirot@fujitsu.com" <naohirot@fujitsu.com>
Cc: 'GNU C Library' <libc-alpha@sourceware.org>
Subject: [PATCH v3 5/5] AArch64: Improve A64FX memset
Date: Thu, 22 Jul 2021 16:04:34 +0000 [thread overview]
Message-ID: <VE1PR08MB5599743405E22528D5F08D7283E49@VE1PR08MB5599.eurprd08.prod.outlook.com> (raw)
Simplify the code for memsets smaller than L1. Improve the unroll8 and L1_prefetch loops.
---
diff --git a/sysdeps/aarch64/multiarch/memset_a64fx.S b/sysdeps/aarch64/multiarch/memset_a64fx.S
index 8665c272431b46dadea53c63ab74829c3aa99312..36628e101db33a9a8ff5234b98dd5a3a5c9ed73c 100644
--- a/sysdeps/aarch64/multiarch/memset_a64fx.S
+++ b/sysdeps/aarch64/multiarch/memset_a64fx.S
@@ -30,7 +30,6 @@
#define L2_SIZE (8*1024*1024) // L2 8MB - 1MB
#define CACHE_LINE_SIZE 256
#define PF_DIST_L1 (CACHE_LINE_SIZE * 16) // Prefetch distance L1
-#define rest x2
#define vector_length x9
#if HAVE_AARCH64_SVE_ASM
@@ -89,29 +88,19 @@ ENTRY (MEMSET)
.p2align 4
L(vl_agnostic): // VL Agnostic
- mov rest, count
mov dst, dstin
- add dstend, dstin, count
- // if rest >= L2_SIZE && vector_length == 64 then L(L2)
- mov tmp1, 64
- cmp rest, L2_SIZE
- ccmp vector_length, tmp1, 0, cs
- b.eq L(L2)
- // if rest >= L1_SIZE && vector_length == 64 then L(L1_prefetch)
- cmp rest, L1_SIZE
- ccmp vector_length, tmp1, 0, cs
- b.eq L(L1_prefetch)
-
+ cmp count, L1_SIZE
+ b.hi L(L1_prefetch)
+ // count >= 8 * vector_length
L(unroll8):
- lsl tmp1, vector_length, 3
- .p2align 3
-1: cmp rest, tmp1
- b.cc L(last)
- st1b_unroll
+ sub count, count, tmp1
+ .p2align 4
+1: subs count, count, tmp1
+ st1b_unroll 0, 7
add dst, dst, tmp1
- sub rest, rest, tmp1
- b 1b
+ b.hi 1b
+ add count, count, tmp1
L(last):
cmp count, vector_length, lsl 1
@@ -129,18 +118,22 @@ L(last):
st1b z0.b, p0, [dstend, -1, mul vl]
ret
-L(L1_prefetch): // if rest >= L1_SIZE
+ // count >= L1_SIZE
.p2align 3
+L(L1_prefetch):
+ cmp count, L2_SIZE
+ b.hs L(L2)
+ cmp vector_length, 64
+ b.ne L(unroll8)
1: st1b_unroll 0, 3
prfm pstl1keep, [dst, PF_DIST_L1]
st1b_unroll 4, 7
prfm pstl1keep, [dst, PF_DIST_L1 + CACHE_LINE_SIZE]
add dst, dst, CACHE_LINE_SIZE * 2
- sub rest, rest, CACHE_LINE_SIZE * 2
- cmp rest, L1_SIZE
- b.ge 1b
- cbnz rest, L(unroll8)
- ret
+ sub count, count, CACHE_LINE_SIZE * 2
+ cmp count, PF_DIST_L1
+ b.hs 1b
+ b L(unroll8)
// count >= L2_SIZE
L(L2):
next reply other threads:[~2021-07-22 16:06 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-22 16:04 Wilco Dijkstra via Libc-alpha [this message]
2021-08-03 11:22 ` [PATCH v3 5/5] AArch64: Improve A64FX memset naohirot--- via Libc-alpha
2021-08-09 14:52 ` Wilco Dijkstra via Libc-alpha
2021-08-17 6:40 ` naohirot--- via Libc-alpha
2021-08-19 13:06 ` Wilco Dijkstra via Libc-alpha
2021-08-20 5:41 ` naohirot--- via Libc-alpha
2021-08-23 16:50 ` Wilco Dijkstra via Libc-alpha
2021-08-24 7:56 ` naohirot--- via Libc-alpha
2021-08-24 8:07 ` naohirot--- via Libc-alpha
2021-08-24 15:46 ` Wilco Dijkstra via Libc-alpha
2021-08-26 1:44 ` naohirot--- via Libc-alpha
2021-08-26 14:13 ` Wilco Dijkstra via Libc-alpha
2021-08-27 5:05 ` naohirot--- via Libc-alpha
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://www.gnu.org/software/libc/involved.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=VE1PR08MB5599743405E22528D5F08D7283E49@VE1PR08MB5599.eurprd08.prod.outlook.com \
--to=libc-alpha@sourceware.org \
--cc=Wilco.Dijkstra@arm.com \
--cc=naohirot@fujitsu.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).