From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: AS3215 2.6.0.0/16 X-Spam-Status: No, score=-4.1 required=3.0 tests=AWL,BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,SPF_HELO_PASS,SPF_PASS shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from sourceware.org (server2.sourceware.org [IPv6:2620:52:3:1:0:246e:9693:128c]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by dcvr.yhbt.net (Postfix) with ESMTPS id A74CF1F601 for ; Wed, 7 Dec 2022 08:57:18 +0000 (UTC) Authentication-Results: dcvr.yhbt.net; dkim=pass (1024-bit key; secure) header.d=sourceware.org header.i=@sourceware.org header.b="NTFEbYUP"; dkim-atps=neutral Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id D86F439DDD9B for ; Wed, 7 Dec 2022 08:57:17 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org D86F439DDD9B DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1670403437; bh=UI9VFp9pUuiS2ICUTXskabQQNG5fcrV8256mIa6jmhA=; h=To:Cc:Subject:Date:In-Reply-To:References:List-Id: List-Unsubscribe:List-Archive:List-Post:List-Help:List-Subscribe: From:Reply-To:From; b=NTFEbYUPs8QPZuh6aryEpXkBMRxWfBIHqf+qMTwiZTymS4hGai3zP+bZnPbqLNg6y IRZMc8DbhFY2VpTGw4Wx+IbkAEz9Lbq3qFrdnpNXBMx1GF3IKHuyHR7eBZ+hkYGOoW CZaLeb/fShrgx8JHuPUgzBluKX93mcpW8+d6v1Ak= Received: from mail-ej1-x635.google.com (mail-ej1-x635.google.com [IPv6:2a00:1450:4864:20::635]) by sourceware.org (Postfix) with ESMTPS id 05E91390CE95 for ; Wed, 7 Dec 2022 08:53:16 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 05E91390CE95 Received: by mail-ej1-x635.google.com with SMTP id n21so12284163ejb.9 for ; Wed, 07 Dec 2022 00:53:15 -0800 (PST) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=UI9VFp9pUuiS2ICUTXskabQQNG5fcrV8256mIa6jmhA=; b=ymtQDWEk1gJX/URPBRF/z/U3N0G8AEq6YSYAAlwvo0L0UaSXrX/RDkjWDpdSWNyZR3 tZVXfcK0MKom5qqvJRnqTcIaERWdbchKwPdzgZtFD9nz6K3GNSKeV+PWLqgMNoRzE+SQ bHlchEM8FEX+BQ27HhhltQxRubd+YxJwFKULcBvRWdNMtBg0ImkZ7trDgOoDdxtowR9G 0S3x9XC0mdJBZWMFJHXA68UAnGqAQEodbbMgXVeZ8Smlz1VsDv5rvmisj6z7G8HZq4OW bVZuXrQ3L2fyomy6UzmPEI89kPer/NskV1mVWdkMrYbpRWcFzGToEzFgxo2vzM0537C6 Sm8w== X-Gm-Message-State: ANoB5plUtWvMbpTka5oVOpbt0eEsoMPv+49VIHGVatzApNSPjOHJRWqA cUnBS0TAfBasNJ03M2cjxAiWnxiSrJo= X-Google-Smtp-Source: AA0mqf5cMRe4lykoPZ0jScwXJLchiMtH7AEvNLvb9SC7crgnIBV102n+XBZznAwJjuOPr71VKmtubg== X-Received: by 2002:a17:906:65c4:b0:7ad:d250:b907 with SMTP id z4-20020a17090665c400b007add250b907mr75186559ejn.737.1670403195241; Wed, 07 Dec 2022 00:53:15 -0800 (PST) Received: from noahgold-desk.lan (2603-8080-1301-76c6-feb7-1b9b-f2dd-08f7.res6.spectrum.com. [2603:8080:1301:76c6:feb7:1b9b:f2dd:8f7]) by smtp.gmail.com with ESMTPSA id k17-20020aa7c051000000b0046bd3b366f9sm1931767edo.32.2022.12.07.00.53.13 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 07 Dec 2022 00:53:14 -0800 (PST) To: libc-alpha@sourceware.org Cc: goldstein.w.n@gmail.com, hjl.tools@gmail.com, andrey.kolesov@intel.com, carlos@systemhalted.org Subject: [PATCH v1 19/27] x86/fpu: Optimize svml_s_log10f4_core_sse4.S Date: Wed, 7 Dec 2022 00:52:28 -0800 Message-Id: <20221207085236.1424424-19-goldstein.w.n@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221207085236.1424424-1-goldstein.w.n@gmail.com> References: <20221207085236.1424424-1-goldstein.w.n@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: libc-alpha@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Libc-alpha mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: Noah Goldstein via Libc-alpha Reply-To: Noah Goldstein Errors-To: libc-alpha-bounces+e=80x24.org@sourceware.org Sender: "Libc-alpha" 1. Improve special values case which ends up covering ~half of all float bit patterns. 2. Cleanup some missed optimizations in instruction selection / unnecissary repeated rodata references. 3. Remove unused rodata. 4. Use common data definitions where possible. Code Size Change: -61 Bytes (279 - 340) Input New Time / Old Time 0F (0x00000000) -> 0.9395 0F (0x0000ffff, Denorm) -> 0.9729 .1F (0x3dcccccd) -> 0.9458 5F (0x40a00000) -> 0.9499 2315255808F (0x4f0a0000) -> 0.9437 -NaN (0xffffffff) -> 0.8284 --- .../fpu/multiarch/svml_s_log10f4_core_sse4.S | 319 +++++++----------- 1 file changed, 123 insertions(+), 196 deletions(-) diff --git a/sysdeps/x86_64/fpu/multiarch/svml_s_log10f4_core_sse4.S b/sysdeps/x86_64/fpu/multiarch/svml_s_log10f4_core_sse4.S index 58f54d62a3..faa0e79a24 100644 --- a/sysdeps/x86_64/fpu/multiarch/svml_s_log10f4_core_sse4.S +++ b/sysdeps/x86_64/fpu/multiarch/svml_s_log10f4_core_sse4.S @@ -27,216 +27,143 @@ * */ -/* Offsets for data table __svml_slog10_data_internal - */ -#define MinNorm 0 -#define MaxNorm 16 -#define L2H 32 -#define L2L 48 -#define iBrkValue 64 -#define iOffExpoMask 80 -#define One 96 -#define sPoly 112 -#define L2 256 +#define LOCAL_DATA_NAME __svml_slog10_data_internal +#include "svml_s_common_sse4_rodata_offsets.h" + +/* Offsets for data table __svml_slog10_data_internal. */ +#define _L2L 0 +#define _Coeff_9 16 +#define _Coeff_8 32 +#define _Coeff_7 48 +#define _Coeff_6 64 +#define _Coeff_5 80 +#define _Coeff_4 96 +#define _Coeff_3 112 +#define _Coeff_2 128 +#define _Coeff_1 144 +#define _L2H 160 #include .section .text.sse4, "ax", @progbits ENTRY(_ZGVbN4v_log10f_sse4) - subq $72, %rsp - cfi_def_cfa_offset(80) - movaps %xmm0, %xmm1 - - /* reduction: compute r, n */ - movdqu iBrkValue+__svml_slog10_data_internal(%rip), %xmm2 - movaps %xmm0, %xmm4 - movdqu iOffExpoMask+__svml_slog10_data_internal(%rip), %xmm10 - psubd %xmm2, %xmm1 - pand %xmm1, %xmm10 - psrad $23, %xmm1 - paddd %xmm2, %xmm10 + movdqu COMMON_DATA(_NotiOffExpoMask)(%rip), %xmm2 movaps %xmm0, %xmm3 - movups sPoly+__svml_slog10_data_internal(%rip), %xmm5 - movups sPoly+32+__svml_slog10_data_internal(%rip), %xmm6 - movups sPoly+64+__svml_slog10_data_internal(%rip), %xmm7 - movups sPoly+96+__svml_slog10_data_internal(%rip), %xmm9 - cvtdq2ps %xmm1, %xmm12 - cmpltps MinNorm+__svml_slog10_data_internal(%rip), %xmm4 - cmpnleps MaxNorm+__svml_slog10_data_internal(%rip), %xmm3 - subps One+__svml_slog10_data_internal(%rip), %xmm10 - mulps %xmm10, %xmm5 - movaps %xmm10, %xmm8 - mulps %xmm10, %xmm6 - mulps %xmm10, %xmm8 - addps sPoly+16+__svml_slog10_data_internal(%rip), %xmm5 - mulps %xmm10, %xmm7 - addps sPoly+48+__svml_slog10_data_internal(%rip), %xmm6 - mulps %xmm10, %xmm9 - mulps %xmm8, %xmm5 - addps sPoly+80+__svml_slog10_data_internal(%rip), %xmm7 - addps sPoly+112+__svml_slog10_data_internal(%rip), %xmm9 - addps %xmm5, %xmm6 - mulps %xmm8, %xmm6 - orps %xmm3, %xmm4 - - /* combine and get argument value range mask */ - movmskps %xmm4, %edx - movups L2L+__svml_slog10_data_internal(%rip), %xmm1 - addps %xmm6, %xmm7 - mulps %xmm12, %xmm1 - mulps %xmm7, %xmm8 - movups L2H+__svml_slog10_data_internal(%rip), %xmm11 - addps %xmm8, %xmm9 - mulps %xmm11, %xmm12 - mulps %xmm10, %xmm9 - addps sPoly+128+__svml_slog10_data_internal(%rip), %xmm9 - mulps %xmm9, %xmm10 - addps %xmm10, %xmm1 - addps %xmm12, %xmm1 - testl %edx, %edx - - /* Go to special inputs processing branch */ + psubd %xmm2, %xmm0 + movaps COMMON_DATA(_ILoRange)(%rip), %xmm4 + pcmpgtd %xmm0, %xmm4 + /* combine and get argument value range mask. */ + movmskps %xmm4, %eax + movups LOCAL_DATA(_L2L)(%rip), %xmm0 + /* reduction: compute r, n. */ + movdqu COMMON_DATA(_IBrkValue)(%rip), %xmm4 + movaps %xmm3, %xmm6 + psubd %xmm4, %xmm3 + pandn %xmm3, %xmm2 + paddd %xmm4, %xmm2 + subps COMMON_DATA(_OneF)(%rip), %xmm2 + psrad $0x17, %xmm3 + cvtdq2ps %xmm3, %xmm4 + mulps %xmm4, %xmm0 + movaps %xmm2, %xmm3 + mulps %xmm2, %xmm2 + movups LOCAL_DATA(_Coeff_9)(%rip), %xmm1 + mulps %xmm3, %xmm1 + addps LOCAL_DATA(_Coeff_8)(%rip), %xmm1 + mulps %xmm2, %xmm1 + movups LOCAL_DATA(_Coeff_7)(%rip), %xmm5 + mulps %xmm3, %xmm5 + addps LOCAL_DATA(_Coeff_6)(%rip), %xmm5 + addps %xmm1, %xmm5 + mulps %xmm2, %xmm5 + movups LOCAL_DATA(_Coeff_5)(%rip), %xmm1 + mulps %xmm3, %xmm1 + addps LOCAL_DATA(_Coeff_4)(%rip), %xmm1 + addps %xmm5, %xmm1 + mulps %xmm1, %xmm2 + movups LOCAL_DATA(_Coeff_3)(%rip), %xmm1 + mulps %xmm3, %xmm1 + addps LOCAL_DATA(_Coeff_2)(%rip), %xmm1 + addps %xmm2, %xmm1 + mulps %xmm3, %xmm1 + addps LOCAL_DATA(_Coeff_1)(%rip), %xmm1 + mulps %xmm1, %xmm3 + addps %xmm3, %xmm0 + movups LOCAL_DATA(_L2H)(%rip), %xmm2 + mulps %xmm4, %xmm2 + addps %xmm2, %xmm0 + testl %eax, %eax + /* Go to special inputs processing branch. */ jne L(SPECIAL_VALUES_BRANCH) - # LOE rbx rbp r12 r13 r14 r15 edx xmm0 xmm1 - - /* Restore registers - * and exit the function - */ - -L(EXIT): - movaps %xmm1, %xmm0 - addq $72, %rsp - cfi_def_cfa_offset(8) ret - cfi_def_cfa_offset(80) - - /* Branch to process - * special inputs - */ + /* Cold case. edx has 1s where there was a special value that + more so than speed here. */ L(SPECIAL_VALUES_BRANCH): - movups %xmm0, 32(%rsp) - movups %xmm1, 48(%rsp) - # LOE rbx rbp r12 r13 r14 r15 edx - - xorl %eax, %eax - movq %r12, 16(%rsp) - cfi_offset(12, -64) - movl %eax, %r12d - movq %r13, 8(%rsp) - cfi_offset(13, -72) - movl %edx, %r13d - movq %r14, (%rsp) - cfi_offset(14, -80) - # LOE rbx rbp r15 r12d r13d - - /* Range mask - * bits check - */ - -L(RANGEMASK_CHECK): - btl %r12d, %r13d - - /* Call scalar math function */ - jc L(SCALAR_MATH_CALL) - # LOE rbx rbp r15 r12d r13d - - /* Special inputs - * processing loop - */ - + /* Stack coming in 16-byte aligned. Set 8-byte misaligned so on + call entry will be 16-byte aligned. */ + + subq $0x38, %rsp + movups %xmm0, 24(%rsp) + movups %xmm6, 40(%rsp) + + /* Use rbx/rbp for callee save registers as they get short + encoding for many instructions (as compared with r12/r13). */ + movq %rbx, (%rsp) + cfi_offset (rbx, -64) + movq %rbp, 8(%rsp) + cfi_offset (rbp, -56) + /* edx has 1s where there was a special value that needs to be + handled by a tanhf call. */ + movl %eax, %ebx L(SPECIAL_VALUES_LOOP): - incl %r12d - cmpl $4, %r12d - - /* Check bits in range mask */ - jl L(RANGEMASK_CHECK) - # LOE rbx rbp r15 r12d r13d - - movq 16(%rsp), %r12 - cfi_restore(12) - movq 8(%rsp), %r13 - cfi_restore(13) - movq (%rsp), %r14 - cfi_restore(14) - movups 48(%rsp), %xmm1 - - /* Go to exit */ - jmp L(EXIT) - cfi_offset(12, -64) - cfi_offset(13, -72) - cfi_offset(14, -80) - # LOE rbx rbp r12 r13 r14 r15 xmm1 - - /* Scalar math fucntion call - * to process special input - */ - -L(SCALAR_MATH_CALL): - movl %r12d, %r14d - movss 32(%rsp, %r14, 4), %xmm0 - call log10f@PLT - # LOE rbx rbp r14 r15 r12d r13d xmm0 - movss %xmm0, 48(%rsp, %r14, 4) + /* use rbp as index for special value that is saved across calls + to tanhf. We technically don't need a callee save register + here as offset to rsp is always [0, 12] so we can restore + rsp by realigning to 64. Essentially the tradeoff is 1 extra + save/restore vs 2 extra instructions in the loop. */ + xorl %ebp, %ebp + bsfl %ebx, %ebp + + /* Scalar math fucntion call to process special input. */ + movss 40(%rsp, %rbp, 4), %xmm0 + call log10f@PLT - /* Process special inputs in loop */ - jmp L(SPECIAL_VALUES_LOOP) - # LOE rbx rbp r15 r12d r13d + /* No good way to avoid the store-forwarding fault this will + cause on return. `lfence` avoids the SF fault but at greater + cost as it serialized stack/callee save restoration. */ + movss %xmm0, 24(%rsp, %rbp, 4) + + leal -1(%rbx), %eax + andl %eax, %ebx + jnz L(SPECIAL_VALUES_LOOP) + + /* All results have been written to 24(%rsp). */ + movups 24(%rsp), %xmm0 + movq (%rsp), %rbx + cfi_restore (rbx) + movq 8(%rsp), %rbp + cfi_restore (rbp) + addq $56, %rsp + cfi_def_cfa_offset (8) + ret END(_ZGVbN4v_log10f_sse4) - .section .rodata, "a" + .section .rodata.sse4, "a" .align 16 -#ifdef __svml_slog10_data_internal_typedef -typedef unsigned int VUINT32; -typedef struct { - __declspec(align(16)) VUINT32 MinNorm[4][1]; - __declspec(align(16)) VUINT32 MaxNorm[4][1]; - __declspec(align(16)) VUINT32 L2H[4][1]; - __declspec(align(16)) VUINT32 L2L[4][1]; - __declspec(align(16)) VUINT32 iBrkValue[4][1]; - __declspec(align(16)) VUINT32 iOffExpoMask[4][1]; - __declspec(align(16)) VUINT32 One[4][1]; - __declspec(align(16)) VUINT32 sPoly[9][4][1]; - __declspec(align(16)) VUINT32 L2[4][1]; -} __svml_slog10_data_internal; -#endif -__svml_slog10_data_internal: - /* MinNorm */ - .long 0x00800000, 0x00800000, 0x00800000, 0x00800000 - /* MaxNorm */ - .align 16 - .long 0x7f7fffff, 0x7f7fffff, 0x7f7fffff, 0x7f7fffff - /* L2H */ - .align 16 - .long 0x3e9a2100, 0x3e9a2100, 0x3e9a2100, 0x3e9a2100 - /* L2L */ - .align 16 - .long 0xb64AF600, 0xb64AF600, 0xb64AF600, 0xb64AF600 - /* iBrkValue = SP 2/3 */ - .align 16 - .long 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab, 0x3f2aaaab - /* iOffExpoMask = SP significand mask */ - .align 16 - .long 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff - /* sOne = SP 1.0 */ - .align 16 - .long 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 - /* spoly[9] */ - .align 16 - .long 0x3d8063B4, 0x3d8063B4, 0x3d8063B4, 0x3d8063B4 /* coeff9 */ - .long 0xbd890073, 0xbd890073, 0xbd890073, 0xbd890073 /* coeff8 */ - .long 0x3d775317, 0x3d775317, 0x3d775317, 0x3d775317 /* coeff7 */ - .long 0xbd91FB27, 0xbd91FB27, 0xbd91FB27, 0xbd91FB27 /* coeff6 */ - .long 0x3dB20B96, 0x3dB20B96, 0x3dB20B96, 0x3dB20B96 /* coeff5 */ - .long 0xbdDE6E20, 0xbdDE6E20, 0xbdDE6E20, 0xbdDE6E20 /* coeff4 */ - .long 0x3e143CE5, 0x3e143CE5, 0x3e143CE5, 0x3e143CE5 /* coeff3 */ - .long 0xbe5E5BC5, 0xbe5E5BC5, 0xbe5E5BC5, 0xbe5E5BC5 /* coeff2 */ - .long 0x3eDE5BD9, 0x3eDE5BD9, 0x3eDE5BD9, 0x3eDE5BD9 /* coeff1 */ - /* L2 */ - .align 16 - .long 0x3e9a209b, 0x3e9a209b, 0x3e9a209b, 0x3e9a209b - .align 16 - .type __svml_slog10_data_internal, @object - .size __svml_slog10_data_internal, .-__svml_slog10_data_internal +LOCAL_DATA_NAME: + DATA_VEC (LOCAL_DATA_NAME, _L2L, 0xb64af600) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_9, 0x3d8063b4) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_8, 0xbd890073) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_7, 0x3d775317) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_6, 0xbd91fb27) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_5, 0x3db20b96) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_4, 0xbdde6e20) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_3, 0x3e143ce5) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_2, 0xbe5e5bc5) + DATA_VEC (LOCAL_DATA_NAME, _Coeff_1, 0x3ede5bd9) + DATA_VEC (LOCAL_DATA_NAME, _L2H, 0x3e9a2100) + .type LOCAL_DATA_NAME, @object + .size LOCAL_DATA_NAME, .-LOCAL_DATA_NAME -- 2.34.1