From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: AS3215 2.6.0.0/16 X-Spam-Status: No, score=-4.2 required=3.0 tests=AWL,BAYES_00,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED,SPF_HELO_PASS,SPF_PASS shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from sourceware.org (server2.sourceware.org [IPv6:2620:52:3:1:0:246e:9693:128c]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) by dcvr.yhbt.net (Postfix) with ESMTPS id C06EA1F5AE for ; Fri, 23 Apr 2021 17:03:23 +0000 (UTC) Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id 37B18393BC3A; Fri, 23 Apr 2021 17:03:22 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 37B18393BC3A DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=sourceware.org; s=default; t=1619197402; bh=2HG1EgbEpeqFgYfdiNPdV444D6GdEJzVUPuxpR/8GRo=; h=Date:To:Subject:References:In-Reply-To:List-Id:List-Unsubscribe: List-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:Cc: From; b=Nkmn/yXF861P1SDW2eo1wKT5hYZl4QcXbz1Nyizmm7cDkSuvLg5Cj/1LKgf83dwDB wDUUsHdtoEpI9ttIf3G5HOJEuTNZXboLbHfCLrNByfA7tj/c89CcCOSUrSkCWzRZ/Z KdQoNF0m3kK+gEH2WG+uEHXxUGFe/zzpTzFp7L5o= Received: from mail-pj1-x1033.google.com (mail-pj1-x1033.google.com [IPv6:2607:f8b0:4864:20::1033]) by sourceware.org (Postfix) with ESMTPS id 29AE23854835 for ; Fri, 23 Apr 2021 17:03:18 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.3.2 sourceware.org 29AE23854835 Received: by mail-pj1-x1033.google.com with SMTP id e8-20020a17090a7288b029014e51f5a6baso1529048pjg.2 for ; Fri, 23 Apr 2021 10:03:18 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:date:from:to:cc:subject:message-id:references :mime-version:content-disposition:in-reply-to; bh=2HG1EgbEpeqFgYfdiNPdV444D6GdEJzVUPuxpR/8GRo=; b=CZiEr93lDpGCvjVoerBnOd/c8h8uZ26f6v26XG0RF1CigGeAvselsWrWli6lI4TYht 6gBF7gcxqbYI+seYIuMZ97FrVtYBMBg2/QfpsHuCJ3CIYulmJ2dWmiEZKpG9FRs15t9L cEzTtiLnautm/WlqjMNJPMzxpvd+o1ngmXG0MPeSVJYOL4z+V2tu5cTdAhCOG80sgvy3 GxK0w55k7ETndive9LHnsP0upfccg+Hpfkn3bja1pa+3yRq5VbSmpquopv51MqdWVesJ uOOx5jIxNmQm2fVBpFeR4l1c4FxSZhX+j/J84KJx47RjB0msLCydZvMD7m9zljW7tQ29 JO5w== X-Gm-Message-State: AOAM532KQEEdpR2V3ikb+jfMZTZCOZJ1pGOTliJx4UIXm2iAy+a+oqof lmr/pO4JUJsn0p0kU43kjyc= X-Google-Smtp-Source: ABdhPJwIn0xfxgQ2hbKDe96EUmKV3qkzzQB9z2IVhGEDSv06nP+KPvBaTjTXdXt3kBAjIMY20/Dveg== X-Received: by 2002:a17:90a:a4c9:: with SMTP id l9mr6694639pjw.21.1619197396946; Fri, 23 Apr 2021 10:03:16 -0700 (PDT) Received: from gnu-cfl-2.localdomain ([2607:fb90:a62d:2d5e:38da:e56e:0:c66]) by smtp.gmail.com with ESMTPSA id o3sm5007543pfk.203.2021.04.23.10.03.16 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 23 Apr 2021 10:03:16 -0700 (PDT) Received: by gnu-cfl-2.localdomain (Postfix, from userid 1000) id 1DC561A044D; Fri, 23 Apr 2021 10:03:15 -0700 (PDT) Date: Fri, 23 Apr 2021 10:03:15 -0700 To: Noah Goldstein Subject: Re: [PATCH v1 2/2] x86: Optimize strchr-evex.S Message-ID: References: <20210421213951.404588-1-goldstein.w.n@gmail.com> <20210421213951.404588-2-goldstein.w.n@gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20210421213951.404588-2-goldstein.w.n@gmail.com> X-BeenThere: libc-alpha@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Libc-alpha mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , From: "H.J. Lu via Libc-alpha" Reply-To: "H.J. Lu" Cc: libc-alpha@sourceware.org Errors-To: libc-alpha-bounces@sourceware.org Sender: "Libc-alpha" On Wed, Apr 21, 2021 at 05:39:53PM -0400, Noah Goldstein wrote: > No bug. This commit optimizes strlen-evex.S. The optimizations are > mostly small things such as save an ALU in the alignment process, > saving a few instructions in the loop return. The one significant > change is saving 2 instructions in the 4x loop. test-strchr, > test-strchrnul, test-wcschr, and test-wcschrnul are all passing. > > Signed-off-by: Noah Goldstein > --- > sysdeps/x86_64/multiarch/strchr-evex.S | 388 ++++++++++++++----------- > 1 file changed, 214 insertions(+), 174 deletions(-) > > diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S > index ddc86a7058..7cd111e96c 100644 > --- a/sysdeps/x86_64/multiarch/strchr-evex.S > +++ b/sysdeps/x86_64/multiarch/strchr-evex.S > @@ -24,23 +24,26 @@ > # define STRCHR __strchr_evex > # endif > > -# define VMOVU vmovdqu64 > -# define VMOVA vmovdqa64 > +# define VMOVU vmovdqu64 > +# define VMOVA vmovdqa64 These changes aren't needed. > > # ifdef USE_AS_WCSCHR > # define VPBROADCAST vpbroadcastd > # define VPCMP vpcmpd > # define VPMINU vpminud > # define CHAR_REG esi > -# define SHIFT_REG r8d > +# define SHIFT_REG ecx > +# define CHAR_SIZE 4 > # else > # define VPBROADCAST vpbroadcastb > # define VPCMP vpcmpb > # define VPMINU vpminub > # define CHAR_REG sil > -# define SHIFT_REG ecx > +# define SHIFT_REG edx > +# define CHAR_SIZE 1 > # endif > > + No need to add a blank line here. > # define XMMZERO xmm16 > > # define YMMZERO ymm16 > @@ -56,23 +59,20 @@ > > # define VEC_SIZE 32 > # define PAGE_SIZE 4096 > +# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) > > .section .text.evex,"ax",@progbits > ENTRY (STRCHR) > - movl %edi, %ecx > -# ifndef USE_AS_STRCHRNUL > - xorl %edx, %edx > -# endif > - > /* Broadcast CHAR to YMM0. */ > - VPBROADCAST %esi, %YMM0 > - > + VPBROADCAST %esi, %YMM0 > + movl %edi, %eax > + andl $(PAGE_SIZE - 1), %eax > vpxorq %XMMZERO, %XMMZERO, %XMMZERO > > - /* Check if we cross page boundary with one vector load. */ > - andl $(PAGE_SIZE - 1), %ecx > - cmpl $(PAGE_SIZE - VEC_SIZE), %ecx > - ja L(cross_page_boundary) > + /* Check if we cross page boundary with one vector load. Otherwise > + it is safe to use an unaligned load. */ > + cmpl $(PAGE_SIZE - VEC_SIZE), %eax > + ja L(cross_page_boundary) > > /* Check the first VEC_SIZE bytes. Search for both CHAR and the > null bytes. */ > @@ -83,251 +83,291 @@ ENTRY (STRCHR) > VPMINU %YMM2, %YMM1, %YMM2 > /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > VPCMP $0, %YMMZERO, %YMM2, %k0 > - ktestd %k0, %k0 > - jz L(more_vecs) > kmovd %k0, %eax > + testl %eax, %eax > + jz L(aligned_more) > tzcntl %eax, %eax > - /* Found CHAR or the null byte. */ > # ifdef USE_AS_WCSCHR > /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (%rdi, %rax, 4), %rax > + leaq (%rdi, %rax, CHAR_SIZE), %rax > # else > addq %rdi, %rax > # endif > # ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + /* Found CHAR or the null byte. */ > + cmp (%rax), %CHAR_REG > + jne L(zero) > # endif > ret > > - .p2align 4 > -L(more_vecs): > - /* Align data for aligned loads in the loop. */ > - andq $-VEC_SIZE, %rdi > -L(aligned_more): > - > - /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time > - since data is only aligned to VEC_SIZE. */ > - VMOVA VEC_SIZE(%rdi), %YMM1 > - addq $VEC_SIZE, %rdi > - > - /* Leaves only CHARS matching esi as 0. */ > - vpxorq %YMM1, %YMM0, %YMM2 > - VPMINU %YMM2, %YMM1, %YMM2 > - /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > - VPCMP $0, %YMMZERO, %YMM2, %k0 > - kmovd %k0, %eax > - testl %eax, %eax > - jnz L(first_vec_x0) > - > - VMOVA VEC_SIZE(%rdi), %YMM1 > - /* Leaves only CHARS matching esi as 0. */ > - vpxorq %YMM1, %YMM0, %YMM2 > - VPMINU %YMM2, %YMM1, %YMM2 > - /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > - VPCMP $0, %YMMZERO, %YMM2, %k0 > - kmovd %k0, %eax > - testl %eax, %eax > - jnz L(first_vec_x1) > - > - VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 > - /* Leaves only CHARS matching esi as 0. */ > - vpxorq %YMM1, %YMM0, %YMM2 > - VPMINU %YMM2, %YMM1, %YMM2 > - /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > - VPCMP $0, %YMMZERO, %YMM2, %k0 > - kmovd %k0, %eax > - testl %eax, %eax > - jnz L(first_vec_x2) > - > - VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 > - /* Leaves only CHARS matching esi as 0. */ > - vpxorq %YMM1, %YMM0, %YMM2 > - VPMINU %YMM2, %YMM1, %YMM2 > - /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > - VPCMP $0, %YMMZERO, %YMM2, %k0 > - ktestd %k0, %k0 > - jz L(prep_loop_4x) > - > - kmovd %k0, %eax > + /* .p2align 5 helps keep performance more consistent if ENTRY() > + alignment % 32 was either 16 or 0. As well this makes the > + alignment % 32 of the loop_4x_vec fixed which makes tuning it > + easier. */ > + .p2align 5 > +L(first_vec_x3): > tzcntl %eax, %eax > +# ifndef USE_AS_STRCHRNUL > /* Found CHAR or the null byte. */ > -# ifdef USE_AS_WCSCHR > - /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax > -# else > - leaq (VEC_SIZE * 3)(%rdi, %rax), %rax > + cmp (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero) > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax > + ret > + > # ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > -# endif > +L(zero): > + xorl %eax, %eax > ret > +# endif > > .p2align 4 > -L(first_vec_x0): > +L(first_vec_x4): > +# ifndef USE_AS_STRCHRNUL > + /* Check to see if first match was CHAR (k0) or null (k1). */ > + kmovd %k0, %eax > tzcntl %eax, %eax > - /* Found CHAR or the null byte. */ > -# ifdef USE_AS_WCSCHR > - /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (%rdi, %rax, 4), %rax > + kmovd %k1, %ecx > + /* bzhil will not be 0 if first match was null. */ > + bzhil %eax, %ecx, %ecx > + jne L(zero) > # else > - addq %rdi, %rax > -# endif > -# ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + /* Combine CHAR and null matches. */ > + kord %k0, %k1, %k0 > + kmovd %k0, %eax > + tzcntl %eax, %eax > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax > ret > > .p2align 4 > L(first_vec_x1): > tzcntl %eax, %eax > - /* Found CHAR or the null byte. */ > -# ifdef USE_AS_WCSCHR > - /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq VEC_SIZE(%rdi, %rax, 4), %rax > -# else > - leaq VEC_SIZE(%rdi, %rax), %rax > -# endif > # ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + /* Found CHAR or the null byte. */ > + cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero) > + > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax > ret > > .p2align 4 > L(first_vec_x2): > +# ifndef USE_AS_STRCHRNUL > + /* Check to see if first match was CHAR (k0) or null (k1). */ > + kmovd %k0, %eax > tzcntl %eax, %eax > - /* Found CHAR or the null byte. */ > -# ifdef USE_AS_WCSCHR > - /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax > + kmovd %k1, %ecx > + /* bzhil will not be 0 if first match was null. */ > + bzhil %eax, %ecx, %ecx > + jne L(zero) > # else > - leaq (VEC_SIZE * 2)(%rdi, %rax), %rax > -# endif > -# ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + /* Combine CHAR and null matches. */ > + kord %k0, %k1, %k0 > + kmovd %k0, %eax > + tzcntl %eax, %eax > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax > ret > > -L(prep_loop_4x): > - /* Align data to 4 * VEC_SIZE. */ > + .p2align 4 > +L(aligned_more): > + /* Align data to VEC_SIZE. */ > + andq $-VEC_SIZE, %rdi > +L(cross_page_continue): > + /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time since > + data is only aligned to VEC_SIZE. Use two alternating methods for > + checking VEC to balance latency and port contention. */ Please limit lines to 72 columns. > + > + /* This method has higher latency but has better port > + distribution. */ > + VMOVA (VEC_SIZE)(%rdi), %YMM1 > + /* Leaves only CHARS matching esi as 0. */ > + vpxorq %YMM1, %YMM0, %YMM2 > + VPMINU %YMM2, %YMM1, %YMM2 > + /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > + VPCMP $0, %YMMZERO, %YMM2, %k0 > + kmovd %k0, %eax > + testl %eax, %eax > + jnz L(first_vec_x1) > + > + /* This method has higher latency but has better port > + distribution. */ > + VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 > + /* Each bit in K0 represents a CHAR in YMM1. */ > + VPCMP $0, %YMM1, %YMM0, %k0 > + /* Each bit in K1 represents a CHAR in YMM1. */ > + VPCMP $0, %YMM1, %YMMZERO, %k1 > + kortestd %k0, %k1 > + jnz L(first_vec_x2) > + > + VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 > + /* Leaves only CHARS matching esi as 0. */ > + vpxorq %YMM1, %YMM0, %YMM2 > + VPMINU %YMM2, %YMM1, %YMM2 > + /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > + VPCMP $0, %YMMZERO, %YMM2, %k0 > + kmovd %k0, %eax > + testl %eax, %eax > + jnz L(first_vec_x3) > + > + VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 > + /* Each bit in K0 represents a CHAR in YMM1. */ > + VPCMP $0, %YMM1, %YMM0, %k0 > + /* Each bit in K1 represents a CHAR in YMM1. */ > + VPCMP $0, %YMM1, %YMMZERO, %k1 > + kortestd %k0, %k1 > + jnz L(first_vec_x4) > + > + /* Align data to VEC_SIZE * 4 for the loop. */ > + addq $VEC_SIZE, %rdi > andq $-(VEC_SIZE * 4), %rdi > > .p2align 4 > L(loop_4x_vec): > - /* Compare 4 * VEC at a time forward. */ > + /* Check 4x VEC at a time. No penalty to imm32 offset with evex > + encoding. */ > VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 > VMOVA (VEC_SIZE * 5)(%rdi), %YMM2 > VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 > VMOVA (VEC_SIZE * 7)(%rdi), %YMM4 > > - /* Leaves only CHARS matching esi as 0. */ > + /* For YMM1 and YMM3 use xor to set the CHARs matching esi to zero. */ Please limit lines to 72 columns. > vpxorq %YMM1, %YMM0, %YMM5 > - vpxorq %YMM2, %YMM0, %YMM6 > + /* For YMM2 and YMM4 cmp not equals to CHAR and store result in k > + register. Its possible to save either 1 or 2 instructions using cmp no > + equals method for either YMM1 or YMM1 and YMM3 respectively but > + bottleneck on p5 makes it no worth it. */ Please limit lines to 72 columns. > + VPCMP $4, %YMM0, %YMM2, %k2 > vpxorq %YMM3, %YMM0, %YMM7 > - vpxorq %YMM4, %YMM0, %YMM8 > - > - VPMINU %YMM5, %YMM1, %YMM5 > - VPMINU %YMM6, %YMM2, %YMM6 > - VPMINU %YMM7, %YMM3, %YMM7 > - VPMINU %YMM8, %YMM4, %YMM8 > - > - VPMINU %YMM5, %YMM6, %YMM1 > - VPMINU %YMM7, %YMM8, %YMM2 > - > - VPMINU %YMM1, %YMM2, %YMM1 > - > - /* Each bit in K0 represents a CHAR or a null byte. */ > - VPCMP $0, %YMMZERO, %YMM1, %k0 > - > - addq $(VEC_SIZE * 4), %rdi > - > - ktestd %k0, %k0 > + VPCMP $4, %YMM0, %YMM4, %k4 > + > + /* Use min to select all zeros (either from xor or end of string). */ Please limit lines to 72 columns. > + VPMINU %YMM1, %YMM5, %YMM1 > + VPMINU %YMM3, %YMM7, %YMM3 > + > + /* Use min + zeromask to select for zeros. Since k2 and k4 will be > + have 0 as positions that matched with CHAR which will set zero in > + the corresponding destination bytes in YMM2 / YMM4. */ Please limit lines to 72 columns. > + VPMINU %YMM1, %YMM2, %YMM2{%k2}{z} > + VPMINU %YMM3, %YMM4, %YMM4 > + VPMINU %YMM2, %YMM4, %YMM4{%k4}{z} > + > + VPCMP $0, %YMMZERO, %YMM4, %k1 > + kmovd %k1, %ecx > + subq $-(VEC_SIZE * 4), %rdi > + testl %ecx, %ecx > jz L(loop_4x_vec) > > - /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > - VPCMP $0, %YMMZERO, %YMM5, %k0 > + VPCMP $0, %YMMZERO, %YMM1, %k0 > kmovd %k0, %eax > testl %eax, %eax > - jnz L(first_vec_x0) > + jnz L(last_vec_x1) > > - /* Each bit in K1 represents a CHAR or a null byte in YMM2. */ > - VPCMP $0, %YMMZERO, %YMM6, %k1 > - kmovd %k1, %eax > + VPCMP $0, %YMMZERO, %YMM2, %k0 > + kmovd %k0, %eax > testl %eax, %eax > - jnz L(first_vec_x1) > - > - /* Each bit in K2 represents a CHAR or a null byte in YMM3. */ > - VPCMP $0, %YMMZERO, %YMM7, %k2 > - /* Each bit in K3 represents a CHAR or a null byte in YMM4. */ > - VPCMP $0, %YMMZERO, %YMM8, %k3 > + jnz L(last_vec_x2) > > + VPCMP $0, %YMMZERO, %YMM3, %k0 > + kmovd %k0, %eax > + /* Combine YMM3 matches (eax) with YMM4 matches (ecx). */ > # ifdef USE_AS_WCSCHR > - /* NB: Each bit in K2/K3 represents 4-byte element. */ > - kshiftlw $8, %k3, %k1 > + sall $8, %ecx > + orl %ecx, %eax > + tzcntl %eax, %eax > # else > - kshiftlq $32, %k3, %k1 > + salq $32, %rcx > + orq %rcx, %rax > + tzcntq %rax, %rax > # endif > +# ifndef USE_AS_STRCHRNUL > + /* Check if match was CHAR or null. */ > + cmp (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero_end) > +# endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax > + ret > > - /* Each bit in K1 represents a NULL or a mismatch. */ > - korq %k1, %k2, %k1 > - kmovq %k1, %rax > +# ifndef USE_AS_STRCHRNUL > +L(zero_end): > + xorl %eax, %eax > + ret > +# endif > > - tzcntq %rax, %rax > -# ifdef USE_AS_WCSCHR > - /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax > -# else > - leaq (VEC_SIZE * 2)(%rdi, %rax), %rax > + .p2align 4 > +L(last_vec_x1): > + tzcntl %eax, %eax > +# ifndef USE_AS_STRCHRNUL > + /* Check if match was null. */ > + cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero_end) > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (%rdi, %rax, CHAR_SIZE), %rax > + ret > + > + .p2align 4 > +L(last_vec_x2): > + tzcntl %eax, %eax > # ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + /* Check if match was null. */ > + cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero_end) > # endif > + /* NB: Multiply sizeof char type (1 or 4) to get the number of > + bytes. */ > + leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax > ret > > /* Cold case for crossing page with first load. */ > .p2align 4 > L(cross_page_boundary): > + movq %rdi, %rdx > + /* Align rdi. */ > andq $-VEC_SIZE, %rdi > - andl $(VEC_SIZE - 1), %ecx > - > VMOVA (%rdi), %YMM1 > - > /* Leaves only CHARS matching esi as 0. */ > vpxorq %YMM1, %YMM0, %YMM2 > VPMINU %YMM2, %YMM1, %YMM2 > /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ > VPCMP $0, %YMMZERO, %YMM2, %k0 > kmovd %k0, %eax > - testl %eax, %eax > - > + /* Remove the leading bits. */ > # ifdef USE_AS_WCSCHR > + movl %edx, %SHIFT_REG > /* NB: Divide shift count by 4 since each bit in K1 represent 4 > bytes. */ > - movl %ecx, %SHIFT_REG > - sarl $2, %SHIFT_REG > + sarl $2, %SHIFT_REG > + andl $(CHAR_PER_VEC - 1), %SHIFT_REG > # endif > - > - /* Remove the leading bits. */ > sarxl %SHIFT_REG, %eax, %eax > + /* If eax is zero continue. */ > testl %eax, %eax > - > - jz L(aligned_more) > + jz L(cross_page_continue) > tzcntl %eax, %eax > - addq %rcx, %rdi > +# ifndef USE_AS_STRCHRNUL > + /* Check to see if match was CHAR or null. */ > + cmp (%rdx, %rax, CHAR_SIZE), %CHAR_REG > + jne L(zero_end) > +# endif > # ifdef USE_AS_WCSCHR > /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ > - leaq (%rdi, %rax, 4), %rax > + leaq (%rdx, %rax, CHAR_SIZE), %rax > # else > - addq %rdi, %rax > -# endif > -# ifndef USE_AS_STRCHRNUL > - cmp (%rax), %CHAR_REG > - cmovne %rdx, %rax > + addq %rdx, %rax > # endif > ret > > -- > 2.29.2 > Thanks. H.J.