From be3fa748e35802871ce61291e22966ae10d62901 Mon Sep 17 00:00:00 2001
From: Amit Pawar <Amit.Pawar@amd.com>
Date: Fri, 18 Mar 2016 17:33:22 +0530
Subject: [PATCH] x86_64 Update memcpy, mempcpy and memmove selection order for
 Excavator CPU.

Performance of Fast_Copy_Backward based implementations are better compare to
AVX_Fast_Unaligned_Load based implementation of memcpy, memcpy_chk, mempcpy,
mempcpy_chk, memmove and memmove_chk functions on Excavator cpu. New feature
bit is required to fix this issue in IFUNC functions without affecting other
targets. So defining two new bit_arch_Avoid_AVX_Fast_Unaligned_Load and
index_arch_Avoid_AVX_Fast_Unaligned_Load feature bit macros and selection
order of these functions are updated.

	[BZ #19583]
	* sysdeps/x86/cpu-features.h (bit_arch_Avoid_AVX_Fast_Unaligned_Load):
	New.
	(index_arch_Avoid_AVX_Fast_Unaligned_Load) Likewise.
	* sysdeps/x86/cpu-features.c (init_cpu_features): Set Fast_Copy_Backward
	and Avoid_AVX_Fast_Unaligned_Load bits for Excavator core.
	* sysdeps/x86_64/multiarch/memcpy.S
	(__new_memcpy, Avoid_AVX_Fast_Unaligned_Load): Add check for
	Avoid_AVX_Fast_Unaligned_Load bit and select on Excavator core instead
	of AVX_Fast_Unaligned_Load.
	* sysdeps/x86_64/multiarch/memcpy_chk.S
	(__memcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
	* sysdeps/x86_64/multiarch/mempcpy.S
	(__mempcpy, Avoid_AVX_Fast_Unaligned_Load): Likewise.
	* sysdeps/x86_64/multiarch/mempcpy_chk.S
	(__mempcpy_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
	* sysdeps/x86_64/multiarch/memmove.c
	(__libc_memmove, Avoid_AVX_Fast_Unaligned_Load): Likewise.
	* sysdeps/x86_64/multiarch/memmove_chk.c
	(__memmove_chk, Avoid_AVX_Fast_Unaligned_Load): Likewise.
---
 sysdeps/x86/cpu-features.c             | 15 +++++++++++++--
 sysdeps/x86/cpu-features.h             |  3 +++
 sysdeps/x86_64/multiarch/memcpy.S      |  5 ++++-
 sysdeps/x86_64/multiarch/memcpy_chk.S  |  3 +++
 sysdeps/x86_64/multiarch/memmove.c     |  3 ++-
 sysdeps/x86_64/multiarch/memmove_chk.c |  4 +++-
 sysdeps/x86_64/multiarch/mempcpy.S     |  3 +++
 sysdeps/x86_64/multiarch/mempcpy_chk.S |  3 +++
 8 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 1787716..f0baf0e 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -160,8 +160,19 @@ init_cpu_features (struct cpu_features *cpu_features)
 	{
 	  /* "Excavator"   */
 	  if (model >= 0x60 && model <= 0x7f)
-	    cpu_features->feature[index_arch_Fast_Unaligned_Load]
-	      |= bit_arch_Fast_Unaligned_Load;
+	    {
+#if index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+# error index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
+#endif
+#if index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Load
+# error index_arch_Avoid_AVX_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Load
+#endif
+	      cpu_features->feature[index_arch_Avoid_AVX_Fast_Unaligned_Load]
+		|= (bit_arch_Fast_Unaligned_Load
+		    | bit_arch_Fast_Copy_Backward
+		    | bit_arch_Avoid_AVX_Fast_Unaligned_Load);
+
+	    }
 	}
     }
   else
diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h
index 0624a92..fe59828 100644
--- a/sysdeps/x86/cpu-features.h
+++ b/sysdeps/x86/cpu-features.h
@@ -35,6 +35,7 @@
 #define bit_arch_I686				(1 << 15)
 #define bit_arch_Prefer_MAP_32BIT_EXEC		(1 << 16)
 #define bit_arch_Prefer_No_VZEROUPPER		(1 << 17)
+#define bit_arch_Avoid_AVX_Fast_Unaligned_Load	(1 << 18)
 
 /* CPUID Feature flags.  */
 
@@ -101,6 +102,7 @@
 # define index_arch_I686		FEATURE_INDEX_1*FEATURE_SIZE
 # define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1*FEATURE_SIZE
 # define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1*FEATURE_SIZE
+# define index_arch_Avoid_AVX_Fast_Unaligned_Load FEATURE_INDEX_1*FEATURE_SIZE
 
 
 # if defined (_LIBC) && !IS_IN (nonlib)
@@ -259,6 +261,7 @@ extern const struct cpu_features *__get_cpu_features (void)
 # define index_arch_I686		FEATURE_INDEX_1
 # define index_arch_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1
 # define index_arch_Prefer_No_VZEROUPPER FEATURE_INDEX_1
+# define index_arch_Avoid_AVX_Fast_Unaligned_Load FEATURE_INDEX_1
 
 #endif	/* !__ASSEMBLER__ */
 
diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S
index 8882590..7983153 100644
--- a/sysdeps/x86_64/multiarch/memcpy.S
+++ b/sysdeps/x86_64/multiarch/memcpy.S
@@ -38,7 +38,10 @@ ENTRY(__new_memcpy)
 	lea    __memcpy_avx512_no_vzeroupper(%rip), %RAX_LP
 	ret
 #endif
-1:	lea	__memcpy_avx_unaligned(%rip), %RAX_LP
+1:	lea	__memcpy_ssse3_back(%rip), %RAX_LP
+	HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	jnz	2f
+	lea	__memcpy_avx_unaligned(%rip), %RAX_LP
 	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
 	jnz	2f
 	lea	__memcpy_sse2_unaligned(%rip), %RAX_LP
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.S b/sysdeps/x86_64/multiarch/memcpy_chk.S
index 648217e..5b608f6 100644
--- a/sysdeps/x86_64/multiarch/memcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/memcpy_chk.S
@@ -48,6 +48,9 @@ ENTRY(__memcpy_chk)
 	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
 	jz  2f
 	leaq    __memcpy_chk_avx_unaligned(%rip), %rax
+	HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	jz  2f
+	leaq    __memcpy_chk_ssse3_back(%rip), %rax
 2:	ret
 END(__memcpy_chk)
 # else
diff --git a/sysdeps/x86_64/multiarch/memmove.c b/sysdeps/x86_64/multiarch/memmove.c
index 8da5640..b9d9439 100644
--- a/sysdeps/x86_64/multiarch/memmove.c
+++ b/sysdeps/x86_64/multiarch/memmove.c
@@ -59,7 +59,8 @@ libc_ifunc (__libc_memmove,
 	    :
 #endif
 	    (HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
-	    ? __memmove_avx_unaligned
+	    ? (HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	       ? __memmove_ssse3_back : __memmove_avx_unaligned)
 	    : (HAS_CPU_FEATURE (SSSE3)
 	       ? (HAS_ARCH_FEATURE (Fast_Copy_Backward)
 	          ? __memmove_ssse3_back : __memmove_ssse3)
diff --git a/sysdeps/x86_64/multiarch/memmove_chk.c b/sysdeps/x86_64/multiarch/memmove_chk.c
index f64da63..b8a4282 100644
--- a/sysdeps/x86_64/multiarch/memmove_chk.c
+++ b/sysdeps/x86_64/multiarch/memmove_chk.c
@@ -39,7 +39,9 @@ libc_ifunc (__memmove_chk,
 	    ? __memmove_chk_avx512_no_vzeroupper
 	    :
 #endif
-	    HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) ? __memmove_chk_avx_unaligned :
+	    HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
+	    ? (HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	       ? __memmove_chk_ssse3_back : __memmove_chk_avx_unaligned) :
 	    (HAS_CPU_FEATURE (SSSE3)
 	    ? (HAS_ARCH_FEATURE (Fast_Copy_Backward)
 	       ? __memmove_chk_ssse3_back : __memmove_chk_ssse3)
diff --git a/sysdeps/x86_64/multiarch/mempcpy.S b/sysdeps/x86_64/multiarch/mempcpy.S
index ed78623..6a8898d 100644
--- a/sysdeps/x86_64/multiarch/mempcpy.S
+++ b/sysdeps/x86_64/multiarch/mempcpy.S
@@ -46,6 +46,9 @@ ENTRY(__mempcpy)
 	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
 	jz	2f
 	leaq	__mempcpy_avx_unaligned(%rip), %rax
+	HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	jz	2f
+	leaq	__mempcpy_ssse3_back(%rip), %rax
 2:	ret
 END(__mempcpy)
 
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.S b/sysdeps/x86_64/multiarch/mempcpy_chk.S
index 6e8a89d..ecea9c2 100644
--- a/sysdeps/x86_64/multiarch/mempcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/mempcpy_chk.S
@@ -48,6 +48,9 @@ ENTRY(__mempcpy_chk)
 	HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
 	jz	2f
 	leaq	__mempcpy_chk_avx_unaligned(%rip), %rax
+	HAS_ARCH_FEATURE (Avoid_AVX_Fast_Unaligned_Load)
+	jz	2f
+	leaq	__mempcpy_chk_ssse3_back(%rip), %rax
 2:	ret
 END(__mempcpy_chk)
 # else
-- 
2.1.4

