[crux-commits] ports/core (3.6): glibc: sync with upstream 2.23 branch

commit 701f83bc54d5b4f1919603381694362c57bb001c Author: Juergen Daubert <jue@jue.li> Date: Wed Feb 9 11:53:46 2022 +0100 glibc: sync with upstream 2.23 branch diff --git a/glibc/.signature b/glibc/.signature index e5d5a599..dbf06195 100644 --- a/glibc/.signature +++ b/glibc/.signature @@ -1,10 +1,10 @@ untrusted comment: verify with /etc/ports/core.pub -RWRJc1FUaeVeqk2uwhHr5qBGXvnIwjOlb4TZ9ezJT2bi3Z+8Ym4D2uHdm5+VLuIIRBBmUYjvoKwBTb1m7j6tV9D+0QxRS+wJeA0= -SHA256 (Pkgfile) = d319ce6af6100d9570cdff2c31542c6404ffb7461283de1ae4d098d0853d5d5a +RWRJc1FUaeVeqjtuBDt9yWal+iJYK0LWLJghiZ1hQdV2j6Ll57N+c9XCp1EGfeMSTdC+Gvnan8BuI2jIRtKjih5DTvd+TEJDrw8= +SHA256 (Pkgfile) = bf6a5730d314fecf8a04f0029193a9c64eafabde1e8fc77a0860076d7eb37d7b SHA256 (.footprint) = aa29daaba0d990bb954964b7605cf132588bcb5ee9cf56d219f2ed26e60eba7b SHA256 (glibc-2.32.tar.xz) = 1627ea54f5a1a8467032563393e0901077626dc66f37f10ee6363bb722222836 SHA256 (linux-5.4.72.tar.xz) = 0e24645bd56fe5b55a7a662895f5562c103d71b54d097281f0c9c71ff22c1172 -SHA256 (glibc-2.32-6.patch) = 70f33f14f62cb2daddd9bbfe0ffdfb3bb01880f7cedbc71cb534e82343d4a3d4 +SHA256 (glibc-2.32-7.patch) = 765880b63e386bfab3898bbc273dbc683f737d6327289dcf968da2fe807575e4 SHA256 (hosts) = 5c02b256c105f1d4a12fb738d71c1bab9eb126533074d7a0c8a14b92670c9431 SHA256 (resolv.conf) = 72ccb58768a72a771ec37142bc361a18478a07ec9de6e925a20760794389bf51 SHA256 (nsswitch.conf) = 859b8984e5e90aff3cce8f9779996ae4033b280d2122840e9411e2f44a1c2e61 diff --git a/glibc/Pkgfile b/glibc/Pkgfile index 3c179148..a761b676 100644 --- a/glibc/Pkgfile +++ b/glibc/Pkgfile @@ -4,10 +4,10 @@ name=glibc version=2.32 -release=6 +release=7 source=(https://ftp.gnu.org/gnu/glibc/glibc-$version.tar.xz \ https://www.kernel.org/pub/linux/kernel/v5.x/linux-5.4.72.tar.xz \ - glibc-$version-6.patch \ + glibc-$version-7.patch \ hosts resolv.conf nsswitch.conf host.conf ld.so.conf) build() { @@ -18,7 +18,7 @@ build() { make INSTALL_HDR_PATH=$PKG/usr headers_install chown root:root $PKG/usr - patch -p1 -d $SRC/$name-${version:0:4} -i $SRC/$name-$version-6.patch + patch -p1 -d $SRC/$name-${version:0:4} -i $SRC/$name-$version-7.patch mkdir $SRC/build cd $SRC/build diff --git a/glibc/glibc-2.32-6.patch b/glibc/glibc-2.32-6.patch deleted file mode 100644 index 5d4a58d2..00000000 --- a/glibc/glibc-2.32-6.patch +++ /dev/null @@ -1,6671 +0,0 @@ -diff --git a/NEWS b/NEWS -index 485b8ddffa..b29826f4f5 100644 ---- a/NEWS -+++ b/NEWS -@@ -5,6 +5,27 @@ See the end for copying conditions. - Please send GNU C library bug reports via <https://sourceware.org/bugzilla/> - using `glibc' in the "product" field. -  -+The following bugs are resolved with this release: -+ -+ [20019] NULL pointer dereference in libc.so.6 IFUNC due to uninitialized GOT -+ [26224] iconv hangs when converting some invalid inputs from several IBM -+ character sets (CVE-2020-27618) -+ [26534] libm.so 2.32 SIGILL in pow() due to FMA4 instruction on non-FMA4 -+ system -+ [26555] string: strerrorname_np does not return the documented value -+ [26600] Transaction ID collisions cause slow DNS lookups in getaddrinfo -+ [26636] libc: 32-bit shmctl(IPC_INFO) crashes when shminfo struct is -+ at the end of a memory mapping -+ [26637] libc: semctl SEM_STAT_ANY fails to pass the buffer specified -+ by the caller to the kernel -+ [26639] libc: msgctl IPC_INFO and MSG_INFO return garbage -+ [26853] aarch64: Missing unwind information in statically linked startup code -+ [26932] libc: sh: Multiple floating point functions defined as stubs only -+ [27130] "rep movsb" performance issue -+ [27177] GLIBC_TUNABLES=glibc.cpu.x86_ibt=on:glibc.cpu.x86_shstk=on doesn't work -+ [28524] Conversion from ISO-2022-JP-3 with iconv may emit spurious NULs -+ [28607] Masked signals are delivered on thread exit -+ - Version 2.32 - - Major new features: -@@ -185,6 +206,14 @@ Security related changes: - Dytrych of the Cisco Security Assessment and Penetration Team (See - TALOS-2020-1019). - -+ CVE-2020-27618: An infinite loop has been fixed in the iconv program when -+ invoked with input containing redundant shift sequences in the IBM1364, -+ IBM1371, IBM1388, IBM1390, or IBM1399 character sets. -+ -+ CVE-2021-33574: The mq_notify function has a potential use-after-free -+ issue when using a notification type of SIGEV_THREAD and a thread -+ attribute with a non-default affinity mask. -+ - The following bugs are resolved with this release: - - [9809] localedata: ckb_IQ: new Kurdish Sorani locale -diff --git a/Rules b/Rules -index 8b771f6095..beab969fde 100644 ---- a/Rules -+++ b/Rules -@@ -155,6 +155,7 @@ xtests: tests $(xtests-special) - else - tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \ - $(tests-container:%=$(objpfx)%.out) \ -+ $(tests-mcheck:%=$(objpfx)%-mcheck.out) \ - $(tests-special) $(tests-printers-out) - xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special) - endif -@@ -165,7 +166,7 @@ ifeq ($(run-built-tests),no) - tests-expected = - else - tests-expected = $(tests) $(tests-internal) $(tests-printers) \ -- $(tests-container) -+ $(tests-container) $(tests-mcheck:%=%-mcheck) - endif - tests: - $(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \ -@@ -191,6 +192,7 @@ else - binaries-pie-tests = - binaries-pie-notests = - endif -+binaries-mcheck-tests = $(tests-mcheck:%=%-mcheck) - else - binaries-all-notests = - binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) -@@ -200,6 +202,7 @@ binaries-static-tests = - binaries-static = - binaries-pie-tests = - binaries-pie-notests = -+binaries-mcheck-tests = - endif - - binaries-pie = $(binaries-pie-tests) $(binaries-pie-notests) -@@ -223,6 +226,14 @@ $(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \ - $(+link-tests) - endif - -+ifneq "$(strip $(binaries-mcheck-tests))" "" -+$(addprefix $(objpfx),$(binaries-mcheck-tests)): %-mcheck: %.o \ -+ $(link-extra-libs-tests) \ -+ $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \ -+ $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit) -+ $(+link-tests) -+endif -+ - ifneq "$(strip $(binaries-pie-tests))" "" - $(addprefix $(objpfx),$(binaries-pie-tests)): %: %.o \ - $(link-extra-libs-tests) \ -@@ -253,6 +264,12 @@ $(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \ - $(+link-static-tests) - endif - -+# All mcheck tests will be run with MALLOC_CHECK_=3 -+define mcheck-ENVS -+$(1)-mcheck-ENV = MALLOC_CHECK_=3 -+endef -+$(foreach t,$(tests-mcheck),$(eval $(call mcheck-ENVS,$(t)))) -+ - ifneq "$(strip $(tests) $(tests-internal) $(xtests) $(test-srcs))" "" - # These are the implicit rules for making test outputs - # from the test programs and whatever input files are present. -diff --git a/debug/Makefile b/debug/Makefile -index 3a60d7af7a..0036edd187 100644 ---- a/debug/Makefile -+++ b/debug/Makefile -@@ -51,7 +51,7 @@ routines = backtrace backtracesyms backtracesymsfd noophooks \ - explicit_bzero_chk \ - stack_chk_fail fortify_fail \ - $(static-only-routines) --static-only-routines := warning-nop stack_chk_fail_local -+static-only-routines := stack_chk_fail_local - - # Don't add stack_chk_fail_local.o to libc.a since __stack_chk_fail_local - # is an alias of __stack_chk_fail in stack_chk_fail.o. -diff --git a/debug/warning-nop.c b/debug/warning-nop.c -deleted file mode 100644 -index 4ab7e182b7..0000000000 ---- a/debug/warning-nop.c -+++ /dev/null -@@ -1,70 +0,0 @@ --/* Dummy nop functions to elicit link-time warnings. -- Copyright (C) 2005-2020 Free Software Foundation, Inc. -- This file is part of the GNU C Library. -- -- The GNU C Library is free software; you can redistribute it and/or -- modify it under the terms of the GNU Lesser General Public -- License as published by the Free Software Foundation; either -- version 2.1 of the License, or (at your option) any later version. -- -- In addition to the permissions in the GNU Lesser General Public -- License, the Free Software Foundation gives you unlimited -- permission to link the compiled version of this file with other -- programs, and to distribute those programs without any restriction -- coming from the use of this file. (The GNU Lesser General Public -- License restrictions do apply in other respects; for example, they -- cover modification of the file, and distribution when not linked -- into another program.) -- -- Note that people who make modified versions of this file are not -- obligated to grant this special exception for their modified -- versions; it is their choice whether to do so. The GNU Lesser -- General Public License gives permission to release a modified -- version without this exception; this exception also makes it -- possible to release a modified version which carries forward this -- exception. -- -- The GNU C Library is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- Lesser General Public License for more details. -- -- You should have received a copy of the GNU Lesser General Public -- License along with the GNU C Library; if not, see -- <https://www.gnu.org/licenses/>. */ -- --#include <sys/cdefs.h> -- --static void --__attribute__ ((used)) --nop (void) --{ --} -- --/* Don't insert any other #include's before this #undef! */ -- --#undef __warndecl --#define __warndecl(name, msg) \ -- extern void name (void) __attribute__ ((alias ("nop"))) attribute_hidden; \ -- link_warning (name, msg) -- --#undef __USE_FORTIFY_LEVEL --#define __USE_FORTIFY_LEVEL 99 -- --/* Following here we need an #include for each public header file -- that uses __warndecl. */ -- --/* Define away to avoid warnings with compilers that do not have these -- builtins. */ --#define __builtin___memcpy_chk(dest, src, len, bos) NULL --#define __builtin___memmove_chk(dest, src, len, bos) NULL --#define __builtin___mempcpy_chk(dest, src, len, bos) NULL --#define __builtin___memset_chk(dest, ch, len, bos) NULL --#define __builtin___stpcpy_chk(dest, src, bos) NULL --#define __builtin___strcat_chk(dest, src, bos) NULL --#define __builtin___strcpy_chk(dest, src, bos) NULL --#define __builtin___strncat_chk(dest, src, len, bos) NULL --#define __builtin___strncpy_chk(dest, src, len, bos) NULL --#define __builtin_object_size(bos, level) 0 -- --#include <string.h> -diff --git a/elf/Makefile b/elf/Makefile -index 0b78721848..3ba7f4ecfc 100644 ---- a/elf/Makefile -+++ b/elf/Makefile -@@ -1381,6 +1381,8 @@ CFLAGS-ifuncmain7pie.c += $(pie-ccflag) - CFLAGS-ifuncmain9pie.c += $(pie-ccflag) - CFLAGS-tst-ifunc-textrel.c += $(pic-ccflag) - -+LDFLAGS-ifuncmain6pie = -Wl,-z,lazy -+ - $(objpfx)ifuncmain1pie: $(objpfx)ifuncmod1.so - $(objpfx)ifuncmain1staticpie: $(objpfx)ifuncdep1pic.o - $(objpfx)ifuncmain1vispie: $(objpfx)ifuncmod1.so -@@ -1630,8 +1632,6 @@ $(objpfx)tst-nodelete-dlclose.out: $(objpfx)tst-nodelete-dlclose-dso.so \ - - tst-env-setuid-ENV = MALLOC_CHECK_=2 MALLOC_MMAP_THRESHOLD_=4096 \ - LD_HWCAP_MASK=0x1 --tst-env-setuid-tunables-ENV = \ -- GLIBC_TUNABLES=glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096 - - $(objpfx)tst-debug1: $(libdl) - $(objpfx)tst-debug1.out: $(objpfx)tst-debug1mod1.so -diff --git a/elf/dl-load.c b/elf/dl-load.c -index e39980fb19..71867e7c1a 100644 ---- a/elf/dl-load.c -+++ b/elf/dl-load.c -@@ -855,10 +855,12 @@ lose (int code, int fd, const char *name, char *realname, struct link_map *l, - - /* Process PT_GNU_PROPERTY program header PH in module L after - PT_LOAD segments are mapped. Only one NT_GNU_PROPERTY_TYPE_0 -- note is handled which contains processor specific properties. */ -+ note is handled which contains processor specific properties. -+ FD is -1 for the kernel mapped main executable otherwise it is -+ the fd used for loading module L. */ - - void --_dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph) -+_dl_process_pt_gnu_property (struct link_map *l, int fd, const ElfW(Phdr) *ph) - { - const ElfW(Nhdr) *note = (const void *) (ph->p_vaddr + l->l_addr); - const ElfW(Addr) size = ph->p_memsz; -@@ -905,7 +907,7 @@ _dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph) - last_type = type; - - /* Target specific property processing. */ -- if (_dl_process_gnu_property (l, type, datasz, ptr) == 0) -+ if (_dl_process_gnu_property (l, fd, type, datasz, ptr) == 0) - return; - - /* Check the next property item. */ -@@ -1251,21 +1253,6 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, - maplength, has_holes, loader); - if (__glibc_unlikely (errstring != NULL)) - goto call_lose; -- -- /* Process program headers again after load segments are mapped in -- case processing requires accessing those segments. Scan program -- headers backward so that PT_NOTE can be skipped if PT_GNU_PROPERTY -- exits. */ -- for (ph = &phdr[l->l_phnum]; ph != phdr; --ph) -- switch (ph[-1].p_type) -- { -- case PT_NOTE: -- _dl_process_pt_note (l, &ph[-1]); -- break; -- case PT_GNU_PROPERTY: -- _dl_process_pt_gnu_property (l, &ph[-1]); -- break; -- } - } - - if (l->l_ld == 0) -@@ -1377,6 +1364,21 @@ cannot enable executable stack as shared object requires"); - if (l->l_tls_initimage != NULL) - l->l_tls_initimage = (char *) l->l_tls_initimage + l->l_addr; - -+ /* Process program headers again after load segments are mapped in -+ case processing requires accessing those segments. Scan program -+ headers backward so that PT_NOTE can be skipped if PT_GNU_PROPERTY -+ exits. */ -+ for (ph = &l->l_phdr[l->l_phnum]; ph != l->l_phdr; --ph) -+ switch (ph[-1].p_type) -+ { -+ case PT_NOTE: -+ _dl_process_pt_note (l, fd, &ph[-1]); -+ break; -+ case PT_GNU_PROPERTY: -+ _dl_process_pt_gnu_property (l, fd, &ph[-1]); -+ break; -+ } -+ - /* We are done mapping in the file. We no longer need the descriptor. */ - if (__glibc_unlikely (__close_nocancel (fd) != 0)) - { -diff --git a/elf/dl-open.c b/elf/dl-open.c -index 8769e47051..55b39e1bbe 100644 ---- a/elf/dl-open.c -+++ b/elf/dl-open.c -@@ -887,7 +887,7 @@ no more namespaces available for dlmopen()")); - /* Avoid keeping around a dangling reference to the libc.so link - map in case it has been cached in libc_map. */ - if (!args.libc_already_loaded) -- GL(dl_ns)[nsid].libc_map = NULL; -+ GL(dl_ns)[args.nsid].libc_map = NULL; - - /* Remove the object from memory. It may be in an inconsistent - state if relocation failed, for example. */ -diff --git a/elf/dl-tunables.c b/elf/dl-tunables.c -index 26e6e26612..15b29bcb90 100644 ---- a/elf/dl-tunables.c -+++ b/elf/dl-tunables.c -@@ -177,6 +177,7 @@ parse_tunables (char *tunestr, char *valstring) - return; - - char *p = tunestr; -+ size_t off = 0; - - while (true) - { -@@ -190,7 +191,11 @@ parse_tunables (char *tunestr, char *valstring) - /* If we reach the end of the string before getting a valid name-value - pair, bail out. */ - if (p[len] == '\0') -- return; -+ { -+ if (__libc_enable_secure) -+ tunestr[off] = '\0'; -+ return; -+ } - - /* We did not find a valid name-value pair before encountering the - colon. */ -@@ -216,35 +221,28 @@ parse_tunables (char *tunestr, char *valstring) - - if (tunable_is_name (cur->name, name)) - { -- /* If we are in a secure context (AT_SECURE) then ignore the tunable -- unless it is explicitly marked as secure. Tunable values take -- precedence over their envvar aliases. */ -+ /* If we are in a secure context (AT_SECURE) then ignore the -+ tunable unless it is explicitly marked as secure. Tunable -+ values take precedence over their envvar aliases. We write -+ the tunables that are not SXID_ERASE back to TUNESTR, thus -+ dropping all SXID_ERASE tunables and any invalid or -+ unrecognized tunables. */ - if (__libc_enable_secure) - { -- if (cur->security_level == TUNABLE_SECLEVEL_SXID_ERASE) -+ if (cur->security_level != TUNABLE_SECLEVEL_SXID_ERASE) - { -- if (p[len] == '\0') -- { -- /* Last tunable in the valstring. Null-terminate and -- return. */ -- *name = '\0'; -- return; -- } -- else -- { -- /* Remove the current tunable from the string. We do -- this by overwriting the string starting from NAME -- (which is where the current tunable begins) with -- the remainder of the string. We then have P point -- to NAME so that we continue in the correct -- position in the valstring. */ -- char *q = &p[len + 1]; -- p = name; -- while (*q != '\0') -- *name++ = *q++; -- name[0] = '\0'; -- len = 0; -- } -+ if (off > 0) -+ tunestr[off++] = ':'; -+ -+ const char *n = cur->name; -+ -+ while (*n != '\0') -+ tunestr[off++] = *n++; -+ -+ tunestr[off++] = '='; -+ -+ for (size_t j = 0; j < len; j++) -+ tunestr[off++] = value[j]; - } - - if (cur->security_level != TUNABLE_SECLEVEL_NONE) -@@ -257,9 +255,7 @@ parse_tunables (char *tunestr, char *valstring) - } - } - -- if (p[len] == '\0') -- return; -- else -+ if (p[len] != '\0') - p += len + 1; - } - } -diff --git a/elf/ifuncmain6pie.c b/elf/ifuncmain6pie.c -index 04faeb86ef..4a01906836 100644 ---- a/elf/ifuncmain6pie.c -+++ b/elf/ifuncmain6pie.c -@@ -9,7 +9,6 @@ - #include "ifunc-sel.h" - - typedef int (*foo_p) (void); --extern foo_p foo_ptr; - - static int - one (void) -@@ -28,20 +27,17 @@ foo_ifunc (void) - } - - extern int foo (void); --extern foo_p get_foo (void); -+extern int call_foo (void); - extern foo_p get_foo_p (void); - --foo_p my_foo_ptr = foo; -+foo_p foo_ptr = foo; - - int - main (void) - { - foo_p p; - -- p = get_foo (); -- if (p != foo) -- abort (); -- if ((*p) () != -30) -+ if (call_foo () != -30) - abort (); - - p = get_foo_p (); -@@ -52,12 +48,8 @@ main (void) - - if (foo_ptr != foo) - abort (); -- if (my_foo_ptr != foo) -- abort (); - if ((*foo_ptr) () != -30) - abort (); -- if ((*my_foo_ptr) () != -30) -- abort (); - if (foo () != -30) - abort (); - -diff --git a/elf/ifuncmod6.c b/elf/ifuncmod6.c -index 2e16c1d06d..2f6d0715e6 100644 ---- a/elf/ifuncmod6.c -+++ b/elf/ifuncmod6.c -@@ -4,7 +4,7 @@ extern int foo (void); - - typedef int (*foo_p) (void); - --foo_p foo_ptr = foo; -+extern foo_p foo_ptr; - - foo_p - get_foo_p (void) -@@ -12,8 +12,8 @@ get_foo_p (void) - return foo_ptr; - } - --foo_p --get_foo (void) -+int -+call_foo (void) - { -- return foo; -+ return foo (); - } -diff --git a/elf/rtld.c b/elf/rtld.c -index 5b882163fa..14a42ed00a 100644 ---- a/elf/rtld.c -+++ b/elf/rtld.c -@@ -1534,10 +1534,10 @@ of this helper program; chances are you did not intend to run this program.\n\ - switch (ph[-1].p_type) - { - case PT_NOTE: -- _dl_process_pt_note (main_map, &ph[-1]); -+ _dl_process_pt_note (main_map, -1, &ph[-1]); - break; - case PT_GNU_PROPERTY: -- _dl_process_pt_gnu_property (main_map, &ph[-1]); -+ _dl_process_pt_gnu_property (main_map, -1, &ph[-1]); - break; - } - -diff --git a/elf/tst-env-setuid-tunables.c b/elf/tst-env-setuid-tunables.c -index 971d5892b1..ca0c8c245c 100644 ---- a/elf/tst-env-setuid-tunables.c -+++ b/elf/tst-env-setuid-tunables.c -@@ -25,35 +25,76 @@ - #include "config.h" - #undef _LIBC - --#define test_parent test_parent_tunables --#define test_child test_child_tunables -- --static int test_child_tunables (void); --static int test_parent_tunables (void); -- --#include "tst-env-setuid.c" -- --#define CHILD_VALSTRING_VALUE "glibc.malloc.mmap_threshold=4096" --#define PARENT_VALSTRING_VALUE \ -- "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096" -+#include <errno.h> -+#include <fcntl.h> -+#include <stdlib.h> -+#include <stdint.h> -+#include <stdio.h> -+#include <string.h> -+#include <sys/stat.h> -+#include <sys/wait.h> -+#include <unistd.h> -+#include <intprops.h> -+#include <array_length.h> -+ -+#include <support/check.h> -+#include <support/support.h> -+#include <support/test-driver.h> -+#include <support/capture_subprocess.h> -+ -+const char *teststrings[] = -+{ -+ "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.check=2:glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096:glibc.malloc.check=2", -+ "glibc.malloc.perturb=0x800", -+ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.perturb=0x800:not_valid.malloc.check=2:glibc.malloc.mmap_threshold=4096", -+ "glibc.not_valid.check=2:glibc.malloc.mmap_threshold=4096", -+ "not_valid.malloc.check=2:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.garbage=2:glibc.maoc.mmap_threshold=4096:glibc.malloc.check=2", -+ "glibc.malloc.check=4:glibc.malloc.garbage=2:glibc.maoc.mmap_threshold=4096", -+ ":glibc.malloc.garbage=2:glibc.malloc.check=1", -+ "glibc.malloc.check=1:glibc.malloc.check=2", -+ "not_valid.malloc.check=2", -+ "glibc.not_valid.check=2", -+}; -+ -+const char *resultstrings[] = -+{ -+ "glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.perturb=0x800", -+ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.mmap_threshold=4096", -+ "glibc.malloc.mmap_threshold=4096", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+}; - - static int --test_child_tunables (void) -+test_child (int off) - { - const char *val = getenv ("GLIBC_TUNABLES"); - - #if HAVE_TUNABLES -- if (val != NULL && strcmp (val, CHILD_VALSTRING_VALUE) == 0) -+ if (val != NULL && strcmp (val, resultstrings[off]) == 0) - return 0; - - if (val != NULL) -- printf ("Unexpected GLIBC_TUNABLES VALUE %s\n", val); -+ printf ("[%d] Unexpected GLIBC_TUNABLES VALUE %s\n", off, val); - - return 1; - #else - if (val != NULL) - { -- printf ("GLIBC_TUNABLES not cleared\n"); -+ printf ("[%d] GLIBC_TUNABLES not cleared\n", off); - return 1; - } - return 0; -@@ -61,15 +102,48 @@ test_child_tunables (void) - } - - static int --test_parent_tunables (void) -+do_test (int argc, char **argv) - { -- const char *val = getenv ("GLIBC_TUNABLES"); -+ /* Setgid child process. */ -+ if (argc == 2) -+ { -+ if (getgid () == getegid ()) -+ /* This can happen if the file system is mounted nosuid. */ -+ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", -+ (intmax_t) getgid ()); - -- if (val != NULL && strcmp (val, PARENT_VALSTRING_VALUE) == 0) -- return 0; -+ int ret = test_child (atoi (argv[1])); - -- if (val != NULL) -- printf ("Unexpected GLIBC_TUNABLES VALUE %s\n", val); -+ if (ret != 0) -+ exit (1); - -- return 1; -+ exit (EXIT_SUCCESS); -+ } -+ else -+ { -+ int ret = 0; -+ -+ /* Spawn tests. */ -+ for (int i = 0; i < array_length (teststrings); i++) -+ { -+ char buf[INT_BUFSIZE_BOUND (int)]; -+ -+ printf ("Spawned test for %s (%d)\n", teststrings[i], i); -+ snprintf (buf, sizeof (buf), "%d\n", i); -+ if (setenv ("GLIBC_TUNABLES", teststrings[i], 1) != 0) -+ exit (1); -+ -+ int status = support_capture_subprogram_self_sgid (buf); -+ -+ /* Bail out early if unsupported. */ -+ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) -+ return EXIT_UNSUPPORTED; -+ -+ ret |= status; -+ } -+ return ret; -+ } - } -+ -+#define TEST_FUNCTION_ARGV do_test -+#include <support/test-driver.c> -diff --git a/elf/tst-env-setuid.c b/elf/tst-env-setuid.c -index 41dc79e83a..2dbccdb69e 100644 ---- a/elf/tst-env-setuid.c -+++ b/elf/tst-env-setuid.c -@@ -29,173 +29,12 @@ - #include <sys/wait.h> - #include <unistd.h> - -+#include <support/check.h> - #include <support/support.h> - #include <support/test-driver.h> -+#include <support/capture_subprocess.h> - - static char SETGID_CHILD[] = "setgid-child"; --#define CHILD_STATUS 42 -- --/* Return a GID which is not our current GID, but is present in the -- supplementary group list. */ --static gid_t --choose_gid (void) --{ -- const int count = 64; -- gid_t groups[count]; -- int ret = getgroups (count, groups); -- if (ret < 0) -- { -- printf ("getgroups: %m\n"); -- exit (1); -- } -- gid_t current = getgid (); -- for (int i = 0; i < ret; ++i) -- { -- if (groups[i] != current) -- return groups[i]; -- } -- return 0; --} -- --/* Spawn and execute a program and verify that it returns the CHILD_STATUS. */ --static pid_t --do_execve (char **args) --{ -- pid_t kid = vfork (); -- -- if (kid < 0) -- { -- printf ("vfork: %m\n"); -- return -1; -- } -- -- if (kid == 0) -- { -- /* Child process. */ -- execve (args[0], args, environ); -- _exit (-errno); -- } -- -- if (kid < 0) -- return 1; -- -- int status; -- -- if (waitpid (kid, &status, 0) < 0) -- { -- printf ("waitpid: %m\n"); -- return 1; -- } -- -- if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) -- return EXIT_UNSUPPORTED; -- -- if (!WIFEXITED (status) || WEXITSTATUS (status) != CHILD_STATUS) -- { -- printf ("Unexpected exit status %d from child process\n", -- WEXITSTATUS (status)); -- return 1; -- } -- return 0; --} -- --/* Copies the executable into a restricted directory, so that we can -- safely make it SGID with the TARGET group ID. Then runs the -- executable. */ --static int --run_executable_sgid (gid_t target) --{ -- char *dirname = xasprintf ("%s/tst-tunables-setuid.%jd", -- test_dir, (intmax_t) getpid ()); -- char *execname = xasprintf ("%s/bin", dirname); -- int infd = -1; -- int outfd = -1; -- int ret = 0; -- if (mkdir (dirname, 0700) < 0) -- { -- printf ("mkdir: %m\n"); -- goto err; -- } -- infd = open ("/proc/self/exe", O_RDONLY); -- if (infd < 0) -- { -- printf ("open (/proc/self/exe): %m\n"); -- goto err; -- } -- outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); -- if (outfd < 0) -- { -- printf ("open (%s): %m\n", execname); -- goto err; -- } -- char buf[4096]; -- for (;;) -- { -- ssize_t rdcount = read (infd, buf, sizeof (buf)); -- if (rdcount < 0) -- { -- printf ("read: %m\n"); -- goto err; -- } -- if (rdcount == 0) -- break; -- char *p = buf; -- char *end = buf + rdcount; -- while (p != end) -- { -- ssize_t wrcount = write (outfd, buf, end - p); -- if (wrcount == 0) -- errno = ENOSPC; -- if (wrcount <= 0) -- { -- printf ("write: %m\n"); -- goto err; -- } -- p += wrcount; -- } -- } -- if (fchown (outfd, getuid (), target) < 0) -- { -- printf ("fchown (%s): %m\n", execname); -- goto err; -- } -- if (fchmod (outfd, 02750) < 0) -- { -- printf ("fchmod (%s): %m\n", execname); -- goto err; -- } -- if (close (outfd) < 0) -- { -- printf ("close (outfd): %m\n"); -- goto err; -- } -- if (close (infd) < 0) -- { -- printf ("close (infd): %m\n"); -- goto err; -- } -- -- char *args[] = {execname, SETGID_CHILD, NULL}; -- -- ret = do_execve (args); -- --err: -- if (outfd >= 0) -- close (outfd); -- if (infd >= 0) -- close (infd); -- if (execname) -- { -- unlink (execname); -- free (execname); -- } -- if (dirname) -- { -- rmdir (dirname); -- free (dirname); -- } -- return ret; --} - - #ifndef test_child - static int -@@ -256,40 +95,32 @@ do_test (int argc, char **argv) - if (argc == 2 && strcmp (argv[1], SETGID_CHILD) == 0) - { - if (getgid () == getegid ()) -- { -- /* This can happen if the file system is mounted nosuid. */ -- fprintf (stderr, "SGID failed: GID and EGID match (%jd)\n", -- (intmax_t) getgid ()); -- exit (EXIT_UNSUPPORTED); -- } -+ /* This can happen if the file system is mounted nosuid. */ -+ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", -+ (intmax_t) getgid ()); - - int ret = test_child (); - - if (ret != 0) - exit (1); - -- exit (CHILD_STATUS); -+ exit (EXIT_SUCCESS); - } - else - { - if (test_parent () != 0) - exit (1); - -- /* Try running a setgid program. */ -- gid_t target = choose_gid (); -- if (target == 0) -- { -- fprintf (stderr, -- "Could not find a suitable GID for user %jd, skipping test\n", -- (intmax_t) getuid ()); -- exit (0); -- } -+ int status = support_capture_subprogram_self_sgid (SETGID_CHILD); - -- return run_executable_sgid (target); -- } -+ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) -+ return EXIT_UNSUPPORTED; -+ -+ if (!WIFEXITED (status)) -+ FAIL_EXIT1 ("Unexpected exit status %d from child process\n", status); - -- /* Something went wrong and our argv was corrupted. */ -- _exit (1); -+ return 0; -+ } - } - - #define TEST_FUNCTION_ARGV do_test -diff --git a/iconv/Versions b/iconv/Versions -index 8a5f4cf780..d51af52fa3 100644 ---- a/iconv/Versions -+++ b/iconv/Versions -@@ -6,7 +6,9 @@ libc { - GLIBC_PRIVATE { - # functions shared with iconv program - __gconv_get_alias_db; __gconv_get_cache; __gconv_get_modules_db; -- __gconv_open; __gconv_create_spec; -+ -+ # functions used elsewhere in glibc -+ __gconv_open; __gconv_create_spec; __gconv_destroy_spec; - - # function used by the gconv modules - __gconv_transliterate; -diff --git a/iconv/gconv_charset.c b/iconv/gconv_charset.c -index 6ccd0773cc..4ba0aa99f5 100644 ---- a/iconv/gconv_charset.c -+++ b/iconv/gconv_charset.c -@@ -216,3 +216,13 @@ out: - return ret; - } - libc_hidden_def (__gconv_create_spec) -+ -+ -+void -+__gconv_destroy_spec (struct gconv_spec *conv_spec) -+{ -+ free (conv_spec->fromcode); -+ free (conv_spec->tocode); -+ return; -+} -+libc_hidden_def (__gconv_destroy_spec) -diff --git a/iconv/gconv_charset.h b/iconv/gconv_charset.h -index b39b09aea1..e9c122cf7e 100644 ---- a/iconv/gconv_charset.h -+++ b/iconv/gconv_charset.h -@@ -48,33 +48,6 @@ - #define GCONV_IGNORE_ERRORS_SUFFIX "IGNORE" - - --/* This function accepts the charset names of the source and destination of the -- conversion and populates *conv_spec with an equivalent conversion -- specification that may later be used by __gconv_open. The charset names -- might contain options in the form of suffixes that alter the conversion, -- e.g. "ISO-10646/UTF-8/TRANSLIT". It processes the charset names, ignoring -- and truncating any suffix options in fromcode, and processing and truncating -- any suffix options in tocode. Supported suffix options ("TRANSLIT" or -- "IGNORE") when found in tocode lead to the corresponding flag in *conv_spec -- to be set to true. Unrecognized suffix options are silently discarded. If -- the function succeeds, it returns conv_spec back to the caller. It returns -- NULL upon failure. */ --struct gconv_spec * --__gconv_create_spec (struct gconv_spec *conv_spec, const char *fromcode, -- const char *tocode); --libc_hidden_proto (__gconv_create_spec) -- -- --/* This function frees all heap memory allocated by __gconv_create_spec. */ --static void __attribute__ ((unused)) --gconv_destroy_spec (struct gconv_spec *conv_spec) --{ -- free (conv_spec->fromcode); -- free (conv_spec->tocode); -- return; --} -- -- - /* This function copies in-order, characters from the source 's' that are - either alpha-numeric or one in one of these: "_-.,:/" - into the destination - 'wp' while dropping all other characters. In the process, it converts all -diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h -index e86938dae7..f721ce30ff 100644 ---- a/iconv/gconv_int.h -+++ b/iconv/gconv_int.h -@@ -152,6 +152,27 @@ extern int __gconv_open (struct gconv_spec *conv_spec, - __gconv_t *handle, int flags); - libc_hidden_proto (__gconv_open) - -+/* This function accepts the charset names of the source and destination of the -+ conversion and populates *conv_spec with an equivalent conversion -+ specification that may later be used by __gconv_open. The charset names -+ might contain options in the form of suffixes that alter the conversion, -+ e.g. "ISO-10646/UTF-8/TRANSLIT". It processes the charset names, ignoring -+ and truncating any suffix options in fromcode, and processing and truncating -+ any suffix options in tocode. Supported suffix options ("TRANSLIT" or -+ "IGNORE") when found in tocode lead to the corresponding flag in *conv_spec -+ to be set to true. Unrecognized suffix options are silently discarded. If -+ the function succeeds, it returns conv_spec back to the caller. It returns -+ NULL upon failure. */ -+extern struct gconv_spec * -+__gconv_create_spec (struct gconv_spec *conv_spec, const char *fromcode, -+ const char *tocode); -+libc_hidden_proto (__gconv_create_spec) -+ -+/* This function frees all heap memory allocated by __gconv_create_spec. */ -+extern void -+__gconv_destroy_spec (struct gconv_spec *conv_spec); -+libc_hidden_proto (__gconv_destroy_spec) -+ - /* Free resources associated with transformation descriptor CD. */ - extern int __gconv_close (__gconv_t cd) - attribute_hidden; -diff --git a/iconv/iconv_open.c b/iconv/iconv_open.c -index dd54bc12e0..5b30055c04 100644 ---- a/iconv/iconv_open.c -+++ b/iconv/iconv_open.c -@@ -39,7 +39,7 @@ iconv_open (const char *tocode, const char *fromcode) - - int res = __gconv_open (&conv_spec, &cd, 0); - -- gconv_destroy_spec (&conv_spec); -+ __gconv_destroy_spec (&conv_spec); - - if (__builtin_expect (res, __GCONV_OK) != __GCONV_OK) - { -diff --git a/iconv/iconv_prog.c b/iconv/iconv_prog.c -index b4334faa57..d59979759c 100644 ---- a/iconv/iconv_prog.c -+++ b/iconv/iconv_prog.c -@@ -184,7 +184,7 @@ main (int argc, char *argv[]) - /* Let's see whether we have these coded character sets. */ - res = __gconv_open (&conv_spec, &cd, 0); - -- gconv_destroy_spec (&conv_spec); -+ __gconv_destroy_spec (&conv_spec); - - if (res != __GCONV_OK) - { -diff --git a/iconv/tst-iconv_prog.sh b/iconv/tst-iconv_prog.sh -index 8298136b7f..d8db7b335c 100644 ---- a/iconv/tst-iconv_prog.sh -+++ b/iconv/tst-iconv_prog.sh -@@ -102,12 +102,16 @@ hangarray=( - "\x00\x80;-c;IBM1161;UTF-8//TRANSLIT//IGNORE" - "\x00\xdb;-c;IBM1162;UTF-8//TRANSLIT//IGNORE" - "\x00\x70;-c;IBM12712;UTF-8//TRANSLIT//IGNORE" --# These are known hangs that are yet to be fixed: --# "\x00\x0f;-c;IBM1364;UTF-8" --# "\x00\x0f;-c;IBM1371;UTF-8" --# "\x00\x0f;-c;IBM1388;UTF-8" --# "\x00\x0f;-c;IBM1390;UTF-8" --# "\x00\x0f;-c;IBM1399;UTF-8" -+"\x00\x0f;-c;IBM1364;UTF-8" -+"\x0e\x0e;-c;IBM1364;UTF-8" -+"\x00\x0f;-c;IBM1371;UTF-8" -+"\x0e\x0e;-c;IBM1371;UTF-8" -+"\x00\x0f;-c;IBM1388;UTF-8" -+"\x0e\x0e;-c;IBM1388;UTF-8" -+"\x00\x0f;-c;IBM1390;UTF-8" -+"\x0e\x0e;-c;IBM1390;UTF-8" -+"\x00\x0f;-c;IBM1399;UTF-8" -+"\x0e\x0e;-c;IBM1399;UTF-8" - "\x00\x53;-c;IBM16804;UTF-8//TRANSLIT//IGNORE" - "\x00\x41;-c;IBM274;UTF-8//TRANSLIT//IGNORE" - "\x00\x41;-c;IBM275;UTF-8//TRANSLIT//IGNORE" -diff --git a/iconvdata/Makefile b/iconvdata/Makefile -index 4ec2741cdc..b67b4feeb4 100644 ---- a/iconvdata/Makefile -+++ b/iconvdata/Makefile -@@ -1,4 +1,5 @@ - # Copyright (C) 1997-2020 Free Software Foundation, Inc. -+# Copyright (C) The GNU Toolchain Authors. - # This file is part of the GNU C Library. - - # The GNU C Library is free software; you can redistribute it and/or -@@ -73,7 +74,8 @@ modules.so := $(addsuffix .so, $(modules)) - ifeq (yes,$(build-shared)) - tests = bug-iconv1 bug-iconv2 tst-loading tst-e2big tst-iconv4 bug-iconv4 \ - tst-iconv6 bug-iconv5 bug-iconv6 tst-iconv7 bug-iconv8 bug-iconv9 \ -- bug-iconv10 bug-iconv11 bug-iconv12 tst-iconv-big5-hkscs-to-2ucs4 -+ bug-iconv10 bug-iconv11 bug-iconv12 tst-iconv-big5-hkscs-to-2ucs4 \ -+ bug-iconv13 bug-iconv14 bug-iconv15 - ifeq ($(have-thread-library),yes) - tests += bug-iconv3 - endif -@@ -321,6 +323,10 @@ $(objpfx)bug-iconv10.out: $(objpfx)gconv-modules \ - $(addprefix $(objpfx),$(modules.so)) - $(objpfx)bug-iconv12.out: $(objpfx)gconv-modules \ - $(addprefix $(objpfx),$(modules.so)) -+$(objpfx)bug-iconv14.out: $(objpfx)gconv-modules \ -+ $(addprefix $(objpfx),$(modules.so)) -+$(objpfx)bug-iconv15.out: $(addprefix $(objpfx), $(gconv-modules)) \ -+ $(addprefix $(objpfx),$(modules.so)) - - $(objpfx)iconv-test.out: run-iconv-test.sh $(objpfx)gconv-modules \ - $(addprefix $(objpfx),$(modules.so)) \ -diff --git a/iconvdata/bug-iconv13.c b/iconvdata/bug-iconv13.c -new file mode 100644 -index 0000000000..87aaff398e ---- /dev/null -+++ b/iconvdata/bug-iconv13.c -@@ -0,0 +1,53 @@ -+/* bug 24973: Test EUC-KR module -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <errno.h> -+#include <iconv.h> -+#include <stdio.h> -+#include <support/check.h> -+ -+static int -+do_test (void) -+{ -+ iconv_t cd = iconv_open ("UTF-8//IGNORE", "EUC-KR"); -+ TEST_VERIFY_EXIT (cd != (iconv_t) -1); -+ -+ /* 0xfe (->0x7e : row 94) and 0xc9 (->0x49 : row 41) are user-defined -+ areas, which are not allowed and should be skipped over due to -+ //IGNORE. The trailing 0xfe also is an incomplete sequence, which -+ should be checked first. */ -+ char input[4] = { '\xc9', '\xa1', '\0', '\xfe' }; -+ char *inptr = input; -+ size_t insize = sizeof (input); -+ char output[4]; -+ char *outptr = output; -+ size_t outsize = sizeof (output); -+ -+ /* This used to crash due to buffer overrun. */ -+ TEST_VERIFY (iconv (cd, &inptr, &insize, &outptr, &outsize) == (size_t) -1); -+ TEST_VERIFY (errno == EINVAL); -+ /* The conversion should produce one character, the converted null -+ character. */ -+ TEST_VERIFY (sizeof (output) - outsize == 1); -+ -+ TEST_VERIFY_EXIT (iconv_close (cd) != -1); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/iconvdata/bug-iconv14.c b/iconvdata/bug-iconv14.c -new file mode 100644 -index 0000000000..902f140fa9 ---- /dev/null -+++ b/iconvdata/bug-iconv14.c -@@ -0,0 +1,127 @@ -+/* Assertion in ISO-2022-JP-3 due to two-character sequence (bug 27256). -+ Copyright (C) 2021 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <iconv.h> -+#include <string.h> -+#include <errno.h> -+#include <support/check.h> -+ -+/* Use an escape sequence to return to the initial state. */ -+static void -+with_escape_sequence (void) -+{ -+ iconv_t c = iconv_open ("UTF-8", "ISO-2022-JP-3"); -+ TEST_VERIFY_EXIT (c != (iconv_t) -1); -+ -+ char in[] = "\e$(O+D\e(B"; -+ char *inbuf = in; -+ size_t inleft = strlen (in); -+ char out[3]; /* Space for one output character. */ -+ char *outbuf; -+ size_t outleft; -+ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), (size_t) -1); -+ TEST_COMPARE (errno, E2BIG); -+ TEST_COMPARE (inleft, 3); -+ TEST_COMPARE (inbuf - in, strlen (in) - 3); -+ TEST_COMPARE (outleft, sizeof (out) - 2); -+ TEST_COMPARE (outbuf - out, 2); -+ TEST_COMPARE (out[0] & 0xff, 0xc3); -+ TEST_COMPARE (out[1] & 0xff, 0xa6); -+ -+ /* Return to the initial shift state, producing the pending -+ character. */ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), 0); -+ TEST_COMPARE (inleft, 0); -+ TEST_COMPARE (inbuf - in, strlen (in)); -+ TEST_COMPARE (outleft, sizeof (out) - 2); -+ TEST_COMPARE (outbuf - out, 2); -+ TEST_COMPARE (out[0] & 0xff, 0xcc); -+ TEST_COMPARE (out[1] & 0xff, 0x80); -+ -+ /* Nothing should be flushed the second time. */ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); -+ TEST_COMPARE (outleft, sizeof (out)); -+ TEST_COMPARE (outbuf - out, 0); -+ TEST_COMPARE (out[0] & 0xff, 0xcc); -+ TEST_COMPARE (out[1] & 0xff, 0x80); -+ -+ TEST_COMPARE (iconv_close (c), 0); -+} -+ -+/* Use an explicit flush to return to the initial state. */ -+static void -+with_flush (void) -+{ -+ iconv_t c = iconv_open ("UTF-8", "ISO-2022-JP-3"); -+ TEST_VERIFY_EXIT (c != (iconv_t) -1); -+ -+ char in[] = "\e$(O+D"; -+ char *inbuf = in; -+ size_t inleft = strlen (in); -+ char out[3]; /* Space for one output character. */ -+ char *outbuf; -+ size_t outleft; -+ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), (size_t) -1); -+ TEST_COMPARE (errno, E2BIG); -+ TEST_COMPARE (inleft, 0); -+ TEST_COMPARE (inbuf - in, strlen (in)); -+ TEST_COMPARE (outleft, sizeof (out) - 2); -+ TEST_COMPARE (outbuf - out, 2); -+ TEST_COMPARE (out[0] & 0xff, 0xc3); -+ TEST_COMPARE (out[1] & 0xff, 0xa6); -+ -+ /* Flush the pending character. */ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); -+ TEST_COMPARE (outleft, sizeof (out) - 2); -+ TEST_COMPARE (outbuf - out, 2); -+ TEST_COMPARE (out[0] & 0xff, 0xcc); -+ TEST_COMPARE (out[1] & 0xff, 0x80); -+ -+ /* Nothing should be flushed the second time. */ -+ outbuf = out; -+ outleft = sizeof (out); -+ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); -+ TEST_COMPARE (outleft, sizeof (out)); -+ TEST_COMPARE (outbuf - out, 0); -+ TEST_COMPARE (out[0] & 0xff, 0xcc); -+ TEST_COMPARE (out[1] & 0xff, 0x80); -+ -+ TEST_COMPARE (iconv_close (c), 0); -+} -+ -+static int -+do_test (void) -+{ -+ with_escape_sequence (); -+ with_flush (); -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/iconvdata/bug-iconv15.c b/iconvdata/bug-iconv15.c -new file mode 100644 -index 0000000000..cc04bd0313 ---- /dev/null -+++ b/iconvdata/bug-iconv15.c -@@ -0,0 +1,60 @@ -+/* Bug 28524: Conversion from ISO-2022-JP-3 with iconv -+ may emit spurious NUL character on state reset. -+ Copyright (C) The GNU Toolchain Authors. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <stddef.h> -+#include <iconv.h> -+#include <support/check.h> -+ -+static int -+do_test (void) -+{ -+ char in[] = "\x1b(I"; -+ char *inbuf = in; -+ size_t inleft = sizeof (in) - 1; -+ char out[1]; -+ char *outbuf = out; -+ size_t outleft = sizeof (out); -+ iconv_t cd; -+ -+ cd = iconv_open ("UTF8", "ISO-2022-JP-3"); -+ TEST_VERIFY_EXIT (cd != (iconv_t) -1); -+ -+ /* First call to iconv should alter internal state. -+ Now, JISX0201_Kana_set is selected and -+ state value != ASCII_set. */ -+ TEST_VERIFY (iconv (cd, &inbuf, &inleft, &outbuf, &outleft) != (size_t) -1); -+ -+ /* No bytes should have been added to -+ the output buffer at this point. */ -+ TEST_VERIFY (outbuf == out); -+ TEST_VERIFY (outleft == sizeof (out)); -+ -+ /* Second call shall emit spurious NUL character in unpatched glibc. */ -+ TEST_VERIFY (iconv (cd, NULL, NULL, &outbuf, &outleft) != (size_t) -1); -+ -+ /* No characters are expected to be produced. */ -+ TEST_VERIFY (outbuf == out); -+ TEST_VERIFY (outleft == sizeof (out)); -+ -+ TEST_VERIFY_EXIT (iconv_close (cd) != -1); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/iconvdata/euc-kr.c b/iconvdata/euc-kr.c -index b0d56cf3ee..1045bae926 100644 ---- a/iconvdata/euc-kr.c -+++ b/iconvdata/euc-kr.c -@@ -80,11 +80,7 @@ euckr_from_ucs4 (uint32_t ch, unsigned char *cp) - \ - if (ch <= 0x9f) \ - ++inptr; \ -- /* 0xfe(->0x7e : row 94) and 0xc9(->0x59 : row 41) are \ -- user-defined areas. */ \ -- else if (__builtin_expect (ch == 0xa0, 0) \ -- || __builtin_expect (ch > 0xfe, 0) \ -- || __builtin_expect (ch == 0xc9, 0)) \ -+ else if (__glibc_unlikely (ch == 0xa0)) \ - { \ - /* This is illegal. */ \ - STANDARD_FROM_LOOP_ERR_HANDLER (1); \ -diff --git a/iconvdata/ibm1364.c b/iconvdata/ibm1364.c -index 49e7267ab4..521f0825b7 100644 ---- a/iconvdata/ibm1364.c -+++ b/iconvdata/ibm1364.c -@@ -158,24 +158,14 @@ enum - \ - if (__builtin_expect (ch, 0) == SO) \ - { \ -- /* Shift OUT, change to DBCS converter. */ \ -- if (curcs == db) \ -- { \ -- result = __GCONV_ILLEGAL_INPUT; \ -- break; \ -- } \ -+ /* Shift OUT, change to DBCS converter (redundant escape okay). */ \ - curcs = db; \ - ++inptr; \ - continue; \ - } \ - if (__builtin_expect (ch, 0) == SI) \ - { \ -- /* Shift IN, change to SBCS converter. */ \ -- if (curcs == sb) \ -- { \ -- result = __GCONV_ILLEGAL_INPUT; \ -- break; \ -- } \ -+ /* Shift IN, change to SBCS converter (redundant escape okay). */ \ - curcs = sb; \ - ++inptr; \ - continue; \ -diff --git a/iconvdata/iso-2022-jp-3.c b/iconvdata/iso-2022-jp-3.c -index 8c3b7e627e..c7b470db61 100644 ---- a/iconvdata/iso-2022-jp-3.c -+++ b/iconvdata/iso-2022-jp-3.c -@@ -1,5 +1,6 @@ - /* Conversion module for ISO-2022-JP-3. - Copyright (C) 1998-2020 Free Software Foundation, Inc. -+ Copyright (C) The GNU Toolchain Authors. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998, - and Bruno Haible <bruno@clisp.org>, 2002. -@@ -67,10 +68,15 @@ enum - CURRENT_SEL_MASK = 7 << 3 - }; - --/* During UCS-4 to ISO-2022-JP-3 conversion, the COUNT element of the state -- also contains the last two bytes to be output, shifted by 6 bits, and a -- one-bit indicator whether they must be preceded by the shift sequence, -- in bit 22. */ -+/* During UCS-4 to ISO-2022-JP-3 conversion, the COUNT element of the -+ state also contains the last two bytes to be output, shifted by 6 -+ bits, and a one-bit indicator whether they must be preceded by the -+ shift sequence, in bit 22. During ISO-2022-JP-3 to UCS-4 -+ conversion, COUNT may also contain a non-zero pending wide -+ character, shifted by six bits. This happens for certain inputs in -+ JISX0213_1_2004_set and JISX0213_2_set if the second wide character -+ in a combining sequence cannot be written because the buffer is -+ full. */ - - /* Since this is a stateful encoding we have to provide code which resets - the output state to the initial state. This has to be done during the -@@ -80,10 +86,27 @@ enum - { \ - if (FROM_DIRECTION) \ - { \ -- /* It's easy, we don't have to emit anything, we just reset the \ -- state for the input. */ \ -- data->__statep->__count &= 7; \ -- data->__statep->__count |= ASCII_set; \ -+ uint32_t ch = data->__statep->__count >> 6; \ -+ \ -+ if (__glibc_unlikely (ch != 0)) \ -+ { \ -+ if (__glibc_likely (outbuf + 4 <= outend)) \ -+ { \ -+ /* Write out the last character. */ \ -+ put32u (outbuf, ch); \ -+ outbuf += 4; \ -+ data->__statep->__count &= 7; \ -+ data->__statep->__count |= ASCII_set; \ -+ } \ -+ else \ -+ /* We don't have enough room in the output buffer. */ \ -+ status = __GCONV_FULL_OUTPUT; \ -+ } \ -+ else \ -+ { \ -+ data->__statep->__count &= 7; \ -+ data->__statep->__count |= ASCII_set; \ -+ } \ - } \ - else \ - { \ -@@ -151,7 +174,21 @@ enum - #define LOOPFCT FROM_LOOP - #define BODY \ - { \ -- uint32_t ch = *inptr; \ -+ uint32_t ch; \ -+ \ -+ /* Output any pending character. */ \ -+ ch = set >> 6; \ -+ if (__glibc_unlikely (ch != 0)) \ -+ { \ -+ put32 (outptr, ch); \ -+ outptr += 4; \ -+ /* Remove the pending character, but preserve state bits. */ \ -+ set &= (1 << 6) - 1; \ -+ continue; \ -+ } \ -+ \ -+ /* Otherwise read the next input byte. */ \ -+ ch = *inptr; \ - \ - /* Recognize escape sequences. */ \ - if (__glibc_unlikely (ch == ESC)) \ -@@ -297,21 +334,25 @@ enum - uint32_t u1 = __jisx0213_to_ucs_combining[ch - 1][0]; \ - uint32_t u2 = __jisx0213_to_ucs_combining[ch - 1][1]; \ - \ -+ inptr += 2; \ -+ \ -+ put32 (outptr, u1); \ -+ outptr += 4; \ -+ \ - /* See whether we have room for two characters. */ \ -- if (outptr + 8 <= outend) \ -+ if (outptr + 4 <= outend) \ - { \ -- inptr += 2; \ -- put32 (outptr, u1); \ -- outptr += 4; \ - put32 (outptr, u2); \ - outptr += 4; \ - continue; \ - } \ -- else \ -- { \ -- result = __GCONV_FULL_OUTPUT; \ -- break; \ -- } \ -+ \ -+ /* Otherwise store only the first character now, and \ -+ put the second one into the queue. */ \ -+ set |= u2 << 6; \ -+ /* Tell the caller why we terminate the loop. */ \ -+ result = __GCONV_FULL_OUTPUT; \ -+ break; \ - } \ - \ - inptr += 2; \ -diff --git a/iconvdata/ksc5601.h b/iconvdata/ksc5601.h -index d3eb3a4ff8..f5cdc72797 100644 ---- a/iconvdata/ksc5601.h -+++ b/iconvdata/ksc5601.h -@@ -50,15 +50,15 @@ ksc5601_to_ucs4 (const unsigned char **s, size_t avail, unsigned char offset) - unsigned char ch2; - int idx; - -+ if (avail < 2) -+ return 0; -+ - /* row 94(0x7e) and row 41(0x49) are user-defined area in KS C 5601 */ - - if (ch < offset || (ch - offset) <= 0x20 || (ch - offset) >= 0x7e - || (ch - offset) == 0x49) - return __UNKNOWN_10646_CHAR; - -- if (avail < 2) -- return 0; -- - ch2 = (*s)[1]; - if (ch2 < offset || (ch2 - offset) <= 0x20 || (ch2 - offset) >= 0x7f) - return __UNKNOWN_10646_CHAR; -diff --git a/intl/dcigettext.c b/intl/dcigettext.c -index 2e7c662bc7..bd332e71da 100644 ---- a/intl/dcigettext.c -+++ b/intl/dcigettext.c -@@ -1120,15 +1120,18 @@ _nl_find_msg (struct loaded_l10nfile *domain_file, - - # ifdef _LIBC - -- struct gconv_spec conv_spec -- = { .fromcode = norm_add_slashes (charset, ""), -- .tocode = norm_add_slashes (outcharset, ""), -- /* We always want to use transliteration. */ -- .translit = true, -- .ignore = false -- }; -+ struct gconv_spec conv_spec; -+ -+ __gconv_create_spec (&conv_spec, charset, outcharset); -+ -+ /* We always want to use transliteration. */ -+ conv_spec.translit = true; -+ - int r = __gconv_open (&conv_spec, &convd->conv, - GCONV_AVOID_NOCONV); -+ -+ __gconv_destroy_spec (&conv_spec); -+ - if (__builtin_expect (r != __GCONV_OK, 0)) - { - /* If the output encoding is the same there is -diff --git a/intl/tst-codeset.c b/intl/tst-codeset.c -index fd70432eca..e9f6e5e09f 100644 ---- a/intl/tst-codeset.c -+++ b/intl/tst-codeset.c -@@ -22,13 +22,11 @@ - #include <stdio.h> - #include <stdlib.h> - #include <string.h> -+#include <support/check.h> - - static int - do_test (void) - { -- char *s; -- int result = 0; -- - unsetenv ("LANGUAGE"); - unsetenv ("OUTPUT_CHARSET"); - setlocale (LC_ALL, "de_DE.ISO-8859-1"); -@@ -36,25 +34,21 @@ do_test (void) - bindtextdomain ("codeset", OBJPFX "domaindir"); - - /* Here we expect output in ISO-8859-1. */ -- s = gettext ("cheese"); -- if (strcmp (s, "K\344se")) -- { -- printf ("call 1 returned: %s\n", s); -- result = 1; -- } -+ TEST_COMPARE_STRING (gettext ("cheese"), "K\344se"); - -+ /* Here we expect output in UTF-8. */ - bind_textdomain_codeset ("codeset", "UTF-8"); -+ TEST_COMPARE_STRING (gettext ("cheese"), "K\303\244se"); - -- /* Here we expect output in UTF-8. */ -- s = gettext ("cheese"); -- if (strcmp (s, "K\303\244se")) -- { -- printf ("call 2 returned: %s\n", s); -- result = 1; -- } -- -- return result; -+ /* `a with umlaut' is transliterated to `ae'. */ -+ bind_textdomain_codeset ("codeset", "ASCII//TRANSLIT"); -+ TEST_COMPARE_STRING (gettext ("cheese"), "Kaese"); -+ -+ /* Transliteration also works by default even if not set. */ -+ bind_textdomain_codeset ("codeset", "ASCII"); -+ TEST_COMPARE_STRING (gettext ("cheese"), "Kaese"); -+ -+ return 0; - } - --#define TEST_FUNCTION do_test () --#include "../test-skeleton.c" -+#include <support/test-driver.c> -diff --git a/malloc/Makefile b/malloc/Makefile -index e22cbde22d..5093e8730e 100644 ---- a/malloc/Makefile -+++ b/malloc/Makefile -@@ -62,6 +62,16 @@ endif - tests += $(tests-static) - test-srcs = tst-mtrace - -+# These tests either are run with MALLOC_CHECK_=3 by default or do not work -+# with MALLOC_CHECK_=3 because they expect a specific failure. -+tests-exclude-mcheck = tst-mcheck tst-malloc-usable \ -+ tst-interpose-nothread tst-interpose-static-nothread \ -+ tst-interpose-static-thread tst-malloc-too-large \ -+ tst-mxfast tst-safe-linking -+ -+# Run all tests with MALLOC_CHECK_=3 -+tests-mcheck = $(filter-out $(tests-exclude-mcheck),$(tests)) -+ - routines = malloc morecore mcheck mtrace obstack reallocarray \ - scratch_buffer_grow scratch_buffer_grow_preserve \ - scratch_buffer_set_array_size \ -@@ -100,6 +110,11 @@ $(objpfx)tst-malloc-thread-exit: $(shared-thread-library) - $(objpfx)tst-malloc-thread-fail: $(shared-thread-library) - $(objpfx)tst-malloc-fork-deadlock: $(shared-thread-library) - $(objpfx)tst-malloc-stats-cancellation: $(shared-thread-library) -+$(objpfx)tst-malloc-backtrace-mcheck: $(shared-thread-library) -+$(objpfx)tst-malloc-thread-exit-mcheck: $(shared-thread-library) -+$(objpfx)tst-malloc-thread-fail-mcheck: $(shared-thread-library) -+$(objpfx)tst-malloc-fork-deadlock-mcheck: $(shared-thread-library) -+$(objpfx)tst-malloc-stats-cancellation-mcheck: $(shared-thread-library) - - # Export the __malloc_initialize_hook variable to libc.so. - LDFLAGS-tst-mallocstate = -rdynamic -@@ -239,6 +254,8 @@ $(tests:%=$(objpfx)%.o): CPPFLAGS += -DTEST_NO_MALLOPT - $(objpfx)tst-interpose-nothread: $(objpfx)tst-interpose-aux-nothread.o - $(objpfx)tst-interpose-thread: \ - $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) -+$(objpfx)tst-interpose-thread-mcheck: \ -+ $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) - $(objpfx)tst-interpose-static-nothread: $(objpfx)tst-interpose-aux-nothread.o - $(objpfx)tst-interpose-static-thread: \ - $(objpfx)tst-interpose-aux-thread.o $(static-thread-library) -@@ -256,3 +273,6 @@ $(objpfx)tst-dynarray-fail-mem.out: $(objpfx)tst-dynarray-fail.out - $(objpfx)tst-malloc-tcache-leak: $(shared-thread-library) - $(objpfx)tst-malloc_info: $(shared-thread-library) - $(objpfx)tst-mallocfork2: $(shared-thread-library) -+$(objpfx)tst-malloc-tcache-leak-mcheck: $(shared-thread-library) -+$(objpfx)tst-malloc_info-mcheck: $(shared-thread-library) -+$(objpfx)tst-mallocfork2-mcheck: $(shared-thread-library) -diff --git a/manual/tunables.texi b/manual/tunables.texi -index 23ef0d40e7..d72d7a5ec0 100644 ---- a/manual/tunables.texi -+++ b/manual/tunables.texi -@@ -432,7 +432,11 @@ set shared cache size in bytes for use in memory and string routines. - - @deftp Tunable glibc.cpu.x86_non_temporal_threshold - The @code{glibc.cpu.x86_non_temporal_threshold} tunable allows the user --to set threshold in bytes for non temporal store. -+to set threshold in bytes for non temporal store. Non temporal stores -+give a hint to the hardware to move data directly to memory without -+displacing other data from the cache. This tunable is used by some -+platforms to determine when to use non temporal stores in operations -+like memmove and memcpy. - - This tunable is specific to i386 and x86-64. - @end deftp -diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h -index 19d9cc5cfe..38221d0b2a 100644 ---- a/misc/sys/cdefs.h -+++ b/misc/sys/cdefs.h -@@ -124,13 +124,10 @@ - #define __bos0(ptr) __builtin_object_size (ptr, 0) - - #if __GNUC_PREREQ (4,3) --# define __warndecl(name, msg) \ -- extern void name (void) __attribute__((__warning__ (msg))) - # define __warnattr(msg) __attribute__((__warning__ (msg))) - # define __errordecl(name, msg) \ - extern void name (void) __attribute__((__error__ (msg))) - #else --# define __warndecl(name, msg) extern void name (void) - # define __warnattr(msg) - # define __errordecl(name, msg) extern void name (void) - #endif -diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c -index 2cba3da38c..c217cda608 100644 ---- a/nptl/pthread_create.c -+++ b/nptl/pthread_create.c -@@ -416,8 +416,6 @@ START_THREAD_DEFN - unwind_buf.priv.data.prev = NULL; - unwind_buf.priv.data.cleanup = NULL; - -- __libc_signal_restore_set (&pd->sigmask); -- - /* Allow setxid from now onwards. */ - if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) - futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); -@@ -427,6 +425,8 @@ START_THREAD_DEFN - /* Store the new cleanup handler info. */ - THREAD_SETMEM (pd, cleanup_jmp_buf, &unwind_buf); - -+ __libc_signal_restore_set (&pd->sigmask); -+ - /* We are either in (a) or (b), and in either case we either own - PD already (2) or are about to own PD (1), and so our only - restriction would be that we can't free PD until we know we -diff --git a/nscd/netgroupcache.c b/nscd/netgroupcache.c -index 88c69d1e9c..381aa721ef 100644 ---- a/nscd/netgroupcache.c -+++ b/nscd/netgroupcache.c -@@ -248,7 +248,7 @@ addgetnetgrentX (struct database_dyn *db, int fd, request_header *req, - : NULL); - ndomain = (ndomain ? newbuf + ndomaindiff - : NULL); -- buffer = newbuf; -+ *tofreep = buffer = newbuf; - } - - nhost = memcpy (buffer + bufused, -@@ -319,7 +319,7 @@ addgetnetgrentX (struct database_dyn *db, int fd, request_header *req, - else if (status == NSS_STATUS_TRYAGAIN && e == ERANGE) - { - buflen *= 2; -- buffer = xrealloc (buffer, buflen); -+ *tofreep = buffer = xrealloc (buffer, buflen); - } - else if (status == NSS_STATUS_RETURN - || status == NSS_STATUS_NOTFOUND -diff --git a/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf b/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf -new file mode 100644 -index 0000000000..5b0c6a4199 ---- /dev/null -+++ b/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf -@@ -0,0 +1 @@ -+hosts: files -diff --git a/posix/bits/unistd.h b/posix/bits/unistd.h -index 725a83eb0d..7e5bb6fb1e 100644 ---- a/posix/bits/unistd.h -+++ b/posix/bits/unistd.h -@@ -193,10 +193,9 @@ __NTH (readlinkat (int __fd, const char *__restrict __path, - #endif - - extern char *__getcwd_chk (char *__buf, size_t __size, size_t __buflen) -- __THROW __wur __attr_access ((__write_only__, 1, 2)); -+ __THROW __wur; - extern char *__REDIRECT_NTH (__getcwd_alias, -- (char *__buf, size_t __size), getcwd) -- __wur __attr_access ((__write_only__, 1, 2)); -+ (char *__buf, size_t __size), getcwd) __wur; - extern char *__REDIRECT_NTH (__getcwd_chk_warn, - (char *__buf, size_t __size, size_t __buflen), - __getcwd_chk) -diff --git a/posix/unistd.h b/posix/unistd.h -index 32b8161619..acf9ee7e79 100644 ---- a/posix/unistd.h -+++ b/posix/unistd.h -@@ -517,8 +517,7 @@ extern int fchdir (int __fd) __THROW __wur; - an array is allocated with `malloc'; the array is SIZE - bytes long, unless SIZE == 0, in which case it is as - big as necessary. */ --extern char *getcwd (char *__buf, size_t __size) __THROW __wur -- __attr_access ((__write_only__, 1, 2)); -+extern char *getcwd (char *__buf, size_t __size) __THROW __wur; - - #ifdef __USE_GNU - /* Return a malloc'd string containing the current directory name. -@@ -831,7 +830,7 @@ extern int symlinkat (const char *__from, int __tofd, - /* Like readlink but a relative PATH is interpreted relative to FD. */ - extern ssize_t readlinkat (int __fd, const char *__restrict __path, - char *__restrict __buf, size_t __len) -- __THROW __nonnull ((2, 3)) __wur __attr_access ((__read_only__, 3, 4)); -+ __THROW __nonnull ((2, 3)) __wur __attr_access ((__write_only__, 3, 4)); - #endif - - /* Remove the link NAME. */ -diff --git a/posix/wordexp-test.c b/posix/wordexp-test.c -index ed1b22308e..cb3f989cba 100644 ---- a/posix/wordexp-test.c -+++ b/posix/wordexp-test.c -@@ -183,6 +183,7 @@ struct test_case_struct - { 0, NULL, "$var", 0, 0, { NULL, }, IFS }, - { 0, NULL, "\"\\n\"", 0, 1, { "\\n", }, IFS }, - { 0, NULL, "", 0, 0, { NULL, }, IFS }, -+ { 0, NULL, "${1234567890123456789012}", 0, 0, { NULL, }, IFS }, - - /* Flags not already covered (testit() has special handling for these) */ - { 0, NULL, "one two", WRDE_DOOFFS, 2, { "one", "two", }, IFS }, -diff --git a/posix/wordexp.c b/posix/wordexp.c -index e082d94895..56289503a1 100644 ---- a/posix/wordexp.c -+++ b/posix/wordexp.c -@@ -1399,7 +1399,7 @@ envsubst: - /* Is it a numeric parameter? */ - else if (isdigit (env[0])) - { -- int n = atoi (env); -+ unsigned long n = strtoul (env, NULL, 10); - - if (n >= __libc_argc) - /* Substitute NULL. */ -diff --git a/resolv/Makefile b/resolv/Makefile -index b61c0c3e0c..dbd8f8bf4f 100644 ---- a/resolv/Makefile -+++ b/resolv/Makefile -@@ -61,6 +61,11 @@ tests += \ - tst-resolv-search \ - tst-resolv-trailing \ - -+# This test calls __res_context_send directly, which is not exported -+# from libresolv. -+tests-internal += tst-resolv-txnid-collision -+tests-static += tst-resolv-txnid-collision -+ - # These tests need libdl. - ifeq (yes,$(build-shared)) - tests += \ -@@ -191,6 +196,8 @@ $(objpfx)tst-resolv-search: $(objpfx)libresolv.so $(shared-thread-library) - $(objpfx)tst-resolv-trailing: $(objpfx)libresolv.so $(shared-thread-library) - $(objpfx)tst-resolv-threads: \ - $(libdl) $(objpfx)libresolv.so $(shared-thread-library) -+$(objpfx)tst-resolv-txnid-collision: $(objpfx)libresolv.a \ -+ $(static-thread-library) - $(objpfx)tst-resolv-canonname: \ - $(libdl) $(objpfx)libresolv.so $(shared-thread-library) - $(objpfx)tst-resolv-trustad: $(objpfx)libresolv.so $(shared-thread-library) -diff --git a/resolv/res_send.c b/resolv/res_send.c -index 7e5fec6646..70e5066031 100644 ---- a/resolv/res_send.c -+++ b/resolv/res_send.c -@@ -1342,15 +1342,6 @@ send_dg(res_state statp, - *terrno = EMSGSIZE; - return close_and_return_error (statp, resplen2); - } -- if ((recvresp1 || hp->id != anhp->id) -- && (recvresp2 || hp2->id != anhp->id)) { -- /* -- * response from old query, ignore it. -- * XXX - potential security hazard could -- * be detected here. -- */ -- goto wait; -- } - - /* Paranoia check. Due to the connected UDP socket, - the kernel has already filtered invalid addresses -@@ -1360,15 +1351,24 @@ send_dg(res_state statp, - - /* Check for the correct header layout and a matching - question. */ -- if ((recvresp1 || !res_queriesmatch(buf, buf + buflen, -- *thisansp, -- *thisansp -- + *thisanssizp)) -- && (recvresp2 || !res_queriesmatch(buf2, buf2 + buflen2, -- *thisansp, -- *thisansp -- + *thisanssizp))) -- goto wait; -+ int matching_query = 0; /* Default to no matching query. */ -+ if (!recvresp1 -+ && anhp->id == hp->id -+ && res_queriesmatch (buf, buf + buflen, -+ *thisansp, *thisansp + *thisanssizp)) -+ matching_query = 1; -+ if (!recvresp2 -+ && anhp->id == hp2->id -+ && res_queriesmatch (buf2, buf2 + buflen2, -+ *thisansp, *thisansp + *thisanssizp)) -+ matching_query = 2; -+ if (matching_query == 0) -+ /* Spurious UDP packet. Drop it and continue -+ waiting. */ -+ { -+ need_recompute = 1; -+ goto wait; -+ } - - if (anhp->rcode == SERVFAIL || - anhp->rcode == NOTIMP || -@@ -1383,7 +1383,7 @@ send_dg(res_state statp, - /* No data from the first reply. */ - resplen = 0; - /* We are waiting for a possible second reply. */ -- if (hp->id == anhp->id) -+ if (matching_query == 1) - recvresp1 = 1; - else - recvresp2 = 1; -@@ -1414,7 +1414,7 @@ send_dg(res_state statp, - return (1); - } - /* Mark which reply we received. */ -- if (recvresp1 == 0 && hp->id == anhp->id) -+ if (matching_query == 1) - recvresp1 = 1; - else - recvresp2 = 1; -diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c -new file mode 100644 -index 0000000000..189b76f126 ---- /dev/null -+++ b/resolv/tst-resolv-txnid-collision.c -@@ -0,0 +1,334 @@ -+/* Test parallel queries with transaction ID collisions. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <arpa/nameser.h> -+#include <array_length.h> -+#include <resolv-internal.h> -+#include <resolv_context.h> -+#include <stdbool.h> -+#include <stdio.h> -+#include <string.h> -+#include <support/check.h> -+#include <support/check_nss.h> -+#include <support/resolv_test.h> -+#include <support/support.h> -+#include <support/test-driver.h> -+ -+/* Result of parsing a DNS question name. -+ -+ A question name has the form reorder-N-M-rcode-C.example.net, where -+ N and M are either 0 and 1, corresponding to the reorder member, -+ and C is a number that will be stored in the rcode field. -+ -+ Also see parse_qname below. */ -+struct parsed_qname -+{ -+ /* The DNS response code requested from the first server. The -+ second server always responds with RCODE zero. */ -+ int rcode; -+ -+ /* Indicates whether to perform reordering in the responses from the -+ respective server. */ -+ bool reorder[2]; -+}; -+ -+/* Fills *PARSED based on QNAME. */ -+static void -+parse_qname (struct parsed_qname *parsed, const char *qname) -+{ -+ int reorder0; -+ int reorder1; -+ int rcode; -+ char *suffix; -+ if (sscanf (qname, "reorder-%d-%d.rcode-%d.%ms", -+ &reorder0, &reorder1, &rcode, &suffix) == 4) -+ { -+ if (reorder0 != 0) -+ TEST_COMPARE (reorder0, 1); -+ if (reorder1 != 0) -+ TEST_COMPARE (reorder1, 1); -+ TEST_VERIFY (rcode >= 0 && rcode <= 15); -+ TEST_COMPARE_STRING (suffix, "example.net"); -+ free (suffix); -+ -+ parsed->rcode = rcode; -+ parsed->reorder[0] = reorder0; -+ parsed->reorder[1] = reorder1; -+ } -+ else -+ FAIL_EXIT1 ("unexpected query: %s", qname); -+} -+ -+/* Used to construct a response. The first server responds with an -+ error, the second server succeeds. */ -+static void -+build_response (const struct resolv_response_context *ctx, -+ struct resolv_response_builder *b, -+ const char *qname, uint16_t qclass, uint16_t qtype) -+{ -+ struct parsed_qname parsed; -+ parse_qname (&parsed, qname); -+ -+ switch (ctx->server_index) -+ { -+ case 0: -+ { -+ struct resolv_response_flags flags = { 0 }; -+ if (parsed.rcode == 0) -+ /* Simulate a delegation in case a NODATA (RCODE zero) -+ response is requested. */ -+ flags.clear_ra = true; -+ else -+ flags.rcode = parsed.rcode; -+ -+ resolv_response_init (b, flags); -+ resolv_response_add_question (b, qname, qclass, qtype); -+ } -+ break; -+ -+ case 1: -+ { -+ struct resolv_response_flags flags = { 0, }; -+ resolv_response_init (b, flags); -+ resolv_response_add_question (b, qname, qclass, qtype); -+ -+ resolv_response_section (b, ns_s_an); -+ resolv_response_open_record (b, qname, qclass, qtype, 0); -+ if (qtype == T_A) -+ { -+ char ipv4[4] = { 192, 0, 2, 1 }; -+ resolv_response_add_data (b, &ipv4, sizeof (ipv4)); -+ } -+ else -+ { -+ char ipv6[16] -+ = { 0x20, 0x01, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }; -+ resolv_response_add_data (b, &ipv6, sizeof (ipv6)); -+ } -+ resolv_response_close_record (b); -+ } -+ break; -+ } -+} -+ -+/* Used to reorder responses. */ -+struct resolv_response_context *previous_query; -+ -+/* Used to keep track of the queries received. */ -+static int previous_server_index = -1; -+static uint16_t previous_qtype; -+ -+/* For each server, buffer the first query and then send both answers -+ to the second query, reordered if requested. */ -+static void -+response (const struct resolv_response_context *ctx, -+ struct resolv_response_builder *b, -+ const char *qname, uint16_t qclass, uint16_t qtype) -+{ -+ TEST_VERIFY (qtype == T_A || qtype == T_AAAA); -+ if (ctx->server_index != 0) -+ TEST_COMPARE (ctx->server_index, 1); -+ -+ struct parsed_qname parsed; -+ parse_qname (&parsed, qname); -+ -+ if (previous_query == NULL) -+ { -+ /* No buffered query. Record this query and do not send a -+ response. */ -+ TEST_COMPARE (previous_qtype, 0); -+ previous_query = resolv_response_context_duplicate (ctx); -+ previous_qtype = qtype; -+ resolv_response_drop (b); -+ previous_server_index = ctx->server_index; -+ -+ if (test_verbose) -+ printf ("info: buffering first query for: %s\n", qname); -+ } -+ else -+ { -+ TEST_VERIFY (previous_query != 0); -+ TEST_COMPARE (ctx->server_index, previous_server_index); -+ TEST_VERIFY (previous_qtype != qtype); /* Not a duplicate. */ -+ -+ /* If reordering, send a response for this query explicitly, and -+ then skip the implicit send. */ -+ if (parsed.reorder[ctx->server_index]) -+ { -+ if (test_verbose) -+ printf ("info: sending reordered second response for: %s\n", -+ qname); -+ build_response (ctx, b, qname, qclass, qtype); -+ resolv_response_send_udp (ctx, b); -+ resolv_response_drop (b); -+ } -+ -+ /* Build a response for the previous query and send it, thus -+ reordering the two responses. */ -+ { -+ if (test_verbose) -+ printf ("info: sending first response for: %s\n", qname); -+ struct resolv_response_builder *btmp -+ = resolv_response_builder_allocate (previous_query->query_buffer, -+ previous_query->query_length); -+ build_response (ctx, btmp, qname, qclass, previous_qtype); -+ resolv_response_send_udp (ctx, btmp); -+ resolv_response_builder_free (btmp); -+ } -+ -+ /* If not reordering, send the reply as usual. */ -+ if (!parsed.reorder[ctx->server_index]) -+ { -+ if (test_verbose) -+ printf ("info: sending non-reordered second response for: %s\n", -+ qname); -+ build_response (ctx, b, qname, qclass, qtype); -+ } -+ -+ /* Unbuffer the response and prepare for the next query. */ -+ resolv_response_context_free (previous_query); -+ previous_query = NULL; -+ previous_qtype = 0; -+ previous_server_index = -1; -+ } -+} -+ -+/* Runs a query for QNAME and checks for the expected reply. See -+ struct parsed_qname for the expected format for QNAME. */ -+static void -+test_qname (const char *qname, int rcode) -+{ -+ struct resolv_context *ctx = __resolv_context_get (); -+ TEST_VERIFY_EXIT (ctx != NULL); -+ -+ unsigned char q1[512]; -+ int q1len = res_mkquery (QUERY, qname, C_IN, T_A, NULL, 0, NULL, -+ q1, sizeof (q1)); -+ TEST_VERIFY_EXIT (q1len > 12); -+ -+ unsigned char q2[512]; -+ int q2len = res_mkquery (QUERY, qname, C_IN, T_AAAA, NULL, 0, NULL, -+ q2, sizeof (q2)); -+ TEST_VERIFY_EXIT (q2len > 12); -+ -+ /* Produce a transaction ID collision. */ -+ memcpy (q2, q1, 2); -+ -+ unsigned char ans1[512]; -+ unsigned char *ans1p = ans1; -+ unsigned char *ans2p = NULL; -+ int nans2p = 0; -+ int resplen2 = 0; -+ int ans2p_malloced = 0; -+ -+ /* Perform a parallel A/AAAA query. */ -+ int resplen1 = __res_context_send (ctx, q1, q1len, q2, q2len, -+ ans1, sizeof (ans1), &ans1p, -+ &ans2p, &nans2p, -+ &resplen2, &ans2p_malloced); -+ -+ TEST_VERIFY (resplen1 > 12); -+ TEST_VERIFY (resplen2 > 12); -+ if (resplen1 <= 12 || resplen2 <= 12) -+ return; -+ -+ if (rcode == 1 || rcode == 3) -+ { -+ /* Format Error and Name Error responses does not trigger -+ switching to the next server. */ -+ TEST_COMPARE (ans1p[3] & 0x0f, rcode); -+ TEST_COMPARE (ans2p[3] & 0x0f, rcode); -+ return; -+ } -+ -+ /* The response should be successful. */ -+ TEST_COMPARE (ans1p[3] & 0x0f, 0); -+ TEST_COMPARE (ans2p[3] & 0x0f, 0); -+ -+ /* Due to bug 19691, the answer may not be in the slot matching the -+ query. Assume that the AAAA response is the longer one. */ -+ unsigned char *a_answer; -+ int a_answer_length; -+ unsigned char *aaaa_answer; -+ int aaaa_answer_length; -+ if (resplen2 > resplen1) -+ { -+ a_answer = ans1p; -+ a_answer_length = resplen1; -+ aaaa_answer = ans2p; -+ aaaa_answer_length = resplen2; -+ } -+ else -+ { -+ a_answer = ans2p; -+ a_answer_length = resplen2; -+ aaaa_answer = ans1p; -+ aaaa_answer_length = resplen1; -+ } -+ -+ { -+ char *expected = xasprintf ("name: %s\n" -+ "address: 192.0.2.1\n", -+ qname); -+ check_dns_packet (qname, a_answer, a_answer_length, expected); -+ free (expected); -+ } -+ { -+ char *expected = xasprintf ("name: %s\n" -+ "address: 2001:db8::1\n", -+ qname); -+ check_dns_packet (qname, aaaa_answer, aaaa_answer_length, expected); -+ free (expected); -+ } -+ -+ if (ans2p_malloced) -+ free (ans2p); -+ -+ __resolv_context_put (ctx); -+} -+ -+static int -+do_test (void) -+{ -+ struct resolv_test *aux = resolv_test_start -+ ((struct resolv_redirect_config) -+ { -+ .response_callback = response, -+ -+ /* The response callback use global state (the previous_* -+ variables), and query processing must therefore be -+ serialized. */ -+ .single_thread_udp = true, -+ }); -+ -+ for (int rcode = 0; rcode <= 5; ++rcode) -+ for (int do_reorder_0 = 0; do_reorder_0 < 2; ++do_reorder_0) -+ for (int do_reorder_1 = 0; do_reorder_1 < 2; ++do_reorder_1) -+ { -+ char *qname = xasprintf ("reorder-%d-%d.rcode-%d.example.net", -+ do_reorder_0, do_reorder_1, rcode); -+ test_qname (qname, rcode); -+ free (qname); -+ } -+ -+ resolv_test_end (aux); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/rt/Makefile b/rt/Makefile -index dab5d62a57..93502cfaa7 100644 ---- a/rt/Makefile -+++ b/rt/Makefile -@@ -44,6 +44,7 @@ tests := tst-shm tst-timer tst-timer2 \ - tst-aio7 tst-aio8 tst-aio9 tst-aio10 \ - tst-mqueue1 tst-mqueue2 tst-mqueue3 tst-mqueue4 \ - tst-mqueue5 tst-mqueue6 tst-mqueue7 tst-mqueue8 tst-mqueue9 \ -+ tst-bz28213 \ - tst-timer3 tst-timer4 tst-timer5 \ - tst-cpuclock2 tst-cputimer1 tst-cputimer2 tst-cputimer3 \ - tst-shm-cancel -diff --git a/rt/tst-bz28213.c b/rt/tst-bz28213.c -new file mode 100644 -index 0000000000..0c096b5a0a ---- /dev/null -+++ b/rt/tst-bz28213.c -@@ -0,0 +1,101 @@ -+/* Bug 28213: test for NULL pointer dereference in mq_notify. -+ Copyright (C) The GNU Toolchain Authors. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <errno.h> -+#include <sys/types.h> -+#include <sys/stat.h> -+#include <fcntl.h> -+#include <unistd.h> -+#include <mqueue.h> -+#include <signal.h> -+#include <stdlib.h> -+#include <string.h> -+#include <support/check.h> -+ -+static mqd_t m = -1; -+static const char msg[] = "hello"; -+ -+static void -+check_bz28213_cb (union sigval sv) -+{ -+ char buf[sizeof (msg)]; -+ -+ (void) sv; -+ -+ TEST_VERIFY_EXIT ((size_t) mq_receive (m, buf, sizeof (buf), NULL) -+ == sizeof (buf)); -+ TEST_VERIFY_EXIT (memcmp (buf, msg, sizeof (buf)) == 0); -+ -+ exit (0); -+} -+ -+static void -+check_bz28213 (void) -+{ -+ struct sigevent sev; -+ -+ memset (&sev, '\0', sizeof (sev)); -+ sev.sigev_notify = SIGEV_THREAD; -+ sev.sigev_notify_function = check_bz28213_cb; -+ -+ /* Step 1: Register & unregister notifier. -+ Helper thread should receive NOTIFY_REMOVED notification. -+ In a vulnerable version of glibc, NULL pointer dereference follows. */ -+ TEST_VERIFY_EXIT (mq_notify (m, &sev) == 0); -+ TEST_VERIFY_EXIT (mq_notify (m, NULL) == 0); -+ -+ /* Step 2: Once again, register notification. -+ Try to send one message. -+ Test is considered successful, if the callback does exit (0). */ -+ TEST_VERIFY_EXIT (mq_notify (m, &sev) == 0); -+ TEST_VERIFY_EXIT (mq_send (m, msg, sizeof (msg), 1) == 0); -+ -+ /* Wait... */ -+ pause (); -+} -+ -+static int -+do_test (void) -+{ -+ static const char m_name[] = "/bz28213_queue"; -+ struct mq_attr m_attr; -+ -+ memset (&m_attr, '\0', sizeof (m_attr)); -+ m_attr.mq_maxmsg = 1; -+ m_attr.mq_msgsize = sizeof (msg); -+ -+ m = mq_open (m_name, -+ O_RDWR | O_CREAT | O_EXCL, -+ 0600, -+ &m_attr); -+ -+ if (m < 0) -+ { -+ if (errno == ENOSYS) -+ FAIL_UNSUPPORTED ("POSIX message queues are not implemented\n"); -+ FAIL_EXIT1 ("Failed to create POSIX message queue: %m\n"); -+ } -+ -+ TEST_VERIFY_EXIT (mq_unlink (m_name) == 0); -+ -+ check_bz28213 (); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/stdio-common/Makefile b/stdio-common/Makefile -index 8475fd1f09..eff0c98d82 100644 ---- a/stdio-common/Makefile -+++ b/stdio-common/Makefile -@@ -69,7 +69,8 @@ tests := tstscanf test_rdwr test-popen tstgetln test-fseek \ - tst-printf-bz25691 \ - tst-vfprintf-width-prec-alloc \ - tst-printf-fp-free \ -- tst-printf-fp-leak -+ tst-printf-fp-leak \ -+ test-strerr - - - test-srcs = tst-unbputc tst-printf tst-printfsz-islongdouble -diff --git a/stdio-common/errlist.c b/stdio-common/errlist.c -index d15f13a22a..2ecf121674 100644 ---- a/stdio-common/errlist.c -+++ b/stdio-common/errlist.c -@@ -20,9 +20,13 @@ - #include <libintl.h> - #include <array_length.h> - -+#ifndef ERR_MAP -+# define ERR_MAP(n) n -+#endif -+ - const char *const _sys_errlist_internal[] = - { --#define _S(n, str) [n] = str, -+#define _S(n, str) [ERR_MAP(n)] = str, - #include <errlist.h> - #undef _S - }; -@@ -41,20 +45,21 @@ static const union sys_errname_t - { - #define MSGSTRFIELD1(line) str##line - #define MSGSTRFIELD(line) MSGSTRFIELD1(line) --#define _S(n, str) char MSGSTRFIELD(__LINE__)[sizeof(str)]; -+#define _S(n, str) char MSGSTRFIELD(__LINE__)[sizeof(#n)]; - #include <errlist.h> - #undef _S - }; - char str[0]; - } _sys_errname = { { --#define _S(n, s) s, -+#define _S(n, s) #n, - #include <errlist.h> - #undef _S - } }; - - static const unsigned short _sys_errnameidx[] = - { --#define _S(n, s) [n] = offsetof(union sys_errname_t, MSGSTRFIELD(__LINE__)), -+#define _S(n, s) \ -+ [ERR_MAP(n)] = offsetof(union sys_errname_t, MSGSTRFIELD(__LINE__)), - #include <errlist.h> - #undef _S - }; -diff --git a/stdio-common/test-strerr.c b/stdio-common/test-strerr.c -index fded208118..d77b81d507 100644 ---- a/stdio-common/test-strerr.c -+++ b/stdio-common/test-strerr.c -@@ -18,46 +18,672 @@ - - #include <string.h> - #include <errno.h> --#include <array_length.h> - - #include <support/support.h> - #include <support/check.h> - --#define N_(name) name -- --static const char *const errlist[] = -- { --/* This file is auto-generated from errlist.def. */ --#include <errlist.h> -- }; -- --#define MSGSTR_T errname_t --#define MSGSTR errname --#define MSGIDX errnameidx --#include <errlist-name.h> --#undef MSGSTR --#undef MSGIDX -- - static int - do_test (void) - { -- TEST_VERIFY (strerrordesc_np (-1) == NULL); -- TEST_VERIFY (strerrordesc_np (array_length (errlist)) == NULL); -- for (size_t i = 0; i < array_length (errlist); i++) -- { -- if (errlist[i] == NULL) -- continue; -- TEST_COMPARE_STRING (strerrordesc_np (i), errlist[i]); -- } -+ TEST_COMPARE_STRING (strerrordesc_np (0), "Success"); -+ TEST_COMPARE_STRING (strerrorname_np (0), "0"); - -- TEST_VERIFY (strerrorname_np (-1) == NULL); -- TEST_VERIFY (strerrorname_np (array_length (errlist)) == NULL); -- for (size_t i = 0; i < array_length (errlist); i++) -- { -- if (errlist[i] == NULL) -- continue; -- TEST_COMPARE_STRING (strerrorname_np (i), errname.str + errnameidx[i]); -- } -+#ifdef EPERM -+ TEST_COMPARE_STRING (strerrordesc_np (EPERM), "Operation not permitted"); -+ TEST_COMPARE_STRING (strerrorname_np (EPERM), "EPERM"); -+#endif -+#ifdef ENOENT -+ TEST_COMPARE_STRING (strerrordesc_np (ENOENT), -+ "No such file or directory"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOENT), "ENOENT"); -+#endif -+#ifdef ESRCH -+ TEST_COMPARE_STRING (strerrordesc_np (ESRCH), "No such process"); -+ TEST_COMPARE_STRING (strerrorname_np (ESRCH), "ESRCH"); -+#endif -+#ifdef EINTR -+ TEST_COMPARE_STRING (strerrordesc_np (EINTR), "Interrupted system call"); -+ TEST_COMPARE_STRING (strerrorname_np (EINTR), "EINTR"); -+#endif -+#ifdef EIO -+ TEST_COMPARE_STRING (strerrordesc_np (EIO), "Input/output error"); -+ TEST_COMPARE_STRING (strerrorname_np (EIO), "EIO"); -+#endif -+#ifdef ENXIO -+ TEST_COMPARE_STRING (strerrordesc_np (ENXIO), "No such device or address"); -+ TEST_COMPARE_STRING (strerrorname_np (ENXIO), "ENXIO"); -+#endif -+#ifdef E2BIG -+ TEST_COMPARE_STRING (strerrordesc_np (E2BIG), "Argument list too long"); -+ TEST_COMPARE_STRING (strerrorname_np (E2BIG), "E2BIG"); -+#endif -+#ifdef ENOEXEC -+ TEST_COMPARE_STRING (strerrordesc_np (ENOEXEC), "Exec format error"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOEXEC), "ENOEXEC"); -+#endif -+#ifdef EBADF -+ TEST_COMPARE_STRING (strerrordesc_np (EBADF), "Bad file descriptor"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADF), "EBADF"); -+#endif -+#ifdef ECHILD -+ TEST_COMPARE_STRING (strerrordesc_np (ECHILD), "No child processes"); -+ TEST_COMPARE_STRING (strerrorname_np (ECHILD), "ECHILD"); -+#endif -+#ifdef EDEADLK -+ TEST_COMPARE_STRING (strerrordesc_np (EDEADLK), -+ "Resource deadlock avoided"); -+ TEST_COMPARE_STRING (strerrorname_np (EDEADLK), "EDEADLK"); -+#endif -+#ifdef ENOMEM -+ TEST_COMPARE_STRING (strerrordesc_np (ENOMEM), "Cannot allocate memory"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOMEM), "ENOMEM"); -+#endif -+#ifdef EACCES -+ TEST_COMPARE_STRING (strerrordesc_np (EACCES), "Permission denied"); -+ TEST_COMPARE_STRING (strerrorname_np (EACCES), "EACCES"); -+#endif -+#ifdef EFAULT -+ TEST_COMPARE_STRING (strerrordesc_np (EFAULT), "Bad address"); -+ TEST_COMPARE_STRING (strerrorname_np (EFAULT), "EFAULT"); -+#endif -+#ifdef ENOTBLK -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTBLK), "Block device required"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTBLK), "ENOTBLK"); -+#endif -+#ifdef EBUSY -+ TEST_COMPARE_STRING (strerrordesc_np (EBUSY), "Device or resource busy"); -+ TEST_COMPARE_STRING (strerrorname_np (EBUSY), "EBUSY"); -+#endif -+#ifdef EEXIST -+ TEST_COMPARE_STRING (strerrordesc_np (EEXIST), "File exists"); -+ TEST_COMPARE_STRING (strerrorname_np (EEXIST), "EEXIST"); -+#endif -+#ifdef EXDEV -+ TEST_COMPARE_STRING (strerrordesc_np (EXDEV), "Invalid cross-device link"); -+ TEST_COMPARE_STRING (strerrorname_np (EXDEV), "EXDEV"); -+#endif -+#ifdef ENODEV -+ TEST_COMPARE_STRING (strerrordesc_np (ENODEV), "No such device"); -+ TEST_COMPARE_STRING (strerrorname_np (ENODEV), "ENODEV"); -+#endif -+#ifdef ENOTDIR -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTDIR), "Not a directory"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTDIR), "ENOTDIR"); -+#endif -+#ifdef EISDIR -+ TEST_COMPARE_STRING (strerrordesc_np (EISDIR), "Is a directory"); -+ TEST_COMPARE_STRING (strerrorname_np (EISDIR), "EISDIR"); -+#endif -+#ifdef EINVAL -+ TEST_COMPARE_STRING (strerrordesc_np (EINVAL), "Invalid argument"); -+ TEST_COMPARE_STRING (strerrorname_np (EINVAL), "EINVAL"); -+#endif -+#ifdef EMFILE -+ TEST_COMPARE_STRING (strerrordesc_np (EMFILE), "Too many open files"); -+ TEST_COMPARE_STRING (strerrorname_np (EMFILE), "EMFILE"); -+#endif -+#ifdef ENFILE -+ TEST_COMPARE_STRING (strerrordesc_np (ENFILE), -+ "Too many open files in system"); -+ TEST_COMPARE_STRING (strerrorname_np (ENFILE), "ENFILE"); -+#endif -+#ifdef ENOTTY -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTTY), -+ "Inappropriate ioctl for device"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTTY), "ENOTTY"); -+#endif -+#ifdef ETXTBSY -+ TEST_COMPARE_STRING (strerrordesc_np (ETXTBSY), "Text file busy"); -+ TEST_COMPARE_STRING (strerrorname_np (ETXTBSY), "ETXTBSY"); -+#endif -+#ifdef EFBIG -+ TEST_COMPARE_STRING (strerrordesc_np (EFBIG), "File too large"); -+ TEST_COMPARE_STRING (strerrorname_np (EFBIG), "EFBIG"); -+#endif -+#ifdef ENOSPC -+ TEST_COMPARE_STRING (strerrordesc_np (ENOSPC), "No space left on device"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOSPC), "ENOSPC"); -+#endif -+#ifdef ESPIPE -+ TEST_COMPARE_STRING (strerrordesc_np (ESPIPE), "Illegal seek"); -+ TEST_COMPARE_STRING (strerrorname_np (ESPIPE), "ESPIPE"); -+#endif -+#ifdef EROFS -+ TEST_COMPARE_STRING (strerrordesc_np (EROFS), "Read-only file system"); -+ TEST_COMPARE_STRING (strerrorname_np (EROFS), "EROFS"); -+#endif -+#ifdef EMLINK -+ TEST_COMPARE_STRING (strerrordesc_np (EMLINK), "Too many links"); -+ TEST_COMPARE_STRING (strerrorname_np (EMLINK), "EMLINK"); -+#endif -+#ifdef EPIPE -+ TEST_COMPARE_STRING (strerrordesc_np (EPIPE), "Broken pipe"); -+ TEST_COMPARE_STRING (strerrorname_np (EPIPE), "EPIPE"); -+#endif -+#ifdef EDOM -+ TEST_COMPARE_STRING (strerrordesc_np (EDOM), -+ "Numerical argument out of domain"); -+ TEST_COMPARE_STRING (strerrorname_np (EDOM), "EDOM"); -+#endif -+#ifdef ERANGE -+ TEST_COMPARE_STRING (strerrordesc_np (ERANGE), -+ "Numerical result out of range"); -+ TEST_COMPARE_STRING (strerrorname_np (ERANGE), "ERANGE"); -+#endif -+#ifdef EAGAIN -+ TEST_COMPARE_STRING (strerrordesc_np (EAGAIN), -+ "Resource temporarily unavailable"); -+ TEST_COMPARE_STRING (strerrorname_np (EAGAIN), "EAGAIN"); -+#endif -+#ifdef EINPROGRESS -+ TEST_COMPARE_STRING (strerrordesc_np (EINPROGRESS), -+ "Operation now in progress"); -+ TEST_COMPARE_STRING (strerrorname_np (EINPROGRESS), "EINPROGRESS"); -+#endif -+#ifdef EALREADY -+ TEST_COMPARE_STRING (strerrordesc_np (EALREADY), -+ "Operation already in progress"); -+ TEST_COMPARE_STRING (strerrorname_np (EALREADY), "EALREADY"); -+#endif -+#ifdef ENOTSOCK -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTSOCK), -+ "Socket operation on non-socket"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTSOCK), "ENOTSOCK"); -+#endif -+#ifdef EMSGSIZE -+ TEST_COMPARE_STRING (strerrordesc_np (EMSGSIZE), "Message too long"); -+ TEST_COMPARE_STRING (strerrorname_np (EMSGSIZE), "EMSGSIZE"); -+#endif -+#ifdef EPROTOTYPE -+ TEST_COMPARE_STRING (strerrordesc_np (EPROTOTYPE), -+ "Protocol wrong type for socket"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROTOTYPE), "EPROTOTYPE"); -+#endif -+#ifdef ENOPROTOOPT -+ TEST_COMPARE_STRING (strerrordesc_np (ENOPROTOOPT), -+ "Protocol not available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOPROTOOPT), "ENOPROTOOPT"); -+#endif -+#ifdef EPROTONOSUPPORT -+ TEST_COMPARE_STRING (strerrordesc_np (EPROTONOSUPPORT), -+ "Protocol not supported"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROTONOSUPPORT), "EPROTONOSUPPORT"); -+#endif -+#ifdef ESOCKTNOSUPPORT -+ TEST_COMPARE_STRING (strerrordesc_np (ESOCKTNOSUPPORT), -+ "Socket type not supported"); -+ TEST_COMPARE_STRING (strerrorname_np (ESOCKTNOSUPPORT), "ESOCKTNOSUPPORT"); -+#endif -+#ifdef EOPNOTSUPP -+ TEST_COMPARE_STRING (strerrordesc_np (EOPNOTSUPP), -+ "Operation not supported"); -+ TEST_COMPARE_STRING (strerrorname_np (EOPNOTSUPP), "EOPNOTSUPP"); -+#endif -+#ifdef EPFNOSUPPORT -+ TEST_COMPARE_STRING (strerrordesc_np (EPFNOSUPPORT), -+ "Protocol family not supported"); -+ TEST_COMPARE_STRING (strerrorname_np (EPFNOSUPPORT), "EPFNOSUPPORT"); -+#endif -+#ifdef EAFNOSUPPORT -+ TEST_COMPARE_STRING (strerrordesc_np (EAFNOSUPPORT), -+ "Address family not supported by protocol"); -+ TEST_COMPARE_STRING (strerrorname_np (EAFNOSUPPORT), "EAFNOSUPPORT"); -+#endif -+#ifdef EADDRINUSE -+ TEST_COMPARE_STRING (strerrordesc_np (EADDRINUSE), -+ "Address already in use"); -+ TEST_COMPARE_STRING (strerrorname_np (EADDRINUSE), "EADDRINUSE"); -+#endif -+#ifdef EADDRNOTAVAIL -+ TEST_COMPARE_STRING (strerrordesc_np (EADDRNOTAVAIL), -+ "Cannot assign requested address"); -+ TEST_COMPARE_STRING (strerrorname_np (EADDRNOTAVAIL), "EADDRNOTAVAIL"); -+#endif -+#ifdef ENETDOWN -+ TEST_COMPARE_STRING (strerrordesc_np (ENETDOWN), "Network is down"); -+ TEST_COMPARE_STRING (strerrorname_np (ENETDOWN), "ENETDOWN"); -+#endif -+#ifdef ENETUNREACH -+ TEST_COMPARE_STRING (strerrordesc_np (ENETUNREACH), -+ "Network is unreachable"); -+ TEST_COMPARE_STRING (strerrorname_np (ENETUNREACH), "ENETUNREACH"); -+#endif -+#ifdef ENETRESET -+ TEST_COMPARE_STRING (strerrordesc_np (ENETRESET), -+ "Network dropped connection on reset"); -+ TEST_COMPARE_STRING (strerrorname_np (ENETRESET), "ENETRESET"); -+#endif -+#ifdef ECONNABORTED -+ TEST_COMPARE_STRING (strerrordesc_np (ECONNABORTED), -+ "Software caused connection abort"); -+ TEST_COMPARE_STRING (strerrorname_np (ECONNABORTED), "ECONNABORTED"); -+#endif -+#ifdef ECONNRESET -+ TEST_COMPARE_STRING (strerrordesc_np (ECONNRESET), -+ "Connection reset by peer"); -+ TEST_COMPARE_STRING (strerrorname_np (ECONNRESET), "ECONNRESET"); -+#endif -+#ifdef ENOBUFS -+ TEST_COMPARE_STRING (strerrordesc_np (ENOBUFS), -+ "No buffer space available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOBUFS), "ENOBUFS"); -+#endif -+#ifdef EISCONN -+ TEST_COMPARE_STRING (strerrordesc_np (EISCONN), -+ "Transport endpoint is already connected"); -+ TEST_COMPARE_STRING (strerrorname_np (EISCONN), "EISCONN"); -+#endif -+#ifdef ENOTCONN -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTCONN), -+ "Transport endpoint is not connected"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTCONN), "ENOTCONN"); -+#endif -+#ifdef EDESTADDRREQ -+ TEST_COMPARE_STRING (strerrordesc_np (EDESTADDRREQ), -+ "Destination address required"); -+ TEST_COMPARE_STRING (strerrorname_np (EDESTADDRREQ), "EDESTADDRREQ"); -+#endif -+#ifdef ESHUTDOWN -+ TEST_COMPARE_STRING (strerrordesc_np (ESHUTDOWN), -+ "Cannot send after transport endpoint shutdown"); -+ TEST_COMPARE_STRING (strerrorname_np (ESHUTDOWN), "ESHUTDOWN"); -+#endif -+#ifdef ETOOMANYREFS -+ TEST_COMPARE_STRING (strerrordesc_np (ETOOMANYREFS), -+ "Too many references: cannot splice"); -+ TEST_COMPARE_STRING (strerrorname_np (ETOOMANYREFS), "ETOOMANYREFS"); -+#endif -+#ifdef ETIMEDOUT -+ TEST_COMPARE_STRING (strerrordesc_np (ETIMEDOUT), "Connection timed out"); -+ TEST_COMPARE_STRING (strerrorname_np (ETIMEDOUT), "ETIMEDOUT"); -+#endif -+#ifdef ECONNREFUSED -+ TEST_COMPARE_STRING (strerrordesc_np (ECONNREFUSED), "Connection refused"); -+ TEST_COMPARE_STRING (strerrorname_np (ECONNREFUSED), "ECONNREFUSED"); -+#endif -+#ifdef ELOOP -+ TEST_COMPARE_STRING (strerrordesc_np (ELOOP), -+ "Too many levels of symbolic links"); -+ TEST_COMPARE_STRING (strerrorname_np (ELOOP), "ELOOP"); -+#endif -+#ifdef ENAMETOOLONG -+ TEST_COMPARE_STRING (strerrordesc_np (ENAMETOOLONG), "File name too long"); -+ TEST_COMPARE_STRING (strerrorname_np (ENAMETOOLONG), "ENAMETOOLONG"); -+#endif -+#ifdef EHOSTDOWN -+ TEST_COMPARE_STRING (strerrordesc_np (EHOSTDOWN), "Host is down"); -+ TEST_COMPARE_STRING (strerrorname_np (EHOSTDOWN), "EHOSTDOWN"); -+#endif -+#ifdef EHOSTUNREACH -+ TEST_COMPARE_STRING (strerrordesc_np (EHOSTUNREACH), "No route to host"); -+ TEST_COMPARE_STRING (strerrorname_np (EHOSTUNREACH), "EHOSTUNREACH"); -+#endif -+#ifdef ENOTEMPTY -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTEMPTY), "Directory not empty"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTEMPTY), "ENOTEMPTY"); -+#endif -+#ifdef EUSERS -+ TEST_COMPARE_STRING (strerrordesc_np (EUSERS), "Too many users"); -+ TEST_COMPARE_STRING (strerrorname_np (EUSERS), "EUSERS"); -+#endif -+#ifdef EDQUOT -+ TEST_COMPARE_STRING (strerrordesc_np (EDQUOT), "Disk quota exceeded"); -+ TEST_COMPARE_STRING (strerrorname_np (EDQUOT), "EDQUOT"); -+#endif -+#ifdef ESTALE -+ TEST_COMPARE_STRING (strerrordesc_np (ESTALE), "Stale file handle"); -+ TEST_COMPARE_STRING (strerrorname_np (ESTALE), "ESTALE"); -+#endif -+#ifdef EREMOTE -+ TEST_COMPARE_STRING (strerrordesc_np (EREMOTE), "Object is remote"); -+ TEST_COMPARE_STRING (strerrorname_np (EREMOTE), "EREMOTE"); -+#endif -+#ifdef ENOLCK -+ TEST_COMPARE_STRING (strerrordesc_np (ENOLCK), "No locks available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOLCK), "ENOLCK"); -+#endif -+#ifdef ENOSYS -+ TEST_COMPARE_STRING (strerrordesc_np (ENOSYS), "Function not implemented"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOSYS), "ENOSYS"); -+#endif -+#ifdef EILSEQ -+ TEST_COMPARE_STRING (strerrordesc_np (EILSEQ), -+ "Invalid or incomplete multibyte or wide character"); -+ TEST_COMPARE_STRING (strerrorname_np (EILSEQ), "EILSEQ"); -+#endif -+#ifdef EBADMSG -+ TEST_COMPARE_STRING (strerrordesc_np (EBADMSG), "Bad message"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADMSG), "EBADMSG"); -+#endif -+#ifdef EIDRM -+ TEST_COMPARE_STRING (strerrordesc_np (EIDRM), "Identifier removed"); -+ TEST_COMPARE_STRING (strerrorname_np (EIDRM), "EIDRM"); -+#endif -+#ifdef EMULTIHOP -+ TEST_COMPARE_STRING (strerrordesc_np (EMULTIHOP), "Multihop attempted"); -+ TEST_COMPARE_STRING (strerrorname_np (EMULTIHOP), "EMULTIHOP"); -+#endif -+#ifdef ENODATA -+ TEST_COMPARE_STRING (strerrordesc_np (ENODATA), "No data available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENODATA), "ENODATA"); -+#endif -+#ifdef ENOLINK -+ TEST_COMPARE_STRING (strerrordesc_np (ENOLINK), "Link has been severed"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOLINK), "ENOLINK"); -+#endif -+#ifdef ENOMSG -+ TEST_COMPARE_STRING (strerrordesc_np (ENOMSG), -+ "No message of desired type"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOMSG), "ENOMSG"); -+#endif -+#ifdef ENOSR -+ TEST_COMPARE_STRING (strerrordesc_np (ENOSR), "Out of streams resources"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOSR), "ENOSR"); -+#endif -+#ifdef ENOSTR -+ TEST_COMPARE_STRING (strerrordesc_np (ENOSTR), "Device not a stream"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOSTR), "ENOSTR"); -+#endif -+#ifdef EOVERFLOW -+ TEST_COMPARE_STRING (strerrordesc_np (EOVERFLOW), -+ "Value too large for defined data type"); -+ TEST_COMPARE_STRING (strerrorname_np (EOVERFLOW), "EOVERFLOW"); -+#endif -+#ifdef EPROTO -+ TEST_COMPARE_STRING (strerrordesc_np (EPROTO), "Protocol error"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROTO), "EPROTO"); -+#endif -+#ifdef ETIME -+ TEST_COMPARE_STRING (strerrordesc_np (ETIME), "Timer expired"); -+ TEST_COMPARE_STRING (strerrorname_np (ETIME), "ETIME"); -+#endif -+#ifdef ECANCELED -+ TEST_COMPARE_STRING (strerrordesc_np (ECANCELED), "Operation canceled"); -+ TEST_COMPARE_STRING (strerrorname_np (ECANCELED), "ECANCELED"); -+#endif -+#ifdef EOWNERDEAD -+ TEST_COMPARE_STRING (strerrordesc_np (EOWNERDEAD), "Owner died"); -+ TEST_COMPARE_STRING (strerrorname_np (EOWNERDEAD), "EOWNERDEAD"); -+#endif -+#ifdef ENOTRECOVERABLE -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTRECOVERABLE), -+ "State not recoverable"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTRECOVERABLE), "ENOTRECOVERABLE"); -+#endif -+#ifdef ERESTART -+ TEST_COMPARE_STRING (strerrordesc_np (ERESTART), -+ "Interrupted system call should be restarted"); -+ TEST_COMPARE_STRING (strerrorname_np (ERESTART), "ERESTART"); -+#endif -+#ifdef ECHRNG -+ TEST_COMPARE_STRING (strerrordesc_np (ECHRNG), -+ "Channel number out of range"); -+ TEST_COMPARE_STRING (strerrorname_np (ECHRNG), "ECHRNG"); -+#endif -+#ifdef EL2NSYNC -+ TEST_COMPARE_STRING (strerrordesc_np (EL2NSYNC), -+ "Level 2 not synchronized"); -+ TEST_COMPARE_STRING (strerrorname_np (EL2NSYNC), "EL2NSYNC"); -+#endif -+#ifdef EL3HLT -+ TEST_COMPARE_STRING (strerrordesc_np (EL3HLT), "Level 3 halted"); -+ TEST_COMPARE_STRING (strerrorname_np (EL3HLT), "EL3HLT"); -+#endif -+#ifdef EL3RST -+ TEST_COMPARE_STRING (strerrordesc_np (EL3RST), "Level 3 reset"); -+ TEST_COMPARE_STRING (strerrorname_np (EL3RST), "EL3RST"); -+#endif -+#ifdef ELNRNG -+ TEST_COMPARE_STRING (strerrordesc_np (ELNRNG), "Link number out of range"); -+ TEST_COMPARE_STRING (strerrorname_np (ELNRNG), "ELNRNG"); -+#endif -+#ifdef EUNATCH -+ TEST_COMPARE_STRING (strerrordesc_np (EUNATCH), -+ "Protocol driver not attached"); -+ TEST_COMPARE_STRING (strerrorname_np (EUNATCH), "EUNATCH"); -+#endif -+#ifdef ENOCSI -+ TEST_COMPARE_STRING (strerrordesc_np (ENOCSI), -+ "No CSI structure available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOCSI), "ENOCSI"); -+#endif -+#ifdef EL2HLT -+ TEST_COMPARE_STRING (strerrordesc_np (EL2HLT), "Level 2 halted"); -+ TEST_COMPARE_STRING (strerrorname_np (EL2HLT), "EL2HLT"); -+#endif -+#ifdef EBADE -+ TEST_COMPARE_STRING (strerrordesc_np (EBADE), "Invalid exchange"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADE), "EBADE"); -+#endif -+#ifdef EBADR -+ TEST_COMPARE_STRING (strerrordesc_np (EBADR), -+ "Invalid request descriptor"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADR), "EBADR"); -+#endif -+#ifdef EXFULL -+ TEST_COMPARE_STRING (strerrordesc_np (EXFULL), "Exchange full"); -+ TEST_COMPARE_STRING (strerrorname_np (EXFULL), "EXFULL"); -+#endif -+#ifdef ENOANO -+ TEST_COMPARE_STRING (strerrordesc_np (ENOANO), "No anode"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOANO), "ENOANO"); -+#endif -+#ifdef EBADRQC -+ TEST_COMPARE_STRING (strerrordesc_np (EBADRQC), "Invalid request code"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADRQC), "EBADRQC"); -+#endif -+#ifdef EBADSLT -+ TEST_COMPARE_STRING (strerrordesc_np (EBADSLT), "Invalid slot"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADSLT), "EBADSLT"); -+#endif -+#ifdef EBFONT -+ TEST_COMPARE_STRING (strerrordesc_np (EBFONT), "Bad font file format"); -+ TEST_COMPARE_STRING (strerrorname_np (EBFONT), "EBFONT"); -+#endif -+#ifdef ENONET -+ TEST_COMPARE_STRING (strerrordesc_np (ENONET), -+ "Machine is not on the network"); -+ TEST_COMPARE_STRING (strerrorname_np (ENONET), "ENONET"); -+#endif -+#ifdef ENOPKG -+ TEST_COMPARE_STRING (strerrordesc_np (ENOPKG), "Package not installed"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOPKG), "ENOPKG"); -+#endif -+#ifdef EADV -+ TEST_COMPARE_STRING (strerrordesc_np (EADV), "Advertise error"); -+ TEST_COMPARE_STRING (strerrorname_np (EADV), "EADV"); -+#endif -+#ifdef ESRMNT -+ TEST_COMPARE_STRING (strerrordesc_np (ESRMNT), "Srmount error"); -+ TEST_COMPARE_STRING (strerrorname_np (ESRMNT), "ESRMNT"); -+#endif -+#ifdef ECOMM -+ TEST_COMPARE_STRING (strerrordesc_np (ECOMM), -+ "Communication error on send"); -+ TEST_COMPARE_STRING (strerrorname_np (ECOMM), "ECOMM"); -+#endif -+#ifdef EDOTDOT -+ TEST_COMPARE_STRING (strerrordesc_np (EDOTDOT), "RFS specific error"); -+ TEST_COMPARE_STRING (strerrorname_np (EDOTDOT), "EDOTDOT"); -+#endif -+#ifdef ENOTUNIQ -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTUNIQ), -+ "Name not unique on network"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTUNIQ), "ENOTUNIQ"); -+#endif -+#ifdef EBADFD -+ TEST_COMPARE_STRING (strerrordesc_np (EBADFD), -+ "File descriptor in bad state"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADFD), "EBADFD"); -+#endif -+#ifdef EREMCHG -+ TEST_COMPARE_STRING (strerrordesc_np (EREMCHG), "Remote address changed"); -+ TEST_COMPARE_STRING (strerrorname_np (EREMCHG), "EREMCHG"); -+#endif -+#ifdef ELIBACC -+ TEST_COMPARE_STRING (strerrordesc_np (ELIBACC), -+ "Can not access a needed shared library"); -+ TEST_COMPARE_STRING (strerrorname_np (ELIBACC), "ELIBACC"); -+#endif -+#ifdef ELIBBAD -+ TEST_COMPARE_STRING (strerrordesc_np (ELIBBAD), -+ "Accessing a corrupted shared library"); -+ TEST_COMPARE_STRING (strerrorname_np (ELIBBAD), "ELIBBAD"); -+#endif -+#ifdef ELIBSCN -+ TEST_COMPARE_STRING (strerrordesc_np (ELIBSCN), -+ ".lib section in a.out corrupted"); -+ TEST_COMPARE_STRING (strerrorname_np (ELIBSCN), "ELIBSCN"); -+#endif -+#ifdef ELIBMAX -+ TEST_COMPARE_STRING (strerrordesc_np (ELIBMAX), -+ "Attempting to link in too many shared libraries"); -+ TEST_COMPARE_STRING (strerrorname_np (ELIBMAX), "ELIBMAX"); -+#endif -+#ifdef ELIBEXEC -+ TEST_COMPARE_STRING (strerrordesc_np (ELIBEXEC), -+ "Cannot exec a shared library directly"); -+ TEST_COMPARE_STRING (strerrorname_np (ELIBEXEC), "ELIBEXEC"); -+#endif -+#ifdef ESTRPIPE -+ TEST_COMPARE_STRING (strerrordesc_np (ESTRPIPE), "Streams pipe error"); -+ TEST_COMPARE_STRING (strerrorname_np (ESTRPIPE), "ESTRPIPE"); -+#endif -+#ifdef EUCLEAN -+ TEST_COMPARE_STRING (strerrordesc_np (EUCLEAN), -+ "Structure needs cleaning"); -+ TEST_COMPARE_STRING (strerrorname_np (EUCLEAN), "EUCLEAN"); -+#endif -+#ifdef ENOTNAM -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTNAM), -+ "Not a XENIX named type file"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTNAM), "ENOTNAM"); -+#endif -+#ifdef ENAVAIL -+ TEST_COMPARE_STRING (strerrordesc_np (ENAVAIL), -+ "No XENIX semaphores available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENAVAIL), "ENAVAIL"); -+#endif -+#ifdef EISNAM -+ TEST_COMPARE_STRING (strerrordesc_np (EISNAM), "Is a named type file"); -+ TEST_COMPARE_STRING (strerrorname_np (EISNAM), "EISNAM"); -+#endif -+#ifdef EREMOTEIO -+ TEST_COMPARE_STRING (strerrordesc_np (EREMOTEIO), "Remote I/O error"); -+ TEST_COMPARE_STRING (strerrorname_np (EREMOTEIO), "EREMOTEIO"); -+#endif -+#ifdef ENOMEDIUM -+ TEST_COMPARE_STRING (strerrordesc_np (ENOMEDIUM), "No medium found"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOMEDIUM), "ENOMEDIUM"); -+#endif -+#ifdef EMEDIUMTYPE -+ TEST_COMPARE_STRING (strerrordesc_np (EMEDIUMTYPE), "Wrong medium type"); -+ TEST_COMPARE_STRING (strerrorname_np (EMEDIUMTYPE), "EMEDIUMTYPE"); -+#endif -+#ifdef ENOKEY -+ TEST_COMPARE_STRING (strerrordesc_np (ENOKEY), -+ "Required key not available"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOKEY), "ENOKEY"); -+#endif -+#ifdef EKEYEXPIRED -+ TEST_COMPARE_STRING (strerrordesc_np (EKEYEXPIRED), "Key has expired"); -+ TEST_COMPARE_STRING (strerrorname_np (EKEYEXPIRED), "EKEYEXPIRED"); -+#endif -+#ifdef EKEYREVOKED -+ TEST_COMPARE_STRING (strerrordesc_np (EKEYREVOKED), -+ "Key has been revoked"); -+ TEST_COMPARE_STRING (strerrorname_np (EKEYREVOKED), "EKEYREVOKED"); -+#endif -+#ifdef EKEYREJECTED -+ TEST_COMPARE_STRING (strerrordesc_np (EKEYREJECTED), -+ "Key was rejected by service"); -+ TEST_COMPARE_STRING (strerrorname_np (EKEYREJECTED), "EKEYREJECTED"); -+#endif -+#ifdef ERFKILL -+ TEST_COMPARE_STRING (strerrordesc_np (ERFKILL), -+ "Operation not possible due to RF-kill"); -+ TEST_COMPARE_STRING (strerrorname_np (ERFKILL), "ERFKILL"); -+#endif -+#ifdef EHWPOISON -+ TEST_COMPARE_STRING (strerrordesc_np (EHWPOISON), -+ "Memory page has hardware error"); -+ TEST_COMPARE_STRING (strerrorname_np (EHWPOISON), "EHWPOISON"); -+#endif -+#ifdef EBADRPC -+ TEST_COMPARE_STRING (strerrordesc_np (EBADRPC), "RPC struct is bad"); -+ TEST_COMPARE_STRING (strerrorname_np (EBADRPC), "EBADRPC"); -+#endif -+#ifdef EFTYPE -+ TEST_COMPARE_STRING (strerrordesc_np (EFTYPE), -+ "Inappropriate file type or format"); -+ TEST_COMPARE_STRING (strerrorname_np (EFTYPE), "EFTYPE"); -+#endif -+#ifdef EPROCUNAVAIL -+ TEST_COMPARE_STRING (strerrordesc_np (EPROCUNAVAIL), -+ "RPC bad procedure for program"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROCUNAVAIL), "EPROCUNAVAIL"); -+#endif -+#ifdef EAUTH -+ TEST_COMPARE_STRING (strerrordesc_np (EAUTH), "Authentication error"); -+ TEST_COMPARE_STRING (strerrorname_np (EAUTH), "EAUTH"); -+#endif -+#ifdef EDIED -+ TEST_COMPARE_STRING (strerrordesc_np (EDIED), "Translator died"); -+ TEST_COMPARE_STRING (strerrorname_np (EDIED), "EDIED"); -+#endif -+#ifdef ERPCMISMATCH -+ TEST_COMPARE_STRING (strerrordesc_np (ERPCMISMATCH), "RPC version wrong"); -+ TEST_COMPARE_STRING (strerrorname_np (ERPCMISMATCH), "ERPCMISMATCH"); -+#endif -+#ifdef EGREGIOUS -+ TEST_COMPARE_STRING (strerrordesc_np (EGREGIOUS), -+ "You really blew it this time"); -+ TEST_COMPARE_STRING (strerrorname_np (EGREGIOUS), "EGREGIOUS"); -+#endif -+#ifdef EPROCLIM -+ TEST_COMPARE_STRING (strerrordesc_np (EPROCLIM), "Too many processes"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROCLIM), "EPROCLIM"); -+#endif -+#ifdef EGRATUITOUS -+ TEST_COMPARE_STRING (strerrordesc_np (EGRATUITOUS), "Gratuitous error"); -+ TEST_COMPARE_STRING (strerrorname_np (EGRATUITOUS), "EGRATUITOUS"); -+#endif -+#if defined (ENOTSUP) && ENOTSUP != EOPNOTSUPP -+ TEST_COMPARE_STRING (strerrordesc_np (ENOTSUP), "Not supported"); -+ TEST_COMPARE_STRING (strerrorname_np (ENOTSUP), "ENOTSUP"); -+#endif -+#ifdef EPROGMISMATCH -+ TEST_COMPARE_STRING (strerrordesc_np (EPROGMISMATCH), -+ "RPC program version wrong"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROGMISMATCH), "EPROGMISMATCH"); -+#endif -+#ifdef EBACKGROUND -+ TEST_COMPARE_STRING (strerrordesc_np (EBACKGROUND), -+ "Inappropriate operation for background process"); -+ TEST_COMPARE_STRING (strerrorname_np (EBACKGROUND), "EBACKGROUND"); -+#endif -+#ifdef EIEIO -+ TEST_COMPARE_STRING (strerrordesc_np (EIEIO), "Computer bought the farm"); -+ TEST_COMPARE_STRING (strerrorname_np (EIEIO), "EIEIO"); -+#endif -+#if defined (EWOULDBLOCK) && EWOULDBLOCK != EAGAIN -+ TEST_COMPARE_STRING (strerrordesc_np (EWOULDBLOCK), -+ "Operation would block"); -+ TEST_COMPARE_STRING (strerrorname_np (EWOULDBLOCK), "EWOULDBLOCK"); -+#endif -+#ifdef ENEEDAUTH -+ TEST_COMPARE_STRING (strerrordesc_np (ENEEDAUTH), "Need authenticator"); -+ TEST_COMPARE_STRING (strerrorname_np (ENEEDAUTH), "ENEEDAUTH"); -+#endif -+#ifdef ED -+ TEST_COMPARE_STRING (strerrordesc_np (ED), "?"); -+ TEST_COMPARE_STRING (strerrorname_np (ED), "ED"); -+#endif -+#ifdef EPROGUNAVAIL -+ TEST_COMPARE_STRING (strerrordesc_np (EPROGUNAVAIL), -+ "RPC program not available"); -+ TEST_COMPARE_STRING (strerrorname_np (EPROGUNAVAIL), "EPROGUNAVAIL"); -+#endif - - return 0; - } -diff --git a/stdio-common/vfscanf-internal.c b/stdio-common/vfscanf-internal.c -index 95b46dcbeb..3a323547f9 100644 ---- a/stdio-common/vfscanf-internal.c -+++ b/stdio-common/vfscanf-internal.c -@@ -277,7 +277,7 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - #endif - { - va_list arg; -- const CHAR_T *f = format; -+ const UCHAR_T *f = (const UCHAR_T *) format; - UCHAR_T fc; /* Current character of the format. */ - WINT_T done = 0; /* Assignments done. */ - size_t read_in = 0; /* Chars read in. */ -@@ -415,10 +415,11 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - #endif - - #ifndef COMPILE_WSCANF -- if (!isascii ((unsigned char) *f)) -+ if (!isascii (*f)) - { - /* Non-ASCII, may be a multibyte. */ -- int len = __mbrlen (f, strlen (f), &state); -+ int len = __mbrlen ((const char *) f, strlen ((const char *) f), -+ &state); - if (len > 0) - { - do -@@ -426,7 +427,7 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - c = inchar (); - if (__glibc_unlikely (c == EOF)) - input_error (); -- else if (c != (unsigned char) *f++) -+ else if (c != *f++) - { - ungetc_not_eof (c, s); - conv_error (); -@@ -484,9 +485,9 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - char_buffer_rewind (&charbuf); - - /* Check for a positional parameter specification. */ -- if (ISDIGIT ((UCHAR_T) *f)) -+ if (ISDIGIT (*f)) - { -- argpos = read_int ((const UCHAR_T **) &f); -+ argpos = read_int (&f); - if (*f == L_('$')) - ++f; - else -@@ -521,8 +522,8 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - - /* Find the maximum field width. */ - width = 0; -- if (ISDIGIT ((UCHAR_T) *f)) -- width = read_int ((const UCHAR_T **) &f); -+ if (ISDIGIT (*f)) -+ width = read_int (&f); - got_width: - if (width == 0) - width = -1; -@@ -2522,12 +2523,11 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, - } - - while ((fc = *f++) != '\0' && fc != ']') -- if (fc == '-' && *f != '\0' && *f != ']' -- && (unsigned char) f[-2] <= (unsigned char) *f) -+ if (fc == '-' && *f != '\0' && *f != ']' && f[-2] <= *f) - { - /* Add all characters from the one before the '-' - up to (but not including) the next format char. */ -- for (fc = (unsigned char) f[-2]; fc < (unsigned char) *f; ++fc) -+ for (fc = f[-2]; fc < *f; ++fc) - ((char *)charbuf.scratch.data)[fc] = 1; - } - else -diff --git a/stdlib/tst-secure-getenv.c b/stdlib/tst-secure-getenv.c -index 3cfe9a05c3..d4b1139c5e 100644 ---- a/stdlib/tst-secure-getenv.c -+++ b/stdlib/tst-secure-getenv.c -@@ -30,167 +30,12 @@ - #include <sys/wait.h> - #include <unistd.h> - -+#include <support/check.h> - #include <support/support.h> -+#include <support/capture_subprocess.h> - #include <support/test-driver.h> - - static char MAGIC_ARGUMENT[] = "run-actual-test"; --#define MAGIC_STATUS 19 -- --/* Return a GID which is not our current GID, but is present in the -- supplementary group list. */ --static gid_t --choose_gid (void) --{ -- int count = getgroups (0, NULL); -- if (count < 0) -- { -- printf ("getgroups: %m\n"); -- exit (1); -- } -- gid_t *groups; -- groups = xcalloc (count, sizeof (*groups)); -- int ret = getgroups (count, groups); -- if (ret < 0) -- { -- printf ("getgroups: %m\n"); -- exit (1); -- } -- gid_t current = getgid (); -- gid_t not_current = 0; -- for (int i = 0; i < ret; ++i) -- { -- if (groups[i] != current) -- { -- not_current = groups[i]; -- break; -- } -- } -- free (groups); -- return not_current; --} -- -- --/* Copies the executable into a restricted directory, so that we can -- safely make it SGID with the TARGET group ID. Then runs the -- executable. */ --static int --run_executable_sgid (gid_t target) --{ -- char *dirname = xasprintf ("%s/secure-getenv.%jd", -- test_dir, (intmax_t) getpid ()); -- char *execname = xasprintf ("%s/bin", dirname); -- int infd = -1; -- int outfd = -1; -- int ret = -1; -- if (mkdir (dirname, 0700) < 0) -- { -- printf ("mkdir: %m\n"); -- goto err; -- } -- infd = open ("/proc/self/exe", O_RDONLY); -- if (infd < 0) -- { -- printf ("open (/proc/self/exe): %m\n"); -- goto err; -- } -- outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); -- if (outfd < 0) -- { -- printf ("open (%s): %m\n", execname); -- goto err; -- } -- char buf[4096]; -- for (;;) -- { -- ssize_t rdcount = read (infd, buf, sizeof (buf)); -- if (rdcount < 0) -- { -- printf ("read: %m\n"); -- goto err; -- } -- if (rdcount == 0) -- break; -- char *p = buf; -- char *end = buf + rdcount; -- while (p != end) -- { -- ssize_t wrcount = write (outfd, buf, end - p); -- if (wrcount == 0) -- errno = ENOSPC; -- if (wrcount <= 0) -- { -- printf ("write: %m\n"); -- goto err; -- } -- p += wrcount; -- } -- } -- if (fchown (outfd, getuid (), target) < 0) -- { -- printf ("fchown (%s): %m\n", execname); -- goto err; -- } -- if (fchmod (outfd, 02750) < 0) -- { -- printf ("fchmod (%s): %m\n", execname); -- goto err; -- } -- if (close (outfd) < 0) -- { -- printf ("close (outfd): %m\n"); -- goto err; -- } -- if (close (infd) < 0) -- { -- printf ("close (infd): %m\n"); -- goto err; -- } -- -- int kid = fork (); -- if (kid < 0) -- { -- printf ("fork: %m\n"); -- goto err; -- } -- if (kid == 0) -- { -- /* Child process. */ -- char *args[] = { execname, MAGIC_ARGUMENT, NULL }; -- execve (execname, args, environ); -- printf ("execve (%s): %m\n", execname); -- _exit (1); -- } -- int status; -- if (waitpid (kid, &status, 0) < 0) -- { -- printf ("waitpid: %m\n"); -- goto err; -- } -- if (!WIFEXITED (status) || WEXITSTATUS (status) != MAGIC_STATUS) -- { -- printf ("Unexpected exit status %d from child process\n", -- status); -- goto err; -- } -- ret = 0; -- --err: -- if (outfd >= 0) -- close (outfd); -- if (infd >= 0) -- close (infd); -- if (execname) -- { -- unlink (execname); -- free (execname); -- } -- if (dirname) -- { -- rmdir (dirname); -- free (dirname); -- } -- return ret; --} - - static int - do_test (void) -@@ -212,15 +57,15 @@ do_test (void) - exit (1); - } - -- gid_t target = choose_gid (); -- if (target == 0) -- { -- fprintf (stderr, -- "Could not find a suitable GID for user %jd, skipping test\n", -- (intmax_t) getuid ()); -- exit (0); -- } -- return run_executable_sgid (target); -+ int status = support_capture_subprogram_self_sgid (MAGIC_ARGUMENT); -+ -+ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) -+ return EXIT_UNSUPPORTED; -+ -+ if (!WIFEXITED (status)) -+ FAIL_EXIT1 ("Unexpected exit status %d from child process\n", status); -+ -+ return 0; - } - - static void -@@ -229,23 +74,15 @@ alternative_main (int argc, char **argv) - if (argc == 2 && strcmp (argv[1], MAGIC_ARGUMENT) == 0) - { - if (getgid () == getegid ()) -- { -- /* This can happen if the file system is mounted nosuid. */ -- fprintf (stderr, "SGID failed: GID and EGID match (%jd)\n", -- (intmax_t) getgid ()); -- exit (MAGIC_STATUS); -- } -+ /* This can happen if the file system is mounted nosuid. */ -+ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", -+ (intmax_t) getgid ()); - if (getenv ("PATH") == NULL) -- { -- printf ("PATH variable not present\n"); -- exit (3); -- } -+ FAIL_EXIT (3, "PATH variable not present\n"); - if (secure_getenv ("PATH") != NULL) -- { -- printf ("PATH variable not filtered out\n"); -- exit (4); -- } -- exit (MAGIC_STATUS); -+ FAIL_EXIT (4, "PATH variable not filtered out\n"); -+ -+ exit (EXIT_SUCCESS); - } - } - -diff --git a/string/bits/string_fortified.h b/string/bits/string_fortified.h -index 309d0f39b2..c8d3051af8 100644 ---- a/string/bits/string_fortified.h -+++ b/string/bits/string_fortified.h -@@ -22,11 +22,6 @@ - # error "Never use <bits/string_fortified.h> directly; include <string.h> instead." - #endif - --#if !__GNUC_PREREQ (5,0) --__warndecl (__warn_memset_zero_len, -- "memset used with constant zero length parameter; this could be due to transposed parameters"); --#endif -- - __fortify_function void * - __NTH (memcpy (void *__restrict __dest, const void *__restrict __src, - size_t __len)) -@@ -58,16 +53,6 @@ __NTH (mempcpy (void *__restrict __dest, const void *__restrict __src, - __fortify_function void * - __NTH (memset (void *__dest, int __ch, size_t __len)) - { -- /* GCC-5.0 and newer implements these checks in the compiler, so we don't -- need them here. */ --#if !__GNUC_PREREQ (5,0) -- if (__builtin_constant_p (__len) && __len == 0 -- && (!__builtin_constant_p (__ch) || __ch != 0)) -- { -- __warn_memset_zero_len (); -- return __dest; -- } --#endif - return __builtin___memset_chk (__dest, __ch, __len, __bos0 (__dest)); - } - -diff --git a/support/Makefile b/support/Makefile -index 93faafddf9..3d3aff5ff9 100644 ---- a/support/Makefile -+++ b/support/Makefile -@@ -35,6 +35,8 @@ libsupport-routines = \ - ignore_stderr \ - next_to_fault \ - oom_error \ -+ resolv_response_context_duplicate \ -+ resolv_response_context_free \ - resolv_test \ - set_fortify_handler \ - support-xfstat \ -@@ -133,6 +135,7 @@ libsupport-routines = \ - xpthread_join \ - xpthread_key_create \ - xpthread_key_delete \ -+ xpthread_kill \ - xpthread_mutex_consistent \ - xpthread_mutex_destroy \ - xpthread_mutex_init \ -diff --git a/support/capture_subprocess.h b/support/capture_subprocess.h -index 9808750f80..421f657678 100644 ---- a/support/capture_subprocess.h -+++ b/support/capture_subprocess.h -@@ -41,6 +41,12 @@ struct support_capture_subprocess support_capture_subprocess - struct support_capture_subprocess support_capture_subprogram - (const char *file, char *const argv[]); - -+/* Copy the running program into a setgid binary and run it with CHILD_ID -+ argument. If execution is successful, return the exit status of the child -+ program, otherwise return a non-zero failure exit code. */ -+int support_capture_subprogram_self_sgid -+ (char *child_id); -+ - /* Deallocate the subprocess data captured by - support_capture_subprocess. */ - void support_capture_subprocess_free (struct support_capture_subprocess *); -diff --git a/support/resolv_response_context_duplicate.c b/support/resolv_response_context_duplicate.c -new file mode 100644 -index 0000000000..f9c5c3462a ---- /dev/null -+++ b/support/resolv_response_context_duplicate.c -@@ -0,0 +1,37 @@ -+/* Duplicate a response context used in DNS resolver tests. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <string.h> -+#include <support/resolv_test.h> -+#include <support/support.h> -+ -+struct resolv_response_context * -+resolv_response_context_duplicate (const struct resolv_response_context *ctx) -+{ -+ struct resolv_response_context *result = xmalloc (sizeof (*result)); -+ memcpy (result, ctx, sizeof (*result)); -+ if (result->client_address != NULL) -+ { -+ result->client_address = xmalloc (result->client_address_length); -+ memcpy (result->client_address, ctx->client_address, -+ result->client_address_length); -+ } -+ result->query_buffer = xmalloc (result->query_length); -+ memcpy (result->query_buffer, ctx->query_buffer, result->query_length); -+ return result; -+} -diff --git a/support/resolv_response_context_free.c b/support/resolv_response_context_free.c -new file mode 100644 -index 0000000000..b88c05ffd4 ---- /dev/null -+++ b/support/resolv_response_context_free.c -@@ -0,0 +1,28 @@ -+/* Free a response context used in DNS resolver tests. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <stdlib.h> -+#include <support/resolv_test.h> -+ -+void -+resolv_response_context_free (struct resolv_response_context *ctx) -+{ -+ free (ctx->query_buffer); -+ free (ctx->client_address); -+ free (ctx); -+} -diff --git a/support/resolv_test.c b/support/resolv_test.c -index 53b7fc41ab..9878a040a3 100644 ---- a/support/resolv_test.c -+++ b/support/resolv_test.c -@@ -181,7 +181,9 @@ resolv_response_init (struct resolv_response_builder *b, - b->buffer[2] |= b->query_buffer[2] & 0x01; /* Copy the RD bit. */ - if (flags.tc) - b->buffer[2] |= 0x02; -- b->buffer[3] = 0x80 | flags.rcode; /* Always set RA. */ -+ b->buffer[3] = flags.rcode; -+ if (!flags.clear_ra) -+ b->buffer[3] |= 0x80; - if (flags.ad) - b->buffer[3] |= 0x20; - -@@ -434,9 +436,9 @@ resolv_response_buffer (const struct resolv_response_builder *b) - return result; - } - --static struct resolv_response_builder * --response_builder_allocate -- (const unsigned char *query_buffer, size_t query_length) -+struct resolv_response_builder * -+resolv_response_builder_allocate (const unsigned char *query_buffer, -+ size_t query_length) - { - struct resolv_response_builder *b = xmalloc (sizeof (*b)); - memset (b, 0, offsetof (struct resolv_response_builder, buffer)); -@@ -445,8 +447,8 @@ response_builder_allocate - return b; - } - --static void --response_builder_free (struct resolv_response_builder *b) -+void -+resolv_response_builder_free (struct resolv_response_builder *b) - { - tdestroy (b->compression_offsets, free); - free (b); -@@ -661,13 +663,17 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) - - struct resolv_response_context ctx = - { -+ .test = obj, -+ .client_address = &peer, -+ .client_address_length = peerlen, - .query_buffer = query, - .query_length = length, - .server_index = server_index, - .tcp = false, - .edns = qinfo.edns, - }; -- struct resolv_response_builder *b = response_builder_allocate (query, length); -+ struct resolv_response_builder *b -+ = resolv_response_builder_allocate (query, length); - obj->config.response_callback - (&ctx, b, qinfo.qname, qinfo.qclass, qinfo.qtype); - -@@ -684,7 +690,7 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) - if (b->offset >= 12) - printf ("info: UDP server %d: sending response:" - " %zu bytes, RCODE %d (for %s/%u/%u)\n", -- server_index, b->offset, b->buffer[3] & 0x0f, -+ ctx.server_index, b->offset, b->buffer[3] & 0x0f, - qinfo.qname, qinfo.qclass, qinfo.qtype); - else - printf ("info: UDP server %d: sending response: %zu bytes" -@@ -694,23 +700,31 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) - if (b->truncate_bytes > 0) - printf ("info: truncated by %u bytes\n", b->truncate_bytes); - } -- size_t to_send = b->offset; -- if (to_send < b->truncate_bytes) -- to_send = 0; -- else -- to_send -= b->truncate_bytes; -- -- /* Ignore most errors here because the other end may have closed -- the socket. */ -- if (sendto (obj->servers[server_index].socket_udp, -- b->buffer, to_send, 0, -- (struct sockaddr *) &peer, peerlen) < 0) -- TEST_VERIFY_EXIT (errno != EBADF); -+ resolv_response_send_udp (&ctx, b); - } -- response_builder_free (b); -+ resolv_response_builder_free (b); - return true; - } - -+void -+resolv_response_send_udp (const struct resolv_response_context *ctx, -+ struct resolv_response_builder *b) -+{ -+ TEST_VERIFY_EXIT (!ctx->tcp); -+ size_t to_send = b->offset; -+ if (to_send < b->truncate_bytes) -+ to_send = 0; -+ else -+ to_send -= b->truncate_bytes; -+ -+ /* Ignore most errors here because the other end may have closed -+ the socket. */ -+ if (sendto (ctx->test->servers[ctx->server_index].socket_udp, -+ b->buffer, to_send, 0, -+ ctx->client_address, ctx->client_address_length) < 0) -+ TEST_VERIFY_EXIT (errno != EBADF); -+} -+ - /* UDP thread_callback function. Variant for one thread per - server. */ - static void -@@ -897,14 +911,15 @@ server_thread_tcp_client (void *arg) - - struct resolv_response_context ctx = - { -+ .test = closure->obj, - .query_buffer = query_buffer, - .query_length = query_length, - .server_index = closure->server_index, - .tcp = true, - .edns = qinfo.edns, - }; -- struct resolv_response_builder *b = response_builder_allocate -- (query_buffer, query_length); -+ struct resolv_response_builder *b -+ = resolv_response_builder_allocate (query_buffer, query_length); - closure->obj->config.response_callback - (&ctx, b, qinfo.qname, qinfo.qclass, qinfo.qtype); - -@@ -936,7 +951,7 @@ server_thread_tcp_client (void *arg) - writev_fully (closure->client_socket, buffers, 2); - } - bool close_flag = b->close; -- response_builder_free (b); -+ resolv_response_builder_free (b); - free (query_buffer); - if (close_flag) - break; -diff --git a/support/resolv_test.h b/support/resolv_test.h -index 67819469a0..31a5c1c3e7 100644 ---- a/support/resolv_test.h -+++ b/support/resolv_test.h -@@ -35,25 +35,36 @@ struct resolv_edns_info - uint16_t payload_size; - }; - -+/* This opaque struct collects information about the resolver testing -+ currently in progress. */ -+struct resolv_test; -+ - /* This struct provides context information when the response callback - specified in struct resolv_redirect_config is invoked. */ - struct resolv_response_context - { -- const unsigned char *query_buffer; -+ struct resolv_test *test; -+ void *client_address; -+ size_t client_address_length; -+ unsigned char *query_buffer; - size_t query_length; - int server_index; - bool tcp; - struct resolv_edns_info edns; - }; - -+/* Produces a deep copy of the context. */ -+struct resolv_response_context * -+ resolv_response_context_duplicate (const struct resolv_response_context *); -+ -+/* Frees the copy. For the context passed to the response function, -+ this happens implicitly. */ -+void resolv_response_context_free (struct resolv_response_context *); -+ - /* This opaque struct is used to construct responses from within the - response callback function. */ - struct resolv_response_builder; - --/* This opaque struct collects information about the resolver testing -- currently in progress. */ --struct resolv_test; -- - enum - { - /* Maximum number of test servers supported by the framework. */ -@@ -137,6 +148,10 @@ struct resolv_response_flags - /* If true, the AD (authenticated data) flag will be set. */ - bool ad; - -+ /* If true, do not set the RA (recursion available) flag in the -+ response. */ -+ bool clear_ra; -+ - /* Initial section count values. Can be used to artificially - increase the counts, for malformed packet testing.*/ - unsigned short qdcount; -@@ -188,6 +203,22 @@ void resolv_response_close (struct resolv_response_builder *); - /* The size of the response packet built so far. */ - size_t resolv_response_length (const struct resolv_response_builder *); - -+/* Allocates a response builder tied to a specific query packet, -+ starting at QUERY_BUFFER, containing QUERY_LENGTH bytes. */ -+struct resolv_response_builder * -+ resolv_response_builder_allocate (const unsigned char *query_buffer, -+ size_t query_length); -+ -+/* Deallocates a response buffer. */ -+void resolv_response_builder_free (struct resolv_response_builder *); -+ -+/* Sends a UDP response using a specific context. This can be used to -+ reorder or duplicate responses, along with -+ resolv_response_context_duplicate and -+ response_builder_allocate. */ -+void resolv_response_send_udp (const struct resolv_response_context *, -+ struct resolv_response_builder *); -+ - __END_DECLS - - #endif /* SUPPORT_RESOLV_TEST_H */ -diff --git a/support/subprocess.h b/support/subprocess.h -index 8b442fd5c0..34ffd02e8e 100644 ---- a/support/subprocess.h -+++ b/support/subprocess.h -@@ -38,6 +38,11 @@ struct support_subprocess support_subprocess - struct support_subprocess support_subprogram - (const char *file, char *const argv[]); - -+/* Invoke program FILE with ARGV arguments by using posix_spawn and wait for it -+ to complete. Return program exit status. */ -+int support_subprogram_wait -+ (const char *file, char *const argv[]); -+ - /* Wait for the subprocess indicated by PROC::PID. Return the status - indicate by waitpid call. */ - int support_process_wait (struct support_subprocess *proc); -diff --git a/support/support_capture_subprocess.c b/support/support_capture_subprocess.c -index eeed676e3d..28a37df67f 100644 ---- a/support/support_capture_subprocess.c -+++ b/support/support_capture_subprocess.c -@@ -20,11 +20,14 @@ - #include <support/capture_subprocess.h> - - #include <errno.h> -+#include <fcntl.h> - #include <stdlib.h> - #include <support/check.h> - #include <support/xunistd.h> - #include <support/xsocket.h> - #include <support/xspawn.h> -+#include <support/support.h> -+#include <support/test-driver.h> - - static void - transfer (const char *what, struct pollfd *pfd, struct xmemstream *stream) -@@ -36,7 +39,7 @@ transfer (const char *what, struct pollfd *pfd, struct xmemstream *stream) - if (ret < 0) - { - support_record_failure (); -- printf ("error: reading from subprocess %s: %m", what); -+ printf ("error: reading from subprocess %s: %m\n", what); - pfd->events = 0; - pfd->revents = 0; - } -@@ -102,6 +105,129 @@ support_capture_subprogram (const char *file, char *const argv[]) - return result; - } - -+/* Copies the executable into a restricted directory, so that we can -+ safely make it SGID with the TARGET group ID. Then runs the -+ executable. */ -+static int -+copy_and_spawn_sgid (char *child_id, gid_t gid) -+{ -+ char *dirname = xasprintf ("%s/tst-tunables-setuid.%jd", -+ test_dir, (intmax_t) getpid ()); -+ char *execname = xasprintf ("%s/bin", dirname); -+ int infd = -1; -+ int outfd = -1; -+ int ret = 1, status = 1; -+ -+ TEST_VERIFY (mkdir (dirname, 0700) == 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ -+ infd = open ("/proc/self/exe", O_RDONLY); -+ if (infd < 0) -+ FAIL_UNSUPPORTED ("unsupported: Cannot read binary from procfs\n"); -+ -+ outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); -+ TEST_VERIFY (outfd >= 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ -+ char buf[4096]; -+ for (;;) -+ { -+ ssize_t rdcount = read (infd, buf, sizeof (buf)); -+ TEST_VERIFY (rdcount >= 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ if (rdcount == 0) -+ break; -+ char *p = buf; -+ char *end = buf + rdcount; -+ while (p != end) -+ { -+ ssize_t wrcount = write (outfd, buf, end - p); -+ if (wrcount == 0) -+ errno = ENOSPC; -+ TEST_VERIFY (wrcount > 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ p += wrcount; -+ } -+ } -+ TEST_VERIFY (fchown (outfd, getuid (), gid) == 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ TEST_VERIFY (fchmod (outfd, 02750) == 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ TEST_VERIFY (close (outfd) == 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ TEST_VERIFY (close (infd) == 0); -+ if (support_record_failure_is_failed ()) -+ goto err; -+ -+ /* We have the binary, now spawn the subprocess. Avoid using -+ support_subprogram because we only want the program exit status, not the -+ contents. */ -+ ret = 0; -+ -+ char * const args[] = {execname, child_id, NULL}; -+ -+ status = support_subprogram_wait (args[0], args); -+ -+err: -+ if (outfd >= 0) -+ close (outfd); -+ if (infd >= 0) -+ close (infd); -+ if (execname != NULL) -+ { -+ unlink (execname); -+ free (execname); -+ } -+ if (dirname != NULL) -+ { -+ rmdir (dirname); -+ free (dirname); -+ } -+ -+ if (ret != 0) -+ FAIL_EXIT1("Failed to make sgid executable for test\n"); -+ -+ return status; -+} -+ -+int -+support_capture_subprogram_self_sgid (char *child_id) -+{ -+ gid_t target = 0; -+ const int count = 64; -+ gid_t groups[count]; -+ -+ /* Get a GID which is not our current GID, but is present in the -+ supplementary group list. */ -+ int ret = getgroups (count, groups); -+ if (ret < 0) -+ FAIL_UNSUPPORTED("Could not get group list for user %jd\n", -+ (intmax_t) getuid ()); -+ -+ gid_t current = getgid (); -+ for (int i = 0; i < ret; ++i) -+ { -+ if (groups[i] != current) -+ { -+ target = groups[i]; -+ break; -+ } -+ } -+ -+ if (target == 0) -+ FAIL_UNSUPPORTED("Could not find a suitable GID for user %jd\n", -+ (intmax_t) getuid ()); -+ -+ return copy_and_spawn_sgid (child_id, target); -+} -+ - void - support_capture_subprocess_free (struct support_capture_subprocess *p) - { -diff --git a/support/support_subprocess.c b/support/support_subprocess.c -index 36e3a77af2..4a25828111 100644 ---- a/support/support_subprocess.c -+++ b/support/support_subprocess.c -@@ -27,7 +27,7 @@ - #include <support/subprocess.h> - - static struct support_subprocess --support_suprocess_init (void) -+support_subprocess_init (void) - { - struct support_subprocess result; - -@@ -48,7 +48,7 @@ support_suprocess_init (void) - struct support_subprocess - support_subprocess (void (*callback) (void *), void *closure) - { -- struct support_subprocess result = support_suprocess_init (); -+ struct support_subprocess result = support_subprocess_init (); - - result.pid = xfork (); - if (result.pid == 0) -@@ -71,7 +71,7 @@ support_subprocess (void (*callback) (void *), void *closure) - struct support_subprocess - support_subprogram (const char *file, char *const argv[]) - { -- struct support_subprocess result = support_suprocess_init (); -+ struct support_subprocess result = support_subprocess_init (); - - posix_spawn_file_actions_t fa; - /* posix_spawn_file_actions_init does not fail. */ -@@ -84,7 +84,7 @@ support_subprogram (const char *file, char *const argv[]) - xposix_spawn_file_actions_addclose (&fa, result.stdout_pipe[1]); - xposix_spawn_file_actions_addclose (&fa, result.stderr_pipe[1]); - -- result.pid = xposix_spawn (file, &fa, NULL, argv, NULL); -+ result.pid = xposix_spawn (file, &fa, NULL, argv, environ); - - xclose (result.stdout_pipe[1]); - xclose (result.stderr_pipe[1]); -@@ -92,6 +92,19 @@ support_subprogram (const char *file, char *const argv[]) - return result; - } - -+int -+support_subprogram_wait (const char *file, char *const argv[]) -+{ -+ posix_spawn_file_actions_t fa; -+ -+ posix_spawn_file_actions_init (&fa); -+ struct support_subprocess res = support_subprocess_init (); -+ -+ res.pid = xposix_spawn (file, &fa, NULL, argv, environ); -+ -+ return support_process_wait (&res); -+} -+ - int - support_process_wait (struct support_subprocess *proc) - { -diff --git a/support/xpthread_kill.c b/support/xpthread_kill.c -new file mode 100644 -index 0000000000..111a75d85e ---- /dev/null -+++ b/support/xpthread_kill.c -@@ -0,0 +1,26 @@ -+/* pthread_kill with error checking. -+ Copyright (C) 2021 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <signal.h> -+#include <support/xthread.h> -+ -+void -+xpthread_kill (pthread_t thr, int signo) -+{ -+ xpthread_check_return ("pthread_kill", pthread_kill (thr, signo)); -+} -diff --git a/support/xthread.h b/support/xthread.h -index 05f8d4a7d9..cb1fc30da0 100644 ---- a/support/xthread.h -+++ b/support/xthread.h -@@ -75,6 +75,8 @@ void xpthread_attr_setstacksize (pthread_attr_t *attr, - void xpthread_attr_setguardsize (pthread_attr_t *attr, - size_t guardsize); - -+void xpthread_kill (pthread_t thr, int signo); -+ - /* Set the stack size in ATTR to a small value, but still large enough - to cover most internal glibc stack usage. */ - void support_set_small_thread_stack_size (pthread_attr_t *attr); -diff --git a/sysdeps/aarch64/dl-bti.c b/sysdeps/aarch64/dl-bti.c -index 196e462520..cf7624aaa2 100644 ---- a/sysdeps/aarch64/dl-bti.c -+++ b/sysdeps/aarch64/dl-bti.c -@@ -19,43 +19,76 @@ - #include <errno.h> - #include <libintl.h> - #include <ldsodefs.h> -+#include <sys/mman.h> - --static int --enable_bti (struct link_map *map, const char *program) -+/* See elf/dl-load.h. */ -+#ifndef MAP_COPY -+# define MAP_COPY (MAP_PRIVATE | MAP_DENYWRITE) -+#endif -+ -+/* Enable BTI protection for MAP. */ -+ -+void -+_dl_bti_protect (struct link_map *map, int fd) - { -+ const size_t pagesz = GLRO(dl_pagesize); - const ElfW(Phdr) *phdr; -- unsigned prot; - - for (phdr = map->l_phdr; phdr < &map->l_phdr[map->l_phnum]; ++phdr) - if (phdr->p_type == PT_LOAD && (phdr->p_flags & PF_X)) - { -- void *start = (void *) (phdr->p_vaddr + map->l_addr); -- size_t len = phdr->p_memsz; -+ size_t vstart = ALIGN_DOWN (phdr->p_vaddr, pagesz); -+ size_t vend = ALIGN_UP (phdr->p_vaddr + phdr->p_filesz, pagesz); -+ off_t off = ALIGN_DOWN (phdr->p_offset, pagesz); -+ void *start = (void *) (vstart + map->l_addr); -+ size_t len = vend - vstart; - -- prot = PROT_EXEC | PROT_BTI; -+ unsigned prot = PROT_EXEC | PROT_BTI; - if (phdr->p_flags & PF_R) - prot |= PROT_READ; - if (phdr->p_flags & PF_W) - prot |= PROT_WRITE; - -- if (__mprotect (start, len, prot) < 0) -- { -- if (program) -- _dl_fatal_printf ("%s: mprotect failed to turn on BTI\n", -- map->l_name); -- else -- _dl_signal_error (errno, map->l_name, "dlopen", -- N_("mprotect failed to turn on BTI")); -- } -+ if (fd == -1) -+ /* Ignore failures for kernel mapped binaries. */ -+ __mprotect (start, len, prot); -+ else -+ map->l_mach.bti_fail = __mmap (start, len, prot, -+ MAP_FIXED|MAP_COPY|MAP_FILE, -+ fd, off) == MAP_FAILED; - } -- return 0; - } - --/* Enable BTI for L if required. */ -+ -+static void -+bti_failed (struct link_map *l, const char *program) -+{ -+ if (program) -+ _dl_fatal_printf ("%s: %s: failed to turn on BTI protection\n", -+ program, l->l_name); -+ else -+ /* Note: the errno value is not available any more. */ -+ _dl_signal_error (0, l->l_name, "dlopen", -+ N_("failed to turn on BTI protection")); -+} -+ -+ -+/* Enable BTI for L and its dependencies. */ - - void - _dl_bti_check (struct link_map *l, const char *program) - { -- if (GLRO(dl_aarch64_cpu_features).bti && l->l_mach.bti) -- enable_bti (l, program); -+ if (!GLRO(dl_aarch64_cpu_features).bti) -+ return; -+ -+ if (l->l_mach.bti_fail) -+ bti_failed (l, program); -+ -+ unsigned int i = l->l_searchlist.r_nlist; -+ while (i-- > 0) -+ { -+ struct link_map *dep = l->l_initfini[i]; -+ if (dep->l_mach.bti_fail) -+ bti_failed (dep, program); -+ } - } -diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h -index 70b9ed3925..fde7cfd9e2 100644 ---- a/sysdeps/aarch64/dl-machine.h -+++ b/sysdeps/aarch64/dl-machine.h -@@ -395,13 +395,6 @@ elf_machine_lazy_rel (struct link_map *map, - /* Check for unexpected PLT reloc type. */ - if (__builtin_expect (r_type == AARCH64_R(JUMP_SLOT), 1)) - { -- if (map->l_mach.plt == 0) -- { -- /* Prelinking. */ -- *reloc_addr += l_addr; -- return; -- } -- - if (__glibc_unlikely (map->l_info[DT_AARCH64 (VARIANT_PCS)] != NULL)) - { - /* Check the symbol table for variant PCS symbols. */ -@@ -425,7 +418,10 @@ elf_machine_lazy_rel (struct link_map *map, - } - } - -- *reloc_addr = map->l_mach.plt; -+ if (map->l_mach.plt == 0) -+ *reloc_addr += l_addr; -+ else -+ *reloc_addr = map->l_mach.plt; - } - else if (__builtin_expect (r_type == AARCH64_R(TLSDESC), 1)) - { -diff --git a/sysdeps/aarch64/dl-prop.h b/sysdeps/aarch64/dl-prop.h -index b0785bda83..e926e54984 100644 ---- a/sysdeps/aarch64/dl-prop.h -+++ b/sysdeps/aarch64/dl-prop.h -@@ -19,6 +19,8 @@ - #ifndef _DL_PROP_H - #define _DL_PROP_H - -+extern void _dl_bti_protect (struct link_map *, int) attribute_hidden; -+ - extern void _dl_bti_check (struct link_map *, const char *) - attribute_hidden; - -@@ -35,14 +37,18 @@ _dl_open_check (struct link_map *m) - } - - static inline void __attribute__ ((always_inline)) --_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) -+_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) - { - } - - static inline int --_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, -- void *data) -+_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, -+ uint32_t datasz, void *data) - { -+ if (!GLRO(dl_aarch64_cpu_features).bti) -+ /* Skip note processing. */ -+ return 0; -+ - if (type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) - { - /* Stop if the property note is ill-formed. */ -@@ -51,7 +57,7 @@ _dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, - - unsigned int feature_1 = *(unsigned int *) data; - if (feature_1 & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) -- l->l_mach.bti = true; -+ _dl_bti_protect (l, fd); - - /* Stop if we processed the property note. */ - return 0; -diff --git a/sysdeps/aarch64/linkmap.h b/sysdeps/aarch64/linkmap.h -index 847a03ace2..b3f7663b07 100644 ---- a/sysdeps/aarch64/linkmap.h -+++ b/sysdeps/aarch64/linkmap.h -@@ -22,5 +22,5 @@ struct link_map_machine - { - ElfW(Addr) plt; /* Address of .plt */ - void *tlsdesc_table; /* Address of TLS descriptor hash table. */ -- bool bti; /* Branch Target Identification is enabled. */ -+ bool bti_fail; /* Failed to enable Branch Target Identification. */ - }; -diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c -index 7cf5f033e8..799d60c98c 100644 ---- a/sysdeps/aarch64/multiarch/memcpy.c -+++ b/sysdeps/aarch64/multiarch/memcpy.c -@@ -41,7 +41,8 @@ libc_ifunc (__libc_memcpy, - ? __memcpy_falkor - : (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr) - ? __memcpy_thunderx2 -- : (IS_NEOVERSE_N1 (midr) -+ : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr) -+ || IS_NEOVERSE_V1 (midr) - ? __memcpy_simd - : __memcpy_generic))))); - -diff --git a/sysdeps/aarch64/multiarch/memcpy_advsimd.S b/sysdeps/aarch64/multiarch/memcpy_advsimd.S -index d4ba747777..48bb6d7ca4 100644 ---- a/sysdeps/aarch64/multiarch/memcpy_advsimd.S -+++ b/sysdeps/aarch64/multiarch/memcpy_advsimd.S -@@ -223,12 +223,13 @@ L(copy_long_backwards): - b.ls L(copy64_from_start) - - L(loop64_backwards): -- stp A_q, B_q, [dstend, -32] -+ str B_q, [dstend, -16] -+ str A_q, [dstend, -32] - ldp A_q, B_q, [srcend, -96] -- stp C_q, D_q, [dstend, -64] -+ str D_q, [dstend, -48] -+ str C_q, [dstend, -64]! - ldp C_q, D_q, [srcend, -128] - sub srcend, srcend, 64 -- sub dstend, dstend, 64 - subs count, count, 64 - b.hi L(loop64_backwards) - -diff --git a/sysdeps/aarch64/multiarch/memmove.c b/sysdeps/aarch64/multiarch/memmove.c -index ad10aa8ac6..46a4cb3a54 100644 ---- a/sysdeps/aarch64/multiarch/memmove.c -+++ b/sysdeps/aarch64/multiarch/memmove.c -@@ -41,7 +41,8 @@ libc_ifunc (__libc_memmove, - ? __memmove_falkor - : (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr) - ? __memmove_thunderx2 -- : (IS_NEOVERSE_N1 (midr) -+ : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr) -+ || IS_NEOVERSE_V1 (midr) - ? __memmove_simd - : __memmove_generic))))); - -diff --git a/sysdeps/aarch64/start.S b/sysdeps/aarch64/start.S -index 75393e1c18..1998ea95d4 100644 ---- a/sysdeps/aarch64/start.S -+++ b/sysdeps/aarch64/start.S -@@ -43,11 +43,9 @@ - */ - - .text -- .globl _start -- .type _start,#function --_start: -- BTI_C -+ENTRY(_start) - /* Create an initial frame with 0 LR and FP */ -+ cfi_undefined (x30) - mov x29, #0 - mov x30, #0 - -@@ -101,8 +99,10 @@ _start: - because crt1.o and rcrt1.o share code and the later must avoid the - use of GOT relocations before __libc_start_main is called. */ - __wrap_main: -+ BTI_C - b main - #endif -+END(_start) - - /* Define a symbol for the first piece of initialized data. */ - .data -diff --git a/sysdeps/generic/dl-prop.h b/sysdeps/generic/dl-prop.h -index f1cf576fe3..df27ff8e6a 100644 ---- a/sysdeps/generic/dl-prop.h -+++ b/sysdeps/generic/dl-prop.h -@@ -37,15 +37,15 @@ _dl_open_check (struct link_map *m) - } - - static inline void __attribute__ ((always_inline)) --_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) -+_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) - { - } - - /* Called for each property in the NT_GNU_PROPERTY_TYPE_0 note of L, - processing of the properties continues until this returns 0. */ - static inline int __attribute__ ((always_inline)) --_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, -- void *data) -+_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, -+ uint32_t datasz, void *data) - { - return 0; - } -diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h -index ba114ab4b1..62ac40d81b 100644 ---- a/sysdeps/generic/ldsodefs.h -+++ b/sysdeps/generic/ldsodefs.h -@@ -919,8 +919,9 @@ extern void _dl_rtld_di_serinfo (struct link_map *loader, - Dl_serinfo *si, bool counting); - - /* Process PT_GNU_PROPERTY program header PH in module L after -- PT_LOAD segments are mapped. */ --void _dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph); -+ PT_LOAD segments are mapped from file FD. */ -+void _dl_process_pt_gnu_property (struct link_map *l, int fd, -+ const ElfW(Phdr) *ph); - - - /* Search loaded objects' symbol tables for a definition of the symbol -diff --git a/sysdeps/generic/unwind.h b/sysdeps/generic/unwind.h -index b667a5b652..c229603af3 100644 ---- a/sysdeps/generic/unwind.h -+++ b/sysdeps/generic/unwind.h -@@ -75,15 +75,21 @@ typedef void (*_Unwind_Exception_Cleanup_Fn) (_Unwind_Reason_Code, - - struct _Unwind_Exception - { -- _Unwind_Exception_Class exception_class; -- _Unwind_Exception_Cleanup_Fn exception_cleanup; -- _Unwind_Word private_1; -- _Unwind_Word private_2; -- -- /* @@@ The IA-64 ABI says that this structure must be double-word aligned. -- Taking that literally does not make much sense generically. Instead we -- provide the maximum alignment required by any type for the machine. */ --} __attribute__((__aligned__)); -+ union -+ { -+ struct -+ { -+ _Unwind_Exception_Class exception_class; -+ _Unwind_Exception_Cleanup_Fn exception_cleanup; -+ _Unwind_Word private_1; -+ _Unwind_Word private_2; -+ }; -+ -+ /* The IA-64 ABI says that this structure must be double-word aligned. */ -+ _Unwind_Word unwind_exception_align[2] -+ __attribute__ ((__aligned__ (2 * sizeof (_Unwind_Word)))); -+ }; -+}; - - - /* The ACTIONS argument to the personality routine is a bitwise OR of one -diff --git a/sysdeps/gnu/errlist.h b/sysdeps/gnu/errlist.h -index 5d11ed723d..6329e5f393 100644 ---- a/sysdeps/gnu/errlist.h -+++ b/sysdeps/gnu/errlist.h -@@ -1,24 +1,21 @@ --#ifndef ERR_MAP --#define ERR_MAP(value) value --#endif --_S(ERR_MAP(0), N_("Success")) -+_S(0, N_("Success")) - #ifdef EPERM - /* - TRANS Only the owner of the file (or other resource) - TRANS or processes with special privileges can perform the operation. */ --_S(ERR_MAP(EPERM), N_("Operation not permitted")) -+_S(EPERM, N_("Operation not permitted")) - #endif - #ifdef ENOENT - /* - TRANS This is a ``file doesn't exist'' error - TRANS for ordinary files that are referenced in contexts where they are - TRANS expected to already exist. */ --_S(ERR_MAP(ENOENT), N_("No such file or directory")) -+_S(ENOENT, N_("No such file or directory")) - #endif - #ifdef ESRCH - /* - TRANS No process matches the specified process ID. */ --_S(ERR_MAP(ESRCH), N_("No such process")) -+_S(ESRCH, N_("No such process")) - #endif - #ifdef EINTR - /* -@@ -29,12 +26,12 @@ TRANS - TRANS You can choose to have functions resume after a signal that is handled, - TRANS rather than failing with @code{EINTR}; see @ref{Interrupted - TRANS Primitives}. */ --_S(ERR_MAP(EINTR), N_("Interrupted system call")) -+_S(EINTR, N_("Interrupted system call")) - #endif - #ifdef EIO - /* - TRANS Usually used for physical read or write errors. */ --_S(ERR_MAP(EIO), N_("Input/output error")) -+_S(EIO, N_("Input/output error")) - #endif - #ifdef ENXIO - /* -@@ -43,7 +40,7 @@ TRANS represented by a file you specified, and it couldn't find the device. - TRANS This can mean that the device file was installed incorrectly, or that - TRANS the physical device is missing or not correctly attached to the - TRANS computer. */ --_S(ERR_MAP(ENXIO), N_("No such device or address")) -+_S(ENXIO, N_("No such device or address")) - #endif - #ifdef E2BIG - /* -@@ -51,27 +48,27 @@ TRANS Used when the arguments passed to a new program - TRANS being executed with one of the @code{exec} functions (@pxref{Executing a - TRANS File}) occupy too much memory space. This condition never arises on - TRANS @gnuhurdsystems{}. */ --_S(ERR_MAP(E2BIG), N_("Argument list too long")) -+_S(E2BIG, N_("Argument list too long")) - #endif - #ifdef ENOEXEC - /* - TRANS Invalid executable file format. This condition is detected by the - TRANS @code{exec} functions; see @ref{Executing a File}. */ --_S(ERR_MAP(ENOEXEC), N_("Exec format error")) -+_S(ENOEXEC, N_("Exec format error")) - #endif - #ifdef EBADF - /* - TRANS For example, I/O on a descriptor that has been - TRANS closed or reading from a descriptor open only for writing (or vice - TRANS versa). */ --_S(ERR_MAP(EBADF), N_("Bad file descriptor")) -+_S(EBADF, N_("Bad file descriptor")) - #endif - #ifdef ECHILD - /* - TRANS This error happens on operations that are - TRANS supposed to manipulate child processes, when there aren't any processes - TRANS to manipulate. */ --_S(ERR_MAP(ECHILD), N_("No child processes")) -+_S(ECHILD, N_("No child processes")) - #endif - #ifdef EDEADLK - /* -@@ -79,74 +76,74 @@ TRANS Allocating a system resource would have resulted in a - TRANS deadlock situation. The system does not guarantee that it will notice - TRANS all such situations. This error means you got lucky and the system - TRANS noticed; it might just hang. @xref{File Locks}, for an example. */ --_S(ERR_MAP(EDEADLK), N_("Resource deadlock avoided")) -+_S(EDEADLK, N_("Resource deadlock avoided")) - #endif - #ifdef ENOMEM - /* - TRANS The system cannot allocate more virtual memory - TRANS because its capacity is full. */ --_S(ERR_MAP(ENOMEM), N_("Cannot allocate memory")) -+_S(ENOMEM, N_("Cannot allocate memory")) - #endif - #ifdef EACCES - /* - TRANS The file permissions do not allow the attempted operation. */ --_S(ERR_MAP(EACCES), N_("Permission denied")) -+_S(EACCES, N_("Permission denied")) - #endif - #ifdef EFAULT - /* - TRANS An invalid pointer was detected. - TRANS On @gnuhurdsystems{}, this error never happens; you get a signal instead. */ --_S(ERR_MAP(EFAULT), N_("Bad address")) -+_S(EFAULT, N_("Bad address")) - #endif - #ifdef ENOTBLK - /* - TRANS A file that isn't a block special file was given in a situation that - TRANS requires one. For example, trying to mount an ordinary file as a file - TRANS system in Unix gives this error. */ --_S(ERR_MAP(ENOTBLK), N_("Block device required")) -+_S(ENOTBLK, N_("Block device required")) - #endif - #ifdef EBUSY - /* - TRANS A system resource that can't be shared is already in use. - TRANS For example, if you try to delete a file that is the root of a currently - TRANS mounted filesystem, you get this error. */ --_S(ERR_MAP(EBUSY), N_("Device or resource busy")) -+_S(EBUSY, N_("Device or resource busy")) - #endif - #ifdef EEXIST - /* - TRANS An existing file was specified in a context where it only - TRANS makes sense to specify a new file. */ --_S(ERR_MAP(EEXIST), N_("File exists")) -+_S(EEXIST, N_("File exists")) - #endif - #ifdef EXDEV - /* - TRANS An attempt to make an improper link across file systems was detected. - TRANS This happens not only when you use @code{link} (@pxref{Hard Links}) but - TRANS also when you rename a file with @code{rename} (@pxref{Renaming Files}). */ --_S(ERR_MAP(EXDEV), N_("Invalid cross-device link")) -+_S(EXDEV, N_("Invalid cross-device link")) - #endif - #ifdef ENODEV - /* - TRANS The wrong type of device was given to a function that expects a - TRANS particular sort of device. */ --_S(ERR_MAP(ENODEV), N_("No such device")) -+_S(ENODEV, N_("No such device")) - #endif - #ifdef ENOTDIR - /* - TRANS A file that isn't a directory was specified when a directory is required. */ --_S(ERR_MAP(ENOTDIR), N_("Not a directory")) -+_S(ENOTDIR, N_("Not a directory")) - #endif - #ifdef EISDIR - /* - TRANS You cannot open a directory for writing, - TRANS or create or remove hard links to it. */ --_S(ERR_MAP(EISDIR), N_("Is a directory")) -+_S(EISDIR, N_("Is a directory")) - #endif - #ifdef EINVAL - /* - TRANS This is used to indicate various kinds of problems - TRANS with passing the wrong argument to a library function. */ --_S(ERR_MAP(EINVAL), N_("Invalid argument")) -+_S(EINVAL, N_("Invalid argument")) - #endif - #ifdef EMFILE - /* -@@ -157,20 +154,20 @@ TRANS In BSD and GNU, the number of open files is controlled by a resource - TRANS limit that can usually be increased. If you get this error, you might - TRANS want to increase the @code{RLIMIT_NOFILE} limit or make it unlimited; - TRANS @pxref{Limits on Resources}. */ --_S(ERR_MAP(EMFILE), N_("Too many open files")) -+_S(EMFILE, N_("Too many open files")) - #endif - #ifdef ENFILE - /* - TRANS There are too many distinct file openings in the entire system. Note - TRANS that any number of linked channels count as just one file opening; see - TRANS @ref{Linked Channels}. This error never occurs on @gnuhurdsystems{}. */ --_S(ERR_MAP(ENFILE), N_("Too many open files in system")) -+_S(ENFILE, N_("Too many open files in system")) - #endif - #ifdef ENOTTY - /* - TRANS Inappropriate I/O control operation, such as trying to set terminal - TRANS modes on an ordinary file. */ --_S(ERR_MAP(ENOTTY), N_("Inappropriate ioctl for device")) -+_S(ENOTTY, N_("Inappropriate ioctl for device")) - #endif - #ifdef ETXTBSY - /* -@@ -179,35 +176,35 @@ TRANS write to a file that is currently being executed. Often using a - TRANS debugger to run a program is considered having it open for writing and - TRANS will cause this error. (The name stands for ``text file busy''.) This - TRANS is not an error on @gnuhurdsystems{}; the text is copied as necessary. */ --_S(ERR_MAP(ETXTBSY), N_("Text file busy")) -+_S(ETXTBSY, N_("Text file busy")) - #endif - #ifdef EFBIG - /* - TRANS The size of a file would be larger than allowed by the system. */ --_S(ERR_MAP(EFBIG), N_("File too large")) -+_S(EFBIG, N_("File too large")) - #endif - #ifdef ENOSPC - /* - TRANS Write operation on a file failed because the - TRANS disk is full. */ --_S(ERR_MAP(ENOSPC), N_("No space left on device")) -+_S(ENOSPC, N_("No space left on device")) - #endif - #ifdef ESPIPE - /* - TRANS Invalid seek operation (such as on a pipe). */ --_S(ERR_MAP(ESPIPE), N_("Illegal seek")) -+_S(ESPIPE, N_("Illegal seek")) - #endif - #ifdef EROFS - /* - TRANS An attempt was made to modify something on a read-only file system. */ --_S(ERR_MAP(EROFS), N_("Read-only file system")) -+_S(EROFS, N_("Read-only file system")) - #endif - #ifdef EMLINK - /* - TRANS The link count of a single file would become too large. - TRANS @code{rename} can cause this error if the file being renamed already has - TRANS as many links as it can take (@pxref{Renaming Files}). */ --_S(ERR_MAP(EMLINK), N_("Too many links")) -+_S(EMLINK, N_("Too many links")) - #endif - #ifdef EPIPE - /* -@@ -216,19 +213,19 @@ TRANS Every library function that returns this error code also generates a - TRANS @code{SIGPIPE} signal; this signal terminates the program if not handled - TRANS or blocked. Thus, your program will never actually see @code{EPIPE} - TRANS unless it has handled or blocked @code{SIGPIPE}. */ --_S(ERR_MAP(EPIPE), N_("Broken pipe")) -+_S(EPIPE, N_("Broken pipe")) - #endif - #ifdef EDOM - /* - TRANS Used by mathematical functions when an argument value does - TRANS not fall into the domain over which the function is defined. */ --_S(ERR_MAP(EDOM), N_("Numerical argument out of domain")) -+_S(EDOM, N_("Numerical argument out of domain")) - #endif - #ifdef ERANGE - /* - TRANS Used by mathematical functions when the result value is - TRANS not representable because of overflow or underflow. */ --_S(ERR_MAP(ERANGE), N_("Numerical result out of range")) -+_S(ERANGE, N_("Numerical result out of range")) - #endif - #ifdef EAGAIN - /* -@@ -261,7 +258,7 @@ TRANS Such shortages are usually fairly serious and affect the whole system, - TRANS so usually an interactive program should report the error to the user - TRANS and return to its command loop. - TRANS @end itemize */ --_S(ERR_MAP(EAGAIN), N_("Resource temporarily unavailable")) -+_S(EAGAIN, N_("Resource temporarily unavailable")) - #endif - #ifdef EINPROGRESS - /* -@@ -273,47 +270,47 @@ TRANS the operation has begun and will take some time. Attempts to manipulate - TRANS the object before the call completes return @code{EALREADY}. You can - TRANS use the @code{select} function to find out when the pending operation - TRANS has completed; @pxref{Waiting for I/O}. */ --_S(ERR_MAP(EINPROGRESS), N_("Operation now in progress")) -+_S(EINPROGRESS, N_("Operation now in progress")) - #endif - #ifdef EALREADY - /* - TRANS An operation is already in progress on an object that has non-blocking - TRANS mode selected. */ --_S(ERR_MAP(EALREADY), N_("Operation already in progress")) -+_S(EALREADY, N_("Operation already in progress")) - #endif - #ifdef ENOTSOCK - /* - TRANS A file that isn't a socket was specified when a socket is required. */ --_S(ERR_MAP(ENOTSOCK), N_("Socket operation on non-socket")) -+_S(ENOTSOCK, N_("Socket operation on non-socket")) - #endif - #ifdef EMSGSIZE - /* - TRANS The size of a message sent on a socket was larger than the supported - TRANS maximum size. */ --_S(ERR_MAP(EMSGSIZE), N_("Message too long")) -+_S(EMSGSIZE, N_("Message too long")) - #endif - #ifdef EPROTOTYPE - /* - TRANS The socket type does not support the requested communications protocol. */ --_S(ERR_MAP(EPROTOTYPE), N_("Protocol wrong type for socket")) -+_S(EPROTOTYPE, N_("Protocol wrong type for socket")) - #endif - #ifdef ENOPROTOOPT - /* - TRANS You specified a socket option that doesn't make sense for the - TRANS particular protocol being used by the socket. @xref{Socket Options}. */ --_S(ERR_MAP(ENOPROTOOPT), N_("Protocol not available")) -+_S(ENOPROTOOPT, N_("Protocol not available")) - #endif - #ifdef EPROTONOSUPPORT - /* - TRANS The socket domain does not support the requested communications protocol - TRANS (perhaps because the requested protocol is completely invalid). - TRANS @xref{Creating a Socket}. */ --_S(ERR_MAP(EPROTONOSUPPORT), N_("Protocol not supported")) -+_S(EPROTONOSUPPORT, N_("Protocol not supported")) - #endif - #ifdef ESOCKTNOSUPPORT - /* - TRANS The socket type is not supported. */ --_S(ERR_MAP(ESOCKTNOSUPPORT), N_("Socket type not supported")) -+_S(ESOCKTNOSUPPORT, N_("Socket type not supported")) - #endif - #ifdef EOPNOTSUPP - /* -@@ -323,71 +320,71 @@ TRANS implemented for all communications protocols. On @gnuhurdsystems{}, this - TRANS error can happen for many calls when the object does not support the - TRANS particular operation; it is a generic indication that the server knows - TRANS nothing to do for that call. */ --_S(ERR_MAP(EOPNOTSUPP), N_("Operation not supported")) -+_S(EOPNOTSUPP, N_("Operation not supported")) - #endif - #ifdef EPFNOSUPPORT - /* - TRANS The socket communications protocol family you requested is not supported. */ --_S(ERR_MAP(EPFNOSUPPORT), N_("Protocol family not supported")) -+_S(EPFNOSUPPORT, N_("Protocol family not supported")) - #endif - #ifdef EAFNOSUPPORT - /* - TRANS The address family specified for a socket is not supported; it is - TRANS inconsistent with the protocol being used on the socket. @xref{Sockets}. */ --_S(ERR_MAP(EAFNOSUPPORT), N_("Address family not supported by protocol")) -+_S(EAFNOSUPPORT, N_("Address family not supported by protocol")) - #endif - #ifdef EADDRINUSE - /* - TRANS The requested socket address is already in use. @xref{Socket Addresses}. */ --_S(ERR_MAP(EADDRINUSE), N_("Address already in use")) -+_S(EADDRINUSE, N_("Address already in use")) - #endif - #ifdef EADDRNOTAVAIL - /* - TRANS The requested socket address is not available; for example, you tried - TRANS to give a socket a name that doesn't match the local host name. - TRANS @xref{Socket Addresses}. */ --_S(ERR_MAP(EADDRNOTAVAIL), N_("Cannot assign requested address")) -+_S(EADDRNOTAVAIL, N_("Cannot assign requested address")) - #endif - #ifdef ENETDOWN - /* - TRANS A socket operation failed because the network was down. */ --_S(ERR_MAP(ENETDOWN), N_("Network is down")) -+_S(ENETDOWN, N_("Network is down")) - #endif - #ifdef ENETUNREACH - /* - TRANS A socket operation failed because the subnet containing the remote host - TRANS was unreachable. */ --_S(ERR_MAP(ENETUNREACH), N_("Network is unreachable")) -+_S(ENETUNREACH, N_("Network is unreachable")) - #endif - #ifdef ENETRESET - /* - TRANS A network connection was reset because the remote host crashed. */ --_S(ERR_MAP(ENETRESET), N_("Network dropped connection on reset")) -+_S(ENETRESET, N_("Network dropped connection on reset")) - #endif - #ifdef ECONNABORTED - /* - TRANS A network connection was aborted locally. */ --_S(ERR_MAP(ECONNABORTED), N_("Software caused connection abort")) -+_S(ECONNABORTED, N_("Software caused connection abort")) - #endif - #ifdef ECONNRESET - /* - TRANS A network connection was closed for reasons outside the control of the - TRANS local host, such as by the remote machine rebooting or an unrecoverable - TRANS protocol violation. */ --_S(ERR_MAP(ECONNRESET), N_("Connection reset by peer")) -+_S(ECONNRESET, N_("Connection reset by peer")) - #endif - #ifdef ENOBUFS - /* - TRANS The kernel's buffers for I/O operations are all in use. In GNU, this - TRANS error is always synonymous with @code{ENOMEM}; you may get one or the - TRANS other from network operations. */ --_S(ERR_MAP(ENOBUFS), N_("No buffer space available")) -+_S(ENOBUFS, N_("No buffer space available")) - #endif - #ifdef EISCONN - /* - TRANS You tried to connect a socket that is already connected. - TRANS @xref{Connecting}. */ --_S(ERR_MAP(EISCONN), N_("Transport endpoint is already connected")) -+_S(EISCONN, N_("Transport endpoint is already connected")) - #endif - #ifdef ENOTCONN - /* -@@ -395,74 +392,74 @@ TRANS The socket is not connected to anything. You get this error when you - TRANS try to transmit data over a socket, without first specifying a - TRANS destination for the data. For a connectionless socket (for datagram - TRANS protocols, such as UDP), you get @code{EDESTADDRREQ} instead. */ --_S(ERR_MAP(ENOTCONN), N_("Transport endpoint is not connected")) -+_S(ENOTCONN, N_("Transport endpoint is not connected")) - #endif - #ifdef EDESTADDRREQ - /* - TRANS No default destination address was set for the socket. You get this - TRANS error when you try to transmit data over a connectionless socket, - TRANS without first specifying a destination for the data with @code{connect}. */ --_S(ERR_MAP(EDESTADDRREQ), N_("Destination address required")) -+_S(EDESTADDRREQ, N_("Destination address required")) - #endif - #ifdef ESHUTDOWN - /* - TRANS The socket has already been shut down. */ --_S(ERR_MAP(ESHUTDOWN), N_("Cannot send after transport endpoint shutdown")) -+_S(ESHUTDOWN, N_("Cannot send after transport endpoint shutdown")) - #endif - #ifdef ETOOMANYREFS --_S(ERR_MAP(ETOOMANYREFS), N_("Too many references: cannot splice")) -+_S(ETOOMANYREFS, N_("Too many references: cannot splice")) - #endif - #ifdef ETIMEDOUT - /* - TRANS A socket operation with a specified timeout received no response during - TRANS the timeout period. */ --_S(ERR_MAP(ETIMEDOUT), N_("Connection timed out")) -+_S(ETIMEDOUT, N_("Connection timed out")) - #endif - #ifdef ECONNREFUSED - /* - TRANS A remote host refused to allow the network connection (typically because - TRANS it is not running the requested service). */ --_S(ERR_MAP(ECONNREFUSED), N_("Connection refused")) -+_S(ECONNREFUSED, N_("Connection refused")) - #endif - #ifdef ELOOP - /* - TRANS Too many levels of symbolic links were encountered in looking up a file name. - TRANS This often indicates a cycle of symbolic links. */ --_S(ERR_MAP(ELOOP), N_("Too many levels of symbolic links")) -+_S(ELOOP, N_("Too many levels of symbolic links")) - #endif - #ifdef ENAMETOOLONG - /* - TRANS Filename too long (longer than @code{PATH_MAX}; @pxref{Limits for - TRANS Files}) or host name too long (in @code{gethostname} or - TRANS @code{sethostname}; @pxref{Host Identification}). */ --_S(ERR_MAP(ENAMETOOLONG), N_("File name too long")) -+_S(ENAMETOOLONG, N_("File name too long")) - #endif - #ifdef EHOSTDOWN - /* - TRANS The remote host for a requested network connection is down. */ --_S(ERR_MAP(EHOSTDOWN), N_("Host is down")) -+_S(EHOSTDOWN, N_("Host is down")) - #endif - /* - TRANS The remote host for a requested network connection is not reachable. */ - #ifdef EHOSTUNREACH --_S(ERR_MAP(EHOSTUNREACH), N_("No route to host")) -+_S(EHOSTUNREACH, N_("No route to host")) - #endif - #ifdef ENOTEMPTY - /* - TRANS Directory not empty, where an empty directory was expected. Typically, - TRANS this error occurs when you are trying to delete a directory. */ --_S(ERR_MAP(ENOTEMPTY), N_("Directory not empty")) -+_S(ENOTEMPTY, N_("Directory not empty")) - #endif - #ifdef EUSERS - /* - TRANS The file quota system is confused because there are too many users. - TRANS @c This can probably happen in a GNU system when using NFS. */ --_S(ERR_MAP(EUSERS), N_("Too many users")) -+_S(EUSERS, N_("Too many users")) - #endif - #ifdef EDQUOT - /* - TRANS The user's disk quota was exceeded. */ --_S(ERR_MAP(EDQUOT), N_("Disk quota exceeded")) -+_S(EDQUOT, N_("Disk quota exceeded")) - #endif - #ifdef ESTALE - /* -@@ -471,7 +468,7 @@ TRANS file system which is due to file system rearrangements on the server host - TRANS for NFS file systems or corruption in other file systems. - TRANS Repairing this condition usually requires unmounting, possibly repairing - TRANS and remounting the file system. */ --_S(ERR_MAP(ESTALE), N_("Stale file handle")) -+_S(ESTALE, N_("Stale file handle")) - #endif - #ifdef EREMOTE - /* -@@ -479,7 +476,7 @@ TRANS An attempt was made to NFS-mount a remote file system with a file name tha - TRANS already specifies an NFS-mounted file. - TRANS (This is an error on some operating systems, but we expect it to work - TRANS properly on @gnuhurdsystems{}, making this error code impossible.) */ --_S(ERR_MAP(EREMOTE), N_("Object is remote")) -+_S(EREMOTE, N_("Object is remote")) - #endif - #ifdef ENOLCK - /* -@@ -487,7 +484,7 @@ TRANS This is used by the file locking facilities; see - TRANS @ref{File Locks}. This error is never generated by @gnuhurdsystems{}, but - TRANS it can result from an operation to an NFS server running another - TRANS operating system. */ --_S(ERR_MAP(ENOLCK), N_("No locks available")) -+_S(ENOLCK, N_("No locks available")) - #endif - #ifdef ENOSYS - /* -@@ -496,46 +493,46 @@ TRANS not implemented at all, either in the C library itself or in the - TRANS operating system. When you get this error, you can be sure that this - TRANS particular function will always fail with @code{ENOSYS} unless you - TRANS install a new version of the C library or the operating system. */ --_S(ERR_MAP(ENOSYS), N_("Function not implemented")) -+_S(ENOSYS, N_("Function not implemented")) - #endif - #ifdef EILSEQ - /* - TRANS While decoding a multibyte character the function came along an invalid - TRANS or an incomplete sequence of bytes or the given wide character is invalid. */ --_S(ERR_MAP(EILSEQ), N_("Invalid or incomplete multibyte or wide character")) -+_S(EILSEQ, N_("Invalid or incomplete multibyte or wide character")) - #endif - #ifdef EBADMSG --_S(ERR_MAP(EBADMSG), N_("Bad message")) -+_S(EBADMSG, N_("Bad message")) - #endif - #ifdef EIDRM --_S(ERR_MAP(EIDRM), N_("Identifier removed")) -+_S(EIDRM, N_("Identifier removed")) - #endif - #ifdef EMULTIHOP --_S(ERR_MAP(EMULTIHOP), N_("Multihop attempted")) -+_S(EMULTIHOP, N_("Multihop attempted")) - #endif - #ifdef ENODATA --_S(ERR_MAP(ENODATA), N_("No data available")) -+_S(ENODATA, N_("No data available")) - #endif - #ifdef ENOLINK --_S(ERR_MAP(ENOLINK), N_("Link has been severed")) -+_S(ENOLINK, N_("Link has been severed")) - #endif - #ifdef ENOMSG --_S(ERR_MAP(ENOMSG), N_("No message of desired type")) -+_S(ENOMSG, N_("No message of desired type")) - #endif - #ifdef ENOSR --_S(ERR_MAP(ENOSR), N_("Out of streams resources")) -+_S(ENOSR, N_("Out of streams resources")) - #endif - #ifdef ENOSTR --_S(ERR_MAP(ENOSTR), N_("Device not a stream")) -+_S(ENOSTR, N_("Device not a stream")) - #endif - #ifdef EOVERFLOW --_S(ERR_MAP(EOVERFLOW), N_("Value too large for defined data type")) -+_S(EOVERFLOW, N_("Value too large for defined data type")) - #endif - #ifdef EPROTO --_S(ERR_MAP(EPROTO), N_("Protocol error")) -+_S(EPROTO, N_("Protocol error")) - #endif - #ifdef ETIME --_S(ERR_MAP(ETIME), N_("Timer expired")) -+_S(ETIME, N_("Timer expired")) - #endif - #ifdef ECANCELED - /* -@@ -543,148 +540,148 @@ TRANS An asynchronous operation was canceled before it - TRANS completed. @xref{Asynchronous I/O}. When you call @code{aio_cancel}, - TRANS the normal result is for the operations affected to complete with this - TRANS error; @pxref{Cancel AIO Operations}. */ --_S(ERR_MAP(ECANCELED), N_("Operation canceled")) -+_S(ECANCELED, N_("Operation canceled")) - #endif - #ifdef EOWNERDEAD --_S(ERR_MAP(EOWNERDEAD), N_("Owner died")) -+_S(EOWNERDEAD, N_("Owner died")) - #endif - #ifdef ENOTRECOVERABLE --_S(ERR_MAP(ENOTRECOVERABLE), N_("State not recoverable")) -+_S(ENOTRECOVERABLE, N_("State not recoverable")) - #endif - #ifdef ERESTART --_S(ERR_MAP(ERESTART), N_("Interrupted system call should be restarted")) -+_S(ERESTART, N_("Interrupted system call should be restarted")) - #endif - #ifdef ECHRNG --_S(ERR_MAP(ECHRNG), N_("Channel number out of range")) -+_S(ECHRNG, N_("Channel number out of range")) - #endif - #ifdef EL2NSYNC --_S(ERR_MAP(EL2NSYNC), N_("Level 2 not synchronized")) -+_S(EL2NSYNC, N_("Level 2 not synchronized")) - #endif - #ifdef EL3HLT --_S(ERR_MAP(EL3HLT), N_("Level 3 halted")) -+_S(EL3HLT, N_("Level 3 halted")) - #endif - #ifdef EL3RST --_S(ERR_MAP(EL3RST), N_("Level 3 reset")) -+_S(EL3RST, N_("Level 3 reset")) - #endif - #ifdef ELNRNG --_S(ERR_MAP(ELNRNG), N_("Link number out of range")) -+_S(ELNRNG, N_("Link number out of range")) - #endif - #ifdef EUNATCH --_S(ERR_MAP(EUNATCH), N_("Protocol driver not attached")) -+_S(EUNATCH, N_("Protocol driver not attached")) - #endif - #ifdef ENOCSI --_S(ERR_MAP(ENOCSI), N_("No CSI structure available")) -+_S(ENOCSI, N_("No CSI structure available")) - #endif - #ifdef EL2HLT --_S(ERR_MAP(EL2HLT), N_("Level 2 halted")) -+_S(EL2HLT, N_("Level 2 halted")) - #endif - #ifdef EBADE --_S(ERR_MAP(EBADE), N_("Invalid exchange")) -+_S(EBADE, N_("Invalid exchange")) - #endif - #ifdef EBADR --_S(ERR_MAP(EBADR), N_("Invalid request descriptor")) -+_S(EBADR, N_("Invalid request descriptor")) - #endif - #ifdef EXFULL --_S(ERR_MAP(EXFULL), N_("Exchange full")) -+_S(EXFULL, N_("Exchange full")) - #endif - #ifdef ENOANO --_S(ERR_MAP(ENOANO), N_("No anode")) -+_S(ENOANO, N_("No anode")) - #endif - #ifdef EBADRQC --_S(ERR_MAP(EBADRQC), N_("Invalid request code")) -+_S(EBADRQC, N_("Invalid request code")) - #endif - #ifdef EBADSLT --_S(ERR_MAP(EBADSLT), N_("Invalid slot")) -+_S(EBADSLT, N_("Invalid slot")) - #endif - #ifdef EBFONT --_S(ERR_MAP(EBFONT), N_("Bad font file format")) -+_S(EBFONT, N_("Bad font file format")) - #endif - #ifdef ENONET --_S(ERR_MAP(ENONET), N_("Machine is not on the network")) -+_S(ENONET, N_("Machine is not on the network")) - #endif - #ifdef ENOPKG --_S(ERR_MAP(ENOPKG), N_("Package not installed")) -+_S(ENOPKG, N_("Package not installed")) - #endif - #ifdef EADV --_S(ERR_MAP(EADV), N_("Advertise error")) -+_S(EADV, N_("Advertise error")) - #endif - #ifdef ESRMNT --_S(ERR_MAP(ESRMNT), N_("Srmount error")) -+_S(ESRMNT, N_("Srmount error")) - #endif - #ifdef ECOMM --_S(ERR_MAP(ECOMM), N_("Communication error on send")) -+_S(ECOMM, N_("Communication error on send")) - #endif - #ifdef EDOTDOT --_S(ERR_MAP(EDOTDOT), N_("RFS specific error")) -+_S(EDOTDOT, N_("RFS specific error")) - #endif - #ifdef ENOTUNIQ --_S(ERR_MAP(ENOTUNIQ), N_("Name not unique on network")) -+_S(ENOTUNIQ, N_("Name not unique on network")) - #endif - #ifdef EBADFD --_S(ERR_MAP(EBADFD), N_("File descriptor in bad state")) -+_S(EBADFD, N_("File descriptor in bad state")) - #endif - #ifdef EREMCHG --_S(ERR_MAP(EREMCHG), N_("Remote address changed")) -+_S(EREMCHG, N_("Remote address changed")) - #endif - #ifdef ELIBACC --_S(ERR_MAP(ELIBACC), N_("Can not access a needed shared library")) -+_S(ELIBACC, N_("Can not access a needed shared library")) - #endif - #ifdef ELIBBAD --_S(ERR_MAP(ELIBBAD), N_("Accessing a corrupted shared library")) -+_S(ELIBBAD, N_("Accessing a corrupted shared library")) - #endif - #ifdef ELIBSCN --_S(ERR_MAP(ELIBSCN), N_(".lib section in a.out corrupted")) -+_S(ELIBSCN, N_(".lib section in a.out corrupted")) - #endif - #ifdef ELIBMAX --_S(ERR_MAP(ELIBMAX), N_("Attempting to link in too many shared libraries")) -+_S(ELIBMAX, N_("Attempting to link in too many shared libraries")) - #endif - #ifdef ELIBEXEC --_S(ERR_MAP(ELIBEXEC), N_("Cannot exec a shared library directly")) -+_S(ELIBEXEC, N_("Cannot exec a shared library directly")) - #endif - #ifdef ESTRPIPE --_S(ERR_MAP(ESTRPIPE), N_("Streams pipe error")) -+_S(ESTRPIPE, N_("Streams pipe error")) - #endif - #ifdef EUCLEAN --_S(ERR_MAP(EUCLEAN), N_("Structure needs cleaning")) -+_S(EUCLEAN, N_("Structure needs cleaning")) - #endif - #ifdef ENOTNAM --_S(ERR_MAP(ENOTNAM), N_("Not a XENIX named type file")) -+_S(ENOTNAM, N_("Not a XENIX named type file")) - #endif - #ifdef ENAVAIL --_S(ERR_MAP(ENAVAIL), N_("No XENIX semaphores available")) -+_S(ENAVAIL, N_("No XENIX semaphores available")) - #endif - #ifdef EISNAM --_S(ERR_MAP(EISNAM), N_("Is a named type file")) -+_S(EISNAM, N_("Is a named type file")) - #endif - #ifdef EREMOTEIO --_S(ERR_MAP(EREMOTEIO), N_("Remote I/O error")) -+_S(EREMOTEIO, N_("Remote I/O error")) - #endif - #ifdef ENOMEDIUM --_S(ERR_MAP(ENOMEDIUM), N_("No medium found")) -+_S(ENOMEDIUM, N_("No medium found")) - #endif - #ifdef EMEDIUMTYPE --_S(ERR_MAP(EMEDIUMTYPE), N_("Wrong medium type")) -+_S(EMEDIUMTYPE, N_("Wrong medium type")) - #endif - #ifdef ENOKEY --_S(ERR_MAP(ENOKEY), N_("Required key not available")) -+_S(ENOKEY, N_("Required key not available")) - #endif - #ifdef EKEYEXPIRED --_S(ERR_MAP(EKEYEXPIRED), N_("Key has expired")) -+_S(EKEYEXPIRED, N_("Key has expired")) - #endif - #ifdef EKEYREVOKED --_S(ERR_MAP(EKEYREVOKED), N_("Key has been revoked")) -+_S(EKEYREVOKED, N_("Key has been revoked")) - #endif - #ifdef EKEYREJECTED --_S(ERR_MAP(EKEYREJECTED), N_("Key was rejected by service")) -+_S(EKEYREJECTED, N_("Key was rejected by service")) - #endif - #ifdef ERFKILL --_S(ERR_MAP(ERFKILL), N_("Operation not possible due to RF-kill")) -+_S(ERFKILL, N_("Operation not possible due to RF-kill")) - #endif - #ifdef EHWPOISON --_S(ERR_MAP(EHWPOISON), N_("Memory page has hardware error")) -+_S(EHWPOISON, N_("Memory page has hardware error")) - #endif - #ifdef EBADRPC --_S(ERR_MAP(EBADRPC), N_("RPC struct is bad")) -+_S(EBADRPC, N_("RPC struct is bad")) - #endif - #ifdef EFTYPE - /* -@@ -693,40 +690,40 @@ TRANS operation, or a data file had the wrong format. - TRANS - TRANS On some systems @code{chmod} returns this error if you try to set the - TRANS sticky bit on a non-directory file; @pxref{Setting Permissions}. */ --_S(ERR_MAP(EFTYPE), N_("Inappropriate file type or format")) -+_S(EFTYPE, N_("Inappropriate file type or format")) - #endif - #ifdef EPROCUNAVAIL --_S(ERR_MAP(EPROCUNAVAIL), N_("RPC bad procedure for program")) -+_S(EPROCUNAVAIL, N_("RPC bad procedure for program")) - #endif - #ifdef EAUTH --_S(ERR_MAP(EAUTH), N_("Authentication error")) -+_S(EAUTH, N_("Authentication error")) - #endif - #ifdef EDIED - /* - TRANS On @gnuhurdsystems{}, opening a file returns this error when the file is - TRANS translated by a program and the translator program dies while starting - TRANS up, before it has connected to the file. */ --_S(ERR_MAP(EDIED), N_("Translator died")) -+_S(EDIED, N_("Translator died")) - #endif - #ifdef ERPCMISMATCH --_S(ERR_MAP(ERPCMISMATCH), N_("RPC version wrong")) -+_S(ERPCMISMATCH, N_("RPC version wrong")) - #endif - #ifdef EGREGIOUS - /* - TRANS You did @strong{what}? */ --_S(ERR_MAP(EGREGIOUS), N_("You really blew it this time")) -+_S(EGREGIOUS, N_("You really blew it this time")) - #endif - #ifdef EPROCLIM - /* - TRANS This means that the per-user limit on new process would be exceeded by - TRANS an attempted @code{fork}. @xref{Limits on Resources}, for details on - TRANS the @code{RLIMIT_NPROC} limit. */ --_S(ERR_MAP(EPROCLIM), N_("Too many processes")) -+_S(EPROCLIM, N_("Too many processes")) - #endif - #ifdef EGRATUITOUS - /* - TRANS This error code has no purpose. */ --_S(ERR_MAP(EGRATUITOUS), N_("Gratuitous error")) -+_S(EGRATUITOUS, N_("Gratuitous error")) - #endif - #if defined (ENOTSUP) && ENOTSUP != EOPNOTSUPP - /* -@@ -742,10 +739,10 @@ TRANS values. - TRANS - TRANS If the entire function is not available at all in the implementation, - TRANS it returns @code{ENOSYS} instead. */ --_S(ERR_MAP(ENOTSUP), N_("Not supported")) -+_S(ENOTSUP, N_("Not supported")) - #endif - #ifdef EPROGMISMATCH --_S(ERR_MAP(EPROGMISMATCH), N_("RPC program version wrong")) -+_S(EPROGMISMATCH, N_("RPC program version wrong")) - #endif - #ifdef EBACKGROUND - /* -@@ -755,7 +752,7 @@ TRANS foreground process group of the terminal. Users do not usually see this - TRANS error because functions such as @code{read} and @code{write} translate - TRANS it into a @code{SIGTTIN} or @code{SIGTTOU} signal. @xref{Job Control}, - TRANS for information on process groups and these signals. */ --_S(ERR_MAP(EBACKGROUND), N_("Inappropriate operation for background process")) -+_S(EBACKGROUND, N_("Inappropriate operation for background process")) - #endif - #ifdef EIEIO - /* -@@ -773,7 +770,7 @@ TRANS @c "bought the farm" means "died". -jtobey - TRANS @c - TRANS @c Translators, please do not translate this litteraly, translate it into - TRANS @c an idiomatic funny way of saying that the computer died. */ --_S(ERR_MAP(EIEIO), N_("Computer bought the farm")) -+_S(EIEIO, N_("Computer bought the farm")) - #endif - #if defined (EWOULDBLOCK) && EWOULDBLOCK != EAGAIN - /* -@@ -782,18 +779,18 @@ TRANS The values are always the same, on every operating system. - TRANS - TRANS C libraries in many older Unix systems have @code{EWOULDBLOCK} as a - TRANS separate error code. */ --_S(ERR_MAP(EWOULDBLOCK), N_("Operation would block")) -+_S(EWOULDBLOCK, N_("Operation would block")) - #endif - #ifdef ENEEDAUTH --_S(ERR_MAP(ENEEDAUTH), N_("Need authenticator")) -+_S(ENEEDAUTH, N_("Need authenticator")) - #endif - #ifdef ED - /* - TRANS The experienced user will know what is wrong. - TRANS @c This error code is a joke. Its perror text is part of the joke. - TRANS @c Don't change it. */ --_S(ERR_MAP(ED), N_("?")) -+_S(ED, N_("?")) - #endif - #ifdef EPROGUNAVAIL --_S(ERR_MAP(EPROGUNAVAIL), N_("RPC program not available")) -+_S(EPROGUNAVAIL, N_("RPC program not available")) - #endif -diff --git a/sysdeps/i386/dl-machine.h b/sysdeps/i386/dl-machine.h -index 0f08079e48..672d8f27ce 100644 ---- a/sysdeps/i386/dl-machine.h -+++ b/sysdeps/i386/dl-machine.h -@@ -338,16 +338,22 @@ elf_machine_rel (struct link_map *map, const Elf32_Rel *reloc, - { - # ifndef RTLD_BOOTSTRAP - if (sym_map != map -- && sym_map->l_type != lt_executable - && !sym_map->l_relocated) - { - const char *strtab - = (const char *) D_PTR (map, l_info[DT_STRTAB]); -- _dl_error_printf ("\ -+ if (sym_map->l_type == lt_executable) -+ _dl_fatal_printf ("\ -+%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \ -+and creates an unsatisfiable circular dependency.\n", -+ RTLD_PROGNAME, strtab + refsym->st_name, -+ map->l_name); -+ else -+ _dl_error_printf ("\ - %s: Relink `%s' with `%s' for IFUNC symbol `%s'\n", -- RTLD_PROGNAME, map->l_name, -- sym_map->l_name, -- strtab + refsym->st_name); -+ RTLD_PROGNAME, map->l_name, -+ sym_map->l_name, -+ strtab + refsym->st_name); - } - # endif - value = ((Elf32_Addr (*) (void)) value) (); -diff --git a/sysdeps/powerpc/powerpc64/backtrace.c b/sysdeps/powerpc/powerpc64/backtrace.c -index 8a53a1088f..362a2b713c 100644 ---- a/sysdeps/powerpc/powerpc64/backtrace.c -+++ b/sysdeps/powerpc/powerpc64/backtrace.c -@@ -54,11 +54,22 @@ struct signal_frame_64 { - /* We don't care about the rest, since the IP value is at 'uc' field. */ - }; - -+/* Test if the address match to the inside the trampoline code. -+ Up to and including kernel 5.8, returning from an interrupt or syscall to a -+ signal handler starts execution directly at the handler's entry point, with -+ LR set to address of the sigreturn trampoline (the vDSO symbol). -+ Newer kernels will branch to signal handler from the trampoline instead, so -+ checking the stacktrace against the vDSO entrypoint does not work in such -+ case. -+ The vDSO branches with a 'bctrl' instruction, so checking either the -+ vDSO address itself and the next instruction should cover all kernel -+ versions. */ - static inline bool - is_sigtramp_address (void *nip) - { - #ifdef HAVE_SIGTRAMP_RT64 -- if (nip == GLRO (dl_vdso_sigtramp_rt64)) -+ if (nip == GLRO (dl_vdso_sigtramp_rt64) || -+ nip == GLRO (dl_vdso_sigtramp_rt64) + 4) - return true; - #endif - return false; -diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile -index 920d875420..bf9b7f7223 100644 ---- a/sysdeps/pthread/Makefile -+++ b/sysdeps/pthread/Makefile -@@ -107,6 +107,7 @@ tests += tst-cnd-basic tst-mtx-trylock tst-cnd-broadcast \ - tst-unload \ - tst-unwind-thread \ - tst-pt-vfork1 tst-pt-vfork2 tst-vfork1x tst-vfork2x \ -+ tst-pthread-exit-signal \ - - - # Files which must not be linked with libpthread. -diff --git a/sysdeps/pthread/tst-pthread-exit-signal.c b/sysdeps/pthread/tst-pthread-exit-signal.c -new file mode 100644 -index 0000000000..b4526fe663 ---- /dev/null -+++ b/sysdeps/pthread/tst-pthread-exit-signal.c -@@ -0,0 +1,45 @@ -+/* Test that pending signals are not delivered on thread exit (bug 28607). -+ Copyright (C) 2021 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+/* Due to bug 28607, pthread_kill (or pthread_cancel) restored the -+ signal mask during during thread exit, triggering the delivery of a -+ blocked pending signal (SIGUSR1 in this test). */ -+ -+#include <support/xthread.h> -+#include <support/xsignal.h> -+ -+static void * -+threadfunc (void *closure) -+{ -+ sigset_t sigmask; -+ sigfillset (&sigmask); -+ xpthread_sigmask (SIG_SETMASK, &sigmask, NULL); -+ xpthread_kill (pthread_self (), SIGUSR1); -+ pthread_exit (NULL); -+ return NULL; -+} -+ -+static int -+do_test (void) -+{ -+ pthread_t thr = xpthread_create (NULL, threadfunc, NULL); -+ xpthread_join (thr); -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/sysdeps/s390/configure b/sysdeps/s390/configure -index fa46e9e351..e7f576338d 100644 ---- a/sysdeps/s390/configure -+++ b/sysdeps/s390/configure -@@ -123,7 +123,9 @@ void testinsn (char *buf) - __asm__ (".machine \"arch13\" \n\t" - ".machinemode \"zarch_nohighgprs\" \n\t" - "lghi %%r0,16 \n\t" -- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); -+ "mvcrl 0(%0),32(%0) \n\t" -+ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" -+ : : "a" (buf) : "memory", "r0"); - } - EOF - if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS --shared conftest.c -@@ -271,7 +273,9 @@ else - void testinsn (char *buf) - { - __asm__ ("lghi %%r0,16 \n\t" -- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); -+ "mvcrl 0(%0),32(%0) \n\t" -+ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" -+ : : "a" (buf) : "memory", "r0"); - } - EOF - if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS --shared conftest.c -diff --git a/sysdeps/s390/configure.ac b/sysdeps/s390/configure.ac -index 3ed5a8ef87..5c3479e8cf 100644 ---- a/sysdeps/s390/configure.ac -+++ b/sysdeps/s390/configure.ac -@@ -88,7 +88,9 @@ void testinsn (char *buf) - __asm__ (".machine \"arch13\" \n\t" - ".machinemode \"zarch_nohighgprs\" \n\t" - "lghi %%r0,16 \n\t" -- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); -+ "mvcrl 0(%0),32(%0) \n\t" -+ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" -+ : : "a" (buf) : "memory", "r0"); - } - EOF - dnl test, if assembler supports S390 arch13 instructions -@@ -195,7 +197,9 @@ cat > conftest.c <<\EOF - void testinsn (char *buf) - { - __asm__ ("lghi %%r0,16 \n\t" -- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); -+ "mvcrl 0(%0),32(%0) \n\t" -+ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" -+ : : "a" (buf) : "memory", "r0"); - } - EOF - dnl test, if assembler supports S390 arch13 zarch instructions as default -diff --git a/sysdeps/s390/memmove.c b/sysdeps/s390/memmove.c -index 5fc85e129f..ee59b5de14 100644 ---- a/sysdeps/s390/memmove.c -+++ b/sysdeps/s390/memmove.c -@@ -43,7 +43,7 @@ extern __typeof (__redirect_memmove) MEMMOVE_ARCH13 attribute_hidden; - s390_libc_ifunc_expr (__redirect_memmove, memmove, - ({ - s390_libc_ifunc_expr_stfle_init (); -- (HAVE_MEMMOVE_ARCH13 -+ (HAVE_MEMMOVE_ARCH13 && (hwcap & HWCAP_S390_VXRS_EXT2) - && S390_IS_ARCH13_MIE3 (stfle_bits)) - ? MEMMOVE_ARCH13 - : (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX)) -diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c -index e6195c6e26..17c0cc3952 100644 ---- a/sysdeps/s390/multiarch/ifunc-impl-list.c -+++ b/sysdeps/s390/multiarch/ifunc-impl-list.c -@@ -171,7 +171,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, - IFUNC_IMPL (i, name, memmove, - # if HAVE_MEMMOVE_ARCH13 - IFUNC_IMPL_ADD (array, i, memmove, -- S390_IS_ARCH13_MIE3 (stfle_bits), -+ ((dl_hwcap & HWCAP_S390_VXRS_EXT2) -+ && S390_IS_ARCH13_MIE3 (stfle_bits)), - MEMMOVE_ARCH13) - # endif - # if HAVE_MEMMOVE_Z13 -diff --git a/sysdeps/sh/be/sh4/fpu/Implies b/sysdeps/sh/be/sh4/fpu/Implies -new file mode 100644 -index 0000000000..71b28ee1a4 ---- /dev/null -+++ b/sysdeps/sh/be/sh4/fpu/Implies -@@ -0,0 +1 @@ -+sh/sh4/fpu -diff --git a/sysdeps/sh/le/sh4/fpu/Implies b/sysdeps/sh/le/sh4/fpu/Implies -new file mode 100644 -index 0000000000..71b28ee1a4 ---- /dev/null -+++ b/sysdeps/sh/le/sh4/fpu/Implies -@@ -0,0 +1 @@ -+sh/sh4/fpu -diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile -index 9b2a253032..34748ffcd1 100644 ---- a/sysdeps/unix/sysv/linux/Makefile -+++ b/sysdeps/unix/sysv/linux/Makefile -@@ -100,7 +100,7 @@ tests += tst-clone tst-clone2 tst-clone3 tst-fanotify tst-personality \ - tst-quota tst-sync_file_range tst-sysconf-iov_max tst-ttyname \ - test-errno-linux tst-memfd_create tst-mlock2 tst-pkey \ - tst-rlimit-infinity tst-ofdlocks tst-gettid tst-gettid-kill \ -- tst-tgkill -+ tst-tgkill tst-sysvsem-linux tst-sysvmsg-linux tst-sysvshm-linux - tests-internal += tst-ofdlocks-compat tst-sigcontext-get_pc - - CFLAGS-tst-sigcontext-get_pc.c = -fasynchronous-unwind-tables -diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h -index fc688450ee..00a4d0c8e7 100644 ---- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h -+++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h -@@ -54,6 +54,10 @@ - && MIDR_PARTNUM(midr) == 0x000) - #define IS_NEOVERSE_N1(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ - && MIDR_PARTNUM(midr) == 0xd0c) -+#define IS_NEOVERSE_N2(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ -+ && MIDR_PARTNUM(midr) == 0xd49) -+#define IS_NEOVERSE_V1(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ -+ && MIDR_PARTNUM(midr) == 0xd40) - - #define IS_EMAG(midr) (MIDR_IMPLEMENTOR(midr) == 'P' \ - && MIDR_PARTNUM(midr) == 0x000) -diff --git a/sysdeps/unix/sysv/linux/mq_notify.c b/sysdeps/unix/sysv/linux/mq_notify.c -index 61bbb03b64..2bb98172c8 100644 ---- a/sysdeps/unix/sysv/linux/mq_notify.c -+++ b/sysdeps/unix/sysv/linux/mq_notify.c -@@ -132,9 +132,12 @@ helper_thread (void *arg) - to wait until it is done with it. */ - (void) __pthread_barrier_wait (¬ify_barrier); - } -- else if (data.raw[NOTIFY_COOKIE_LEN - 1] == NOTIFY_REMOVED) -- /* The only state we keep is the copy of the thread attributes. */ -- free (data.attr); -+ else if (data.raw[NOTIFY_COOKIE_LEN - 1] == NOTIFY_REMOVED && data.attr != NULL) -+ { -+ /* The only state we keep is the copy of the thread attributes. */ -+ pthread_attr_destroy (data.attr); -+ free (data.attr); -+ } - } - return NULL; - } -@@ -255,8 +258,14 @@ mq_notify (mqd_t mqdes, const struct sigevent *notification) - if (data.attr == NULL) - return -1; - -- memcpy (data.attr, notification->sigev_notify_attributes, -- sizeof (pthread_attr_t)); -+ int ret = __pthread_attr_copy (data.attr, -+ notification->sigev_notify_attributes); -+ if (ret != 0) -+ { -+ free (data.attr); -+ __set_errno (ret); -+ return -1; -+ } - } - - /* Construct the new request. */ -@@ -269,8 +278,11 @@ mq_notify (mqd_t mqdes, const struct sigevent *notification) - int retval = INLINE_SYSCALL (mq_notify, 2, mqdes, &se); - - /* If it failed, free the allocated memory. */ -- if (__glibc_unlikely (retval != 0)) -- free (data.attr); -+ if (retval != 0 && data.attr != NULL) -+ { -+ pthread_attr_destroy (data.attr); -+ free (data.attr); -+ } - - return retval; - } -diff --git a/sysdeps/unix/sysv/linux/msgctl.c b/sysdeps/unix/sysv/linux/msgctl.c -index 0776472d5e..a1f24ab242 100644 ---- a/sysdeps/unix/sysv/linux/msgctl.c -+++ b/sysdeps/unix/sysv/linux/msgctl.c -@@ -90,8 +90,15 @@ __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf) - struct kernel_msqid64_ds ksemid, *arg = NULL; - if (buf != NULL) - { -- msqid64_to_kmsqid64 (buf, &ksemid); -- arg = &ksemid; -+ /* This is a Linux extension where kernel returns a 'struct msginfo' -+ instead. */ -+ if (cmd == IPC_INFO || cmd == MSG_INFO) -+ arg = (struct kernel_msqid64_ds *) buf; -+ else -+ { -+ msqid64_to_kmsqid64 (buf, &ksemid); -+ arg = &ksemid; -+ } - } - # ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T - if (cmd == IPC_SET) -@@ -169,8 +176,15 @@ __msgctl (int msqid, int cmd, struct msqid_ds *buf) - struct __msqid64_ds msqid64, *buf64 = NULL; - if (buf != NULL) - { -- msqid_to_msqid64 (&msqid64, buf); -- buf64 = &msqid64; -+ /* This is a Linux extension where kernel returns a 'struct msginfo' -+ instead. */ -+ if (cmd == IPC_INFO || cmd == MSG_INFO) -+ buf64 = (struct __msqid64_ds *) buf; -+ else -+ { -+ msqid_to_msqid64 (&msqid64, buf); -+ buf64 = &msqid64; -+ } - } - - int ret = __msgctl64 (msqid, cmd, buf64); -diff --git a/sysdeps/unix/sysv/linux/semctl.c b/sysdeps/unix/sysv/linux/semctl.c -index f131a26fc7..1cdabde8f2 100644 ---- a/sysdeps/unix/sysv/linux/semctl.c -+++ b/sysdeps/unix/sysv/linux/semctl.c -@@ -102,6 +102,7 @@ semun64_to_ksemun64 (int cmd, union semun64 semun64, - r.array = semun64.array; - break; - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_STAT: - case IPC_SET: - r.buf = buf; -@@ -150,6 +151,7 @@ __semctl64 (int semid, int semnum, int cmd, ...) - case IPC_STAT: /* arg.buf */ - case IPC_SET: - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_INFO: /* arg.__buf */ - case SEM_INFO: - va_start (ap, cmd); -@@ -238,6 +240,7 @@ semun_to_semun64 (int cmd, union semun semun, struct __semid64_ds *semid64) - r.array = semun.array; - break; - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_STAT: - case IPC_SET: - r.buf = semid64; -@@ -267,6 +270,7 @@ __semctl (int semid, int semnum, int cmd, ...) - case IPC_STAT: /* arg.buf */ - case IPC_SET: - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_INFO: /* arg.__buf */ - case SEM_INFO: - va_start (ap, cmd); -@@ -321,6 +325,7 @@ __semctl_mode16 (int semid, int semnum, int cmd, ...) - case IPC_STAT: /* arg.buf */ - case IPC_SET: - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_INFO: /* arg.__buf */ - case SEM_INFO: - va_start (ap, cmd); -@@ -354,6 +359,7 @@ __old_semctl (int semid, int semnum, int cmd, ...) - case IPC_STAT: /* arg.buf */ - case IPC_SET: - case SEM_STAT: -+ case SEM_STAT_ANY: - case IPC_INFO: /* arg.__buf */ - case SEM_INFO: - va_start (ap, cmd); -diff --git a/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies b/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies -new file mode 100644 -index 0000000000..7eeaf15a5a ---- /dev/null -+++ b/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies -@@ -0,0 +1 @@ -+unix/sysv/linux/sh/sh4/fpu -diff --git a/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies b/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies -new file mode 100644 -index 0000000000..7eeaf15a5a ---- /dev/null -+++ b/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies -@@ -0,0 +1 @@ -+unix/sysv/linux/sh/sh4/fpu -diff --git a/sysdeps/unix/sysv/linux/shmctl.c b/sysdeps/unix/sysv/linux/shmctl.c -index 76d88441f1..1d19a798b1 100644 ---- a/sysdeps/unix/sysv/linux/shmctl.c -+++ b/sysdeps/unix/sysv/linux/shmctl.c -@@ -90,8 +90,15 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf) - struct kernel_shmid64_ds kshmid, *arg = NULL; - if (buf != NULL) - { -- shmid64_to_kshmid64 (buf, &kshmid); -- arg = &kshmid; -+ /* This is a Linux extension where kernel expects either a -+ 'struct shminfo' (IPC_INFO) or 'struct shm_info' (SHM_INFO). */ -+ if (cmd == IPC_INFO || cmd == SHM_INFO) -+ arg = (struct kernel_shmid64_ds *) buf; -+ else -+ { -+ shmid64_to_kshmid64 (buf, &kshmid); -+ arg = &kshmid; -+ } - } - # ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T - if (cmd == IPC_SET) -@@ -107,7 +114,6 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf) - - switch (cmd) - { -- case IPC_INFO: - case IPC_STAT: - case SHM_STAT: - case SHM_STAT_ANY: -@@ -168,8 +174,15 @@ __shmctl (int shmid, int cmd, struct shmid_ds *buf) - struct __shmid64_ds shmid64, *buf64 = NULL; - if (buf != NULL) - { -- shmid_to_shmid64 (&shmid64, buf); -- buf64 = &shmid64; -+ /* This is a Linux extension where kernel expects either a -+ 'struct shminfo' (IPC_INFO) or 'struct shm_info' (SHM_INFO). */ -+ if (cmd == IPC_INFO || cmd == SHM_INFO) -+ buf64 = (struct __shmid64_ds *) buf; -+ else -+ { -+ shmid_to_shmid64 (&shmid64, buf); -+ buf64 = &shmid64; -+ } - } - - int ret = __shmctl64 (shmid, cmd, buf64); -@@ -178,7 +191,6 @@ __shmctl (int shmid, int cmd, struct shmid_ds *buf) - - switch (cmd) - { -- case IPC_INFO: - case IPC_STAT: - case SHM_STAT: - case SHM_STAT_ANY: -diff --git a/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c b/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c -new file mode 100644 -index 0000000000..630f4f792c ---- /dev/null -+++ b/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c -@@ -0,0 +1,177 @@ -+/* Basic tests for Linux SYSV message queue extensions. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <sys/ipc.h> -+#include <sys/msg.h> -+#include <errno.h> -+#include <stdlib.h> -+#include <stdbool.h> -+#include <stdio.h> -+ -+#include <support/check.h> -+#include <support/temp_file.h> -+ -+#define MSGQ_MODE 0644 -+ -+/* These are for the temporary file we generate. */ -+static char *name; -+static int msqid; -+ -+static void -+remove_msq (void) -+{ -+ /* Enforce message queue removal in case of early test failure. -+ Ignore error since the msg may already have being removed. */ -+ msgctl (msqid, IPC_RMID, NULL); -+} -+ -+static void -+do_prepare (int argc, char *argv[]) -+{ -+ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvmsg.", &name) != -1); -+} -+ -+#define PREPARE do_prepare -+ -+struct test_msginfo -+{ -+ int msgmax; -+ int msgmnb; -+ int msgmni; -+}; -+ -+/* It tries to obtain some system-wide SysV messsage queue information from -+ /proc to check against IPC_INFO/MSG_INFO. The /proc only returns the -+ tunables value of MSGMAX, MSGMNB, and MSGMNI. -+ -+ The kernel also returns constant value for MSGSSZ, MSGSEG and also MSGMAP, -+ MSGPOOL, and MSGTQL (for IPC_INFO). The issue to check them is they might -+ change over kernel releases. */ -+ -+static int -+read_proc_file (const char *file) -+{ -+ FILE *f = fopen (file, "r"); -+ if (f == NULL) -+ FAIL_UNSUPPORTED ("/proc is not mounted or %s is not available", file); -+ -+ int v; -+ int r = fscanf (f, "%d", & v); -+ TEST_VERIFY_EXIT (r == 1); -+ -+ fclose (f); -+ return v; -+} -+ -+ -+/* Check if the message queue with IDX (index into the kernel's internal -+ array) matches the one with KEY. The CMD is either MSG_STAT or -+ MSG_STAT_ANY. */ -+ -+static bool -+check_msginfo (int idx, key_t key, int cmd) -+{ -+ struct msqid_ds msginfo; -+ int mid = msgctl (idx, cmd, &msginfo); -+ /* Ignore unused array slot returned by the kernel or information from -+ unknown message queue. */ -+ if ((mid == -1 && errno == EINVAL) || mid != msqid) -+ return false; -+ -+ if (mid == -1) -+ FAIL_EXIT1 ("msgctl with %s failed: %m", -+ cmd == MSG_STAT ? "MSG_STAT" : "MSG_STAT_ANY"); -+ -+ TEST_COMPARE (msginfo.msg_perm.__key, key); -+ TEST_COMPARE (msginfo.msg_perm.mode, MSGQ_MODE); -+ TEST_COMPARE (msginfo.msg_qnum, 0); -+ -+ return true; -+} -+ -+static int -+do_test (void) -+{ -+ atexit (remove_msq); -+ -+ key_t key = ftok (name, 'G'); -+ if (key == -1) -+ FAIL_EXIT1 ("ftok failed: %m"); -+ -+ msqid = msgget (key, MSGQ_MODE | IPC_CREAT); -+ if (msqid == -1) -+ FAIL_EXIT1 ("msgget failed: %m"); -+ -+ struct test_msginfo tipcinfo; -+ tipcinfo.msgmax = read_proc_file ("/proc/sys/kernel/msgmax"); -+ tipcinfo.msgmnb = read_proc_file ("/proc/sys/kernel/msgmnb"); -+ tipcinfo.msgmni = read_proc_file ("/proc/sys/kernel/msgmni"); -+ -+ int msqidx; -+ -+ { -+ struct msginfo ipcinfo; -+ msqidx = msgctl (msqid, IPC_INFO, (struct msqid_ds *) &ipcinfo); -+ if (msqidx == -1) -+ FAIL_EXIT1 ("msgctl with IPC_INFO failed: %m"); -+ -+ TEST_COMPARE (ipcinfo.msgmax, tipcinfo.msgmax); -+ TEST_COMPARE (ipcinfo.msgmnb, tipcinfo.msgmnb); -+ TEST_COMPARE (ipcinfo.msgmni, tipcinfo.msgmni); -+ } -+ -+ /* Same as before but with MSG_INFO. */ -+ { -+ struct msginfo ipcinfo; -+ msqidx = msgctl (msqid, MSG_INFO, (struct msqid_ds *) &ipcinfo); -+ if (msqidx == -1) -+ FAIL_EXIT1 ("msgctl with IPC_INFO failed: %m"); -+ -+ TEST_COMPARE (ipcinfo.msgmax, tipcinfo.msgmax); -+ TEST_COMPARE (ipcinfo.msgmnb, tipcinfo.msgmnb); -+ TEST_COMPARE (ipcinfo.msgmni, tipcinfo.msgmni); -+ } -+ -+ /* We check if the created message queue shows in global list. */ -+ bool found = false; -+ for (int i = 0; i <= msqidx; i++) -+ { -+ /* We can't tell apart if MSG_STAT_ANY is not supported (kernel older -+ than 4.17) or if the index used is invalid. So it just check if the -+ value returned from a valid call matches the created message -+ queue. */ -+ check_msginfo (i, key, MSG_STAT_ANY); -+ -+ if (check_msginfo (i, key, MSG_STAT)) -+ { -+ found = true; -+ break; -+ } -+ } -+ -+ if (!found) -+ FAIL_EXIT1 ("msgctl with MSG_STAT/MSG_STAT_ANY could not find the " -+ "created message queue"); -+ -+ if (msgctl (msqid, IPC_RMID, NULL) == -1) -+ FAIL_EXIT1 ("msgctl failed"); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c b/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c -new file mode 100644 -index 0000000000..45f19e2d37 ---- /dev/null -+++ b/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c -@@ -0,0 +1,184 @@ -+/* Basic tests for Linux SYSV semaphore extensions. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <sys/ipc.h> -+#include <sys/sem.h> -+#include <errno.h> -+#include <stdlib.h> -+#include <stdbool.h> -+#include <stdio.h> -+ -+#include <support/check.h> -+#include <support/temp_file.h> -+ -+/* These are for the temporary file we generate. */ -+static char *name; -+static int semid; -+ -+static void -+remove_sem (void) -+{ -+ /* Enforce message queue removal in case of early test failure. -+ Ignore error since the sem may already have being removed. */ -+ semctl (semid, 0, IPC_RMID, 0); -+} -+ -+static void -+do_prepare (int argc, char *argv[]) -+{ -+ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvsem.", &name) != -1); -+} -+ -+#define PREPARE do_prepare -+ -+#define SEM_MODE 0644 -+ -+union semun -+{ -+ int val; -+ struct semid_ds *buf; -+ unsigned short *array; -+ struct seminfo *__buf; -+}; -+ -+struct test_seminfo -+{ -+ int semmsl; -+ int semmns; -+ int semopm; -+ int semmni; -+}; -+ -+/* It tries to obtain some system-wide SysV semaphore information from /proc -+ to check against IPC_INFO/SEM_INFO. The /proc only returns the tunables -+ value of SEMMSL, SEMMNS, SEMOPM, and SEMMNI. -+ -+ The kernel also returns constant value for SEMVMX, SEMMNU, SEMMAP, SEMUME, -+ and also SEMUSZ and SEMAEM (for IPC_INFO). The issue to check them is they -+ might change over kernel releases. */ -+ -+static void -+read_sem_stat (struct test_seminfo *tseminfo) -+{ -+ FILE *f = fopen ("/proc/sys/kernel/sem", "r"); -+ if (f == NULL) -+ FAIL_UNSUPPORTED ("/proc is not mounted or /proc/sys/kernel/sem is not " -+ "available"); -+ -+ int r = fscanf (f, "%d %d %d %d", -+ &tseminfo->semmsl, &tseminfo->semmns, &tseminfo->semopm, -+ &tseminfo->semmni); -+ TEST_VERIFY_EXIT (r == 4); -+ -+ fclose (f); -+} -+ -+ -+/* Check if the semaphore with IDX (index into the kernel's internal array) -+ matches the one with KEY. The CMD is either SEM_STAT or SEM_STAT_ANY. */ -+ -+static bool -+check_seminfo (int idx, key_t key, int cmd) -+{ -+ struct semid_ds seminfo; -+ int sid = semctl (idx, 0, cmd, (union semun) { .buf = &seminfo }); -+ /* Ignore unused array slot returned by the kernel or information from -+ unknown semaphores. */ -+ if ((sid == -1 && errno == EINVAL) || sid != semid) -+ return false; -+ -+ if (sid == -1) -+ FAIL_EXIT1 ("semctl with SEM_STAT failed (errno=%d)", errno); -+ -+ TEST_COMPARE (seminfo.sem_perm.__key, key); -+ TEST_COMPARE (seminfo.sem_perm.mode, SEM_MODE); -+ TEST_COMPARE (seminfo.sem_nsems, 1); -+ -+ return true; -+} -+ -+static int -+do_test (void) -+{ -+ atexit (remove_sem); -+ -+ key_t key = ftok (name, 'G'); -+ if (key == -1) -+ FAIL_EXIT1 ("ftok failed: %m"); -+ -+ semid = semget (key, 1, IPC_CREAT | IPC_EXCL | SEM_MODE); -+ if (semid == -1) -+ FAIL_EXIT1 ("semget failed: %m"); -+ -+ struct test_seminfo tipcinfo; -+ read_sem_stat (&tipcinfo); -+ -+ int semidx; -+ -+ { -+ struct seminfo ipcinfo; -+ semidx = semctl (semid, 0, IPC_INFO, (union semun) { .__buf = &ipcinfo }); -+ if (semidx == -1) -+ FAIL_EXIT1 ("semctl with IPC_INFO failed: %m"); -+ -+ TEST_COMPARE (ipcinfo.semmsl, tipcinfo.semmsl); -+ TEST_COMPARE (ipcinfo.semmns, tipcinfo.semmns); -+ TEST_COMPARE (ipcinfo.semopm, tipcinfo.semopm); -+ TEST_COMPARE (ipcinfo.semmni, tipcinfo.semmni); -+ } -+ -+ /* Same as before but with SEM_INFO. */ -+ { -+ struct seminfo ipcinfo; -+ semidx = semctl (semid, 0, SEM_INFO, (union semun) { .__buf = &ipcinfo }); -+ if (semidx == -1) -+ FAIL_EXIT1 ("semctl with IPC_INFO failed: %m"); -+ -+ TEST_COMPARE (ipcinfo.semmsl, tipcinfo.semmsl); -+ TEST_COMPARE (ipcinfo.semmns, tipcinfo.semmns); -+ TEST_COMPARE (ipcinfo.semopm, tipcinfo.semopm); -+ TEST_COMPARE (ipcinfo.semmni, tipcinfo.semmni); -+ } -+ -+ /* We check if the created semaphore shows in the system-wide status. */ -+ bool found = false; -+ for (int i = 0; i <= semidx; i++) -+ { -+ /* We can't tell apart if SEM_STAT_ANY is not supported (kernel older -+ than 4.17) or if the index used is invalid. So it just check if -+ value returned from a valid call matches the created semaphore. */ -+ check_seminfo (i, key, SEM_STAT_ANY); -+ -+ if (check_seminfo (i, key, SEM_STAT)) -+ { -+ found = true; -+ break; -+ } -+ } -+ -+ if (!found) -+ FAIL_EXIT1 ("semctl with SEM_STAT/SEM_STAT_ANY could not find the " -+ "created semaphore"); -+ -+ if (semctl (semid, 0, IPC_RMID, 0) == -1) -+ FAIL_EXIT1 ("semctl failed: %m"); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c b/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c -new file mode 100644 -index 0000000000..bb154592a6 ---- /dev/null -+++ b/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c -@@ -0,0 +1,188 @@ -+/* Basic tests for Linux SYSV shared memory extensions. -+ Copyright (C) 2020 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <https://www.gnu.org/licenses/>. */ -+ -+#include <sys/ipc.h> -+#include <sys/shm.h> -+#include <errno.h> -+#include <stdlib.h> -+#include <stdbool.h> -+#include <stdio.h> -+#include <unistd.h> -+#include <inttypes.h> -+#include <limits.h> -+ -+#include <support/check.h> -+#include <support/temp_file.h> -+ -+#define SHM_MODE 0644 -+ -+/* These are for the temporary file we generate. */ -+static char *name; -+static int shmid; -+static long int pgsz; -+ -+static void -+remove_shm (void) -+{ -+ /* Enforce message queue removal in case of early test failure. -+ Ignore error since the shm may already have being removed. */ -+ shmctl (shmid, IPC_RMID, NULL); -+} -+ -+static void -+do_prepare (int argc, char *argv[]) -+{ -+ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvshm.", &name) != -1); -+} -+ -+#define PREPARE do_prepare -+ -+struct test_shminfo -+{ -+ __syscall_ulong_t shmall; -+ __syscall_ulong_t shmmax; -+ __syscall_ulong_t shmmni; -+}; -+ -+/* It tries to obtain some system-wide SysV shared memory information from -+ /proc to check against IPC_INFO/SHM_INFO. The /proc only returns the -+ tunables value of SHMALL, SHMMAX, and SHMMNI. */ -+ -+static uint64_t -+read_proc_file (const char *file) -+{ -+ FILE *f = fopen (file, "r"); -+ if (f == NULL) -+ FAIL_UNSUPPORTED ("/proc is not mounted or %s is not available", file); -+ -+ /* Handle 32-bit binaries running on 64-bit kernels. */ -+ uint64_t v; -+ int r = fscanf (f, "%" SCNu64, &v); -+ TEST_VERIFY_EXIT (r == 1); -+ -+ fclose (f); -+ return v; -+} -+ -+ -+/* Check if the message queue with IDX (index into the kernel's internal -+ array) matches the one with KEY. The CMD is either SHM_STAT or -+ SHM_STAT_ANY. */ -+ -+static bool -+check_shminfo (int idx, key_t key, int cmd) -+{ -+ struct shmid_ds shminfo; -+ int sid = shmctl (idx, cmd, &shminfo); -+ /* Ignore unused array slot returned by the kernel or information from -+ unknown message queue. */ -+ if ((sid == -1 && errno == EINVAL) || sid != shmid) -+ return false; -+ -+ if (sid == -1) -+ FAIL_EXIT1 ("shmctl with %s failed: %m", -+ cmd == SHM_STAT ? "SHM_STAT" : "SHM_STAT_ANY"); -+ -+ TEST_COMPARE (shminfo.shm_perm.__key, key); -+ TEST_COMPARE (shminfo.shm_perm.mode, SHM_MODE); -+ TEST_COMPARE (shminfo.shm_segsz, pgsz); -+ -+ return true; -+} -+ -+static int -+do_test (void) -+{ -+ atexit (remove_shm); -+ -+ pgsz = sysconf (_SC_PAGESIZE); -+ if (pgsz == -1) -+ FAIL_EXIT1 ("sysconf (_SC_PAGESIZE) failed: %m"); -+ -+ key_t key = ftok (name, 'G'); -+ if (key == -1) -+ FAIL_EXIT1 ("ftok failed: %m"); -+ -+ shmid = shmget (key, pgsz, IPC_CREAT | IPC_EXCL | SHM_MODE); -+ if (shmid == -1) -+ FAIL_EXIT1 ("shmget failed: %m"); -+ -+ /* It does not check shmmax because kernel clamp its value to INT_MAX for: -+ -+ 1. Compat symbols with IPC_64, i.e, 32-bit binaries running on 64-bit -+ kernels. -+ -+ 2. Default symbol without IPC_64 (defined as IPC_OLD within Linux) and -+ glibc always use IPC_64 for 32-bit ABIs (to support 64-bit time_t). -+ It means that 32-bit binaries running on 32-bit kernels will not see -+ shmmax being clamped. -+ -+ And finding out whether the compat symbol is used would require checking -+ the underlying kernel against the current ABI. The shmall and shmmni -+ already provided enough coverage. */ -+ -+ struct test_shminfo tipcinfo; -+ tipcinfo.shmall = read_proc_file ("/proc/sys/kernel/shmall"); -+ tipcinfo.shmmni = read_proc_file ("/proc/sys/kernel/shmmni"); -+ -+ int shmidx; -+ -+ /* Note: SHM_INFO does not return a shminfo, but rather a 'struct shm_info'. -+ It is tricky to verify its values since the syscall returns system wide -+ resources consumed by shared memory. The shmctl implementation handles -+ SHM_INFO as IPC_INFO, so the IPC_INFO test should validate SHM_INFO as -+ well. */ -+ -+ { -+ struct shminfo ipcinfo; -+ shmidx = shmctl (shmid, IPC_INFO, (struct shmid_ds *) &ipcinfo); -+ if (shmidx == -1) -+ FAIL_EXIT1 ("shmctl with IPC_INFO failed: %m"); -+ -+ TEST_COMPARE (ipcinfo.shmall, tipcinfo.shmall); -+ TEST_COMPARE (ipcinfo.shmmni, tipcinfo.shmmni); -+ } -+ -+ /* We check if the created shared memory shows in the global list. */ -+ bool found = false; -+ for (int i = 0; i <= shmidx; i++) -+ { -+ /* We can't tell apart if SHM_STAT_ANY is not supported (kernel older -+ than 4.17) or if the index used is invalid. So it just check if -+ value returned from a valid call matches the created message -+ queue. */ -+ check_shminfo (i, key, SHM_STAT_ANY); -+ -+ if (check_shminfo (i, key, SHM_STAT)) -+ { -+ found = true; -+ break; -+ } -+ } -+ -+ if (!found) -+ FAIL_EXIT1 ("shmctl with SHM_STAT/SHM_STAT_ANY could not find the " -+ "created shared memory"); -+ -+ if (shmctl (shmid, IPC_RMID, NULL) == -1) -+ FAIL_EXIT1 ("shmctl failed"); -+ -+ return 0; -+} -+ -+#include <support/test-driver.c> -diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile -index a6736aef25..9736a13e7b 100644 ---- a/sysdeps/x86/Makefile -+++ b/sysdeps/x86/Makefile -@@ -12,6 +12,12 @@ endif - ifeq ($(subdir),setjmp) - gen-as-const-headers += jmp_buf-ssp.sym - sysdep_routines += __longjmp_cancel -+ifneq ($(enable-cet),no) -+ifneq ($(have-tunables),no) -+tests += tst-setjmp-cet -+tst-setjmp-cet-ENV = GLIBC_TUNABLES=glibc.cpu.x86_ibt=on:glibc.cpu.x86_shstk=on -+endif -+endif - endif - - ifeq ($(subdir),string) -diff --git a/sysdeps/x86/cacheinfo.c b/sysdeps/x86/cacheinfo.c -index 217c21c34f..3fb4a028d8 100644 ---- a/sysdeps/x86/cacheinfo.c -+++ b/sysdeps/x86/cacheinfo.c -@@ -808,7 +808,7 @@ init_cacheinfo (void) - threads = 1 << ((ecx >> 12) & 0x0f); - } - -- if (threads == 0) -+ if (threads == 0 || cpu_features->basic.family >= 0x17) - { - /* If APIC ID width is not available, use logical - processor count. */ -@@ -823,8 +823,22 @@ init_cacheinfo (void) - if (threads > 0) - shared /= threads; - -- /* Account for exclusive L2 and L3 caches. */ -- shared += core; -+ /* Get shared cache per ccx for Zen architectures. */ -+ if (cpu_features->basic.family >= 0x17) -+ { -+ unsigned int eax; -+ -+ /* Get number of threads share the L3 cache in CCX. */ -+ __cpuid_count (0x8000001D, 0x3, eax, ebx, ecx, edx); -+ -+ unsigned int threads_per_ccx = ((eax >> 14) & 0xfff) + 1; -+ shared *= threads_per_ccx; -+ } -+ else -+ { -+ /* Account for exclusive L2 and L3 caches. */ -+ shared += core; -+ } - } - } - -@@ -854,14 +868,20 @@ init_cacheinfo (void) - __x86_shared_cache_size = shared; - } - -- /* The large memcpy micro benchmark in glibc shows that 6 times of -- shared cache size is the approximate value above which non-temporal -- store becomes faster on a 8-core processor. This is the 3/4 of the -- total shared cache size. */ -+ /* The default setting for the non_temporal threshold is 3/4 of one -+ thread's share of the chip's cache. For most Intel and AMD processors -+ with an initial release date between 2017 and 2020, a thread's typical -+ share of the cache is from 500 KBytes to 2 MBytes. Using the 3/4 -+ threshold leaves 125 KBytes to 500 KBytes of the thread's data -+ in cache after a maximum temporal copy, which will maintain -+ in cache a reasonable portion of the thread's stack and other -+ active data. If the threshold is set higher than one thread's -+ share of the cache, it has a substantial risk of negatively -+ impacting the performance of other threads running on the chip. */ - __x86_shared_non_temporal_threshold - = (cpu_features->non_temporal_threshold != 0 - ? cpu_features->non_temporal_threshold -- : __x86_shared_cache_size * threads * 3 / 4); -+ : __x86_shared_cache_size * 3 / 4); - - /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8. */ - unsigned int minimum_rep_movsb_threshold; -diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c -index 03572f7af6..3cc54a8d53 100644 ---- a/sysdeps/x86/dl-cet.c -+++ b/sysdeps/x86/dl-cet.c -@@ -47,7 +47,10 @@ dl_cet_check (struct link_map *m, const char *program) - /* No legacy object check if both IBT and SHSTK are always on. */ - if (enable_ibt_type == cet_always_on - && enable_shstk_type == cet_always_on) -- return; -+ { -+ THREAD_SETMEM (THREAD_SELF, header.feature_1, GL(dl_x86_feature_1)); -+ return; -+ } - - /* Check if IBT is enabled by kernel. */ - bool ibt_enabled -diff --git a/sysdeps/x86/dl-prop.h b/sysdeps/x86/dl-prop.h -index 89911e19e2..4eb3b85a7b 100644 ---- a/sysdeps/x86/dl-prop.h -+++ b/sysdeps/x86/dl-prop.h -@@ -145,15 +145,15 @@ _dl_process_cet_property_note (struct link_map *l, - } - - static inline void __attribute__ ((unused)) --_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) -+_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) - { - const ElfW(Nhdr) *note = (const void *) (ph->p_vaddr + l->l_addr); - _dl_process_cet_property_note (l, note, ph->p_memsz, ph->p_align); - } - - static inline int __attribute__ ((always_inline)) --_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, -- void *data) -+_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, -+ uint32_t datasz, void *data) - { - return 0; - } -diff --git a/sysdeps/x86/tst-setjmp-cet.c b/sysdeps/x86/tst-setjmp-cet.c -new file mode 100644 -index 0000000000..42c795d2a8 ---- /dev/null -+++ b/sysdeps/x86/tst-setjmp-cet.c -@@ -0,0 +1 @@ -+#include <setjmp/tst-setjmp.c> -diff --git a/sysdeps/x86_64/configure b/sysdeps/x86_64/configure -old mode 100644 -new mode 100755 -index 84f82c2406..fc1840e23f ---- a/sysdeps/x86_64/configure -+++ b/sysdeps/x86_64/configure -@@ -107,39 +107,6 @@ if test x"$build_mathvec" = xnotset; then - build_mathvec=yes - fi - --if test "$static_pie" = yes; then -- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for linker static PIE support" >&5 --$as_echo_n "checking for linker static PIE support... " >&6; } --if ${libc_cv_ld_static_pie+:} false; then : -- $as_echo_n "(cached) " >&6 --else -- cat > conftest.s <<\EOF -- .text -- .global _start -- .weak foo --_start: -- leaq foo(%rip), %rax --EOF -- libc_cv_pie_option="-Wl,-pie" -- if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS -nostartfiles -nostdlib $no_ssp $libc_cv_pie_option -o conftest conftest.s 1>&5' -- { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 -- (eval $ac_try) 2>&5 -- ac_status=$? -- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 -- test $ac_status = 0; }; }; then -- libc_cv_ld_static_pie=yes -- else -- libc_cv_ld_static_pie=no -- fi --rm -f conftest* --fi --{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $libc_cv_ld_static_pie" >&5 --$as_echo "$libc_cv_ld_static_pie" >&6; } -- if test "$libc_cv_ld_static_pie" != yes; then -- as_fn_error $? "linker support for static PIE needed" "$LINENO" 5 -- fi --fi -- - $as_echo "#define PI_STATIC_AND_HIDDEN 1" >>confdefs.h - - -diff --git a/sysdeps/x86_64/configure.ac b/sysdeps/x86_64/configure.ac -index cdaba0c075..611a7d9ba3 100644 ---- a/sysdeps/x86_64/configure.ac -+++ b/sysdeps/x86_64/configure.ac -@@ -53,31 +53,6 @@ if test x"$build_mathvec" = xnotset; then - build_mathvec=yes - fi - --dnl Check if linker supports static PIE with the fix for --dnl --dnl https://sourceware.org/bugzilla/show_bug.cgi?id=21782 --dnl --if test "$static_pie" = yes; then -- AC_CACHE_CHECK(for linker static PIE support, libc_cv_ld_static_pie, [dnl --cat > conftest.s <<\EOF -- .text -- .global _start -- .weak foo --_start: -- leaq foo(%rip), %rax --EOF -- libc_cv_pie_option="-Wl,-pie" -- if AC_TRY_COMMAND(${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS -nostartfiles -nostdlib $no_ssp $libc_cv_pie_option -o conftest conftest.s 1>&AS_MESSAGE_LOG_FD); then -- libc_cv_ld_static_pie=yes -- else -- libc_cv_ld_static_pie=no -- fi --rm -f conftest*]) -- if test "$libc_cv_ld_static_pie" != yes; then -- AC_MSG_ERROR([linker support for static PIE needed]) -- fi --fi -- - dnl It is always possible to access static and hidden symbols in an - dnl position independent way. - AC_DEFINE(PI_STATIC_AND_HIDDEN) -diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h -index ca73d8fef9..363a749cb2 100644 ---- a/sysdeps/x86_64/dl-machine.h -+++ b/sysdeps/x86_64/dl-machine.h -@@ -315,16 +315,22 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc, - { - # ifndef RTLD_BOOTSTRAP - if (sym_map != map -- && sym_map->l_type != lt_executable - && !sym_map->l_relocated) - { - const char *strtab - = (const char *) D_PTR (map, l_info[DT_STRTAB]); -- _dl_error_printf ("\ -+ if (sym_map->l_type == lt_executable) -+ _dl_fatal_printf ("\ -+%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \ -+and creates an unsatisfiable circular dependency.\n", -+ RTLD_PROGNAME, strtab + refsym->st_name, -+ map->l_name); -+ else -+ _dl_error_printf ("\ - %s: Relink `%s' with `%s' for IFUNC symbol `%s'\n", -- RTLD_PROGNAME, map->l_name, -- sym_map->l_name, -- strtab + refsym->st_name); -+ RTLD_PROGNAME, map->l_name, -+ sym_map->l_name, -+ strtab + refsym->st_name); - } - # endif - value = ((ElfW(Addr) (*) (void)) value) (); -diff --git a/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h b/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h -index 7659758972..e5fd5ac9cb 100644 ---- a/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h -+++ b/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h -@@ -32,7 +32,7 @@ IFUNC_SELECTOR (void) - && CPU_FEATURE_USABLE_P (cpu_features, AVX2)) - return OPTIMIZE (fma); - -- if (CPU_FEATURE_USABLE_P (cpu_features, FMA)) -+ if (CPU_FEATURE_USABLE_P (cpu_features, FMA4)) - return OPTIMIZE (fma4); - - return OPTIMIZE (sse2); -diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S -index bd5dc1a3f3..092f364bb6 100644 ---- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S -+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S -@@ -56,6 +56,13 @@ - # endif - #endif - -+/* Avoid short distance rep movsb only with non-SSE vector. */ -+#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB -+# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16) -+#else -+# define AVOID_SHORT_DISTANCE_REP_MOVSB 0 -+#endif -+ - #ifndef PREFETCH - # define PREFETCH(addr) prefetcht0 addr - #endif -@@ -243,7 +250,21 @@ L(movsb): - cmpq %r9, %rdi - /* Avoid slow backward REP MOVSB. */ - jb L(more_8x_vec_backward) -+# if AVOID_SHORT_DISTANCE_REP_MOVSB -+ movq %rdi, %rcx -+ subq %rsi, %rcx -+ jmp 2f -+# endif - 1: -+# if AVOID_SHORT_DISTANCE_REP_MOVSB -+ movq %rsi, %rcx -+ subq %rdi, %rcx -+2: -+/* Avoid "rep movsb" if RCX, the distance between source and destination, -+ is N*4GB + [1..63] with N >= 0. */ -+ cmpl $63, %ecx -+ jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ -+# endif - mov %RDX_LP, %RCX_LP - rep movsb - L(nop): -diff --git a/sysvipc/test-sysvsem.c b/sysvipc/test-sysvsem.c -index 01dbff343a..b7284e0b48 100644 ---- a/sysvipc/test-sysvsem.c -+++ b/sysvipc/test-sysvsem.c -@@ -20,6 +20,7 @@ - #include <stdlib.h> - #include <errno.h> - #include <string.h> -+#include <stdbool.h> - #include <sys/types.h> - #include <sys/ipc.h> - #include <sys/sem.h> -diff --git a/version.h b/version.h -index 83cd196798..e6ca7a8857 100644 ---- a/version.h -+++ b/version.h -@@ -1,4 +1,4 @@ - /* This file just defines the current version number of libc. */ - --#define RELEASE "release" -+#define RELEASE "stable" - #define VERSION "2.32" -diff -pruN glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/64/configure glibc-2.32/sysdeps/unix/sysv/linux/x86_64/64/configure ---- glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/64/configure 2021-09-18 21:02:32.741186019 +1000 -+++ glibc-2.32/sysdeps/unix/sysv/linux/x86_64/64/configure 2021-09-18 21:03:05.314302356 +1000 -@@ -4,10 +4,10 @@ - test -n "$libc_cv_slibdir" || - case "$prefix" in - /usr | /usr/) -- libc_cv_slibdir='/lib64' -- libc_cv_rtlddir='/lib64' -+ libc_cv_slibdir='/lib' -+ libc_cv_rtlddir='/lib' - if test "$libdir" = '${exec_prefix}/lib'; then -- libdir='${exec_prefix}/lib64'; -+ libdir='${exec_prefix}/lib'; - # Locale data can be shared between 32-bit and 64-bit libraries. - libc_cv_complocaledir='${exec_prefix}/lib/locale' - fi -diff -pruN glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/ldconfig.h glibc-2.32/sysdeps/unix/sysv/linux/x86_64/ldconfig.h ---- glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/ldconfig.h 2021-09-18 21:02:32.742186053 +1000 -+++ glibc-2.32/sysdeps/unix/sysv/linux/x86_64/ldconfig.h 2021-09-18 21:03:05.314302356 +1000 -@@ -18,9 +18,9 @@ - #include <sysdeps/generic/ldconfig.h> - - #define SYSDEP_KNOWN_INTERPRETER_NAMES \ -- { "/lib/ld-linux.so.2", FLAG_ELF_LIBC6 }, \ -+ { "/lib32/ld-linux.so.2", FLAG_ELF_LIBC6 }, \ - { "/libx32/ld-linux-x32.so.2", FLAG_ELF_LIBC6 }, \ -- { "/lib64/ld-linux-x86-64.so.2", FLAG_ELF_LIBC6 }, -+ { "/lib/ld-linux-x86-64.so.2", FLAG_ELF_LIBC6 }, - #define SYSDEP_KNOWN_LIBRARY_NAMES \ - { "libc.so.6", FLAG_ELF_LIBC6 }, \ - { "libm.so.6", FLAG_ELF_LIBC6 }, diff --git a/glibc/glibc-2.32-7.patch b/glibc/glibc-2.32-7.patch new file mode 100644 index 00000000..d7949e11 --- /dev/null +++ b/glibc/glibc-2.32-7.patch @@ -0,0 +1,17967 @@ +diff --git a/NEWS b/NEWS +index 485b8ddffa..d138a45519 100644 +--- a/NEWS ++++ b/NEWS +@@ -5,6 +5,30 @@ See the end for copying conditions. + Please send GNU C library bug reports via <https://sourceware.org/bugzilla/> + using `glibc' in the "product" field. +  ++The following bugs are resolved with this release: ++ ++ [20019] NULL pointer dereference in libc.so.6 IFUNC due to uninitialized GOT ++ [26224] iconv hangs when converting some invalid inputs from several IBM ++ character sets (CVE-2020-27618) ++ [26534] libm.so 2.32 SIGILL in pow() due to FMA4 instruction on non-FMA4 ++ system ++ [26555] string: strerrorname_np does not return the documented value ++ [26600] Transaction ID collisions cause slow DNS lookups in getaddrinfo ++ [26636] libc: 32-bit shmctl(IPC_INFO) crashes when shminfo struct is ++ at the end of a memory mapping ++ [26637] libc: semctl SEM_STAT_ANY fails to pass the buffer specified ++ by the caller to the kernel ++ [26639] libc: msgctl IPC_INFO and MSG_INFO return garbage ++ [26853] aarch64: Missing unwind information in statically linked startup code ++ [26932] libc: sh: Multiple floating point functions defined as stubs only ++ [27130] "rep movsb" performance issue ++ [27177] GLIBC_TUNABLES=glibc.cpu.x86_ibt=on:glibc.cpu.x86_shstk=on doesn't work ++ [27457] vzeroupper use in AVX2 multiarch string functions cause HTM aborts ++ [27974] Overflow bug in some implementation of wcsnlen, wmemchr, and wcsncat ++ [28524] Conversion from ISO-2022-JP-3 with iconv may emit spurious NULs ++ [28607] Masked signals are delivered on thread exit ++ [28755] overflow bug in wcsncmp_avx2 and wcsncmp_evex ++ + Version 2.32 + + Major new features: +@@ -185,6 +209,14 @@ Security related changes: + Dytrych of the Cisco Security Assessment and Penetration Team (See + TALOS-2020-1019). + ++ CVE-2020-27618: An infinite loop has been fixed in the iconv program when ++ invoked with input containing redundant shift sequences in the IBM1364, ++ IBM1371, IBM1388, IBM1390, or IBM1399 character sets. ++ ++ CVE-2021-33574: The mq_notify function has a potential use-after-free ++ issue when using a notification type of SIGEV_THREAD and a thread ++ attribute with a non-default affinity mask. ++ + The following bugs are resolved with this release: + + [9809] localedata: ckb_IQ: new Kurdish Sorani locale +diff --git a/Rules b/Rules +index 8b771f6095..beab969fde 100644 +--- a/Rules ++++ b/Rules +@@ -155,6 +155,7 @@ xtests: tests $(xtests-special) + else + tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \ + $(tests-container:%=$(objpfx)%.out) \ ++ $(tests-mcheck:%=$(objpfx)%-mcheck.out) \ + $(tests-special) $(tests-printers-out) + xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special) + endif +@@ -165,7 +166,7 @@ ifeq ($(run-built-tests),no) + tests-expected = + else + tests-expected = $(tests) $(tests-internal) $(tests-printers) \ +- $(tests-container) ++ $(tests-container) $(tests-mcheck:%=%-mcheck) + endif + tests: + $(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \ +@@ -191,6 +192,7 @@ else + binaries-pie-tests = + binaries-pie-notests = + endif ++binaries-mcheck-tests = $(tests-mcheck:%=%-mcheck) + else + binaries-all-notests = + binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) +@@ -200,6 +202,7 @@ binaries-static-tests = + binaries-static = + binaries-pie-tests = + binaries-pie-notests = ++binaries-mcheck-tests = + endif + + binaries-pie = $(binaries-pie-tests) $(binaries-pie-notests) +@@ -223,6 +226,14 @@ $(addprefix $(objpfx),$(binaries-shared-tests)): %: %.o \ + $(+link-tests) + endif + ++ifneq "$(strip $(binaries-mcheck-tests))" "" ++$(addprefix $(objpfx),$(binaries-mcheck-tests)): %-mcheck: %.o \ ++ $(link-extra-libs-tests) \ ++ $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \ ++ $(addprefix $(csu-objpfx),start.o) $(+preinit) $(+postinit) ++ $(+link-tests) ++endif ++ + ifneq "$(strip $(binaries-pie-tests))" "" + $(addprefix $(objpfx),$(binaries-pie-tests)): %: %.o \ + $(link-extra-libs-tests) \ +@@ -253,6 +264,12 @@ $(addprefix $(objpfx),$(binaries-static-tests)): %: %.o \ + $(+link-static-tests) + endif + ++# All mcheck tests will be run with MALLOC_CHECK_=3 ++define mcheck-ENVS ++$(1)-mcheck-ENV = MALLOC_CHECK_=3 ++endef ++$(foreach t,$(tests-mcheck),$(eval $(call mcheck-ENVS,$(t)))) ++ + ifneq "$(strip $(tests) $(tests-internal) $(xtests) $(test-srcs))" "" + # These are the implicit rules for making test outputs + # from the test programs and whatever input files are present. +diff --git a/debug/Makefile b/debug/Makefile +index 3a60d7af7a..0036edd187 100644 +--- a/debug/Makefile ++++ b/debug/Makefile +@@ -51,7 +51,7 @@ routines = backtrace backtracesyms backtracesymsfd noophooks \ + explicit_bzero_chk \ + stack_chk_fail fortify_fail \ + $(static-only-routines) +-static-only-routines := warning-nop stack_chk_fail_local ++static-only-routines := stack_chk_fail_local + + # Don't add stack_chk_fail_local.o to libc.a since __stack_chk_fail_local + # is an alias of __stack_chk_fail in stack_chk_fail.o. +diff --git a/debug/warning-nop.c b/debug/warning-nop.c +deleted file mode 100644 +index 4ab7e182b7..0000000000 +--- a/debug/warning-nop.c ++++ /dev/null +@@ -1,70 +0,0 @@ +-/* Dummy nop functions to elicit link-time warnings. +- Copyright (C) 2005-2020 Free Software Foundation, Inc. +- This file is part of the GNU C Library. +- +- The GNU C Library is free software; you can redistribute it and/or +- modify it under the terms of the GNU Lesser General Public +- License as published by the Free Software Foundation; either +- version 2.1 of the License, or (at your option) any later version. +- +- In addition to the permissions in the GNU Lesser General Public +- License, the Free Software Foundation gives you unlimited +- permission to link the compiled version of this file with other +- programs, and to distribute those programs without any restriction +- coming from the use of this file. (The GNU Lesser General Public +- License restrictions do apply in other respects; for example, they +- cover modification of the file, and distribution when not linked +- into another program.) +- +- Note that people who make modified versions of this file are not +- obligated to grant this special exception for their modified +- versions; it is their choice whether to do so. The GNU Lesser +- General Public License gives permission to release a modified +- version without this exception; this exception also makes it +- possible to release a modified version which carries forward this +- exception. +- +- The GNU C Library is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- Lesser General Public License for more details. +- +- You should have received a copy of the GNU Lesser General Public +- License along with the GNU C Library; if not, see +- <https://www.gnu.org/licenses/>. */ +- +-#include <sys/cdefs.h> +- +-static void +-__attribute__ ((used)) +-nop (void) +-{ +-} +- +-/* Don't insert any other #include's before this #undef! */ +- +-#undef __warndecl +-#define __warndecl(name, msg) \ +- extern void name (void) __attribute__ ((alias ("nop"))) attribute_hidden; \ +- link_warning (name, msg) +- +-#undef __USE_FORTIFY_LEVEL +-#define __USE_FORTIFY_LEVEL 99 +- +-/* Following here we need an #include for each public header file +- that uses __warndecl. */ +- +-/* Define away to avoid warnings with compilers that do not have these +- builtins. */ +-#define __builtin___memcpy_chk(dest, src, len, bos) NULL +-#define __builtin___memmove_chk(dest, src, len, bos) NULL +-#define __builtin___mempcpy_chk(dest, src, len, bos) NULL +-#define __builtin___memset_chk(dest, ch, len, bos) NULL +-#define __builtin___stpcpy_chk(dest, src, bos) NULL +-#define __builtin___strcat_chk(dest, src, bos) NULL +-#define __builtin___strcpy_chk(dest, src, bos) NULL +-#define __builtin___strncat_chk(dest, src, len, bos) NULL +-#define __builtin___strncpy_chk(dest, src, len, bos) NULL +-#define __builtin_object_size(bos, level) 0 +- +-#include <string.h> +diff --git a/elf/Makefile b/elf/Makefile +index 0b78721848..3ba7f4ecfc 100644 +--- a/elf/Makefile ++++ b/elf/Makefile +@@ -1381,6 +1381,8 @@ CFLAGS-ifuncmain7pie.c += $(pie-ccflag) + CFLAGS-ifuncmain9pie.c += $(pie-ccflag) + CFLAGS-tst-ifunc-textrel.c += $(pic-ccflag) + ++LDFLAGS-ifuncmain6pie = -Wl,-z,lazy ++ + $(objpfx)ifuncmain1pie: $(objpfx)ifuncmod1.so + $(objpfx)ifuncmain1staticpie: $(objpfx)ifuncdep1pic.o + $(objpfx)ifuncmain1vispie: $(objpfx)ifuncmod1.so +@@ -1630,8 +1632,6 @@ $(objpfx)tst-nodelete-dlclose.out: $(objpfx)tst-nodelete-dlclose-dso.so \ + + tst-env-setuid-ENV = MALLOC_CHECK_=2 MALLOC_MMAP_THRESHOLD_=4096 \ + LD_HWCAP_MASK=0x1 +-tst-env-setuid-tunables-ENV = \ +- GLIBC_TUNABLES=glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096 + + $(objpfx)tst-debug1: $(libdl) + $(objpfx)tst-debug1.out: $(objpfx)tst-debug1mod1.so +diff --git a/elf/dl-load.c b/elf/dl-load.c +index e39980fb19..71867e7c1a 100644 +--- a/elf/dl-load.c ++++ b/elf/dl-load.c +@@ -855,10 +855,12 @@ lose (int code, int fd, const char *name, char *realname, struct link_map *l, + + /* Process PT_GNU_PROPERTY program header PH in module L after + PT_LOAD segments are mapped. Only one NT_GNU_PROPERTY_TYPE_0 +- note is handled which contains processor specific properties. */ ++ note is handled which contains processor specific properties. ++ FD is -1 for the kernel mapped main executable otherwise it is ++ the fd used for loading module L. */ + + void +-_dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph) ++_dl_process_pt_gnu_property (struct link_map *l, int fd, const ElfW(Phdr) *ph) + { + const ElfW(Nhdr) *note = (const void *) (ph->p_vaddr + l->l_addr); + const ElfW(Addr) size = ph->p_memsz; +@@ -905,7 +907,7 @@ _dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph) + last_type = type; + + /* Target specific property processing. */ +- if (_dl_process_gnu_property (l, type, datasz, ptr) == 0) ++ if (_dl_process_gnu_property (l, fd, type, datasz, ptr) == 0) + return; + + /* Check the next property item. */ +@@ -1251,21 +1253,6 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, + maplength, has_holes, loader); + if (__glibc_unlikely (errstring != NULL)) + goto call_lose; +- +- /* Process program headers again after load segments are mapped in +- case processing requires accessing those segments. Scan program +- headers backward so that PT_NOTE can be skipped if PT_GNU_PROPERTY +- exits. */ +- for (ph = &phdr[l->l_phnum]; ph != phdr; --ph) +- switch (ph[-1].p_type) +- { +- case PT_NOTE: +- _dl_process_pt_note (l, &ph[-1]); +- break; +- case PT_GNU_PROPERTY: +- _dl_process_pt_gnu_property (l, &ph[-1]); +- break; +- } + } + + if (l->l_ld == 0) +@@ -1377,6 +1364,21 @@ cannot enable executable stack as shared object requires"); + if (l->l_tls_initimage != NULL) + l->l_tls_initimage = (char *) l->l_tls_initimage + l->l_addr; + ++ /* Process program headers again after load segments are mapped in ++ case processing requires accessing those segments. Scan program ++ headers backward so that PT_NOTE can be skipped if PT_GNU_PROPERTY ++ exits. */ ++ for (ph = &l->l_phdr[l->l_phnum]; ph != l->l_phdr; --ph) ++ switch (ph[-1].p_type) ++ { ++ case PT_NOTE: ++ _dl_process_pt_note (l, fd, &ph[-1]); ++ break; ++ case PT_GNU_PROPERTY: ++ _dl_process_pt_gnu_property (l, fd, &ph[-1]); ++ break; ++ } ++ + /* We are done mapping in the file. We no longer need the descriptor. */ + if (__glibc_unlikely (__close_nocancel (fd) != 0)) + { +diff --git a/elf/dl-open.c b/elf/dl-open.c +index 8769e47051..55b39e1bbe 100644 +--- a/elf/dl-open.c ++++ b/elf/dl-open.c +@@ -887,7 +887,7 @@ no more namespaces available for dlmopen()")); + /* Avoid keeping around a dangling reference to the libc.so link + map in case it has been cached in libc_map. */ + if (!args.libc_already_loaded) +- GL(dl_ns)[nsid].libc_map = NULL; ++ GL(dl_ns)[args.nsid].libc_map = NULL; + + /* Remove the object from memory. It may be in an inconsistent + state if relocation failed, for example. */ +diff --git a/elf/dl-tunables.c b/elf/dl-tunables.c +index 26e6e26612..15b29bcb90 100644 +--- a/elf/dl-tunables.c ++++ b/elf/dl-tunables.c +@@ -177,6 +177,7 @@ parse_tunables (char *tunestr, char *valstring) + return; + + char *p = tunestr; ++ size_t off = 0; + + while (true) + { +@@ -190,7 +191,11 @@ parse_tunables (char *tunestr, char *valstring) + /* If we reach the end of the string before getting a valid name-value + pair, bail out. */ + if (p[len] == '\0') +- return; ++ { ++ if (__libc_enable_secure) ++ tunestr[off] = '\0'; ++ return; ++ } + + /* We did not find a valid name-value pair before encountering the + colon. */ +@@ -216,35 +221,28 @@ parse_tunables (char *tunestr, char *valstring) + + if (tunable_is_name (cur->name, name)) + { +- /* If we are in a secure context (AT_SECURE) then ignore the tunable +- unless it is explicitly marked as secure. Tunable values take +- precedence over their envvar aliases. */ ++ /* If we are in a secure context (AT_SECURE) then ignore the ++ tunable unless it is explicitly marked as secure. Tunable ++ values take precedence over their envvar aliases. We write ++ the tunables that are not SXID_ERASE back to TUNESTR, thus ++ dropping all SXID_ERASE tunables and any invalid or ++ unrecognized tunables. */ + if (__libc_enable_secure) + { +- if (cur->security_level == TUNABLE_SECLEVEL_SXID_ERASE) ++ if (cur->security_level != TUNABLE_SECLEVEL_SXID_ERASE) + { +- if (p[len] == '\0') +- { +- /* Last tunable in the valstring. Null-terminate and +- return. */ +- *name = '\0'; +- return; +- } +- else +- { +- /* Remove the current tunable from the string. We do +- this by overwriting the string starting from NAME +- (which is where the current tunable begins) with +- the remainder of the string. We then have P point +- to NAME so that we continue in the correct +- position in the valstring. */ +- char *q = &p[len + 1]; +- p = name; +- while (*q != '\0') +- *name++ = *q++; +- name[0] = '\0'; +- len = 0; +- } ++ if (off > 0) ++ tunestr[off++] = ':'; ++ ++ const char *n = cur->name; ++ ++ while (*n != '\0') ++ tunestr[off++] = *n++; ++ ++ tunestr[off++] = '='; ++ ++ for (size_t j = 0; j < len; j++) ++ tunestr[off++] = value[j]; + } + + if (cur->security_level != TUNABLE_SECLEVEL_NONE) +@@ -257,9 +255,7 @@ parse_tunables (char *tunestr, char *valstring) + } + } + +- if (p[len] == '\0') +- return; +- else ++ if (p[len] != '\0') + p += len + 1; + } + } +diff --git a/elf/ifuncmain6pie.c b/elf/ifuncmain6pie.c +index 04faeb86ef..4a01906836 100644 +--- a/elf/ifuncmain6pie.c ++++ b/elf/ifuncmain6pie.c +@@ -9,7 +9,6 @@ + #include "ifunc-sel.h" + + typedef int (*foo_p) (void); +-extern foo_p foo_ptr; + + static int + one (void) +@@ -28,20 +27,17 @@ foo_ifunc (void) + } + + extern int foo (void); +-extern foo_p get_foo (void); ++extern int call_foo (void); + extern foo_p get_foo_p (void); + +-foo_p my_foo_ptr = foo; ++foo_p foo_ptr = foo; + + int + main (void) + { + foo_p p; + +- p = get_foo (); +- if (p != foo) +- abort (); +- if ((*p) () != -30) ++ if (call_foo () != -30) + abort (); + + p = get_foo_p (); +@@ -52,12 +48,8 @@ main (void) + + if (foo_ptr != foo) + abort (); +- if (my_foo_ptr != foo) +- abort (); + if ((*foo_ptr) () != -30) + abort (); +- if ((*my_foo_ptr) () != -30) +- abort (); + if (foo () != -30) + abort (); + +diff --git a/elf/ifuncmod6.c b/elf/ifuncmod6.c +index 2e16c1d06d..2f6d0715e6 100644 +--- a/elf/ifuncmod6.c ++++ b/elf/ifuncmod6.c +@@ -4,7 +4,7 @@ extern int foo (void); + + typedef int (*foo_p) (void); + +-foo_p foo_ptr = foo; ++extern foo_p foo_ptr; + + foo_p + get_foo_p (void) +@@ -12,8 +12,8 @@ get_foo_p (void) + return foo_ptr; + } + +-foo_p +-get_foo (void) ++int ++call_foo (void) + { +- return foo; ++ return foo (); + } +diff --git a/elf/rtld.c b/elf/rtld.c +index 5b882163fa..14a42ed00a 100644 +--- a/elf/rtld.c ++++ b/elf/rtld.c +@@ -1534,10 +1534,10 @@ of this helper program; chances are you did not intend to run this program.\n\ + switch (ph[-1].p_type) + { + case PT_NOTE: +- _dl_process_pt_note (main_map, &ph[-1]); ++ _dl_process_pt_note (main_map, -1, &ph[-1]); + break; + case PT_GNU_PROPERTY: +- _dl_process_pt_gnu_property (main_map, &ph[-1]); ++ _dl_process_pt_gnu_property (main_map, -1, &ph[-1]); + break; + } + +diff --git a/elf/tst-env-setuid-tunables.c b/elf/tst-env-setuid-tunables.c +index 971d5892b1..ca0c8c245c 100644 +--- a/elf/tst-env-setuid-tunables.c ++++ b/elf/tst-env-setuid-tunables.c +@@ -25,35 +25,76 @@ + #include "config.h" + #undef _LIBC + +-#define test_parent test_parent_tunables +-#define test_child test_child_tunables +- +-static int test_child_tunables (void); +-static int test_parent_tunables (void); +- +-#include "tst-env-setuid.c" +- +-#define CHILD_VALSTRING_VALUE "glibc.malloc.mmap_threshold=4096" +-#define PARENT_VALSTRING_VALUE \ +- "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096" ++#include <errno.h> ++#include <fcntl.h> ++#include <stdlib.h> ++#include <stdint.h> ++#include <stdio.h> ++#include <string.h> ++#include <sys/stat.h> ++#include <sys/wait.h> ++#include <unistd.h> ++#include <intprops.h> ++#include <array_length.h> ++ ++#include <support/check.h> ++#include <support/support.h> ++#include <support/test-driver.h> ++#include <support/capture_subprocess.h> ++ ++const char *teststrings[] = ++{ ++ "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.check=2:glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.check=2:glibc.malloc.mmap_threshold=4096:glibc.malloc.check=2", ++ "glibc.malloc.perturb=0x800", ++ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.perturb=0x800:not_valid.malloc.check=2:glibc.malloc.mmap_threshold=4096", ++ "glibc.not_valid.check=2:glibc.malloc.mmap_threshold=4096", ++ "not_valid.malloc.check=2:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.garbage=2:glibc.maoc.mmap_threshold=4096:glibc.malloc.check=2", ++ "glibc.malloc.check=4:glibc.malloc.garbage=2:glibc.maoc.mmap_threshold=4096", ++ ":glibc.malloc.garbage=2:glibc.malloc.check=1", ++ "glibc.malloc.check=1:glibc.malloc.check=2", ++ "not_valid.malloc.check=2", ++ "glibc.not_valid.check=2", ++}; ++ ++const char *resultstrings[] = ++{ ++ "glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.perturb=0x800", ++ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.perturb=0x800:glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.mmap_threshold=4096", ++ "glibc.malloc.mmap_threshold=4096", ++ "", ++ "", ++ "", ++ "", ++ "", ++ "", ++}; + + static int +-test_child_tunables (void) ++test_child (int off) + { + const char *val = getenv ("GLIBC_TUNABLES"); + + #if HAVE_TUNABLES +- if (val != NULL && strcmp (val, CHILD_VALSTRING_VALUE) == 0) ++ if (val != NULL && strcmp (val, resultstrings[off]) == 0) + return 0; + + if (val != NULL) +- printf ("Unexpected GLIBC_TUNABLES VALUE %s\n", val); ++ printf ("[%d] Unexpected GLIBC_TUNABLES VALUE %s\n", off, val); + + return 1; + #else + if (val != NULL) + { +- printf ("GLIBC_TUNABLES not cleared\n"); ++ printf ("[%d] GLIBC_TUNABLES not cleared\n", off); + return 1; + } + return 0; +@@ -61,15 +102,48 @@ test_child_tunables (void) + } + + static int +-test_parent_tunables (void) ++do_test (int argc, char **argv) + { +- const char *val = getenv ("GLIBC_TUNABLES"); ++ /* Setgid child process. */ ++ if (argc == 2) ++ { ++ if (getgid () == getegid ()) ++ /* This can happen if the file system is mounted nosuid. */ ++ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", ++ (intmax_t) getgid ()); + +- if (val != NULL && strcmp (val, PARENT_VALSTRING_VALUE) == 0) +- return 0; ++ int ret = test_child (atoi (argv[1])); + +- if (val != NULL) +- printf ("Unexpected GLIBC_TUNABLES VALUE %s\n", val); ++ if (ret != 0) ++ exit (1); + +- return 1; ++ exit (EXIT_SUCCESS); ++ } ++ else ++ { ++ int ret = 0; ++ ++ /* Spawn tests. */ ++ for (int i = 0; i < array_length (teststrings); i++) ++ { ++ char buf[INT_BUFSIZE_BOUND (int)]; ++ ++ printf ("Spawned test for %s (%d)\n", teststrings[i], i); ++ snprintf (buf, sizeof (buf), "%d\n", i); ++ if (setenv ("GLIBC_TUNABLES", teststrings[i], 1) != 0) ++ exit (1); ++ ++ int status = support_capture_subprogram_self_sgid (buf); ++ ++ /* Bail out early if unsupported. */ ++ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) ++ return EXIT_UNSUPPORTED; ++ ++ ret |= status; ++ } ++ return ret; ++ } + } ++ ++#define TEST_FUNCTION_ARGV do_test ++#include <support/test-driver.c> +diff --git a/elf/tst-env-setuid.c b/elf/tst-env-setuid.c +index 41dc79e83a..2dbccdb69e 100644 +--- a/elf/tst-env-setuid.c ++++ b/elf/tst-env-setuid.c +@@ -29,173 +29,12 @@ + #include <sys/wait.h> + #include <unistd.h> + ++#include <support/check.h> + #include <support/support.h> + #include <support/test-driver.h> ++#include <support/capture_subprocess.h> + + static char SETGID_CHILD[] = "setgid-child"; +-#define CHILD_STATUS 42 +- +-/* Return a GID which is not our current GID, but is present in the +- supplementary group list. */ +-static gid_t +-choose_gid (void) +-{ +- const int count = 64; +- gid_t groups[count]; +- int ret = getgroups (count, groups); +- if (ret < 0) +- { +- printf ("getgroups: %m\n"); +- exit (1); +- } +- gid_t current = getgid (); +- for (int i = 0; i < ret; ++i) +- { +- if (groups[i] != current) +- return groups[i]; +- } +- return 0; +-} +- +-/* Spawn and execute a program and verify that it returns the CHILD_STATUS. */ +-static pid_t +-do_execve (char **args) +-{ +- pid_t kid = vfork (); +- +- if (kid < 0) +- { +- printf ("vfork: %m\n"); +- return -1; +- } +- +- if (kid == 0) +- { +- /* Child process. */ +- execve (args[0], args, environ); +- _exit (-errno); +- } +- +- if (kid < 0) +- return 1; +- +- int status; +- +- if (waitpid (kid, &status, 0) < 0) +- { +- printf ("waitpid: %m\n"); +- return 1; +- } +- +- if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) +- return EXIT_UNSUPPORTED; +- +- if (!WIFEXITED (status) || WEXITSTATUS (status) != CHILD_STATUS) +- { +- printf ("Unexpected exit status %d from child process\n", +- WEXITSTATUS (status)); +- return 1; +- } +- return 0; +-} +- +-/* Copies the executable into a restricted directory, so that we can +- safely make it SGID with the TARGET group ID. Then runs the +- executable. */ +-static int +-run_executable_sgid (gid_t target) +-{ +- char *dirname = xasprintf ("%s/tst-tunables-setuid.%jd", +- test_dir, (intmax_t) getpid ()); +- char *execname = xasprintf ("%s/bin", dirname); +- int infd = -1; +- int outfd = -1; +- int ret = 0; +- if (mkdir (dirname, 0700) < 0) +- { +- printf ("mkdir: %m\n"); +- goto err; +- } +- infd = open ("/proc/self/exe", O_RDONLY); +- if (infd < 0) +- { +- printf ("open (/proc/self/exe): %m\n"); +- goto err; +- } +- outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); +- if (outfd < 0) +- { +- printf ("open (%s): %m\n", execname); +- goto err; +- } +- char buf[4096]; +- for (;;) +- { +- ssize_t rdcount = read (infd, buf, sizeof (buf)); +- if (rdcount < 0) +- { +- printf ("read: %m\n"); +- goto err; +- } +- if (rdcount == 0) +- break; +- char *p = buf; +- char *end = buf + rdcount; +- while (p != end) +- { +- ssize_t wrcount = write (outfd, buf, end - p); +- if (wrcount == 0) +- errno = ENOSPC; +- if (wrcount <= 0) +- { +- printf ("write: %m\n"); +- goto err; +- } +- p += wrcount; +- } +- } +- if (fchown (outfd, getuid (), target) < 0) +- { +- printf ("fchown (%s): %m\n", execname); +- goto err; +- } +- if (fchmod (outfd, 02750) < 0) +- { +- printf ("fchmod (%s): %m\n", execname); +- goto err; +- } +- if (close (outfd) < 0) +- { +- printf ("close (outfd): %m\n"); +- goto err; +- } +- if (close (infd) < 0) +- { +- printf ("close (infd): %m\n"); +- goto err; +- } +- +- char *args[] = {execname, SETGID_CHILD, NULL}; +- +- ret = do_execve (args); +- +-err: +- if (outfd >= 0) +- close (outfd); +- if (infd >= 0) +- close (infd); +- if (execname) +- { +- unlink (execname); +- free (execname); +- } +- if (dirname) +- { +- rmdir (dirname); +- free (dirname); +- } +- return ret; +-} + + #ifndef test_child + static int +@@ -256,40 +95,32 @@ do_test (int argc, char **argv) + if (argc == 2 && strcmp (argv[1], SETGID_CHILD) == 0) + { + if (getgid () == getegid ()) +- { +- /* This can happen if the file system is mounted nosuid. */ +- fprintf (stderr, "SGID failed: GID and EGID match (%jd)\n", +- (intmax_t) getgid ()); +- exit (EXIT_UNSUPPORTED); +- } ++ /* This can happen if the file system is mounted nosuid. */ ++ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", ++ (intmax_t) getgid ()); + + int ret = test_child (); + + if (ret != 0) + exit (1); + +- exit (CHILD_STATUS); ++ exit (EXIT_SUCCESS); + } + else + { + if (test_parent () != 0) + exit (1); + +- /* Try running a setgid program. */ +- gid_t target = choose_gid (); +- if (target == 0) +- { +- fprintf (stderr, +- "Could not find a suitable GID for user %jd, skipping test\n", +- (intmax_t) getuid ()); +- exit (0); +- } ++ int status = support_capture_subprogram_self_sgid (SETGID_CHILD); + +- return run_executable_sgid (target); +- } ++ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) ++ return EXIT_UNSUPPORTED; ++ ++ if (!WIFEXITED (status)) ++ FAIL_EXIT1 ("Unexpected exit status %d from child process\n", status); + +- /* Something went wrong and our argv was corrupted. */ +- _exit (1); ++ return 0; ++ } + } + + #define TEST_FUNCTION_ARGV do_test +diff --git a/iconv/Versions b/iconv/Versions +index 8a5f4cf780..d51af52fa3 100644 +--- a/iconv/Versions ++++ b/iconv/Versions +@@ -6,7 +6,9 @@ libc { + GLIBC_PRIVATE { + # functions shared with iconv program + __gconv_get_alias_db; __gconv_get_cache; __gconv_get_modules_db; +- __gconv_open; __gconv_create_spec; ++ ++ # functions used elsewhere in glibc ++ __gconv_open; __gconv_create_spec; __gconv_destroy_spec; + + # function used by the gconv modules + __gconv_transliterate; +diff --git a/iconv/gconv_charset.c b/iconv/gconv_charset.c +index 6ccd0773cc..4ba0aa99f5 100644 +--- a/iconv/gconv_charset.c ++++ b/iconv/gconv_charset.c +@@ -216,3 +216,13 @@ out: + return ret; + } + libc_hidden_def (__gconv_create_spec) ++ ++ ++void ++__gconv_destroy_spec (struct gconv_spec *conv_spec) ++{ ++ free (conv_spec->fromcode); ++ free (conv_spec->tocode); ++ return; ++} ++libc_hidden_def (__gconv_destroy_spec) +diff --git a/iconv/gconv_charset.h b/iconv/gconv_charset.h +index b39b09aea1..e9c122cf7e 100644 +--- a/iconv/gconv_charset.h ++++ b/iconv/gconv_charset.h +@@ -48,33 +48,6 @@ + #define GCONV_IGNORE_ERRORS_SUFFIX "IGNORE" + + +-/* This function accepts the charset names of the source and destination of the +- conversion and populates *conv_spec with an equivalent conversion +- specification that may later be used by __gconv_open. The charset names +- might contain options in the form of suffixes that alter the conversion, +- e.g. "ISO-10646/UTF-8/TRANSLIT". It processes the charset names, ignoring +- and truncating any suffix options in fromcode, and processing and truncating +- any suffix options in tocode. Supported suffix options ("TRANSLIT" or +- "IGNORE") when found in tocode lead to the corresponding flag in *conv_spec +- to be set to true. Unrecognized suffix options are silently discarded. If +- the function succeeds, it returns conv_spec back to the caller. It returns +- NULL upon failure. */ +-struct gconv_spec * +-__gconv_create_spec (struct gconv_spec *conv_spec, const char *fromcode, +- const char *tocode); +-libc_hidden_proto (__gconv_create_spec) +- +- +-/* This function frees all heap memory allocated by __gconv_create_spec. */ +-static void __attribute__ ((unused)) +-gconv_destroy_spec (struct gconv_spec *conv_spec) +-{ +- free (conv_spec->fromcode); +- free (conv_spec->tocode); +- return; +-} +- +- + /* This function copies in-order, characters from the source 's' that are + either alpha-numeric or one in one of these: "_-.,:/" - into the destination + 'wp' while dropping all other characters. In the process, it converts all +diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h +index e86938dae7..f721ce30ff 100644 +--- a/iconv/gconv_int.h ++++ b/iconv/gconv_int.h +@@ -152,6 +152,27 @@ extern int __gconv_open (struct gconv_spec *conv_spec, + __gconv_t *handle, int flags); + libc_hidden_proto (__gconv_open) + ++/* This function accepts the charset names of the source and destination of the ++ conversion and populates *conv_spec with an equivalent conversion ++ specification that may later be used by __gconv_open. The charset names ++ might contain options in the form of suffixes that alter the conversion, ++ e.g. "ISO-10646/UTF-8/TRANSLIT". It processes the charset names, ignoring ++ and truncating any suffix options in fromcode, and processing and truncating ++ any suffix options in tocode. Supported suffix options ("TRANSLIT" or ++ "IGNORE") when found in tocode lead to the corresponding flag in *conv_spec ++ to be set to true. Unrecognized suffix options are silently discarded. If ++ the function succeeds, it returns conv_spec back to the caller. It returns ++ NULL upon failure. */ ++extern struct gconv_spec * ++__gconv_create_spec (struct gconv_spec *conv_spec, const char *fromcode, ++ const char *tocode); ++libc_hidden_proto (__gconv_create_spec) ++ ++/* This function frees all heap memory allocated by __gconv_create_spec. */ ++extern void ++__gconv_destroy_spec (struct gconv_spec *conv_spec); ++libc_hidden_proto (__gconv_destroy_spec) ++ + /* Free resources associated with transformation descriptor CD. */ + extern int __gconv_close (__gconv_t cd) + attribute_hidden; +diff --git a/iconv/iconv_open.c b/iconv/iconv_open.c +index dd54bc12e0..5b30055c04 100644 +--- a/iconv/iconv_open.c ++++ b/iconv/iconv_open.c +@@ -39,7 +39,7 @@ iconv_open (const char *tocode, const char *fromcode) + + int res = __gconv_open (&conv_spec, &cd, 0); + +- gconv_destroy_spec (&conv_spec); ++ __gconv_destroy_spec (&conv_spec); + + if (__builtin_expect (res, __GCONV_OK) != __GCONV_OK) + { +diff --git a/iconv/iconv_prog.c b/iconv/iconv_prog.c +index b4334faa57..d59979759c 100644 +--- a/iconv/iconv_prog.c ++++ b/iconv/iconv_prog.c +@@ -184,7 +184,7 @@ main (int argc, char *argv[]) + /* Let's see whether we have these coded character sets. */ + res = __gconv_open (&conv_spec, &cd, 0); + +- gconv_destroy_spec (&conv_spec); ++ __gconv_destroy_spec (&conv_spec); + + if (res != __GCONV_OK) + { +diff --git a/iconv/tst-iconv_prog.sh b/iconv/tst-iconv_prog.sh +index 8298136b7f..d8db7b335c 100644 +--- a/iconv/tst-iconv_prog.sh ++++ b/iconv/tst-iconv_prog.sh +@@ -102,12 +102,16 @@ hangarray=( + "\x00\x80;-c;IBM1161;UTF-8//TRANSLIT//IGNORE" + "\x00\xdb;-c;IBM1162;UTF-8//TRANSLIT//IGNORE" + "\x00\x70;-c;IBM12712;UTF-8//TRANSLIT//IGNORE" +-# These are known hangs that are yet to be fixed: +-# "\x00\x0f;-c;IBM1364;UTF-8" +-# "\x00\x0f;-c;IBM1371;UTF-8" +-# "\x00\x0f;-c;IBM1388;UTF-8" +-# "\x00\x0f;-c;IBM1390;UTF-8" +-# "\x00\x0f;-c;IBM1399;UTF-8" ++"\x00\x0f;-c;IBM1364;UTF-8" ++"\x0e\x0e;-c;IBM1364;UTF-8" ++"\x00\x0f;-c;IBM1371;UTF-8" ++"\x0e\x0e;-c;IBM1371;UTF-8" ++"\x00\x0f;-c;IBM1388;UTF-8" ++"\x0e\x0e;-c;IBM1388;UTF-8" ++"\x00\x0f;-c;IBM1390;UTF-8" ++"\x0e\x0e;-c;IBM1390;UTF-8" ++"\x00\x0f;-c;IBM1399;UTF-8" ++"\x0e\x0e;-c;IBM1399;UTF-8" + "\x00\x53;-c;IBM16804;UTF-8//TRANSLIT//IGNORE" + "\x00\x41;-c;IBM274;UTF-8//TRANSLIT//IGNORE" + "\x00\x41;-c;IBM275;UTF-8//TRANSLIT//IGNORE" +diff --git a/iconvdata/Makefile b/iconvdata/Makefile +index 4ec2741cdc..b67b4feeb4 100644 +--- a/iconvdata/Makefile ++++ b/iconvdata/Makefile +@@ -1,4 +1,5 @@ + # Copyright (C) 1997-2020 Free Software Foundation, Inc. ++# Copyright (C) The GNU Toolchain Authors. + # This file is part of the GNU C Library. + + # The GNU C Library is free software; you can redistribute it and/or +@@ -73,7 +74,8 @@ modules.so := $(addsuffix .so, $(modules)) + ifeq (yes,$(build-shared)) + tests = bug-iconv1 bug-iconv2 tst-loading tst-e2big tst-iconv4 bug-iconv4 \ + tst-iconv6 bug-iconv5 bug-iconv6 tst-iconv7 bug-iconv8 bug-iconv9 \ +- bug-iconv10 bug-iconv11 bug-iconv12 tst-iconv-big5-hkscs-to-2ucs4 ++ bug-iconv10 bug-iconv11 bug-iconv12 tst-iconv-big5-hkscs-to-2ucs4 \ ++ bug-iconv13 bug-iconv14 bug-iconv15 + ifeq ($(have-thread-library),yes) + tests += bug-iconv3 + endif +@@ -321,6 +323,10 @@ $(objpfx)bug-iconv10.out: $(objpfx)gconv-modules \ + $(addprefix $(objpfx),$(modules.so)) + $(objpfx)bug-iconv12.out: $(objpfx)gconv-modules \ + $(addprefix $(objpfx),$(modules.so)) ++$(objpfx)bug-iconv14.out: $(objpfx)gconv-modules \ ++ $(addprefix $(objpfx),$(modules.so)) ++$(objpfx)bug-iconv15.out: $(addprefix $(objpfx), $(gconv-modules)) \ ++ $(addprefix $(objpfx),$(modules.so)) + + $(objpfx)iconv-test.out: run-iconv-test.sh $(objpfx)gconv-modules \ + $(addprefix $(objpfx),$(modules.so)) \ +diff --git a/iconvdata/bug-iconv13.c b/iconvdata/bug-iconv13.c +new file mode 100644 +index 0000000000..87aaff398e +--- /dev/null ++++ b/iconvdata/bug-iconv13.c +@@ -0,0 +1,53 @@ ++/* bug 24973: Test EUC-KR module ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <errno.h> ++#include <iconv.h> ++#include <stdio.h> ++#include <support/check.h> ++ ++static int ++do_test (void) ++{ ++ iconv_t cd = iconv_open ("UTF-8//IGNORE", "EUC-KR"); ++ TEST_VERIFY_EXIT (cd != (iconv_t) -1); ++ ++ /* 0xfe (->0x7e : row 94) and 0xc9 (->0x49 : row 41) are user-defined ++ areas, which are not allowed and should be skipped over due to ++ //IGNORE. The trailing 0xfe also is an incomplete sequence, which ++ should be checked first. */ ++ char input[4] = { '\xc9', '\xa1', '\0', '\xfe' }; ++ char *inptr = input; ++ size_t insize = sizeof (input); ++ char output[4]; ++ char *outptr = output; ++ size_t outsize = sizeof (output); ++ ++ /* This used to crash due to buffer overrun. */ ++ TEST_VERIFY (iconv (cd, &inptr, &insize, &outptr, &outsize) == (size_t) -1); ++ TEST_VERIFY (errno == EINVAL); ++ /* The conversion should produce one character, the converted null ++ character. */ ++ TEST_VERIFY (sizeof (output) - outsize == 1); ++ ++ TEST_VERIFY_EXIT (iconv_close (cd) != -1); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/iconvdata/bug-iconv14.c b/iconvdata/bug-iconv14.c +new file mode 100644 +index 0000000000..902f140fa9 +--- /dev/null ++++ b/iconvdata/bug-iconv14.c +@@ -0,0 +1,127 @@ ++/* Assertion in ISO-2022-JP-3 due to two-character sequence (bug 27256). ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <iconv.h> ++#include <string.h> ++#include <errno.h> ++#include <support/check.h> ++ ++/* Use an escape sequence to return to the initial state. */ ++static void ++with_escape_sequence (void) ++{ ++ iconv_t c = iconv_open ("UTF-8", "ISO-2022-JP-3"); ++ TEST_VERIFY_EXIT (c != (iconv_t) -1); ++ ++ char in[] = "\e$(O+D\e(B"; ++ char *inbuf = in; ++ size_t inleft = strlen (in); ++ char out[3]; /* Space for one output character. */ ++ char *outbuf; ++ size_t outleft; ++ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), (size_t) -1); ++ TEST_COMPARE (errno, E2BIG); ++ TEST_COMPARE (inleft, 3); ++ TEST_COMPARE (inbuf - in, strlen (in) - 3); ++ TEST_COMPARE (outleft, sizeof (out) - 2); ++ TEST_COMPARE (outbuf - out, 2); ++ TEST_COMPARE (out[0] & 0xff, 0xc3); ++ TEST_COMPARE (out[1] & 0xff, 0xa6); ++ ++ /* Return to the initial shift state, producing the pending ++ character. */ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), 0); ++ TEST_COMPARE (inleft, 0); ++ TEST_COMPARE (inbuf - in, strlen (in)); ++ TEST_COMPARE (outleft, sizeof (out) - 2); ++ TEST_COMPARE (outbuf - out, 2); ++ TEST_COMPARE (out[0] & 0xff, 0xcc); ++ TEST_COMPARE (out[1] & 0xff, 0x80); ++ ++ /* Nothing should be flushed the second time. */ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); ++ TEST_COMPARE (outleft, sizeof (out)); ++ TEST_COMPARE (outbuf - out, 0); ++ TEST_COMPARE (out[0] & 0xff, 0xcc); ++ TEST_COMPARE (out[1] & 0xff, 0x80); ++ ++ TEST_COMPARE (iconv_close (c), 0); ++} ++ ++/* Use an explicit flush to return to the initial state. */ ++static void ++with_flush (void) ++{ ++ iconv_t c = iconv_open ("UTF-8", "ISO-2022-JP-3"); ++ TEST_VERIFY_EXIT (c != (iconv_t) -1); ++ ++ char in[] = "\e$(O+D"; ++ char *inbuf = in; ++ size_t inleft = strlen (in); ++ char out[3]; /* Space for one output character. */ ++ char *outbuf; ++ size_t outleft; ++ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, &inbuf, &inleft, &outbuf, &outleft), (size_t) -1); ++ TEST_COMPARE (errno, E2BIG); ++ TEST_COMPARE (inleft, 0); ++ TEST_COMPARE (inbuf - in, strlen (in)); ++ TEST_COMPARE (outleft, sizeof (out) - 2); ++ TEST_COMPARE (outbuf - out, 2); ++ TEST_COMPARE (out[0] & 0xff, 0xc3); ++ TEST_COMPARE (out[1] & 0xff, 0xa6); ++ ++ /* Flush the pending character. */ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); ++ TEST_COMPARE (outleft, sizeof (out) - 2); ++ TEST_COMPARE (outbuf - out, 2); ++ TEST_COMPARE (out[0] & 0xff, 0xcc); ++ TEST_COMPARE (out[1] & 0xff, 0x80); ++ ++ /* Nothing should be flushed the second time. */ ++ outbuf = out; ++ outleft = sizeof (out); ++ TEST_COMPARE (iconv (c, NULL, 0, &outbuf, &outleft), 0); ++ TEST_COMPARE (outleft, sizeof (out)); ++ TEST_COMPARE (outbuf - out, 0); ++ TEST_COMPARE (out[0] & 0xff, 0xcc); ++ TEST_COMPARE (out[1] & 0xff, 0x80); ++ ++ TEST_COMPARE (iconv_close (c), 0); ++} ++ ++static int ++do_test (void) ++{ ++ with_escape_sequence (); ++ with_flush (); ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/iconvdata/bug-iconv15.c b/iconvdata/bug-iconv15.c +new file mode 100644 +index 0000000000..cc04bd0313 +--- /dev/null ++++ b/iconvdata/bug-iconv15.c +@@ -0,0 +1,60 @@ ++/* Bug 28524: Conversion from ISO-2022-JP-3 with iconv ++ may emit spurious NUL character on state reset. ++ Copyright (C) The GNU Toolchain Authors. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <stddef.h> ++#include <iconv.h> ++#include <support/check.h> ++ ++static int ++do_test (void) ++{ ++ char in[] = "\x1b(I"; ++ char *inbuf = in; ++ size_t inleft = sizeof (in) - 1; ++ char out[1]; ++ char *outbuf = out; ++ size_t outleft = sizeof (out); ++ iconv_t cd; ++ ++ cd = iconv_open ("UTF8", "ISO-2022-JP-3"); ++ TEST_VERIFY_EXIT (cd != (iconv_t) -1); ++ ++ /* First call to iconv should alter internal state. ++ Now, JISX0201_Kana_set is selected and ++ state value != ASCII_set. */ ++ TEST_VERIFY (iconv (cd, &inbuf, &inleft, &outbuf, &outleft) != (size_t) -1); ++ ++ /* No bytes should have been added to ++ the output buffer at this point. */ ++ TEST_VERIFY (outbuf == out); ++ TEST_VERIFY (outleft == sizeof (out)); ++ ++ /* Second call shall emit spurious NUL character in unpatched glibc. */ ++ TEST_VERIFY (iconv (cd, NULL, NULL, &outbuf, &outleft) != (size_t) -1); ++ ++ /* No characters are expected to be produced. */ ++ TEST_VERIFY (outbuf == out); ++ TEST_VERIFY (outleft == sizeof (out)); ++ ++ TEST_VERIFY_EXIT (iconv_close (cd) != -1); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/iconvdata/euc-kr.c b/iconvdata/euc-kr.c +index b0d56cf3ee..1045bae926 100644 +--- a/iconvdata/euc-kr.c ++++ b/iconvdata/euc-kr.c +@@ -80,11 +80,7 @@ euckr_from_ucs4 (uint32_t ch, unsigned char *cp) + \ + if (ch <= 0x9f) \ + ++inptr; \ +- /* 0xfe(->0x7e : row 94) and 0xc9(->0x59 : row 41) are \ +- user-defined areas. */ \ +- else if (__builtin_expect (ch == 0xa0, 0) \ +- || __builtin_expect (ch > 0xfe, 0) \ +- || __builtin_expect (ch == 0xc9, 0)) \ ++ else if (__glibc_unlikely (ch == 0xa0)) \ + { \ + /* This is illegal. */ \ + STANDARD_FROM_LOOP_ERR_HANDLER (1); \ +diff --git a/iconvdata/ibm1364.c b/iconvdata/ibm1364.c +index 49e7267ab4..521f0825b7 100644 +--- a/iconvdata/ibm1364.c ++++ b/iconvdata/ibm1364.c +@@ -158,24 +158,14 @@ enum + \ + if (__builtin_expect (ch, 0) == SO) \ + { \ +- /* Shift OUT, change to DBCS converter. */ \ +- if (curcs == db) \ +- { \ +- result = __GCONV_ILLEGAL_INPUT; \ +- break; \ +- } \ ++ /* Shift OUT, change to DBCS converter (redundant escape okay). */ \ + curcs = db; \ + ++inptr; \ + continue; \ + } \ + if (__builtin_expect (ch, 0) == SI) \ + { \ +- /* Shift IN, change to SBCS converter. */ \ +- if (curcs == sb) \ +- { \ +- result = __GCONV_ILLEGAL_INPUT; \ +- break; \ +- } \ ++ /* Shift IN, change to SBCS converter (redundant escape okay). */ \ + curcs = sb; \ + ++inptr; \ + continue; \ +diff --git a/iconvdata/iso-2022-jp-3.c b/iconvdata/iso-2022-jp-3.c +index 8c3b7e627e..c7b470db61 100644 +--- a/iconvdata/iso-2022-jp-3.c ++++ b/iconvdata/iso-2022-jp-3.c +@@ -1,5 +1,6 @@ + /* Conversion module for ISO-2022-JP-3. + Copyright (C) 1998-2020 Free Software Foundation, Inc. ++ Copyright (C) The GNU Toolchain Authors. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998, + and Bruno Haible <bruno@clisp.org>, 2002. +@@ -67,10 +68,15 @@ enum + CURRENT_SEL_MASK = 7 << 3 + }; + +-/* During UCS-4 to ISO-2022-JP-3 conversion, the COUNT element of the state +- also contains the last two bytes to be output, shifted by 6 bits, and a +- one-bit indicator whether they must be preceded by the shift sequence, +- in bit 22. */ ++/* During UCS-4 to ISO-2022-JP-3 conversion, the COUNT element of the ++ state also contains the last two bytes to be output, shifted by 6 ++ bits, and a one-bit indicator whether they must be preceded by the ++ shift sequence, in bit 22. During ISO-2022-JP-3 to UCS-4 ++ conversion, COUNT may also contain a non-zero pending wide ++ character, shifted by six bits. This happens for certain inputs in ++ JISX0213_1_2004_set and JISX0213_2_set if the second wide character ++ in a combining sequence cannot be written because the buffer is ++ full. */ + + /* Since this is a stateful encoding we have to provide code which resets + the output state to the initial state. This has to be done during the +@@ -80,10 +86,27 @@ enum + { \ + if (FROM_DIRECTION) \ + { \ +- /* It's easy, we don't have to emit anything, we just reset the \ +- state for the input. */ \ +- data->__statep->__count &= 7; \ +- data->__statep->__count |= ASCII_set; \ ++ uint32_t ch = data->__statep->__count >> 6; \ ++ \ ++ if (__glibc_unlikely (ch != 0)) \ ++ { \ ++ if (__glibc_likely (outbuf + 4 <= outend)) \ ++ { \ ++ /* Write out the last character. */ \ ++ put32u (outbuf, ch); \ ++ outbuf += 4; \ ++ data->__statep->__count &= 7; \ ++ data->__statep->__count |= ASCII_set; \ ++ } \ ++ else \ ++ /* We don't have enough room in the output buffer. */ \ ++ status = __GCONV_FULL_OUTPUT; \ ++ } \ ++ else \ ++ { \ ++ data->__statep->__count &= 7; \ ++ data->__statep->__count |= ASCII_set; \ ++ } \ + } \ + else \ + { \ +@@ -151,7 +174,21 @@ enum + #define LOOPFCT FROM_LOOP + #define BODY \ + { \ +- uint32_t ch = *inptr; \ ++ uint32_t ch; \ ++ \ ++ /* Output any pending character. */ \ ++ ch = set >> 6; \ ++ if (__glibc_unlikely (ch != 0)) \ ++ { \ ++ put32 (outptr, ch); \ ++ outptr += 4; \ ++ /* Remove the pending character, but preserve state bits. */ \ ++ set &= (1 << 6) - 1; \ ++ continue; \ ++ } \ ++ \ ++ /* Otherwise read the next input byte. */ \ ++ ch = *inptr; \ + \ + /* Recognize escape sequences. */ \ + if (__glibc_unlikely (ch == ESC)) \ +@@ -297,21 +334,25 @@ enum + uint32_t u1 = __jisx0213_to_ucs_combining[ch - 1][0]; \ + uint32_t u2 = __jisx0213_to_ucs_combining[ch - 1][1]; \ + \ ++ inptr += 2; \ ++ \ ++ put32 (outptr, u1); \ ++ outptr += 4; \ ++ \ + /* See whether we have room for two characters. */ \ +- if (outptr + 8 <= outend) \ ++ if (outptr + 4 <= outend) \ + { \ +- inptr += 2; \ +- put32 (outptr, u1); \ +- outptr += 4; \ + put32 (outptr, u2); \ + outptr += 4; \ + continue; \ + } \ +- else \ +- { \ +- result = __GCONV_FULL_OUTPUT; \ +- break; \ +- } \ ++ \ ++ /* Otherwise store only the first character now, and \ ++ put the second one into the queue. */ \ ++ set |= u2 << 6; \ ++ /* Tell the caller why we terminate the loop. */ \ ++ result = __GCONV_FULL_OUTPUT; \ ++ break; \ + } \ + \ + inptr += 2; \ +diff --git a/iconvdata/ksc5601.h b/iconvdata/ksc5601.h +index d3eb3a4ff8..f5cdc72797 100644 +--- a/iconvdata/ksc5601.h ++++ b/iconvdata/ksc5601.h +@@ -50,15 +50,15 @@ ksc5601_to_ucs4 (const unsigned char **s, size_t avail, unsigned char offset) + unsigned char ch2; + int idx; + ++ if (avail < 2) ++ return 0; ++ + /* row 94(0x7e) and row 41(0x49) are user-defined area in KS C 5601 */ + + if (ch < offset || (ch - offset) <= 0x20 || (ch - offset) >= 0x7e + || (ch - offset) == 0x49) + return __UNKNOWN_10646_CHAR; + +- if (avail < 2) +- return 0; +- + ch2 = (*s)[1]; + if (ch2 < offset || (ch2 - offset) <= 0x20 || (ch2 - offset) >= 0x7f) + return __UNKNOWN_10646_CHAR; +diff --git a/intl/dcigettext.c b/intl/dcigettext.c +index 2e7c662bc7..bd332e71da 100644 +--- a/intl/dcigettext.c ++++ b/intl/dcigettext.c +@@ -1120,15 +1120,18 @@ _nl_find_msg (struct loaded_l10nfile *domain_file, + + # ifdef _LIBC + +- struct gconv_spec conv_spec +- = { .fromcode = norm_add_slashes (charset, ""), +- .tocode = norm_add_slashes (outcharset, ""), +- /* We always want to use transliteration. */ +- .translit = true, +- .ignore = false +- }; ++ struct gconv_spec conv_spec; ++ ++ __gconv_create_spec (&conv_spec, charset, outcharset); ++ ++ /* We always want to use transliteration. */ ++ conv_spec.translit = true; ++ + int r = __gconv_open (&conv_spec, &convd->conv, + GCONV_AVOID_NOCONV); ++ ++ __gconv_destroy_spec (&conv_spec); ++ + if (__builtin_expect (r != __GCONV_OK, 0)) + { + /* If the output encoding is the same there is +diff --git a/intl/tst-codeset.c b/intl/tst-codeset.c +index fd70432eca..e9f6e5e09f 100644 +--- a/intl/tst-codeset.c ++++ b/intl/tst-codeset.c +@@ -22,13 +22,11 @@ + #include <stdio.h> + #include <stdlib.h> + #include <string.h> ++#include <support/check.h> + + static int + do_test (void) + { +- char *s; +- int result = 0; +- + unsetenv ("LANGUAGE"); + unsetenv ("OUTPUT_CHARSET"); + setlocale (LC_ALL, "de_DE.ISO-8859-1"); +@@ -36,25 +34,21 @@ do_test (void) + bindtextdomain ("codeset", OBJPFX "domaindir"); + + /* Here we expect output in ISO-8859-1. */ +- s = gettext ("cheese"); +- if (strcmp (s, "K\344se")) +- { +- printf ("call 1 returned: %s\n", s); +- result = 1; +- } ++ TEST_COMPARE_STRING (gettext ("cheese"), "K\344se"); + ++ /* Here we expect output in UTF-8. */ + bind_textdomain_codeset ("codeset", "UTF-8"); ++ TEST_COMPARE_STRING (gettext ("cheese"), "K\303\244se"); + +- /* Here we expect output in UTF-8. */ +- s = gettext ("cheese"); +- if (strcmp (s, "K\303\244se")) +- { +- printf ("call 2 returned: %s\n", s); +- result = 1; +- } +- +- return result; ++ /* `a with umlaut' is transliterated to `ae'. */ ++ bind_textdomain_codeset ("codeset", "ASCII//TRANSLIT"); ++ TEST_COMPARE_STRING (gettext ("cheese"), "Kaese"); ++ ++ /* Transliteration also works by default even if not set. */ ++ bind_textdomain_codeset ("codeset", "ASCII"); ++ TEST_COMPARE_STRING (gettext ("cheese"), "Kaese"); ++ ++ return 0; + } + +-#define TEST_FUNCTION do_test () +-#include "../test-skeleton.c" ++#include <support/test-driver.c> +diff --git a/malloc/Makefile b/malloc/Makefile +index e22cbde22d..5093e8730e 100644 +--- a/malloc/Makefile ++++ b/malloc/Makefile +@@ -62,6 +62,16 @@ endif + tests += $(tests-static) + test-srcs = tst-mtrace + ++# These tests either are run with MALLOC_CHECK_=3 by default or do not work ++# with MALLOC_CHECK_=3 because they expect a specific failure. ++tests-exclude-mcheck = tst-mcheck tst-malloc-usable \ ++ tst-interpose-nothread tst-interpose-static-nothread \ ++ tst-interpose-static-thread tst-malloc-too-large \ ++ tst-mxfast tst-safe-linking ++ ++# Run all tests with MALLOC_CHECK_=3 ++tests-mcheck = $(filter-out $(tests-exclude-mcheck),$(tests)) ++ + routines = malloc morecore mcheck mtrace obstack reallocarray \ + scratch_buffer_grow scratch_buffer_grow_preserve \ + scratch_buffer_set_array_size \ +@@ -100,6 +110,11 @@ $(objpfx)tst-malloc-thread-exit: $(shared-thread-library) + $(objpfx)tst-malloc-thread-fail: $(shared-thread-library) + $(objpfx)tst-malloc-fork-deadlock: $(shared-thread-library) + $(objpfx)tst-malloc-stats-cancellation: $(shared-thread-library) ++$(objpfx)tst-malloc-backtrace-mcheck: $(shared-thread-library) ++$(objpfx)tst-malloc-thread-exit-mcheck: $(shared-thread-library) ++$(objpfx)tst-malloc-thread-fail-mcheck: $(shared-thread-library) ++$(objpfx)tst-malloc-fork-deadlock-mcheck: $(shared-thread-library) ++$(objpfx)tst-malloc-stats-cancellation-mcheck: $(shared-thread-library) + + # Export the __malloc_initialize_hook variable to libc.so. + LDFLAGS-tst-mallocstate = -rdynamic +@@ -239,6 +254,8 @@ $(tests:%=$(objpfx)%.o): CPPFLAGS += -DTEST_NO_MALLOPT + $(objpfx)tst-interpose-nothread: $(objpfx)tst-interpose-aux-nothread.o + $(objpfx)tst-interpose-thread: \ + $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) ++$(objpfx)tst-interpose-thread-mcheck: \ ++ $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) + $(objpfx)tst-interpose-static-nothread: $(objpfx)tst-interpose-aux-nothread.o + $(objpfx)tst-interpose-static-thread: \ + $(objpfx)tst-interpose-aux-thread.o $(static-thread-library) +@@ -256,3 +273,6 @@ $(objpfx)tst-dynarray-fail-mem.out: $(objpfx)tst-dynarray-fail.out + $(objpfx)tst-malloc-tcache-leak: $(shared-thread-library) + $(objpfx)tst-malloc_info: $(shared-thread-library) + $(objpfx)tst-mallocfork2: $(shared-thread-library) ++$(objpfx)tst-malloc-tcache-leak-mcheck: $(shared-thread-library) ++$(objpfx)tst-malloc_info-mcheck: $(shared-thread-library) ++$(objpfx)tst-mallocfork2-mcheck: $(shared-thread-library) +diff --git a/manual/tunables.texi b/manual/tunables.texi +index 23ef0d40e7..d72d7a5ec0 100644 +--- a/manual/tunables.texi ++++ b/manual/tunables.texi +@@ -432,7 +432,11 @@ set shared cache size in bytes for use in memory and string routines. + + @deftp Tunable glibc.cpu.x86_non_temporal_threshold + The @code{glibc.cpu.x86_non_temporal_threshold} tunable allows the user +-to set threshold in bytes for non temporal store. ++to set threshold in bytes for non temporal store. Non temporal stores ++give a hint to the hardware to move data directly to memory without ++displacing other data from the cache. This tunable is used by some ++platforms to determine when to use non temporal stores in operations ++like memmove and memcpy. + + This tunable is specific to i386 and x86-64. + @end deftp +diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h +index 19d9cc5cfe..38221d0b2a 100644 +--- a/misc/sys/cdefs.h ++++ b/misc/sys/cdefs.h +@@ -124,13 +124,10 @@ + #define __bos0(ptr) __builtin_object_size (ptr, 0) + + #if __GNUC_PREREQ (4,3) +-# define __warndecl(name, msg) \ +- extern void name (void) __attribute__((__warning__ (msg))) + # define __warnattr(msg) __attribute__((__warning__ (msg))) + # define __errordecl(name, msg) \ + extern void name (void) __attribute__((__error__ (msg))) + #else +-# define __warndecl(name, msg) extern void name (void) + # define __warnattr(msg) + # define __errordecl(name, msg) extern void name (void) + #endif +diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c +index 2cba3da38c..c217cda608 100644 +--- a/nptl/pthread_create.c ++++ b/nptl/pthread_create.c +@@ -416,8 +416,6 @@ START_THREAD_DEFN + unwind_buf.priv.data.prev = NULL; + unwind_buf.priv.data.cleanup = NULL; + +- __libc_signal_restore_set (&pd->sigmask); +- + /* Allow setxid from now onwards. */ + if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) + futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); +@@ -427,6 +425,8 @@ START_THREAD_DEFN + /* Store the new cleanup handler info. */ + THREAD_SETMEM (pd, cleanup_jmp_buf, &unwind_buf); + ++ __libc_signal_restore_set (&pd->sigmask); ++ + /* We are either in (a) or (b), and in either case we either own + PD already (2) or are about to own PD (1), and so our only + restriction would be that we can't free PD until we know we +diff --git a/nscd/netgroupcache.c b/nscd/netgroupcache.c +index 88c69d1e9c..381aa721ef 100644 +--- a/nscd/netgroupcache.c ++++ b/nscd/netgroupcache.c +@@ -248,7 +248,7 @@ addgetnetgrentX (struct database_dyn *db, int fd, request_header *req, + : NULL); + ndomain = (ndomain ? newbuf + ndomaindiff + : NULL); +- buffer = newbuf; ++ *tofreep = buffer = newbuf; + } + + nhost = memcpy (buffer + bufused, +@@ -319,7 +319,7 @@ addgetnetgrentX (struct database_dyn *db, int fd, request_header *req, + else if (status == NSS_STATUS_TRYAGAIN && e == ERANGE) + { + buflen *= 2; +- buffer = xrealloc (buffer, buflen); ++ *tofreep = buffer = xrealloc (buffer, buflen); + } + else if (status == NSS_STATUS_RETURN + || status == NSS_STATUS_NOTFOUND +diff --git a/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf b/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf +new file mode 100644 +index 0000000000..5b0c6a4199 +--- /dev/null ++++ b/nss/tst-nss-files-hosts-long.root/etc/nsswitch.conf +@@ -0,0 +1 @@ ++hosts: files +diff --git a/posix/bits/unistd.h b/posix/bits/unistd.h +index 725a83eb0d..7e5bb6fb1e 100644 +--- a/posix/bits/unistd.h ++++ b/posix/bits/unistd.h +@@ -193,10 +193,9 @@ __NTH (readlinkat (int __fd, const char *__restrict __path, + #endif + + extern char *__getcwd_chk (char *__buf, size_t __size, size_t __buflen) +- __THROW __wur __attr_access ((__write_only__, 1, 2)); ++ __THROW __wur; + extern char *__REDIRECT_NTH (__getcwd_alias, +- (char *__buf, size_t __size), getcwd) +- __wur __attr_access ((__write_only__, 1, 2)); ++ (char *__buf, size_t __size), getcwd) __wur; + extern char *__REDIRECT_NTH (__getcwd_chk_warn, + (char *__buf, size_t __size, size_t __buflen), + __getcwd_chk) +diff --git a/posix/unistd.h b/posix/unistd.h +index 32b8161619..acf9ee7e79 100644 +--- a/posix/unistd.h ++++ b/posix/unistd.h +@@ -517,8 +517,7 @@ extern int fchdir (int __fd) __THROW __wur; + an array is allocated with `malloc'; the array is SIZE + bytes long, unless SIZE == 0, in which case it is as + big as necessary. */ +-extern char *getcwd (char *__buf, size_t __size) __THROW __wur +- __attr_access ((__write_only__, 1, 2)); ++extern char *getcwd (char *__buf, size_t __size) __THROW __wur; + + #ifdef __USE_GNU + /* Return a malloc'd string containing the current directory name. +@@ -831,7 +830,7 @@ extern int symlinkat (const char *__from, int __tofd, + /* Like readlink but a relative PATH is interpreted relative to FD. */ + extern ssize_t readlinkat (int __fd, const char *__restrict __path, + char *__restrict __buf, size_t __len) +- __THROW __nonnull ((2, 3)) __wur __attr_access ((__read_only__, 3, 4)); ++ __THROW __nonnull ((2, 3)) __wur __attr_access ((__write_only__, 3, 4)); + #endif + + /* Remove the link NAME. */ +diff --git a/posix/wordexp-test.c b/posix/wordexp-test.c +index ed1b22308e..cb3f989cba 100644 +--- a/posix/wordexp-test.c ++++ b/posix/wordexp-test.c +@@ -183,6 +183,7 @@ struct test_case_struct + { 0, NULL, "$var", 0, 0, { NULL, }, IFS }, + { 0, NULL, "\"\\n\"", 0, 1, { "\\n", }, IFS }, + { 0, NULL, "", 0, 0, { NULL, }, IFS }, ++ { 0, NULL, "${1234567890123456789012}", 0, 0, { NULL, }, IFS }, + + /* Flags not already covered (testit() has special handling for these) */ + { 0, NULL, "one two", WRDE_DOOFFS, 2, { "one", "two", }, IFS }, +diff --git a/posix/wordexp.c b/posix/wordexp.c +index e082d94895..56289503a1 100644 +--- a/posix/wordexp.c ++++ b/posix/wordexp.c +@@ -1399,7 +1399,7 @@ envsubst: + /* Is it a numeric parameter? */ + else if (isdigit (env[0])) + { +- int n = atoi (env); ++ unsigned long n = strtoul (env, NULL, 10); + + if (n >= __libc_argc) + /* Substitute NULL. */ +diff --git a/resolv/Makefile b/resolv/Makefile +index b61c0c3e0c..dbd8f8bf4f 100644 +--- a/resolv/Makefile ++++ b/resolv/Makefile +@@ -61,6 +61,11 @@ tests += \ + tst-resolv-search \ + tst-resolv-trailing \ + ++# This test calls __res_context_send directly, which is not exported ++# from libresolv. ++tests-internal += tst-resolv-txnid-collision ++tests-static += tst-resolv-txnid-collision ++ + # These tests need libdl. + ifeq (yes,$(build-shared)) + tests += \ +@@ -191,6 +196,8 @@ $(objpfx)tst-resolv-search: $(objpfx)libresolv.so $(shared-thread-library) + $(objpfx)tst-resolv-trailing: $(objpfx)libresolv.so $(shared-thread-library) + $(objpfx)tst-resolv-threads: \ + $(libdl) $(objpfx)libresolv.so $(shared-thread-library) ++$(objpfx)tst-resolv-txnid-collision: $(objpfx)libresolv.a \ ++ $(static-thread-library) + $(objpfx)tst-resolv-canonname: \ + $(libdl) $(objpfx)libresolv.so $(shared-thread-library) + $(objpfx)tst-resolv-trustad: $(objpfx)libresolv.so $(shared-thread-library) +diff --git a/resolv/res_send.c b/resolv/res_send.c +index 7e5fec6646..70e5066031 100644 +--- a/resolv/res_send.c ++++ b/resolv/res_send.c +@@ -1342,15 +1342,6 @@ send_dg(res_state statp, + *terrno = EMSGSIZE; + return close_and_return_error (statp, resplen2); + } +- if ((recvresp1 || hp->id != anhp->id) +- && (recvresp2 || hp2->id != anhp->id)) { +- /* +- * response from old query, ignore it. +- * XXX - potential security hazard could +- * be detected here. +- */ +- goto wait; +- } + + /* Paranoia check. Due to the connected UDP socket, + the kernel has already filtered invalid addresses +@@ -1360,15 +1351,24 @@ send_dg(res_state statp, + + /* Check for the correct header layout and a matching + question. */ +- if ((recvresp1 || !res_queriesmatch(buf, buf + buflen, +- *thisansp, +- *thisansp +- + *thisanssizp)) +- && (recvresp2 || !res_queriesmatch(buf2, buf2 + buflen2, +- *thisansp, +- *thisansp +- + *thisanssizp))) +- goto wait; ++ int matching_query = 0; /* Default to no matching query. */ ++ if (!recvresp1 ++ && anhp->id == hp->id ++ && res_queriesmatch (buf, buf + buflen, ++ *thisansp, *thisansp + *thisanssizp)) ++ matching_query = 1; ++ if (!recvresp2 ++ && anhp->id == hp2->id ++ && res_queriesmatch (buf2, buf2 + buflen2, ++ *thisansp, *thisansp + *thisanssizp)) ++ matching_query = 2; ++ if (matching_query == 0) ++ /* Spurious UDP packet. Drop it and continue ++ waiting. */ ++ { ++ need_recompute = 1; ++ goto wait; ++ } + + if (anhp->rcode == SERVFAIL || + anhp->rcode == NOTIMP || +@@ -1383,7 +1383,7 @@ send_dg(res_state statp, + /* No data from the first reply. */ + resplen = 0; + /* We are waiting for a possible second reply. */ +- if (hp->id == anhp->id) ++ if (matching_query == 1) + recvresp1 = 1; + else + recvresp2 = 1; +@@ -1414,7 +1414,7 @@ send_dg(res_state statp, + return (1); + } + /* Mark which reply we received. */ +- if (recvresp1 == 0 && hp->id == anhp->id) ++ if (matching_query == 1) + recvresp1 = 1; + else + recvresp2 = 1; +diff --git a/resolv/tst-resolv-txnid-collision.c b/resolv/tst-resolv-txnid-collision.c +new file mode 100644 +index 0000000000..189b76f126 +--- /dev/null ++++ b/resolv/tst-resolv-txnid-collision.c +@@ -0,0 +1,334 @@ ++/* Test parallel queries with transaction ID collisions. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <arpa/nameser.h> ++#include <array_length.h> ++#include <resolv-internal.h> ++#include <resolv_context.h> ++#include <stdbool.h> ++#include <stdio.h> ++#include <string.h> ++#include <support/check.h> ++#include <support/check_nss.h> ++#include <support/resolv_test.h> ++#include <support/support.h> ++#include <support/test-driver.h> ++ ++/* Result of parsing a DNS question name. ++ ++ A question name has the form reorder-N-M-rcode-C.example.net, where ++ N and M are either 0 and 1, corresponding to the reorder member, ++ and C is a number that will be stored in the rcode field. ++ ++ Also see parse_qname below. */ ++struct parsed_qname ++{ ++ /* The DNS response code requested from the first server. The ++ second server always responds with RCODE zero. */ ++ int rcode; ++ ++ /* Indicates whether to perform reordering in the responses from the ++ respective server. */ ++ bool reorder[2]; ++}; ++ ++/* Fills *PARSED based on QNAME. */ ++static void ++parse_qname (struct parsed_qname *parsed, const char *qname) ++{ ++ int reorder0; ++ int reorder1; ++ int rcode; ++ char *suffix; ++ if (sscanf (qname, "reorder-%d-%d.rcode-%d.%ms", ++ &reorder0, &reorder1, &rcode, &suffix) == 4) ++ { ++ if (reorder0 != 0) ++ TEST_COMPARE (reorder0, 1); ++ if (reorder1 != 0) ++ TEST_COMPARE (reorder1, 1); ++ TEST_VERIFY (rcode >= 0 && rcode <= 15); ++ TEST_COMPARE_STRING (suffix, "example.net"); ++ free (suffix); ++ ++ parsed->rcode = rcode; ++ parsed->reorder[0] = reorder0; ++ parsed->reorder[1] = reorder1; ++ } ++ else ++ FAIL_EXIT1 ("unexpected query: %s", qname); ++} ++ ++/* Used to construct a response. The first server responds with an ++ error, the second server succeeds. */ ++static void ++build_response (const struct resolv_response_context *ctx, ++ struct resolv_response_builder *b, ++ const char *qname, uint16_t qclass, uint16_t qtype) ++{ ++ struct parsed_qname parsed; ++ parse_qname (&parsed, qname); ++ ++ switch (ctx->server_index) ++ { ++ case 0: ++ { ++ struct resolv_response_flags flags = { 0 }; ++ if (parsed.rcode == 0) ++ /* Simulate a delegation in case a NODATA (RCODE zero) ++ response is requested. */ ++ flags.clear_ra = true; ++ else ++ flags.rcode = parsed.rcode; ++ ++ resolv_response_init (b, flags); ++ resolv_response_add_question (b, qname, qclass, qtype); ++ } ++ break; ++ ++ case 1: ++ { ++ struct resolv_response_flags flags = { 0, }; ++ resolv_response_init (b, flags); ++ resolv_response_add_question (b, qname, qclass, qtype); ++ ++ resolv_response_section (b, ns_s_an); ++ resolv_response_open_record (b, qname, qclass, qtype, 0); ++ if (qtype == T_A) ++ { ++ char ipv4[4] = { 192, 0, 2, 1 }; ++ resolv_response_add_data (b, &ipv4, sizeof (ipv4)); ++ } ++ else ++ { ++ char ipv6[16] ++ = { 0x20, 0x01, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }; ++ resolv_response_add_data (b, &ipv6, sizeof (ipv6)); ++ } ++ resolv_response_close_record (b); ++ } ++ break; ++ } ++} ++ ++/* Used to reorder responses. */ ++struct resolv_response_context *previous_query; ++ ++/* Used to keep track of the queries received. */ ++static int previous_server_index = -1; ++static uint16_t previous_qtype; ++ ++/* For each server, buffer the first query and then send both answers ++ to the second query, reordered if requested. */ ++static void ++response (const struct resolv_response_context *ctx, ++ struct resolv_response_builder *b, ++ const char *qname, uint16_t qclass, uint16_t qtype) ++{ ++ TEST_VERIFY (qtype == T_A || qtype == T_AAAA); ++ if (ctx->server_index != 0) ++ TEST_COMPARE (ctx->server_index, 1); ++ ++ struct parsed_qname parsed; ++ parse_qname (&parsed, qname); ++ ++ if (previous_query == NULL) ++ { ++ /* No buffered query. Record this query and do not send a ++ response. */ ++ TEST_COMPARE (previous_qtype, 0); ++ previous_query = resolv_response_context_duplicate (ctx); ++ previous_qtype = qtype; ++ resolv_response_drop (b); ++ previous_server_index = ctx->server_index; ++ ++ if (test_verbose) ++ printf ("info: buffering first query for: %s\n", qname); ++ } ++ else ++ { ++ TEST_VERIFY (previous_query != 0); ++ TEST_COMPARE (ctx->server_index, previous_server_index); ++ TEST_VERIFY (previous_qtype != qtype); /* Not a duplicate. */ ++ ++ /* If reordering, send a response for this query explicitly, and ++ then skip the implicit send. */ ++ if (parsed.reorder[ctx->server_index]) ++ { ++ if (test_verbose) ++ printf ("info: sending reordered second response for: %s\n", ++ qname); ++ build_response (ctx, b, qname, qclass, qtype); ++ resolv_response_send_udp (ctx, b); ++ resolv_response_drop (b); ++ } ++ ++ /* Build a response for the previous query and send it, thus ++ reordering the two responses. */ ++ { ++ if (test_verbose) ++ printf ("info: sending first response for: %s\n", qname); ++ struct resolv_response_builder *btmp ++ = resolv_response_builder_allocate (previous_query->query_buffer, ++ previous_query->query_length); ++ build_response (ctx, btmp, qname, qclass, previous_qtype); ++ resolv_response_send_udp (ctx, btmp); ++ resolv_response_builder_free (btmp); ++ } ++ ++ /* If not reordering, send the reply as usual. */ ++ if (!parsed.reorder[ctx->server_index]) ++ { ++ if (test_verbose) ++ printf ("info: sending non-reordered second response for: %s\n", ++ qname); ++ build_response (ctx, b, qname, qclass, qtype); ++ } ++ ++ /* Unbuffer the response and prepare for the next query. */ ++ resolv_response_context_free (previous_query); ++ previous_query = NULL; ++ previous_qtype = 0; ++ previous_server_index = -1; ++ } ++} ++ ++/* Runs a query for QNAME and checks for the expected reply. See ++ struct parsed_qname for the expected format for QNAME. */ ++static void ++test_qname (const char *qname, int rcode) ++{ ++ struct resolv_context *ctx = __resolv_context_get (); ++ TEST_VERIFY_EXIT (ctx != NULL); ++ ++ unsigned char q1[512]; ++ int q1len = res_mkquery (QUERY, qname, C_IN, T_A, NULL, 0, NULL, ++ q1, sizeof (q1)); ++ TEST_VERIFY_EXIT (q1len > 12); ++ ++ unsigned char q2[512]; ++ int q2len = res_mkquery (QUERY, qname, C_IN, T_AAAA, NULL, 0, NULL, ++ q2, sizeof (q2)); ++ TEST_VERIFY_EXIT (q2len > 12); ++ ++ /* Produce a transaction ID collision. */ ++ memcpy (q2, q1, 2); ++ ++ unsigned char ans1[512]; ++ unsigned char *ans1p = ans1; ++ unsigned char *ans2p = NULL; ++ int nans2p = 0; ++ int resplen2 = 0; ++ int ans2p_malloced = 0; ++ ++ /* Perform a parallel A/AAAA query. */ ++ int resplen1 = __res_context_send (ctx, q1, q1len, q2, q2len, ++ ans1, sizeof (ans1), &ans1p, ++ &ans2p, &nans2p, ++ &resplen2, &ans2p_malloced); ++ ++ TEST_VERIFY (resplen1 > 12); ++ TEST_VERIFY (resplen2 > 12); ++ if (resplen1 <= 12 || resplen2 <= 12) ++ return; ++ ++ if (rcode == 1 || rcode == 3) ++ { ++ /* Format Error and Name Error responses does not trigger ++ switching to the next server. */ ++ TEST_COMPARE (ans1p[3] & 0x0f, rcode); ++ TEST_COMPARE (ans2p[3] & 0x0f, rcode); ++ return; ++ } ++ ++ /* The response should be successful. */ ++ TEST_COMPARE (ans1p[3] & 0x0f, 0); ++ TEST_COMPARE (ans2p[3] & 0x0f, 0); ++ ++ /* Due to bug 19691, the answer may not be in the slot matching the ++ query. Assume that the AAAA response is the longer one. */ ++ unsigned char *a_answer; ++ int a_answer_length; ++ unsigned char *aaaa_answer; ++ int aaaa_answer_length; ++ if (resplen2 > resplen1) ++ { ++ a_answer = ans1p; ++ a_answer_length = resplen1; ++ aaaa_answer = ans2p; ++ aaaa_answer_length = resplen2; ++ } ++ else ++ { ++ a_answer = ans2p; ++ a_answer_length = resplen2; ++ aaaa_answer = ans1p; ++ aaaa_answer_length = resplen1; ++ } ++ ++ { ++ char *expected = xasprintf ("name: %s\n" ++ "address: 192.0.2.1\n", ++ qname); ++ check_dns_packet (qname, a_answer, a_answer_length, expected); ++ free (expected); ++ } ++ { ++ char *expected = xasprintf ("name: %s\n" ++ "address: 2001:db8::1\n", ++ qname); ++ check_dns_packet (qname, aaaa_answer, aaaa_answer_length, expected); ++ free (expected); ++ } ++ ++ if (ans2p_malloced) ++ free (ans2p); ++ ++ __resolv_context_put (ctx); ++} ++ ++static int ++do_test (void) ++{ ++ struct resolv_test *aux = resolv_test_start ++ ((struct resolv_redirect_config) ++ { ++ .response_callback = response, ++ ++ /* The response callback use global state (the previous_* ++ variables), and query processing must therefore be ++ serialized. */ ++ .single_thread_udp = true, ++ }); ++ ++ for (int rcode = 0; rcode <= 5; ++rcode) ++ for (int do_reorder_0 = 0; do_reorder_0 < 2; ++do_reorder_0) ++ for (int do_reorder_1 = 0; do_reorder_1 < 2; ++do_reorder_1) ++ { ++ char *qname = xasprintf ("reorder-%d-%d.rcode-%d.example.net", ++ do_reorder_0, do_reorder_1, rcode); ++ test_qname (qname, rcode); ++ free (qname); ++ } ++ ++ resolv_test_end (aux); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/rt/Makefile b/rt/Makefile +index dab5d62a57..93502cfaa7 100644 +--- a/rt/Makefile ++++ b/rt/Makefile +@@ -44,6 +44,7 @@ tests := tst-shm tst-timer tst-timer2 \ + tst-aio7 tst-aio8 tst-aio9 tst-aio10 \ + tst-mqueue1 tst-mqueue2 tst-mqueue3 tst-mqueue4 \ + tst-mqueue5 tst-mqueue6 tst-mqueue7 tst-mqueue8 tst-mqueue9 \ ++ tst-bz28213 \ + tst-timer3 tst-timer4 tst-timer5 \ + tst-cpuclock2 tst-cputimer1 tst-cputimer2 tst-cputimer3 \ + tst-shm-cancel +diff --git a/rt/tst-bz28213.c b/rt/tst-bz28213.c +new file mode 100644 +index 0000000000..0c096b5a0a +--- /dev/null ++++ b/rt/tst-bz28213.c +@@ -0,0 +1,101 @@ ++/* Bug 28213: test for NULL pointer dereference in mq_notify. ++ Copyright (C) The GNU Toolchain Authors. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <errno.h> ++#include <sys/types.h> ++#include <sys/stat.h> ++#include <fcntl.h> ++#include <unistd.h> ++#include <mqueue.h> ++#include <signal.h> ++#include <stdlib.h> ++#include <string.h> ++#include <support/check.h> ++ ++static mqd_t m = -1; ++static const char msg[] = "hello"; ++ ++static void ++check_bz28213_cb (union sigval sv) ++{ ++ char buf[sizeof (msg)]; ++ ++ (void) sv; ++ ++ TEST_VERIFY_EXIT ((size_t) mq_receive (m, buf, sizeof (buf), NULL) ++ == sizeof (buf)); ++ TEST_VERIFY_EXIT (memcmp (buf, msg, sizeof (buf)) == 0); ++ ++ exit (0); ++} ++ ++static void ++check_bz28213 (void) ++{ ++ struct sigevent sev; ++ ++ memset (&sev, '\0', sizeof (sev)); ++ sev.sigev_notify = SIGEV_THREAD; ++ sev.sigev_notify_function = check_bz28213_cb; ++ ++ /* Step 1: Register & unregister notifier. ++ Helper thread should receive NOTIFY_REMOVED notification. ++ In a vulnerable version of glibc, NULL pointer dereference follows. */ ++ TEST_VERIFY_EXIT (mq_notify (m, &sev) == 0); ++ TEST_VERIFY_EXIT (mq_notify (m, NULL) == 0); ++ ++ /* Step 2: Once again, register notification. ++ Try to send one message. ++ Test is considered successful, if the callback does exit (0). */ ++ TEST_VERIFY_EXIT (mq_notify (m, &sev) == 0); ++ TEST_VERIFY_EXIT (mq_send (m, msg, sizeof (msg), 1) == 0); ++ ++ /* Wait... */ ++ pause (); ++} ++ ++static int ++do_test (void) ++{ ++ static const char m_name[] = "/bz28213_queue"; ++ struct mq_attr m_attr; ++ ++ memset (&m_attr, '\0', sizeof (m_attr)); ++ m_attr.mq_maxmsg = 1; ++ m_attr.mq_msgsize = sizeof (msg); ++ ++ m = mq_open (m_name, ++ O_RDWR | O_CREAT | O_EXCL, ++ 0600, ++ &m_attr); ++ ++ if (m < 0) ++ { ++ if (errno == ENOSYS) ++ FAIL_UNSUPPORTED ("POSIX message queues are not implemented\n"); ++ FAIL_EXIT1 ("Failed to create POSIX message queue: %m\n"); ++ } ++ ++ TEST_VERIFY_EXIT (mq_unlink (m_name) == 0); ++ ++ check_bz28213 (); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/stdio-common/Makefile b/stdio-common/Makefile +index 8475fd1f09..eff0c98d82 100644 +--- a/stdio-common/Makefile ++++ b/stdio-common/Makefile +@@ -69,7 +69,8 @@ tests := tstscanf test_rdwr test-popen tstgetln test-fseek \ + tst-printf-bz25691 \ + tst-vfprintf-width-prec-alloc \ + tst-printf-fp-free \ +- tst-printf-fp-leak ++ tst-printf-fp-leak \ ++ test-strerr + + + test-srcs = tst-unbputc tst-printf tst-printfsz-islongdouble +diff --git a/stdio-common/errlist.c b/stdio-common/errlist.c +index d15f13a22a..2ecf121674 100644 +--- a/stdio-common/errlist.c ++++ b/stdio-common/errlist.c +@@ -20,9 +20,13 @@ + #include <libintl.h> + #include <array_length.h> + ++#ifndef ERR_MAP ++# define ERR_MAP(n) n ++#endif ++ + const char *const _sys_errlist_internal[] = + { +-#define _S(n, str) [n] = str, ++#define _S(n, str) [ERR_MAP(n)] = str, + #include <errlist.h> + #undef _S + }; +@@ -41,20 +45,21 @@ static const union sys_errname_t + { + #define MSGSTRFIELD1(line) str##line + #define MSGSTRFIELD(line) MSGSTRFIELD1(line) +-#define _S(n, str) char MSGSTRFIELD(__LINE__)[sizeof(str)]; ++#define _S(n, str) char MSGSTRFIELD(__LINE__)[sizeof(#n)]; + #include <errlist.h> + #undef _S + }; + char str[0]; + } _sys_errname = { { +-#define _S(n, s) s, ++#define _S(n, s) #n, + #include <errlist.h> + #undef _S + } }; + + static const unsigned short _sys_errnameidx[] = + { +-#define _S(n, s) [n] = offsetof(union sys_errname_t, MSGSTRFIELD(__LINE__)), ++#define _S(n, s) \ ++ [ERR_MAP(n)] = offsetof(union sys_errname_t, MSGSTRFIELD(__LINE__)), + #include <errlist.h> + #undef _S + }; +diff --git a/stdio-common/test-strerr.c b/stdio-common/test-strerr.c +index fded208118..d77b81d507 100644 +--- a/stdio-common/test-strerr.c ++++ b/stdio-common/test-strerr.c +@@ -18,46 +18,672 @@ + + #include <string.h> + #include <errno.h> +-#include <array_length.h> + + #include <support/support.h> + #include <support/check.h> + +-#define N_(name) name +- +-static const char *const errlist[] = +- { +-/* This file is auto-generated from errlist.def. */ +-#include <errlist.h> +- }; +- +-#define MSGSTR_T errname_t +-#define MSGSTR errname +-#define MSGIDX errnameidx +-#include <errlist-name.h> +-#undef MSGSTR +-#undef MSGIDX +- + static int + do_test (void) + { +- TEST_VERIFY (strerrordesc_np (-1) == NULL); +- TEST_VERIFY (strerrordesc_np (array_length (errlist)) == NULL); +- for (size_t i = 0; i < array_length (errlist); i++) +- { +- if (errlist[i] == NULL) +- continue; +- TEST_COMPARE_STRING (strerrordesc_np (i), errlist[i]); +- } ++ TEST_COMPARE_STRING (strerrordesc_np (0), "Success"); ++ TEST_COMPARE_STRING (strerrorname_np (0), "0"); + +- TEST_VERIFY (strerrorname_np (-1) == NULL); +- TEST_VERIFY (strerrorname_np (array_length (errlist)) == NULL); +- for (size_t i = 0; i < array_length (errlist); i++) +- { +- if (errlist[i] == NULL) +- continue; +- TEST_COMPARE_STRING (strerrorname_np (i), errname.str + errnameidx[i]); +- } ++#ifdef EPERM ++ TEST_COMPARE_STRING (strerrordesc_np (EPERM), "Operation not permitted"); ++ TEST_COMPARE_STRING (strerrorname_np (EPERM), "EPERM"); ++#endif ++#ifdef ENOENT ++ TEST_COMPARE_STRING (strerrordesc_np (ENOENT), ++ "No such file or directory"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOENT), "ENOENT"); ++#endif ++#ifdef ESRCH ++ TEST_COMPARE_STRING (strerrordesc_np (ESRCH), "No such process"); ++ TEST_COMPARE_STRING (strerrorname_np (ESRCH), "ESRCH"); ++#endif ++#ifdef EINTR ++ TEST_COMPARE_STRING (strerrordesc_np (EINTR), "Interrupted system call"); ++ TEST_COMPARE_STRING (strerrorname_np (EINTR), "EINTR"); ++#endif ++#ifdef EIO ++ TEST_COMPARE_STRING (strerrordesc_np (EIO), "Input/output error"); ++ TEST_COMPARE_STRING (strerrorname_np (EIO), "EIO"); ++#endif ++#ifdef ENXIO ++ TEST_COMPARE_STRING (strerrordesc_np (ENXIO), "No such device or address"); ++ TEST_COMPARE_STRING (strerrorname_np (ENXIO), "ENXIO"); ++#endif ++#ifdef E2BIG ++ TEST_COMPARE_STRING (strerrordesc_np (E2BIG), "Argument list too long"); ++ TEST_COMPARE_STRING (strerrorname_np (E2BIG), "E2BIG"); ++#endif ++#ifdef ENOEXEC ++ TEST_COMPARE_STRING (strerrordesc_np (ENOEXEC), "Exec format error"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOEXEC), "ENOEXEC"); ++#endif ++#ifdef EBADF ++ TEST_COMPARE_STRING (strerrordesc_np (EBADF), "Bad file descriptor"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADF), "EBADF"); ++#endif ++#ifdef ECHILD ++ TEST_COMPARE_STRING (strerrordesc_np (ECHILD), "No child processes"); ++ TEST_COMPARE_STRING (strerrorname_np (ECHILD), "ECHILD"); ++#endif ++#ifdef EDEADLK ++ TEST_COMPARE_STRING (strerrordesc_np (EDEADLK), ++ "Resource deadlock avoided"); ++ TEST_COMPARE_STRING (strerrorname_np (EDEADLK), "EDEADLK"); ++#endif ++#ifdef ENOMEM ++ TEST_COMPARE_STRING (strerrordesc_np (ENOMEM), "Cannot allocate memory"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOMEM), "ENOMEM"); ++#endif ++#ifdef EACCES ++ TEST_COMPARE_STRING (strerrordesc_np (EACCES), "Permission denied"); ++ TEST_COMPARE_STRING (strerrorname_np (EACCES), "EACCES"); ++#endif ++#ifdef EFAULT ++ TEST_COMPARE_STRING (strerrordesc_np (EFAULT), "Bad address"); ++ TEST_COMPARE_STRING (strerrorname_np (EFAULT), "EFAULT"); ++#endif ++#ifdef ENOTBLK ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTBLK), "Block device required"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTBLK), "ENOTBLK"); ++#endif ++#ifdef EBUSY ++ TEST_COMPARE_STRING (strerrordesc_np (EBUSY), "Device or resource busy"); ++ TEST_COMPARE_STRING (strerrorname_np (EBUSY), "EBUSY"); ++#endif ++#ifdef EEXIST ++ TEST_COMPARE_STRING (strerrordesc_np (EEXIST), "File exists"); ++ TEST_COMPARE_STRING (strerrorname_np (EEXIST), "EEXIST"); ++#endif ++#ifdef EXDEV ++ TEST_COMPARE_STRING (strerrordesc_np (EXDEV), "Invalid cross-device link"); ++ TEST_COMPARE_STRING (strerrorname_np (EXDEV), "EXDEV"); ++#endif ++#ifdef ENODEV ++ TEST_COMPARE_STRING (strerrordesc_np (ENODEV), "No such device"); ++ TEST_COMPARE_STRING (strerrorname_np (ENODEV), "ENODEV"); ++#endif ++#ifdef ENOTDIR ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTDIR), "Not a directory"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTDIR), "ENOTDIR"); ++#endif ++#ifdef EISDIR ++ TEST_COMPARE_STRING (strerrordesc_np (EISDIR), "Is a directory"); ++ TEST_COMPARE_STRING (strerrorname_np (EISDIR), "EISDIR"); ++#endif ++#ifdef EINVAL ++ TEST_COMPARE_STRING (strerrordesc_np (EINVAL), "Invalid argument"); ++ TEST_COMPARE_STRING (strerrorname_np (EINVAL), "EINVAL"); ++#endif ++#ifdef EMFILE ++ TEST_COMPARE_STRING (strerrordesc_np (EMFILE), "Too many open files"); ++ TEST_COMPARE_STRING (strerrorname_np (EMFILE), "EMFILE"); ++#endif ++#ifdef ENFILE ++ TEST_COMPARE_STRING (strerrordesc_np (ENFILE), ++ "Too many open files in system"); ++ TEST_COMPARE_STRING (strerrorname_np (ENFILE), "ENFILE"); ++#endif ++#ifdef ENOTTY ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTTY), ++ "Inappropriate ioctl for device"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTTY), "ENOTTY"); ++#endif ++#ifdef ETXTBSY ++ TEST_COMPARE_STRING (strerrordesc_np (ETXTBSY), "Text file busy"); ++ TEST_COMPARE_STRING (strerrorname_np (ETXTBSY), "ETXTBSY"); ++#endif ++#ifdef EFBIG ++ TEST_COMPARE_STRING (strerrordesc_np (EFBIG), "File too large"); ++ TEST_COMPARE_STRING (strerrorname_np (EFBIG), "EFBIG"); ++#endif ++#ifdef ENOSPC ++ TEST_COMPARE_STRING (strerrordesc_np (ENOSPC), "No space left on device"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOSPC), "ENOSPC"); ++#endif ++#ifdef ESPIPE ++ TEST_COMPARE_STRING (strerrordesc_np (ESPIPE), "Illegal seek"); ++ TEST_COMPARE_STRING (strerrorname_np (ESPIPE), "ESPIPE"); ++#endif ++#ifdef EROFS ++ TEST_COMPARE_STRING (strerrordesc_np (EROFS), "Read-only file system"); ++ TEST_COMPARE_STRING (strerrorname_np (EROFS), "EROFS"); ++#endif ++#ifdef EMLINK ++ TEST_COMPARE_STRING (strerrordesc_np (EMLINK), "Too many links"); ++ TEST_COMPARE_STRING (strerrorname_np (EMLINK), "EMLINK"); ++#endif ++#ifdef EPIPE ++ TEST_COMPARE_STRING (strerrordesc_np (EPIPE), "Broken pipe"); ++ TEST_COMPARE_STRING (strerrorname_np (EPIPE), "EPIPE"); ++#endif ++#ifdef EDOM ++ TEST_COMPARE_STRING (strerrordesc_np (EDOM), ++ "Numerical argument out of domain"); ++ TEST_COMPARE_STRING (strerrorname_np (EDOM), "EDOM"); ++#endif ++#ifdef ERANGE ++ TEST_COMPARE_STRING (strerrordesc_np (ERANGE), ++ "Numerical result out of range"); ++ TEST_COMPARE_STRING (strerrorname_np (ERANGE), "ERANGE"); ++#endif ++#ifdef EAGAIN ++ TEST_COMPARE_STRING (strerrordesc_np (EAGAIN), ++ "Resource temporarily unavailable"); ++ TEST_COMPARE_STRING (strerrorname_np (EAGAIN), "EAGAIN"); ++#endif ++#ifdef EINPROGRESS ++ TEST_COMPARE_STRING (strerrordesc_np (EINPROGRESS), ++ "Operation now in progress"); ++ TEST_COMPARE_STRING (strerrorname_np (EINPROGRESS), "EINPROGRESS"); ++#endif ++#ifdef EALREADY ++ TEST_COMPARE_STRING (strerrordesc_np (EALREADY), ++ "Operation already in progress"); ++ TEST_COMPARE_STRING (strerrorname_np (EALREADY), "EALREADY"); ++#endif ++#ifdef ENOTSOCK ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTSOCK), ++ "Socket operation on non-socket"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTSOCK), "ENOTSOCK"); ++#endif ++#ifdef EMSGSIZE ++ TEST_COMPARE_STRING (strerrordesc_np (EMSGSIZE), "Message too long"); ++ TEST_COMPARE_STRING (strerrorname_np (EMSGSIZE), "EMSGSIZE"); ++#endif ++#ifdef EPROTOTYPE ++ TEST_COMPARE_STRING (strerrordesc_np (EPROTOTYPE), ++ "Protocol wrong type for socket"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROTOTYPE), "EPROTOTYPE"); ++#endif ++#ifdef ENOPROTOOPT ++ TEST_COMPARE_STRING (strerrordesc_np (ENOPROTOOPT), ++ "Protocol not available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOPROTOOPT), "ENOPROTOOPT"); ++#endif ++#ifdef EPROTONOSUPPORT ++ TEST_COMPARE_STRING (strerrordesc_np (EPROTONOSUPPORT), ++ "Protocol not supported"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROTONOSUPPORT), "EPROTONOSUPPORT"); ++#endif ++#ifdef ESOCKTNOSUPPORT ++ TEST_COMPARE_STRING (strerrordesc_np (ESOCKTNOSUPPORT), ++ "Socket type not supported"); ++ TEST_COMPARE_STRING (strerrorname_np (ESOCKTNOSUPPORT), "ESOCKTNOSUPPORT"); ++#endif ++#ifdef EOPNOTSUPP ++ TEST_COMPARE_STRING (strerrordesc_np (EOPNOTSUPP), ++ "Operation not supported"); ++ TEST_COMPARE_STRING (strerrorname_np (EOPNOTSUPP), "EOPNOTSUPP"); ++#endif ++#ifdef EPFNOSUPPORT ++ TEST_COMPARE_STRING (strerrordesc_np (EPFNOSUPPORT), ++ "Protocol family not supported"); ++ TEST_COMPARE_STRING (strerrorname_np (EPFNOSUPPORT), "EPFNOSUPPORT"); ++#endif ++#ifdef EAFNOSUPPORT ++ TEST_COMPARE_STRING (strerrordesc_np (EAFNOSUPPORT), ++ "Address family not supported by protocol"); ++ TEST_COMPARE_STRING (strerrorname_np (EAFNOSUPPORT), "EAFNOSUPPORT"); ++#endif ++#ifdef EADDRINUSE ++ TEST_COMPARE_STRING (strerrordesc_np (EADDRINUSE), ++ "Address already in use"); ++ TEST_COMPARE_STRING (strerrorname_np (EADDRINUSE), "EADDRINUSE"); ++#endif ++#ifdef EADDRNOTAVAIL ++ TEST_COMPARE_STRING (strerrordesc_np (EADDRNOTAVAIL), ++ "Cannot assign requested address"); ++ TEST_COMPARE_STRING (strerrorname_np (EADDRNOTAVAIL), "EADDRNOTAVAIL"); ++#endif ++#ifdef ENETDOWN ++ TEST_COMPARE_STRING (strerrordesc_np (ENETDOWN), "Network is down"); ++ TEST_COMPARE_STRING (strerrorname_np (ENETDOWN), "ENETDOWN"); ++#endif ++#ifdef ENETUNREACH ++ TEST_COMPARE_STRING (strerrordesc_np (ENETUNREACH), ++ "Network is unreachable"); ++ TEST_COMPARE_STRING (strerrorname_np (ENETUNREACH), "ENETUNREACH"); ++#endif ++#ifdef ENETRESET ++ TEST_COMPARE_STRING (strerrordesc_np (ENETRESET), ++ "Network dropped connection on reset"); ++ TEST_COMPARE_STRING (strerrorname_np (ENETRESET), "ENETRESET"); ++#endif ++#ifdef ECONNABORTED ++ TEST_COMPARE_STRING (strerrordesc_np (ECONNABORTED), ++ "Software caused connection abort"); ++ TEST_COMPARE_STRING (strerrorname_np (ECONNABORTED), "ECONNABORTED"); ++#endif ++#ifdef ECONNRESET ++ TEST_COMPARE_STRING (strerrordesc_np (ECONNRESET), ++ "Connection reset by peer"); ++ TEST_COMPARE_STRING (strerrorname_np (ECONNRESET), "ECONNRESET"); ++#endif ++#ifdef ENOBUFS ++ TEST_COMPARE_STRING (strerrordesc_np (ENOBUFS), ++ "No buffer space available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOBUFS), "ENOBUFS"); ++#endif ++#ifdef EISCONN ++ TEST_COMPARE_STRING (strerrordesc_np (EISCONN), ++ "Transport endpoint is already connected"); ++ TEST_COMPARE_STRING (strerrorname_np (EISCONN), "EISCONN"); ++#endif ++#ifdef ENOTCONN ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTCONN), ++ "Transport endpoint is not connected"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTCONN), "ENOTCONN"); ++#endif ++#ifdef EDESTADDRREQ ++ TEST_COMPARE_STRING (strerrordesc_np (EDESTADDRREQ), ++ "Destination address required"); ++ TEST_COMPARE_STRING (strerrorname_np (EDESTADDRREQ), "EDESTADDRREQ"); ++#endif ++#ifdef ESHUTDOWN ++ TEST_COMPARE_STRING (strerrordesc_np (ESHUTDOWN), ++ "Cannot send after transport endpoint shutdown"); ++ TEST_COMPARE_STRING (strerrorname_np (ESHUTDOWN), "ESHUTDOWN"); ++#endif ++#ifdef ETOOMANYREFS ++ TEST_COMPARE_STRING (strerrordesc_np (ETOOMANYREFS), ++ "Too many references: cannot splice"); ++ TEST_COMPARE_STRING (strerrorname_np (ETOOMANYREFS), "ETOOMANYREFS"); ++#endif ++#ifdef ETIMEDOUT ++ TEST_COMPARE_STRING (strerrordesc_np (ETIMEDOUT), "Connection timed out"); ++ TEST_COMPARE_STRING (strerrorname_np (ETIMEDOUT), "ETIMEDOUT"); ++#endif ++#ifdef ECONNREFUSED ++ TEST_COMPARE_STRING (strerrordesc_np (ECONNREFUSED), "Connection refused"); ++ TEST_COMPARE_STRING (strerrorname_np (ECONNREFUSED), "ECONNREFUSED"); ++#endif ++#ifdef ELOOP ++ TEST_COMPARE_STRING (strerrordesc_np (ELOOP), ++ "Too many levels of symbolic links"); ++ TEST_COMPARE_STRING (strerrorname_np (ELOOP), "ELOOP"); ++#endif ++#ifdef ENAMETOOLONG ++ TEST_COMPARE_STRING (strerrordesc_np (ENAMETOOLONG), "File name too long"); ++ TEST_COMPARE_STRING (strerrorname_np (ENAMETOOLONG), "ENAMETOOLONG"); ++#endif ++#ifdef EHOSTDOWN ++ TEST_COMPARE_STRING (strerrordesc_np (EHOSTDOWN), "Host is down"); ++ TEST_COMPARE_STRING (strerrorname_np (EHOSTDOWN), "EHOSTDOWN"); ++#endif ++#ifdef EHOSTUNREACH ++ TEST_COMPARE_STRING (strerrordesc_np (EHOSTUNREACH), "No route to host"); ++ TEST_COMPARE_STRING (strerrorname_np (EHOSTUNREACH), "EHOSTUNREACH"); ++#endif ++#ifdef ENOTEMPTY ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTEMPTY), "Directory not empty"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTEMPTY), "ENOTEMPTY"); ++#endif ++#ifdef EUSERS ++ TEST_COMPARE_STRING (strerrordesc_np (EUSERS), "Too many users"); ++ TEST_COMPARE_STRING (strerrorname_np (EUSERS), "EUSERS"); ++#endif ++#ifdef EDQUOT ++ TEST_COMPARE_STRING (strerrordesc_np (EDQUOT), "Disk quota exceeded"); ++ TEST_COMPARE_STRING (strerrorname_np (EDQUOT), "EDQUOT"); ++#endif ++#ifdef ESTALE ++ TEST_COMPARE_STRING (strerrordesc_np (ESTALE), "Stale file handle"); ++ TEST_COMPARE_STRING (strerrorname_np (ESTALE), "ESTALE"); ++#endif ++#ifdef EREMOTE ++ TEST_COMPARE_STRING (strerrordesc_np (EREMOTE), "Object is remote"); ++ TEST_COMPARE_STRING (strerrorname_np (EREMOTE), "EREMOTE"); ++#endif ++#ifdef ENOLCK ++ TEST_COMPARE_STRING (strerrordesc_np (ENOLCK), "No locks available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOLCK), "ENOLCK"); ++#endif ++#ifdef ENOSYS ++ TEST_COMPARE_STRING (strerrordesc_np (ENOSYS), "Function not implemented"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOSYS), "ENOSYS"); ++#endif ++#ifdef EILSEQ ++ TEST_COMPARE_STRING (strerrordesc_np (EILSEQ), ++ "Invalid or incomplete multibyte or wide character"); ++ TEST_COMPARE_STRING (strerrorname_np (EILSEQ), "EILSEQ"); ++#endif ++#ifdef EBADMSG ++ TEST_COMPARE_STRING (strerrordesc_np (EBADMSG), "Bad message"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADMSG), "EBADMSG"); ++#endif ++#ifdef EIDRM ++ TEST_COMPARE_STRING (strerrordesc_np (EIDRM), "Identifier removed"); ++ TEST_COMPARE_STRING (strerrorname_np (EIDRM), "EIDRM"); ++#endif ++#ifdef EMULTIHOP ++ TEST_COMPARE_STRING (strerrordesc_np (EMULTIHOP), "Multihop attempted"); ++ TEST_COMPARE_STRING (strerrorname_np (EMULTIHOP), "EMULTIHOP"); ++#endif ++#ifdef ENODATA ++ TEST_COMPARE_STRING (strerrordesc_np (ENODATA), "No data available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENODATA), "ENODATA"); ++#endif ++#ifdef ENOLINK ++ TEST_COMPARE_STRING (strerrordesc_np (ENOLINK), "Link has been severed"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOLINK), "ENOLINK"); ++#endif ++#ifdef ENOMSG ++ TEST_COMPARE_STRING (strerrordesc_np (ENOMSG), ++ "No message of desired type"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOMSG), "ENOMSG"); ++#endif ++#ifdef ENOSR ++ TEST_COMPARE_STRING (strerrordesc_np (ENOSR), "Out of streams resources"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOSR), "ENOSR"); ++#endif ++#ifdef ENOSTR ++ TEST_COMPARE_STRING (strerrordesc_np (ENOSTR), "Device not a stream"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOSTR), "ENOSTR"); ++#endif ++#ifdef EOVERFLOW ++ TEST_COMPARE_STRING (strerrordesc_np (EOVERFLOW), ++ "Value too large for defined data type"); ++ TEST_COMPARE_STRING (strerrorname_np (EOVERFLOW), "EOVERFLOW"); ++#endif ++#ifdef EPROTO ++ TEST_COMPARE_STRING (strerrordesc_np (EPROTO), "Protocol error"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROTO), "EPROTO"); ++#endif ++#ifdef ETIME ++ TEST_COMPARE_STRING (strerrordesc_np (ETIME), "Timer expired"); ++ TEST_COMPARE_STRING (strerrorname_np (ETIME), "ETIME"); ++#endif ++#ifdef ECANCELED ++ TEST_COMPARE_STRING (strerrordesc_np (ECANCELED), "Operation canceled"); ++ TEST_COMPARE_STRING (strerrorname_np (ECANCELED), "ECANCELED"); ++#endif ++#ifdef EOWNERDEAD ++ TEST_COMPARE_STRING (strerrordesc_np (EOWNERDEAD), "Owner died"); ++ TEST_COMPARE_STRING (strerrorname_np (EOWNERDEAD), "EOWNERDEAD"); ++#endif ++#ifdef ENOTRECOVERABLE ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTRECOVERABLE), ++ "State not recoverable"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTRECOVERABLE), "ENOTRECOVERABLE"); ++#endif ++#ifdef ERESTART ++ TEST_COMPARE_STRING (strerrordesc_np (ERESTART), ++ "Interrupted system call should be restarted"); ++ TEST_COMPARE_STRING (strerrorname_np (ERESTART), "ERESTART"); ++#endif ++#ifdef ECHRNG ++ TEST_COMPARE_STRING (strerrordesc_np (ECHRNG), ++ "Channel number out of range"); ++ TEST_COMPARE_STRING (strerrorname_np (ECHRNG), "ECHRNG"); ++#endif ++#ifdef EL2NSYNC ++ TEST_COMPARE_STRING (strerrordesc_np (EL2NSYNC), ++ "Level 2 not synchronized"); ++ TEST_COMPARE_STRING (strerrorname_np (EL2NSYNC), "EL2NSYNC"); ++#endif ++#ifdef EL3HLT ++ TEST_COMPARE_STRING (strerrordesc_np (EL3HLT), "Level 3 halted"); ++ TEST_COMPARE_STRING (strerrorname_np (EL3HLT), "EL3HLT"); ++#endif ++#ifdef EL3RST ++ TEST_COMPARE_STRING (strerrordesc_np (EL3RST), "Level 3 reset"); ++ TEST_COMPARE_STRING (strerrorname_np (EL3RST), "EL3RST"); ++#endif ++#ifdef ELNRNG ++ TEST_COMPARE_STRING (strerrordesc_np (ELNRNG), "Link number out of range"); ++ TEST_COMPARE_STRING (strerrorname_np (ELNRNG), "ELNRNG"); ++#endif ++#ifdef EUNATCH ++ TEST_COMPARE_STRING (strerrordesc_np (EUNATCH), ++ "Protocol driver not attached"); ++ TEST_COMPARE_STRING (strerrorname_np (EUNATCH), "EUNATCH"); ++#endif ++#ifdef ENOCSI ++ TEST_COMPARE_STRING (strerrordesc_np (ENOCSI), ++ "No CSI structure available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOCSI), "ENOCSI"); ++#endif ++#ifdef EL2HLT ++ TEST_COMPARE_STRING (strerrordesc_np (EL2HLT), "Level 2 halted"); ++ TEST_COMPARE_STRING (strerrorname_np (EL2HLT), "EL2HLT"); ++#endif ++#ifdef EBADE ++ TEST_COMPARE_STRING (strerrordesc_np (EBADE), "Invalid exchange"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADE), "EBADE"); ++#endif ++#ifdef EBADR ++ TEST_COMPARE_STRING (strerrordesc_np (EBADR), ++ "Invalid request descriptor"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADR), "EBADR"); ++#endif ++#ifdef EXFULL ++ TEST_COMPARE_STRING (strerrordesc_np (EXFULL), "Exchange full"); ++ TEST_COMPARE_STRING (strerrorname_np (EXFULL), "EXFULL"); ++#endif ++#ifdef ENOANO ++ TEST_COMPARE_STRING (strerrordesc_np (ENOANO), "No anode"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOANO), "ENOANO"); ++#endif ++#ifdef EBADRQC ++ TEST_COMPARE_STRING (strerrordesc_np (EBADRQC), "Invalid request code"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADRQC), "EBADRQC"); ++#endif ++#ifdef EBADSLT ++ TEST_COMPARE_STRING (strerrordesc_np (EBADSLT), "Invalid slot"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADSLT), "EBADSLT"); ++#endif ++#ifdef EBFONT ++ TEST_COMPARE_STRING (strerrordesc_np (EBFONT), "Bad font file format"); ++ TEST_COMPARE_STRING (strerrorname_np (EBFONT), "EBFONT"); ++#endif ++#ifdef ENONET ++ TEST_COMPARE_STRING (strerrordesc_np (ENONET), ++ "Machine is not on the network"); ++ TEST_COMPARE_STRING (strerrorname_np (ENONET), "ENONET"); ++#endif ++#ifdef ENOPKG ++ TEST_COMPARE_STRING (strerrordesc_np (ENOPKG), "Package not installed"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOPKG), "ENOPKG"); ++#endif ++#ifdef EADV ++ TEST_COMPARE_STRING (strerrordesc_np (EADV), "Advertise error"); ++ TEST_COMPARE_STRING (strerrorname_np (EADV), "EADV"); ++#endif ++#ifdef ESRMNT ++ TEST_COMPARE_STRING (strerrordesc_np (ESRMNT), "Srmount error"); ++ TEST_COMPARE_STRING (strerrorname_np (ESRMNT), "ESRMNT"); ++#endif ++#ifdef ECOMM ++ TEST_COMPARE_STRING (strerrordesc_np (ECOMM), ++ "Communication error on send"); ++ TEST_COMPARE_STRING (strerrorname_np (ECOMM), "ECOMM"); ++#endif ++#ifdef EDOTDOT ++ TEST_COMPARE_STRING (strerrordesc_np (EDOTDOT), "RFS specific error"); ++ TEST_COMPARE_STRING (strerrorname_np (EDOTDOT), "EDOTDOT"); ++#endif ++#ifdef ENOTUNIQ ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTUNIQ), ++ "Name not unique on network"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTUNIQ), "ENOTUNIQ"); ++#endif ++#ifdef EBADFD ++ TEST_COMPARE_STRING (strerrordesc_np (EBADFD), ++ "File descriptor in bad state"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADFD), "EBADFD"); ++#endif ++#ifdef EREMCHG ++ TEST_COMPARE_STRING (strerrordesc_np (EREMCHG), "Remote address changed"); ++ TEST_COMPARE_STRING (strerrorname_np (EREMCHG), "EREMCHG"); ++#endif ++#ifdef ELIBACC ++ TEST_COMPARE_STRING (strerrordesc_np (ELIBACC), ++ "Can not access a needed shared library"); ++ TEST_COMPARE_STRING (strerrorname_np (ELIBACC), "ELIBACC"); ++#endif ++#ifdef ELIBBAD ++ TEST_COMPARE_STRING (strerrordesc_np (ELIBBAD), ++ "Accessing a corrupted shared library"); ++ TEST_COMPARE_STRING (strerrorname_np (ELIBBAD), "ELIBBAD"); ++#endif ++#ifdef ELIBSCN ++ TEST_COMPARE_STRING (strerrordesc_np (ELIBSCN), ++ ".lib section in a.out corrupted"); ++ TEST_COMPARE_STRING (strerrorname_np (ELIBSCN), "ELIBSCN"); ++#endif ++#ifdef ELIBMAX ++ TEST_COMPARE_STRING (strerrordesc_np (ELIBMAX), ++ "Attempting to link in too many shared libraries"); ++ TEST_COMPARE_STRING (strerrorname_np (ELIBMAX), "ELIBMAX"); ++#endif ++#ifdef ELIBEXEC ++ TEST_COMPARE_STRING (strerrordesc_np (ELIBEXEC), ++ "Cannot exec a shared library directly"); ++ TEST_COMPARE_STRING (strerrorname_np (ELIBEXEC), "ELIBEXEC"); ++#endif ++#ifdef ESTRPIPE ++ TEST_COMPARE_STRING (strerrordesc_np (ESTRPIPE), "Streams pipe error"); ++ TEST_COMPARE_STRING (strerrorname_np (ESTRPIPE), "ESTRPIPE"); ++#endif ++#ifdef EUCLEAN ++ TEST_COMPARE_STRING (strerrordesc_np (EUCLEAN), ++ "Structure needs cleaning"); ++ TEST_COMPARE_STRING (strerrorname_np (EUCLEAN), "EUCLEAN"); ++#endif ++#ifdef ENOTNAM ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTNAM), ++ "Not a XENIX named type file"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTNAM), "ENOTNAM"); ++#endif ++#ifdef ENAVAIL ++ TEST_COMPARE_STRING (strerrordesc_np (ENAVAIL), ++ "No XENIX semaphores available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENAVAIL), "ENAVAIL"); ++#endif ++#ifdef EISNAM ++ TEST_COMPARE_STRING (strerrordesc_np (EISNAM), "Is a named type file"); ++ TEST_COMPARE_STRING (strerrorname_np (EISNAM), "EISNAM"); ++#endif ++#ifdef EREMOTEIO ++ TEST_COMPARE_STRING (strerrordesc_np (EREMOTEIO), "Remote I/O error"); ++ TEST_COMPARE_STRING (strerrorname_np (EREMOTEIO), "EREMOTEIO"); ++#endif ++#ifdef ENOMEDIUM ++ TEST_COMPARE_STRING (strerrordesc_np (ENOMEDIUM), "No medium found"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOMEDIUM), "ENOMEDIUM"); ++#endif ++#ifdef EMEDIUMTYPE ++ TEST_COMPARE_STRING (strerrordesc_np (EMEDIUMTYPE), "Wrong medium type"); ++ TEST_COMPARE_STRING (strerrorname_np (EMEDIUMTYPE), "EMEDIUMTYPE"); ++#endif ++#ifdef ENOKEY ++ TEST_COMPARE_STRING (strerrordesc_np (ENOKEY), ++ "Required key not available"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOKEY), "ENOKEY"); ++#endif ++#ifdef EKEYEXPIRED ++ TEST_COMPARE_STRING (strerrordesc_np (EKEYEXPIRED), "Key has expired"); ++ TEST_COMPARE_STRING (strerrorname_np (EKEYEXPIRED), "EKEYEXPIRED"); ++#endif ++#ifdef EKEYREVOKED ++ TEST_COMPARE_STRING (strerrordesc_np (EKEYREVOKED), ++ "Key has been revoked"); ++ TEST_COMPARE_STRING (strerrorname_np (EKEYREVOKED), "EKEYREVOKED"); ++#endif ++#ifdef EKEYREJECTED ++ TEST_COMPARE_STRING (strerrordesc_np (EKEYREJECTED), ++ "Key was rejected by service"); ++ TEST_COMPARE_STRING (strerrorname_np (EKEYREJECTED), "EKEYREJECTED"); ++#endif ++#ifdef ERFKILL ++ TEST_COMPARE_STRING (strerrordesc_np (ERFKILL), ++ "Operation not possible due to RF-kill"); ++ TEST_COMPARE_STRING (strerrorname_np (ERFKILL), "ERFKILL"); ++#endif ++#ifdef EHWPOISON ++ TEST_COMPARE_STRING (strerrordesc_np (EHWPOISON), ++ "Memory page has hardware error"); ++ TEST_COMPARE_STRING (strerrorname_np (EHWPOISON), "EHWPOISON"); ++#endif ++#ifdef EBADRPC ++ TEST_COMPARE_STRING (strerrordesc_np (EBADRPC), "RPC struct is bad"); ++ TEST_COMPARE_STRING (strerrorname_np (EBADRPC), "EBADRPC"); ++#endif ++#ifdef EFTYPE ++ TEST_COMPARE_STRING (strerrordesc_np (EFTYPE), ++ "Inappropriate file type or format"); ++ TEST_COMPARE_STRING (strerrorname_np (EFTYPE), "EFTYPE"); ++#endif ++#ifdef EPROCUNAVAIL ++ TEST_COMPARE_STRING (strerrordesc_np (EPROCUNAVAIL), ++ "RPC bad procedure for program"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROCUNAVAIL), "EPROCUNAVAIL"); ++#endif ++#ifdef EAUTH ++ TEST_COMPARE_STRING (strerrordesc_np (EAUTH), "Authentication error"); ++ TEST_COMPARE_STRING (strerrorname_np (EAUTH), "EAUTH"); ++#endif ++#ifdef EDIED ++ TEST_COMPARE_STRING (strerrordesc_np (EDIED), "Translator died"); ++ TEST_COMPARE_STRING (strerrorname_np (EDIED), "EDIED"); ++#endif ++#ifdef ERPCMISMATCH ++ TEST_COMPARE_STRING (strerrordesc_np (ERPCMISMATCH), "RPC version wrong"); ++ TEST_COMPARE_STRING (strerrorname_np (ERPCMISMATCH), "ERPCMISMATCH"); ++#endif ++#ifdef EGREGIOUS ++ TEST_COMPARE_STRING (strerrordesc_np (EGREGIOUS), ++ "You really blew it this time"); ++ TEST_COMPARE_STRING (strerrorname_np (EGREGIOUS), "EGREGIOUS"); ++#endif ++#ifdef EPROCLIM ++ TEST_COMPARE_STRING (strerrordesc_np (EPROCLIM), "Too many processes"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROCLIM), "EPROCLIM"); ++#endif ++#ifdef EGRATUITOUS ++ TEST_COMPARE_STRING (strerrordesc_np (EGRATUITOUS), "Gratuitous error"); ++ TEST_COMPARE_STRING (strerrorname_np (EGRATUITOUS), "EGRATUITOUS"); ++#endif ++#if defined (ENOTSUP) && ENOTSUP != EOPNOTSUPP ++ TEST_COMPARE_STRING (strerrordesc_np (ENOTSUP), "Not supported"); ++ TEST_COMPARE_STRING (strerrorname_np (ENOTSUP), "ENOTSUP"); ++#endif ++#ifdef EPROGMISMATCH ++ TEST_COMPARE_STRING (strerrordesc_np (EPROGMISMATCH), ++ "RPC program version wrong"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROGMISMATCH), "EPROGMISMATCH"); ++#endif ++#ifdef EBACKGROUND ++ TEST_COMPARE_STRING (strerrordesc_np (EBACKGROUND), ++ "Inappropriate operation for background process"); ++ TEST_COMPARE_STRING (strerrorname_np (EBACKGROUND), "EBACKGROUND"); ++#endif ++#ifdef EIEIO ++ TEST_COMPARE_STRING (strerrordesc_np (EIEIO), "Computer bought the farm"); ++ TEST_COMPARE_STRING (strerrorname_np (EIEIO), "EIEIO"); ++#endif ++#if defined (EWOULDBLOCK) && EWOULDBLOCK != EAGAIN ++ TEST_COMPARE_STRING (strerrordesc_np (EWOULDBLOCK), ++ "Operation would block"); ++ TEST_COMPARE_STRING (strerrorname_np (EWOULDBLOCK), "EWOULDBLOCK"); ++#endif ++#ifdef ENEEDAUTH ++ TEST_COMPARE_STRING (strerrordesc_np (ENEEDAUTH), "Need authenticator"); ++ TEST_COMPARE_STRING (strerrorname_np (ENEEDAUTH), "ENEEDAUTH"); ++#endif ++#ifdef ED ++ TEST_COMPARE_STRING (strerrordesc_np (ED), "?"); ++ TEST_COMPARE_STRING (strerrorname_np (ED), "ED"); ++#endif ++#ifdef EPROGUNAVAIL ++ TEST_COMPARE_STRING (strerrordesc_np (EPROGUNAVAIL), ++ "RPC program not available"); ++ TEST_COMPARE_STRING (strerrorname_np (EPROGUNAVAIL), "EPROGUNAVAIL"); ++#endif + + return 0; + } +diff --git a/stdio-common/vfscanf-internal.c b/stdio-common/vfscanf-internal.c +index 95b46dcbeb..3a323547f9 100644 +--- a/stdio-common/vfscanf-internal.c ++++ b/stdio-common/vfscanf-internal.c +@@ -277,7 +277,7 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + #endif + { + va_list arg; +- const CHAR_T *f = format; ++ const UCHAR_T *f = (const UCHAR_T *) format; + UCHAR_T fc; /* Current character of the format. */ + WINT_T done = 0; /* Assignments done. */ + size_t read_in = 0; /* Chars read in. */ +@@ -415,10 +415,11 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + #endif + + #ifndef COMPILE_WSCANF +- if (!isascii ((unsigned char) *f)) ++ if (!isascii (*f)) + { + /* Non-ASCII, may be a multibyte. */ +- int len = __mbrlen (f, strlen (f), &state); ++ int len = __mbrlen ((const char *) f, strlen ((const char *) f), ++ &state); + if (len > 0) + { + do +@@ -426,7 +427,7 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + c = inchar (); + if (__glibc_unlikely (c == EOF)) + input_error (); +- else if (c != (unsigned char) *f++) ++ else if (c != *f++) + { + ungetc_not_eof (c, s); + conv_error (); +@@ -484,9 +485,9 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + char_buffer_rewind (&charbuf); + + /* Check for a positional parameter specification. */ +- if (ISDIGIT ((UCHAR_T) *f)) ++ if (ISDIGIT (*f)) + { +- argpos = read_int ((const UCHAR_T **) &f); ++ argpos = read_int (&f); + if (*f == L_('$')) + ++f; + else +@@ -521,8 +522,8 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + + /* Find the maximum field width. */ + width = 0; +- if (ISDIGIT ((UCHAR_T) *f)) +- width = read_int ((const UCHAR_T **) &f); ++ if (ISDIGIT (*f)) ++ width = read_int (&f); + got_width: + if (width == 0) + width = -1; +@@ -2522,12 +2523,11 @@ __vfscanf_internal (FILE *s, const char *format, va_list argptr, + } + + while ((fc = *f++) != '\0' && fc != ']') +- if (fc == '-' && *f != '\0' && *f != ']' +- && (unsigned char) f[-2] <= (unsigned char) *f) ++ if (fc == '-' && *f != '\0' && *f != ']' && f[-2] <= *f) + { + /* Add all characters from the one before the '-' + up to (but not including) the next format char. */ +- for (fc = (unsigned char) f[-2]; fc < (unsigned char) *f; ++fc) ++ for (fc = f[-2]; fc < *f; ++fc) + ((char *)charbuf.scratch.data)[fc] = 1; + } + else +diff --git a/stdlib/tst-secure-getenv.c b/stdlib/tst-secure-getenv.c +index 3cfe9a05c3..d4b1139c5e 100644 +--- a/stdlib/tst-secure-getenv.c ++++ b/stdlib/tst-secure-getenv.c +@@ -30,167 +30,12 @@ + #include <sys/wait.h> + #include <unistd.h> + ++#include <support/check.h> + #include <support/support.h> ++#include <support/capture_subprocess.h> + #include <support/test-driver.h> + + static char MAGIC_ARGUMENT[] = "run-actual-test"; +-#define MAGIC_STATUS 19 +- +-/* Return a GID which is not our current GID, but is present in the +- supplementary group list. */ +-static gid_t +-choose_gid (void) +-{ +- int count = getgroups (0, NULL); +- if (count < 0) +- { +- printf ("getgroups: %m\n"); +- exit (1); +- } +- gid_t *groups; +- groups = xcalloc (count, sizeof (*groups)); +- int ret = getgroups (count, groups); +- if (ret < 0) +- { +- printf ("getgroups: %m\n"); +- exit (1); +- } +- gid_t current = getgid (); +- gid_t not_current = 0; +- for (int i = 0; i < ret; ++i) +- { +- if (groups[i] != current) +- { +- not_current = groups[i]; +- break; +- } +- } +- free (groups); +- return not_current; +-} +- +- +-/* Copies the executable into a restricted directory, so that we can +- safely make it SGID with the TARGET group ID. Then runs the +- executable. */ +-static int +-run_executable_sgid (gid_t target) +-{ +- char *dirname = xasprintf ("%s/secure-getenv.%jd", +- test_dir, (intmax_t) getpid ()); +- char *execname = xasprintf ("%s/bin", dirname); +- int infd = -1; +- int outfd = -1; +- int ret = -1; +- if (mkdir (dirname, 0700) < 0) +- { +- printf ("mkdir: %m\n"); +- goto err; +- } +- infd = open ("/proc/self/exe", O_RDONLY); +- if (infd < 0) +- { +- printf ("open (/proc/self/exe): %m\n"); +- goto err; +- } +- outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); +- if (outfd < 0) +- { +- printf ("open (%s): %m\n", execname); +- goto err; +- } +- char buf[4096]; +- for (;;) +- { +- ssize_t rdcount = read (infd, buf, sizeof (buf)); +- if (rdcount < 0) +- { +- printf ("read: %m\n"); +- goto err; +- } +- if (rdcount == 0) +- break; +- char *p = buf; +- char *end = buf + rdcount; +- while (p != end) +- { +- ssize_t wrcount = write (outfd, buf, end - p); +- if (wrcount == 0) +- errno = ENOSPC; +- if (wrcount <= 0) +- { +- printf ("write: %m\n"); +- goto err; +- } +- p += wrcount; +- } +- } +- if (fchown (outfd, getuid (), target) < 0) +- { +- printf ("fchown (%s): %m\n", execname); +- goto err; +- } +- if (fchmod (outfd, 02750) < 0) +- { +- printf ("fchmod (%s): %m\n", execname); +- goto err; +- } +- if (close (outfd) < 0) +- { +- printf ("close (outfd): %m\n"); +- goto err; +- } +- if (close (infd) < 0) +- { +- printf ("close (infd): %m\n"); +- goto err; +- } +- +- int kid = fork (); +- if (kid < 0) +- { +- printf ("fork: %m\n"); +- goto err; +- } +- if (kid == 0) +- { +- /* Child process. */ +- char *args[] = { execname, MAGIC_ARGUMENT, NULL }; +- execve (execname, args, environ); +- printf ("execve (%s): %m\n", execname); +- _exit (1); +- } +- int status; +- if (waitpid (kid, &status, 0) < 0) +- { +- printf ("waitpid: %m\n"); +- goto err; +- } +- if (!WIFEXITED (status) || WEXITSTATUS (status) != MAGIC_STATUS) +- { +- printf ("Unexpected exit status %d from child process\n", +- status); +- goto err; +- } +- ret = 0; +- +-err: +- if (outfd >= 0) +- close (outfd); +- if (infd >= 0) +- close (infd); +- if (execname) +- { +- unlink (execname); +- free (execname); +- } +- if (dirname) +- { +- rmdir (dirname); +- free (dirname); +- } +- return ret; +-} + + static int + do_test (void) +@@ -212,15 +57,15 @@ do_test (void) + exit (1); + } + +- gid_t target = choose_gid (); +- if (target == 0) +- { +- fprintf (stderr, +- "Could not find a suitable GID for user %jd, skipping test\n", +- (intmax_t) getuid ()); +- exit (0); +- } +- return run_executable_sgid (target); ++ int status = support_capture_subprogram_self_sgid (MAGIC_ARGUMENT); ++ ++ if (WEXITSTATUS (status) == EXIT_UNSUPPORTED) ++ return EXIT_UNSUPPORTED; ++ ++ if (!WIFEXITED (status)) ++ FAIL_EXIT1 ("Unexpected exit status %d from child process\n", status); ++ ++ return 0; + } + + static void +@@ -229,23 +74,15 @@ alternative_main (int argc, char **argv) + if (argc == 2 && strcmp (argv[1], MAGIC_ARGUMENT) == 0) + { + if (getgid () == getegid ()) +- { +- /* This can happen if the file system is mounted nosuid. */ +- fprintf (stderr, "SGID failed: GID and EGID match (%jd)\n", +- (intmax_t) getgid ()); +- exit (MAGIC_STATUS); +- } ++ /* This can happen if the file system is mounted nosuid. */ ++ FAIL_UNSUPPORTED ("SGID failed: GID and EGID match (%jd)\n", ++ (intmax_t) getgid ()); + if (getenv ("PATH") == NULL) +- { +- printf ("PATH variable not present\n"); +- exit (3); +- } ++ FAIL_EXIT (3, "PATH variable not present\n"); + if (secure_getenv ("PATH") != NULL) +- { +- printf ("PATH variable not filtered out\n"); +- exit (4); +- } +- exit (MAGIC_STATUS); ++ FAIL_EXIT (4, "PATH variable not filtered out\n"); ++ ++ exit (EXIT_SUCCESS); + } + } + +diff --git a/string/bits/string_fortified.h b/string/bits/string_fortified.h +index 309d0f39b2..c8d3051af8 100644 +--- a/string/bits/string_fortified.h ++++ b/string/bits/string_fortified.h +@@ -22,11 +22,6 @@ + # error "Never use <bits/string_fortified.h> directly; include <string.h> instead." + #endif + +-#if !__GNUC_PREREQ (5,0) +-__warndecl (__warn_memset_zero_len, +- "memset used with constant zero length parameter; this could be due to transposed parameters"); +-#endif +- + __fortify_function void * + __NTH (memcpy (void *__restrict __dest, const void *__restrict __src, + size_t __len)) +@@ -58,16 +53,6 @@ __NTH (mempcpy (void *__restrict __dest, const void *__restrict __src, + __fortify_function void * + __NTH (memset (void *__dest, int __ch, size_t __len)) + { +- /* GCC-5.0 and newer implements these checks in the compiler, so we don't +- need them here. */ +-#if !__GNUC_PREREQ (5,0) +- if (__builtin_constant_p (__len) && __len == 0 +- && (!__builtin_constant_p (__ch) || __ch != 0)) +- { +- __warn_memset_zero_len (); +- return __dest; +- } +-#endif + return __builtin___memset_chk (__dest, __ch, __len, __bos0 (__dest)); + } + +diff --git a/string/test-memchr.c b/string/test-memchr.c +index 5dd0aa5470..de70e794d9 100644 +--- a/string/test-memchr.c ++++ b/string/test-memchr.c +@@ -65,8 +65,8 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n, CHAR *exp_res) + CHAR *res = CALL (impl, s, c, n); + if (res != exp_res) + { +- error (0, 0, "Wrong result in function %s %p %p", impl->name, +- res, exp_res); ++ error (0, 0, "Wrong result in function %s (%p, %d, %zu) -> %p != %p", ++ impl->name, s, c, n, res, exp_res); + ret = 1; + return; + } +@@ -91,7 +91,7 @@ do_test (size_t align, size_t pos, size_t len, size_t n, int seek_char) + } + buf[align + len] = 0; + +- if (pos < len) ++ if (pos < MIN(n, len)) + { + buf[align + pos] = seek_char; + buf[align + len] = -seek_char; +@@ -107,6 +107,38 @@ do_test (size_t align, size_t pos, size_t len, size_t n, int seek_char) + do_one_test (impl, (CHAR *) (buf + align), seek_char, n, result); + } + ++static void ++do_overflow_tests (void) ++{ ++ size_t i, j, len; ++ const size_t one = 1; ++ uintptr_t buf_addr = (uintptr_t) buf1; ++ ++ for (i = 0; i < 750; ++i) ++ { ++ do_test (0, i, 751, SIZE_MAX - i, BIG_CHAR); ++ do_test (0, i, 751, i - buf_addr, BIG_CHAR); ++ do_test (0, i, 751, -buf_addr - i, BIG_CHAR); ++ do_test (0, i, 751, SIZE_MAX - buf_addr - i, BIG_CHAR); ++ do_test (0, i, 751, SIZE_MAX - buf_addr + i, BIG_CHAR); ++ ++ len = 0; ++ for (j = 8 * sizeof(size_t) - 1; j ; --j) ++ { ++ len |= one << j; ++ do_test (0, i, 751, len - i, BIG_CHAR); ++ do_test (0, i, 751, len + i, BIG_CHAR); ++ do_test (0, i, 751, len - buf_addr - i, BIG_CHAR); ++ do_test (0, i, 751, len - buf_addr + i, BIG_CHAR); ++ ++ do_test (0, i, 751, ~len - i, BIG_CHAR); ++ do_test (0, i, 751, ~len + i, BIG_CHAR); ++ do_test (0, i, 751, ~len - buf_addr - i, BIG_CHAR); ++ do_test (0, i, 751, ~len - buf_addr + i, BIG_CHAR); ++ } ++ } ++} ++ + static void + do_random_tests (void) + { +@@ -221,6 +253,7 @@ test_main (void) + do_test (page_size / 2 - i, i, i, 1, 0x9B); + + do_random_tests (); ++ do_overflow_tests (); + return ret; + } + +diff --git a/string/test-strncat.c b/string/test-strncat.c +index abbacb95c6..0c7f68d086 100644 +--- a/string/test-strncat.c ++++ b/string/test-strncat.c +@@ -134,6 +134,66 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2, + } + } + ++static void ++do_overflow_tests (void) ++{ ++ size_t i, j, len; ++ const size_t one = 1; ++ CHAR *s1, *s2; ++ uintptr_t s1_addr; ++ s1 = (CHAR *) buf1; ++ s2 = (CHAR *) buf2; ++ s1_addr = (uintptr_t)s1; ++ for (j = 0; j < 200; ++j) ++ s2[j] = 32 + 23 * j % (BIG_CHAR - 32); ++ s2[200] = 0; ++ for (i = 0; i < 750; ++i) { ++ for (j = 0; j < i; ++j) ++ s1[j] = 32 + 23 * j % (BIG_CHAR - 32); ++ s1[i] = '\0'; ++ ++ FOR_EACH_IMPL (impl, 0) ++ { ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, SIZE_MAX - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, i - s1_addr); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, -s1_addr - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, SIZE_MAX - s1_addr - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, SIZE_MAX - s1_addr + i); ++ } ++ ++ len = 0; ++ for (j = 8 * sizeof(size_t) - 1; j ; --j) ++ { ++ len |= one << j; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, len - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, len + i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, len - s1_addr - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, len - s1_addr + i); ++ ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, ~len - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, ~len + i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, ~len - s1_addr - i); ++ s2[200] = '\0'; ++ do_one_test (impl, s2, s1, ~len - s1_addr + i); ++ } ++ } ++ } ++} ++ + static void + do_random_tests (void) + { +@@ -316,6 +376,7 @@ test_main (void) + } + + do_random_tests (); ++ do_overflow_tests (); + return ret; + } + +diff --git a/string/test-strnlen.c b/string/test-strnlen.c +index 80ac9e8602..a1a6746cc9 100644 +--- a/string/test-strnlen.c ++++ b/string/test-strnlen.c +@@ -27,6 +27,7 @@ + + #ifndef WIDE + # define STRNLEN strnlen ++# define MEMSET memset + # define CHAR char + # define BIG_CHAR CHAR_MAX + # define MIDDLE_CHAR 127 +@@ -34,6 +35,7 @@ + #else + # include <wchar.h> + # define STRNLEN wcsnlen ++# define MEMSET wmemset + # define CHAR wchar_t + # define BIG_CHAR WCHAR_MAX + # define MIDDLE_CHAR 1121 +@@ -87,6 +89,38 @@ do_test (size_t align, size_t len, size_t maxlen, int max_char) + do_one_test (impl, (CHAR *) (buf + align), maxlen, MIN (len, maxlen)); + } + ++static void ++do_overflow_tests (void) ++{ ++ size_t i, j, len; ++ const size_t one = 1; ++ uintptr_t buf_addr = (uintptr_t) buf1; ++ ++ for (i = 0; i < 750; ++i) ++ { ++ do_test (0, i, SIZE_MAX - i, BIG_CHAR); ++ do_test (0, i, i - buf_addr, BIG_CHAR); ++ do_test (0, i, -buf_addr - i, BIG_CHAR); ++ do_test (0, i, SIZE_MAX - buf_addr - i, BIG_CHAR); ++ do_test (0, i, SIZE_MAX - buf_addr + i, BIG_CHAR); ++ ++ len = 0; ++ for (j = 8 * sizeof(size_t) - 1; j ; --j) ++ { ++ len |= one << j; ++ do_test (0, i, len - i, BIG_CHAR); ++ do_test (0, i, len + i, BIG_CHAR); ++ do_test (0, i, len - buf_addr - i, BIG_CHAR); ++ do_test (0, i, len - buf_addr + i, BIG_CHAR); ++ ++ do_test (0, i, ~len - i, BIG_CHAR); ++ do_test (0, i, ~len + i, BIG_CHAR); ++ do_test (0, i, ~len - buf_addr - i, BIG_CHAR); ++ do_test (0, i, ~len - buf_addr + i, BIG_CHAR); ++ } ++ } ++} ++ + static void + do_random_tests (void) + { +@@ -153,7 +187,7 @@ do_page_tests (void) + size_t last_offset = (page_size / sizeof (CHAR)) - 1; + + CHAR *s = (CHAR *) buf2; +- memset (s, 65, (last_offset - 1)); ++ MEMSET (s, 65, (last_offset - 1)); + s[last_offset] = 0; + + /* Place short strings ending at page boundary. */ +@@ -196,6 +230,35 @@ do_page_tests (void) + } + } + ++/* Tests meant to unveil fail on implementations that access bytes ++ beyond the maxium length. */ ++ ++static void ++do_page_2_tests (void) ++{ ++ size_t i, exp_len, offset; ++ size_t last_offset = page_size / sizeof (CHAR); ++ ++ CHAR *s = (CHAR *) buf2; ++ MEMSET (s, 65, last_offset); ++ ++ /* Place short strings ending at page boundary without the null ++ byte. */ ++ offset = last_offset; ++ for (i = 0; i < 128; i++) ++ { ++ /* Decrease offset to stress several sizes and alignments. */ ++ offset--; ++ exp_len = last_offset - offset; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ /* If an implementation goes beyond EXP_LEN, it will trigger ++ the segfault. */ ++ do_one_test (impl, (CHAR *) (s + offset), exp_len, exp_len); ++ } ++ } ++} ++ + int + test_main (void) + { +@@ -242,6 +305,8 @@ test_main (void) + + do_random_tests (); + do_page_tests (); ++ do_page_2_tests (); ++ do_overflow_tests (); + return ret; + } + +diff --git a/support/Makefile b/support/Makefile +index 93faafddf9..3d3aff5ff9 100644 +--- a/support/Makefile ++++ b/support/Makefile +@@ -35,6 +35,8 @@ libsupport-routines = \ + ignore_stderr \ + next_to_fault \ + oom_error \ ++ resolv_response_context_duplicate \ ++ resolv_response_context_free \ + resolv_test \ + set_fortify_handler \ + support-xfstat \ +@@ -133,6 +135,7 @@ libsupport-routines = \ + xpthread_join \ + xpthread_key_create \ + xpthread_key_delete \ ++ xpthread_kill \ + xpthread_mutex_consistent \ + xpthread_mutex_destroy \ + xpthread_mutex_init \ +diff --git a/support/capture_subprocess.h b/support/capture_subprocess.h +index 9808750f80..421f657678 100644 +--- a/support/capture_subprocess.h ++++ b/support/capture_subprocess.h +@@ -41,6 +41,12 @@ struct support_capture_subprocess support_capture_subprocess + struct support_capture_subprocess support_capture_subprogram + (const char *file, char *const argv[]); + ++/* Copy the running program into a setgid binary and run it with CHILD_ID ++ argument. If execution is successful, return the exit status of the child ++ program, otherwise return a non-zero failure exit code. */ ++int support_capture_subprogram_self_sgid ++ (char *child_id); ++ + /* Deallocate the subprocess data captured by + support_capture_subprocess. */ + void support_capture_subprocess_free (struct support_capture_subprocess *); +diff --git a/support/resolv_response_context_duplicate.c b/support/resolv_response_context_duplicate.c +new file mode 100644 +index 0000000000..f9c5c3462a +--- /dev/null ++++ b/support/resolv_response_context_duplicate.c +@@ -0,0 +1,37 @@ ++/* Duplicate a response context used in DNS resolver tests. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <string.h> ++#include <support/resolv_test.h> ++#include <support/support.h> ++ ++struct resolv_response_context * ++resolv_response_context_duplicate (const struct resolv_response_context *ctx) ++{ ++ struct resolv_response_context *result = xmalloc (sizeof (*result)); ++ memcpy (result, ctx, sizeof (*result)); ++ if (result->client_address != NULL) ++ { ++ result->client_address = xmalloc (result->client_address_length); ++ memcpy (result->client_address, ctx->client_address, ++ result->client_address_length); ++ } ++ result->query_buffer = xmalloc (result->query_length); ++ memcpy (result->query_buffer, ctx->query_buffer, result->query_length); ++ return result; ++} +diff --git a/support/resolv_response_context_free.c b/support/resolv_response_context_free.c +new file mode 100644 +index 0000000000..b88c05ffd4 +--- /dev/null ++++ b/support/resolv_response_context_free.c +@@ -0,0 +1,28 @@ ++/* Free a response context used in DNS resolver tests. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <stdlib.h> ++#include <support/resolv_test.h> ++ ++void ++resolv_response_context_free (struct resolv_response_context *ctx) ++{ ++ free (ctx->query_buffer); ++ free (ctx->client_address); ++ free (ctx); ++} +diff --git a/support/resolv_test.c b/support/resolv_test.c +index 53b7fc41ab..9878a040a3 100644 +--- a/support/resolv_test.c ++++ b/support/resolv_test.c +@@ -181,7 +181,9 @@ resolv_response_init (struct resolv_response_builder *b, + b->buffer[2] |= b->query_buffer[2] & 0x01; /* Copy the RD bit. */ + if (flags.tc) + b->buffer[2] |= 0x02; +- b->buffer[3] = 0x80 | flags.rcode; /* Always set RA. */ ++ b->buffer[3] = flags.rcode; ++ if (!flags.clear_ra) ++ b->buffer[3] |= 0x80; + if (flags.ad) + b->buffer[3] |= 0x20; + +@@ -434,9 +436,9 @@ resolv_response_buffer (const struct resolv_response_builder *b) + return result; + } + +-static struct resolv_response_builder * +-response_builder_allocate +- (const unsigned char *query_buffer, size_t query_length) ++struct resolv_response_builder * ++resolv_response_builder_allocate (const unsigned char *query_buffer, ++ size_t query_length) + { + struct resolv_response_builder *b = xmalloc (sizeof (*b)); + memset (b, 0, offsetof (struct resolv_response_builder, buffer)); +@@ -445,8 +447,8 @@ response_builder_allocate + return b; + } + +-static void +-response_builder_free (struct resolv_response_builder *b) ++void ++resolv_response_builder_free (struct resolv_response_builder *b) + { + tdestroy (b->compression_offsets, free); + free (b); +@@ -661,13 +663,17 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) + + struct resolv_response_context ctx = + { ++ .test = obj, ++ .client_address = &peer, ++ .client_address_length = peerlen, + .query_buffer = query, + .query_length = length, + .server_index = server_index, + .tcp = false, + .edns = qinfo.edns, + }; +- struct resolv_response_builder *b = response_builder_allocate (query, length); ++ struct resolv_response_builder *b ++ = resolv_response_builder_allocate (query, length); + obj->config.response_callback + (&ctx, b, qinfo.qname, qinfo.qclass, qinfo.qtype); + +@@ -684,7 +690,7 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) + if (b->offset >= 12) + printf ("info: UDP server %d: sending response:" + " %zu bytes, RCODE %d (for %s/%u/%u)\n", +- server_index, b->offset, b->buffer[3] & 0x0f, ++ ctx.server_index, b->offset, b->buffer[3] & 0x0f, + qinfo.qname, qinfo.qclass, qinfo.qtype); + else + printf ("info: UDP server %d: sending response: %zu bytes" +@@ -694,23 +700,31 @@ server_thread_udp_process_one (struct resolv_test *obj, int server_index) + if (b->truncate_bytes > 0) + printf ("info: truncated by %u bytes\n", b->truncate_bytes); + } +- size_t to_send = b->offset; +- if (to_send < b->truncate_bytes) +- to_send = 0; +- else +- to_send -= b->truncate_bytes; +- +- /* Ignore most errors here because the other end may have closed +- the socket. */ +- if (sendto (obj->servers[server_index].socket_udp, +- b->buffer, to_send, 0, +- (struct sockaddr *) &peer, peerlen) < 0) +- TEST_VERIFY_EXIT (errno != EBADF); ++ resolv_response_send_udp (&ctx, b); + } +- response_builder_free (b); ++ resolv_response_builder_free (b); + return true; + } + ++void ++resolv_response_send_udp (const struct resolv_response_context *ctx, ++ struct resolv_response_builder *b) ++{ ++ TEST_VERIFY_EXIT (!ctx->tcp); ++ size_t to_send = b->offset; ++ if (to_send < b->truncate_bytes) ++ to_send = 0; ++ else ++ to_send -= b->truncate_bytes; ++ ++ /* Ignore most errors here because the other end may have closed ++ the socket. */ ++ if (sendto (ctx->test->servers[ctx->server_index].socket_udp, ++ b->buffer, to_send, 0, ++ ctx->client_address, ctx->client_address_length) < 0) ++ TEST_VERIFY_EXIT (errno != EBADF); ++} ++ + /* UDP thread_callback function. Variant for one thread per + server. */ + static void +@@ -897,14 +911,15 @@ server_thread_tcp_client (void *arg) + + struct resolv_response_context ctx = + { ++ .test = closure->obj, + .query_buffer = query_buffer, + .query_length = query_length, + .server_index = closure->server_index, + .tcp = true, + .edns = qinfo.edns, + }; +- struct resolv_response_builder *b = response_builder_allocate +- (query_buffer, query_length); ++ struct resolv_response_builder *b ++ = resolv_response_builder_allocate (query_buffer, query_length); + closure->obj->config.response_callback + (&ctx, b, qinfo.qname, qinfo.qclass, qinfo.qtype); + +@@ -936,7 +951,7 @@ server_thread_tcp_client (void *arg) + writev_fully (closure->client_socket, buffers, 2); + } + bool close_flag = b->close; +- response_builder_free (b); ++ resolv_response_builder_free (b); + free (query_buffer); + if (close_flag) + break; +diff --git a/support/resolv_test.h b/support/resolv_test.h +index 67819469a0..31a5c1c3e7 100644 +--- a/support/resolv_test.h ++++ b/support/resolv_test.h +@@ -35,25 +35,36 @@ struct resolv_edns_info + uint16_t payload_size; + }; + ++/* This opaque struct collects information about the resolver testing ++ currently in progress. */ ++struct resolv_test; ++ + /* This struct provides context information when the response callback + specified in struct resolv_redirect_config is invoked. */ + struct resolv_response_context + { +- const unsigned char *query_buffer; ++ struct resolv_test *test; ++ void *client_address; ++ size_t client_address_length; ++ unsigned char *query_buffer; + size_t query_length; + int server_index; + bool tcp; + struct resolv_edns_info edns; + }; + ++/* Produces a deep copy of the context. */ ++struct resolv_response_context * ++ resolv_response_context_duplicate (const struct resolv_response_context *); ++ ++/* Frees the copy. For the context passed to the response function, ++ this happens implicitly. */ ++void resolv_response_context_free (struct resolv_response_context *); ++ + /* This opaque struct is used to construct responses from within the + response callback function. */ + struct resolv_response_builder; + +-/* This opaque struct collects information about the resolver testing +- currently in progress. */ +-struct resolv_test; +- + enum + { + /* Maximum number of test servers supported by the framework. */ +@@ -137,6 +148,10 @@ struct resolv_response_flags + /* If true, the AD (authenticated data) flag will be set. */ + bool ad; + ++ /* If true, do not set the RA (recursion available) flag in the ++ response. */ ++ bool clear_ra; ++ + /* Initial section count values. Can be used to artificially + increase the counts, for malformed packet testing.*/ + unsigned short qdcount; +@@ -188,6 +203,22 @@ void resolv_response_close (struct resolv_response_builder *); + /* The size of the response packet built so far. */ + size_t resolv_response_length (const struct resolv_response_builder *); + ++/* Allocates a response builder tied to a specific query packet, ++ starting at QUERY_BUFFER, containing QUERY_LENGTH bytes. */ ++struct resolv_response_builder * ++ resolv_response_builder_allocate (const unsigned char *query_buffer, ++ size_t query_length); ++ ++/* Deallocates a response buffer. */ ++void resolv_response_builder_free (struct resolv_response_builder *); ++ ++/* Sends a UDP response using a specific context. This can be used to ++ reorder or duplicate responses, along with ++ resolv_response_context_duplicate and ++ response_builder_allocate. */ ++void resolv_response_send_udp (const struct resolv_response_context *, ++ struct resolv_response_builder *); ++ + __END_DECLS + + #endif /* SUPPORT_RESOLV_TEST_H */ +diff --git a/support/subprocess.h b/support/subprocess.h +index 8b442fd5c0..34ffd02e8e 100644 +--- a/support/subprocess.h ++++ b/support/subprocess.h +@@ -38,6 +38,11 @@ struct support_subprocess support_subprocess + struct support_subprocess support_subprogram + (const char *file, char *const argv[]); + ++/* Invoke program FILE with ARGV arguments by using posix_spawn and wait for it ++ to complete. Return program exit status. */ ++int support_subprogram_wait ++ (const char *file, char *const argv[]); ++ + /* Wait for the subprocess indicated by PROC::PID. Return the status + indicate by waitpid call. */ + int support_process_wait (struct support_subprocess *proc); +diff --git a/support/support_capture_subprocess.c b/support/support_capture_subprocess.c +index eeed676e3d..28a37df67f 100644 +--- a/support/support_capture_subprocess.c ++++ b/support/support_capture_subprocess.c +@@ -20,11 +20,14 @@ + #include <support/capture_subprocess.h> + + #include <errno.h> ++#include <fcntl.h> + #include <stdlib.h> + #include <support/check.h> + #include <support/xunistd.h> + #include <support/xsocket.h> + #include <support/xspawn.h> ++#include <support/support.h> ++#include <support/test-driver.h> + + static void + transfer (const char *what, struct pollfd *pfd, struct xmemstream *stream) +@@ -36,7 +39,7 @@ transfer (const char *what, struct pollfd *pfd, struct xmemstream *stream) + if (ret < 0) + { + support_record_failure (); +- printf ("error: reading from subprocess %s: %m", what); ++ printf ("error: reading from subprocess %s: %m\n", what); + pfd->events = 0; + pfd->revents = 0; + } +@@ -102,6 +105,129 @@ support_capture_subprogram (const char *file, char *const argv[]) + return result; + } + ++/* Copies the executable into a restricted directory, so that we can ++ safely make it SGID with the TARGET group ID. Then runs the ++ executable. */ ++static int ++copy_and_spawn_sgid (char *child_id, gid_t gid) ++{ ++ char *dirname = xasprintf ("%s/tst-tunables-setuid.%jd", ++ test_dir, (intmax_t) getpid ()); ++ char *execname = xasprintf ("%s/bin", dirname); ++ int infd = -1; ++ int outfd = -1; ++ int ret = 1, status = 1; ++ ++ TEST_VERIFY (mkdir (dirname, 0700) == 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ ++ infd = open ("/proc/self/exe", O_RDONLY); ++ if (infd < 0) ++ FAIL_UNSUPPORTED ("unsupported: Cannot read binary from procfs\n"); ++ ++ outfd = open (execname, O_WRONLY | O_CREAT | O_EXCL, 0700); ++ TEST_VERIFY (outfd >= 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ ++ char buf[4096]; ++ for (;;) ++ { ++ ssize_t rdcount = read (infd, buf, sizeof (buf)); ++ TEST_VERIFY (rdcount >= 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ if (rdcount == 0) ++ break; ++ char *p = buf; ++ char *end = buf + rdcount; ++ while (p != end) ++ { ++ ssize_t wrcount = write (outfd, buf, end - p); ++ if (wrcount == 0) ++ errno = ENOSPC; ++ TEST_VERIFY (wrcount > 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ p += wrcount; ++ } ++ } ++ TEST_VERIFY (fchown (outfd, getuid (), gid) == 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ TEST_VERIFY (fchmod (outfd, 02750) == 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ TEST_VERIFY (close (outfd) == 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ TEST_VERIFY (close (infd) == 0); ++ if (support_record_failure_is_failed ()) ++ goto err; ++ ++ /* We have the binary, now spawn the subprocess. Avoid using ++ support_subprogram because we only want the program exit status, not the ++ contents. */ ++ ret = 0; ++ ++ char * const args[] = {execname, child_id, NULL}; ++ ++ status = support_subprogram_wait (args[0], args); ++ ++err: ++ if (outfd >= 0) ++ close (outfd); ++ if (infd >= 0) ++ close (infd); ++ if (execname != NULL) ++ { ++ unlink (execname); ++ free (execname); ++ } ++ if (dirname != NULL) ++ { ++ rmdir (dirname); ++ free (dirname); ++ } ++ ++ if (ret != 0) ++ FAIL_EXIT1("Failed to make sgid executable for test\n"); ++ ++ return status; ++} ++ ++int ++support_capture_subprogram_self_sgid (char *child_id) ++{ ++ gid_t target = 0; ++ const int count = 64; ++ gid_t groups[count]; ++ ++ /* Get a GID which is not our current GID, but is present in the ++ supplementary group list. */ ++ int ret = getgroups (count, groups); ++ if (ret < 0) ++ FAIL_UNSUPPORTED("Could not get group list for user %jd\n", ++ (intmax_t) getuid ()); ++ ++ gid_t current = getgid (); ++ for (int i = 0; i < ret; ++i) ++ { ++ if (groups[i] != current) ++ { ++ target = groups[i]; ++ break; ++ } ++ } ++ ++ if (target == 0) ++ FAIL_UNSUPPORTED("Could not find a suitable GID for user %jd\n", ++ (intmax_t) getuid ()); ++ ++ return copy_and_spawn_sgid (child_id, target); ++} ++ + void + support_capture_subprocess_free (struct support_capture_subprocess *p) + { +diff --git a/support/support_subprocess.c b/support/support_subprocess.c +index 36e3a77af2..4a25828111 100644 +--- a/support/support_subprocess.c ++++ b/support/support_subprocess.c +@@ -27,7 +27,7 @@ + #include <support/subprocess.h> + + static struct support_subprocess +-support_suprocess_init (void) ++support_subprocess_init (void) + { + struct support_subprocess result; + +@@ -48,7 +48,7 @@ support_suprocess_init (void) + struct support_subprocess + support_subprocess (void (*callback) (void *), void *closure) + { +- struct support_subprocess result = support_suprocess_init (); ++ struct support_subprocess result = support_subprocess_init (); + + result.pid = xfork (); + if (result.pid == 0) +@@ -71,7 +71,7 @@ support_subprocess (void (*callback) (void *), void *closure) + struct support_subprocess + support_subprogram (const char *file, char *const argv[]) + { +- struct support_subprocess result = support_suprocess_init (); ++ struct support_subprocess result = support_subprocess_init (); + + posix_spawn_file_actions_t fa; + /* posix_spawn_file_actions_init does not fail. */ +@@ -84,7 +84,7 @@ support_subprogram (const char *file, char *const argv[]) + xposix_spawn_file_actions_addclose (&fa, result.stdout_pipe[1]); + xposix_spawn_file_actions_addclose (&fa, result.stderr_pipe[1]); + +- result.pid = xposix_spawn (file, &fa, NULL, argv, NULL); ++ result.pid = xposix_spawn (file, &fa, NULL, argv, environ); + + xclose (result.stdout_pipe[1]); + xclose (result.stderr_pipe[1]); +@@ -92,6 +92,19 @@ support_subprogram (const char *file, char *const argv[]) + return result; + } + ++int ++support_subprogram_wait (const char *file, char *const argv[]) ++{ ++ posix_spawn_file_actions_t fa; ++ ++ posix_spawn_file_actions_init (&fa); ++ struct support_subprocess res = support_subprocess_init (); ++ ++ res.pid = xposix_spawn (file, &fa, NULL, argv, environ); ++ ++ return support_process_wait (&res); ++} ++ + int + support_process_wait (struct support_subprocess *proc) + { +diff --git a/support/xpthread_kill.c b/support/xpthread_kill.c +new file mode 100644 +index 0000000000..111a75d85e +--- /dev/null ++++ b/support/xpthread_kill.c +@@ -0,0 +1,26 @@ ++/* pthread_kill with error checking. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <signal.h> ++#include <support/xthread.h> ++ ++void ++xpthread_kill (pthread_t thr, int signo) ++{ ++ xpthread_check_return ("pthread_kill", pthread_kill (thr, signo)); ++} +diff --git a/support/xthread.h b/support/xthread.h +index 05f8d4a7d9..cb1fc30da0 100644 +--- a/support/xthread.h ++++ b/support/xthread.h +@@ -75,6 +75,8 @@ void xpthread_attr_setstacksize (pthread_attr_t *attr, + void xpthread_attr_setguardsize (pthread_attr_t *attr, + size_t guardsize); + ++void xpthread_kill (pthread_t thr, int signo); ++ + /* Set the stack size in ATTR to a small value, but still large enough + to cover most internal glibc stack usage. */ + void support_set_small_thread_stack_size (pthread_attr_t *attr); +diff --git a/sysdeps/aarch64/dl-bti.c b/sysdeps/aarch64/dl-bti.c +index 196e462520..cf7624aaa2 100644 +--- a/sysdeps/aarch64/dl-bti.c ++++ b/sysdeps/aarch64/dl-bti.c +@@ -19,43 +19,76 @@ + #include <errno.h> + #include <libintl.h> + #include <ldsodefs.h> ++#include <sys/mman.h> + +-static int +-enable_bti (struct link_map *map, const char *program) ++/* See elf/dl-load.h. */ ++#ifndef MAP_COPY ++# define MAP_COPY (MAP_PRIVATE | MAP_DENYWRITE) ++#endif ++ ++/* Enable BTI protection for MAP. */ ++ ++void ++_dl_bti_protect (struct link_map *map, int fd) + { ++ const size_t pagesz = GLRO(dl_pagesize); + const ElfW(Phdr) *phdr; +- unsigned prot; + + for (phdr = map->l_phdr; phdr < &map->l_phdr[map->l_phnum]; ++phdr) + if (phdr->p_type == PT_LOAD && (phdr->p_flags & PF_X)) + { +- void *start = (void *) (phdr->p_vaddr + map->l_addr); +- size_t len = phdr->p_memsz; ++ size_t vstart = ALIGN_DOWN (phdr->p_vaddr, pagesz); ++ size_t vend = ALIGN_UP (phdr->p_vaddr + phdr->p_filesz, pagesz); ++ off_t off = ALIGN_DOWN (phdr->p_offset, pagesz); ++ void *start = (void *) (vstart + map->l_addr); ++ size_t len = vend - vstart; + +- prot = PROT_EXEC | PROT_BTI; ++ unsigned prot = PROT_EXEC | PROT_BTI; + if (phdr->p_flags & PF_R) + prot |= PROT_READ; + if (phdr->p_flags & PF_W) + prot |= PROT_WRITE; + +- if (__mprotect (start, len, prot) < 0) +- { +- if (program) +- _dl_fatal_printf ("%s: mprotect failed to turn on BTI\n", +- map->l_name); +- else +- _dl_signal_error (errno, map->l_name, "dlopen", +- N_("mprotect failed to turn on BTI")); +- } ++ if (fd == -1) ++ /* Ignore failures for kernel mapped binaries. */ ++ __mprotect (start, len, prot); ++ else ++ map->l_mach.bti_fail = __mmap (start, len, prot, ++ MAP_FIXED|MAP_COPY|MAP_FILE, ++ fd, off) == MAP_FAILED; + } +- return 0; + } + +-/* Enable BTI for L if required. */ ++ ++static void ++bti_failed (struct link_map *l, const char *program) ++{ ++ if (program) ++ _dl_fatal_printf ("%s: %s: failed to turn on BTI protection\n", ++ program, l->l_name); ++ else ++ /* Note: the errno value is not available any more. */ ++ _dl_signal_error (0, l->l_name, "dlopen", ++ N_("failed to turn on BTI protection")); ++} ++ ++ ++/* Enable BTI for L and its dependencies. */ + + void + _dl_bti_check (struct link_map *l, const char *program) + { +- if (GLRO(dl_aarch64_cpu_features).bti && l->l_mach.bti) +- enable_bti (l, program); ++ if (!GLRO(dl_aarch64_cpu_features).bti) ++ return; ++ ++ if (l->l_mach.bti_fail) ++ bti_failed (l, program); ++ ++ unsigned int i = l->l_searchlist.r_nlist; ++ while (i-- > 0) ++ { ++ struct link_map *dep = l->l_initfini[i]; ++ if (dep->l_mach.bti_fail) ++ bti_failed (dep, program); ++ } + } +diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h +index 70b9ed3925..fde7cfd9e2 100644 +--- a/sysdeps/aarch64/dl-machine.h ++++ b/sysdeps/aarch64/dl-machine.h +@@ -395,13 +395,6 @@ elf_machine_lazy_rel (struct link_map *map, + /* Check for unexpected PLT reloc type. */ + if (__builtin_expect (r_type == AARCH64_R(JUMP_SLOT), 1)) + { +- if (map->l_mach.plt == 0) +- { +- /* Prelinking. */ +- *reloc_addr += l_addr; +- return; +- } +- + if (__glibc_unlikely (map->l_info[DT_AARCH64 (VARIANT_PCS)] != NULL)) + { + /* Check the symbol table for variant PCS symbols. */ +@@ -425,7 +418,10 @@ elf_machine_lazy_rel (struct link_map *map, + } + } + +- *reloc_addr = map->l_mach.plt; ++ if (map->l_mach.plt == 0) ++ *reloc_addr += l_addr; ++ else ++ *reloc_addr = map->l_mach.plt; + } + else if (__builtin_expect (r_type == AARCH64_R(TLSDESC), 1)) + { +diff --git a/sysdeps/aarch64/dl-prop.h b/sysdeps/aarch64/dl-prop.h +index b0785bda83..e926e54984 100644 +--- a/sysdeps/aarch64/dl-prop.h ++++ b/sysdeps/aarch64/dl-prop.h +@@ -19,6 +19,8 @@ + #ifndef _DL_PROP_H + #define _DL_PROP_H + ++extern void _dl_bti_protect (struct link_map *, int) attribute_hidden; ++ + extern void _dl_bti_check (struct link_map *, const char *) + attribute_hidden; + +@@ -35,14 +37,18 @@ _dl_open_check (struct link_map *m) + } + + static inline void __attribute__ ((always_inline)) +-_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) ++_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) + { + } + + static inline int +-_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, +- void *data) ++_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, ++ uint32_t datasz, void *data) + { ++ if (!GLRO(dl_aarch64_cpu_features).bti) ++ /* Skip note processing. */ ++ return 0; ++ + if (type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) + { + /* Stop if the property note is ill-formed. */ +@@ -51,7 +57,7 @@ _dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, + + unsigned int feature_1 = *(unsigned int *) data; + if (feature_1 & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) +- l->l_mach.bti = true; ++ _dl_bti_protect (l, fd); + + /* Stop if we processed the property note. */ + return 0; +diff --git a/sysdeps/aarch64/linkmap.h b/sysdeps/aarch64/linkmap.h +index 847a03ace2..b3f7663b07 100644 +--- a/sysdeps/aarch64/linkmap.h ++++ b/sysdeps/aarch64/linkmap.h +@@ -22,5 +22,5 @@ struct link_map_machine + { + ElfW(Addr) plt; /* Address of .plt */ + void *tlsdesc_table; /* Address of TLS descriptor hash table. */ +- bool bti; /* Branch Target Identification is enabled. */ ++ bool bti_fail; /* Failed to enable Branch Target Identification. */ + }; +diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c +index 7cf5f033e8..799d60c98c 100644 +--- a/sysdeps/aarch64/multiarch/memcpy.c ++++ b/sysdeps/aarch64/multiarch/memcpy.c +@@ -41,7 +41,8 @@ libc_ifunc (__libc_memcpy, + ? __memcpy_falkor + : (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr) + ? __memcpy_thunderx2 +- : (IS_NEOVERSE_N1 (midr) ++ : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr) ++ || IS_NEOVERSE_V1 (midr) + ? __memcpy_simd + : __memcpy_generic))))); + +diff --git a/sysdeps/aarch64/multiarch/memcpy_advsimd.S b/sysdeps/aarch64/multiarch/memcpy_advsimd.S +index d4ba747777..48bb6d7ca4 100644 +--- a/sysdeps/aarch64/multiarch/memcpy_advsimd.S ++++ b/sysdeps/aarch64/multiarch/memcpy_advsimd.S +@@ -223,12 +223,13 @@ L(copy_long_backwards): + b.ls L(copy64_from_start) + + L(loop64_backwards): +- stp A_q, B_q, [dstend, -32] ++ str B_q, [dstend, -16] ++ str A_q, [dstend, -32] + ldp A_q, B_q, [srcend, -96] +- stp C_q, D_q, [dstend, -64] ++ str D_q, [dstend, -48] ++ str C_q, [dstend, -64]! + ldp C_q, D_q, [srcend, -128] + sub srcend, srcend, 64 +- sub dstend, dstend, 64 + subs count, count, 64 + b.hi L(loop64_backwards) + +diff --git a/sysdeps/aarch64/multiarch/memmove.c b/sysdeps/aarch64/multiarch/memmove.c +index ad10aa8ac6..46a4cb3a54 100644 +--- a/sysdeps/aarch64/multiarch/memmove.c ++++ b/sysdeps/aarch64/multiarch/memmove.c +@@ -41,7 +41,8 @@ libc_ifunc (__libc_memmove, + ? __memmove_falkor + : (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr) + ? __memmove_thunderx2 +- : (IS_NEOVERSE_N1 (midr) ++ : (IS_NEOVERSE_N1 (midr) || IS_NEOVERSE_N2 (midr) ++ || IS_NEOVERSE_V1 (midr) + ? __memmove_simd + : __memmove_generic))))); + +diff --git a/sysdeps/aarch64/start.S b/sysdeps/aarch64/start.S +index 75393e1c18..1998ea95d4 100644 +--- a/sysdeps/aarch64/start.S ++++ b/sysdeps/aarch64/start.S +@@ -43,11 +43,9 @@ + */ + + .text +- .globl _start +- .type _start,#function +-_start: +- BTI_C ++ENTRY(_start) + /* Create an initial frame with 0 LR and FP */ ++ cfi_undefined (x30) + mov x29, #0 + mov x30, #0 + +@@ -101,8 +99,10 @@ _start: + because crt1.o and rcrt1.o share code and the later must avoid the + use of GOT relocations before __libc_start_main is called. */ + __wrap_main: ++ BTI_C + b main + #endif ++END(_start) + + /* Define a symbol for the first piece of initialized data. */ + .data +diff --git a/sysdeps/generic/dl-prop.h b/sysdeps/generic/dl-prop.h +index f1cf576fe3..df27ff8e6a 100644 +--- a/sysdeps/generic/dl-prop.h ++++ b/sysdeps/generic/dl-prop.h +@@ -37,15 +37,15 @@ _dl_open_check (struct link_map *m) + } + + static inline void __attribute__ ((always_inline)) +-_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) ++_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) + { + } + + /* Called for each property in the NT_GNU_PROPERTY_TYPE_0 note of L, + processing of the properties continues until this returns 0. */ + static inline int __attribute__ ((always_inline)) +-_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, +- void *data) ++_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, ++ uint32_t datasz, void *data) + { + return 0; + } +diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h +index ba114ab4b1..62ac40d81b 100644 +--- a/sysdeps/generic/ldsodefs.h ++++ b/sysdeps/generic/ldsodefs.h +@@ -919,8 +919,9 @@ extern void _dl_rtld_di_serinfo (struct link_map *loader, + Dl_serinfo *si, bool counting); + + /* Process PT_GNU_PROPERTY program header PH in module L after +- PT_LOAD segments are mapped. */ +-void _dl_process_pt_gnu_property (struct link_map *l, const ElfW(Phdr) *ph); ++ PT_LOAD segments are mapped from file FD. */ ++void _dl_process_pt_gnu_property (struct link_map *l, int fd, ++ const ElfW(Phdr) *ph); + + + /* Search loaded objects' symbol tables for a definition of the symbol +diff --git a/sysdeps/generic/unwind.h b/sysdeps/generic/unwind.h +index b667a5b652..c229603af3 100644 +--- a/sysdeps/generic/unwind.h ++++ b/sysdeps/generic/unwind.h +@@ -75,15 +75,21 @@ typedef void (*_Unwind_Exception_Cleanup_Fn) (_Unwind_Reason_Code, + + struct _Unwind_Exception + { +- _Unwind_Exception_Class exception_class; +- _Unwind_Exception_Cleanup_Fn exception_cleanup; +- _Unwind_Word private_1; +- _Unwind_Word private_2; +- +- /* @@@ The IA-64 ABI says that this structure must be double-word aligned. +- Taking that literally does not make much sense generically. Instead we +- provide the maximum alignment required by any type for the machine. */ +-} __attribute__((__aligned__)); ++ union ++ { ++ struct ++ { ++ _Unwind_Exception_Class exception_class; ++ _Unwind_Exception_Cleanup_Fn exception_cleanup; ++ _Unwind_Word private_1; ++ _Unwind_Word private_2; ++ }; ++ ++ /* The IA-64 ABI says that this structure must be double-word aligned. */ ++ _Unwind_Word unwind_exception_align[2] ++ __attribute__ ((__aligned__ (2 * sizeof (_Unwind_Word)))); ++ }; ++}; + + + /* The ACTIONS argument to the personality routine is a bitwise OR of one +diff --git a/sysdeps/gnu/errlist.h b/sysdeps/gnu/errlist.h +index 5d11ed723d..6329e5f393 100644 +--- a/sysdeps/gnu/errlist.h ++++ b/sysdeps/gnu/errlist.h +@@ -1,24 +1,21 @@ +-#ifndef ERR_MAP +-#define ERR_MAP(value) value +-#endif +-_S(ERR_MAP(0), N_("Success")) ++_S(0, N_("Success")) + #ifdef EPERM + /* + TRANS Only the owner of the file (or other resource) + TRANS or processes with special privileges can perform the operation. */ +-_S(ERR_MAP(EPERM), N_("Operation not permitted")) ++_S(EPERM, N_("Operation not permitted")) + #endif + #ifdef ENOENT + /* + TRANS This is a ``file doesn't exist'' error + TRANS for ordinary files that are referenced in contexts where they are + TRANS expected to already exist. */ +-_S(ERR_MAP(ENOENT), N_("No such file or directory")) ++_S(ENOENT, N_("No such file or directory")) + #endif + #ifdef ESRCH + /* + TRANS No process matches the specified process ID. */ +-_S(ERR_MAP(ESRCH), N_("No such process")) ++_S(ESRCH, N_("No such process")) + #endif + #ifdef EINTR + /* +@@ -29,12 +26,12 @@ TRANS + TRANS You can choose to have functions resume after a signal that is handled, + TRANS rather than failing with @code{EINTR}; see @ref{Interrupted + TRANS Primitives}. */ +-_S(ERR_MAP(EINTR), N_("Interrupted system call")) ++_S(EINTR, N_("Interrupted system call")) + #endif + #ifdef EIO + /* + TRANS Usually used for physical read or write errors. */ +-_S(ERR_MAP(EIO), N_("Input/output error")) ++_S(EIO, N_("Input/output error")) + #endif + #ifdef ENXIO + /* +@@ -43,7 +40,7 @@ TRANS represented by a file you specified, and it couldn't find the device. + TRANS This can mean that the device file was installed incorrectly, or that + TRANS the physical device is missing or not correctly attached to the + TRANS computer. */ +-_S(ERR_MAP(ENXIO), N_("No such device or address")) ++_S(ENXIO, N_("No such device or address")) + #endif + #ifdef E2BIG + /* +@@ -51,27 +48,27 @@ TRANS Used when the arguments passed to a new program + TRANS being executed with one of the @code{exec} functions (@pxref{Executing a + TRANS File}) occupy too much memory space. This condition never arises on + TRANS @gnuhurdsystems{}. */ +-_S(ERR_MAP(E2BIG), N_("Argument list too long")) ++_S(E2BIG, N_("Argument list too long")) + #endif + #ifdef ENOEXEC + /* + TRANS Invalid executable file format. This condition is detected by the + TRANS @code{exec} functions; see @ref{Executing a File}. */ +-_S(ERR_MAP(ENOEXEC), N_("Exec format error")) ++_S(ENOEXEC, N_("Exec format error")) + #endif + #ifdef EBADF + /* + TRANS For example, I/O on a descriptor that has been + TRANS closed or reading from a descriptor open only for writing (or vice + TRANS versa). */ +-_S(ERR_MAP(EBADF), N_("Bad file descriptor")) ++_S(EBADF, N_("Bad file descriptor")) + #endif + #ifdef ECHILD + /* + TRANS This error happens on operations that are + TRANS supposed to manipulate child processes, when there aren't any processes + TRANS to manipulate. */ +-_S(ERR_MAP(ECHILD), N_("No child processes")) ++_S(ECHILD, N_("No child processes")) + #endif + #ifdef EDEADLK + /* +@@ -79,74 +76,74 @@ TRANS Allocating a system resource would have resulted in a + TRANS deadlock situation. The system does not guarantee that it will notice + TRANS all such situations. This error means you got lucky and the system + TRANS noticed; it might just hang. @xref{File Locks}, for an example. */ +-_S(ERR_MAP(EDEADLK), N_("Resource deadlock avoided")) ++_S(EDEADLK, N_("Resource deadlock avoided")) + #endif + #ifdef ENOMEM + /* + TRANS The system cannot allocate more virtual memory + TRANS because its capacity is full. */ +-_S(ERR_MAP(ENOMEM), N_("Cannot allocate memory")) ++_S(ENOMEM, N_("Cannot allocate memory")) + #endif + #ifdef EACCES + /* + TRANS The file permissions do not allow the attempted operation. */ +-_S(ERR_MAP(EACCES), N_("Permission denied")) ++_S(EACCES, N_("Permission denied")) + #endif + #ifdef EFAULT + /* + TRANS An invalid pointer was detected. + TRANS On @gnuhurdsystems{}, this error never happens; you get a signal instead. */ +-_S(ERR_MAP(EFAULT), N_("Bad address")) ++_S(EFAULT, N_("Bad address")) + #endif + #ifdef ENOTBLK + /* + TRANS A file that isn't a block special file was given in a situation that + TRANS requires one. For example, trying to mount an ordinary file as a file + TRANS system in Unix gives this error. */ +-_S(ERR_MAP(ENOTBLK), N_("Block device required")) ++_S(ENOTBLK, N_("Block device required")) + #endif + #ifdef EBUSY + /* + TRANS A system resource that can't be shared is already in use. + TRANS For example, if you try to delete a file that is the root of a currently + TRANS mounted filesystem, you get this error. */ +-_S(ERR_MAP(EBUSY), N_("Device or resource busy")) ++_S(EBUSY, N_("Device or resource busy")) + #endif + #ifdef EEXIST + /* + TRANS An existing file was specified in a context where it only + TRANS makes sense to specify a new file. */ +-_S(ERR_MAP(EEXIST), N_("File exists")) ++_S(EEXIST, N_("File exists")) + #endif + #ifdef EXDEV + /* + TRANS An attempt to make an improper link across file systems was detected. + TRANS This happens not only when you use @code{link} (@pxref{Hard Links}) but + TRANS also when you rename a file with @code{rename} (@pxref{Renaming Files}). */ +-_S(ERR_MAP(EXDEV), N_("Invalid cross-device link")) ++_S(EXDEV, N_("Invalid cross-device link")) + #endif + #ifdef ENODEV + /* + TRANS The wrong type of device was given to a function that expects a + TRANS particular sort of device. */ +-_S(ERR_MAP(ENODEV), N_("No such device")) ++_S(ENODEV, N_("No such device")) + #endif + #ifdef ENOTDIR + /* + TRANS A file that isn't a directory was specified when a directory is required. */ +-_S(ERR_MAP(ENOTDIR), N_("Not a directory")) ++_S(ENOTDIR, N_("Not a directory")) + #endif + #ifdef EISDIR + /* + TRANS You cannot open a directory for writing, + TRANS or create or remove hard links to it. */ +-_S(ERR_MAP(EISDIR), N_("Is a directory")) ++_S(EISDIR, N_("Is a directory")) + #endif + #ifdef EINVAL + /* + TRANS This is used to indicate various kinds of problems + TRANS with passing the wrong argument to a library function. */ +-_S(ERR_MAP(EINVAL), N_("Invalid argument")) ++_S(EINVAL, N_("Invalid argument")) + #endif + #ifdef EMFILE + /* +@@ -157,20 +154,20 @@ TRANS In BSD and GNU, the number of open files is controlled by a resource + TRANS limit that can usually be increased. If you get this error, you might + TRANS want to increase the @code{RLIMIT_NOFILE} limit or make it unlimited; + TRANS @pxref{Limits on Resources}. */ +-_S(ERR_MAP(EMFILE), N_("Too many open files")) ++_S(EMFILE, N_("Too many open files")) + #endif + #ifdef ENFILE + /* + TRANS There are too many distinct file openings in the entire system. Note + TRANS that any number of linked channels count as just one file opening; see + TRANS @ref{Linked Channels}. This error never occurs on @gnuhurdsystems{}. */ +-_S(ERR_MAP(ENFILE), N_("Too many open files in system")) ++_S(ENFILE, N_("Too many open files in system")) + #endif + #ifdef ENOTTY + /* + TRANS Inappropriate I/O control operation, such as trying to set terminal + TRANS modes on an ordinary file. */ +-_S(ERR_MAP(ENOTTY), N_("Inappropriate ioctl for device")) ++_S(ENOTTY, N_("Inappropriate ioctl for device")) + #endif + #ifdef ETXTBSY + /* +@@ -179,35 +176,35 @@ TRANS write to a file that is currently being executed. Often using a + TRANS debugger to run a program is considered having it open for writing and + TRANS will cause this error. (The name stands for ``text file busy''.) This + TRANS is not an error on @gnuhurdsystems{}; the text is copied as necessary. */ +-_S(ERR_MAP(ETXTBSY), N_("Text file busy")) ++_S(ETXTBSY, N_("Text file busy")) + #endif + #ifdef EFBIG + /* + TRANS The size of a file would be larger than allowed by the system. */ +-_S(ERR_MAP(EFBIG), N_("File too large")) ++_S(EFBIG, N_("File too large")) + #endif + #ifdef ENOSPC + /* + TRANS Write operation on a file failed because the + TRANS disk is full. */ +-_S(ERR_MAP(ENOSPC), N_("No space left on device")) ++_S(ENOSPC, N_("No space left on device")) + #endif + #ifdef ESPIPE + /* + TRANS Invalid seek operation (such as on a pipe). */ +-_S(ERR_MAP(ESPIPE), N_("Illegal seek")) ++_S(ESPIPE, N_("Illegal seek")) + #endif + #ifdef EROFS + /* + TRANS An attempt was made to modify something on a read-only file system. */ +-_S(ERR_MAP(EROFS), N_("Read-only file system")) ++_S(EROFS, N_("Read-only file system")) + #endif + #ifdef EMLINK + /* + TRANS The link count of a single file would become too large. + TRANS @code{rename} can cause this error if the file being renamed already has + TRANS as many links as it can take (@pxref{Renaming Files}). */ +-_S(ERR_MAP(EMLINK), N_("Too many links")) ++_S(EMLINK, N_("Too many links")) + #endif + #ifdef EPIPE + /* +@@ -216,19 +213,19 @@ TRANS Every library function that returns this error code also generates a + TRANS @code{SIGPIPE} signal; this signal terminates the program if not handled + TRANS or blocked. Thus, your program will never actually see @code{EPIPE} + TRANS unless it has handled or blocked @code{SIGPIPE}. */ +-_S(ERR_MAP(EPIPE), N_("Broken pipe")) ++_S(EPIPE, N_("Broken pipe")) + #endif + #ifdef EDOM + /* + TRANS Used by mathematical functions when an argument value does + TRANS not fall into the domain over which the function is defined. */ +-_S(ERR_MAP(EDOM), N_("Numerical argument out of domain")) ++_S(EDOM, N_("Numerical argument out of domain")) + #endif + #ifdef ERANGE + /* + TRANS Used by mathematical functions when the result value is + TRANS not representable because of overflow or underflow. */ +-_S(ERR_MAP(ERANGE), N_("Numerical result out of range")) ++_S(ERANGE, N_("Numerical result out of range")) + #endif + #ifdef EAGAIN + /* +@@ -261,7 +258,7 @@ TRANS Such shortages are usually fairly serious and affect the whole system, + TRANS so usually an interactive program should report the error to the user + TRANS and return to its command loop. + TRANS @end itemize */ +-_S(ERR_MAP(EAGAIN), N_("Resource temporarily unavailable")) ++_S(EAGAIN, N_("Resource temporarily unavailable")) + #endif + #ifdef EINPROGRESS + /* +@@ -273,47 +270,47 @@ TRANS the operation has begun and will take some time. Attempts to manipulate + TRANS the object before the call completes return @code{EALREADY}. You can + TRANS use the @code{select} function to find out when the pending operation + TRANS has completed; @pxref{Waiting for I/O}. */ +-_S(ERR_MAP(EINPROGRESS), N_("Operation now in progress")) ++_S(EINPROGRESS, N_("Operation now in progress")) + #endif + #ifdef EALREADY + /* + TRANS An operation is already in progress on an object that has non-blocking + TRANS mode selected. */ +-_S(ERR_MAP(EALREADY), N_("Operation already in progress")) ++_S(EALREADY, N_("Operation already in progress")) + #endif + #ifdef ENOTSOCK + /* + TRANS A file that isn't a socket was specified when a socket is required. */ +-_S(ERR_MAP(ENOTSOCK), N_("Socket operation on non-socket")) ++_S(ENOTSOCK, N_("Socket operation on non-socket")) + #endif + #ifdef EMSGSIZE + /* + TRANS The size of a message sent on a socket was larger than the supported + TRANS maximum size. */ +-_S(ERR_MAP(EMSGSIZE), N_("Message too long")) ++_S(EMSGSIZE, N_("Message too long")) + #endif + #ifdef EPROTOTYPE + /* + TRANS The socket type does not support the requested communications protocol. */ +-_S(ERR_MAP(EPROTOTYPE), N_("Protocol wrong type for socket")) ++_S(EPROTOTYPE, N_("Protocol wrong type for socket")) + #endif + #ifdef ENOPROTOOPT + /* + TRANS You specified a socket option that doesn't make sense for the + TRANS particular protocol being used by the socket. @xref{Socket Options}. */ +-_S(ERR_MAP(ENOPROTOOPT), N_("Protocol not available")) ++_S(ENOPROTOOPT, N_("Protocol not available")) + #endif + #ifdef EPROTONOSUPPORT + /* + TRANS The socket domain does not support the requested communications protocol + TRANS (perhaps because the requested protocol is completely invalid). + TRANS @xref{Creating a Socket}. */ +-_S(ERR_MAP(EPROTONOSUPPORT), N_("Protocol not supported")) ++_S(EPROTONOSUPPORT, N_("Protocol not supported")) + #endif + #ifdef ESOCKTNOSUPPORT + /* + TRANS The socket type is not supported. */ +-_S(ERR_MAP(ESOCKTNOSUPPORT), N_("Socket type not supported")) ++_S(ESOCKTNOSUPPORT, N_("Socket type not supported")) + #endif + #ifdef EOPNOTSUPP + /* +@@ -323,71 +320,71 @@ TRANS implemented for all communications protocols. On @gnuhurdsystems{}, this + TRANS error can happen for many calls when the object does not support the + TRANS particular operation; it is a generic indication that the server knows + TRANS nothing to do for that call. */ +-_S(ERR_MAP(EOPNOTSUPP), N_("Operation not supported")) ++_S(EOPNOTSUPP, N_("Operation not supported")) + #endif + #ifdef EPFNOSUPPORT + /* + TRANS The socket communications protocol family you requested is not supported. */ +-_S(ERR_MAP(EPFNOSUPPORT), N_("Protocol family not supported")) ++_S(EPFNOSUPPORT, N_("Protocol family not supported")) + #endif + #ifdef EAFNOSUPPORT + /* + TRANS The address family specified for a socket is not supported; it is + TRANS inconsistent with the protocol being used on the socket. @xref{Sockets}. */ +-_S(ERR_MAP(EAFNOSUPPORT), N_("Address family not supported by protocol")) ++_S(EAFNOSUPPORT, N_("Address family not supported by protocol")) + #endif + #ifdef EADDRINUSE + /* + TRANS The requested socket address is already in use. @xref{Socket Addresses}. */ +-_S(ERR_MAP(EADDRINUSE), N_("Address already in use")) ++_S(EADDRINUSE, N_("Address already in use")) + #endif + #ifdef EADDRNOTAVAIL + /* + TRANS The requested socket address is not available; for example, you tried + TRANS to give a socket a name that doesn't match the local host name. + TRANS @xref{Socket Addresses}. */ +-_S(ERR_MAP(EADDRNOTAVAIL), N_("Cannot assign requested address")) ++_S(EADDRNOTAVAIL, N_("Cannot assign requested address")) + #endif + #ifdef ENETDOWN + /* + TRANS A socket operation failed because the network was down. */ +-_S(ERR_MAP(ENETDOWN), N_("Network is down")) ++_S(ENETDOWN, N_("Network is down")) + #endif + #ifdef ENETUNREACH + /* + TRANS A socket operation failed because the subnet containing the remote host + TRANS was unreachable. */ +-_S(ERR_MAP(ENETUNREACH), N_("Network is unreachable")) ++_S(ENETUNREACH, N_("Network is unreachable")) + #endif + #ifdef ENETRESET + /* + TRANS A network connection was reset because the remote host crashed. */ +-_S(ERR_MAP(ENETRESET), N_("Network dropped connection on reset")) ++_S(ENETRESET, N_("Network dropped connection on reset")) + #endif + #ifdef ECONNABORTED + /* + TRANS A network connection was aborted locally. */ +-_S(ERR_MAP(ECONNABORTED), N_("Software caused connection abort")) ++_S(ECONNABORTED, N_("Software caused connection abort")) + #endif + #ifdef ECONNRESET + /* + TRANS A network connection was closed for reasons outside the control of the + TRANS local host, such as by the remote machine rebooting or an unrecoverable + TRANS protocol violation. */ +-_S(ERR_MAP(ECONNRESET), N_("Connection reset by peer")) ++_S(ECONNRESET, N_("Connection reset by peer")) + #endif + #ifdef ENOBUFS + /* + TRANS The kernel's buffers for I/O operations are all in use. In GNU, this + TRANS error is always synonymous with @code{ENOMEM}; you may get one or the + TRANS other from network operations. */ +-_S(ERR_MAP(ENOBUFS), N_("No buffer space available")) ++_S(ENOBUFS, N_("No buffer space available")) + #endif + #ifdef EISCONN + /* + TRANS You tried to connect a socket that is already connected. + TRANS @xref{Connecting}. */ +-_S(ERR_MAP(EISCONN), N_("Transport endpoint is already connected")) ++_S(EISCONN, N_("Transport endpoint is already connected")) + #endif + #ifdef ENOTCONN + /* +@@ -395,74 +392,74 @@ TRANS The socket is not connected to anything. You get this error when you + TRANS try to transmit data over a socket, without first specifying a + TRANS destination for the data. For a connectionless socket (for datagram + TRANS protocols, such as UDP), you get @code{EDESTADDRREQ} instead. */ +-_S(ERR_MAP(ENOTCONN), N_("Transport endpoint is not connected")) ++_S(ENOTCONN, N_("Transport endpoint is not connected")) + #endif + #ifdef EDESTADDRREQ + /* + TRANS No default destination address was set for the socket. You get this + TRANS error when you try to transmit data over a connectionless socket, + TRANS without first specifying a destination for the data with @code{connect}. */ +-_S(ERR_MAP(EDESTADDRREQ), N_("Destination address required")) ++_S(EDESTADDRREQ, N_("Destination address required")) + #endif + #ifdef ESHUTDOWN + /* + TRANS The socket has already been shut down. */ +-_S(ERR_MAP(ESHUTDOWN), N_("Cannot send after transport endpoint shutdown")) ++_S(ESHUTDOWN, N_("Cannot send after transport endpoint shutdown")) + #endif + #ifdef ETOOMANYREFS +-_S(ERR_MAP(ETOOMANYREFS), N_("Too many references: cannot splice")) ++_S(ETOOMANYREFS, N_("Too many references: cannot splice")) + #endif + #ifdef ETIMEDOUT + /* + TRANS A socket operation with a specified timeout received no response during + TRANS the timeout period. */ +-_S(ERR_MAP(ETIMEDOUT), N_("Connection timed out")) ++_S(ETIMEDOUT, N_("Connection timed out")) + #endif + #ifdef ECONNREFUSED + /* + TRANS A remote host refused to allow the network connection (typically because + TRANS it is not running the requested service). */ +-_S(ERR_MAP(ECONNREFUSED), N_("Connection refused")) ++_S(ECONNREFUSED, N_("Connection refused")) + #endif + #ifdef ELOOP + /* + TRANS Too many levels of symbolic links were encountered in looking up a file name. + TRANS This often indicates a cycle of symbolic links. */ +-_S(ERR_MAP(ELOOP), N_("Too many levels of symbolic links")) ++_S(ELOOP, N_("Too many levels of symbolic links")) + #endif + #ifdef ENAMETOOLONG + /* + TRANS Filename too long (longer than @code{PATH_MAX}; @pxref{Limits for + TRANS Files}) or host name too long (in @code{gethostname} or + TRANS @code{sethostname}; @pxref{Host Identification}). */ +-_S(ERR_MAP(ENAMETOOLONG), N_("File name too long")) ++_S(ENAMETOOLONG, N_("File name too long")) + #endif + #ifdef EHOSTDOWN + /* + TRANS The remote host for a requested network connection is down. */ +-_S(ERR_MAP(EHOSTDOWN), N_("Host is down")) ++_S(EHOSTDOWN, N_("Host is down")) + #endif + /* + TRANS The remote host for a requested network connection is not reachable. */ + #ifdef EHOSTUNREACH +-_S(ERR_MAP(EHOSTUNREACH), N_("No route to host")) ++_S(EHOSTUNREACH, N_("No route to host")) + #endif + #ifdef ENOTEMPTY + /* + TRANS Directory not empty, where an empty directory was expected. Typically, + TRANS this error occurs when you are trying to delete a directory. */ +-_S(ERR_MAP(ENOTEMPTY), N_("Directory not empty")) ++_S(ENOTEMPTY, N_("Directory not empty")) + #endif + #ifdef EUSERS + /* + TRANS The file quota system is confused because there are too many users. + TRANS @c This can probably happen in a GNU system when using NFS. */ +-_S(ERR_MAP(EUSERS), N_("Too many users")) ++_S(EUSERS, N_("Too many users")) + #endif + #ifdef EDQUOT + /* + TRANS The user's disk quota was exceeded. */ +-_S(ERR_MAP(EDQUOT), N_("Disk quota exceeded")) ++_S(EDQUOT, N_("Disk quota exceeded")) + #endif + #ifdef ESTALE + /* +@@ -471,7 +468,7 @@ TRANS file system which is due to file system rearrangements on the server host + TRANS for NFS file systems or corruption in other file systems. + TRANS Repairing this condition usually requires unmounting, possibly repairing + TRANS and remounting the file system. */ +-_S(ERR_MAP(ESTALE), N_("Stale file handle")) ++_S(ESTALE, N_("Stale file handle")) + #endif + #ifdef EREMOTE + /* +@@ -479,7 +476,7 @@ TRANS An attempt was made to NFS-mount a remote file system with a file name tha + TRANS already specifies an NFS-mounted file. + TRANS (This is an error on some operating systems, but we expect it to work + TRANS properly on @gnuhurdsystems{}, making this error code impossible.) */ +-_S(ERR_MAP(EREMOTE), N_("Object is remote")) ++_S(EREMOTE, N_("Object is remote")) + #endif + #ifdef ENOLCK + /* +@@ -487,7 +484,7 @@ TRANS This is used by the file locking facilities; see + TRANS @ref{File Locks}. This error is never generated by @gnuhurdsystems{}, but + TRANS it can result from an operation to an NFS server running another + TRANS operating system. */ +-_S(ERR_MAP(ENOLCK), N_("No locks available")) ++_S(ENOLCK, N_("No locks available")) + #endif + #ifdef ENOSYS + /* +@@ -496,46 +493,46 @@ TRANS not implemented at all, either in the C library itself or in the + TRANS operating system. When you get this error, you can be sure that this + TRANS particular function will always fail with @code{ENOSYS} unless you + TRANS install a new version of the C library or the operating system. */ +-_S(ERR_MAP(ENOSYS), N_("Function not implemented")) ++_S(ENOSYS, N_("Function not implemented")) + #endif + #ifdef EILSEQ + /* + TRANS While decoding a multibyte character the function came along an invalid + TRANS or an incomplete sequence of bytes or the given wide character is invalid. */ +-_S(ERR_MAP(EILSEQ), N_("Invalid or incomplete multibyte or wide character")) ++_S(EILSEQ, N_("Invalid or incomplete multibyte or wide character")) + #endif + #ifdef EBADMSG +-_S(ERR_MAP(EBADMSG), N_("Bad message")) ++_S(EBADMSG, N_("Bad message")) + #endif + #ifdef EIDRM +-_S(ERR_MAP(EIDRM), N_("Identifier removed")) ++_S(EIDRM, N_("Identifier removed")) + #endif + #ifdef EMULTIHOP +-_S(ERR_MAP(EMULTIHOP), N_("Multihop attempted")) ++_S(EMULTIHOP, N_("Multihop attempted")) + #endif + #ifdef ENODATA +-_S(ERR_MAP(ENODATA), N_("No data available")) ++_S(ENODATA, N_("No data available")) + #endif + #ifdef ENOLINK +-_S(ERR_MAP(ENOLINK), N_("Link has been severed")) ++_S(ENOLINK, N_("Link has been severed")) + #endif + #ifdef ENOMSG +-_S(ERR_MAP(ENOMSG), N_("No message of desired type")) ++_S(ENOMSG, N_("No message of desired type")) + #endif + #ifdef ENOSR +-_S(ERR_MAP(ENOSR), N_("Out of streams resources")) ++_S(ENOSR, N_("Out of streams resources")) + #endif + #ifdef ENOSTR +-_S(ERR_MAP(ENOSTR), N_("Device not a stream")) ++_S(ENOSTR, N_("Device not a stream")) + #endif + #ifdef EOVERFLOW +-_S(ERR_MAP(EOVERFLOW), N_("Value too large for defined data type")) ++_S(EOVERFLOW, N_("Value too large for defined data type")) + #endif + #ifdef EPROTO +-_S(ERR_MAP(EPROTO), N_("Protocol error")) ++_S(EPROTO, N_("Protocol error")) + #endif + #ifdef ETIME +-_S(ERR_MAP(ETIME), N_("Timer expired")) ++_S(ETIME, N_("Timer expired")) + #endif + #ifdef ECANCELED + /* +@@ -543,148 +540,148 @@ TRANS An asynchronous operation was canceled before it + TRANS completed. @xref{Asynchronous I/O}. When you call @code{aio_cancel}, + TRANS the normal result is for the operations affected to complete with this + TRANS error; @pxref{Cancel AIO Operations}. */ +-_S(ERR_MAP(ECANCELED), N_("Operation canceled")) ++_S(ECANCELED, N_("Operation canceled")) + #endif + #ifdef EOWNERDEAD +-_S(ERR_MAP(EOWNERDEAD), N_("Owner died")) ++_S(EOWNERDEAD, N_("Owner died")) + #endif + #ifdef ENOTRECOVERABLE +-_S(ERR_MAP(ENOTRECOVERABLE), N_("State not recoverable")) ++_S(ENOTRECOVERABLE, N_("State not recoverable")) + #endif + #ifdef ERESTART +-_S(ERR_MAP(ERESTART), N_("Interrupted system call should be restarted")) ++_S(ERESTART, N_("Interrupted system call should be restarted")) + #endif + #ifdef ECHRNG +-_S(ERR_MAP(ECHRNG), N_("Channel number out of range")) ++_S(ECHRNG, N_("Channel number out of range")) + #endif + #ifdef EL2NSYNC +-_S(ERR_MAP(EL2NSYNC), N_("Level 2 not synchronized")) ++_S(EL2NSYNC, N_("Level 2 not synchronized")) + #endif + #ifdef EL3HLT +-_S(ERR_MAP(EL3HLT), N_("Level 3 halted")) ++_S(EL3HLT, N_("Level 3 halted")) + #endif + #ifdef EL3RST +-_S(ERR_MAP(EL3RST), N_("Level 3 reset")) ++_S(EL3RST, N_("Level 3 reset")) + #endif + #ifdef ELNRNG +-_S(ERR_MAP(ELNRNG), N_("Link number out of range")) ++_S(ELNRNG, N_("Link number out of range")) + #endif + #ifdef EUNATCH +-_S(ERR_MAP(EUNATCH), N_("Protocol driver not attached")) ++_S(EUNATCH, N_("Protocol driver not attached")) + #endif + #ifdef ENOCSI +-_S(ERR_MAP(ENOCSI), N_("No CSI structure available")) ++_S(ENOCSI, N_("No CSI structure available")) + #endif + #ifdef EL2HLT +-_S(ERR_MAP(EL2HLT), N_("Level 2 halted")) ++_S(EL2HLT, N_("Level 2 halted")) + #endif + #ifdef EBADE +-_S(ERR_MAP(EBADE), N_("Invalid exchange")) ++_S(EBADE, N_("Invalid exchange")) + #endif + #ifdef EBADR +-_S(ERR_MAP(EBADR), N_("Invalid request descriptor")) ++_S(EBADR, N_("Invalid request descriptor")) + #endif + #ifdef EXFULL +-_S(ERR_MAP(EXFULL), N_("Exchange full")) ++_S(EXFULL, N_("Exchange full")) + #endif + #ifdef ENOANO +-_S(ERR_MAP(ENOANO), N_("No anode")) ++_S(ENOANO, N_("No anode")) + #endif + #ifdef EBADRQC +-_S(ERR_MAP(EBADRQC), N_("Invalid request code")) ++_S(EBADRQC, N_("Invalid request code")) + #endif + #ifdef EBADSLT +-_S(ERR_MAP(EBADSLT), N_("Invalid slot")) ++_S(EBADSLT, N_("Invalid slot")) + #endif + #ifdef EBFONT +-_S(ERR_MAP(EBFONT), N_("Bad font file format")) ++_S(EBFONT, N_("Bad font file format")) + #endif + #ifdef ENONET +-_S(ERR_MAP(ENONET), N_("Machine is not on the network")) ++_S(ENONET, N_("Machine is not on the network")) + #endif + #ifdef ENOPKG +-_S(ERR_MAP(ENOPKG), N_("Package not installed")) ++_S(ENOPKG, N_("Package not installed")) + #endif + #ifdef EADV +-_S(ERR_MAP(EADV), N_("Advertise error")) ++_S(EADV, N_("Advertise error")) + #endif + #ifdef ESRMNT +-_S(ERR_MAP(ESRMNT), N_("Srmount error")) ++_S(ESRMNT, N_("Srmount error")) + #endif + #ifdef ECOMM +-_S(ERR_MAP(ECOMM), N_("Communication error on send")) ++_S(ECOMM, N_("Communication error on send")) + #endif + #ifdef EDOTDOT +-_S(ERR_MAP(EDOTDOT), N_("RFS specific error")) ++_S(EDOTDOT, N_("RFS specific error")) + #endif + #ifdef ENOTUNIQ +-_S(ERR_MAP(ENOTUNIQ), N_("Name not unique on network")) ++_S(ENOTUNIQ, N_("Name not unique on network")) + #endif + #ifdef EBADFD +-_S(ERR_MAP(EBADFD), N_("File descriptor in bad state")) ++_S(EBADFD, N_("File descriptor in bad state")) + #endif + #ifdef EREMCHG +-_S(ERR_MAP(EREMCHG), N_("Remote address changed")) ++_S(EREMCHG, N_("Remote address changed")) + #endif + #ifdef ELIBACC +-_S(ERR_MAP(ELIBACC), N_("Can not access a needed shared library")) ++_S(ELIBACC, N_("Can not access a needed shared library")) + #endif + #ifdef ELIBBAD +-_S(ERR_MAP(ELIBBAD), N_("Accessing a corrupted shared library")) ++_S(ELIBBAD, N_("Accessing a corrupted shared library")) + #endif + #ifdef ELIBSCN +-_S(ERR_MAP(ELIBSCN), N_(".lib section in a.out corrupted")) ++_S(ELIBSCN, N_(".lib section in a.out corrupted")) + #endif + #ifdef ELIBMAX +-_S(ERR_MAP(ELIBMAX), N_("Attempting to link in too many shared libraries")) ++_S(ELIBMAX, N_("Attempting to link in too many shared libraries")) + #endif + #ifdef ELIBEXEC +-_S(ERR_MAP(ELIBEXEC), N_("Cannot exec a shared library directly")) ++_S(ELIBEXEC, N_("Cannot exec a shared library directly")) + #endif + #ifdef ESTRPIPE +-_S(ERR_MAP(ESTRPIPE), N_("Streams pipe error")) ++_S(ESTRPIPE, N_("Streams pipe error")) + #endif + #ifdef EUCLEAN +-_S(ERR_MAP(EUCLEAN), N_("Structure needs cleaning")) ++_S(EUCLEAN, N_("Structure needs cleaning")) + #endif + #ifdef ENOTNAM +-_S(ERR_MAP(ENOTNAM), N_("Not a XENIX named type file")) ++_S(ENOTNAM, N_("Not a XENIX named type file")) + #endif + #ifdef ENAVAIL +-_S(ERR_MAP(ENAVAIL), N_("No XENIX semaphores available")) ++_S(ENAVAIL, N_("No XENIX semaphores available")) + #endif + #ifdef EISNAM +-_S(ERR_MAP(EISNAM), N_("Is a named type file")) ++_S(EISNAM, N_("Is a named type file")) + #endif + #ifdef EREMOTEIO +-_S(ERR_MAP(EREMOTEIO), N_("Remote I/O error")) ++_S(EREMOTEIO, N_("Remote I/O error")) + #endif + #ifdef ENOMEDIUM +-_S(ERR_MAP(ENOMEDIUM), N_("No medium found")) ++_S(ENOMEDIUM, N_("No medium found")) + #endif + #ifdef EMEDIUMTYPE +-_S(ERR_MAP(EMEDIUMTYPE), N_("Wrong medium type")) ++_S(EMEDIUMTYPE, N_("Wrong medium type")) + #endif + #ifdef ENOKEY +-_S(ERR_MAP(ENOKEY), N_("Required key not available")) ++_S(ENOKEY, N_("Required key not available")) + #endif + #ifdef EKEYEXPIRED +-_S(ERR_MAP(EKEYEXPIRED), N_("Key has expired")) ++_S(EKEYEXPIRED, N_("Key has expired")) + #endif + #ifdef EKEYREVOKED +-_S(ERR_MAP(EKEYREVOKED), N_("Key has been revoked")) ++_S(EKEYREVOKED, N_("Key has been revoked")) + #endif + #ifdef EKEYREJECTED +-_S(ERR_MAP(EKEYREJECTED), N_("Key was rejected by service")) ++_S(EKEYREJECTED, N_("Key was rejected by service")) + #endif + #ifdef ERFKILL +-_S(ERR_MAP(ERFKILL), N_("Operation not possible due to RF-kill")) ++_S(ERFKILL, N_("Operation not possible due to RF-kill")) + #endif + #ifdef EHWPOISON +-_S(ERR_MAP(EHWPOISON), N_("Memory page has hardware error")) ++_S(EHWPOISON, N_("Memory page has hardware error")) + #endif + #ifdef EBADRPC +-_S(ERR_MAP(EBADRPC), N_("RPC struct is bad")) ++_S(EBADRPC, N_("RPC struct is bad")) + #endif + #ifdef EFTYPE + /* +@@ -693,40 +690,40 @@ TRANS operation, or a data file had the wrong format. + TRANS + TRANS On some systems @code{chmod} returns this error if you try to set the + TRANS sticky bit on a non-directory file; @pxref{Setting Permissions}. */ +-_S(ERR_MAP(EFTYPE), N_("Inappropriate file type or format")) ++_S(EFTYPE, N_("Inappropriate file type or format")) + #endif + #ifdef EPROCUNAVAIL +-_S(ERR_MAP(EPROCUNAVAIL), N_("RPC bad procedure for program")) ++_S(EPROCUNAVAIL, N_("RPC bad procedure for program")) + #endif + #ifdef EAUTH +-_S(ERR_MAP(EAUTH), N_("Authentication error")) ++_S(EAUTH, N_("Authentication error")) + #endif + #ifdef EDIED + /* + TRANS On @gnuhurdsystems{}, opening a file returns this error when the file is + TRANS translated by a program and the translator program dies while starting + TRANS up, before it has connected to the file. */ +-_S(ERR_MAP(EDIED), N_("Translator died")) ++_S(EDIED, N_("Translator died")) + #endif + #ifdef ERPCMISMATCH +-_S(ERR_MAP(ERPCMISMATCH), N_("RPC version wrong")) ++_S(ERPCMISMATCH, N_("RPC version wrong")) + #endif + #ifdef EGREGIOUS + /* + TRANS You did @strong{what}? */ +-_S(ERR_MAP(EGREGIOUS), N_("You really blew it this time")) ++_S(EGREGIOUS, N_("You really blew it this time")) + #endif + #ifdef EPROCLIM + /* + TRANS This means that the per-user limit on new process would be exceeded by + TRANS an attempted @code{fork}. @xref{Limits on Resources}, for details on + TRANS the @code{RLIMIT_NPROC} limit. */ +-_S(ERR_MAP(EPROCLIM), N_("Too many processes")) ++_S(EPROCLIM, N_("Too many processes")) + #endif + #ifdef EGRATUITOUS + /* + TRANS This error code has no purpose. */ +-_S(ERR_MAP(EGRATUITOUS), N_("Gratuitous error")) ++_S(EGRATUITOUS, N_("Gratuitous error")) + #endif + #if defined (ENOTSUP) && ENOTSUP != EOPNOTSUPP + /* +@@ -742,10 +739,10 @@ TRANS values. + TRANS + TRANS If the entire function is not available at all in the implementation, + TRANS it returns @code{ENOSYS} instead. */ +-_S(ERR_MAP(ENOTSUP), N_("Not supported")) ++_S(ENOTSUP, N_("Not supported")) + #endif + #ifdef EPROGMISMATCH +-_S(ERR_MAP(EPROGMISMATCH), N_("RPC program version wrong")) ++_S(EPROGMISMATCH, N_("RPC program version wrong")) + #endif + #ifdef EBACKGROUND + /* +@@ -755,7 +752,7 @@ TRANS foreground process group of the terminal. Users do not usually see this + TRANS error because functions such as @code{read} and @code{write} translate + TRANS it into a @code{SIGTTIN} or @code{SIGTTOU} signal. @xref{Job Control}, + TRANS for information on process groups and these signals. */ +-_S(ERR_MAP(EBACKGROUND), N_("Inappropriate operation for background process")) ++_S(EBACKGROUND, N_("Inappropriate operation for background process")) + #endif + #ifdef EIEIO + /* +@@ -773,7 +770,7 @@ TRANS @c "bought the farm" means "died". -jtobey + TRANS @c + TRANS @c Translators, please do not translate this litteraly, translate it into + TRANS @c an idiomatic funny way of saying that the computer died. */ +-_S(ERR_MAP(EIEIO), N_("Computer bought the farm")) ++_S(EIEIO, N_("Computer bought the farm")) + #endif + #if defined (EWOULDBLOCK) && EWOULDBLOCK != EAGAIN + /* +@@ -782,18 +779,18 @@ TRANS The values are always the same, on every operating system. + TRANS + TRANS C libraries in many older Unix systems have @code{EWOULDBLOCK} as a + TRANS separate error code. */ +-_S(ERR_MAP(EWOULDBLOCK), N_("Operation would block")) ++_S(EWOULDBLOCK, N_("Operation would block")) + #endif + #ifdef ENEEDAUTH +-_S(ERR_MAP(ENEEDAUTH), N_("Need authenticator")) ++_S(ENEEDAUTH, N_("Need authenticator")) + #endif + #ifdef ED + /* + TRANS The experienced user will know what is wrong. + TRANS @c This error code is a joke. Its perror text is part of the joke. + TRANS @c Don't change it. */ +-_S(ERR_MAP(ED), N_("?")) ++_S(ED, N_("?")) + #endif + #ifdef EPROGUNAVAIL +-_S(ERR_MAP(EPROGUNAVAIL), N_("RPC program not available")) ++_S(EPROGUNAVAIL, N_("RPC program not available")) + #endif +diff --git a/sysdeps/i386/dl-machine.h b/sysdeps/i386/dl-machine.h +index 0f08079e48..672d8f27ce 100644 +--- a/sysdeps/i386/dl-machine.h ++++ b/sysdeps/i386/dl-machine.h +@@ -338,16 +338,22 @@ elf_machine_rel (struct link_map *map, const Elf32_Rel *reloc, + { + # ifndef RTLD_BOOTSTRAP + if (sym_map != map +- && sym_map->l_type != lt_executable + && !sym_map->l_relocated) + { + const char *strtab + = (const char *) D_PTR (map, l_info[DT_STRTAB]); +- _dl_error_printf ("\ ++ if (sym_map->l_type == lt_executable) ++ _dl_fatal_printf ("\ ++%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \ ++and creates an unsatisfiable circular dependency.\n", ++ RTLD_PROGNAME, strtab + refsym->st_name, ++ map->l_name); ++ else ++ _dl_error_printf ("\ + %s: Relink `%s' with `%s' for IFUNC symbol `%s'\n", +- RTLD_PROGNAME, map->l_name, +- sym_map->l_name, +- strtab + refsym->st_name); ++ RTLD_PROGNAME, map->l_name, ++ sym_map->l_name, ++ strtab + refsym->st_name); + } + # endif + value = ((Elf32_Addr (*) (void)) value) (); +diff --git a/sysdeps/powerpc/powerpc64/backtrace.c b/sysdeps/powerpc/powerpc64/backtrace.c +index 8a53a1088f..362a2b713c 100644 +--- a/sysdeps/powerpc/powerpc64/backtrace.c ++++ b/sysdeps/powerpc/powerpc64/backtrace.c +@@ -54,11 +54,22 @@ struct signal_frame_64 { + /* We don't care about the rest, since the IP value is at 'uc' field. */ + }; + ++/* Test if the address match to the inside the trampoline code. ++ Up to and including kernel 5.8, returning from an interrupt or syscall to a ++ signal handler starts execution directly at the handler's entry point, with ++ LR set to address of the sigreturn trampoline (the vDSO symbol). ++ Newer kernels will branch to signal handler from the trampoline instead, so ++ checking the stacktrace against the vDSO entrypoint does not work in such ++ case. ++ The vDSO branches with a 'bctrl' instruction, so checking either the ++ vDSO address itself and the next instruction should cover all kernel ++ versions. */ + static inline bool + is_sigtramp_address (void *nip) + { + #ifdef HAVE_SIGTRAMP_RT64 +- if (nip == GLRO (dl_vdso_sigtramp_rt64)) ++ if (nip == GLRO (dl_vdso_sigtramp_rt64) || ++ nip == GLRO (dl_vdso_sigtramp_rt64) + 4) + return true; + #endif + return false; +diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile +index 920d875420..bf9b7f7223 100644 +--- a/sysdeps/pthread/Makefile ++++ b/sysdeps/pthread/Makefile +@@ -107,6 +107,7 @@ tests += tst-cnd-basic tst-mtx-trylock tst-cnd-broadcast \ + tst-unload \ + tst-unwind-thread \ + tst-pt-vfork1 tst-pt-vfork2 tst-vfork1x tst-vfork2x \ ++ tst-pthread-exit-signal \ + + + # Files which must not be linked with libpthread. +diff --git a/sysdeps/pthread/tst-pthread-exit-signal.c b/sysdeps/pthread/tst-pthread-exit-signal.c +new file mode 100644 +index 0000000000..b4526fe663 +--- /dev/null ++++ b/sysdeps/pthread/tst-pthread-exit-signal.c +@@ -0,0 +1,45 @@ ++/* Test that pending signals are not delivered on thread exit (bug 28607). ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++/* Due to bug 28607, pthread_kill (or pthread_cancel) restored the ++ signal mask during during thread exit, triggering the delivery of a ++ blocked pending signal (SIGUSR1 in this test). */ ++ ++#include <support/xthread.h> ++#include <support/xsignal.h> ++ ++static void * ++threadfunc (void *closure) ++{ ++ sigset_t sigmask; ++ sigfillset (&sigmask); ++ xpthread_sigmask (SIG_SETMASK, &sigmask, NULL); ++ xpthread_kill (pthread_self (), SIGUSR1); ++ pthread_exit (NULL); ++ return NULL; ++} ++ ++static int ++do_test (void) ++{ ++ pthread_t thr = xpthread_create (NULL, threadfunc, NULL); ++ xpthread_join (thr); ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/s390/configure b/sysdeps/s390/configure +index fa46e9e351..e7f576338d 100644 +--- a/sysdeps/s390/configure ++++ b/sysdeps/s390/configure +@@ -123,7 +123,9 @@ void testinsn (char *buf) + __asm__ (".machine \"arch13\" \n\t" + ".machinemode \"zarch_nohighgprs\" \n\t" + "lghi %%r0,16 \n\t" +- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); ++ "mvcrl 0(%0),32(%0) \n\t" ++ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" ++ : : "a" (buf) : "memory", "r0"); + } + EOF + if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS --shared conftest.c +@@ -271,7 +273,9 @@ else + void testinsn (char *buf) + { + __asm__ ("lghi %%r0,16 \n\t" +- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); ++ "mvcrl 0(%0),32(%0) \n\t" ++ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" ++ : : "a" (buf) : "memory", "r0"); + } + EOF + if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS --shared conftest.c +diff --git a/sysdeps/s390/configure.ac b/sysdeps/s390/configure.ac +index 3ed5a8ef87..5c3479e8cf 100644 +--- a/sysdeps/s390/configure.ac ++++ b/sysdeps/s390/configure.ac +@@ -88,7 +88,9 @@ void testinsn (char *buf) + __asm__ (".machine \"arch13\" \n\t" + ".machinemode \"zarch_nohighgprs\" \n\t" + "lghi %%r0,16 \n\t" +- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); ++ "mvcrl 0(%0),32(%0) \n\t" ++ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" ++ : : "a" (buf) : "memory", "r0"); + } + EOF + dnl test, if assembler supports S390 arch13 instructions +@@ -195,7 +197,9 @@ cat > conftest.c <<\EOF + void testinsn (char *buf) + { + __asm__ ("lghi %%r0,16 \n\t" +- "mvcrl 0(%0),32(%0)" : : "a" (buf) : "memory", "r0"); ++ "mvcrl 0(%0),32(%0) \n\t" ++ "vstrs %%v20,%%v20,%%v20,%%v20,0,2" ++ : : "a" (buf) : "memory", "r0"); + } + EOF + dnl test, if assembler supports S390 arch13 zarch instructions as default +diff --git a/sysdeps/s390/memmove.c b/sysdeps/s390/memmove.c +index 5fc85e129f..ee59b5de14 100644 +--- a/sysdeps/s390/memmove.c ++++ b/sysdeps/s390/memmove.c +@@ -43,7 +43,7 @@ extern __typeof (__redirect_memmove) MEMMOVE_ARCH13 attribute_hidden; + s390_libc_ifunc_expr (__redirect_memmove, memmove, + ({ + s390_libc_ifunc_expr_stfle_init (); +- (HAVE_MEMMOVE_ARCH13 ++ (HAVE_MEMMOVE_ARCH13 && (hwcap & HWCAP_S390_VXRS_EXT2) + && S390_IS_ARCH13_MIE3 (stfle_bits)) + ? MEMMOVE_ARCH13 + : (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX)) +diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c +index e6195c6e26..17c0cc3952 100644 +--- a/sysdeps/s390/multiarch/ifunc-impl-list.c ++++ b/sysdeps/s390/multiarch/ifunc-impl-list.c +@@ -171,7 +171,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, memmove, + # if HAVE_MEMMOVE_ARCH13 + IFUNC_IMPL_ADD (array, i, memmove, +- S390_IS_ARCH13_MIE3 (stfle_bits), ++ ((dl_hwcap & HWCAP_S390_VXRS_EXT2) ++ && S390_IS_ARCH13_MIE3 (stfle_bits)), + MEMMOVE_ARCH13) + # endif + # if HAVE_MEMMOVE_Z13 +diff --git a/sysdeps/sh/be/sh4/fpu/Implies b/sysdeps/sh/be/sh4/fpu/Implies +new file mode 100644 +index 0000000000..71b28ee1a4 +--- /dev/null ++++ b/sysdeps/sh/be/sh4/fpu/Implies +@@ -0,0 +1 @@ ++sh/sh4/fpu +diff --git a/sysdeps/sh/le/sh4/fpu/Implies b/sysdeps/sh/le/sh4/fpu/Implies +new file mode 100644 +index 0000000000..71b28ee1a4 +--- /dev/null ++++ b/sysdeps/sh/le/sh4/fpu/Implies +@@ -0,0 +1 @@ ++sh/sh4/fpu +diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile +index 9b2a253032..34748ffcd1 100644 +--- a/sysdeps/unix/sysv/linux/Makefile ++++ b/sysdeps/unix/sysv/linux/Makefile +@@ -100,7 +100,7 @@ tests += tst-clone tst-clone2 tst-clone3 tst-fanotify tst-personality \ + tst-quota tst-sync_file_range tst-sysconf-iov_max tst-ttyname \ + test-errno-linux tst-memfd_create tst-mlock2 tst-pkey \ + tst-rlimit-infinity tst-ofdlocks tst-gettid tst-gettid-kill \ +- tst-tgkill ++ tst-tgkill tst-sysvsem-linux tst-sysvmsg-linux tst-sysvshm-linux + tests-internal += tst-ofdlocks-compat tst-sigcontext-get_pc + + CFLAGS-tst-sigcontext-get_pc.c = -fasynchronous-unwind-tables +diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h +index fc688450ee..00a4d0c8e7 100644 +--- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h ++++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h +@@ -54,6 +54,10 @@ + && MIDR_PARTNUM(midr) == 0x000) + #define IS_NEOVERSE_N1(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ + && MIDR_PARTNUM(midr) == 0xd0c) ++#define IS_NEOVERSE_N2(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ ++ && MIDR_PARTNUM(midr) == 0xd49) ++#define IS_NEOVERSE_V1(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ ++ && MIDR_PARTNUM(midr) == 0xd40) + + #define IS_EMAG(midr) (MIDR_IMPLEMENTOR(midr) == 'P' \ + && MIDR_PARTNUM(midr) == 0x000) +diff --git a/sysdeps/unix/sysv/linux/mq_notify.c b/sysdeps/unix/sysv/linux/mq_notify.c +index 61bbb03b64..2bb98172c8 100644 +--- a/sysdeps/unix/sysv/linux/mq_notify.c ++++ b/sysdeps/unix/sysv/linux/mq_notify.c +@@ -132,9 +132,12 @@ helper_thread (void *arg) + to wait until it is done with it. */ + (void) __pthread_barrier_wait (¬ify_barrier); + } +- else if (data.raw[NOTIFY_COOKIE_LEN - 1] == NOTIFY_REMOVED) +- /* The only state we keep is the copy of the thread attributes. */ +- free (data.attr); ++ else if (data.raw[NOTIFY_COOKIE_LEN - 1] == NOTIFY_REMOVED && data.attr != NULL) ++ { ++ /* The only state we keep is the copy of the thread attributes. */ ++ pthread_attr_destroy (data.attr); ++ free (data.attr); ++ } + } + return NULL; + } +@@ -255,8 +258,14 @@ mq_notify (mqd_t mqdes, const struct sigevent *notification) + if (data.attr == NULL) + return -1; + +- memcpy (data.attr, notification->sigev_notify_attributes, +- sizeof (pthread_attr_t)); ++ int ret = __pthread_attr_copy (data.attr, ++ notification->sigev_notify_attributes); ++ if (ret != 0) ++ { ++ free (data.attr); ++ __set_errno (ret); ++ return -1; ++ } + } + + /* Construct the new request. */ +@@ -269,8 +278,11 @@ mq_notify (mqd_t mqdes, const struct sigevent *notification) + int retval = INLINE_SYSCALL (mq_notify, 2, mqdes, &se); + + /* If it failed, free the allocated memory. */ +- if (__glibc_unlikely (retval != 0)) +- free (data.attr); ++ if (retval != 0 && data.attr != NULL) ++ { ++ pthread_attr_destroy (data.attr); ++ free (data.attr); ++ } + + return retval; + } +diff --git a/sysdeps/unix/sysv/linux/msgctl.c b/sysdeps/unix/sysv/linux/msgctl.c +index 0776472d5e..a1f24ab242 100644 +--- a/sysdeps/unix/sysv/linux/msgctl.c ++++ b/sysdeps/unix/sysv/linux/msgctl.c +@@ -90,8 +90,15 @@ __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf) + struct kernel_msqid64_ds ksemid, *arg = NULL; + if (buf != NULL) + { +- msqid64_to_kmsqid64 (buf, &ksemid); +- arg = &ksemid; ++ /* This is a Linux extension where kernel returns a 'struct msginfo' ++ instead. */ ++ if (cmd == IPC_INFO || cmd == MSG_INFO) ++ arg = (struct kernel_msqid64_ds *) buf; ++ else ++ { ++ msqid64_to_kmsqid64 (buf, &ksemid); ++ arg = &ksemid; ++ } + } + # ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T + if (cmd == IPC_SET) +@@ -169,8 +176,15 @@ __msgctl (int msqid, int cmd, struct msqid_ds *buf) + struct __msqid64_ds msqid64, *buf64 = NULL; + if (buf != NULL) + { +- msqid_to_msqid64 (&msqid64, buf); +- buf64 = &msqid64; ++ /* This is a Linux extension where kernel returns a 'struct msginfo' ++ instead. */ ++ if (cmd == IPC_INFO || cmd == MSG_INFO) ++ buf64 = (struct __msqid64_ds *) buf; ++ else ++ { ++ msqid_to_msqid64 (&msqid64, buf); ++ buf64 = &msqid64; ++ } + } + + int ret = __msgctl64 (msqid, cmd, buf64); +diff --git a/sysdeps/unix/sysv/linux/semctl.c b/sysdeps/unix/sysv/linux/semctl.c +index f131a26fc7..1cdabde8f2 100644 +--- a/sysdeps/unix/sysv/linux/semctl.c ++++ b/sysdeps/unix/sysv/linux/semctl.c +@@ -102,6 +102,7 @@ semun64_to_ksemun64 (int cmd, union semun64 semun64, + r.array = semun64.array; + break; + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_STAT: + case IPC_SET: + r.buf = buf; +@@ -150,6 +151,7 @@ __semctl64 (int semid, int semnum, int cmd, ...) + case IPC_STAT: /* arg.buf */ + case IPC_SET: + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_INFO: /* arg.__buf */ + case SEM_INFO: + va_start (ap, cmd); +@@ -238,6 +240,7 @@ semun_to_semun64 (int cmd, union semun semun, struct __semid64_ds *semid64) + r.array = semun.array; + break; + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_STAT: + case IPC_SET: + r.buf = semid64; +@@ -267,6 +270,7 @@ __semctl (int semid, int semnum, int cmd, ...) + case IPC_STAT: /* arg.buf */ + case IPC_SET: + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_INFO: /* arg.__buf */ + case SEM_INFO: + va_start (ap, cmd); +@@ -321,6 +325,7 @@ __semctl_mode16 (int semid, int semnum, int cmd, ...) + case IPC_STAT: /* arg.buf */ + case IPC_SET: + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_INFO: /* arg.__buf */ + case SEM_INFO: + va_start (ap, cmd); +@@ -354,6 +359,7 @@ __old_semctl (int semid, int semnum, int cmd, ...) + case IPC_STAT: /* arg.buf */ + case IPC_SET: + case SEM_STAT: ++ case SEM_STAT_ANY: + case IPC_INFO: /* arg.__buf */ + case SEM_INFO: + va_start (ap, cmd); +diff --git a/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies b/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies +new file mode 100644 +index 0000000000..7eeaf15a5a +--- /dev/null ++++ b/sysdeps/unix/sysv/linux/sh/be/sh4/fpu/Implies +@@ -0,0 +1 @@ ++unix/sysv/linux/sh/sh4/fpu +diff --git a/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies b/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies +new file mode 100644 +index 0000000000..7eeaf15a5a +--- /dev/null ++++ b/sysdeps/unix/sysv/linux/sh/le/sh4/fpu/Implies +@@ -0,0 +1 @@ ++unix/sysv/linux/sh/sh4/fpu +diff --git a/sysdeps/unix/sysv/linux/shmctl.c b/sysdeps/unix/sysv/linux/shmctl.c +index 76d88441f1..1d19a798b1 100644 +--- a/sysdeps/unix/sysv/linux/shmctl.c ++++ b/sysdeps/unix/sysv/linux/shmctl.c +@@ -90,8 +90,15 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf) + struct kernel_shmid64_ds kshmid, *arg = NULL; + if (buf != NULL) + { +- shmid64_to_kshmid64 (buf, &kshmid); +- arg = &kshmid; ++ /* This is a Linux extension where kernel expects either a ++ 'struct shminfo' (IPC_INFO) or 'struct shm_info' (SHM_INFO). */ ++ if (cmd == IPC_INFO || cmd == SHM_INFO) ++ arg = (struct kernel_shmid64_ds *) buf; ++ else ++ { ++ shmid64_to_kshmid64 (buf, &kshmid); ++ arg = &kshmid; ++ } + } + # ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T + if (cmd == IPC_SET) +@@ -107,7 +114,6 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf) + + switch (cmd) + { +- case IPC_INFO: + case IPC_STAT: + case SHM_STAT: + case SHM_STAT_ANY: +@@ -168,8 +174,15 @@ __shmctl (int shmid, int cmd, struct shmid_ds *buf) + struct __shmid64_ds shmid64, *buf64 = NULL; + if (buf != NULL) + { +- shmid_to_shmid64 (&shmid64, buf); +- buf64 = &shmid64; ++ /* This is a Linux extension where kernel expects either a ++ 'struct shminfo' (IPC_INFO) or 'struct shm_info' (SHM_INFO). */ ++ if (cmd == IPC_INFO || cmd == SHM_INFO) ++ buf64 = (struct __shmid64_ds *) buf; ++ else ++ { ++ shmid_to_shmid64 (&shmid64, buf); ++ buf64 = &shmid64; ++ } + } + + int ret = __shmctl64 (shmid, cmd, buf64); +@@ -178,7 +191,6 @@ __shmctl (int shmid, int cmd, struct shmid_ds *buf) + + switch (cmd) + { +- case IPC_INFO: + case IPC_STAT: + case SHM_STAT: + case SHM_STAT_ANY: +diff --git a/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c b/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c +new file mode 100644 +index 0000000000..630f4f792c +--- /dev/null ++++ b/sysdeps/unix/sysv/linux/tst-sysvmsg-linux.c +@@ -0,0 +1,177 @@ ++/* Basic tests for Linux SYSV message queue extensions. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sys/ipc.h> ++#include <sys/msg.h> ++#include <errno.h> ++#include <stdlib.h> ++#include <stdbool.h> ++#include <stdio.h> ++ ++#include <support/check.h> ++#include <support/temp_file.h> ++ ++#define MSGQ_MODE 0644 ++ ++/* These are for the temporary file we generate. */ ++static char *name; ++static int msqid; ++ ++static void ++remove_msq (void) ++{ ++ /* Enforce message queue removal in case of early test failure. ++ Ignore error since the msg may already have being removed. */ ++ msgctl (msqid, IPC_RMID, NULL); ++} ++ ++static void ++do_prepare (int argc, char *argv[]) ++{ ++ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvmsg.", &name) != -1); ++} ++ ++#define PREPARE do_prepare ++ ++struct test_msginfo ++{ ++ int msgmax; ++ int msgmnb; ++ int msgmni; ++}; ++ ++/* It tries to obtain some system-wide SysV messsage queue information from ++ /proc to check against IPC_INFO/MSG_INFO. The /proc only returns the ++ tunables value of MSGMAX, MSGMNB, and MSGMNI. ++ ++ The kernel also returns constant value for MSGSSZ, MSGSEG and also MSGMAP, ++ MSGPOOL, and MSGTQL (for IPC_INFO). The issue to check them is they might ++ change over kernel releases. */ ++ ++static int ++read_proc_file (const char *file) ++{ ++ FILE *f = fopen (file, "r"); ++ if (f == NULL) ++ FAIL_UNSUPPORTED ("/proc is not mounted or %s is not available", file); ++ ++ int v; ++ int r = fscanf (f, "%d", & v); ++ TEST_VERIFY_EXIT (r == 1); ++ ++ fclose (f); ++ return v; ++} ++ ++ ++/* Check if the message queue with IDX (index into the kernel's internal ++ array) matches the one with KEY. The CMD is either MSG_STAT or ++ MSG_STAT_ANY. */ ++ ++static bool ++check_msginfo (int idx, key_t key, int cmd) ++{ ++ struct msqid_ds msginfo; ++ int mid = msgctl (idx, cmd, &msginfo); ++ /* Ignore unused array slot returned by the kernel or information from ++ unknown message queue. */ ++ if ((mid == -1 && errno == EINVAL) || mid != msqid) ++ return false; ++ ++ if (mid == -1) ++ FAIL_EXIT1 ("msgctl with %s failed: %m", ++ cmd == MSG_STAT ? "MSG_STAT" : "MSG_STAT_ANY"); ++ ++ TEST_COMPARE (msginfo.msg_perm.__key, key); ++ TEST_COMPARE (msginfo.msg_perm.mode, MSGQ_MODE); ++ TEST_COMPARE (msginfo.msg_qnum, 0); ++ ++ return true; ++} ++ ++static int ++do_test (void) ++{ ++ atexit (remove_msq); ++ ++ key_t key = ftok (name, 'G'); ++ if (key == -1) ++ FAIL_EXIT1 ("ftok failed: %m"); ++ ++ msqid = msgget (key, MSGQ_MODE | IPC_CREAT); ++ if (msqid == -1) ++ FAIL_EXIT1 ("msgget failed: %m"); ++ ++ struct test_msginfo tipcinfo; ++ tipcinfo.msgmax = read_proc_file ("/proc/sys/kernel/msgmax"); ++ tipcinfo.msgmnb = read_proc_file ("/proc/sys/kernel/msgmnb"); ++ tipcinfo.msgmni = read_proc_file ("/proc/sys/kernel/msgmni"); ++ ++ int msqidx; ++ ++ { ++ struct msginfo ipcinfo; ++ msqidx = msgctl (msqid, IPC_INFO, (struct msqid_ds *) &ipcinfo); ++ if (msqidx == -1) ++ FAIL_EXIT1 ("msgctl with IPC_INFO failed: %m"); ++ ++ TEST_COMPARE (ipcinfo.msgmax, tipcinfo.msgmax); ++ TEST_COMPARE (ipcinfo.msgmnb, tipcinfo.msgmnb); ++ TEST_COMPARE (ipcinfo.msgmni, tipcinfo.msgmni); ++ } ++ ++ /* Same as before but with MSG_INFO. */ ++ { ++ struct msginfo ipcinfo; ++ msqidx = msgctl (msqid, MSG_INFO, (struct msqid_ds *) &ipcinfo); ++ if (msqidx == -1) ++ FAIL_EXIT1 ("msgctl with IPC_INFO failed: %m"); ++ ++ TEST_COMPARE (ipcinfo.msgmax, tipcinfo.msgmax); ++ TEST_COMPARE (ipcinfo.msgmnb, tipcinfo.msgmnb); ++ TEST_COMPARE (ipcinfo.msgmni, tipcinfo.msgmni); ++ } ++ ++ /* We check if the created message queue shows in global list. */ ++ bool found = false; ++ for (int i = 0; i <= msqidx; i++) ++ { ++ /* We can't tell apart if MSG_STAT_ANY is not supported (kernel older ++ than 4.17) or if the index used is invalid. So it just check if the ++ value returned from a valid call matches the created message ++ queue. */ ++ check_msginfo (i, key, MSG_STAT_ANY); ++ ++ if (check_msginfo (i, key, MSG_STAT)) ++ { ++ found = true; ++ break; ++ } ++ } ++ ++ if (!found) ++ FAIL_EXIT1 ("msgctl with MSG_STAT/MSG_STAT_ANY could not find the " ++ "created message queue"); ++ ++ if (msgctl (msqid, IPC_RMID, NULL) == -1) ++ FAIL_EXIT1 ("msgctl failed"); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c b/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c +new file mode 100644 +index 0000000000..45f19e2d37 +--- /dev/null ++++ b/sysdeps/unix/sysv/linux/tst-sysvsem-linux.c +@@ -0,0 +1,184 @@ ++/* Basic tests for Linux SYSV semaphore extensions. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sys/ipc.h> ++#include <sys/sem.h> ++#include <errno.h> ++#include <stdlib.h> ++#include <stdbool.h> ++#include <stdio.h> ++ ++#include <support/check.h> ++#include <support/temp_file.h> ++ ++/* These are for the temporary file we generate. */ ++static char *name; ++static int semid; ++ ++static void ++remove_sem (void) ++{ ++ /* Enforce message queue removal in case of early test failure. ++ Ignore error since the sem may already have being removed. */ ++ semctl (semid, 0, IPC_RMID, 0); ++} ++ ++static void ++do_prepare (int argc, char *argv[]) ++{ ++ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvsem.", &name) != -1); ++} ++ ++#define PREPARE do_prepare ++ ++#define SEM_MODE 0644 ++ ++union semun ++{ ++ int val; ++ struct semid_ds *buf; ++ unsigned short *array; ++ struct seminfo *__buf; ++}; ++ ++struct test_seminfo ++{ ++ int semmsl; ++ int semmns; ++ int semopm; ++ int semmni; ++}; ++ ++/* It tries to obtain some system-wide SysV semaphore information from /proc ++ to check against IPC_INFO/SEM_INFO. The /proc only returns the tunables ++ value of SEMMSL, SEMMNS, SEMOPM, and SEMMNI. ++ ++ The kernel also returns constant value for SEMVMX, SEMMNU, SEMMAP, SEMUME, ++ and also SEMUSZ and SEMAEM (for IPC_INFO). The issue to check them is they ++ might change over kernel releases. */ ++ ++static void ++read_sem_stat (struct test_seminfo *tseminfo) ++{ ++ FILE *f = fopen ("/proc/sys/kernel/sem", "r"); ++ if (f == NULL) ++ FAIL_UNSUPPORTED ("/proc is not mounted or /proc/sys/kernel/sem is not " ++ "available"); ++ ++ int r = fscanf (f, "%d %d %d %d", ++ &tseminfo->semmsl, &tseminfo->semmns, &tseminfo->semopm, ++ &tseminfo->semmni); ++ TEST_VERIFY_EXIT (r == 4); ++ ++ fclose (f); ++} ++ ++ ++/* Check if the semaphore with IDX (index into the kernel's internal array) ++ matches the one with KEY. The CMD is either SEM_STAT or SEM_STAT_ANY. */ ++ ++static bool ++check_seminfo (int idx, key_t key, int cmd) ++{ ++ struct semid_ds seminfo; ++ int sid = semctl (idx, 0, cmd, (union semun) { .buf = &seminfo }); ++ /* Ignore unused array slot returned by the kernel or information from ++ unknown semaphores. */ ++ if ((sid == -1 && errno == EINVAL) || sid != semid) ++ return false; ++ ++ if (sid == -1) ++ FAIL_EXIT1 ("semctl with SEM_STAT failed (errno=%d)", errno); ++ ++ TEST_COMPARE (seminfo.sem_perm.__key, key); ++ TEST_COMPARE (seminfo.sem_perm.mode, SEM_MODE); ++ TEST_COMPARE (seminfo.sem_nsems, 1); ++ ++ return true; ++} ++ ++static int ++do_test (void) ++{ ++ atexit (remove_sem); ++ ++ key_t key = ftok (name, 'G'); ++ if (key == -1) ++ FAIL_EXIT1 ("ftok failed: %m"); ++ ++ semid = semget (key, 1, IPC_CREAT | IPC_EXCL | SEM_MODE); ++ if (semid == -1) ++ FAIL_EXIT1 ("semget failed: %m"); ++ ++ struct test_seminfo tipcinfo; ++ read_sem_stat (&tipcinfo); ++ ++ int semidx; ++ ++ { ++ struct seminfo ipcinfo; ++ semidx = semctl (semid, 0, IPC_INFO, (union semun) { .__buf = &ipcinfo }); ++ if (semidx == -1) ++ FAIL_EXIT1 ("semctl with IPC_INFO failed: %m"); ++ ++ TEST_COMPARE (ipcinfo.semmsl, tipcinfo.semmsl); ++ TEST_COMPARE (ipcinfo.semmns, tipcinfo.semmns); ++ TEST_COMPARE (ipcinfo.semopm, tipcinfo.semopm); ++ TEST_COMPARE (ipcinfo.semmni, tipcinfo.semmni); ++ } ++ ++ /* Same as before but with SEM_INFO. */ ++ { ++ struct seminfo ipcinfo; ++ semidx = semctl (semid, 0, SEM_INFO, (union semun) { .__buf = &ipcinfo }); ++ if (semidx == -1) ++ FAIL_EXIT1 ("semctl with IPC_INFO failed: %m"); ++ ++ TEST_COMPARE (ipcinfo.semmsl, tipcinfo.semmsl); ++ TEST_COMPARE (ipcinfo.semmns, tipcinfo.semmns); ++ TEST_COMPARE (ipcinfo.semopm, tipcinfo.semopm); ++ TEST_COMPARE (ipcinfo.semmni, tipcinfo.semmni); ++ } ++ ++ /* We check if the created semaphore shows in the system-wide status. */ ++ bool found = false; ++ for (int i = 0; i <= semidx; i++) ++ { ++ /* We can't tell apart if SEM_STAT_ANY is not supported (kernel older ++ than 4.17) or if the index used is invalid. So it just check if ++ value returned from a valid call matches the created semaphore. */ ++ check_seminfo (i, key, SEM_STAT_ANY); ++ ++ if (check_seminfo (i, key, SEM_STAT)) ++ { ++ found = true; ++ break; ++ } ++ } ++ ++ if (!found) ++ FAIL_EXIT1 ("semctl with SEM_STAT/SEM_STAT_ANY could not find the " ++ "created semaphore"); ++ ++ if (semctl (semid, 0, IPC_RMID, 0) == -1) ++ FAIL_EXIT1 ("semctl failed: %m"); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c b/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c +new file mode 100644 +index 0000000000..bb154592a6 +--- /dev/null ++++ b/sysdeps/unix/sysv/linux/tst-sysvshm-linux.c +@@ -0,0 +1,188 @@ ++/* Basic tests for Linux SYSV shared memory extensions. ++ Copyright (C) 2020 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sys/ipc.h> ++#include <sys/shm.h> ++#include <errno.h> ++#include <stdlib.h> ++#include <stdbool.h> ++#include <stdio.h> ++#include <unistd.h> ++#include <inttypes.h> ++#include <limits.h> ++ ++#include <support/check.h> ++#include <support/temp_file.h> ++ ++#define SHM_MODE 0644 ++ ++/* These are for the temporary file we generate. */ ++static char *name; ++static int shmid; ++static long int pgsz; ++ ++static void ++remove_shm (void) ++{ ++ /* Enforce message queue removal in case of early test failure. ++ Ignore error since the shm may already have being removed. */ ++ shmctl (shmid, IPC_RMID, NULL); ++} ++ ++static void ++do_prepare (int argc, char *argv[]) ++{ ++ TEST_VERIFY_EXIT (create_temp_file ("tst-sysvshm.", &name) != -1); ++} ++ ++#define PREPARE do_prepare ++ ++struct test_shminfo ++{ ++ __syscall_ulong_t shmall; ++ __syscall_ulong_t shmmax; ++ __syscall_ulong_t shmmni; ++}; ++ ++/* It tries to obtain some system-wide SysV shared memory information from ++ /proc to check against IPC_INFO/SHM_INFO. The /proc only returns the ++ tunables value of SHMALL, SHMMAX, and SHMMNI. */ ++ ++static uint64_t ++read_proc_file (const char *file) ++{ ++ FILE *f = fopen (file, "r"); ++ if (f == NULL) ++ FAIL_UNSUPPORTED ("/proc is not mounted or %s is not available", file); ++ ++ /* Handle 32-bit binaries running on 64-bit kernels. */ ++ uint64_t v; ++ int r = fscanf (f, "%" SCNu64, &v); ++ TEST_VERIFY_EXIT (r == 1); ++ ++ fclose (f); ++ return v; ++} ++ ++ ++/* Check if the message queue with IDX (index into the kernel's internal ++ array) matches the one with KEY. The CMD is either SHM_STAT or ++ SHM_STAT_ANY. */ ++ ++static bool ++check_shminfo (int idx, key_t key, int cmd) ++{ ++ struct shmid_ds shminfo; ++ int sid = shmctl (idx, cmd, &shminfo); ++ /* Ignore unused array slot returned by the kernel or information from ++ unknown message queue. */ ++ if ((sid == -1 && errno == EINVAL) || sid != shmid) ++ return false; ++ ++ if (sid == -1) ++ FAIL_EXIT1 ("shmctl with %s failed: %m", ++ cmd == SHM_STAT ? "SHM_STAT" : "SHM_STAT_ANY"); ++ ++ TEST_COMPARE (shminfo.shm_perm.__key, key); ++ TEST_COMPARE (shminfo.shm_perm.mode, SHM_MODE); ++ TEST_COMPARE (shminfo.shm_segsz, pgsz); ++ ++ return true; ++} ++ ++static int ++do_test (void) ++{ ++ atexit (remove_shm); ++ ++ pgsz = sysconf (_SC_PAGESIZE); ++ if (pgsz == -1) ++ FAIL_EXIT1 ("sysconf (_SC_PAGESIZE) failed: %m"); ++ ++ key_t key = ftok (name, 'G'); ++ if (key == -1) ++ FAIL_EXIT1 ("ftok failed: %m"); ++ ++ shmid = shmget (key, pgsz, IPC_CREAT | IPC_EXCL | SHM_MODE); ++ if (shmid == -1) ++ FAIL_EXIT1 ("shmget failed: %m"); ++ ++ /* It does not check shmmax because kernel clamp its value to INT_MAX for: ++ ++ 1. Compat symbols with IPC_64, i.e, 32-bit binaries running on 64-bit ++ kernels. ++ ++ 2. Default symbol without IPC_64 (defined as IPC_OLD within Linux) and ++ glibc always use IPC_64 for 32-bit ABIs (to support 64-bit time_t). ++ It means that 32-bit binaries running on 32-bit kernels will not see ++ shmmax being clamped. ++ ++ And finding out whether the compat symbol is used would require checking ++ the underlying kernel against the current ABI. The shmall and shmmni ++ already provided enough coverage. */ ++ ++ struct test_shminfo tipcinfo; ++ tipcinfo.shmall = read_proc_file ("/proc/sys/kernel/shmall"); ++ tipcinfo.shmmni = read_proc_file ("/proc/sys/kernel/shmmni"); ++ ++ int shmidx; ++ ++ /* Note: SHM_INFO does not return a shminfo, but rather a 'struct shm_info'. ++ It is tricky to verify its values since the syscall returns system wide ++ resources consumed by shared memory. The shmctl implementation handles ++ SHM_INFO as IPC_INFO, so the IPC_INFO test should validate SHM_INFO as ++ well. */ ++ ++ { ++ struct shminfo ipcinfo; ++ shmidx = shmctl (shmid, IPC_INFO, (struct shmid_ds *) &ipcinfo); ++ if (shmidx == -1) ++ FAIL_EXIT1 ("shmctl with IPC_INFO failed: %m"); ++ ++ TEST_COMPARE (ipcinfo.shmall, tipcinfo.shmall); ++ TEST_COMPARE (ipcinfo.shmmni, tipcinfo.shmmni); ++ } ++ ++ /* We check if the created shared memory shows in the global list. */ ++ bool found = false; ++ for (int i = 0; i <= shmidx; i++) ++ { ++ /* We can't tell apart if SHM_STAT_ANY is not supported (kernel older ++ than 4.17) or if the index used is invalid. So it just check if ++ value returned from a valid call matches the created message ++ queue. */ ++ check_shminfo (i, key, SHM_STAT_ANY); ++ ++ if (check_shminfo (i, key, SHM_STAT)) ++ { ++ found = true; ++ break; ++ } ++ } ++ ++ if (!found) ++ FAIL_EXIT1 ("shmctl with SHM_STAT/SHM_STAT_ANY could not find the " ++ "created shared memory"); ++ ++ if (shmctl (shmid, IPC_RMID, NULL) == -1) ++ FAIL_EXIT1 ("shmctl failed"); ++ ++ return 0; ++} ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile +index a6736aef25..e821d95fa3 100644 +--- a/sysdeps/x86/Makefile ++++ b/sysdeps/x86/Makefile +@@ -12,10 +12,39 @@ endif + ifeq ($(subdir),setjmp) + gen-as-const-headers += jmp_buf-ssp.sym + sysdep_routines += __longjmp_cancel ++ifneq ($(enable-cet),no) ++ifneq ($(have-tunables),no) ++tests += tst-setjmp-cet ++tst-setjmp-cet-ENV = GLIBC_TUNABLES=glibc.cpu.x86_ibt=on:glibc.cpu.x86_shstk=on ++endif ++endif + endif + + ifeq ($(subdir),string) + sysdep_routines += cacheinfo ++ ++tests += \ ++ tst-memchr-rtm \ ++ tst-memcmp-rtm \ ++ tst-memmove-rtm \ ++ tst-memrchr-rtm \ ++ tst-memset-rtm \ ++ tst-strchr-rtm \ ++ tst-strcpy-rtm \ ++ tst-strlen-rtm \ ++ tst-strncmp-rtm \ ++ tst-strrchr-rtm ++ ++CFLAGS-tst-memchr-rtm.c += -mrtm ++CFLAGS-tst-memcmp-rtm.c += -mrtm ++CFLAGS-tst-memmove-rtm.c += -mrtm ++CFLAGS-tst-memrchr-rtm.c += -mrtm ++CFLAGS-tst-memset-rtm.c += -mrtm ++CFLAGS-tst-strchr-rtm.c += -mrtm ++CFLAGS-tst-strcpy-rtm.c += -mrtm ++CFLAGS-tst-strlen-rtm.c += -mrtm ++CFLAGS-tst-strncmp-rtm.c += -mrtm ++CFLAGS-tst-strrchr-rtm.c += -mrtm + endif + + ifneq ($(enable-cet),no) +diff --git a/sysdeps/x86/cacheinfo.c b/sysdeps/x86/cacheinfo.c +index 217c21c34f..3fb4a028d8 100644 +--- a/sysdeps/x86/cacheinfo.c ++++ b/sysdeps/x86/cacheinfo.c +@@ -808,7 +808,7 @@ init_cacheinfo (void) + threads = 1 << ((ecx >> 12) & 0x0f); + } + +- if (threads == 0) ++ if (threads == 0 || cpu_features->basic.family >= 0x17) + { + /* If APIC ID width is not available, use logical + processor count. */ +@@ -823,8 +823,22 @@ init_cacheinfo (void) + if (threads > 0) + shared /= threads; + +- /* Account for exclusive L2 and L3 caches. */ +- shared += core; ++ /* Get shared cache per ccx for Zen architectures. */ ++ if (cpu_features->basic.family >= 0x17) ++ { ++ unsigned int eax; ++ ++ /* Get number of threads share the L3 cache in CCX. */ ++ __cpuid_count (0x8000001D, 0x3, eax, ebx, ecx, edx); ++ ++ unsigned int threads_per_ccx = ((eax >> 14) & 0xfff) + 1; ++ shared *= threads_per_ccx; ++ } ++ else ++ { ++ /* Account for exclusive L2 and L3 caches. */ ++ shared += core; ++ } + } + } + +@@ -854,14 +868,20 @@ init_cacheinfo (void) + __x86_shared_cache_size = shared; + } + +- /* The large memcpy micro benchmark in glibc shows that 6 times of +- shared cache size is the approximate value above which non-temporal +- store becomes faster on a 8-core processor. This is the 3/4 of the +- total shared cache size. */ ++ /* The default setting for the non_temporal threshold is 3/4 of one ++ thread's share of the chip's cache. For most Intel and AMD processors ++ with an initial release date between 2017 and 2020, a thread's typical ++ share of the cache is from 500 KBytes to 2 MBytes. Using the 3/4 ++ threshold leaves 125 KBytes to 500 KBytes of the thread's data ++ in cache after a maximum temporal copy, which will maintain ++ in cache a reasonable portion of the thread's stack and other ++ active data. If the threshold is set higher than one thread's ++ share of the cache, it has a substantial risk of negatively ++ impacting the performance of other threads running on the chip. */ + __x86_shared_non_temporal_threshold + = (cpu_features->non_temporal_threshold != 0 + ? cpu_features->non_temporal_threshold +- : __x86_shared_cache_size * threads * 3 / 4); ++ : __x86_shared_cache_size * 3 / 4); + + /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8. */ + unsigned int minimum_rep_movsb_threshold; +diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c +index 4c24ba7c31..484efe7a0f 100644 +--- a/sysdeps/x86/cpu-features.c ++++ b/sysdeps/x86/cpu-features.c +@@ -71,7 +71,6 @@ update_usable (struct cpu_features *cpu_features) + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_6); + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_7); + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_9); +- CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_11); + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_12); + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_13); + CPU_FEATURE_UNSET (cpu_features, INDEX_7_EDX_17); +@@ -318,6 +317,9 @@ update_usable (struct cpu_features *cpu_features) + /* Determine if PKU is usable. */ + if (CPU_FEATURES_CPU_P (cpu_features, OSPKE)) + CPU_FEATURE_SET (cpu_features, PKU); ++ ++ if (CPU_FEATURES_CPU_P (cpu_features, RTM_ALWAYS_ABORT)) ++ CPU_FEATURE_UNSET (cpu_features, RTM); + } + + static void +@@ -516,11 +518,39 @@ init_cpu_features (struct cpu_features *cpu_features) + break; + } + +- /* Disable TSX on some Haswell processors to avoid TSX on kernels that +- weren't updated with the latest microcode package (which disables +- broken feature by default). */ ++ /* Disable TSX on some processors to avoid TSX on kernels that ++ weren't updated with the latest microcode package (which ++ disables broken feature by default). */ + switch (model) + { ++ case 0x55: ++ if (stepping <= 5) ++ goto disable_tsx; ++ break; ++ case 0x8e: ++ /* NB: Although the errata documents that for model == 0x8e, ++ only 0xb stepping or lower are impacted, the intention of ++ the errata was to disable TSX on all client processors on ++ all steppings. Include 0xc stepping which is an Intel ++ Core i7-8665U, a client mobile processor. */ ++ case 0x9e: ++ if (stepping > 0xc) ++ break; ++ /* Fall through. */ ++ case 0x4e: ++ case 0x5e: ++ { ++ /* Disable Intel TSX and enable RTM_ALWAYS_ABORT for ++ processors listed in: ++ ++https://www.intel.com/content/www/us/en/support/articles/000059422/processor... ++ */ ++disable_tsx: ++ CPU_FEATURE_UNSET (cpu_features, HLE); ++ CPU_FEATURE_UNSET (cpu_features, RTM); ++ CPU_FEATURE_SET (cpu_features, RTM_ALWAYS_ABORT); ++ } ++ break; + case 0x3f: + /* Xeon E7 v3 with stepping >= 4 has working TSX. */ + if (stepping >= 4) +@@ -546,8 +576,24 @@ init_cpu_features (struct cpu_features *cpu_features) + cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER] + |= bit_arch_Prefer_No_VZEROUPPER; + else +- cpu_features->preferred[index_arch_Prefer_No_AVX512] +- |= bit_arch_Prefer_No_AVX512; ++ { ++ cpu_features->preferred[index_arch_Prefer_No_AVX512] ++ |= bit_arch_Prefer_No_AVX512; ++ ++ /* Avoid RTM abort triggered by VZEROUPPER inside a ++ transactionally executing RTM region. */ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER] ++ |= bit_arch_Prefer_No_VZEROUPPER; ++ ++ /* Since to compare 2 32-byte strings, 256-bit EVEX strcmp ++ requires 2 loads, 3 VPCMPs and 2 KORDs while AVX2 strcmp ++ requires 1 load, 2 VPCMPEQs, 1 VPMINU and 1 VPMOVMSKB, ++ AVX2 strcmp is faster than EVEX strcmp. */ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) ++ cpu_features->preferred[index_arch_Prefer_AVX2_STRCMP] ++ |= bit_arch_Prefer_AVX2_STRCMP; ++ } + } + /* This spells out "AuthenticAMD" or "HygonGenuine". */ + else if ((ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) +diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h +index a0b9b9177c..8995a15f09 100644 +--- a/sysdeps/x86/cpu-features.h ++++ b/sysdeps/x86/cpu-features.h +@@ -295,7 +295,7 @@ extern const struct cpu_features *__get_cpu_features (void) + #define bit_cpu_AVX512_VP2INTERSECT (1u << 8) + #define bit_cpu_INDEX_7_EDX_9 (1u << 9) + #define bit_cpu_MD_CLEAR (1u << 10) +-#define bit_cpu_INDEX_7_EDX_11 (1u << 11) ++#define bit_cpu_RTM_ALWAYS_ABORT (1u << 11) + #define bit_cpu_INDEX_7_EDX_12 (1u << 12) + #define bit_cpu_INDEX_7_EDX_13 (1u << 13) + #define bit_cpu_SERIALIZE (1u << 14) +@@ -508,7 +508,7 @@ extern const struct cpu_features *__get_cpu_features (void) + #define index_cpu_AVX512_VP2INTERSECT COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_9 COMMON_CPUID_INDEX_7 + #define index_cpu_MD_CLEAR COMMON_CPUID_INDEX_7 +-#define index_cpu_INDEX_7_EDX_11 COMMON_CPUID_INDEX_7 ++#define index_cpu_RTM_ALWAYS_ABORT COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_12 COMMON_CPUID_INDEX_7 + #define index_cpu_INDEX_7_EDX_13 COMMON_CPUID_INDEX_7 + #define index_cpu_SERIALIZE COMMON_CPUID_INDEX_7 +@@ -721,7 +721,7 @@ extern const struct cpu_features *__get_cpu_features (void) + #define reg_AVX512_VP2INTERSECT edx + #define reg_INDEX_7_EDX_9 edx + #define reg_MD_CLEAR edx +-#define reg_INDEX_7_EDX_11 edx ++#define reg_RTM_ALWAYS_ABORT edx + #define reg_INDEX_7_EDX_12 edx + #define reg_INDEX_7_EDX_13 edx + #define reg_SERIALIZE edx +@@ -804,6 +804,7 @@ extern const struct cpu_features *__get_cpu_features (void) + #define bit_arch_Prefer_FSRM (1u << 13) + #define bit_arch_Prefer_No_AVX512 (1u << 14) + #define bit_arch_MathVec_Prefer_No_AVX512 (1u << 15) ++#define bit_arch_Prefer_AVX2_STRCMP (1u << 16) + + #define index_arch_Fast_Rep_String PREFERRED_FEATURE_INDEX_1 + #define index_arch_Fast_Copy_Backward PREFERRED_FEATURE_INDEX_1 +@@ -821,6 +822,7 @@ extern const struct cpu_features *__get_cpu_features (void) + #define index_arch_Prefer_No_AVX512 PREFERRED_FEATURE_INDEX_1 + #define index_arch_MathVec_Prefer_No_AVX512 PREFERRED_FEATURE_INDEX_1 + #define index_arch_Prefer_FSRM PREFERRED_FEATURE_INDEX_1 ++#define index_arch_Prefer_AVX2_STRCMP PREFERRED_FEATURE_INDEX_1 + + /* XCR0 Feature flags. */ + #define bit_XMM_state (1u << 1) +diff --git a/sysdeps/x86/cpu-tunables.c b/sysdeps/x86/cpu-tunables.c +index 588bbf9448..b251a91af3 100644 +--- a/sysdeps/x86/cpu-tunables.c ++++ b/sysdeps/x86/cpu-tunables.c +@@ -238,6 +238,8 @@ TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) + CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, + Fast_Copy_Backward, + disable, 18); ++ CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH ++ (n, cpu_features, Prefer_AVX2_STRCMP, AVX2, disable, 18); + } + break; + case 19: +diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c +index 03572f7af6..3cc54a8d53 100644 +--- a/sysdeps/x86/dl-cet.c ++++ b/sysdeps/x86/dl-cet.c +@@ -47,7 +47,10 @@ dl_cet_check (struct link_map *m, const char *program) + /* No legacy object check if both IBT and SHSTK are always on. */ + if (enable_ibt_type == cet_always_on + && enable_shstk_type == cet_always_on) +- return; ++ { ++ THREAD_SETMEM (THREAD_SELF, header.feature_1, GL(dl_x86_feature_1)); ++ return; ++ } + + /* Check if IBT is enabled by kernel. */ + bool ibt_enabled +diff --git a/sysdeps/x86/dl-prop.h b/sysdeps/x86/dl-prop.h +index 89911e19e2..4eb3b85a7b 100644 +--- a/sysdeps/x86/dl-prop.h ++++ b/sysdeps/x86/dl-prop.h +@@ -145,15 +145,15 @@ _dl_process_cet_property_note (struct link_map *l, + } + + static inline void __attribute__ ((unused)) +-_dl_process_pt_note (struct link_map *l, const ElfW(Phdr) *ph) ++_dl_process_pt_note (struct link_map *l, int fd, const ElfW(Phdr) *ph) + { + const ElfW(Nhdr) *note = (const void *) (ph->p_vaddr + l->l_addr); + _dl_process_cet_property_note (l, note, ph->p_memsz, ph->p_align); + } + + static inline int __attribute__ ((always_inline)) +-_dl_process_gnu_property (struct link_map *l, uint32_t type, uint32_t datasz, +- void *data) ++_dl_process_gnu_property (struct link_map *l, int fd, uint32_t type, ++ uint32_t datasz, void *data) + { + return 0; + } +diff --git a/sysdeps/x86/tst-get-cpu-features.c b/sysdeps/x86/tst-get-cpu-features.c +index 080c58e70b..527de3b5d9 100644 +--- a/sysdeps/x86/tst-get-cpu-features.c ++++ b/sysdeps/x86/tst-get-cpu-features.c +@@ -183,6 +183,7 @@ do_test (void) + CHECK_CPU_FEATURE (FSRM); + CHECK_CPU_FEATURE (AVX512_VP2INTERSECT); + CHECK_CPU_FEATURE (MD_CLEAR); ++ CHECK_CPU_FEATURE (RTM_ALWAYS_ABORT); + CHECK_CPU_FEATURE (SERIALIZE); + CHECK_CPU_FEATURE (HYBRID); + CHECK_CPU_FEATURE (TSXLDTRK); +@@ -336,6 +337,7 @@ do_test (void) + CHECK_CPU_FEATURE_USABLE (FSRM); + CHECK_CPU_FEATURE_USABLE (AVX512_VP2INTERSECT); + CHECK_CPU_FEATURE_USABLE (MD_CLEAR); ++ CHECK_CPU_FEATURE_USABLE (RTM_ALWAYS_ABORT); + CHECK_CPU_FEATURE_USABLE (SERIALIZE); + CHECK_CPU_FEATURE_USABLE (HYBRID); + CHECK_CPU_FEATURE_USABLE (TSXLDTRK); +diff --git a/sysdeps/x86/tst-memchr-rtm.c b/sysdeps/x86/tst-memchr-rtm.c +new file mode 100644 +index 0000000000..e47494011e +--- /dev/null ++++ b/sysdeps/x86/tst-memchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for memchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = memchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = memchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memcmp-rtm.c b/sysdeps/x86/tst-memcmp-rtm.c +new file mode 100644 +index 0000000000..e4c8a623bb +--- /dev/null ++++ b/sysdeps/x86/tst-memcmp-rtm.c +@@ -0,0 +1,52 @@ ++/* Test case for memcmp inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ memset (string2, 'a', STRING_SIZE); ++ if (memcmp (string1, string2, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (memcmp (string1, string2, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memcmp", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memmove-rtm.c b/sysdeps/x86/tst-memmove-rtm.c +new file mode 100644 +index 0000000000..4bf97ef1e3 +--- /dev/null ++++ b/sysdeps/x86/tst-memmove-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for memmove inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ if (memmove (string2, string1, STRING_SIZE) == string2 ++ && memcmp (string2, string1, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (memmove (string2, string1, STRING_SIZE) == string2 ++ && memcmp (string2, string1, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memmove", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memrchr-rtm.c b/sysdeps/x86/tst-memrchr-rtm.c +new file mode 100644 +index 0000000000..a57a5a8eb9 +--- /dev/null ++++ b/sysdeps/x86/tst-memrchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for memrchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = memrchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[STRING_SIZE - 100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = memrchr (string1, 'c', STRING_SIZE); ++ if (p == &string1[STRING_SIZE - 100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memrchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-memset-rtm.c b/sysdeps/x86/tst-memset-rtm.c +new file mode 100644 +index 0000000000..bf343a4dad +--- /dev/null ++++ b/sysdeps/x86/tst-memset-rtm.c +@@ -0,0 +1,45 @@ ++/* Test case for memset inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ return EXIT_SUCCESS; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ memset (string1, 'a', STRING_SIZE); ++ return 0; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("memset", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-setjmp-cet.c b/sysdeps/x86/tst-setjmp-cet.c +new file mode 100644 +index 0000000000..42c795d2a8 +--- /dev/null ++++ b/sysdeps/x86/tst-setjmp-cet.c +@@ -0,0 +1 @@ ++#include <setjmp/tst-setjmp.c> +diff --git a/sysdeps/x86/tst-strchr-rtm.c b/sysdeps/x86/tst-strchr-rtm.c +new file mode 100644 +index 0000000000..a82e29c072 +--- /dev/null ++++ b/sysdeps/x86/tst-strchr-rtm.c +@@ -0,0 +1,54 @@ ++/* Test case for strchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[100] = 'c'; ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = strchr (string1, 'c'); ++ if (p == &string1[100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = strchr (string1, 'c'); ++ if (p == &string1[100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strcpy-rtm.c b/sysdeps/x86/tst-strcpy-rtm.c +new file mode 100644 +index 0000000000..2b2a583fb4 +--- /dev/null ++++ b/sysdeps/x86/tst-strcpy-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strcpy inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ if (strcpy (string2, string1) == string2 ++ && strcmp (string2, string1) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (strcpy (string2, string1) == string2 ++ && strcmp (string2, string1) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strcpy", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-string-rtm.h b/sysdeps/x86/tst-string-rtm.h +new file mode 100644 +index 0000000000..6ed9eca017 +--- /dev/null ++++ b/sysdeps/x86/tst-string-rtm.h +@@ -0,0 +1,72 @@ ++/* Test string function in a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <string.h> ++#include <x86intrin.h> ++#include <cpu-features.h> ++#include <support/check.h> ++#include <support/test-driver.h> ++ ++static int ++do_test_1 (const char *name, unsigned int loop, int (*prepare) (void), ++ int (*function) (void)) ++{ ++ if (!CPU_FEATURE_USABLE (RTM)) ++ return EXIT_UNSUPPORTED; ++ ++ int status = prepare (); ++ if (status != EXIT_SUCCESS) ++ return status; ++ ++ unsigned int i; ++ unsigned int naborts = 0; ++ unsigned int failed = 0; ++ for (i = 0; i < loop; i++) ++ { ++ failed |= function (); ++ if (_xbegin() == _XBEGIN_STARTED) ++ { ++ failed |= function (); ++ _xend(); ++ } ++ else ++ { ++ failed |= function (); ++ ++naborts; ++ } ++ } ++ ++ if (failed) ++ FAIL_EXIT1 ("%s() failed", name); ++ ++ if (naborts) ++ { ++ /* NB: Low single digit (<= 5%) noise-level aborts are normal for ++ TSX. */ ++ double rate = 100 * ((double) naborts) / ((double) loop); ++ if (rate > 5) ++ FAIL_EXIT1 ("TSX abort rate: %.2f%% (%d out of %d)", ++ rate, naborts, loop); ++ } ++ ++ return EXIT_SUCCESS; ++} ++ ++static int do_test (void); ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/x86/tst-strlen-rtm.c b/sysdeps/x86/tst-strlen-rtm.c +new file mode 100644 +index 0000000000..0dcf14db87 +--- /dev/null ++++ b/sysdeps/x86/tst-strlen-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strlen inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[STRING_SIZE - 100] = '\0'; ++ size_t len = strlen (string1); ++ if (len == STRING_SIZE - 100) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ size_t len = strlen (string1); ++ if (len == STRING_SIZE - 100) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strlen", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strncmp-rtm.c b/sysdeps/x86/tst-strncmp-rtm.c +new file mode 100644 +index 0000000000..236ad951b5 +--- /dev/null ++++ b/sysdeps/x86/tst-strncmp-rtm.c +@@ -0,0 +1,52 @@ ++/* Test case for strncmp inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++char string2[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ memset (string2, 'a', STRING_SIZE - 1); ++ if (strncmp (string1, string2, STRING_SIZE) == 0) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ if (strncmp (string1, string2, STRING_SIZE) == 0) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strncmp", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86/tst-strrchr-rtm.c b/sysdeps/x86/tst-strrchr-rtm.c +new file mode 100644 +index 0000000000..e32bfaf5f5 +--- /dev/null ++++ b/sysdeps/x86/tst-strrchr-rtm.c +@@ -0,0 +1,53 @@ ++/* Test case for strrchr inside a transactionally executing RTM region. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <tst-string-rtm.h> ++ ++#define LOOP 3000 ++#define STRING_SIZE 1024 ++char string1[STRING_SIZE]; ++ ++__attribute__ ((noinline, noclone)) ++static int ++prepare (void) ++{ ++ memset (string1, 'a', STRING_SIZE - 1); ++ string1[STRING_SIZE - 100] = 'c'; ++ char *p = strrchr (string1, 'c'); ++ if (p == &string1[STRING_SIZE - 100]) ++ return EXIT_SUCCESS; ++ else ++ return EXIT_FAILURE; ++} ++ ++__attribute__ ((noinline, noclone)) ++static int ++function (void) ++{ ++ char *p = strrchr (string1, 'c'); ++ if (p == &string1[STRING_SIZE - 100]) ++ return 0; ++ else ++ return 1; ++} ++ ++static int ++do_test (void) ++{ ++ return do_test_1 ("strrchr", LOOP, prepare, function); ++} +diff --git a/sysdeps/x86_64/Makefile b/sysdeps/x86_64/Makefile +index 42b97c5cc7..020044da80 100644 +--- a/sysdeps/x86_64/Makefile ++++ b/sysdeps/x86_64/Makefile +@@ -20,6 +20,8 @@ endif + ifeq ($(subdir),string) + sysdep_routines += strcasecmp_l-nonascii strncase_l-nonascii + gen-as-const-headers += locale-defines.sym ++tests += \ ++ tst-rsi-strlen + endif + + ifeq ($(subdir),elf) +@@ -150,6 +152,11 @@ ifeq ($(subdir),csu) + gen-as-const-headers += tlsdesc.sym rtld-offsets.sym + endif + ++ifeq ($(subdir),wcsmbs) ++tests += \ ++ tst-rsi-wcslen ++endif ++ + $(objpfx)x86_64/tst-x86_64mod-1.os: $(objpfx)tst-x86_64mod-1.os + $(make-target-directory) + rm -f $@ +diff --git a/sysdeps/x86_64/configure b/sysdeps/x86_64/configure +old mode 100644 +new mode 100755 +index 84f82c2406..fc1840e23f +--- a/sysdeps/x86_64/configure ++++ b/sysdeps/x86_64/configure +@@ -107,39 +107,6 @@ if test x"$build_mathvec" = xnotset; then + build_mathvec=yes + fi + +-if test "$static_pie" = yes; then +- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for linker static PIE support" >&5 +-$as_echo_n "checking for linker static PIE support... " >&6; } +-if ${libc_cv_ld_static_pie+:} false; then : +- $as_echo_n "(cached) " >&6 +-else +- cat > conftest.s <<\EOF +- .text +- .global _start +- .weak foo +-_start: +- leaq foo(%rip), %rax +-EOF +- libc_cv_pie_option="-Wl,-pie" +- if { ac_try='${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS -nostartfiles -nostdlib $no_ssp $libc_cv_pie_option -o conftest conftest.s 1>&5' +- { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 +- (eval $ac_try) 2>&5 +- ac_status=$? +- $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 +- test $ac_status = 0; }; }; then +- libc_cv_ld_static_pie=yes +- else +- libc_cv_ld_static_pie=no +- fi +-rm -f conftest* +-fi +-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $libc_cv_ld_static_pie" >&5 +-$as_echo "$libc_cv_ld_static_pie" >&6; } +- if test "$libc_cv_ld_static_pie" != yes; then +- as_fn_error $? "linker support for static PIE needed" "$LINENO" 5 +- fi +-fi +- + $as_echo "#define PI_STATIC_AND_HIDDEN 1" >>confdefs.h + + +diff --git a/sysdeps/x86_64/configure.ac b/sysdeps/x86_64/configure.ac +index cdaba0c075..611a7d9ba3 100644 +--- a/sysdeps/x86_64/configure.ac ++++ b/sysdeps/x86_64/configure.ac +@@ -53,31 +53,6 @@ if test x"$build_mathvec" = xnotset; then + build_mathvec=yes + fi + +-dnl Check if linker supports static PIE with the fix for +-dnl +-dnl https://sourceware.org/bugzilla/show_bug.cgi?id=21782 +-dnl +-if test "$static_pie" = yes; then +- AC_CACHE_CHECK(for linker static PIE support, libc_cv_ld_static_pie, [dnl +-cat > conftest.s <<\EOF +- .text +- .global _start +- .weak foo +-_start: +- leaq foo(%rip), %rax +-EOF +- libc_cv_pie_option="-Wl,-pie" +- if AC_TRY_COMMAND(${CC-cc} $CFLAGS $CPPFLAGS $LDFLAGS -nostartfiles -nostdlib $no_ssp $libc_cv_pie_option -o conftest conftest.s 1>&AS_MESSAGE_LOG_FD); then +- libc_cv_ld_static_pie=yes +- else +- libc_cv_ld_static_pie=no +- fi +-rm -f conftest*]) +- if test "$libc_cv_ld_static_pie" != yes; then +- AC_MSG_ERROR([linker support for static PIE needed]) +- fi +-fi +- + dnl It is always possible to access static and hidden symbols in an + dnl position independent way. + AC_DEFINE(PI_STATIC_AND_HIDDEN) +diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h +index ca73d8fef9..363a749cb2 100644 +--- a/sysdeps/x86_64/dl-machine.h ++++ b/sysdeps/x86_64/dl-machine.h +@@ -315,16 +315,22 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc, + { + # ifndef RTLD_BOOTSTRAP + if (sym_map != map +- && sym_map->l_type != lt_executable + && !sym_map->l_relocated) + { + const char *strtab + = (const char *) D_PTR (map, l_info[DT_STRTAB]); +- _dl_error_printf ("\ ++ if (sym_map->l_type == lt_executable) ++ _dl_fatal_printf ("\ ++%s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \ ++and creates an unsatisfiable circular dependency.\n", ++ RTLD_PROGNAME, strtab + refsym->st_name, ++ map->l_name); ++ else ++ _dl_error_printf ("\ + %s: Relink `%s' with `%s' for IFUNC symbol `%s'\n", +- RTLD_PROGNAME, map->l_name, +- sym_map->l_name, +- strtab + refsym->st_name); ++ RTLD_PROGNAME, map->l_name, ++ sym_map->l_name, ++ strtab + refsym->st_name); + } + # endif + value = ((ElfW(Addr) (*) (void)) value) (); +diff --git a/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h b/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h +index 7659758972..e5fd5ac9cb 100644 +--- a/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h ++++ b/sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h +@@ -32,7 +32,7 @@ IFUNC_SELECTOR (void) + && CPU_FEATURE_USABLE_P (cpu_features, AVX2)) + return OPTIMIZE (fma); + +- if (CPU_FEATURE_USABLE_P (cpu_features, FMA)) ++ if (CPU_FEATURE_USABLE_P (cpu_features, FMA4)) + return OPTIMIZE (fma4); + + return OPTIMIZE (sse2); +diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S +index a5c879d2af..070e5ef90b 100644 +--- a/sysdeps/x86_64/memchr.S ++++ b/sysdeps/x86_64/memchr.S +@@ -21,9 +21,11 @@ + #ifdef USE_AS_WMEMCHR + # define MEMCHR wmemchr + # define PCMPEQ pcmpeqd ++# define CHAR_PER_VEC 4 + #else + # define MEMCHR memchr + # define PCMPEQ pcmpeqb ++# define CHAR_PER_VEC 16 + #endif + + /* fast SSE2 version with using pmaxub and 64 byte loop */ +@@ -33,15 +35,14 @@ ENTRY(MEMCHR) + movd %esi, %xmm1 + mov %edi, %ecx + ++#ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++#endif + #ifdef USE_AS_WMEMCHR + test %RDX_LP, %RDX_LP + jz L(return_null) +- shl $2, %RDX_LP + #else +-# ifdef __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %edx, %edx +-# endif + punpcklbw %xmm1, %xmm1 + test %RDX_LP, %RDX_LP + jz L(return_null) +@@ -60,13 +61,16 @@ ENTRY(MEMCHR) + test %eax, %eax + + jnz L(matches_1) +- sub $16, %rdx ++ sub $CHAR_PER_VEC, %rdx + jbe L(return_null) + add $16, %rdi + and $15, %ecx + and $-16, %rdi ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + add %rcx, %rdx +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + jmp L(loop_prolog) + +@@ -77,16 +81,21 @@ L(crosscache): + movdqa (%rdi), %xmm0 + + PCMPEQ %xmm1, %xmm0 +-/* Check if there is a match. */ ++ /* Check if there is a match. */ + pmovmskb %xmm0, %eax +-/* Remove the leading bytes. */ ++ /* Remove the leading bytes. */ + sar %cl, %eax + test %eax, %eax + je L(unaligned_no_match) +-/* Check which byte is a match. */ ++ /* Check which byte is a match. */ + bsf %eax, %eax +- ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + add %rdi, %rax + add %rcx, %rax +@@ -94,15 +103,18 @@ L(crosscache): + + .p2align 4 + L(unaligned_no_match): +- /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using ++ /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using + "rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void + possible addition overflow. */ + neg %rcx + add $16, %rcx ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + sub %rcx, %rdx + jbe L(return_null) + add $16, %rdi +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + + .p2align 4 +@@ -135,7 +147,7 @@ L(loop_prolog): + test $0x3f, %rdi + jz L(align64_loop) + +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + + movdqa (%rdi), %xmm0 +@@ -167,11 +179,14 @@ L(loop_prolog): + mov %rdi, %rcx + and $-64, %rdi + and $63, %ecx ++#ifdef USE_AS_WMEMCHR ++ shr $2, %ecx ++#endif + add %rcx, %rdx + + .p2align 4 + L(align64_loop): +- sub $64, %rdx ++ sub $(CHAR_PER_VEC * 4), %rdx + jbe L(exit_loop) + movdqa (%rdi), %xmm0 + movdqa 16(%rdi), %xmm2 +@@ -218,7 +233,7 @@ L(align64_loop): + + .p2align 4 + L(exit_loop): +- add $32, %edx ++ add $(CHAR_PER_VEC * 2), %edx + jle L(exit_loop_32) + + movdqa (%rdi), %xmm0 +@@ -238,7 +253,7 @@ L(exit_loop): + pmovmskb %xmm3, %eax + test %eax, %eax + jnz L(matches32_1) +- sub $16, %edx ++ sub $CHAR_PER_VEC, %edx + jle L(return_null) + + PCMPEQ 48(%rdi), %xmm1 +@@ -250,13 +265,13 @@ L(exit_loop): + + .p2align 4 + L(exit_loop_32): +- add $32, %edx ++ add $(CHAR_PER_VEC * 2), %edx + movdqa (%rdi), %xmm0 + PCMPEQ %xmm1, %xmm0 + pmovmskb %xmm0, %eax + test %eax, %eax + jnz L(matches_1) +- sub $16, %edx ++ sub $CHAR_PER_VEC, %edx + jbe L(return_null) + + PCMPEQ 16(%rdi), %xmm1 +@@ -293,7 +308,13 @@ L(matches32): + .p2align 4 + L(matches_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + add %rdi, %rax + ret +@@ -301,7 +322,13 @@ L(matches_1): + .p2align 4 + L(matches16_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 16(%rdi, %rax), %rax + ret +@@ -309,7 +336,13 @@ L(matches16_1): + .p2align 4 + L(matches32_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 32(%rdi, %rax), %rax + ret +@@ -317,7 +350,13 @@ L(matches32_1): + .p2align 4 + L(matches48_1): + bsf %eax, %eax ++#ifdef USE_AS_WMEMCHR ++ mov %eax, %esi ++ shr $2, %esi ++ sub %rsi, %rdx ++#else + sub %rax, %rdx ++#endif + jbe L(return_null) + lea 48(%rdi, %rax), %rax + ret +diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile +index 395e432c09..da1446d731 100644 +--- a/sysdeps/x86_64/multiarch/Makefile ++++ b/sysdeps/x86_64/multiarch/Makefile +@@ -43,7 +43,45 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ + memmove-avx512-unaligned-erms \ + memset-sse2-unaligned-erms \ + memset-avx2-unaligned-erms \ +- memset-avx512-unaligned-erms ++ memset-avx512-unaligned-erms \ ++ memchr-avx2-rtm \ ++ memcmp-avx2-movbe-rtm \ ++ memmove-avx-unaligned-erms-rtm \ ++ memrchr-avx2-rtm \ ++ memset-avx2-unaligned-erms-rtm \ ++ rawmemchr-avx2-rtm \ ++ strchr-avx2-rtm \ ++ strcmp-avx2-rtm \ ++ strchrnul-avx2-rtm \ ++ stpcpy-avx2-rtm \ ++ stpncpy-avx2-rtm \ ++ strcat-avx2-rtm \ ++ strcpy-avx2-rtm \ ++ strlen-avx2-rtm \ ++ strncat-avx2-rtm \ ++ strncmp-avx2-rtm \ ++ strncpy-avx2-rtm \ ++ strnlen-avx2-rtm \ ++ strrchr-avx2-rtm \ ++ memchr-evex \ ++ memcmp-evex-movbe \ ++ memmove-evex-unaligned-erms \ ++ memrchr-evex \ ++ memset-evex-unaligned-erms \ ++ rawmemchr-evex \ ++ stpcpy-evex \ ++ stpncpy-evex \ ++ strcat-evex \ ++ strchr-evex \ ++ strchrnul-evex \ ++ strcmp-evex \ ++ strcpy-evex \ ++ strlen-evex \ ++ strncat-evex \ ++ strncmp-evex \ ++ strncpy-evex \ ++ strnlen-evex \ ++ strrchr-evex + CFLAGS-varshift.c += -msse4 + CFLAGS-strcspn-c.c += -msse4 + CFLAGS-strpbrk-c.c += -msse4 +@@ -59,8 +97,24 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ + wcscpy-ssse3 wcscpy-c \ + wcschr-sse2 wcschr-avx2 \ + wcsrchr-sse2 wcsrchr-avx2 \ +- wcsnlen-sse4_1 wcsnlen-c \ +- wcslen-sse2 wcslen-avx2 wcsnlen-avx2 ++ wcslen-sse2 wcslen-sse4_1 wcslen-avx2 \ ++ wcsnlen-c wcsnlen-sse4_1 wcsnlen-avx2 \ ++ wcschr-avx2-rtm \ ++ wcscmp-avx2-rtm \ ++ wcslen-avx2-rtm \ ++ wcsncmp-avx2-rtm \ ++ wcsnlen-avx2-rtm \ ++ wcsrchr-avx2-rtm \ ++ wmemchr-avx2-rtm \ ++ wmemcmp-avx2-movbe-rtm \ ++ wcschr-evex \ ++ wcscmp-evex \ ++ wcslen-evex \ ++ wcsncmp-evex \ ++ wcsnlen-evex \ ++ wcsrchr-evex \ ++ wmemchr-evex \ ++ wmemcmp-evex-movbe + endif + + ifeq ($(subdir),debug) +diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h +index f4e311d470..f450c786f0 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-avx2.h ++++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h +@@ -21,16 +21,28 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + return OPTIMIZE (sse2); + } +diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +index f93ec39d98..920e64241e 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c +@@ -43,6 +43,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memchr, + CPU_FEATURE_USABLE (AVX2), + __memchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, memchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __memchr_evex) + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/memcmp.c. */ +@@ -51,6 +60,16 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __memcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, memcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (MOVBE) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcmp_avx2_movbe_rtm) ++ IFUNC_IMPL_ADD (array, i, memcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (MOVBE)), ++ __memcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1), + __memcmp_sse4_1) + IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3), +@@ -64,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memmove_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __memmove_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memmove_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX), +@@ -75,6 +94,20 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (AVX), + __memmove_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_chk_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memmove_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memmove_chk, + CPU_FEATURE_USABLE (SSSE3), + __memmove_chk_ssse3_back) +@@ -97,14 +130,28 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX), + __memmove_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memmove_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memmove, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memmove_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memmove, + CPU_FEATURE_USABLE (AVX512F), + __memmove_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, memmove, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memmove, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memmove_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3), + __memmove_ssse3_back) +@@ -121,6 +168,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memrchr, + CPU_FEATURE_USABLE (AVX2), + __memrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, memrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memrchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, memrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memrchr_evex) ++ + IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_sse2)) + + #ifdef SHARED +@@ -139,10 +195,28 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX2), + __memset_chk_avx2_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_chk_avx2_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_chk_avx2_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_chk_evex_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memset_chk, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memset_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memset_chk, + CPU_FEATURE_USABLE (AVX512F), +@@ -164,10 +238,28 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX2), + __memset_avx2_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_avx2_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memset_avx2_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __memset_evex_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memset, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memset, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), + __memset_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memset, + CPU_FEATURE_USABLE (AVX512F), +@@ -179,20 +271,51 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, rawmemchr, + CPU_FEATURE_USABLE (AVX2), + __rawmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, rawmemchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __rawmemchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, rawmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __rawmemchr_evex) + IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/strlen.c. */ + IFUNC_IMPL (i, name, strlen, + IFUNC_IMPL_ADD (array, i, strlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strlen_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strlen_evex) + IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2)) + + /* Support sysdeps/x86_64/multiarch/strnlen.c. */ + IFUNC_IMPL (i, name, strnlen, + IFUNC_IMPL_ADD (array, i, strnlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __strnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, strnlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strnlen_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strnlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strnlen_evex) + IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2)) + + /* Support sysdeps/x86_64/multiarch/stpncpy.c. */ +@@ -201,6 +324,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpncpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), + __stpncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpncpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __stpncpy_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, stpncpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __stpncpy_evex) + IFUNC_IMPL_ADD (array, i, stpncpy, 1, + __stpncpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, stpncpy, 1, __stpncpy_sse2)) +@@ -211,6 +342,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + __stpcpy_ssse3) + IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), + __stpcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, stpcpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __stpcpy_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, stpcpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __stpcpy_evex) + IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2)) + +@@ -245,6 +384,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcat, + IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), + __strcat_avx2) ++ IFUNC_IMPL_ADD (array, i, strcat, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcat_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strcat, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strcat_evex) + IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (SSSE3), + __strcat_ssse3) + IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned) +@@ -255,6 +402,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchr, + CPU_FEATURE_USABLE (AVX2), + __strchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strchr_evex) + IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf) + IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2)) + +@@ -263,6 +419,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchrnul, + CPU_FEATURE_USABLE (AVX2), + __strchrnul_avx2) ++ IFUNC_IMPL_ADD (array, i, strchrnul, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strchrnul_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strchrnul, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strchrnul_evex) + IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2)) + + /* Support sysdeps/x86_64/multiarch/strrchr.c. */ +@@ -270,6 +435,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strrchr, + CPU_FEATURE_USABLE (AVX2), + __strrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, strrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strrchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strrchr_evex) + IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/strcmp.c. */ +@@ -277,6 +450,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strcmp, + CPU_FEATURE_USABLE (AVX2), + __strcmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcmp_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __strcmp_evex) + IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSE4_2), + __strcmp_sse42) + IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSSE3), +@@ -288,6 +470,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strcpy, + IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), + __strcpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strcpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strcpy_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strcpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strcpy_evex) + IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (SSSE3), + __strcpy_ssse3) + IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned) +@@ -331,6 +521,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncat, + IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), + __strncat_avx2) ++ IFUNC_IMPL_ADD (array, i, strncat, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncat_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strncat, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncat_evex) + IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (SSSE3), + __strncat_ssse3) + IFUNC_IMPL_ADD (array, i, strncat, 1, +@@ -341,6 +539,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL (i, name, strncpy, + IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), + __strncpy_avx2) ++ IFUNC_IMPL_ADD (array, i, strncpy, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncpy_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strncpy, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncpy_evex) + IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (SSSE3), + __strncpy_ssse3) + IFUNC_IMPL_ADD (array, i, strncpy, 1, +@@ -370,6 +576,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcschr, + CPU_FEATURE_USABLE (AVX2), + __wcschr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcschr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcschr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcschr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcschr_evex) + IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsrchr.c. */ +@@ -377,6 +592,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsrchr, + CPU_FEATURE_USABLE (AVX2), + __wcsrchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsrchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsrchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcsrchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsrchr_evex) + IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcscmp.c. */ +@@ -384,6 +608,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcscmp, + CPU_FEATURE_USABLE (AVX2), + __wcscmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcscmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcscmp_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcscmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcscmp_evex) + IFUNC_IMPL_ADD (array, i, wcscmp, 1, __wcscmp_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsncmp.c. */ +@@ -391,6 +624,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wcsncmp, + CPU_FEATURE_USABLE (AVX2), + __wcsncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsncmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsncmp_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcsncmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsncmp_evex) + IFUNC_IMPL_ADD (array, i, wcsncmp, 1, __wcsncmp_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcscpy.c. */ +@@ -402,15 +644,40 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + /* Support sysdeps/x86_64/multiarch/wcslen.c. */ + IFUNC_IMPL (i, name, wcslen, + IFUNC_IMPL_ADD (array, i, wcslen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __wcslen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcslen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcslen_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcslen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcslen_evex) ++ IFUNC_IMPL_ADD (array, i, wcslen, ++ CPU_FEATURE_USABLE (SSE4_1), ++ __wcslen_sse4_1) + IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2)) + + /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */ + IFUNC_IMPL (i, name, wcsnlen, + IFUNC_IMPL_ADD (array, i, wcsnlen, +- CPU_FEATURE_USABLE (AVX2), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2)), + __wcsnlen_avx2) ++ IFUNC_IMPL_ADD (array, i, wcsnlen, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (BMI2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wcsnlen_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wcsnlen, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wcsnlen_evex) + IFUNC_IMPL_ADD (array, i, wcsnlen, + CPU_FEATURE_USABLE (SSE4_1), + __wcsnlen_sse4_1) +@@ -421,6 +688,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, wmemchr, + CPU_FEATURE_USABLE (AVX2), + __wmemchr_avx2) ++ IFUNC_IMPL_ADD (array, i, wmemchr, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemchr_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, wmemchr, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (BMI2)), ++ __wmemchr_evex) + IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) + + /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */ +@@ -429,6 +705,16 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE)), + __wmemcmp_avx2_movbe) ++ IFUNC_IMPL_ADD (array, i, wmemcmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (MOVBE) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemcmp_avx2_movbe_rtm) ++ IFUNC_IMPL_ADD (array, i, wmemcmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW) ++ && CPU_FEATURE_USABLE (MOVBE)), ++ __wmemcmp_evex_movbe) + IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1), + __wmemcmp_sse4_1) + IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3), +@@ -443,7 +729,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX2), + __wmemset_avx2_unaligned) + IFUNC_IMPL_ADD (array, i, wmemset, +- CPU_FEATURE_USABLE (AVX512F), ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __wmemset_avx2_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, wmemset, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __wmemset_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, wmemset, ++ CPU_FEATURE_USABLE (AVX512VL), + __wmemset_avx512_unaligned)) + + #ifdef SHARED +@@ -453,10 +746,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memcpy_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX), +@@ -464,6 +757,20 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (AVX), + __memcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_chk_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __memcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, + CPU_FEATURE_USABLE (SSSE3), + __memcpy_chk_ssse3_back) +@@ -486,6 +793,20 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, memcpy, + CPU_FEATURE_USABLE (AVX), + __memcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __memcpy_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, memcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __memcpy_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), + __memcpy_ssse3_back) + IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3), +@@ -494,10 +815,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __memcpy_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, memcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, memcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __memcpy_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned) + IFUNC_IMPL_ADD (array, i, memcpy, 1, +@@ -511,10 +832,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __mempcpy_chk_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_chk_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_chk_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX), +@@ -522,6 +843,20 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (AVX), + __mempcpy_chk_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_chk_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_chk_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_chk_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, __mempcpy_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_chk_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, + CPU_FEATURE_USABLE (SSSE3), + __mempcpy_chk_ssse3_back) +@@ -542,10 +877,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + CPU_FEATURE_USABLE (AVX512F), + __mempcpy_avx512_no_vzeroupper) + IFUNC_IMPL_ADD (array, i, mempcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_avx512_unaligned) + IFUNC_IMPL_ADD (array, i, mempcpy, +- CPU_FEATURE_USABLE (AVX512F), ++ CPU_FEATURE_USABLE (AVX512VL), + __mempcpy_avx512_unaligned_erms) + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX), +@@ -553,6 +888,20 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, mempcpy, + CPU_FEATURE_USABLE (AVX), + __mempcpy_avx_unaligned_erms) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_avx_unaligned_rtm) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ (CPU_FEATURE_USABLE (AVX) ++ && CPU_FEATURE_USABLE (RTM)), ++ __mempcpy_avx_unaligned_erms_rtm) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_evex_unaligned) ++ IFUNC_IMPL_ADD (array, i, mempcpy, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __mempcpy_evex_unaligned_erms) + IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), + __mempcpy_ssse3_back) + IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3), +@@ -568,6 +917,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strncmp, + CPU_FEATURE_USABLE (AVX2), + __strncmp_avx2) ++ IFUNC_IMPL_ADD (array, i, strncmp, ++ (CPU_FEATURE_USABLE (AVX2) ++ && CPU_FEATURE_USABLE (RTM)), ++ __strncmp_avx2_rtm) ++ IFUNC_IMPL_ADD (array, i, strncmp, ++ (CPU_FEATURE_USABLE (AVX512VL) ++ && CPU_FEATURE_USABLE (AVX512BW)), ++ __strncmp_evex) + IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSE4_2), + __strncmp_sse42) + IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSSE3), +@@ -582,6 +939,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, __wmemset_chk, + CPU_FEATURE_USABLE (AVX2), + __wmemset_chk_avx2_unaligned) ++ IFUNC_IMPL_ADD (array, i, __wmemset_chk, ++ CPU_FEATURE_USABLE (AVX512VL), ++ __wmemset_chk_evex_unaligned) + IFUNC_IMPL_ADD (array, i, __wmemset_chk, + CPU_FEATURE_USABLE (AVX512F), + __wmemset_chk_avx512_unaligned)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +index 0e21b3a628..4f96c2764a 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memcmp.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h +@@ -23,17 +23,28 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_movbe) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURE_USABLE_P (cpu_features, MOVBE) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2_movbe); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex_movbe); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_movbe_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2_movbe); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) + return OPTIMIZE (sse4_1); +diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h +index 9ada03aa43..db26210e3b 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h +@@ -29,6 +29,14 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms) +@@ -48,21 +56,42 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { +- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx512_no_vzeroupper); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx512_unaligned_erms); + +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx512_unaligned_erms); ++ return OPTIMIZE (avx512_unaligned); ++ } + +- return OPTIMIZE (avx512_unaligned); ++ return OPTIMIZE (avx512_no_vzeroupper); + } + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx_unaligned_erms); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (evex_unaligned_erms); ++ ++ return OPTIMIZE (evex_unaligned); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx_unaligned_erms_rtm); ++ ++ return OPTIMIZE (avx_unaligned_rtm); ++ } ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx_unaligned_erms); + +- return OPTIMIZE (avx_unaligned); ++ return OPTIMIZE (avx_unaligned); ++ } + } + + if (!CPU_FEATURE_USABLE_P (cpu_features, SSSE3) +diff --git a/sysdeps/x86_64/multiarch/ifunc-memset.h b/sysdeps/x86_64/multiarch/ifunc-memset.h +index f52613d372..57029fc17b 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-memset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-memset.h +@@ -27,6 +27,14 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms) + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms) + attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_erms_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms) ++ attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms) +@@ -45,21 +53,44 @@ IFUNC_SELECTOR (void) + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { +- if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) +- return OPTIMIZE (avx512_no_vzeroupper); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx512_unaligned_erms); + +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx512_unaligned_erms); ++ return OPTIMIZE (avx512_unaligned); ++ } + +- return OPTIMIZE (avx512_unaligned); ++ return OPTIMIZE (avx512_no_vzeroupper); + } + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +- return OPTIMIZE (avx2_unaligned_erms); +- else +- return OPTIMIZE (avx2_unaligned); ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (evex_unaligned_erms); ++ ++ return OPTIMIZE (evex_unaligned); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx2_unaligned_erms_rtm); ++ ++ return OPTIMIZE (avx2_unaligned_rtm); ++ } ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) ++ return OPTIMIZE (avx2_unaligned_erms); ++ ++ return OPTIMIZE (avx2_unaligned); ++ } + } + + if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) +diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +index 63b0dc0d96..35741f3ec8 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h ++++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h +@@ -25,16 +25,27 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) + attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + return OPTIMIZE (sse2_unaligned); +diff --git a/sysdeps/x86_64/multiarch/ifunc-wcslen.h b/sysdeps/x86_64/multiarch/ifunc-wcslen.h +new file mode 100644 +index 0000000000..39e3347378 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/ifunc-wcslen.h +@@ -0,0 +1,52 @@ ++/* Common definition for ifunc selections for wcslen and wcsnlen ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2017-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <init-arch.h> ++ ++extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; ++ ++static inline void * ++IFUNC_SELECTOR (void) ++{ ++ const struct cpu_features* cpu_features = __get_cpu_features (); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) ++ return OPTIMIZE (sse4_1); ++ ++ return OPTIMIZE (sse2); ++} +diff --git a/sysdeps/x86_64/multiarch/ifunc-wmemset.h b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +index 8cfce562fc..e06e8b4d80 100644 +--- a/sysdeps/x86_64/multiarch/ifunc-wmemset.h ++++ b/sysdeps/x86_64/multiarch/ifunc-wmemset.h +@@ -20,6 +20,9 @@ + + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_unaligned_rtm) ++ attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) attribute_hidden; + + static inline void * +@@ -27,14 +30,21 @@ IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { +- if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) +- && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) +- return OPTIMIZE (avx512_unaligned); +- else ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)) ++ { ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) ++ return OPTIMIZE (avx512_unaligned); ++ ++ return OPTIMIZE (evex_unaligned); ++ } ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_unaligned_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx2_unaligned); + } + +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S +new file mode 100644 +index 0000000000..87b076c7c4 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMCHR ++# define MEMCHR __memchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S +index e5a9abd211..0987616a1b 100644 +--- a/sysdeps/x86_64/multiarch/memchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memchr-avx2.S +@@ -26,319 +26,407 @@ + + # ifdef USE_AS_WMEMCHR + # define VPCMPEQ vpcmpeqd ++# define VPBROADCAST vpbroadcastd ++# define CHAR_SIZE 4 + # else + # define VPCMPEQ vpcmpeqb ++# define VPBROADCAST vpbroadcastb ++# define CHAR_SIZE 1 ++# endif ++ ++# ifdef USE_AS_RAWMEMCHR ++# define ERAW_PTR_REG ecx ++# define RRAW_PTR_REG rcx ++# define ALGN_PTR_REG rdi ++# else ++# define ERAW_PTR_REG edi ++# define RRAW_PTR_REG rdi ++# define ALGN_PTR_REG rcx + # endif + + # ifndef VZEROUPPER + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (MEMCHR) + # ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ ++# ifdef __ILP32__ ++ /* Clear upper bits. */ ++ and %RDX_LP, %RDX_LP ++# else + test %RDX_LP, %RDX_LP ++# endif + jz L(null) + # endif +- movl %edi, %ecx +- /* Broadcast CHAR to YMM0. */ ++ /* Broadcast CHAR to YMMMATCH. */ + vmovd %esi, %xmm0 +-# ifdef USE_AS_WMEMCHR +- shl $2, %RDX_LP +- vpbroadcastd %xmm0, %ymm0 +-# else +-# ifdef __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %edx, %edx +-# endif +- vpbroadcastb %xmm0, %ymm0 +-# endif ++ VPBROADCAST %xmm0, %ymm0 + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++ VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- + # ifndef USE_AS_RAWMEMCHR +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rdx +- jbe L(zero) +-# else +- jnz L(first_vec_x0) ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $CHAR_PER_VEC, %rdx ++ jbe L(first_vec_x0) + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + + # ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ +- addq %rcx, %rdx ++ .p2align 5 ++L(first_vec_x0): ++ /* Check if first match was before length. */ ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif ++ xorl %ecx, %ecx ++ cmpl %eax, %edx ++ leaq (%rdi, %rax), %rax ++ cmovle %rcx, %rax ++ VZEROUPPER_RETURN + +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) ++L(null): ++ xorl %eax, %eax ++ ret + # endif +- jmp L(more_4x_vec) +- + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++L(cross_page_boundary): ++ /* Save pointer before aligning as its original value is ++ necessary for computer return address if byte is found or ++ adjusting length if it is not and this is memchr. */ ++ movq %rdi, %rcx ++ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr ++ and rdi for rawmemchr. */ ++ orq $(VEC_SIZE - 1), %ALGN_PTR_REG ++ VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Calculate length until end of page (length checked for a ++ match). */ ++ leaq 1(%ALGN_PTR_REG), %rsi ++ subq %RRAW_PTR_REG, %rsi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %esi ++# endif ++# endif + /* Remove the leading bytes. */ +- sarl %cl, %eax +- testl %eax, %eax +- jz L(aligned_more) +- tzcntl %eax, %eax ++ sarxl %ERAW_PTR_REG, %eax, %eax + # ifndef USE_AS_RAWMEMCHR + /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) ++ cmpq %rsi, %rdx ++ jbe L(first_vec_x0) + # endif ++ testl %eax, %eax ++ jz L(cross_page_continue) ++ tzcntl %eax, %eax ++ addq %RRAW_PTR_REG, %rax ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ incq %rdi + addq %rdi, %rax +- addq %rcx, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 +-L(aligned_more): +-# ifndef USE_AS_RAWMEMCHR +- /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)" +- instead of "(rdx + rcx) - VEC_SIZE" to void possible addition +- overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- /* Check the end of data. */ +- subq %rcx, %rdx +- jbe L(zero) +-# endif ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE * 2 + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +- addq $VEC_SIZE, %rdi + +-# ifndef USE_AS_RAWMEMCHR +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ addq $(VEC_SIZE * 3 + 1), %rdi ++ addq %rdi, %rax ++ VZEROUPPER_RETURN + +-L(more_4x_vec): ++ .p2align 4 ++L(aligned_more): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++# ifndef USE_AS_RAWMEMCHR ++L(cross_page_continue): ++ /* Align data to VEC_SIZE - 1. */ ++ xorl %ecx, %ecx ++ subl %edi, %ecx ++ orq $(VEC_SIZE - 1), %rdi ++ /* esi is for adjusting length to see if near the end. */ ++ leal (VEC_SIZE * 4 + 1)(%rdi, %rcx), %esi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %esi ++# endif ++# else ++ orq $(VEC_SIZE - 1), %rdi ++L(cross_page_continue): ++# endif ++ /* Load first VEC regardless. */ ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. If near end handle specially. */ ++ subq %rsi, %rdx ++ jbe L(last_4x_vec_or_less) ++# endif + testl %eax, %eax + jnz L(first_vec_x1) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi +- +-# ifndef USE_AS_RAWMEMCHR +- subq $(VEC_SIZE * 4), %rdx +- jbe L(last_4x_vec_or_less) +-# endif +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) + + # ifndef USE_AS_RAWMEMCHR +- /* Adjust length. */ ++ /* Check if at last VEC_SIZE * 4 length. */ ++ subq $(CHAR_PER_VEC * 4), %rdx ++ jbe L(last_4x_vec_or_less_cmpeq) ++ /* Align data to VEC_SIZE * 4 - 1 for the loop and readjust ++ length. */ ++ incq %rdi ++ movl %edi, %ecx ++ orq $(VEC_SIZE * 4 - 1), %rdi ++ andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif + addq %rcx, %rdx ++# else ++ /* Align data to VEC_SIZE * 4 - 1 for loop. */ ++ incq %rdi ++ orq $(VEC_SIZE * 4 - 1), %rdi + # endif + ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2 +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3 +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4 +- ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm2 ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm3 ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm4 + vpor %ymm1, %ymm2, %ymm5 + vpor %ymm3, %ymm4, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + +- vpmovmskb %ymm5, %eax +- testl %eax, %eax +- jnz L(4x_vec_end) +- +- addq $(VEC_SIZE * 4), %rdi +- ++ vpmovmskb %ymm5, %ecx + # ifdef USE_AS_RAWMEMCHR +- jmp L(loop_4x_vec) ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx ++ jz L(loop_4x_vec) + # else +- subq $(VEC_SIZE * 4), %rdx +- ja L(loop_4x_vec) ++ testl %ecx, %ecx ++ jnz L(loop_4x_vec_end) + +-L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %edx +- jle L(last_2x_vec) ++ subq $-(VEC_SIZE * 4), %rdi + +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) ++ subq $(CHAR_PER_VEC * 4), %rdx ++ ja L(loop_4x_vec) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++ /* Fall through into less than 4 remaining vectors of length ++ case. */ ++ VPCMPEQ (VEC_SIZE * 0 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++ .p2align 4 ++L(last_4x_vec_or_less): ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif ++ /* Check if first VEC contained match. */ + testl %eax, %eax +- jnz L(first_vec_x1) ++ jnz L(first_vec_x1_check) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax ++ /* If remaining length > VEC_SIZE * 2. */ ++ addl $(VEC_SIZE * 2), %edx ++ jg L(last_4x_vec) + +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %edx +- jle L(zero) ++L(last_2x_vec): ++ /* If remaining length < VEC_SIZE. */ ++ addl $VEC_SIZE, %edx ++ jle L(zero_end) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 ++ /* Check VEC2 and compare any match with remaining length. */ ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- +- jnz L(first_vec_x3_check) +- xorl %eax, %eax +- VZEROUPPER +- ret ++ tzcntl %eax, %eax ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ addq $(VEC_SIZE + 1), %rdi ++ addq %rdi, %rax ++L(zero_end): ++ VZEROUPPER_RETURN + + .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %edx +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++L(loop_4x_vec_end): ++# endif ++ /* rawmemchr will fall through into this if match was found in ++ loop. */ ++ + vpmovmskb %ymm1, %eax + testl %eax, %eax ++ jnz L(last_vec_x1_return) + +- jnz L(first_vec_x0_check) +- subl $VEC_SIZE, %edx +- jle L(zero) +- +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ vpmovmskb %ymm2, %eax + testl %eax, %eax +- jnz L(first_vec_x1_check) +- xorl %eax, %eax +- VZEROUPPER +- ret ++ jnz L(last_vec_x2_return) + +- .p2align 4 +-L(first_vec_x0_check): +- tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) ++ vpmovmskb %ymm3, %eax ++ /* Combine VEC3 matches (eax) with VEC4 matches (ecx). */ ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 2 - 1), %rdi ++# else ++ subq $-(VEC_SIZE * 2 + 1), %rdi ++# endif + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN ++# ifndef USE_AS_RAWMEMCHR + + .p2align 4 + L(first_vec_x1_check): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $VEC_SIZE, %rax ++ /* Adjust length. */ ++ subl $-(VEC_SIZE * 4), %edx ++ /* Check if match within remaining length. */ ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ incq %rdi + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN ++ .p2align 4 ++L(set_zero_end): ++ xorl %eax, %eax ++ VZEROUPPER_RETURN ++# endif + + .p2align 4 +-L(first_vec_x2_check): ++L(last_vec_x1_return): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 2), %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 4 - 1), %rdi ++# else ++ incq %rdi ++# endif + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x2_return): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rdx +- jbe L(zero) +- addq $(VEC_SIZE * 3), %rax ++# ifdef USE_AS_RAWMEMCHR ++ subq $(VEC_SIZE * 3 - 1), %rdi ++# else ++ subq $-(VEC_SIZE + 1), %rdi ++# endif + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + ++# ifndef USE_AS_RAWMEMCHR + .p2align 4 +-L(zero): +- VZEROUPPER +-L(null): +- xorl %eax, %eax +- ret +-# endif ++L(last_4x_vec_or_less_cmpeq): ++ VPCMPEQ (VEC_SIZE * 4 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %edx ++# endif ++ subq $-(VEC_SIZE * 4), %rdi ++ /* Check first VEC regardless. */ ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) + ++ /* If remaining length <= CHAR_PER_VEC * 2. */ ++ addl $(VEC_SIZE * 2), %edx ++ jle L(last_2x_vec) + .p2align 4 +-L(first_vec_x0): +- tzcntl %eax, %eax +- addq %rdi, %rax +- VZEROUPPER +- ret ++L(last_4x_vec): ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2_return) + +- .p2align 4 +-L(first_vec_x1): +- tzcntl %eax, %eax +- addq $VEC_SIZE, %rax +- addq %rdi, %rax +- VZEROUPPER +- ret ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax + +- .p2align 4 +-L(first_vec_x2): ++ /* Create mask for possible matches within remaining length. */ ++ movq $-1, %rcx ++ bzhiq %rdx, %rcx, %rcx ++ ++ /* Test matches in data against length match. */ ++ andl %ecx, %eax ++ jnz L(last_vec_x3) ++ ++ /* if remaining length <= VEC_SIZE * 3 (Note this is after ++ remaining length was found to be > VEC_SIZE * 2. */ ++ subl $VEC_SIZE, %edx ++ jbe L(zero_end2) ++ ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ /* Shift remaining length mask for last VEC. */ ++ shrq $32, %rcx ++ andl %ecx, %eax ++ jz L(zero_end2) + tzcntl %eax, %eax +- addq $(VEC_SIZE * 2), %rax ++ addq $(VEC_SIZE * 3 + 1), %rdi + addq %rdi, %rax +- VZEROUPPER +- ret ++L(zero_end2): ++ VZEROUPPER_RETURN + + .p2align 4 +-L(4x_vec_end): +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- vpmovmskb %ymm2, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- vpmovmskb %ymm3, %eax +- testl %eax, %eax +- jnz L(first_vec_x2) +- vpmovmskb %ymm4, %eax +- testl %eax, %eax +-L(first_vec_x3): ++L(last_vec_x3): + tzcntl %eax, %eax +- addq $(VEC_SIZE * 3), %rax ++ subq $-(VEC_SIZE * 2 + 1), %rdi + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN ++# endif + + END (MEMCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/memchr-evex.S b/sysdeps/x86_64/multiarch/memchr-evex.S +new file mode 100644 +index 0000000000..f3fdad4fda +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memchr-evex.S +@@ -0,0 +1,478 @@ ++/* memchr/wmemchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef MEMCHR ++# define MEMCHR __memchr_evex ++# endif ++ ++# ifdef USE_AS_WMEMCHR ++# define VPBROADCAST vpbroadcastd ++# define VPMINU vpminud ++# define VPCMP vpcmpd ++# define VPCMPEQ vpcmpeqd ++# define CHAR_SIZE 4 ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPMINU vpminub ++# define VPCMP vpcmpb ++# define VPCMPEQ vpcmpeqb ++# define CHAR_SIZE 1 ++# endif ++ ++# ifdef USE_AS_RAWMEMCHR ++# define RAW_PTR_REG rcx ++# define ALGN_PTR_REG rdi ++# else ++# define RAW_PTR_REG rdi ++# define ALGN_PTR_REG rcx ++# endif ++ ++# define XMMZERO xmm23 ++# define YMMZERO ymm23 ++# define XMMMATCH xmm16 ++# define YMMMATCH ymm16 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) ++# define PAGE_SIZE 4096 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (MEMCHR) ++# ifndef USE_AS_RAWMEMCHR ++ /* Check for zero length. */ ++ test %RDX_LP, %RDX_LP ++ jz L(zero) ++ ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++# endif ++ /* Broadcast CHAR to YMMMATCH. */ ++ VPBROADCAST %esi, %YMMMATCH ++ /* Check if we may cross page boundary with one vector load. */ ++ movl %edi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. */ ++ VPCMP $0, (%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $CHAR_PER_VEC, %rdx ++ jbe L(first_vec_x0) ++# endif ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++# else ++ addq %rdi, %rax ++# endif ++ ret ++ ++# ifndef USE_AS_RAWMEMCHR ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 5 ++L(first_vec_x0): ++ /* Check if first match was before length. */ ++ tzcntl %eax, %eax ++ xorl %ecx, %ecx ++ cmpl %eax, %edx ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++ cmovle %rcx, %rax ++ ret ++# else ++ /* NB: first_vec_x0 is 17 bytes which will leave ++ cross_page_boundary (which is relatively cold) close enough ++ to ideal alignment. So only realign L(cross_page_boundary) if ++ rawmemchr. */ ++ .p2align 4 ++# endif ++L(cross_page_boundary): ++ /* Save pointer before aligning as its original value is ++ necessary for computer return address if byte is found or ++ adjusting length if it is not and this is memchr. */ ++ movq %rdi, %rcx ++ /* Align data to VEC_SIZE. ALGN_PTR_REG is rcx for memchr and rdi ++ for rawmemchr. */ ++ andq $-VEC_SIZE, %ALGN_PTR_REG ++ VPCMP $0, (%ALGN_PTR_REG), %YMMMATCH, %k0 ++ kmovd %k0, %r8d ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide shift count by 4 since each bit in K0 represent 4 ++ bytes. */ ++ sarl $2, %eax ++# endif ++# ifndef USE_AS_RAWMEMCHR ++ movl $(PAGE_SIZE / CHAR_SIZE), %esi ++ subl %eax, %esi ++# endif ++# ifdef USE_AS_WMEMCHR ++ andl $(CHAR_PER_VEC - 1), %eax ++# endif ++ /* Remove the leading bytes. */ ++ sarxl %eax, %r8d, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Check the end of data. */ ++ cmpq %rsi, %rdx ++ jbe L(first_vec_x0) ++# endif ++ testl %eax, %eax ++ jz L(cross_page_continue) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%RAW_PTR_REG, %rax, CHAR_SIZE), %rax ++# else ++ addq %RAW_PTR_REG, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 5 ++L(aligned_more): ++ /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ ++# ifndef USE_AS_RAWMEMCHR ++ /* Align data to VEC_SIZE. */ ++L(cross_page_continue): ++ xorl %ecx, %ecx ++ subl %edi, %ecx ++ andq $-VEC_SIZE, %rdi ++ /* esi is for adjusting length to see if near the end. */ ++ leal (VEC_SIZE * 5)(%rdi, %rcx), %esi ++# ifdef USE_AS_WMEMCHR ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %esi ++# endif ++# else ++ andq $-VEC_SIZE, %rdi ++L(cross_page_continue): ++# endif ++ /* Load first VEC regardless. */ ++ VPCMP $0, (VEC_SIZE)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++# ifndef USE_AS_RAWMEMCHR ++ /* Adjust length. If near end handle specially. */ ++ subq %rsi, %rdx ++ jbe L(last_4x_vec_or_less) ++# endif ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) ++ ++ ++# ifndef USE_AS_RAWMEMCHR ++ /* Check if at last CHAR_PER_VEC * 4 length. */ ++ subq $(CHAR_PER_VEC * 4), %rdx ++ jbe L(last_4x_vec_or_less_cmpeq) ++ addq $VEC_SIZE, %rdi ++ ++ /* Align data to VEC_SIZE * 4 for the loop and readjust length. ++ */ ++# ifdef USE_AS_WMEMCHR ++ movl %edi, %ecx ++ andq $-(4 * VEC_SIZE), %rdi ++ andl $(VEC_SIZE * 4 - 1), %ecx ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++ addq %rcx, %rdx ++# else ++ addq %rdi, %rdx ++ andq $-(4 * VEC_SIZE), %rdi ++ subq %rdi, %rdx ++# endif ++# else ++ addq $VEC_SIZE, %rdi ++ andq $-(4 * VEC_SIZE), %rdi ++# endif ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Compare 4 * VEC at a time forward. */ ++ .p2align 4 ++L(loop_4x_vec): ++ /* It would be possible to save some instructions using 4x VPCMP ++ but bottleneck on port 5 makes it not woth it. */ ++ VPCMP $4, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k1 ++ /* xor will set bytes match esi to zero. */ ++ vpxorq (VEC_SIZE * 5)(%rdi), %YMMMATCH, %YMM2 ++ vpxorq (VEC_SIZE * 6)(%rdi), %YMMMATCH, %YMM3 ++ VPCMP $0, (VEC_SIZE * 7)(%rdi), %YMMMATCH, %k3 ++ /* Reduce VEC2 / VEC3 with min and VEC1 with zero mask. */ ++ VPMINU %YMM2, %YMM3, %YMM3{%k1}{z} ++ VPCMP $0, %YMM3, %YMMZERO, %k2 ++# ifdef USE_AS_RAWMEMCHR ++ subq $-(VEC_SIZE * 4), %rdi ++ kortestd %k2, %k3 ++ jz L(loop_4x_vec) ++# else ++ kortestd %k2, %k3 ++ jnz L(loop_4x_vec_end) ++ ++ subq $-(VEC_SIZE * 4), %rdi ++ ++ subq $(CHAR_PER_VEC * 4), %rdx ++ ja L(loop_4x_vec) ++ ++ /* Fall through into less than 4 remaining vectors of length case. ++ */ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ addq $(VEC_SIZE * 3), %rdi ++ .p2align 4 ++L(last_4x_vec_or_less): ++ /* Check if first VEC contained match. */ ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) ++ ++ /* If remaining length > CHAR_PER_VEC * 2. */ ++ addl $(CHAR_PER_VEC * 2), %edx ++ jg L(last_4x_vec) ++ ++L(last_2x_vec): ++ /* If remaining length < CHAR_PER_VEC. */ ++ addl $CHAR_PER_VEC, %edx ++ jle L(zero_end) ++ ++ /* Check VEC2 and compare any match with remaining length. */ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++L(zero_end): ++ ret ++ ++ ++ .p2align 4 ++L(first_vec_x1_check): ++ tzcntl %eax, %eax ++ /* Adjust length. */ ++ subl $-(CHAR_PER_VEC * 4), %edx ++ /* Check if match within remaining length. */ ++ cmpl %eax, %edx ++ jbe L(set_zero_end) ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++L(set_zero_end): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(loop_4x_vec_end): ++# endif ++ /* rawmemchr will fall through into this if match was found in ++ loop. */ ++ ++ /* k1 has not of matches with VEC1. */ ++ kmovd %k1, %eax ++# ifdef USE_AS_WMEMCHR ++ subl $((1 << CHAR_PER_VEC) - 1), %eax ++# else ++ incl %eax ++# endif ++ jnz L(last_vec_x1_return) ++ ++ VPCMP $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2_return) ++ ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3_return) ++ ++ kmovd %k3, %eax ++ tzcntl %eax, %eax ++# ifdef USE_AS_RAWMEMCHR ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++# else ++ leaq (VEC_SIZE * 7)(%rdi, %rax, CHAR_SIZE), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(last_vec_x1_return): ++ tzcntl %eax, %eax ++# ifdef USE_AS_RAWMEMCHR ++# ifdef USE_AS_WMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (%rdi, %rax, CHAR_SIZE), %rax ++# else ++ addq %rdi, %rax ++# endif ++# else ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(last_vec_x2_return): ++ tzcntl %eax, %eax ++# ifdef USE_AS_RAWMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq VEC_SIZE(%rdi, %rax, CHAR_SIZE), %rax ++# else ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 5)(%rdi, %rax, CHAR_SIZE), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(last_vec_x3_return): ++ tzcntl %eax, %eax ++# ifdef USE_AS_RAWMEMCHR ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++# else ++ /* NB: Multiply bytes by CHAR_SIZE to get the wchar_t count. */ ++ leaq (VEC_SIZE * 6)(%rdi, %rax, CHAR_SIZE), %rax ++# endif ++ ret ++ ++ ++# ifndef USE_AS_RAWMEMCHR ++L(last_4x_vec_or_less_cmpeq): ++ VPCMP $0, (VEC_SIZE * 5)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ subq $-(VEC_SIZE * 4), %rdi ++ /* Check first VEC regardless. */ ++ testl %eax, %eax ++ jnz L(first_vec_x1_check) ++ ++ /* If remaining length <= CHAR_PER_VEC * 2. */ ++ addl $(CHAR_PER_VEC * 2), %edx ++ jle L(last_2x_vec) ++ ++ .p2align 4 ++L(last_4x_vec): ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ /* Create mask for possible matches within remaining length. */ ++# ifdef USE_AS_WMEMCHR ++ movl $((1 << (CHAR_PER_VEC * 2)) - 1), %ecx ++ bzhil %edx, %ecx, %ecx ++# else ++ movq $-1, %rcx ++ bzhiq %rdx, %rcx, %rcx ++# endif ++ /* Test matches in data against length match. */ ++ andl %ecx, %eax ++ jnz L(last_vec_x3) ++ ++ /* if remaining length <= CHAR_PER_VEC * 3 (Note this is after ++ remaining length was found to be > CHAR_PER_VEC * 2. */ ++ subl $CHAR_PER_VEC, %edx ++ jbe L(zero_end2) ++ ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMMATCH, %k0 ++ kmovd %k0, %eax ++ /* Shift remaining length mask for last VEC. */ ++# ifdef USE_AS_WMEMCHR ++ shrl $CHAR_PER_VEC, %ecx ++# else ++ shrq $CHAR_PER_VEC, %rcx ++# endif ++ andl %ecx, %eax ++ jz L(zero_end2) ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax ++L(zero_end2): ++ ret ++ ++L(last_vec_x2): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3): ++ tzcntl %eax, %eax ++ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax ++ ret ++# endif ++ ++END (MEMCHR) ++#endif +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S +new file mode 100644 +index 0000000000..cf4eff5d4a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMCMP ++# define MEMCMP __memcmp_avx2_movbe_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memcmp-avx2-movbe.S" +diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +index 67fc575b59..87f9478eaf 100644 +--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S ++++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S +@@ -47,6 +47,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + # define VEC_MASK ((1 << VEC_SIZE) - 1) + +@@ -55,7 +59,7 @@ + memcmp has to use UNSIGNED comparison for elemnts. + */ + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (MEMCMP) + # ifdef USE_AS_WMEMCMP + shl $2, %RDX_LP +@@ -123,8 +127,8 @@ ENTRY (MEMCMP) + vptest %ymm0, %ymm5 + jnc L(4x_vec_end) + xorl %eax, %eax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -144,8 +148,7 @@ L(last_vec): + vpmovmskb %ymm2, %eax + subl $VEC_MASK, %eax + jnz L(first_vec) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec): +@@ -164,8 +167,7 @@ L(wmemcmp_return): + movzbl (%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_WMEMCMP + .p2align 4 +@@ -367,8 +369,7 @@ L(last_4x_vec): + vpmovmskb %ymm2, %eax + subl $VEC_MASK, %eax + jnz L(first_vec) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -394,8 +395,7 @@ L(4x_vec_end): + movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): +@@ -410,8 +410,7 @@ L(first_vec_x1): + movzbl VEC_SIZE(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): +@@ -426,7 +425,6 @@ L(first_vec_x2): + movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx + sub %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (MEMCMP) + #endif +diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +new file mode 100644 +index 0000000000..9c093972e1 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S +@@ -0,0 +1,440 @@ ++/* memcmp/wmemcmp optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++/* memcmp/wmemcmp is implemented as: ++ 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap ++ to avoid branches. ++ 2. Use overlapping compare to avoid branch. ++ 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8 ++ bytes for wmemcmp. ++ 4. If size is 8 * VEC_SIZE or less, unroll the loop. ++ 5. Compare 4 * VEC_SIZE at a time with the aligned first memory ++ area. ++ 6. Use 2 vector compares when size is 2 * VEC_SIZE or less. ++ 7. Use 4 vector compares when size is 4 * VEC_SIZE or less. ++ 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. */ ++ ++# include <sysdep.h> ++ ++# ifndef MEMCMP ++# define MEMCMP __memcmp_evex_movbe ++# endif ++ ++# define VMOVU vmovdqu64 ++ ++# ifdef USE_AS_WMEMCMP ++# define VPCMPEQ vpcmpeqd ++# else ++# define VPCMPEQ vpcmpeqb ++# endif ++ ++# define XMM1 xmm17 ++# define XMM2 xmm18 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++# ifdef USE_AS_WMEMCMP ++# define VEC_MASK 0xff ++# define XMM_MASK 0xf ++# else ++# define VEC_MASK 0xffffffff ++# define XMM_MASK 0xffff ++# endif ++ ++/* Warning! ++ wmemcmp has to use SIGNED comparison for elements. ++ memcmp has to use UNSIGNED comparison for elemnts. ++*/ ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (MEMCMP) ++# ifdef USE_AS_WMEMCMP ++ shl $2, %RDX_LP ++# elif defined __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %edx, %edx ++# endif ++ cmp $VEC_SIZE, %RDX_LP ++ jb L(less_vec) ++ ++ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k1 ++ kmovd %k1, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ cmpq $(VEC_SIZE * 2), %rdx ++ jbe L(last_vec) ++ ++ /* More than 2 * VEC. */ ++ cmpq $(VEC_SIZE * 8), %rdx ++ ja L(more_8x_vec) ++ cmpq $(VEC_SIZE * 4), %rdx ++ jb L(last_4x_vec) ++ ++ /* From 4 * VEC to 8 * VEC, inclusively. */ ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ ++ kandd %k1, %k2, %k5 ++ kandd %k3, %k4, %k6 ++ kandd %k5, %k6, %k6 ++ ++ kmovd %k6, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ ++ leaq -(4 * VEC_SIZE)(%rdi, %rdx), %rdi ++ leaq -(4 * VEC_SIZE)(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ kandd %k1, %k2, %k5 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ kandd %k3, %k5, %k5 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ kandd %k4, %k5, %k5 ++ ++ kmovd %k5, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ /* From VEC to 2 * VEC. No branch when size == VEC_SIZE. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++L(last_vec): ++ /* Use overlapping loads to avoid branches. */ ++ leaq -VEC_SIZE(%rdi, %rdx), %rdi ++ leaq -VEC_SIZE(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(first_vec): ++ /* A byte or int32 is different within 16 or 32 bytes. */ ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (%rdi, %rcx, 4), %edx ++ cmpl (%rsi, %rcx, 4), %edx ++L(wmemcmp_return): ++ setl %al ++ negl %eax ++ orl $1, %eax ++# else ++ movzbl (%rdi, %rcx), %eax ++ movzbl (%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++# ifdef USE_AS_WMEMCMP ++ .p2align 4 ++L(4): ++ xorl %eax, %eax ++ movl (%rdi), %edx ++ cmpl (%rsi), %edx ++ jne L(wmemcmp_return) ++ ret ++# else ++ .p2align 4 ++L(between_4_7): ++ /* Load as big endian with overlapping movbe to avoid branches. */ ++ movbe (%rdi), %eax ++ movbe (%rsi), %ecx ++ shlq $32, %rax ++ shlq $32, %rcx ++ movbe -4(%rdi, %rdx), %edi ++ movbe -4(%rsi, %rdx), %esi ++ orq %rdi, %rax ++ orq %rsi, %rcx ++ subq %rcx, %rax ++ je L(exit) ++ sbbl %eax, %eax ++ orl $1, %eax ++ ret ++ ++ .p2align 4 ++L(exit): ++ ret ++ ++ .p2align 4 ++L(between_2_3): ++ /* Load as big endian to avoid branches. */ ++ movzwl (%rdi), %eax ++ movzwl (%rsi), %ecx ++ shll $8, %eax ++ shll $8, %ecx ++ bswap %eax ++ bswap %ecx ++ movb -1(%rdi, %rdx), %al ++ movb -1(%rsi, %rdx), %cl ++ /* Subtraction is okay because the upper 8 bits are zero. */ ++ subl %ecx, %eax ++ ret ++ ++ .p2align 4 ++L(1): ++ movzbl (%rdi), %eax ++ movzbl (%rsi), %ecx ++ subl %ecx, %eax ++ ret ++# endif ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(less_vec): ++# ifdef USE_AS_WMEMCMP ++ /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes. */ ++ cmpb $4, %dl ++ je L(4) ++ jb L(zero) ++# else ++ cmpb $1, %dl ++ je L(1) ++ jb L(zero) ++ cmpb $4, %dl ++ jb L(between_2_3) ++ cmpb $8, %dl ++ jb L(between_4_7) ++# endif ++ cmpb $16, %dl ++ jae L(between_16_31) ++ /* It is between 8 and 15 bytes. */ ++ vmovq (%rdi), %XMM1 ++ vmovq (%rsi), %XMM2 ++ VPCMPEQ %XMM1, %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ /* Use overlapping loads to avoid branches. */ ++ leaq -8(%rdi, %rdx), %rdi ++ leaq -8(%rsi, %rdx), %rsi ++ vmovq (%rdi), %XMM1 ++ vmovq (%rsi), %XMM2 ++ VPCMPEQ %XMM1, %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(between_16_31): ++ /* From 16 to 31 bytes. No branch when size == 16. */ ++ VMOVU (%rsi), %XMM2 ++ VPCMPEQ (%rdi), %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Use overlapping loads to avoid branches. */ ++ leaq -16(%rdi, %rdx), %rdi ++ leaq -16(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %XMM2 ++ VPCMPEQ (%rdi), %XMM2, %k2 ++ kmovw %k2, %eax ++ subl $XMM_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(more_8x_vec): ++ /* More than 8 * VEC. Check the first VEC. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Align the first memory area for aligned loads in the loop. ++ Compute how much the first memory area is misaligned. */ ++ movq %rdi, %rcx ++ andl $(VEC_SIZE - 1), %ecx ++ /* Get the negative of offset for alignment. */ ++ subq $VEC_SIZE, %rcx ++ /* Adjust the second memory area. */ ++ subq %rcx, %rsi ++ /* Adjust the first memory area which should be aligned now. */ ++ subq %rcx, %rdi ++ /* Adjust length. */ ++ addq %rcx, %rdx ++ ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VMOVU (%rsi), %YMM1 ++ VPCMPEQ (%rdi), %YMM1, %k1 ++ ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2 ++ kandd %k2, %k1, %k5 ++ ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM3 ++ VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3 ++ kandd %k3, %k5, %k5 ++ ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM4 ++ VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4 ++ kandd %k4, %k5, %k5 ++ ++ kmovd %k5, %eax ++ cmpl $VEC_MASK, %eax ++ jne L(4x_vec_end) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ addq $(VEC_SIZE * 4), %rsi ++ ++ subq $(VEC_SIZE * 4), %rdx ++ cmpq $(VEC_SIZE * 4), %rdx ++ jae L(loop_4x_vec) ++ ++ /* Less than 4 * VEC. */ ++ cmpq $VEC_SIZE, %rdx ++ jbe L(last_vec) ++ cmpq $(VEC_SIZE * 2), %rdx ++ jbe L(last_2x_vec) ++ ++L(last_4x_vec): ++ /* From 2 * VEC to 4 * VEC. */ ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ /* Use overlapping loads to avoid branches. */ ++ leaq -(3 * VEC_SIZE)(%rdi, %rdx), %rdi ++ leaq -(3 * VEC_SIZE)(%rsi, %rdx), %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rsi ++ VMOVU (%rsi), %YMM2 ++ VPCMPEQ (%rdi), %YMM2, %k2 ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ ret ++ ++ .p2align 4 ++L(4x_vec_end): ++ kmovd %k1, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec) ++ kmovd %k2, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec_x1) ++ kmovd %k3, %eax ++ subl $VEC_MASK, %eax ++ jnz L(first_vec_x2) ++ kmovd %k4, %eax ++ subl $VEC_MASK, %eax ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx ++ cmpl (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl VEC_SIZE(%rdi, %rcx, 4), %edx ++ cmpl VEC_SIZE(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl VEC_SIZE(%rdi, %rcx), %eax ++ movzbl VEC_SIZE(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %ecx ++# ifdef USE_AS_WMEMCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx ++ cmpl (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx ++ jmp L(wmemcmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rdi, %rcx), %eax ++ movzbl (VEC_SIZE * 2)(%rsi, %rcx), %edx ++ sub %edx, %eax ++# endif ++ ret ++END (MEMCMP) ++#endif +diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +new file mode 100644 +index 0000000000..1ec1962e86 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +@@ -0,0 +1,17 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define VEC(i) ymm##i ++# define VMOVNT vmovntdq ++# define VMOVU vmovdqu ++# define VMOVA vmovdqa ++ ++# define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++# define VZEROUPPER_RETURN jmp L(return) ++ ++# define SECTION(p) p##.avx.rtm ++# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm ++ ++# include "memmove-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +index aac1515cf6..848848ab39 100644 +--- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +@@ -1,11 +1,32 @@ + #if IS_IN (libc) + # define VEC_SIZE 64 +-# define VEC(i) zmm##i ++# define XMM0 xmm16 ++# define XMM1 xmm17 ++# define YMM0 ymm16 ++# define YMM1 ymm17 ++# define VEC0 zmm16 ++# define VEC1 zmm17 ++# define VEC2 zmm18 ++# define VEC3 zmm19 ++# define VEC4 zmm20 ++# define VEC5 zmm21 ++# define VEC6 zmm22 ++# define VEC7 zmm23 ++# define VEC8 zmm24 ++# define VEC9 zmm25 ++# define VEC10 zmm26 ++# define VEC11 zmm27 ++# define VEC12 zmm28 ++# define VEC13 zmm29 ++# define VEC14 zmm30 ++# define VEC15 zmm31 ++# define VEC(i) VEC##i + # define VMOVNT vmovntdq + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 ++# define VZEROUPPER + +-# define SECTION(p) p##.avx512 ++# define SECTION(p) p##.evex512 + # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s + + # include "memmove-vec-unaligned-erms.S" +diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +new file mode 100644 +index 0000000000..0cbce8f944 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +@@ -0,0 +1,33 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define XMM0 xmm16 ++# define XMM1 xmm17 ++# define YMM0 ymm16 ++# define YMM1 ymm17 ++# define VEC0 ymm16 ++# define VEC1 ymm17 ++# define VEC2 ymm18 ++# define VEC3 ymm19 ++# define VEC4 ymm20 ++# define VEC5 ymm21 ++# define VEC6 ymm22 ++# define VEC7 ymm23 ++# define VEC8 ymm24 ++# define VEC9 ymm25 ++# define VEC10 ymm26 ++# define VEC11 ymm27 ++# define VEC12 ymm28 ++# define VEC13 ymm29 ++# define VEC14 ymm30 ++# define VEC15 ymm31 ++# define VEC(i) VEC##i ++# define VMOVNT vmovntdq ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++# define VZEROUPPER ++ ++# define SECTION(p) p##.evex ++# define MEMMOVE_SYMBOL(p,s) p##_evex_##s ++ ++# include "memmove-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +index bd5dc1a3f3..f71c343ecb 100644 +--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +@@ -48,6 +48,14 @@ + # define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) + #endif + ++#ifndef XMM0 ++# define XMM0 xmm0 ++#endif ++ ++#ifndef YMM0 ++# define YMM0 ymm0 ++#endif ++ + #ifndef VZEROUPPER + # if VEC_SIZE > 16 + # define VZEROUPPER vzeroupper +@@ -56,6 +64,13 @@ + # endif + #endif + ++/* Avoid short distance rep movsb only with non-SSE vector. */ ++#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB ++# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16) ++#else ++# define AVOID_SHORT_DISTANCE_REP_MOVSB 0 ++#endif ++ + #ifndef PREFETCH + # define PREFETCH(addr) prefetcht0 addr + #endif +@@ -132,11 +147,12 @@ L(last_2x_vec): + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) +- VZEROUPPER + #if !defined USE_MULTIARCH || !IS_IN (libc) + L(nop): +-#endif + ret ++#else ++ VZEROUPPER_RETURN ++#endif + #if defined USE_MULTIARCH && IS_IN (libc) + END (MEMMOVE_SYMBOL (__memmove, unaligned)) + +@@ -229,8 +245,11 @@ L(last_2x_vec): + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + L(return): +- VZEROUPPER ++#if VEC_SIZE > 16 ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++#else + ret ++#endif + + L(movsb): + cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP +@@ -243,7 +262,21 @@ L(movsb): + cmpq %r9, %rdi + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) ++# if AVOID_SHORT_DISTANCE_REP_MOVSB ++ movq %rdi, %rcx ++ subq %rsi, %rcx ++ jmp 2f ++# endif + 1: ++# if AVOID_SHORT_DISTANCE_REP_MOVSB ++ movq %rsi, %rcx ++ subq %rdi, %rcx ++2: ++/* Avoid "rep movsb" if RCX, the distance between source and destination, ++ is N*4GB + [1..63] with N >= 0. */ ++ cmpl $63, %ecx ++ jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ ++# endif + mov %RDX_LP, %RCX_LP + rep movsb + L(nop): +@@ -277,21 +310,20 @@ L(less_vec): + #if VEC_SIZE > 32 + L(between_32_63): + /* From 32 to 63. No branch when size == 32. */ +- vmovdqu (%rsi), %ymm0 +- vmovdqu -32(%rsi,%rdx), %ymm1 +- vmovdqu %ymm0, (%rdi) +- vmovdqu %ymm1, -32(%rdi,%rdx) +- VZEROUPPER +- ret ++ VMOVU (%rsi), %YMM0 ++ VMOVU -32(%rsi,%rdx), %YMM1 ++ VMOVU %YMM0, (%rdi) ++ VMOVU %YMM1, -32(%rdi,%rdx) ++ VZEROUPPER_RETURN + #endif + #if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): +- vmovdqu (%rsi), %xmm0 +- vmovdqu -16(%rsi,%rdx), %xmm1 +- vmovdqu %xmm0, (%rdi) +- vmovdqu %xmm1, -16(%rdi,%rdx) +- ret ++ VMOVU (%rsi), %XMM0 ++ VMOVU -16(%rsi,%rdx), %XMM1 ++ VMOVU %XMM0, (%rdi) ++ VMOVU %XMM1, -16(%rdi,%rdx) ++ VZEROUPPER_RETURN + #endif + L(between_8_15): + /* From 8 to 15. No branch when size == 8. */ +@@ -344,8 +376,7 @@ L(more_2x_vec): + VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) + VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) + VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(last_4x_vec): + /* Copy from 2 * VEC to 4 * VEC. */ + VMOVU (%rsi), %VEC(0) +@@ -356,8 +387,7 @@ L(last_4x_vec): + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(more_8x_vec): + cmpq %rsi, %rdi +@@ -413,8 +443,7 @@ L(loop_4x_vec_forward): + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(more_8x_vec_backward): + /* Load the first 4 * VEC and last VEC to support overlapping +@@ -465,8 +494,7 @@ L(loop_4x_vec_backward): + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc) + L(large_forward): +@@ -501,8 +529,7 @@ L(loop_large_forward): + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(large_backward): + /* Don't use non-temporal store if there is overlap between +@@ -536,8 +563,7 @@ L(loop_large_backward): + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + #endif + END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) + +diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S +new file mode 100644 +index 0000000000..cea2d2a72d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef MEMRCHR ++# define MEMRCHR __memrchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "memrchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2.S b/sysdeps/x86_64/multiarch/memrchr-avx2.S +index f5437b54de..c8d54c08d6 100644 +--- a/sysdeps/x86_64/multiarch/memrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/memrchr-avx2.S +@@ -20,14 +20,22 @@ + + # include <sysdep.h> + ++# ifndef MEMRCHR ++# define MEMRCHR __memrchr_avx2 ++# endif ++ + # ifndef VZEROUPPER + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits +-ENTRY (__memrchr_avx2) ++ .section SECTION(.text),"ax",@progbits ++ENTRY (MEMRCHR) + /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 + vpbroadcastb %xmm0, %ymm0 +@@ -134,8 +142,8 @@ L(loop_4x_vec): + vpmovmskb %ymm1, %eax + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(last_4x_vec_or_less): +@@ -169,8 +177,7 @@ L(last_4x_vec_or_less): + addq %rax, %rdx + jl L(zero) + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_2x_vec): +@@ -191,31 +198,27 @@ L(last_2x_vec): + jl L(zero) + addl $(VEC_SIZE * 2), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x0): + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x1): + bsrl %eax, %eax + addl $VEC_SIZE, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x2): + bsrl %eax, %eax + addl $(VEC_SIZE * 2), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x3): +@@ -232,8 +235,7 @@ L(last_vec_x1_check): + jl L(zero) + addl $VEC_SIZE, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_x3_check): +@@ -243,12 +245,14 @@ L(last_vec_x3_check): + jl L(zero) + addl $(VEC_SIZE * 3), %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(zero): +- VZEROUPPER ++ xorl %eax, %eax ++ VZEROUPPER_RETURN ++ ++ .p2align 4 + L(null): + xorl %eax, %eax + ret +@@ -273,8 +277,7 @@ L(last_vec_or_less_aligned): + + bsrl %eax, %eax + addq %rdi, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_or_less): +@@ -315,8 +318,7 @@ L(last_vec_or_less): + bsrl %eax, %eax + addq %rdi, %rax + addq %r8, %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(last_vec_2x_aligned): +@@ -353,7 +355,6 @@ L(last_vec_2x_aligned): + bsrl %eax, %eax + addq %rdi, %rax + addq %r8, %rax +- VZEROUPPER +- ret +-END (__memrchr_avx2) ++ VZEROUPPER_RETURN ++END (MEMRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/memrchr-evex.S b/sysdeps/x86_64/multiarch/memrchr-evex.S +new file mode 100644 +index 0000000000..16bf8e02b1 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memrchr-evex.S +@@ -0,0 +1,337 @@ ++/* memrchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# define VMOVA vmovdqa64 ++ ++# define YMMMATCH ymm16 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (__memrchr_evex) ++ /* Broadcast CHAR to YMMMATCH. */ ++ vpbroadcastb %esi, %YMMMATCH ++ ++ sub $VEC_SIZE, %RDX_LP ++ jbe L(last_vec_or_less) ++ ++ add %RDX_LP, %RDI_LP ++ ++ /* Check the last VEC_SIZE bytes. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x0) ++ ++ subq $(VEC_SIZE * 4), %rdi ++ movl %edi, %ecx ++ andl $(VEC_SIZE - 1), %ecx ++ jz L(aligned_more) ++ ++ /* Align data for aligned loads in the loop. */ ++ addq $VEC_SIZE, %rdi ++ addq $VEC_SIZE, %rdx ++ andq $-VEC_SIZE, %rdi ++ subq %rcx, %rdx ++ ++ .p2align 4 ++L(aligned_more): ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++ ++ /* Check the last 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k2 ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k3 ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k4 ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x0) ++ ++ /* Align data to 4 * VEC_SIZE for loop with fewer branches. ++ There are some overlaps with above if data isn't aligned ++ to 4 * VEC_SIZE. */ ++ movl %edi, %ecx ++ andl $(VEC_SIZE * 4 - 1), %ecx ++ jz L(loop_4x_vec) ++ ++ addq $(VEC_SIZE * 4), %rdi ++ addq $(VEC_SIZE * 4), %rdx ++ andq $-(VEC_SIZE * 4), %rdi ++ subq %rcx, %rdx ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ subq $(VEC_SIZE * 4), %rdi ++ subq $(VEC_SIZE * 4), %rdx ++ jbe L(last_4x_vec_or_less) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k2 ++ kord %k1, %k2, %k5 ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k3 ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k4 ++ ++ kord %k3, %k4, %k6 ++ kortestd %k5, %k6 ++ jz L(loop_4x_vec) ++ ++ /* There is a match. */ ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ kmovd %k1, %eax ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_4x_vec_or_less): ++ addl $(VEC_SIZE * 4), %edx ++ cmpl $(VEC_SIZE * 2), %edx ++ jbe L(last_2x_vec) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k2 ++ kmovd %k2, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k3 ++ kmovd %k3, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x1_check) ++ cmpl $(VEC_SIZE * 3), %edx ++ jbe L(zero) ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k4 ++ kmovd %k4, %eax ++ testl %eax, %eax ++ jz L(zero) ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 4), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_2x_vec): ++ vpcmpb $0, (VEC_SIZE * 3)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3_check) ++ cmpl $VEC_SIZE, %edx ++ jbe L(zero) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jz L(zero) ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 2), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $(VEC_SIZE * 2), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x0): ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x1): ++ bsrl %eax, %eax ++ addl $VEC_SIZE, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x2): ++ bsrl %eax, %eax ++ addl $(VEC_SIZE * 2), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3): ++ bsrl %eax, %eax ++ addl $(VEC_SIZE * 3), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x1_check): ++ bsrl %eax, %eax ++ subq $(VEC_SIZE * 3), %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $VEC_SIZE, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3_check): ++ bsrl %eax, %eax ++ subq $VEC_SIZE, %rdx ++ addq %rax, %rdx ++ jl L(zero) ++ addl $(VEC_SIZE * 3), %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(last_vec_or_less_aligned): ++ movl %edx, %ecx ++ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ ++ movl $1, %edx ++ /* Support rdx << 32. */ ++ salq %cl, %rdx ++ subq $1, %rdx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the trailing bytes. */ ++ andl %edx, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_or_less): ++ addl $VEC_SIZE, %edx ++ ++ /* Check for zero length. */ ++ testl %edx, %edx ++ jz L(zero) ++ ++ movl %edi, %ecx ++ andl $(VEC_SIZE - 1), %ecx ++ jz L(last_vec_or_less_aligned) ++ ++ movl %ecx, %esi ++ movl %ecx, %r8d ++ addl %edx, %esi ++ andq $-VEC_SIZE, %rdi ++ ++ subl $VEC_SIZE, %esi ++ ja L(last_vec_2x_aligned) ++ ++ /* Check the last VEC. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ kmovd %k1, %eax ++ ++ /* Remove the leading and trailing bytes. */ ++ sarl %cl, %eax ++ movl %edx, %ecx ++ ++ movl $1, %edx ++ sall %cl, %edx ++ subl $1, %edx ++ ++ andl %edx, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ addq %r8, %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_2x_aligned): ++ movl %esi, %ecx ++ ++ /* Check the last VEC. */ ++ vpcmpb $0, VEC_SIZE(%rdi), %YMMMATCH, %k1 ++ ++ movl $1, %edx ++ sall %cl, %edx ++ subl $1, %edx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the trailing bytes. */ ++ andl %edx, %eax ++ ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ /* Check the second last VEC. */ ++ vpcmpb $0, (%rdi), %YMMMATCH, %k1 ++ ++ movl %r8d, %ecx ++ ++ kmovd %k1, %eax ++ ++ /* Remove the leading bytes. Must use unsigned right shift for ++ bsrl below. */ ++ shrl %cl, %eax ++ testl %eax, %eax ++ jz L(zero) ++ ++ bsrl %eax, %eax ++ addq %rdi, %rax ++ addq %r8, %rax ++ ret ++END (__memrchr_evex) ++#endif +diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S +new file mode 100644 +index 0000000000..8ac3e479bb +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S +@@ -0,0 +1,10 @@ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return) ++ ++#define SECTION(p) p##.avx.rtm ++#define MEMSET_SYMBOL(p,s) p##_avx2_##s##_rtm ++#define WMEMSET_SYMBOL(p,s) p##_avx2_##s##_rtm ++ ++#include "memset-avx2-unaligned-erms.S" +diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +index 7ab3d89849..ae0860f36a 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S +@@ -14,9 +14,15 @@ + movq r, %rax; \ + vpbroadcastd %xmm0, %ymm0 + +-# define SECTION(p) p##.avx +-# define MEMSET_SYMBOL(p,s) p##_avx2_##s +-# define WMEMSET_SYMBOL(p,s) p##_avx2_##s ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++# ifndef MEMSET_SYMBOL ++# define MEMSET_SYMBOL(p,s) p##_avx2_##s ++# endif ++# ifndef WMEMSET_SYMBOL ++# define WMEMSET_SYMBOL(p,s) p##_avx2_##s ++# endif + + # include "memset-vec-unaligned-erms.S" + #endif +diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +index 0783979ca5..22e7b187c8 100644 +--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S +@@ -1,22 +1,22 @@ + #if IS_IN (libc) + # define VEC_SIZE 64 +-# define VEC(i) zmm##i ++# define XMM0 xmm16 ++# define YMM0 ymm16 ++# define VEC0 zmm16 ++# define VEC(i) VEC##i + # define VMOVU vmovdqu64 + # define VMOVA vmovdqa64 ++# define VZEROUPPER + + # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +- vmovd d, %xmm0; \ + movq r, %rax; \ +- vpbroadcastb %xmm0, %xmm0; \ +- vpbroadcastq %xmm0, %zmm0 ++ vpbroadcastb d, %VEC0 + + # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ +- vmovd d, %xmm0; \ + movq r, %rax; \ +- vpbroadcastd %xmm0, %xmm0; \ +- vpbroadcastq %xmm0, %zmm0 ++ vpbroadcastd d, %VEC0 + +-# define SECTION(p) p##.avx512 ++# define SECTION(p) p##.evex512 + # define MEMSET_SYMBOL(p,s) p##_avx512_##s + # define WMEMSET_SYMBOL(p,s) p##_avx512_##s + +diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +new file mode 100644 +index 0000000000..ae0a4d6e46 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S +@@ -0,0 +1,24 @@ ++#if IS_IN (libc) ++# define VEC_SIZE 32 ++# define XMM0 xmm16 ++# define YMM0 ymm16 ++# define VEC0 ymm16 ++# define VEC(i) VEC##i ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++# define VZEROUPPER ++ ++# define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ ++ movq r, %rax; \ ++ vpbroadcastb d, %VEC0 ++ ++# define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ ++ movq r, %rax; \ ++ vpbroadcastd d, %VEC0 ++ ++# define SECTION(p) p##.evex ++# define MEMSET_SYMBOL(p,s) p##_evex_##s ++# define WMEMSET_SYMBOL(p,s) p##_evex_##s ++ ++# include "memset-vec-unaligned-erms.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +index 2bfc95de05..de5a8a38f5 100644 +--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S ++++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S +@@ -34,20 +34,25 @@ + # define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s) + #endif + ++#ifndef XMM0 ++# define XMM0 xmm0 ++#endif ++ ++#ifndef YMM0 ++# define YMM0 ymm0 ++#endif ++ + #ifndef VZEROUPPER + # if VEC_SIZE > 16 + # define VZEROUPPER vzeroupper ++# define VZEROUPPER_SHORT_RETURN vzeroupper; ret + # else + # define VZEROUPPER + # endif + #endif + + #ifndef VZEROUPPER_SHORT_RETURN +-# if VEC_SIZE > 16 +-# define VZEROUPPER_SHORT_RETURN vzeroupper +-# else +-# define VZEROUPPER_SHORT_RETURN rep +-# endif ++# define VZEROUPPER_SHORT_RETURN rep; ret + #endif + + #ifndef MOVQ +@@ -67,7 +72,7 @@ + ENTRY (__bzero) + mov %RDI_LP, %RAX_LP /* Set return value. */ + mov %RSI_LP, %RDX_LP /* Set n. */ +- pxor %xmm0, %xmm0 ++ pxor %XMM0, %XMM0 + jmp L(entry_from_bzero) + END (__bzero) + weak_alias (__bzero, bzero) +@@ -109,8 +114,7 @@ L(entry_from_bzero): + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + #if defined USE_MULTIARCH && IS_IN (libc) + END (MEMSET_SYMBOL (__memset, unaligned)) + +@@ -133,14 +137,12 @@ ENTRY (__memset_erms) + ENTRY (MEMSET_SYMBOL (__memset, erms)) + # endif + L(stosb): +- /* Issue vzeroupper before rep stosb. */ +- VZEROUPPER + mov %RDX_LP, %RCX_LP + movzbl %sil, %eax + mov %RDI_LP, %RDX_LP + rep stosb + mov %RDX_LP, %RAX_LP +- ret ++ VZEROUPPER_RETURN + # if VEC_SIZE == 16 + END (__memset_erms) + # else +@@ -167,8 +169,7 @@ ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + L(stosb_more_2x_vec): + cmp __x86_rep_stosb_threshold(%rip), %RDX_LP +@@ -182,8 +183,11 @@ L(more_2x_vec): + VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) + L(return): +- VZEROUPPER ++#if VEC_SIZE > 16 ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++#else + ret ++#endif + + L(loop_start): + leaq (VEC_SIZE * 4)(%rdi), %rcx +@@ -209,7 +213,6 @@ L(loop): + cmpq %rcx, %rdx + jne L(loop) + VZEROUPPER_SHORT_RETURN +- ret + L(less_vec): + /* Less than 1 VEC. */ + # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 +@@ -223,7 +226,7 @@ L(less_vec): + cmpb $16, %dl + jae L(between_16_31) + # endif +- MOVQ %xmm0, %rcx ++ MOVQ %XMM0, %rcx + cmpb $8, %dl + jae L(between_8_15) + cmpb $4, %dl +@@ -233,40 +236,34 @@ L(less_vec): + jb 1f + movb %cl, (%rdi) + 1: +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # if VEC_SIZE > 32 + /* From 32 to 63. No branch when size == 32. */ + L(between_32_63): +- vmovdqu %ymm0, -32(%rdi,%rdx) +- vmovdqu %ymm0, (%rdi) +- VZEROUPPER +- ret ++ VMOVU %YMM0, -32(%rdi,%rdx) ++ VMOVU %YMM0, (%rdi) ++ VZEROUPPER_RETURN + # endif + # if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ + L(between_16_31): +- vmovdqu %xmm0, -16(%rdi,%rdx) +- vmovdqu %xmm0, (%rdi) +- VZEROUPPER +- ret ++ VMOVU %XMM0, -16(%rdi,%rdx) ++ VMOVU %XMM0, (%rdi) ++ VZEROUPPER_RETURN + # endif + /* From 8 to 15. No branch when size == 8. */ + L(between_8_15): + movq %rcx, -8(%rdi,%rdx) + movq %rcx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(between_4_7): + /* From 4 to 7. No branch when size == 4. */ + movl %ecx, -4(%rdi,%rdx) + movl %ecx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + L(between_2_3): + /* From 2 to 3. No branch when size == 2. */ + movw %cx, -2(%rdi,%rdx) + movw %cx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (MEMSET_SYMBOL (__memset, unaligned_erms)) +diff --git a/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S +new file mode 100644 +index 0000000000..acc5f6e2fb +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/rawmemchr-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __rawmemchr_avx2_rtm ++#define USE_AS_RAWMEMCHR 1 ++ ++#include "memchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/rawmemchr-evex.S b/sysdeps/x86_64/multiarch/rawmemchr-evex.S +new file mode 100644 +index 0000000000..ec942b77ba +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/rawmemchr-evex.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __rawmemchr_evex ++#define USE_AS_RAWMEMCHR 1 ++ ++#include "memchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S +new file mode 100644 +index 0000000000..2b9c07a59f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpcpy-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STPCPY ++#define STRCPY __stpcpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/stpcpy-evex.S b/sysdeps/x86_64/multiarch/stpcpy-evex.S +new file mode 100644 +index 0000000000..7c6f26cd98 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpcpy-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STPCPY ++#define STRCPY __stpcpy_evex ++#include "strcpy-evex.S" +diff --git a/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S +new file mode 100644 +index 0000000000..60a2ccfe53 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpncpy-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define USE_AS_STPCPY ++#define USE_AS_STRNCPY ++#define STRCPY __stpncpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/stpncpy-evex.S b/sysdeps/x86_64/multiarch/stpncpy-evex.S +new file mode 100644 +index 0000000000..1570014d1c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/stpncpy-evex.S +@@ -0,0 +1,4 @@ ++#define USE_AS_STPCPY ++#define USE_AS_STRNCPY ++#define STRCPY __stpncpy_evex ++#include "strcpy-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S +new file mode 100644 +index 0000000000..637fb557c4 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcat-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCAT ++# define STRCAT __strcat_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcat-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcat-avx2.S b/sysdeps/x86_64/multiarch/strcat-avx2.S +index a4143bf8f5..1e6d4827ee 100644 +--- a/sysdeps/x86_64/multiarch/strcat-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcat-avx2.S +@@ -30,7 +30,11 @@ + /* Number of bytes in a vector register */ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCAT) + mov %rdi, %r9 + # ifdef USE_AS_STRNCAT +diff --git a/sysdeps/x86_64/multiarch/strcat-evex.S b/sysdeps/x86_64/multiarch/strcat-evex.S +new file mode 100644 +index 0000000000..97c3d85b6d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcat-evex.S +@@ -0,0 +1,283 @@ ++/* strcat with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef STRCAT ++# define STRCAT __strcat_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++/* zero register */ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++ ++# define USE_AS_STRCAT ++ ++/* Number of bytes in a vector register */ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCAT) ++ mov %rdi, %r9 ++# ifdef USE_AS_STRNCAT ++ mov %rdx, %r8 ++# endif ++ ++ xor %eax, %eax ++ mov %edi, %ecx ++ and $((VEC_SIZE * 4) - 1), %ecx ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ cmp $(VEC_SIZE * 3), %ecx ++ ja L(fourth_vector_boundary) ++ vpcmpb $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_first_vector) ++ mov %rdi, %rax ++ and $-VEC_SIZE, %rax ++ jmp L(align_vec_size_start) ++L(fourth_vector_boundary): ++ mov %rdi, %rax ++ and $-VEC_SIZE, %rax ++ vpcmpb $0, (%rax), %YMMZERO, %k0 ++ mov $-1, %r10d ++ sub %rax, %rcx ++ shl %cl, %r10d ++ kmovd %k0, %edx ++ and %r10d, %edx ++ jnz L(exit) ++ ++L(align_vec_size_start): ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 4), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ kmovd %k4, %edx ++ add $(VEC_SIZE * 4), %rax ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 4), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fifth_vector) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, (VEC_SIZE * 5)(%rax), %YMMZERO, %k4 ++ add $(VEC_SIZE * 5), %rax ++ kmovd %k4, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ add $VEC_SIZE, %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k0 ++ add $VEC_SIZE, %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ test $((VEC_SIZE * 4) - 1), %rax ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, VEC_SIZE(%rax), %YMMZERO, %k1 ++ add $VEC_SIZE, %rax ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit) ++ ++ add $VEC_SIZE, %rax ++ ++ .p2align 4 ++L(align_four_vec_loop): ++ VMOVA (%rax), %YMM0 ++ VMOVA (VEC_SIZE * 2)(%rax), %YMM1 ++ vpminub VEC_SIZE(%rax), %YMM0, %YMM0 ++ vpminub (VEC_SIZE * 3)(%rax), %YMM1, %YMM1 ++ vpminub %YMM0, %YMM1, %YMM0 ++ /* If K0 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM0, %YMMZERO, %k0 ++ add $(VEC_SIZE * 4), %rax ++ ktestd %k0, %k0 ++ jz L(align_four_vec_loop) ++ ++ vpcmpb $0, -(VEC_SIZE * 4)(%rax), %YMMZERO, %k0 ++ sub $(VEC_SIZE * 5), %rax ++ kmovd %k0, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_second_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 2)(%rax), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_third_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 3)(%rax), %YMMZERO, %k2 ++ kmovd %k2, %edx ++ test %edx, %edx ++ jnz L(exit_null_on_fourth_vector) ++ ++ vpcmpb $0, (VEC_SIZE * 4)(%rax), %YMMZERO, %k3 ++ kmovd %k3, %edx ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 4), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit): ++ sub %rdi, %rax ++L(exit_null_on_first_vector): ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_second_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $VEC_SIZE, %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_third_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 2), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_fourth_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 3), %rax ++ jmp L(StartStrcpyPart) ++ ++ .p2align 4 ++L(exit_null_on_fifth_vector): ++ sub %rdi, %rax ++ bsf %rdx, %rdx ++ add %rdx, %rax ++ add $(VEC_SIZE * 4), %rax ++ ++ .p2align 4 ++L(StartStrcpyPart): ++ lea (%r9, %rax), %rdi ++ mov %rsi, %rcx ++ mov %r9, %rax /* save result */ ++ ++# ifdef USE_AS_STRNCAT ++ test %r8, %r8 ++ jz L(ExitZero) ++# define USE_AS_STRNCPY ++# endif ++ ++# include "strcpy-evex.S" ++#endif +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S +new file mode 100644 +index 0000000000..81f20d1d8e +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCHR ++# define STRCHR __strchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strchr-avx2.S b/sysdeps/x86_64/multiarch/strchr-avx2.S +index 39fc69da7b..0a5217514a 100644 +--- a/sysdeps/x86_64/multiarch/strchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strchr-avx2.S +@@ -38,9 +38,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCHR) + movl %edi, %ecx + /* Broadcast CHAR to YMM0. */ +@@ -93,8 +97,8 @@ L(cros_page_boundary): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(aligned_more): +@@ -190,8 +194,7 @@ L(first_vec_x0): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x1): +@@ -205,8 +208,7 @@ L(first_vec_x1): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(first_vec_x2): +@@ -220,8 +222,7 @@ L(first_vec_x2): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(4x_vec_end): +@@ -247,8 +248,7 @@ L(first_vec_x3): + cmp (%rax), %CHAR_REG + cmovne %rdx, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (STRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S +new file mode 100644 +index 0000000000..ddc86a7058 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchr-evex.S +@@ -0,0 +1,335 @@ ++/* strchr/strchrnul optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef STRCHR ++# define STRCHR __strchr_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSCHR ++# define VPBROADCAST vpbroadcastd ++# define VPCMP vpcmpd ++# define VPMINU vpminud ++# define CHAR_REG esi ++# define SHIFT_REG r8d ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPCMP vpcmpb ++# define VPMINU vpminub ++# define CHAR_REG sil ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMZERO xmm16 ++ ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++# define YMM2 ymm19 ++# define YMM3 ymm20 ++# define YMM4 ymm21 ++# define YMM5 ymm22 ++# define YMM6 ymm23 ++# define YMM7 ymm24 ++# define YMM8 ymm25 ++ ++# define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCHR) ++ movl %edi, %ecx ++# ifndef USE_AS_STRCHRNUL ++ xorl %edx, %edx ++# endif ++ ++ /* Broadcast CHAR to YMM0. */ ++ VPBROADCAST %esi, %YMM0 ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Check if we cross page boundary with one vector load. */ ++ andl $(PAGE_SIZE - 1), %ecx ++ cmpl $(PAGE_SIZE - VEC_SIZE), %ecx ++ ja L(cross_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. Search for both CHAR and the ++ null bytes. */ ++ VMOVU (%rdi), %YMM1 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ ktestd %k0, %k0 ++ jz L(more_vecs) ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(more_vecs): ++ /* Align data for aligned loads in the loop. */ ++ andq $-VEC_SIZE, %rdi ++L(aligned_more): ++ ++ /* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++ VMOVA VEC_SIZE(%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ VMOVA VEC_SIZE(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VMOVA (VEC_SIZE * 2)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VMOVA (VEC_SIZE * 3)(%rdi), %YMM1 ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ ktestd %k0, %k0 ++ jz L(prep_loop_4x) ++ ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 3)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 3)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x0): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq VEC_SIZE(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ /* Found CHAR or the null byte. */ ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++L(prep_loop_4x): ++ /* Align data to 4 * VEC_SIZE. */ ++ andq $-(VEC_SIZE * 4), %rdi ++ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Compare 4 * VEC at a time forward. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++ VMOVA (VEC_SIZE * 5)(%rdi), %YMM2 ++ VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 ++ VMOVA (VEC_SIZE * 7)(%rdi), %YMM4 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM5 ++ vpxorq %YMM2, %YMM0, %YMM6 ++ vpxorq %YMM3, %YMM0, %YMM7 ++ vpxorq %YMM4, %YMM0, %YMM8 ++ ++ VPMINU %YMM5, %YMM1, %YMM5 ++ VPMINU %YMM6, %YMM2, %YMM6 ++ VPMINU %YMM7, %YMM3, %YMM7 ++ VPMINU %YMM8, %YMM4, %YMM8 ++ ++ VPMINU %YMM5, %YMM6, %YMM1 ++ VPMINU %YMM7, %YMM8, %YMM2 ++ ++ VPMINU %YMM1, %YMM2, %YMM1 ++ ++ /* Each bit in K0 represents a CHAR or a null byte. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ ++ addq $(VEC_SIZE * 4), %rdi ++ ++ ktestd %k0, %k0 ++ jz L(loop_4x_vec) ++ ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM5, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x0) ++ ++ /* Each bit in K1 represents a CHAR or a null byte in YMM2. */ ++ VPCMP $0, %YMMZERO, %YMM6, %k1 ++ kmovd %k1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ /* Each bit in K2 represents a CHAR or a null byte in YMM3. */ ++ VPCMP $0, %YMMZERO, %YMM7, %k2 ++ /* Each bit in K3 represents a CHAR or a null byte in YMM4. */ ++ VPCMP $0, %YMMZERO, %YMM8, %k3 ++ ++# ifdef USE_AS_WCSCHR ++ /* NB: Each bit in K2/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k1 ++# else ++ kshiftlq $32, %k3, %k1 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rax ++ ++ tzcntq %rax, %rax ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (VEC_SIZE * 2)(%rdi, %rax, 4), %rax ++# else ++ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++ /* Cold case for crossing page with first load. */ ++ .p2align 4 ++L(cross_page_boundary): ++ andq $-VEC_SIZE, %rdi ++ andl $(VEC_SIZE - 1), %ecx ++ ++ VMOVA (%rdi), %YMM1 ++ ++ /* Leaves only CHARS matching esi as 0. */ ++ vpxorq %YMM1, %YMM0, %YMM2 ++ VPMINU %YMM2, %YMM1, %YMM2 ++ /* Each bit in K0 represents a CHAR or a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM2, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ ++# ifdef USE_AS_WCSCHR ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ ++ /* Remove the leading bits. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++ ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ addq %rcx, %rdi ++# ifdef USE_AS_WCSCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq (%rdi, %rax, 4), %rax ++# else ++ addq %rdi, %rax ++# endif ++# ifndef USE_AS_STRCHRNUL ++ cmp (%rax), %CHAR_REG ++ cmovne %rdx, %rax ++# endif ++ ret ++ ++END (STRCHR) ++# endif +diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c +index 8df4609bf8..4ed1177c70 100644 +--- a/sysdeps/x86_64/multiarch/strchr.c ++++ b/sysdeps/x86_64/multiarch/strchr.c +@@ -29,16 +29,28 @@ + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_no_bsf) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Slow_BSF)) + return OPTIMIZE (sse2_no_bsf); +diff --git a/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S b/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S +new file mode 100644 +index 0000000000..cdcf818b91 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchrnul-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCHR __strchrnul_avx2_rtm ++#define USE_AS_STRCHRNUL 1 ++#include "strchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strchrnul-evex.S b/sysdeps/x86_64/multiarch/strchrnul-evex.S +new file mode 100644 +index 0000000000..064fe7ca9e +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strchrnul-evex.S +@@ -0,0 +1,3 @@ ++#define STRCHR __strchrnul_evex ++#define USE_AS_STRCHRNUL 1 ++#include "strchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S +new file mode 100644 +index 0000000000..aecd30d97f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCMP ++# define STRCMP __strcmp_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcmp-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcmp-avx2.S b/sysdeps/x86_64/multiarch/strcmp-avx2.S +index d42b04b54f..759e5b64c2 100644 +--- a/sysdeps/x86_64/multiarch/strcmp-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcmp-avx2.S +@@ -55,6 +55,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + /* Warning! + wcscmp/wcsncmp have to use SIGNED comparison for elements. + strcmp/strncmp have to use UNSIGNED comparison for elements. +@@ -75,7 +79,7 @@ + the maximum offset is reached before a difference is found, zero is + returned. */ + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCMP) + # ifdef USE_AS_STRNCMP + /* Check for simple cases (0 or 1) in offset. */ +@@ -83,6 +87,16 @@ ENTRY (STRCMP) + je L(char0) + jb L(zero) + # ifdef USE_AS_WCSCMP ++# ifndef __ILP32__ ++ movq %rdx, %rcx ++ /* Check if length could overflow when multiplied by ++ sizeof(wchar_t). Checking top 8 bits will cover all potential ++ overflow cases as well as redirect cases where its impossible to ++ length to bound a valid memory region. In these cases just use ++ 'wcscmp'. */ ++ shrq $56, %rcx ++ jnz __wcscmp_avx2 ++# endif + /* Convert units: from wide to byte char. */ + shl $2, %RDX_LP + # endif +@@ -127,8 +141,8 @@ L(return): + movzbl (%rsi, %rdx), %edx + subl %edx, %eax + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(return_vec_size): +@@ -161,8 +175,7 @@ L(return_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_2_vec_size): +@@ -195,8 +208,7 @@ L(return_2_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_3_vec_size): +@@ -229,8 +241,7 @@ L(return_3_vec_size): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(next_3_vectors): +@@ -356,8 +367,7 @@ L(back_to_loop): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_vec): +@@ -400,8 +410,7 @@ L(test_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_2_vec): +@@ -444,8 +453,7 @@ L(test_2_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(test_3_vec): +@@ -486,8 +494,7 @@ L(test_3_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(loop_cross_page): +@@ -556,8 +563,7 @@ L(loop_cross_page): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(loop_cross_page_2_vec): +@@ -631,8 +637,7 @@ L(loop_cross_page_2_vec): + subl %edx, %eax + # endif + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNCMP + L(string_nbyte_offset_check): +@@ -674,8 +679,7 @@ L(cross_page_loop): + # ifndef USE_AS_WCSCMP + L(different): + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_WCSCMP + .p2align 4 +@@ -685,16 +689,14 @@ L(different): + setl %al + negl %eax + orl $1, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + + # ifdef USE_AS_STRNCMP + .p2align 4 + L(zero): + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(char0): +@@ -708,8 +710,7 @@ L(char0): + movzbl (%rdi), %eax + subl %ecx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + # endif + + .p2align 4 +@@ -734,8 +735,7 @@ L(last_vector): + movzbl (%rsi, %rdx), %edx + subl %edx, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + /* Comparing on page boundary region requires special treatment: + It must done one vector at the time, starting with the wider +@@ -856,7 +856,6 @@ L(cross_page_4bytes): + testl %eax, %eax + jne L(cross_page_loop) + subl %ecx, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + END (STRCMP) + #endif +diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S +new file mode 100644 +index 0000000000..459eeed09f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcmp-evex.S +@@ -0,0 +1,1043 @@ ++/* strcmp/wcscmp/strncmp/wcsncmp optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef STRCMP ++# define STRCMP __strcmp_evex ++# endif ++ ++# define PAGE_SIZE 4096 ++ ++/* VEC_SIZE = Number of bytes in a ymm register */ ++# define VEC_SIZE 32 ++ ++/* Shift for dividing by (VEC_SIZE * 4). */ ++# define DIVIDE_BY_VEC_4_SHIFT 7 ++# if (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT) ++# error (VEC_SIZE * 4) != (1 << DIVIDE_BY_VEC_4_SHIFT) ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSCMP ++/* Compare packed dwords. */ ++# define VPCMP vpcmpd ++# define SHIFT_REG32 r8d ++# define SHIFT_REG64 r8 ++/* 1 dword char == 4 bytes. */ ++# define SIZE_OF_CHAR 4 ++# else ++/* Compare packed bytes. */ ++# define VPCMP vpcmpb ++# define SHIFT_REG32 ecx ++# define SHIFT_REG64 rcx ++/* 1 byte char == 1 byte. */ ++# define SIZE_OF_CHAR 1 ++# endif ++ ++# define XMMZERO xmm16 ++# define XMM0 xmm17 ++# define XMM1 xmm18 ++ ++# define YMMZERO ymm16 ++# define YMM0 ymm17 ++# define YMM1 ymm18 ++# define YMM2 ymm19 ++# define YMM3 ymm20 ++# define YMM4 ymm21 ++# define YMM5 ymm22 ++# define YMM6 ymm23 ++# define YMM7 ymm24 ++ ++/* Warning! ++ wcscmp/wcsncmp have to use SIGNED comparison for elements. ++ strcmp/strncmp have to use UNSIGNED comparison for elements. ++*/ ++ ++/* The main idea of the string comparison (byte or dword) using 256-bit ++ EVEX instructions consists of comparing (VPCMP) two ymm vectors. The ++ latter can be on either packed bytes or dwords depending on ++ USE_AS_WCSCMP. In order to check the null char, algorithm keeps the ++ matched bytes/dwords, requiring 5 EVEX instructions (3 VPCMP and 2 ++ KORD). In general, the costs of comparing VEC_SIZE bytes (32-bytes) ++ are 3 VPCMP and 2 KORD instructions, together with VMOVU and ktestd ++ instructions. Main loop (away from from page boundary) compares 4 ++ vectors are a time, effectively comparing 4 x VEC_SIZE bytes (128 ++ bytes) on each loop. ++ ++ The routine strncmp/wcsncmp (enabled by defining USE_AS_STRNCMP) logic ++ is the same as strcmp, except that an a maximum offset is tracked. If ++ the maximum offset is reached before a difference is found, zero is ++ returned. */ ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCMP) ++# ifdef USE_AS_STRNCMP ++ /* Check for simple cases (0 or 1) in offset. */ ++ cmp $1, %RDX_LP ++ je L(char0) ++ jb L(zero) ++# ifdef USE_AS_WCSCMP ++ /* Convert units: from wide to byte char. */ ++ shl $2, %RDX_LP ++# endif ++ /* Register %r11 tracks the maximum offset. */ ++ mov %RDX_LP, %R11_LP ++# endif ++ movl %edi, %eax ++ xorl %edx, %edx ++ /* Make %XMMZERO (%YMMZERO) all zeros in this function. */ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ orl %esi, %eax ++ andl $(PAGE_SIZE - 1), %eax ++ cmpl $(PAGE_SIZE - (VEC_SIZE * 4)), %eax ++ jg L(cross_page) ++ /* Start comparing 4 vectors. */ ++ VMOVU (%rdi), %YMM0 ++ VMOVU (%rsi), %YMM1 ++ ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ ++ /* Check for NULL in YMM0. */ ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ /* Check for NULL in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ ++ /* Each bit in K1 represents: ++ 1. A mismatch in YMM0 and YMM1. Or ++ 2. A NULL in YMM0 or YMM1. ++ */ ++ kord %k0, %k1, %k1 ++ ++ ktestd %k1, %k1 ++ je L(next_3_vectors) ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx) is after the maximum ++ offset (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ je L(return) ++L(wcscmp_return): ++ setl %al ++ negl %eax ++ orl $1, %eax ++L(return): ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(return_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + VEC_SIZE) is after ++ the maximum offset (%r11). */ ++ addq $VEC_SIZE, %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl VEC_SIZE(%rdi, %rdx), %ecx ++ cmpl VEC_SIZE(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl VEC_SIZE(%rdi, %rdx), %eax ++ movzbl VEC_SIZE(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(return_2_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + 2 * VEC_SIZE) is ++ after the maximum offset (%r11). */ ++ addq $(VEC_SIZE * 2), %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rdi, %rdx), %ecx ++ cmpl (VEC_SIZE * 2)(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rdi, %rdx), %eax ++ movzbl (VEC_SIZE * 2)(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(return_3_vec_size): ++ kmovd %k1, %ecx ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the mismatched index (%rdx + 3 * VEC_SIZE) is ++ after the maximum offset (%r11). */ ++ addq $(VEC_SIZE * 3), %rdx ++ cmpq %r11, %rdx ++ jae L(zero) ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rdi, %rdx), %ecx ++ cmpl (VEC_SIZE * 3)(%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rdi, %rdx), %eax ++ movzbl (VEC_SIZE * 3)(%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(next_3_vectors): ++ VMOVU VEC_SIZE(%rdi), %YMM0 ++ VMOVU VEC_SIZE(%rsi), %YMM1 ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_vec_size) ++ ++ VMOVU (VEC_SIZE * 2)(%rdi), %YMM2 ++ VMOVU (VEC_SIZE * 3)(%rdi), %YMM3 ++ VMOVU (VEC_SIZE * 2)(%rsi), %YMM4 ++ VMOVU (VEC_SIZE * 3)(%rsi), %YMM5 ++ ++ /* Each bit in K0 represents a mismatch in YMM2 and YMM4. */ ++ VPCMP $4, %YMM2, %YMM4, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM4, %k2 ++ /* Each bit in K1 represents a NULL in YMM2 or YMM4. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_2_vec_size) ++ ++ /* Each bit in K0 represents a mismatch in YMM3 and YMM5. */ ++ VPCMP $4, %YMM3, %YMM5, %k0 ++ VPCMP $0, %YMMZERO, %YMM3, %k1 ++ VPCMP $0, %YMMZERO, %YMM5, %k2 ++ /* Each bit in K1 represents a NULL in YMM3 or YMM5. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ ktestd %k1, %k1 ++ jne L(return_3_vec_size) ++L(main_loop_header): ++ leaq (VEC_SIZE * 4)(%rdi), %rdx ++ movl $PAGE_SIZE, %ecx ++ /* Align load via RAX. */ ++ andq $-(VEC_SIZE * 4), %rdx ++ subq %rdi, %rdx ++ leaq (%rdi, %rdx), %rax ++# ifdef USE_AS_STRNCMP ++ /* Starting from this point, the maximum offset, or simply the ++ 'offset', DECREASES by the same amount when base pointers are ++ moved forward. Return 0 when: ++ 1) On match: offset <= the matched vector index. ++ 2) On mistmach, offset is before the mistmatched index. ++ */ ++ subq %rdx, %r11 ++ jbe L(zero) ++# endif ++ addq %rsi, %rdx ++ movq %rdx, %rsi ++ andl $(PAGE_SIZE - 1), %esi ++ /* Number of bytes before page crossing. */ ++ subq %rsi, %rcx ++ /* Number of VEC_SIZE * 4 blocks before page crossing. */ ++ shrq $DIVIDE_BY_VEC_4_SHIFT, %rcx ++ /* ESI: Number of VEC_SIZE * 4 blocks before page crossing. */ ++ movl %ecx, %esi ++ jmp L(loop_start) ++ ++ .p2align 4 ++L(loop): ++# ifdef USE_AS_STRNCMP ++ /* Base pointers are moved forward by 4 * VEC_SIZE. Decrease ++ the maximum offset (%r11) by the same amount. */ ++ subq $(VEC_SIZE * 4), %r11 ++ jbe L(zero) ++# endif ++ addq $(VEC_SIZE * 4), %rax ++ addq $(VEC_SIZE * 4), %rdx ++L(loop_start): ++ testl %esi, %esi ++ leal -1(%esi), %esi ++ je L(loop_cross_page) ++L(back_to_loop): ++ /* Main loop, comparing 4 vectors are a time. */ ++ VMOVA (%rax), %YMM0 ++ VMOVA VEC_SIZE(%rax), %YMM2 ++ VMOVA (VEC_SIZE * 2)(%rax), %YMM4 ++ VMOVA (VEC_SIZE * 3)(%rax), %YMM6 ++ VMOVU (%rdx), %YMM1 ++ VMOVU VEC_SIZE(%rdx), %YMM3 ++ VMOVU (VEC_SIZE * 2)(%rdx), %YMM5 ++ VMOVU (VEC_SIZE * 3)(%rdx), %YMM7 ++ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K4 represents a NULL or a mismatch in YMM0 and ++ YMM1. */ ++ kord %k0, %k1, %k4 ++ ++ VPCMP $4, %YMM2, %YMM3, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM3, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K5 represents a NULL or a mismatch in YMM2 and ++ YMM3. */ ++ kord %k0, %k1, %k5 ++ ++ VPCMP $4, %YMM4, %YMM5, %k0 ++ VPCMP $0, %YMMZERO, %YMM4, %k1 ++ VPCMP $0, %YMMZERO, %YMM5, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K6 represents a NULL or a mismatch in YMM4 and ++ YMM5. */ ++ kord %k0, %k1, %k6 ++ ++ VPCMP $4, %YMM6, %YMM7, %k0 ++ VPCMP $0, %YMMZERO, %YMM6, %k1 ++ VPCMP $0, %YMMZERO, %YMM7, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K7 represents a NULL or a mismatch in YMM6 and ++ YMM7. */ ++ kord %k0, %k1, %k7 ++ ++ kord %k4, %k5, %k0 ++ kord %k6, %k7, %k1 ++ ++ /* Test each mask (32 bits) individually because for VEC_SIZE ++ == 32 is not possible to OR the four masks and keep all bits ++ in a 64-bit integer register, differing from SSE2 strcmp ++ where ORing is possible. */ ++ kortestd %k0, %k1 ++ je L(loop) ++ ktestd %k4, %k4 ++ je L(test_vec) ++ kmovd %k4, %edi ++ tzcntl %edi, %ecx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first vector matched. Return 0 if the maximum offset ++ (%r11) <= VEC_SIZE. */ ++ cmpq $VEC_SIZE, %r11 ++ jbe L(zero) ++# endif ++ ktestd %k5, %k5 ++ je L(test_2_vec) ++ kmovd %k5, %ecx ++ tzcntl %ecx, %edi ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edi ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $VEC_SIZE, %rdi ++ cmpq %rdi, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rdi), %ecx ++ cmpl (%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rdi), %eax ++ movzbl (%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl VEC_SIZE(%rsi, %rdi), %ecx ++ cmpl VEC_SIZE(%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl VEC_SIZE(%rax, %rdi), %eax ++ movzbl VEC_SIZE(%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_2_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first 2 vectors matched. Return 0 if the maximum offset ++ (%r11) <= 2 * VEC_SIZE. */ ++ cmpq $(VEC_SIZE * 2), %r11 ++ jbe L(zero) ++# endif ++ ktestd %k6, %k6 ++ je L(test_3_vec) ++ kmovd %k6, %ecx ++ tzcntl %ecx, %edi ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edi ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 2), %rdi ++ cmpq %rdi, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rdi), %ecx ++ cmpl (%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rdi), %eax ++ movzbl (%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rsi, %rdi), %ecx ++ cmpl (VEC_SIZE * 2)(%rdx, %rdi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rax, %rdi), %eax ++ movzbl (VEC_SIZE * 2)(%rdx, %rdi), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(test_3_vec): ++# ifdef USE_AS_STRNCMP ++ /* The first 3 vectors matched. Return 0 if the maximum offset ++ (%r11) <= 3 * VEC_SIZE. */ ++ cmpq $(VEC_SIZE * 3), %r11 ++ jbe L(zero) ++# endif ++ kmovd %k7, %esi ++ tzcntl %esi, %ecx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 3), %rcx ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %esi ++ cmpl (%rdx, %rcx), %esi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 3)(%rsi, %rcx), %esi ++ cmpl (VEC_SIZE * 3)(%rdx, %rcx), %esi ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 3)(%rax, %rcx), %eax ++ movzbl (VEC_SIZE * 3)(%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(loop_cross_page): ++ xorl %r10d, %r10d ++ movq %rdx, %rcx ++ /* Align load via RDX. We load the extra ECX bytes which should ++ be ignored. */ ++ andl $((VEC_SIZE * 4) - 1), %ecx ++ /* R10 is -RCX. */ ++ subq %rcx, %r10 ++ ++ /* This works only if VEC_SIZE * 2 == 64. */ ++# if (VEC_SIZE * 2) != 64 ++# error (VEC_SIZE * 2) != 64 ++# endif ++ ++ /* Check if the first VEC_SIZE * 2 bytes should be ignored. */ ++ cmpl $(VEC_SIZE * 2), %ecx ++ jge L(loop_cross_page_2_vec) ++ ++ VMOVU (%rax, %r10), %YMM2 ++ VMOVU VEC_SIZE(%rax, %r10), %YMM3 ++ VMOVU (%rdx, %r10), %YMM4 ++ VMOVU VEC_SIZE(%rdx, %r10), %YMM5 ++ ++ VPCMP $4, %YMM4, %YMM2, %k0 ++ VPCMP $0, %YMMZERO, %YMM2, %k1 ++ VPCMP $0, %YMMZERO, %YMM4, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch in YMM2 and ++ YMM4. */ ++ kord %k0, %k1, %k1 ++ ++ VPCMP $4, %YMM5, %YMM3, %k3 ++ VPCMP $0, %YMMZERO, %YMM3, %k4 ++ VPCMP $0, %YMMZERO, %YMM5, %k5 ++ kord %k4, %k5, %k4 ++ /* Each bit in K3 represents a NULL or a mismatch in YMM3 and ++ YMM5. */ ++ kord %k3, %k4, %k3 ++ ++# ifdef USE_AS_WCSCMP ++ /* NB: Each bit in K1/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k2 ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG32 ++ sarl $2, %SHIFT_REG32 ++# else ++ kshiftlq $32, %k3, %k2 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rdi ++ ++ /* Since ECX < VEC_SIZE * 2, simply skip the first ECX bytes. */ ++ shrxq %SHIFT_REG64, %rdi, %rdi ++ testq %rdi, %rdi ++ je L(loop_cross_page_2_vec) ++ tzcntq %rdi, %rcx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++ .p2align 4 ++L(loop_cross_page_2_vec): ++ /* The first VEC_SIZE * 2 bytes match or are ignored. */ ++ VMOVU (VEC_SIZE * 2)(%rax, %r10), %YMM0 ++ VMOVU (VEC_SIZE * 3)(%rax, %r10), %YMM1 ++ VMOVU (VEC_SIZE * 2)(%rdx, %r10), %YMM2 ++ VMOVU (VEC_SIZE * 3)(%rdx, %r10), %YMM3 ++ ++ VPCMP $4, %YMM0, %YMM2, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM2, %k2 ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch in YMM0 and ++ YMM2. */ ++ kord %k0, %k1, %k1 ++ ++ VPCMP $4, %YMM1, %YMM3, %k3 ++ VPCMP $0, %YMMZERO, %YMM1, %k4 ++ VPCMP $0, %YMMZERO, %YMM3, %k5 ++ kord %k4, %k5, %k4 ++ /* Each bit in K3 represents a NULL or a mismatch in YMM1 and ++ YMM3. */ ++ kord %k3, %k4, %k3 ++ ++# ifdef USE_AS_WCSCMP ++ /* NB: Each bit in K1/K3 represents 4-byte element. */ ++ kshiftlw $8, %k3, %k2 ++# else ++ kshiftlq $32, %k3, %k2 ++# endif ++ ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korq %k1, %k2, %k1 ++ kmovq %k1, %rdi ++ ++ xorl %r8d, %r8d ++ /* If ECX > VEC_SIZE * 2, skip ECX - (VEC_SIZE * 2) bytes. */ ++ subl $(VEC_SIZE * 2), %ecx ++ jle 1f ++ /* R8 has number of bytes skipped. */ ++ movl %ecx, %r8d ++# ifdef USE_AS_WCSCMP ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ sarl $2, %ecx ++# endif ++ /* Skip ECX bytes. */ ++ shrq %cl, %rdi ++1: ++ /* Before jumping back to the loop, set ESI to the number of ++ VEC_SIZE * 4 blocks before page crossing. */ ++ movl $(PAGE_SIZE / (VEC_SIZE * 4) - 1), %esi ++ ++ testq %rdi, %rdi ++# ifdef USE_AS_STRNCMP ++ /* At this point, if %rdi value is 0, it already tested ++ VEC_SIZE*4+%r10 byte starting from %rax. This label ++ checks whether strncmp maximum offset reached or not. */ ++ je L(string_nbyte_offset_check) ++# else ++ je L(back_to_loop) ++# endif ++ tzcntq %rdi, %rcx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %ecx ++# endif ++ addq %r10, %rcx ++ /* Adjust for number of bytes skipped. */ ++ addq %r8, %rcx ++# ifdef USE_AS_STRNCMP ++ addq $(VEC_SIZE * 2), %rcx ++ subq %rcx, %r11 ++ jbe L(zero) ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (%rsi, %rcx), %edi ++ cmpl (%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (%rax, %rcx), %eax ++ movzbl (%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# else ++# ifdef USE_AS_WCSCMP ++ movq %rax, %rsi ++ xorl %eax, %eax ++ movl (VEC_SIZE * 2)(%rsi, %rcx), %edi ++ cmpl (VEC_SIZE * 2)(%rdx, %rcx), %edi ++ jne L(wcscmp_return) ++# else ++ movzbl (VEC_SIZE * 2)(%rax, %rcx), %eax ++ movzbl (VEC_SIZE * 2)(%rdx, %rcx), %edx ++ subl %edx, %eax ++# endif ++# endif ++ ret ++ ++# ifdef USE_AS_STRNCMP ++L(string_nbyte_offset_check): ++ leaq (VEC_SIZE * 4)(%r10), %r10 ++ cmpq %r10, %r11 ++ jbe L(zero) ++ jmp L(back_to_loop) ++# endif ++ ++ .p2align 4 ++L(cross_page_loop): ++ /* Check one byte/dword at a time. */ ++# ifdef USE_AS_WCSCMP ++ cmpl %ecx, %eax ++# else ++ subl %ecx, %eax ++# endif ++ jne L(different) ++ addl $SIZE_OF_CHAR, %edx ++ cmpl $(VEC_SIZE * 4), %edx ++ je L(main_loop_header) ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ movl (%rdi, %rdx), %eax ++ movl (%rsi, %rdx), %ecx ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %ecx ++# endif ++ /* Check null char. */ ++ testl %eax, %eax ++ jne L(cross_page_loop) ++ /* Since %eax == 0, subtract is OK for both SIGNED and UNSIGNED ++ comparisons. */ ++ subl %ecx, %eax ++# ifndef USE_AS_WCSCMP ++L(different): ++# endif ++ ret ++ ++# ifdef USE_AS_WCSCMP ++ .p2align 4 ++L(different): ++ /* Use movl to avoid modifying EFLAGS. */ ++ movl $0, %eax ++ setl %al ++ negl %eax ++ orl $1, %eax ++ ret ++# endif ++ ++# ifdef USE_AS_STRNCMP ++ .p2align 4 ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(char0): ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi), %ecx ++ cmpl (%rsi), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rsi), %ecx ++ movzbl (%rdi), %eax ++ subl %ecx, %eax ++# endif ++ ret ++# endif ++ ++ .p2align 4 ++L(last_vector): ++ addq %rdx, %rdi ++ addq %rdx, %rsi ++# ifdef USE_AS_STRNCMP ++ subq %rdx, %r11 ++# endif ++ tzcntl %ecx, %edx ++# ifdef USE_AS_WCSCMP ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ sall $2, %edx ++# endif ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ xorl %eax, %eax ++ movl (%rdi, %rdx), %ecx ++ cmpl (%rsi, %rdx), %ecx ++ jne L(wcscmp_return) ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %edx ++ subl %edx, %eax ++# endif ++ ret ++ ++ /* Comparing on page boundary region requires special treatment: ++ It must done one vector at the time, starting with the wider ++ ymm vector if possible, if not, with xmm. If fetching 16 bytes ++ (xmm) still passes the boundary, byte comparison must be done. ++ */ ++ .p2align 4 ++L(cross_page): ++ /* Try one ymm vector at a time. */ ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jg L(cross_page_1_vector) ++L(loop_1_vector): ++ VMOVU (%rdi, %rdx), %YMM0 ++ VMOVU (%rsi, %rdx), %YMM1 ++ ++ /* Each bit in K0 represents a mismatch in YMM0 and YMM1. */ ++ VPCMP $4, %YMM0, %YMM1, %k0 ++ VPCMP $0, %YMMZERO, %YMM0, %k1 ++ VPCMP $0, %YMMZERO, %YMM1, %k2 ++ /* Each bit in K1 represents a NULL in YMM0 or YMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $VEC_SIZE, %edx ++ ++ addl $VEC_SIZE, %eax ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ jle L(loop_1_vector) ++L(cross_page_1_vector): ++ /* Less than 32 bytes to check, try one xmm vector. */ ++ cmpl $(PAGE_SIZE - 16), %eax ++ jg L(cross_page_1_xmm) ++ VMOVU (%rdi, %rdx), %XMM0 ++ VMOVU (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ korw %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ korw %k0, %k1, %k1 ++ kmovw %k1, %ecx ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $16, %edx ++# ifndef USE_AS_WCSCMP ++ addl $16, %eax ++# endif ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_1_xmm): ++# ifndef USE_AS_WCSCMP ++ /* Less than 16 bytes to check, try 8 byte vector. NB: No need ++ for wcscmp nor wcsncmp since wide char is 4 bytes. */ ++ cmpl $(PAGE_SIZE - 8), %eax ++ jg L(cross_page_8bytes) ++ vmovq (%rdi, %rdx), %XMM0 ++ vmovq (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ ++# ifdef USE_AS_WCSCMP ++ /* Only last 2 bits are valid. */ ++ andl $0x3, %ecx ++# else ++ /* Only last 8 bits are valid. */ ++ andl $0xff, %ecx ++# endif ++ ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $8, %edx ++ addl $8, %eax ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_8bytes): ++ /* Less than 8 bytes to check, try 4 byte vector. */ ++ cmpl $(PAGE_SIZE - 4), %eax ++ jg L(cross_page_4bytes) ++ vmovd (%rdi, %rdx), %XMM0 ++ vmovd (%rsi, %rdx), %XMM1 ++ ++ /* Each bit in K0 represents a mismatch in XMM0 and XMM1. */ ++ VPCMP $4, %XMM0, %XMM1, %k0 ++ VPCMP $0, %XMMZERO, %XMM0, %k1 ++ VPCMP $0, %XMMZERO, %XMM1, %k2 ++ /* Each bit in K1 represents a NULL in XMM0 or XMM1. */ ++ kord %k1, %k2, %k1 ++ /* Each bit in K1 represents a NULL or a mismatch. */ ++ kord %k0, %k1, %k1 ++ kmovd %k1, %ecx ++ ++# ifdef USE_AS_WCSCMP ++ /* Only the last bit is valid. */ ++ andl $0x1, %ecx ++# else ++ /* Only last 4 bits are valid. */ ++ andl $0xf, %ecx ++# endif ++ ++ testl %ecx, %ecx ++ jne L(last_vector) ++ ++ addl $4, %edx ++# ifdef USE_AS_STRNCMP ++ /* Return 0 if the current offset (%rdx) >= the maximum offset ++ (%r11). */ ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++ ++L(cross_page_4bytes): ++# endif ++ /* Less than 4 bytes to check, try one byte/dword at a time. */ ++# ifdef USE_AS_STRNCMP ++ cmpq %r11, %rdx ++ jae L(zero) ++# endif ++# ifdef USE_AS_WCSCMP ++ movl (%rdi, %rdx), %eax ++ movl (%rsi, %rdx), %ecx ++# else ++ movzbl (%rdi, %rdx), %eax ++ movzbl (%rsi, %rdx), %ecx ++# endif ++ testl %eax, %eax ++ jne L(cross_page_loop) ++ subl %ecx, %eax ++ ret ++END (STRCMP) ++#endif +diff --git a/sysdeps/x86_64/multiarch/strcmp.c b/sysdeps/x86_64/multiarch/strcmp.c +index 16ae72a4c8..df4ba875d9 100644 +--- a/sysdeps/x86_64/multiarch/strcmp.c ++++ b/sysdeps/x86_64/multiarch/strcmp.c +@@ -30,16 +30,29 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + return OPTIMIZE (sse2_unaligned); +diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S +new file mode 100644 +index 0000000000..c2c581ecf7 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcpy-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRCPY ++# define STRCPY __strcpy_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strcpy-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2.S b/sysdeps/x86_64/multiarch/strcpy-avx2.S +index 3f2f9e8170..1ce17253ab 100644 +--- a/sysdeps/x86_64/multiarch/strcpy-avx2.S ++++ b/sysdeps/x86_64/multiarch/strcpy-avx2.S +@@ -37,6 +37,10 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + /* zero register */ + #define xmmZ xmm0 + #define ymmZ ymm0 +@@ -46,7 +50,7 @@ + + # ifndef USE_AS_STRCAT + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRCPY) + # ifdef USE_AS_STRNCPY + mov %RDX_LP, %R8_LP +@@ -369,8 +373,8 @@ L(CopyVecSizeExit): + lea 1(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(CopyTwoVecSize1): +@@ -553,8 +557,7 @@ L(Exit1): + lea 2(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit2): +@@ -569,8 +572,7 @@ L(Exit2): + lea 3(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit3): +@@ -584,8 +586,7 @@ L(Exit3): + lea 4(%rdi), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit4_7): +@@ -602,8 +603,7 @@ L(Exit4_7): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit8_15): +@@ -620,8 +620,7 @@ L(Exit8_15): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit16_31): +@@ -638,8 +637,7 @@ L(Exit16_31): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Exit32_63): +@@ -656,8 +654,7 @@ L(Exit32_63): + lea 1(%rdi, %rdx), %rdi + jnz L(StrncpyFillTailWithZero) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNCPY + +@@ -671,8 +668,7 @@ L(StrncpyExit1): + # ifdef USE_AS_STRCAT + movb $0, 1(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit2): +@@ -684,8 +680,7 @@ L(StrncpyExit2): + # ifdef USE_AS_STRCAT + movb $0, 2(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit3_4): +@@ -699,8 +694,7 @@ L(StrncpyExit3_4): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit5_8): +@@ -714,8 +708,7 @@ L(StrncpyExit5_8): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit9_16): +@@ -729,8 +722,7 @@ L(StrncpyExit9_16): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit17_32): +@@ -744,8 +736,7 @@ L(StrncpyExit17_32): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit33_64): +@@ -760,8 +751,7 @@ L(StrncpyExit33_64): + # ifdef USE_AS_STRCAT + movb $0, (%rdi, %r8) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(StrncpyExit65): +@@ -778,50 +768,43 @@ L(StrncpyExit65): + # ifdef USE_AS_STRCAT + movb $0, 65(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # ifndef USE_AS_STRCAT + + .p2align 4 + L(Fill1): + mov %dl, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill2): + mov %dx, (%rdi) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill3_4): + mov %dx, (%rdi) + mov %dx, -2(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill5_8): + mov %edx, (%rdi) + mov %edx, -4(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill9_16): + mov %rdx, (%rdi) + mov %rdx, -8(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(Fill17_32): + vmovdqu %xmmZ, (%rdi) + vmovdqu %xmmZ, -16(%rdi, %r8) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(CopyVecSizeUnalignedVec2): +@@ -898,8 +881,7 @@ L(Fill): + cmp $1, %r8d + ja L(Fill2) + je L(Fill1) +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + /* end of ifndef USE_AS_STRCAT */ + # endif +@@ -929,8 +911,7 @@ L(UnalignedFourVecSizeLeaveCase3): + # ifdef USE_AS_STRCAT + movb $0, (VEC_SIZE * 4)(%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(UnalignedFourVecSizeLeaveCase2): +@@ -1001,16 +982,14 @@ L(StrncpyExit): + # ifdef USE_AS_STRCAT + movb $0, (%rdi) + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(ExitZero): + # ifndef USE_AS_STRCAT + mov %rdi, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + # endif + +diff --git a/sysdeps/x86_64/multiarch/strcpy-evex.S b/sysdeps/x86_64/multiarch/strcpy-evex.S +new file mode 100644 +index 0000000000..a343a1a692 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strcpy-evex.S +@@ -0,0 +1,1003 @@ ++/* strcpy with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# ifndef USE_AS_STRCAT ++# include <sysdep.h> ++ ++# ifndef STRCPY ++# define STRCPY __strcpy_evex ++# endif ++ ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++/* Number of bytes in a vector register */ ++# ifndef VEC_SIZE ++# define VEC_SIZE 32 ++# endif ++ ++# define XMM2 xmm18 ++# define XMM3 xmm19 ++ ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++# define YMM7 ymm23 ++ ++# ifndef USE_AS_STRCAT ++ ++/* zero register */ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM1 ymm17 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRCPY) ++# ifdef USE_AS_STRNCPY ++ mov %RDX_LP, %R8_LP ++ test %R8_LP, %R8_LP ++ jz L(ExitZero) ++# endif ++ mov %rsi, %rcx ++# ifndef USE_AS_STPCPY ++ mov %rdi, %rax /* save result */ ++# endif ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++# endif ++ ++ and $((VEC_SIZE * 4) - 1), %ecx ++ cmp $(VEC_SIZE * 2), %ecx ++ jbe L(SourceStringAlignmentLessTwoVecSize) ++ ++ and $-VEC_SIZE, %rsi ++ and $(VEC_SIZE - 1), %ecx ++ ++ vpcmpb $0, (%rsi), %YMMZERO, %k0 ++ kmovd %k0, %edx ++ shr %cl, %rdx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ mov $VEC_SIZE, %r10 ++ sub %rcx, %r10 ++ cmp %r10, %r8 ++# else ++ mov $(VEC_SIZE + 1), %r10 ++ sub %rcx, %r10 ++ cmp %r10, %r8 ++# endif ++ jbe L(CopyVecSizeTailCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyVecSizeTail) ++ ++ vpcmpb $0, VEC_SIZE(%rsi), %YMMZERO, %k1 ++ kmovd %k1, %edx ++ ++# ifdef USE_AS_STRNCPY ++ add $VEC_SIZE, %r10 ++ cmp %r10, %r8 ++ jbe L(CopyTwoVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyTwoVecSize) ++ ++ VMOVU (%rsi, %rcx), %YMM2 /* copy VEC_SIZE bytes */ ++ VMOVU %YMM2, (%rdi) ++ ++/* If source address alignment != destination address alignment */ ++ .p2align 4 ++L(UnalignVecSizeBoth): ++ sub %rcx, %rdi ++# ifdef USE_AS_STRNCPY ++ add %rcx, %r8 ++ sbb %rcx, %rcx ++ or %rcx, %r8 ++# endif ++ mov $VEC_SIZE, %rcx ++ VMOVA (%rsi, %rcx), %YMM2 ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 3), %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM3 ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec3) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM3, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM4 ++ vpcmpb $0, %YMM4, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec4) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM4, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM2, (%rdi, %rcx) ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM2 ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec2) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVA VEC_SIZE(%rsi, %rcx), %YMM3 ++ VMOVU %YMM2, (%rdi, %rcx) ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ add $VEC_SIZE, %rcx ++# ifdef USE_AS_STRNCPY ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++# endif ++ test %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec3) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ VMOVU %YMM3, (%rdi, %rcx) ++ mov %rsi, %rdx ++ lea VEC_SIZE(%rsi, %rcx), %rsi ++ and $-(VEC_SIZE * 4), %rsi ++ sub %rsi, %rdx ++ sub %rdx, %rdi ++# ifdef USE_AS_STRNCPY ++ lea (VEC_SIZE * 8)(%r8, %rdx), %r8 ++# endif ++L(UnalignedFourVecSizeLoop): ++ VMOVA (%rsi), %YMM4 ++ VMOVA VEC_SIZE(%rsi), %YMM5 ++ VMOVA (VEC_SIZE * 2)(%rsi), %YMM6 ++ VMOVA (VEC_SIZE * 3)(%rsi), %YMM7 ++ vpminub %YMM5, %YMM4, %YMM2 ++ vpminub %YMM7, %YMM6, %YMM3 ++ vpminub %YMM2, %YMM3, %YMM2 ++ /* If K7 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM2, %YMMZERO, %k7 ++ kmovd %k7, %edx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 4), %r8 ++ jbe L(UnalignedLeaveCase2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(UnalignedFourVecSizeLeave) ++ ++L(UnalignedFourVecSizeLoop_start): ++ add $(VEC_SIZE * 4), %rdi ++ add $(VEC_SIZE * 4), %rsi ++ VMOVU %YMM4, -(VEC_SIZE * 4)(%rdi) ++ VMOVA (%rsi), %YMM4 ++ VMOVU %YMM5, -(VEC_SIZE * 3)(%rdi) ++ VMOVA VEC_SIZE(%rsi), %YMM5 ++ vpminub %YMM5, %YMM4, %YMM2 ++ VMOVU %YMM6, -(VEC_SIZE * 2)(%rdi) ++ VMOVA (VEC_SIZE * 2)(%rsi), %YMM6 ++ VMOVU %YMM7, -VEC_SIZE(%rdi) ++ VMOVA (VEC_SIZE * 3)(%rsi), %YMM7 ++ vpminub %YMM7, %YMM6, %YMM3 ++ vpminub %YMM2, %YMM3, %YMM2 ++ /* If K7 != 0, there is a null byte. */ ++ vpcmpb $0, %YMM2, %YMMZERO, %k7 ++ kmovd %k7, %edx ++# ifdef USE_AS_STRNCPY ++ sub $(VEC_SIZE * 4), %r8 ++ jbe L(UnalignedLeaveCase2OrCase3) ++# endif ++ test %edx, %edx ++ jz L(UnalignedFourVecSizeLoop_start) ++ ++L(UnalignedFourVecSizeLeave): ++ vpcmpb $0, %YMM4, %YMMZERO, %k1 ++ kmovd %k1, %edx ++ test %edx, %edx ++ jnz L(CopyVecSizeUnaligned_0) ++ ++ vpcmpb $0, %YMM5, %YMMZERO, %k2 ++ kmovd %k2, %ecx ++ test %ecx, %ecx ++ jnz L(CopyVecSizeUnaligned_16) ++ ++ vpcmpb $0, %YMM6, %YMMZERO, %k3 ++ kmovd %k3, %edx ++ test %edx, %edx ++ jnz L(CopyVecSizeUnaligned_32) ++ ++ vpcmpb $0, %YMM7, %YMMZERO, %k4 ++ kmovd %k4, %ecx ++ bsf %ecx, %edx ++ VMOVU %YMM4, (%rdi) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 3)(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM7, (VEC_SIZE * 3)(%rdi) ++ add $(VEC_SIZE - 1), %r8 ++ sub %rdx, %r8 ++ lea ((VEC_SIZE * 3) + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $(VEC_SIZE * 3), %rsi ++ add $(VEC_SIZE * 3), %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++/* If source address alignment == destination address alignment */ ++ ++L(SourceStringAlignmentLessTwoVecSize): ++ VMOVU (%rsi), %YMM3 ++ VMOVU VEC_SIZE(%rsi), %YMM2 ++ vpcmpb $0, %YMM3, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ cmp $VEC_SIZE, %r8 ++# else ++ cmp $(VEC_SIZE + 1), %r8 ++# endif ++ jbe L(CopyVecSizeTail1Case2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyVecSizeTail1) ++ ++ VMOVU %YMM3, (%rdi) ++ vpcmpb $0, %YMM2, %YMMZERO, %k0 ++ kmovd %k0, %edx ++ ++# ifdef USE_AS_STRNCPY ++# if defined USE_AS_STPCPY || defined USE_AS_STRCAT ++ cmp $(VEC_SIZE * 2), %r8 ++# else ++ cmp $((VEC_SIZE * 2) + 1), %r8 ++# endif ++ jbe L(CopyTwoVecSize1Case2OrCase3) ++# endif ++ test %edx, %edx ++ jnz L(CopyTwoVecSize1) ++ ++ and $-VEC_SIZE, %rsi ++ and $(VEC_SIZE - 1), %ecx ++ jmp L(UnalignVecSizeBoth) ++ ++/*------End of main part with loops---------------------*/ ++ ++/* Case1 */ ++ ++# if (!defined USE_AS_STRNCPY) || (defined USE_AS_STRCAT) ++ .p2align 4 ++L(CopyVecSize): ++ add %rcx, %rdi ++# endif ++L(CopyVecSizeTail): ++ add %rcx, %rsi ++L(CopyVecSizeTail1): ++ bsf %edx, %edx ++L(CopyVecSizeExit): ++ cmp $32, %edx ++ jae L(Exit32_63) ++ cmp $16, %edx ++ jae L(Exit16_31) ++ cmp $8, %edx ++ jae L(Exit8_15) ++ cmp $4, %edx ++ jae L(Exit4_7) ++ cmp $3, %edx ++ je L(Exit3) ++ cmp $1, %edx ++ ja L(Exit2) ++ je L(Exit1) ++ movb $0, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea (%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $1, %r8 ++ lea 1(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(CopyTwoVecSize1): ++ add $VEC_SIZE, %rsi ++ add $VEC_SIZE, %rdi ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $VEC_SIZE, %r8 ++# endif ++ jmp L(CopyVecSizeTail1) ++ ++ .p2align 4 ++L(CopyTwoVecSize): ++ bsf %edx, %edx ++ add %rcx, %rsi ++ add $VEC_SIZE, %edx ++ sub %ecx, %edx ++ jmp L(CopyVecSizeExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_0): ++ bsf %edx, %edx ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM4, (%rdi) ++ add $((VEC_SIZE * 4) - 1), %r8 ++ sub %rdx, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ jmp L(CopyVecSizeExit) ++# endif ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_16): ++ bsf %ecx, %edx ++ VMOVU %YMM4, (%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea VEC_SIZE(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ add $((VEC_SIZE * 3) - 1), %r8 ++ sub %rdx, %r8 ++ lea (VEC_SIZE + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $VEC_SIZE, %rsi ++ add $VEC_SIZE, %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++ .p2align 4 ++L(CopyVecSizeUnaligned_32): ++ bsf %edx, %edx ++ VMOVU %YMM4, (%rdi) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 2)(%rdi, %rdx), %rax ++# endif ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ add $((VEC_SIZE * 2) - 1), %r8 ++ sub %rdx, %r8 ++ lea ((VEC_SIZE * 2) + 1)(%rdi, %rdx), %rdi ++ jmp L(StrncpyFillTailWithZero) ++# else ++ add $(VEC_SIZE * 2), %rsi ++ add $(VEC_SIZE * 2), %rdi ++ jmp L(CopyVecSizeExit) ++# endif ++ ++# ifdef USE_AS_STRNCPY ++# ifndef USE_AS_STRCAT ++ .p2align 4 ++L(CopyVecSizeUnalignedVec6): ++ VMOVU %YMM6, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec5): ++ VMOVU %YMM5, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec4): ++ VMOVU %YMM4, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec3): ++ VMOVU %YMM3, (%rdi, %rcx) ++ jmp L(CopyVecSizeVecExit) ++# endif ++ ++/* Case2 */ ++ ++ .p2align 4 ++L(CopyVecSizeCase2): ++ add $VEC_SIZE, %r8 ++ add %rcx, %rdi ++ add %rcx, %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSizeCase2): ++ add %rcx, %rsi ++ bsf %edx, %edx ++ add $VEC_SIZE, %edx ++ sub %ecx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++L(CopyVecSizeTailCase2): ++ add %rcx, %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++L(CopyVecSizeTail1Case2): ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++ jmp L(StrncpyExit) ++ ++/* Case2 or Case3, Case3 */ ++ ++ .p2align 4 ++L(CopyVecSizeCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeCase2) ++L(CopyVecSizeCase3): ++ add $VEC_SIZE, %r8 ++ add %rcx, %rdi ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSizeCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyTwoVecSizeCase2) ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyVecSizeTailCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeTailCase2) ++ add %rcx, %rsi ++ jmp L(StrncpyExit) ++ ++ .p2align 4 ++L(CopyTwoVecSize1Case2OrCase3): ++ add $VEC_SIZE, %rdi ++ add $VEC_SIZE, %rsi ++ sub $VEC_SIZE, %r8 ++L(CopyVecSizeTail1Case2OrCase3): ++ test %rdx, %rdx ++ jnz L(CopyVecSizeTail1Case2) ++ jmp L(StrncpyExit) ++# endif ++ ++/*------------End labels regarding with copying 1-VEC_SIZE bytes--and 1-(VEC_SIZE*2) bytes----*/ ++ ++ .p2align 4 ++L(Exit1): ++ movzwl (%rsi), %edx ++ mov %dx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 1(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $2, %r8 ++ lea 2(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit2): ++ movzwl (%rsi), %ecx ++ mov %cx, (%rdi) ++ movb $0, 2(%rdi) ++# ifdef USE_AS_STPCPY ++ lea 2(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $3, %r8 ++ lea 3(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit3): ++ mov (%rsi), %edx ++ mov %edx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 3(%rdi), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub $4, %r8 ++ lea 4(%rdi), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit4_7): ++ mov (%rsi), %ecx ++ mov %ecx, (%rdi) ++ mov -3(%rsi, %rdx), %ecx ++ mov %ecx, -3(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit8_15): ++ mov (%rsi), %rcx ++ mov -7(%rsi, %rdx), %r9 ++ mov %rcx, (%rdi) ++ mov %r9, -7(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit16_31): ++ VMOVU (%rsi), %XMM2 ++ VMOVU -15(%rsi, %rdx), %XMM3 ++ VMOVU %XMM2, (%rdi) ++ VMOVU %XMM3, -15(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++ .p2align 4 ++L(Exit32_63): ++ VMOVU (%rsi), %YMM2 ++ VMOVU -31(%rsi, %rdx), %YMM3 ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, -31(%rdi, %rdx) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++# if defined USE_AS_STRNCPY && !defined USE_AS_STRCAT ++ sub %rdx, %r8 ++ sub $1, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ jnz L(StrncpyFillTailWithZero) ++# endif ++ ret ++ ++# ifdef USE_AS_STRNCPY ++ ++ .p2align 4 ++L(StrncpyExit1): ++ movzbl (%rsi), %edx ++ mov %dl, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 1(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 1(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit2): ++ movzwl (%rsi), %edx ++ mov %dx, (%rdi) ++# ifdef USE_AS_STPCPY ++ lea 2(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 2(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit3_4): ++ movzwl (%rsi), %ecx ++ movzwl -2(%rsi, %r8), %edx ++ mov %cx, (%rdi) ++ mov %dx, -2(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit5_8): ++ mov (%rsi), %ecx ++ mov -4(%rsi, %r8), %edx ++ mov %ecx, (%rdi) ++ mov %edx, -4(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit9_16): ++ mov (%rsi), %rcx ++ mov -8(%rsi, %r8), %rdx ++ mov %rcx, (%rdi) ++ mov %rdx, -8(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit17_32): ++ VMOVU (%rsi), %XMM2 ++ VMOVU -16(%rsi, %r8), %XMM3 ++ VMOVU %XMM2, (%rdi) ++ VMOVU %XMM3, -16(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit33_64): ++ /* 0/32, 31/16 */ ++ VMOVU (%rsi), %YMM2 ++ VMOVU -VEC_SIZE(%rsi, %r8), %YMM3 ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, -VEC_SIZE(%rdi, %r8) ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %r8), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi, %r8) ++# endif ++ ret ++ ++ .p2align 4 ++L(StrncpyExit65): ++ /* 0/32, 32/32, 64/1 */ ++ VMOVU (%rsi), %YMM2 ++ VMOVU 32(%rsi), %YMM3 ++ mov 64(%rsi), %cl ++ VMOVU %YMM2, (%rdi) ++ VMOVU %YMM3, 32(%rdi) ++ mov %cl, 64(%rdi) ++# ifdef USE_AS_STPCPY ++ lea 65(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, 65(%rdi) ++# endif ++ ret ++ ++# ifndef USE_AS_STRCAT ++ ++ .p2align 4 ++L(Fill1): ++ mov %dl, (%rdi) ++ ret ++ ++ .p2align 4 ++L(Fill2): ++ mov %dx, (%rdi) ++ ret ++ ++ .p2align 4 ++L(Fill3_4): ++ mov %dx, (%rdi) ++ mov %dx, -2(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill5_8): ++ mov %edx, (%rdi) ++ mov %edx, -4(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill9_16): ++ mov %rdx, (%rdi) ++ mov %rdx, -8(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(Fill17_32): ++ VMOVU %XMMZERO, (%rdi) ++ VMOVU %XMMZERO, -16(%rdi, %r8) ++ ret ++ ++ .p2align 4 ++L(CopyVecSizeUnalignedVec2): ++ VMOVU %YMM2, (%rdi, %rcx) ++ ++ .p2align 4 ++L(CopyVecSizeVecExit): ++ bsf %edx, %edx ++ add $(VEC_SIZE - 1), %r8 ++ add %rcx, %rdi ++# ifdef USE_AS_STPCPY ++ lea (%rdi, %rdx), %rax ++# endif ++ sub %rdx, %r8 ++ lea 1(%rdi, %rdx), %rdi ++ ++ .p2align 4 ++L(StrncpyFillTailWithZero): ++ xor %edx, %edx ++ sub $VEC_SIZE, %r8 ++ jbe L(StrncpyFillExit) ++ ++ VMOVU %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ ++ mov %rdi, %rsi ++ and $(VEC_SIZE - 1), %esi ++ sub %rsi, %rdi ++ add %rsi, %r8 ++ sub $(VEC_SIZE * 4), %r8 ++ jb L(StrncpyFillLessFourVecSize) ++ ++L(StrncpyFillLoopVmovdqa): ++ VMOVA %YMMZERO, (%rdi) ++ VMOVA %YMMZERO, VEC_SIZE(%rdi) ++ VMOVA %YMMZERO, (VEC_SIZE * 2)(%rdi) ++ VMOVA %YMMZERO, (VEC_SIZE * 3)(%rdi) ++ add $(VEC_SIZE * 4), %rdi ++ sub $(VEC_SIZE * 4), %r8 ++ jae L(StrncpyFillLoopVmovdqa) ++ ++L(StrncpyFillLessFourVecSize): ++ add $(VEC_SIZE * 2), %r8 ++ jl L(StrncpyFillLessTwoVecSize) ++ VMOVA %YMMZERO, (%rdi) ++ VMOVA %YMMZERO, VEC_SIZE(%rdi) ++ add $(VEC_SIZE * 2), %rdi ++ sub $VEC_SIZE, %r8 ++ jl L(StrncpyFillExit) ++ VMOVA %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ jmp L(Fill) ++ ++ .p2align 4 ++L(StrncpyFillLessTwoVecSize): ++ add $VEC_SIZE, %r8 ++ jl L(StrncpyFillExit) ++ VMOVA %YMMZERO, (%rdi) ++ add $VEC_SIZE, %rdi ++ jmp L(Fill) ++ ++ .p2align 4 ++L(StrncpyFillExit): ++ add $VEC_SIZE, %r8 ++L(Fill): ++ cmp $17, %r8d ++ jae L(Fill17_32) ++ cmp $9, %r8d ++ jae L(Fill9_16) ++ cmp $5, %r8d ++ jae L(Fill5_8) ++ cmp $3, %r8d ++ jae L(Fill3_4) ++ cmp $1, %r8d ++ ja L(Fill2) ++ je L(Fill1) ++ ret ++ ++/* end of ifndef USE_AS_STRCAT */ ++# endif ++ ++ .p2align 4 ++L(UnalignedLeaveCase2OrCase3): ++ test %rdx, %rdx ++ jnz L(UnalignedFourVecSizeLeaveCase2) ++L(UnalignedFourVecSizeLeaveCase3): ++ lea (VEC_SIZE * 4)(%r8), %rcx ++ and $-VEC_SIZE, %rcx ++ add $(VEC_SIZE * 3), %r8 ++ jl L(CopyVecSizeCase3) ++ VMOVU %YMM4, (%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ sub $VEC_SIZE, %r8 ++ jb L(CopyVecSizeCase3) ++ VMOVU %YMM7, (VEC_SIZE * 3)(%rdi) ++# ifdef USE_AS_STPCPY ++ lea (VEC_SIZE * 4)(%rdi), %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (VEC_SIZE * 4)(%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(UnalignedFourVecSizeLeaveCase2): ++ xor %ecx, %ecx ++ vpcmpb $0, %YMM4, %YMMZERO, %k1 ++ kmovd %k1, %edx ++ add $(VEC_SIZE * 3), %r8 ++ jle L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec4) ++# else ++ jnz L(CopyVecSize) ++# endif ++ vpcmpb $0, %YMM5, %YMMZERO, %k2 ++ kmovd %k2, %edx ++ VMOVU %YMM4, (%rdi) ++ add $VEC_SIZE, %rcx ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec5) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ vpcmpb $0, %YMM6, %YMMZERO, %k3 ++ kmovd %k3, %edx ++ VMOVU %YMM5, VEC_SIZE(%rdi) ++ add $VEC_SIZE, %rcx ++ sub $VEC_SIZE, %r8 ++ jbe L(CopyVecSizeCase2OrCase3) ++ test %edx, %edx ++# ifndef USE_AS_STRCAT ++ jnz L(CopyVecSizeUnalignedVec6) ++# else ++ jnz L(CopyVecSize) ++# endif ++ ++ vpcmpb $0, %YMM7, %YMMZERO, %k4 ++ kmovd %k4, %edx ++ VMOVU %YMM6, (VEC_SIZE * 2)(%rdi) ++ lea VEC_SIZE(%rdi, %rcx), %rdi ++ lea VEC_SIZE(%rsi, %rcx), %rsi ++ bsf %edx, %edx ++ cmp %r8d, %edx ++ jb L(CopyVecSizeExit) ++L(StrncpyExit): ++ cmp $65, %r8d ++ je L(StrncpyExit65) ++ cmp $33, %r8d ++ jae L(StrncpyExit33_64) ++ cmp $17, %r8d ++ jae L(StrncpyExit17_32) ++ cmp $9, %r8d ++ jae L(StrncpyExit9_16) ++ cmp $5, %r8d ++ jae L(StrncpyExit5_8) ++ cmp $3, %r8d ++ jae L(StrncpyExit3_4) ++ cmp $1, %r8d ++ ja L(StrncpyExit2) ++ je L(StrncpyExit1) ++# ifdef USE_AS_STPCPY ++ mov %rdi, %rax ++# endif ++# ifdef USE_AS_STRCAT ++ movb $0, (%rdi) ++# endif ++ ret ++ ++ .p2align 4 ++L(ExitZero): ++# ifndef USE_AS_STRCAT ++ mov %rdi, %rax ++# endif ++ ret ++ ++# endif ++ ++# ifndef USE_AS_STRCAT ++END (STRCPY) ++# else ++END (STRCAT) ++# endif ++#endif +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S +new file mode 100644 +index 0000000000..75b4b7612c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRLEN ++# define STRLEN __strlen_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strlen-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strlen-avx2.S b/sysdeps/x86_64/multiarch/strlen-avx2.S +index 73421ec1b2..45e08e64d6 100644 +--- a/sysdeps/x86_64/multiarch/strlen-avx2.S ++++ b/sysdeps/x86_64/multiarch/strlen-avx2.S +@@ -27,370 +27,531 @@ + # ifdef USE_AS_WCSLEN + # define VPCMPEQ vpcmpeqd + # define VPMINU vpminud ++# define CHAR_SIZE 4 + # else + # define VPCMPEQ vpcmpeqb + # define VPMINU vpminub ++# define CHAR_SIZE 1 + # endif + + # ifndef VZEROUPPER + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRLEN) + # ifdef USE_AS_STRNLEN +- /* Check for zero length. */ ++ /* Check zero length. */ ++# ifdef __ILP32__ ++ /* Clear upper bits. */ ++ and %RSI_LP, %RSI_LP ++# else + test %RSI_LP, %RSI_LP +- jz L(zero) +-# ifdef USE_AS_WCSLEN +- shl $2, %RSI_LP +-# elif defined __ILP32__ +- /* Clear the upper 32 bits. */ +- movl %esi, %esi + # endif ++ jz L(zero) ++ /* Store max len in R8_LP before adjusting if using WCSLEN. */ + mov %RSI_LP, %R8_LP + # endif +- movl %edi, %ecx ++ movl %edi, %eax + movq %rdi, %rdx + vpxor %xmm0, %xmm0, %xmm0 +- ++ /* Clear high bits from edi. Only keeping bits relevant to page ++ cross check. */ ++ andl $(PAGE_SIZE - 1), %eax + /* Check if we may cross page boundary with one vector load. */ +- andl $(2 * VEC_SIZE - 1), %ecx +- cmpl $VEC_SIZE, %ecx +- ja L(cros_page_boundary) ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) + + /* Check the first VEC_SIZE bytes. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 ++ VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- + # ifdef USE_AS_STRNLEN +- jnz L(first_vec_x0_check) +- /* Adjust length and check the end of data. */ +- subq $VEC_SIZE, %rsi +- jbe L(max) +-# else +- jnz L(first_vec_x0) ++ /* If length < VEC_SIZE handle special. */ ++ cmpq $CHAR_PER_VEC, %rsi ++ jbe L(first_vec_x0) + # endif +- +- /* Align data for aligned loads in the loop. */ +- addq $VEC_SIZE, %rdi +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi ++ /* If empty continue to aligned_more. Otherwise return bit ++ position of first match. */ ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ +- addq %rcx, %rsi ++L(zero): ++ xorl %eax, %eax ++ ret + +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ .p2align 4 ++L(first_vec_x0): ++ /* Set bit for max len so that tzcnt will return min of max len ++ and position of first match. */ ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi ++# endif ++ btsq %rsi, %rax ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + # endif +- jmp L(more_4x_vec) + + .p2align 4 +-L(cros_page_boundary): +- andl $(VEC_SIZE - 1), %ecx +- andq $-VEC_SIZE, %rdi +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- /* Remove the leading bytes. */ +- sarl %cl, %eax +- testl %eax, %eax +- jz L(aligned_more) ++L(first_vec_x1): + tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 4 + 1)(%rax, %rcx, 4), %eax ++# else ++ subl $(VEC_SIZE * 4 + 1), %ecx ++ addl %ecx, %eax ++# endif ++# else ++ subl %edx, %edi ++ incl %edi ++ addl %edi, %eax + # endif +- addq %rdi, %rax +- addq %rcx, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 +-L(aligned_more): ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- /* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE" +- with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE" +- to void possible addition overflow. */ +- negq %rcx +- addq $VEC_SIZE, %rcx +- +- /* Check the end of data. */ +- subq %rcx, %rsi +- jbe L(max) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 3 + 1)(%rax, %rcx, 4), %eax ++# else ++ subl $(VEC_SIZE * 3 + 1), %ecx ++ addl %ecx, %eax ++# endif ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE + 1), %edi ++ addl %edi, %eax + # endif ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + +- addq $VEC_SIZE, %rdi ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 2 + 1)(%rax, %rcx, 4), %eax ++# else ++ subl $(VEC_SIZE * 2 + 1), %ecx ++ addl %ecx, %eax ++# endif ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE * 2 + 1), %edi ++ addl %edi, %eax ++# endif ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ + # ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++# ifdef USE_AS_WCSLEN ++ leal -(VEC_SIZE * 1 + 1)(%rax, %rcx, 4), %eax ++# else ++ subl $(VEC_SIZE + 1), %ecx ++ addl %ecx, %eax ++# endif ++# else ++ subl %edx, %edi ++ addl $(VEC_SIZE * 3 + 1), %edi ++ addl %edi, %eax + # endif ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + +-L(more_4x_vec): ++ .p2align 5 ++L(aligned_more): ++ /* Align data to VEC_SIZE - 1. This is the same number of ++ instructions as using andq with -VEC_SIZE but saves 4 bytes of ++ code on the x4 check. */ ++ orq $(VEC_SIZE - 1), %rdi ++L(cross_page_continue): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++# ifdef USE_AS_STRNLEN ++ /* + 1 because rdi is aligned to VEC_SIZE - 1. + CHAR_SIZE ++ because it simplies the logic in last_4x_vec_or_less. */ ++ leaq (VEC_SIZE * 4 + CHAR_SIZE + 1)(%rdi), %rcx ++ subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++# endif ++ /* Load first VEC regardless. */ ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. If near end handle specially. */ ++ subq %rcx, %rsi ++ jb L(last_4x_vec_or_less) ++# endif + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + +- addq $(VEC_SIZE * 4), %rdi +- +-# ifdef USE_AS_STRNLEN +- subq $(VEC_SIZE * 4), %rsi +- jbe L(last_4x_vec_or_less) +-# endif +- +- /* Align data to 4 * VEC_SIZE. */ +- movq %rdi, %rcx +- andl $(4 * VEC_SIZE - 1), %ecx +- andq $-(4 * VEC_SIZE), %rdi ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) + ++ /* Align data to VEC_SIZE * 4 - 1. */ + # ifdef USE_AS_STRNLEN +- /* Adjust length. */ ++ /* Before adjusting length check if at last VEC_SIZE * 4. */ ++ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi ++ jbe L(last_4x_vec_or_less_load) ++ incq %rdi ++ movl %edi, %ecx ++ orq $(VEC_SIZE * 4 - 1), %rdi ++ andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++ /* Readjust length. */ + addq %rcx, %rsi ++# else ++ incq %rdi ++ orq $(VEC_SIZE * 4 - 1), %rdi + # endif +- ++ /* Compare 4 * VEC at a time forward. */ + .p2align 4 + L(loop_4x_vec): +- /* Compare 4 * VEC at a time forward. */ +- vmovdqa (%rdi), %ymm1 +- vmovdqa VEC_SIZE(%rdi), %ymm2 +- vmovdqa (VEC_SIZE * 2)(%rdi), %ymm3 +- vmovdqa (VEC_SIZE * 3)(%rdi), %ymm4 +- VPMINU %ymm1, %ymm2, %ymm5 +- VPMINU %ymm3, %ymm4, %ymm6 +- VPMINU %ymm5, %ymm6, %ymm5 +- ++# ifdef USE_AS_STRNLEN ++ /* Break if at end of length. */ ++ subq $(CHAR_PER_VEC * 4), %rsi ++ jb L(last_4x_vec_or_less_cmpeq) ++# endif ++ /* Save some code size by microfusing VPMINU with the load. ++ Since the matches in ymm2/ymm4 can only be returned if there ++ where no matches in ymm1/ymm3 respectively there is no issue ++ with overlap. */ ++ vmovdqa 1(%rdi), %ymm1 ++ VPMINU (VEC_SIZE + 1)(%rdi), %ymm1, %ymm2 ++ vmovdqa (VEC_SIZE * 2 + 1)(%rdi), %ymm3 ++ VPMINU (VEC_SIZE * 3 + 1)(%rdi), %ymm3, %ymm4 ++ ++ VPMINU %ymm2, %ymm4, %ymm5 + VPCMPEQ %ymm5, %ymm0, %ymm5 +- vpmovmskb %ymm5, %eax +- testl %eax, %eax +- jnz L(4x_vec_end) ++ vpmovmskb %ymm5, %ecx + +- addq $(VEC_SIZE * 4), %rdi ++ subq $-(VEC_SIZE * 4), %rdi ++ testl %ecx, %ecx ++ jz L(loop_4x_vec) + +-# ifndef USE_AS_STRNLEN +- jmp L(loop_4x_vec) +-# else +- subq $(VEC_SIZE * 4), %rsi +- ja L(loop_4x_vec) + +-L(last_4x_vec_or_less): +- /* Less than 4 * VEC and aligned to VEC_SIZE. */ +- addl $(VEC_SIZE * 2), %esi +- jle L(last_2x_vec) +- +- VPCMPEQ (%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x0) +- +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++ VPCMPEQ %ymm1, %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++ subq %rdx, %rdi + testl %eax, %eax +- jnz L(first_vec_x1) ++ jnz L(last_vec_return_x0) + +- VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax ++ VPCMPEQ %ymm2, %ymm0, %ymm2 ++ vpmovmskb %ymm2, %eax + testl %eax, %eax ++ jnz L(last_vec_return_x1) + +- jnz L(first_vec_x2_check) +- subl $VEC_SIZE, %esi +- jle L(max) ++ /* Combine last 2 VEC. */ ++ VPCMPEQ %ymm3, %ymm0, %ymm3 ++ vpmovmskb %ymm3, %eax ++ /* rcx has combined result from all 4 VEC. It will only be used ++ if the first 3 other VEC all did not contain a match. */ ++ salq $32, %rcx ++ orq %rcx, %rax ++ tzcntq %rax, %rax ++ subq $(VEC_SIZE * 2 - 1), %rdi ++ addq %rdi, %rax ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrq $2, %rax ++# endif ++ VZEROUPPER_RETURN + +- VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 +- vpmovmskb %ymm1, %eax +- testl %eax, %eax + +- jnz L(first_vec_x3_check) +- movq %r8, %rax ++# ifdef USE_AS_STRNLEN ++ .p2align 4 ++L(last_4x_vec_or_less_load): ++ /* Depending on entry adjust rdi / prepare first VEC in ymm1. ++ */ ++ subq $-(VEC_SIZE * 4), %rdi ++L(last_4x_vec_or_less_cmpeq): ++ VPCMPEQ 1(%rdi), %ymm0, %ymm1 ++L(last_4x_vec_or_less): + # ifdef USE_AS_WCSLEN +- shrq $2, %rax ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi + # endif +- VZEROUPPER +- ret +- +- .p2align 4 +-L(last_2x_vec): +- addl $(VEC_SIZE * 2), %esi +- VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++ /* If remaining length > VEC_SIZE * 2. This works if esi is off ++ by VEC_SIZE * 4. */ ++ testl $(VEC_SIZE * 2), %esi ++ jnz L(last_4x_vec) ++ ++ /* length may have been negative or positive by an offset of ++ VEC_SIZE * 4 depending on where this was called from. This fixes ++ that. */ ++ andl $(VEC_SIZE * 4 - 1), %esi + testl %eax, %eax ++ jnz L(last_vec_x1_check) + +- jnz L(first_vec_x0_check) + subl $VEC_SIZE, %esi +- jle L(max) ++ jb L(max) + +- VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax +- testl %eax, %eax +- jnz L(first_vec_x1_check) +- movq %r8, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax +-# endif +- VZEROUPPER +- ret +- +- .p2align 4 +-L(first_vec_x0_check): + tzcntl %eax, %eax + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN ++# endif + + .p2align 4 +-L(first_vec_x1_check): ++L(last_vec_return_x0): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $VEC_SIZE, %rax ++ subq $(VEC_SIZE * 4 - 1), %rdi + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax +-# endif +- VZEROUPPER +- ret ++# endif ++ VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2_check): ++L(last_vec_return_x1): + tzcntl %eax, %eax +- /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 2), %rax ++ subq $(VEC_SIZE * 3 - 1), %rdi + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax +-# endif +- VZEROUPPER +- ret ++# endif ++ VZEROUPPER_RETURN + ++# ifdef USE_AS_STRNLEN + .p2align 4 +-L(first_vec_x3_check): ++L(last_vec_x1_check): ++ + tzcntl %eax, %eax + /* Check the end of data. */ +- cmpq %rax, %rsi +- jbe L(max) +- addq $(VEC_SIZE * 3), %rax ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ incl %eax + addq %rdi, %rax +- subq %rdx, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + +- .p2align 4 + L(max): + movq %r8, %rax ++ VZEROUPPER_RETURN ++ ++ .p2align 4 ++L(last_4x_vec): ++ /* Test first 2x VEC normally. */ ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ VPCMPEQ (VEC_SIZE + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ /* Normalize length. */ ++ andl $(VEC_SIZE * 4 - 1), %esi ++ VPCMPEQ (VEC_SIZE * 2 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ subl $(VEC_SIZE * 3), %esi ++ jb L(max) ++ ++ VPCMPEQ (VEC_SIZE * 3 + 1)(%rdi), %ymm0, %ymm1 ++ vpmovmskb %ymm1, %eax ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE * 3 + 1), %eax ++ addq %rdi, %rax + # ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax + # endif +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + +- .p2align 4 +-L(zero): +- xorl %eax, %eax +- ret +-# endif + + .p2align 4 +-L(first_vec_x0): ++L(last_vec_x1): ++ /* essentially duplicates of first_vec_x1 but use 64 bit ++ instructions. */ + tzcntl %eax, %eax ++ subq %rdx, %rdi ++ incl %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax +-# endif +- VZEROUPPER +- ret ++# endif ++ VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x1): ++L(last_vec_x2): ++ /* essentially duplicates of first_vec_x1 but use 64 bit ++ instructions. */ + tzcntl %eax, %eax +- addq $VEC_SIZE, %rax ++ subq %rdx, %rdi ++ addl $(VEC_SIZE + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax +-# endif +- VZEROUPPER +- ret ++# endif ++ VZEROUPPER_RETURN + + .p2align 4 +-L(first_vec_x2): ++L(last_vec_x3): + tzcntl %eax, %eax +- addq $(VEC_SIZE * 2), %rax ++ subl $(VEC_SIZE * 2), %esi ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max_end) ++ subq %rdx, %rdi ++ addl $(VEC_SIZE * 2 + 1), %eax + addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ + shrq $2, %rax ++# endif ++ VZEROUPPER_RETURN ++L(max_end): ++ movq %r8, %rax ++ VZEROUPPER_RETURN + # endif +- VZEROUPPER +- ret + ++ /* Cold case for crossing page with first load. */ + .p2align 4 +-L(4x_vec_end): +- VPCMPEQ %ymm1, %ymm0, %ymm1 ++L(cross_page_boundary): ++ /* Align data to VEC_SIZE - 1. */ ++ orq $(VEC_SIZE - 1), %rdi ++ VPCMPEQ -(VEC_SIZE - 1)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax ++ /* Remove the leading bytes. sarxl only uses bits [5:0] of COUNT ++ so no need to manually mod rdx. */ ++ sarxl %edx, %eax, %eax ++# ifdef USE_AS_STRNLEN + testl %eax, %eax +- jnz L(first_vec_x0) +- VPCMPEQ %ymm2, %ymm0, %ymm2 +- vpmovmskb %ymm2, %eax +- testl %eax, %eax +- jnz L(first_vec_x1) +- VPCMPEQ %ymm3, %ymm0, %ymm3 +- vpmovmskb %ymm3, %eax ++ jnz L(cross_page_less_vec) ++ leaq 1(%rdi), %rcx ++ subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get wchar_t count. */ ++ shrl $2, %ecx ++# endif ++ /* Check length. */ ++ cmpq %rsi, %rcx ++ jb L(cross_page_continue) ++ movq %r8, %rax ++# else + testl %eax, %eax +- jnz L(first_vec_x2) +- VPCMPEQ %ymm4, %ymm0, %ymm4 +- vpmovmskb %ymm4, %eax +-L(first_vec_x3): ++ jz L(cross_page_continue) + tzcntl %eax, %eax +- addq $(VEC_SIZE * 3), %rax +- addq %rdi, %rax +- subq %rdx, %rax +-# ifdef USE_AS_WCSLEN +- shrq $2, %rax ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide length by 4 to get wchar_t count. */ ++ shrl $2, %eax ++# endif ++# endif ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN ++ ++# ifdef USE_AS_STRNLEN ++ .p2align 4 ++L(cross_page_less_vec): ++ tzcntl %eax, %eax ++# ifdef USE_AS_WCSLEN ++ /* NB: Multiply length by 4 to get byte count. */ ++ sall $2, %esi ++# endif ++ cmpq %rax, %rsi ++ cmovb %esi, %eax ++# ifdef USE_AS_WCSLEN ++ shrl $2, %eax ++# endif ++ VZEROUPPER_RETURN + # endif +- VZEROUPPER +- ret + + END (STRLEN) + #endif +diff --git a/sysdeps/x86_64/multiarch/strlen-evex.S b/sysdeps/x86_64/multiarch/strlen-evex.S +new file mode 100644 +index 0000000000..4bf6874b82 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-evex.S +@@ -0,0 +1,489 @@ ++/* strlen/strnlen/wcslen/wcsnlen optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef STRLEN ++# define STRLEN __strlen_evex ++# endif ++ ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSLEN ++# define VPCMP vpcmpd ++# define VPMINU vpminud ++# define SHIFT_REG ecx ++# define CHAR_SIZE 4 ++# else ++# define VPCMP vpcmpb ++# define VPMINU vpminub ++# define SHIFT_REG edx ++# define CHAR_SIZE 1 ++# endif ++ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMM1 ymm17 ++# define YMM2 ymm18 ++# define YMM3 ymm19 ++# define YMM4 ymm20 ++# define YMM5 ymm21 ++# define YMM6 ymm22 ++ ++# define VEC_SIZE 32 ++# define PAGE_SIZE 4096 ++# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRLEN) ++# ifdef USE_AS_STRNLEN ++ /* Check zero length. */ ++ test %RSI_LP, %RSI_LP ++ jz L(zero) ++# ifdef __ILP32__ ++ /* Clear the upper 32 bits. */ ++ movl %esi, %esi ++# endif ++ mov %RSI_LP, %R8_LP ++# endif ++ movl %edi, %eax ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ /* Clear high bits from edi. Only keeping bits relevant to page ++ cross check. */ ++ andl $(PAGE_SIZE - 1), %eax ++ /* Check if we may cross page boundary with one vector load. */ ++ cmpl $(PAGE_SIZE - VEC_SIZE), %eax ++ ja L(cross_page_boundary) ++ ++ /* Check the first VEC_SIZE bytes. Each bit in K0 represents a ++ null byte. */ ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++# ifdef USE_AS_STRNLEN ++ /* If length < CHAR_PER_VEC handle special. */ ++ cmpq $CHAR_PER_VEC, %rsi ++ jbe L(first_vec_x0) ++# endif ++ testl %eax, %eax ++ jz L(aligned_more) ++ tzcntl %eax, %eax ++ ret ++# ifdef USE_AS_STRNLEN ++L(zero): ++ xorl %eax, %eax ++ ret ++ ++ .p2align 4 ++L(first_vec_x0): ++ /* Set bit for max len so that tzcnt will return min of max len ++ and position of first match. */ ++ btsq %rsi, %rax ++ tzcntl %eax, %eax ++ ret ++# endif ++ ++ .p2align 4 ++L(first_vec_x1): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 4 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal CHAR_PER_VEC(%rdi, %rax), %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x2): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 3 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 2)(%rdi, %rax), %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x3): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC * 2 + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 3)(%rdi, %rax), %eax ++# endif ++ ret ++ ++ .p2align 4 ++L(first_vec_x4): ++ tzcntl %eax, %eax ++ /* Safe to use 32 bit instructions as these are only called for ++ size = [1, 159]. */ ++# ifdef USE_AS_STRNLEN ++ /* Use ecx which was computed earlier to compute correct value. ++ */ ++ leal -(CHAR_PER_VEC + 1)(%rcx, %rax), %eax ++# else ++ subl %edx, %edi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %edi ++# endif ++ leal (CHAR_PER_VEC * 4)(%rdi, %rax), %eax ++# endif ++ ret ++ ++ .p2align 5 ++L(aligned_more): ++ movq %rdi, %rdx ++ /* Align data to VEC_SIZE. */ ++ andq $-(VEC_SIZE), %rdi ++L(cross_page_continue): ++ /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time ++ since data is only aligned to VEC_SIZE. */ ++# ifdef USE_AS_STRNLEN ++ /* + CHAR_SIZE because it simplies the logic in ++ last_4x_vec_or_less. */ ++ leaq (VEC_SIZE * 5 + CHAR_SIZE)(%rdi), %rcx ++ subq %rdx, %rcx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++# endif ++ /* Load first VEC regardless. */ ++ VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 ++# ifdef USE_AS_STRNLEN ++ /* Adjust length. If near end handle specially. */ ++ subq %rcx, %rsi ++ jb L(last_4x_vec_or_less) ++# endif ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ test %eax, %eax ++ jnz L(first_vec_x2) ++ ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x3) ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(first_vec_x4) ++ ++ addq $VEC_SIZE, %rdi ++# ifdef USE_AS_STRNLEN ++ /* Check if at last VEC_SIZE * 4 length. */ ++ cmpq $(CHAR_PER_VEC * 4 - 1), %rsi ++ jbe L(last_4x_vec_or_less_load) ++ movl %edi, %ecx ++ andl $(VEC_SIZE * 4 - 1), %ecx ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarl $2, %ecx ++# endif ++ /* Readjust length. */ ++ addq %rcx, %rsi ++# endif ++ /* Align data to VEC_SIZE * 4. */ ++ andq $-(VEC_SIZE * 4), %rdi ++ ++ /* Compare 4 * VEC at a time forward. */ ++ .p2align 4 ++L(loop_4x_vec): ++ /* Load first VEC regardless. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++# ifdef USE_AS_STRNLEN ++ /* Break if at end of length. */ ++ subq $(CHAR_PER_VEC * 4), %rsi ++ jb L(last_4x_vec_or_less_cmpeq) ++# endif ++ /* Save some code size by microfusing VPMINU with the load. Since ++ the matches in ymm2/ymm4 can only be returned if there where no ++ matches in ymm1/ymm3 respectively there is no issue with overlap. ++ */ ++ VPMINU (VEC_SIZE * 5)(%rdi), %YMM1, %YMM2 ++ VMOVA (VEC_SIZE * 6)(%rdi), %YMM3 ++ VPMINU (VEC_SIZE * 7)(%rdi), %YMM3, %YMM4 ++ ++ VPCMP $0, %YMM2, %YMMZERO, %k0 ++ VPCMP $0, %YMM4, %YMMZERO, %k1 ++ subq $-(VEC_SIZE * 4), %rdi ++ kortestd %k0, %k1 ++ jz L(loop_4x_vec) ++ ++ /* Check if end was in first half. */ ++ kmovd %k0, %eax ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ shrq $2, %rdi ++# endif ++ testl %eax, %eax ++ jz L(second_vec_return) ++ ++ VPCMP $0, %YMM1, %YMMZERO, %k2 ++ kmovd %k2, %edx ++ /* Combine VEC1 matches (edx) with VEC2 matches (eax). */ ++# ifdef USE_AS_WCSLEN ++ sall $CHAR_PER_VEC, %eax ++ orl %edx, %eax ++ tzcntl %eax, %eax ++# else ++ salq $CHAR_PER_VEC, %rax ++ orq %rdx, %rax ++ tzcntq %rax, %rax ++# endif ++ addq %rdi, %rax ++ ret ++ ++ ++# ifdef USE_AS_STRNLEN ++ ++L(last_4x_vec_or_less_load): ++ /* Depending on entry adjust rdi / prepare first VEC in YMM1. */ ++ VMOVA (VEC_SIZE * 4)(%rdi), %YMM1 ++L(last_4x_vec_or_less_cmpeq): ++ VPCMP $0, %YMM1, %YMMZERO, %k0 ++ addq $(VEC_SIZE * 3), %rdi ++L(last_4x_vec_or_less): ++ kmovd %k0, %eax ++ /* If remaining length > VEC_SIZE * 2. This works if esi is off by ++ VEC_SIZE * 4. */ ++ testl $(CHAR_PER_VEC * 2), %esi ++ jnz L(last_4x_vec) ++ ++ /* length may have been negative or positive by an offset of ++ CHAR_PER_VEC * 4 depending on where this was called from. This ++ fixes that. */ ++ andl $(CHAR_PER_VEC * 4 - 1), %esi ++ testl %eax, %eax ++ jnz L(last_vec_x1_check) ++ ++ /* Check the end of data. */ ++ subl $CHAR_PER_VEC, %esi ++ jb L(max) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) ++ ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax ++ ret ++L(max): ++ movq %r8, %rax ++ ret ++# endif ++ ++ /* Placed here in strnlen so that the jcc L(last_4x_vec_or_less) ++ in the 4x VEC loop can use 2 byte encoding. */ ++ .p2align 4 ++L(second_vec_return): ++ VPCMP $0, %YMM3, %YMMZERO, %k0 ++ /* Combine YMM3 matches (k0) with YMM4 matches (k1). */ ++# ifdef USE_AS_WCSLEN ++ kunpckbw %k0, %k1, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++# else ++ kunpckdq %k0, %k1, %k0 ++ kmovq %k0, %rax ++ tzcntq %rax, %rax ++# endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax ++ ret ++ ++ ++# ifdef USE_AS_STRNLEN ++L(last_vec_x1_check): ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max) ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax ++ ret ++ ++ .p2align 4 ++L(last_4x_vec): ++ /* Test first 2x VEC normally. */ ++ testl %eax, %eax ++ jnz L(last_vec_x1) ++ ++ VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x2) ++ ++ /* Normalize length. */ ++ andl $(CHAR_PER_VEC * 4 - 1), %esi ++ VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ testl %eax, %eax ++ jnz L(last_vec_x3) ++ ++ /* Check the end of data. */ ++ subl $(CHAR_PER_VEC * 3), %esi ++ jb L(max) ++ ++ VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ tzcntl %eax, %eax ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max_end) ++ ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC * 4)(%rdi, %rax), %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x1): ++ tzcntl %eax, %eax ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC)(%rdi, %rax), %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x2): ++ tzcntl %eax, %eax ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax ++ ret ++ ++ .p2align 4 ++L(last_vec_x3): ++ tzcntl %eax, %eax ++ subl $(CHAR_PER_VEC * 2), %esi ++ /* Check the end of data. */ ++ cmpl %eax, %esi ++ jb L(max_end) ++ subq %rdx, %rdi ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide bytes by 4 to get the wchar_t count. */ ++ sarq $2, %rdi ++# endif ++ leaq (CHAR_PER_VEC * 3)(%rdi, %rax), %rax ++ ret ++L(max_end): ++ movq %r8, %rax ++ ret ++# endif ++ ++ /* Cold case for crossing page with first load. */ ++ .p2align 4 ++L(cross_page_boundary): ++ movq %rdi, %rdx ++ /* Align data to VEC_SIZE. */ ++ andq $-VEC_SIZE, %rdi ++ VPCMP $0, (%rdi), %YMMZERO, %k0 ++ kmovd %k0, %eax ++ /* Remove the leading bytes. */ ++# ifdef USE_AS_WCSLEN ++ /* NB: Divide shift count by 4 since each bit in K0 represent 4 ++ bytes. */ ++ movl %edx, %ecx ++ shrl $2, %ecx ++ andl $(CHAR_PER_VEC - 1), %ecx ++# endif ++ /* SHIFT_REG is ecx for USE_AS_WCSLEN and edx otherwise. */ ++ sarxl %SHIFT_REG, %eax, %eax ++ testl %eax, %eax ++# ifndef USE_AS_STRNLEN ++ jz L(cross_page_continue) ++ tzcntl %eax, %eax ++ ret ++# else ++ jnz L(cross_page_less_vec) ++# ifndef USE_AS_WCSLEN ++ movl %edx, %ecx ++ andl $(CHAR_PER_VEC - 1), %ecx ++# endif ++ movl $CHAR_PER_VEC, %eax ++ subl %ecx, %eax ++ /* Check the end of data. */ ++ cmpq %rax, %rsi ++ ja L(cross_page_continue) ++ movl %esi, %eax ++ ret ++L(cross_page_less_vec): ++ tzcntl %eax, %eax ++ /* Select min of length and position of first null. */ ++ cmpq %rax, %rsi ++ cmovb %esi, %eax ++ ret ++# endif ++ ++END (STRLEN) ++#endif +diff --git a/sysdeps/x86_64/multiarch/strlen-sse2.S b/sysdeps/x86_64/multiarch/strlen-sse2.S +index 055fbbc690..812af73c13 100644 +--- a/sysdeps/x86_64/multiarch/strlen-sse2.S ++++ b/sysdeps/x86_64/multiarch/strlen-sse2.S +@@ -20,4 +20,4 @@ + # define strlen __strlen_sse2 + #endif + +-#include "../strlen.S" ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/multiarch/strlen-vec.S b/sysdeps/x86_64/multiarch/strlen-vec.S +new file mode 100644 +index 0000000000..439e486a43 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strlen-vec.S +@@ -0,0 +1,270 @@ ++/* SSE2 version of strlen and SSE4.1 version of wcslen. ++ Copyright (C) 2012-2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sysdep.h> ++ ++#ifdef AS_WCSLEN ++# define PMINU pminud ++# define PCMPEQ pcmpeqd ++# define SHIFT_RETURN shrq $2, %rax ++#else ++# define PMINU pminub ++# define PCMPEQ pcmpeqb ++# define SHIFT_RETURN ++#endif ++ ++/* Long lived register in strlen(s), strnlen(s, n) are: ++ ++ %xmm3 - zero ++ %rdi - s ++ %r10 (s+n) & (~(64-1)) ++ %r11 s+n ++*/ ++ ++ ++.text ++ENTRY(strlen) ++ ++/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */ ++#define FIND_ZERO \ ++ PCMPEQ (%rax), %xmm0; \ ++ PCMPEQ 16(%rax), %xmm1; \ ++ PCMPEQ 32(%rax), %xmm2; \ ++ PCMPEQ 48(%rax), %xmm3; \ ++ pmovmskb %xmm0, %esi; \ ++ pmovmskb %xmm1, %edx; \ ++ pmovmskb %xmm2, %r8d; \ ++ pmovmskb %xmm3, %ecx; \ ++ salq $16, %rdx; \ ++ salq $16, %rcx; \ ++ orq %rsi, %rdx; \ ++ orq %r8, %rcx; \ ++ salq $32, %rcx; \ ++ orq %rcx, %rdx; ++ ++#ifdef AS_STRNLEN ++/* Do not read anything when n==0. */ ++ test %RSI_LP, %RSI_LP ++ jne L(n_nonzero) ++ xor %rax, %rax ++ ret ++L(n_nonzero): ++# ifdef AS_WCSLEN ++/* Check for overflow from maxlen * sizeof(wchar_t). If it would ++ overflow the only way this program doesn't have undefined behavior ++ is if there is a null terminator in valid memory so wcslen will ++ suffice. */ ++ mov %RSI_LP, %R10_LP ++ sar $62, %R10_LP ++ test %R10_LP, %R10_LP ++ jnz __wcslen_sse4_1 ++ sal $2, %RSI_LP ++# endif ++ ++ ++/* Initialize long lived registers. */ ++ ++ add %RDI_LP, %RSI_LP ++# ifdef AS_WCSLEN ++/* Check for overflow again from s + maxlen * sizeof(wchar_t). */ ++ jbe __wcslen_sse4_1 ++# endif ++ mov %RSI_LP, %R10_LP ++ and $-64, %R10_LP ++ mov %RSI_LP, %R11_LP ++#endif ++ ++ pxor %xmm0, %xmm0 ++ pxor %xmm1, %xmm1 ++ pxor %xmm2, %xmm2 ++ pxor %xmm3, %xmm3 ++ movq %rdi, %rax ++ movq %rdi, %rcx ++ andq $4095, %rcx ++/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */ ++ cmpq $4047, %rcx ++/* We cannot unify this branching as it would be ~6 cycles slower. */ ++ ja L(cross_page) ++ ++#ifdef AS_STRNLEN ++/* Test if end is among first 64 bytes. */ ++# define STRNLEN_PROLOG \ ++ mov %r11, %rsi; \ ++ subq %rax, %rsi; \ ++ andq $-64, %rax; \ ++ testq $-64, %rsi; \ ++ je L(strnlen_ret) ++#else ++# define STRNLEN_PROLOG andq $-64, %rax; ++#endif ++ ++/* Ignore bits in mask that come before start of string. */ ++#define PROLOG(lab) \ ++ movq %rdi, %rcx; \ ++ xorq %rax, %rcx; \ ++ STRNLEN_PROLOG; \ ++ sarq %cl, %rdx; \ ++ test %rdx, %rdx; \ ++ je L(lab); \ ++ bsfq %rdx, %rax; \ ++ SHIFT_RETURN; \ ++ ret ++ ++#ifdef AS_STRNLEN ++ andq $-16, %rax ++ FIND_ZERO ++#else ++ /* Test first 16 bytes unaligned. */ ++ movdqu (%rax), %xmm4 ++ PCMPEQ %xmm0, %xmm4 ++ pmovmskb %xmm4, %edx ++ test %edx, %edx ++ je L(next48_bytes) ++ bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */ ++ SHIFT_RETURN ++ ret ++ ++L(next48_bytes): ++/* Same as FIND_ZERO except we do not check first 16 bytes. */ ++ andq $-16, %rax ++ PCMPEQ 16(%rax), %xmm1 ++ PCMPEQ 32(%rax), %xmm2 ++ PCMPEQ 48(%rax), %xmm3 ++ pmovmskb %xmm1, %edx ++ pmovmskb %xmm2, %r8d ++ pmovmskb %xmm3, %ecx ++ salq $16, %rdx ++ salq $16, %rcx ++ orq %r8, %rcx ++ salq $32, %rcx ++ orq %rcx, %rdx ++#endif ++ ++ /* When no zero byte is found xmm1-3 are zero so we do not have to ++ zero them. */ ++ PROLOG(loop) ++ ++ .p2align 4 ++L(cross_page): ++ andq $-64, %rax ++ FIND_ZERO ++ PROLOG(loop_init) ++ ++#ifdef AS_STRNLEN ++/* We must do this check to correctly handle strnlen (s, -1). */ ++L(strnlen_ret): ++ bts %rsi, %rdx ++ sarq %cl, %rdx ++ test %rdx, %rdx ++ je L(loop_init) ++ bsfq %rdx, %rax ++ SHIFT_RETURN ++ ret ++#endif ++ .p2align 4 ++L(loop_init): ++ pxor %xmm1, %xmm1 ++ pxor %xmm2, %xmm2 ++ pxor %xmm3, %xmm3 ++#ifdef AS_STRNLEN ++ .p2align 4 ++L(loop): ++ ++ addq $64, %rax ++ cmpq %rax, %r10 ++ je L(exit_end) ++ ++ movdqa (%rax), %xmm0 ++ PMINU 16(%rax), %xmm0 ++ PMINU 32(%rax), %xmm0 ++ PMINU 48(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit) ++ jmp L(loop) ++ ++ .p2align 4 ++L(exit_end): ++ cmp %rax, %r11 ++ je L(first) /* Do not read when end is at page boundary. */ ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++L(first): ++ bts %r11, %rdx ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++ .p2align 4 ++L(exit): ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++#else ++ ++ /* Main loop. Unrolled twice to improve L2 cache performance on core2. */ ++ .p2align 4 ++L(loop): ++ ++ movdqa 64(%rax), %xmm0 ++ PMINU 80(%rax), %xmm0 ++ PMINU 96(%rax), %xmm0 ++ PMINU 112(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit64) ++ ++ subq $-128, %rax ++ ++ movdqa (%rax), %xmm0 ++ PMINU 16(%rax), %xmm0 ++ PMINU 32(%rax), %xmm0 ++ PMINU 48(%rax), %xmm0 ++ PCMPEQ %xmm3, %xmm0 ++ pmovmskb %xmm0, %edx ++ testl %edx, %edx ++ jne L(exit0) ++ jmp L(loop) ++ ++ .p2align 4 ++L(exit64): ++ addq $64, %rax ++L(exit0): ++ pxor %xmm0, %xmm0 ++ FIND_ZERO ++ ++ bsfq %rdx, %rdx ++ addq %rdx, %rax ++ subq %rdi, %rax ++ SHIFT_RETURN ++ ret ++ ++#endif ++ ++END(strlen) +diff --git a/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S +new file mode 100644 +index 0000000000..0dcea18dbb +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncat-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCAT ++#define STRCAT __strncat_avx2_rtm ++#include "strcat-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncat-evex.S b/sysdeps/x86_64/multiarch/strncat-evex.S +new file mode 100644 +index 0000000000..8884f02371 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncat-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCAT ++#define STRCAT __strncat_evex ++#include "strcat-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +new file mode 100644 +index 0000000000..37d1224bb9 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncmp-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCMP __strncmp_avx2_rtm ++#define USE_AS_STRNCMP 1 ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp-evex.S b/sysdeps/x86_64/multiarch/strncmp-evex.S +new file mode 100644 +index 0000000000..a1d53e8c9f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncmp-evex.S +@@ -0,0 +1,3 @@ ++#define STRCMP __strncmp_evex ++#define USE_AS_STRNCMP 1 ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strncmp.c b/sysdeps/x86_64/multiarch/strncmp.c +index 3c94b3ffd9..7accba2b7c 100644 +--- a/sysdeps/x86_64/multiarch/strncmp.c ++++ b/sysdeps/x86_64/multiarch/strncmp.c +@@ -30,16 +30,29 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; ++extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + + static inline void * + IFUNC_SELECTOR (void) + { + const struct cpu_features* cpu_features = __get_cpu_features (); + +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); ++ { ++ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) ++ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW) ++ && CPU_FEATURE_USABLE_P (cpu_features, BMI2) ++ && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_AVX2_STRCMP)) ++ return OPTIMIZE (evex); ++ ++ if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) ++ return OPTIMIZE (avx2_rtm); ++ ++ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) ++ return OPTIMIZE (avx2); ++ } + + if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_2) + && !CPU_FEATURES_ARCH_P (cpu_features, Slow_SSE4_2)) +diff --git a/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S +new file mode 100644 +index 0000000000..79e7083299 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncpy-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCPY ++#define STRCPY __strncpy_avx2_rtm ++#include "strcpy-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strncpy-evex.S b/sysdeps/x86_64/multiarch/strncpy-evex.S +new file mode 100644 +index 0000000000..40e391f0da +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strncpy-evex.S +@@ -0,0 +1,3 @@ ++#define USE_AS_STRNCPY ++#define STRCPY __strncpy_evex ++#include "strcpy-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S +new file mode 100644 +index 0000000000..04f1626a5c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strnlen-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRLEN __strnlen_avx2_rtm ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/strnlen-evex.S b/sysdeps/x86_64/multiarch/strnlen-evex.S +new file mode 100644 +index 0000000000..722022f303 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strnlen-evex.S +@@ -0,0 +1,4 @@ ++#define STRLEN __strnlen_evex ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S +new file mode 100644 +index 0000000000..5def14ec1c +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strrchr-avx2-rtm.S +@@ -0,0 +1,12 @@ ++#ifndef STRRCHR ++# define STRRCHR __strrchr_avx2_rtm ++#endif ++ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST ++ ++#define VZEROUPPER_RETURN jmp L(return_vzeroupper) ++ ++#define SECTION(p) p##.avx.rtm ++ ++#include "strrchr-avx2.S" +diff --git a/sysdeps/x86_64/multiarch/strrchr-avx2.S b/sysdeps/x86_64/multiarch/strrchr-avx2.S +index 146bdd51d0..ad91fab991 100644 +--- a/sysdeps/x86_64/multiarch/strrchr-avx2.S ++++ b/sysdeps/x86_64/multiarch/strrchr-avx2.S +@@ -36,9 +36,13 @@ + # define VZEROUPPER vzeroupper + # endif + ++# ifndef SECTION ++# define SECTION(p) p##.avx ++# endif ++ + # define VEC_SIZE 32 + +- .section .text.avx,"ax",@progbits ++ .section SECTION(.text),"ax",@progbits + ENTRY (STRRCHR) + movd %esi, %xmm4 + movl %edi, %ecx +@@ -166,8 +170,8 @@ L(return_value): + # endif + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++L(return_vzeroupper): ++ ZERO_UPPER_VEC_REGISTERS_RETURN + + .p2align 4 + L(match): +@@ -198,8 +202,7 @@ L(find_nul): + jz L(return_value) + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(char_and_nul): +@@ -222,14 +225,12 @@ L(char_and_nul_in_first_vec): + jz L(return_null) + bsrl %eax, %eax + leaq -VEC_SIZE(%rdi, %rax), %rax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + .p2align 4 + L(return_null): + xorl %eax, %eax +- VZEROUPPER +- ret ++ VZEROUPPER_RETURN + + END (STRRCHR) + #endif +diff --git a/sysdeps/x86_64/multiarch/strrchr-evex.S b/sysdeps/x86_64/multiarch/strrchr-evex.S +new file mode 100644 +index 0000000000..f920b5a584 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/strrchr-evex.S +@@ -0,0 +1,265 @@ ++/* strrchr/wcsrchr optimized with 256-bit EVEX instructions. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#if IS_IN (libc) ++ ++# include <sysdep.h> ++ ++# ifndef STRRCHR ++# define STRRCHR __strrchr_evex ++# endif ++ ++# define VMOVU vmovdqu64 ++# define VMOVA vmovdqa64 ++ ++# ifdef USE_AS_WCSRCHR ++# define VPBROADCAST vpbroadcastd ++# define VPCMP vpcmpd ++# define SHIFT_REG r8d ++# else ++# define VPBROADCAST vpbroadcastb ++# define VPCMP vpcmpb ++# define SHIFT_REG ecx ++# endif ++ ++# define XMMZERO xmm16 ++# define YMMZERO ymm16 ++# define YMMMATCH ymm17 ++# define YMM1 ymm18 ++ ++# define VEC_SIZE 32 ++ ++ .section .text.evex,"ax",@progbits ++ENTRY (STRRCHR) ++ movl %edi, %ecx ++ /* Broadcast CHAR to YMMMATCH. */ ++ VPBROADCAST %esi, %YMMMATCH ++ ++ vpxorq %XMMZERO, %XMMZERO, %XMMZERO ++ ++ /* Check if we may cross page boundary with one vector load. */ ++ andl $(2 * VEC_SIZE - 1), %ecx ++ cmpl $VEC_SIZE, %ecx ++ ja L(cros_page_boundary) ++ ++ VMOVU (%rdi), %YMM1 ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ ++ addq $VEC_SIZE, %rdi ++ ++ testl %eax, %eax ++ jnz L(first_vec) ++ ++ testl %ecx, %ecx ++ jnz L(return_null) ++ ++ andq $-VEC_SIZE, %rdi ++ xorl %edx, %edx ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(first_vec): ++ /* Check if there is a null byte. */ ++ testl %ecx, %ecx ++ jnz L(char_and_nul_in_first_vec) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ movq %rdi, %rsi ++ andq $-VEC_SIZE, %rdi ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(cros_page_boundary): ++ andl $(VEC_SIZE - 1), %ecx ++ andq $-VEC_SIZE, %rdi ++ ++# ifdef USE_AS_WCSRCHR ++ /* NB: Divide shift count by 4 since each bit in K1 represent 4 ++ bytes. */ ++ movl %ecx, %SHIFT_REG ++ sarl $2, %SHIFT_REG ++# endif ++ ++ VMOVA (%rdi), %YMM1 ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %edx ++ kmovd %k1, %eax ++ ++ shrxl %SHIFT_REG, %edx, %edx ++ shrxl %SHIFT_REG, %eax, %eax ++ addq $VEC_SIZE, %rdi ++ ++ /* Check if there is a CHAR. */ ++ testl %eax, %eax ++ jnz L(found_char) ++ ++ testl %edx, %edx ++ jnz L(return_null) ++ ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(found_char): ++ testl %edx, %edx ++ jnz L(char_and_nul) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ leaq (%rdi, %rcx), %rsi ++ ++ .p2align 4 ++L(aligned_loop): ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ add $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jnz L(char_nor_null) ++ ++ VMOVA (%rdi), %YMM1 ++ addq $VEC_SIZE, %rdi ++ ++ /* Each bit in K0 represents a null byte in YMM1. */ ++ VPCMP $0, %YMMZERO, %YMM1, %k0 ++ /* Each bit in K1 represents a CHAR in YMM1. */ ++ VPCMP $0, %YMMMATCH, %YMM1, %k1 ++ kmovd %k0, %ecx ++ kmovd %k1, %eax ++ orl %eax, %ecx ++ jz L(aligned_loop) ++ ++ .p2align 4 ++L(char_nor_null): ++ /* Find a CHAR or a null byte in a loop. */ ++ testl %eax, %eax ++ jnz L(match) ++L(return_value): ++ testl %edx, %edx ++ jz L(return_null) ++ movl %edx, %eax ++ movq %rsi, %rdi ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(match): ++ /* Find a CHAR. Check if there is a null byte. */ ++ kmovd %k0, %ecx ++ testl %ecx, %ecx ++ jnz L(find_nul) ++ ++ /* Remember the match and keep searching. */ ++ movl %eax, %edx ++ movq %rdi, %rsi ++ jmp L(aligned_loop) ++ ++ .p2align 4 ++L(find_nul): ++ /* Mask out any matching bits after the null byte. */ ++ movl %ecx, %r8d ++ subl $1, %r8d ++ xorl %ecx, %r8d ++ andl %r8d, %eax ++ testl %eax, %eax ++ /* If there is no CHAR here, return the remembered one. */ ++ jz L(return_value) ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(char_and_nul): ++ /* Find both a CHAR and a null byte. */ ++ addq %rcx, %rdi ++ movl %edx, %ecx ++L(char_and_nul_in_first_vec): ++ /* Mask out any matching bits after the null byte. */ ++ movl %ecx, %r8d ++ subl $1, %r8d ++ xorl %ecx, %r8d ++ andl %r8d, %eax ++ testl %eax, %eax ++ /* Return null pointer if the null byte comes first. */ ++ jz L(return_null) ++ bsrl %eax, %eax ++# ifdef USE_AS_WCSRCHR ++ /* NB: Multiply wchar_t count by 4 to get the number of bytes. */ ++ leaq -VEC_SIZE(%rdi, %rax, 4), %rax ++# else ++ leaq -VEC_SIZE(%rdi, %rax), %rax ++# endif ++ ret ++ ++ .p2align 4 ++L(return_null): ++ xorl %eax, %eax ++ ret ++ ++END (STRRCHR) ++#endif +diff --git a/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S +new file mode 100644 +index 0000000000..d49dbbf0b4 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcschr-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRCHR __wcschr_avx2_rtm ++#define USE_AS_WCSCHR 1 ++#include "strchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcschr-evex.S b/sysdeps/x86_64/multiarch/wcschr-evex.S +new file mode 100644 +index 0000000000..7cb8f1e41a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcschr-evex.S +@@ -0,0 +1,3 @@ ++#define STRCHR __wcschr_evex ++#define USE_AS_WCSCHR 1 ++#include "strchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S +new file mode 100644 +index 0000000000..d6ca2b8064 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcscmp-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRCMP __wcscmp_avx2_rtm ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcscmp-evex.S b/sysdeps/x86_64/multiarch/wcscmp-evex.S +new file mode 100644 +index 0000000000..42e73e51eb +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcscmp-evex.S +@@ -0,0 +1,4 @@ ++#define STRCMP __wcscmp_evex ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S +new file mode 100644 +index 0000000000..35658d7365 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define STRLEN __wcslen_avx2_rtm ++#define USE_AS_WCSLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen-evex.S b/sysdeps/x86_64/multiarch/wcslen-evex.S +new file mode 100644 +index 0000000000..bdafa83bd5 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-evex.S +@@ -0,0 +1,4 @@ ++#define STRLEN __wcslen_evex ++#define USE_AS_WCSLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen-sse4_1.S b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S +new file mode 100644 +index 0000000000..7e62621afc +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcslen-sse4_1.S +@@ -0,0 +1,4 @@ ++#define AS_WCSLEN ++#define strlen __wcslen_sse4_1 ++ ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/multiarch/wcslen.c b/sysdeps/x86_64/multiarch/wcslen.c +index bb97438c7f..26b5fdffd6 100644 +--- a/sysdeps/x86_64/multiarch/wcslen.c ++++ b/sysdeps/x86_64/multiarch/wcslen.c +@@ -24,7 +24,7 @@ + # undef __wcslen + + # define SYMBOL_NAME wcslen +-# include "ifunc-avx2.h" ++# include "ifunc-wcslen.h" + + libc_ifunc_redirected (__redirect_wcslen, __wcslen, IFUNC_SELECTOR ()); + weak_alias (__wcslen, wcslen); +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +new file mode 100644 +index 0000000000..4e88c70cc6 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsncmp-avx2-rtm.S +@@ -0,0 +1,5 @@ ++#define STRCMP __wcsncmp_avx2_rtm ++#define USE_AS_STRNCMP 1 ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsncmp-evex.S b/sysdeps/x86_64/multiarch/wcsncmp-evex.S +new file mode 100644 +index 0000000000..8a8e310713 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsncmp-evex.S +@@ -0,0 +1,5 @@ ++#define STRCMP __wcsncmp_evex ++#define USE_AS_STRNCMP 1 ++#define USE_AS_WCSCMP 1 ++ ++#include "strcmp-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S +new file mode 100644 +index 0000000000..7437ebee2d +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsnlen-avx2-rtm.S +@@ -0,0 +1,5 @@ ++#define STRLEN __wcsnlen_avx2_rtm ++#define USE_AS_WCSLEN 1 ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-evex.S b/sysdeps/x86_64/multiarch/wcsnlen-evex.S +new file mode 100644 +index 0000000000..24773bb4e2 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsnlen-evex.S +@@ -0,0 +1,5 @@ ++#define STRLEN __wcsnlen_evex ++#define USE_AS_WCSLEN 1 ++#define USE_AS_STRNLEN 1 ++ ++#include "strlen-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S +index a8cab0cb00..5fa51fe07c 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S ++++ b/sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S +@@ -2,4 +2,4 @@ + #define AS_STRNLEN + #define strlen __wcsnlen_sse4_1 + +-#include "../strlen.S" ++#include "strlen-vec.S" +diff --git a/sysdeps/x86_64/multiarch/wcsnlen.c b/sysdeps/x86_64/multiarch/wcsnlen.c +index 52e7e5d4f3..f15c1b328b 100644 +--- a/sysdeps/x86_64/multiarch/wcsnlen.c ++++ b/sysdeps/x86_64/multiarch/wcsnlen.c +@@ -24,27 +24,7 @@ + # undef __wcsnlen + + # define SYMBOL_NAME wcsnlen +-# include <init-arch.h> +- +-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; +-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; +- +-static inline void * +-IFUNC_SELECTOR (void) +-{ +- const struct cpu_features* cpu_features = __get_cpu_features (); +- +- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) +- && CPU_FEATURE_USABLE_P (cpu_features, AVX2) +- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) +- return OPTIMIZE (avx2); +- +- if (CPU_FEATURE_USABLE_P (cpu_features, SSE4_1)) +- return OPTIMIZE (sse4_1); +- +- return OPTIMIZE (sse2); +-} ++# include "ifunc-wcslen.h" + + libc_ifunc_redirected (__redirect_wcsnlen, __wcsnlen, IFUNC_SELECTOR ()); + weak_alias (__wcsnlen, wcsnlen); +diff --git a/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S +new file mode 100644 +index 0000000000..9bf760833f +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsrchr-avx2-rtm.S +@@ -0,0 +1,3 @@ ++#define STRRCHR __wcsrchr_avx2_rtm ++#define USE_AS_WCSRCHR 1 ++#include "strrchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wcsrchr-evex.S b/sysdeps/x86_64/multiarch/wcsrchr-evex.S +new file mode 100644 +index 0000000000..c64602f7dc +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wcsrchr-evex.S +@@ -0,0 +1,3 @@ ++#define STRRCHR __wcsrchr_evex ++#define USE_AS_WCSRCHR 1 ++#include "strrchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S +new file mode 100644 +index 0000000000..58ed21db01 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemchr-avx2-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __wmemchr_avx2_rtm ++#define USE_AS_WMEMCHR 1 ++ ++#include "memchr-avx2-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wmemchr-evex.S b/sysdeps/x86_64/multiarch/wmemchr-evex.S +new file mode 100644 +index 0000000000..06cd0f9f5a +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemchr-evex.S +@@ -0,0 +1,4 @@ ++#define MEMCHR __wmemchr_evex ++#define USE_AS_WMEMCHR 1 ++ ++#include "memchr-evex.S" +diff --git a/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S b/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S +new file mode 100644 +index 0000000000..31104d1215 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemcmp-avx2-movbe-rtm.S +@@ -0,0 +1,4 @@ ++#define MEMCMP __wmemcmp_avx2_movbe_rtm ++#define USE_AS_WMEMCMP 1 ++ ++#include "memcmp-avx2-movbe-rtm.S" +diff --git a/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S +new file mode 100644 +index 0000000000..4726d74aa1 +--- /dev/null ++++ b/sysdeps/x86_64/multiarch/wmemcmp-evex-movbe.S +@@ -0,0 +1,4 @@ ++#define MEMCMP __wmemcmp_evex_movbe ++#define USE_AS_WMEMCMP 1 ++ ++#include "memcmp-evex-movbe.S" +diff --git a/sysdeps/x86_64/strlen.S b/sysdeps/x86_64/strlen.S +index 2e226d0d55..8422c15cc8 100644 +--- a/sysdeps/x86_64/strlen.S ++++ b/sysdeps/x86_64/strlen.S +@@ -1,5 +1,5 @@ +-/* SSE2 version of strlen/wcslen. +- Copyright (C) 2012-2020 Free Software Foundation, Inc. ++/* SSE2 version of strlen. ++ Copyright (C) 2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or +@@ -16,243 +16,6 @@ + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +-#include <sysdep.h> ++#include "multiarch/strlen-vec.S" + +-#ifdef AS_WCSLEN +-# define PMINU pminud +-# define PCMPEQ pcmpeqd +-# define SHIFT_RETURN shrq $2, %rax +-#else +-# define PMINU pminub +-# define PCMPEQ pcmpeqb +-# define SHIFT_RETURN +-#endif +- +-/* Long lived register in strlen(s), strnlen(s, n) are: +- +- %xmm3 - zero +- %rdi - s +- %r10 (s+n) & (~(64-1)) +- %r11 s+n +-*/ +- +- +-.text +-ENTRY(strlen) +- +-/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */ +-#define FIND_ZERO \ +- PCMPEQ (%rax), %xmm0; \ +- PCMPEQ 16(%rax), %xmm1; \ +- PCMPEQ 32(%rax), %xmm2; \ +- PCMPEQ 48(%rax), %xmm3; \ +- pmovmskb %xmm0, %esi; \ +- pmovmskb %xmm1, %edx; \ +- pmovmskb %xmm2, %r8d; \ +- pmovmskb %xmm3, %ecx; \ +- salq $16, %rdx; \ +- salq $16, %rcx; \ +- orq %rsi, %rdx; \ +- orq %r8, %rcx; \ +- salq $32, %rcx; \ +- orq %rcx, %rdx; +- +-#ifdef AS_STRNLEN +-/* Do not read anything when n==0. */ +- test %RSI_LP, %RSI_LP +- jne L(n_nonzero) +- xor %rax, %rax +- ret +-L(n_nonzero): +-# ifdef AS_WCSLEN +- shl $2, %RSI_LP +-# endif +- +-/* Initialize long lived registers. */ +- +- add %RDI_LP, %RSI_LP +- mov %RSI_LP, %R10_LP +- and $-64, %R10_LP +- mov %RSI_LP, %R11_LP +-#endif +- +- pxor %xmm0, %xmm0 +- pxor %xmm1, %xmm1 +- pxor %xmm2, %xmm2 +- pxor %xmm3, %xmm3 +- movq %rdi, %rax +- movq %rdi, %rcx +- andq $4095, %rcx +-/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */ +- cmpq $4047, %rcx +-/* We cannot unify this branching as it would be ~6 cycles slower. */ +- ja L(cross_page) +- +-#ifdef AS_STRNLEN +-/* Test if end is among first 64 bytes. */ +-# define STRNLEN_PROLOG \ +- mov %r11, %rsi; \ +- subq %rax, %rsi; \ +- andq $-64, %rax; \ +- testq $-64, %rsi; \ +- je L(strnlen_ret) +-#else +-# define STRNLEN_PROLOG andq $-64, %rax; +-#endif +- +-/* Ignore bits in mask that come before start of string. */ +-#define PROLOG(lab) \ +- movq %rdi, %rcx; \ +- xorq %rax, %rcx; \ +- STRNLEN_PROLOG; \ +- sarq %cl, %rdx; \ +- test %rdx, %rdx; \ +- je L(lab); \ +- bsfq %rdx, %rax; \ +- SHIFT_RETURN; \ +- ret +- +-#ifdef AS_STRNLEN +- andq $-16, %rax +- FIND_ZERO +-#else +- /* Test first 16 bytes unaligned. */ +- movdqu (%rax), %xmm4 +- PCMPEQ %xmm0, %xmm4 +- pmovmskb %xmm4, %edx +- test %edx, %edx +- je L(next48_bytes) +- bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */ +- SHIFT_RETURN +- ret +- +-L(next48_bytes): +-/* Same as FIND_ZERO except we do not check first 16 bytes. */ +- andq $-16, %rax +- PCMPEQ 16(%rax), %xmm1 +- PCMPEQ 32(%rax), %xmm2 +- PCMPEQ 48(%rax), %xmm3 +- pmovmskb %xmm1, %edx +- pmovmskb %xmm2, %r8d +- pmovmskb %xmm3, %ecx +- salq $16, %rdx +- salq $16, %rcx +- orq %r8, %rcx +- salq $32, %rcx +- orq %rcx, %rdx +-#endif +- +- /* When no zero byte is found xmm1-3 are zero so we do not have to +- zero them. */ +- PROLOG(loop) +- +- .p2align 4 +-L(cross_page): +- andq $-64, %rax +- FIND_ZERO +- PROLOG(loop_init) +- +-#ifdef AS_STRNLEN +-/* We must do this check to correctly handle strnlen (s, -1). */ +-L(strnlen_ret): +- bts %rsi, %rdx +- sarq %cl, %rdx +- test %rdx, %rdx +- je L(loop_init) +- bsfq %rdx, %rax +- SHIFT_RETURN +- ret +-#endif +- .p2align 4 +-L(loop_init): +- pxor %xmm1, %xmm1 +- pxor %xmm2, %xmm2 +- pxor %xmm3, %xmm3 +-#ifdef AS_STRNLEN +- .p2align 4 +-L(loop): +- +- addq $64, %rax +- cmpq %rax, %r10 +- je L(exit_end) +- +- movdqa (%rax), %xmm0 +- PMINU 16(%rax), %xmm0 +- PMINU 32(%rax), %xmm0 +- PMINU 48(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit) +- jmp L(loop) +- +- .p2align 4 +-L(exit_end): +- cmp %rax, %r11 +- je L(first) /* Do not read when end is at page boundary. */ +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +-L(first): +- bts %r11, %rdx +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +- .p2align 4 +-L(exit): +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +-#else +- +- /* Main loop. Unrolled twice to improve L2 cache performance on core2. */ +- .p2align 4 +-L(loop): +- +- movdqa 64(%rax), %xmm0 +- PMINU 80(%rax), %xmm0 +- PMINU 96(%rax), %xmm0 +- PMINU 112(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit64) +- +- subq $-128, %rax +- +- movdqa (%rax), %xmm0 +- PMINU 16(%rax), %xmm0 +- PMINU 32(%rax), %xmm0 +- PMINU 48(%rax), %xmm0 +- PCMPEQ %xmm3, %xmm0 +- pmovmskb %xmm0, %edx +- testl %edx, %edx +- jne L(exit0) +- jmp L(loop) +- +- .p2align 4 +-L(exit64): +- addq $64, %rax +-L(exit0): +- pxor %xmm0, %xmm0 +- FIND_ZERO +- +- bsfq %rdx, %rdx +- addq %rdx, %rax +- subq %rdi, %rax +- SHIFT_RETURN +- ret +- +-#endif +- +-END(strlen) + libc_hidden_builtin_def (strlen) +diff --git a/sysdeps/x86_64/sysdep.h b/sysdeps/x86_64/sysdep.h +index 0b73674f68..c8ad778fee 100644 +--- a/sysdeps/x86_64/sysdep.h ++++ b/sysdeps/x86_64/sysdep.h +@@ -95,6 +95,28 @@ lose: \ + #define R14_LP r14 + #define R15_LP r15 + ++/* Zero upper vector registers and return with xtest. NB: Use VZEROALL ++ to avoid RTM abort triggered by VZEROUPPER inside transactionally. */ ++#define ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST \ ++ xtest; \ ++ jz 1f; \ ++ vzeroall; \ ++ ret; \ ++1: \ ++ vzeroupper; \ ++ ret ++ ++/* Zero upper vector registers and return. */ ++#ifndef ZERO_UPPER_VEC_REGISTERS_RETURN ++# define ZERO_UPPER_VEC_REGISTERS_RETURN \ ++ VZEROUPPER; \ ++ ret ++#endif ++ ++#ifndef VZEROUPPER_RETURN ++# define VZEROUPPER_RETURN VZEROUPPER; ret ++#endif ++ + #else /* __ASSEMBLER__ */ + + /* Long and pointer size in bytes. */ +diff --git a/sysdeps/x86_64/tst-rsi-strlen.c b/sysdeps/x86_64/tst-rsi-strlen.c +new file mode 100644 +index 0000000000..a80c4f85c2 +--- /dev/null ++++ b/sysdeps/x86_64/tst-rsi-strlen.c +@@ -0,0 +1,81 @@ ++/* Test strlen with 0 in the RSI register. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#ifdef WIDE ++# define TEST_NAME "wcslen" ++#else ++# define TEST_NAME "strlen" ++#endif /* WIDE */ ++ ++#define TEST_MAIN ++#include <string/test-string.h> ++ ++#ifdef WIDE ++# include <wchar.h> ++# define STRLEN wcslen ++# define CHAR wchar_t ++#else ++# define STRLEN strlen ++# define CHAR char ++#endif /* WIDE */ ++ ++IMPL (STRLEN, 1) ++ ++typedef size_t (*proto_t) (const CHAR *); ++ ++typedef struct ++{ ++ void (*fn) (void); ++} parameter_t; ++ ++size_t ++__attribute__ ((weak, noinline, noclone)) ++do_strlen (parameter_t *a, int zero, const CHAR *str) ++{ ++ return CALL (a, str); ++} ++ ++static int ++test_main (void) ++{ ++ test_init (); ++ ++ size_t size = page_size / sizeof (CHAR) - 1; ++ CHAR *buf = (CHAR *) buf2; ++ buf[size] = 0; ++ ++ parameter_t a; ++ ++ int ret = 0; ++ FOR_EACH_IMPL (impl, 0) ++ { ++ a.fn = impl->fn; ++ /* NB: Pass 0 in RSI. */ ++ size_t res = do_strlen (&a, 0, buf); ++ if (res != size) ++ { ++ error (0, 0, "Wrong result in function %s: %zu != %zu", ++ impl->name, res, size); ++ ret = 1; ++ } ++ } ++ ++ return ret ? EXIT_FAILURE : EXIT_SUCCESS; ++} ++ ++#include <support/test-driver.c> +diff --git a/sysdeps/x86_64/tst-rsi-wcslen.c b/sysdeps/x86_64/tst-rsi-wcslen.c +new file mode 100644 +index 0000000000..f45a7dfb51 +--- /dev/null ++++ b/sysdeps/x86_64/tst-rsi-wcslen.c +@@ -0,0 +1,20 @@ ++/* Test wcslen with 0 in the RSI register. ++ Copyright (C) 2021 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#define WIDE 1 ++#include "tst-rsi-strlen.c" +diff --git a/sysvipc/test-sysvsem.c b/sysvipc/test-sysvsem.c +index 01dbff343a..b7284e0b48 100644 +--- a/sysvipc/test-sysvsem.c ++++ b/sysvipc/test-sysvsem.c +@@ -20,6 +20,7 @@ + #include <stdlib.h> + #include <errno.h> + #include <string.h> ++#include <stdbool.h> + #include <sys/types.h> + #include <sys/ipc.h> + #include <sys/sem.h> +diff --git a/version.h b/version.h +index 83cd196798..e6ca7a8857 100644 +--- a/version.h ++++ b/version.h +@@ -1,4 +1,4 @@ + /* This file just defines the current version number of libc. */ + +-#define RELEASE "release" ++#define RELEASE "stable" + #define VERSION "2.32" +diff -pruN glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/64/configure glibc-2.32/sysdeps/unix/sysv/linux/x86_64/64/configure +--- glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/64/configure 2021-09-18 21:02:32.741186019 +1000 ++++ glibc-2.32/sysdeps/unix/sysv/linux/x86_64/64/configure 2021-09-18 21:03:05.314302356 +1000 +@@ -4,10 +4,10 @@ + test -n "$libc_cv_slibdir" || + case "$prefix" in + /usr | /usr/) +- libc_cv_slibdir='/lib64' +- libc_cv_rtlddir='/lib64' ++ libc_cv_slibdir='/lib' ++ libc_cv_rtlddir='/lib' + if test "$libdir" = '${exec_prefix}/lib'; then +- libdir='${exec_prefix}/lib64'; ++ libdir='${exec_prefix}/lib'; + # Locale data can be shared between 32-bit and 64-bit libraries. + libc_cv_complocaledir='${exec_prefix}/lib/locale' + fi +diff -pruN glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/ldconfig.h glibc-2.32/sysdeps/unix/sysv/linux/x86_64/ldconfig.h +--- glibc-2.32.orig/sysdeps/unix/sysv/linux/x86_64/ldconfig.h 2021-09-18 21:02:32.742186053 +1000 ++++ glibc-2.32/sysdeps/unix/sysv/linux/x86_64/ldconfig.h 2021-09-18 21:03:05.314302356 +1000 +@@ -18,9 +18,9 @@ + #include <sysdeps/generic/ldconfig.h> + + #define SYSDEP_KNOWN_INTERPRETER_NAMES \ +- { "/lib/ld-linux.so.2", FLAG_ELF_LIBC6 }, \ ++ { "/lib32/ld-linux.so.2", FLAG_ELF_LIBC6 }, \ + { "/libx32/ld-linux-x32.so.2", FLAG_ELF_LIBC6 }, \ +- { "/lib64/ld-linux-x86-64.so.2", FLAG_ELF_LIBC6 }, ++ { "/lib/ld-linux-x86-64.so.2", FLAG_ELF_LIBC6 }, + #define SYSDEP_KNOWN_LIBRARY_NAMES \ + { "libc.so.6", FLAG_ELF_LIBC6 }, \ + { "libm.so.6", FLAG_ELF_LIBC6 },