From 7cbf1c8416b04c65dc3d253061d8a674ee3c616e Mon Sep 17 00:00:00 2001 From: Florian Weimer Date: Wed, 5 May 2021 06:20:31 +0200 Subject: [PATCH] elf, nptl: Initialize static TLS directly in ld.so The stack list is available in ld.so since commit 1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list variables into _rtld_global"), so it's possible to walk the stack list directly in ld.so and perform the initialization there. This eliminates an unprotected function pointer from _rtld_global and reduces the libpthread initialization code. --- elf/dl-open.c | 2 +- elf/dl-reloc.c | 5 +++-- elf/dl-support.c | 3 +-- elf/dl-tls.c | 39 +++++++++++++++++++++++++++++++++++++++ elf/rtld.c | 2 ++ nptl/allocatestack.c | 35 ----------------------------------- nptl/nptl-init.c | 2 -- nptl/pthreadP.h | 2 -- sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++ 9 files changed, 65 insertions(+), 44 deletions(-) diff --git a/elf/dl-open.c b/elf/dl-open.c index ab7aaa3..09f0df7 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -426,7 +426,7 @@ TLS generation counter wrapped! Please report this.")); _dl_update_slotinfo (imap->l_tls_modid); #endif - GL(dl_init_static_tls) (imap); + dl_init_static_tls (imap); assert (imap->l_need_tls_init == 0); } } diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c index c2df26d..bb9ca1a 100644 --- a/elf/dl-reloc.c +++ b/elf/dl-reloc.c @@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional) (void) _dl_update_slotinfo (map->l_tls_modid); #endif - GL(dl_init_static_tls) (map); + dl_init_static_tls (map); } else map->l_need_tls_init = 1; @@ -141,6 +141,7 @@ cannot allocate memory in static TLS block")); } } +#if !THREAD_GSCOPE_IN_TCB /* Initialize static TLS area and DTV for current (only) thread. libpthread implementations should provide their own hook to handle all threads. */ @@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map) memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), '\0', map->l_tls_blocksize - map->l_tls_initimage_size); } - +#endif /* !THREAD_GSCOPE_IN_TCB */ void _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[], diff --git a/elf/dl-support.c b/elf/dl-support.c index 7fc2ee7..f966a2e 100644 --- a/elf/dl-support.c +++ b/elf/dl-support.c @@ -138,8 +138,6 @@ void *_dl_random; #include #include -void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; - size_t _dl_pagesize = EXEC_PAGESIZE; size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ; @@ -197,6 +195,7 @@ list_t _dl_stack_user; int _dl_stack_cache_lock; #else int _dl_thread_gscope_count; +void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls; #endif struct dl_scope_free_list *_dl_scope_free_list; diff --git a/elf/dl-tls.c b/elf/dl-tls.c index f8b32b3..6baff0c 100644 --- a/elf/dl-tls.c +++ b/elf/dl-tls.c @@ -29,6 +29,10 @@ #include #include +#if THREAD_GSCOPE_IN_TCB +# include +#endif + #define TUNABLE_NAMESPACE rtld #include @@ -1005,3 +1009,38 @@ cannot create TLS data structures")); listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; } } + +#if THREAD_GSCOPE_IN_TCB +static inline void __attribute__((always_inline)) +init_one_static_tls (struct pthread *curp, struct link_map *map) +{ +# if TLS_TCB_AT_TP + void *dest = (char *) curp - map->l_tls_offset; +# elif TLS_DTV_AT_TP + void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; +# else +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" +# endif + + /* Initialize the memory. */ + memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), + '\0', map->l_tls_blocksize - map->l_tls_initimage_size); +} + +void +_dl_init_static_tls (struct link_map *map) +{ + lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); + + /* Iterate over the list with system-allocated threads first. */ + list_t *runp; + list_for_each (runp, &GL (dl_stack_used)) + init_one_static_tls (list_entry (runp, struct pthread, list), map); + + /* Now the list with threads using user-allocated stacks. */ + list_for_each (runp, &GL (dl_stack_user)) + init_one_static_tls (list_entry (runp, struct pthread, list), map); + + lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); +} +#endif /* THREAD_GSCOPE_IN_TCB */ diff --git a/elf/rtld.c b/elf/rtld.c index 3487901..ad325d4 100644 --- a/elf/rtld.c +++ b/elf/rtld.c @@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr, struct dl_main_state state; dl_main_state_init (&state); +#if !THREAD_GSCOPE_IN_TCB GL(dl_init_static_tls) = &_dl_nothread_init_static_tls; +#endif #if defined SHARED && defined _LIBC_REENTRANT \ && defined __rtld_lock_default_lock_recursive diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index f1270c3..8aaba08 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -951,38 +951,3 @@ __reclaim_stacks (void) GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER; __default_pthread_attr_lock = LLL_LOCK_INITIALIZER; } - - -static inline void __attribute__((always_inline)) -init_one_static_tls (struct pthread *curp, struct link_map *map) -{ -# if TLS_TCB_AT_TP - void *dest = (char *) curp - map->l_tls_offset; -# elif TLS_DTV_AT_TP - void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; -# else -# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" -# endif - - /* Initialize the memory. */ - memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), - '\0', map->l_tls_blocksize - map->l_tls_initimage_size); -} - -void -attribute_hidden -__pthread_init_static_tls (struct link_map *map) -{ - lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE); - - /* Iterate over the list with system-allocated threads first. */ - list_t *runp; - list_for_each (runp, &GL (dl_stack_used)) - init_one_static_tls (list_entry (runp, struct pthread, list), map); - - /* Now the list with threads using user-allocated stacks. */ - list_for_each (runp, &GL (dl_stack_user)) - init_one_static_tls (list_entry (runp, struct pthread, list), map); - - lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE); -} diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c index b0879bd..fcab5a0 100644 --- a/nptl/nptl-init.c +++ b/nptl/nptl-init.c @@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void) GL(dl_make_stack_executable_hook) = &__make_stacks_executable; #endif - GL(dl_init_static_tls) = &__pthread_init_static_tls; - /* Register the fork generation counter with the libc. */ #ifndef TLS_MULTIPLE_THREADS_IN_TCB __libc_multiple_threads_ptr = diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index 00d2cfe..0ce6367 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden; extern int *__libc_multiple_threads_ptr attribute_hidden; #endif -extern void __pthread_init_static_tls (struct link_map *) attribute_hidden; - extern size_t __pthread_get_minstack (const pthread_attr_t *attr); /* Namespace save aliases. */ diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h index 67c6686..1b064c5 100644 --- a/sysdeps/generic/ldsodefs.h +++ b/sysdeps/generic/ldsodefs.h @@ -464,7 +464,9 @@ struct rtld_global /* Generation counter for the dtv. */ EXTERN size_t _dl_tls_generation; +#if !THREAD_GSCOPE_IN_TCB EXTERN void (*_dl_init_static_tls) (struct link_map *); +#endif /* Scopes to free after next THREAD_GSCOPE_WAIT (). */ EXTERN struct dl_scope_free_list @@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void) extern void _dl_aux_init (ElfW(auxv_t) *av) attribute_hidden; +/* Initialize the static TLS space for the link map in all existing + threads. */ +#if THREAD_GSCOPE_IN_TCB +void _dl_init_static_tls (struct link_map *map) attribute_hidden; +#endif +static inline void +dl_init_static_tls (struct link_map *map) +{ +#if THREAD_GSCOPE_IN_TCB + /* The stack list is available to ld.so, so the initialization can + be handled within ld.so directly. */ + _dl_init_static_tls (map); +#else + GL (dl_init_static_tls) (map); +#endif +} + /* Return true if the ld.so copy in this namespace is actually active and working. If false, the dl_open/dlfcn hooks have to be used to call into the outer dynamic linker (which happens after static -- 2.7.4