123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900 |
- #if defined SHARED || defined NOT_IN_libc
- # error in buildsystem: This file is for libc.a
- #endif
- #include <libintl.h>
- #include <signal.h>
- #include <stdlib.h>
- #include <sys/param.h>
- #include <tls.h>
- #include <dl-tls.h>
- #include <ldsodefs.h>
- #include <dl-elf.h>
- #include <dl-hash.h>
- #include <assert.h>
- #include <link.h>
- #include <string.h>
- #include <unistd.h>
- #include <stdio.h>
- #define _dl_malloc malloc
- #define _dl_memset memset
- #define _dl_mempcpy mempcpy
- #define _dl_dprintf fprintf
- #define _dl_debug_file stderr
- #define _dl_exit exit
- # define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
- # define TLS_DTV_UNALLOCATED ((void *) -1l)
- # ifdef SHARED
- static void
- __attribute__ ((__noreturn__))
- oom (void)
- {
- do {
- _dl_dprintf (_dl_debug_file,
- "cannot allocate thread-local memory: ABORT\n");
- _dl_exit (127);
- } while (1);
- }
- # endif
- void *_dl_memalign(size_t alignment, size_t bytes);
- void *_dl_memalign(size_t alignment, size_t bytes)
- {
- return _dl_malloc(bytes);
- }
- void
- internal_function __attribute_noinline__
- _dl_allocate_static_tls (struct link_map *map)
- {
-
- if (map->l_tls_align > _dl_tls_static_align)
- {
- fail:
- _dl_dprintf(_dl_debug_file, "cannot allocate memory in static TLS block");
- _dl_exit(30);
- }
- # if defined(TLS_TCB_AT_TP)
- size_t freebytes;
- size_t n;
- size_t blsize;
- freebytes = _dl_tls_static_size - _dl_tls_static_used - TLS_TCB_SIZE;
- blsize = map->l_tls_blocksize + map->l_tls_firstbyte_offset;
- if (freebytes < blsize)
- goto fail;
- n = (freebytes - blsize) / map->l_tls_align;
- size_t offset = _dl_tls_static_used + (freebytes - n * map->l_tls_align
- - map->l_tls_firstbyte_offset);
- map->l_tls_offset = _dl_tls_static_used = offset;
- # elif defined(TLS_DTV_AT_TP)
- size_t used;
- size_t check;
- size_t offset = roundup (_dl_tls_static_used, map->l_tls_align);
- used = offset + map->l_tls_blocksize;
- check = used;
-
- if (check > _dl_tls_static_size)
- goto fail;
- map->l_tls_offset = offset;
- _dl_tls_static_used = used;
- # else
- # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
- # endif
-
- if (((struct elf_resolve *) map)->init_flag & RELOCS_DONE)
- {
- #ifdef SHARED
-
- if (__builtin_expect (THREAD_DTV()[0].counter != _dl_tls_generation, 0))
- (void) _dl_update_slotinfo (map->l_tls_modid);
- #endif
- _dl_init_static_tls (map);
- }
- else
- map->l_need_tls_init = 1;
- }
- size_t
- internal_function
- _dl_next_tls_modid (void)
- {
- size_t result;
- if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
- {
- size_t disp = 0;
- struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
-
- result = GL(dl_tls_static_nelem) + 1;
- if (result <= GL(dl_tls_max_dtv_idx))
- do
- {
- while (result - disp < runp->len)
- {
- if (runp->slotinfo[result - disp].map == NULL)
- break;
- ++result;
- assert (result <= GL(dl_tls_max_dtv_idx) + 1);
- }
- if (result - disp < runp->len)
- break;
- disp += runp->len;
- }
- while ((runp = runp->next) != NULL);
- if (result > GL(dl_tls_max_dtv_idx))
- {
-
- assert (result == GL(dl_tls_max_dtv_idx) + 1);
-
- GL(dl_tls_dtv_gaps) = false;
- goto nogaps;
- }
- }
- else
- {
-
- nogaps:
- result = ++GL(dl_tls_max_dtv_idx);
- }
- return result;
- }
- # ifdef SHARED
- void
- internal_function
- _dl_determine_tlsoffset (void)
- {
- size_t max_align = TLS_TCB_ALIGN;
- size_t freetop = 0;
- size_t freebottom = 0;
-
- assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
-
- assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
- struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
-
- # if defined(TLS_TCB_AT_TP)
-
- size_t offset = 0;
- size_t cnt;
- for (cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
- {
- assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
- size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
- & (slotinfo[cnt].map->l_tls_align - 1));
- size_t off;
- max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
- if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
- {
- off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
- - firstbyte, slotinfo[cnt].map->l_tls_align)
- + firstbyte;
- if (off <= freebottom)
- {
- freetop = off;
-
- slotinfo[cnt].map->l_tls_offset = off;
- continue;
- }
- }
- off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
- slotinfo[cnt].map->l_tls_align) + firstbyte;
- if (off > offset + slotinfo[cnt].map->l_tls_blocksize
- + (freebottom - freetop))
- {
- freetop = offset;
- freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
- }
- offset = off;
-
- slotinfo[cnt].map->l_tls_offset = off;
- }
- GL(dl_tls_static_used) = offset;
- GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
- + TLS_TCB_SIZE);
- # elif defined(TLS_DTV_AT_TP)
-
- size_t offset = TLS_TCB_SIZE;
- size_t cnt;
- for (cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
- {
- assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
- size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
- & (slotinfo[cnt].map->l_tls_align - 1));
- size_t off;
- max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
- if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
- {
- off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
- if (off - freebottom < firstbyte)
- off += slotinfo[cnt].map->l_tls_align;
- if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
- {
- slotinfo[cnt].map->l_tls_offset = off - firstbyte;
- freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
- - firstbyte);
- continue;
- }
- }
- off = roundup (offset, slotinfo[cnt].map->l_tls_align);
- if (off - offset < firstbyte)
- off += slotinfo[cnt].map->l_tls_align;
- slotinfo[cnt].map->l_tls_offset = off - firstbyte;
- if (off - firstbyte - offset > freetop - freebottom)
- {
- freebottom = offset;
- freetop = off - firstbyte;
- }
- offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
- }
- GL(dl_tls_static_used) = offset;
- GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
- TLS_TCB_ALIGN);
- # else
- # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
- # endif
-
- GL(dl_tls_static_align) = max_align;
- }
- int
- internal_function
- _dl_tls_setup (void)
- {
- assert (GL(dl_tls_dtv_slotinfo_list) == NULL);
- assert (GL(dl_tls_max_dtv_idx) == 0);
- const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS;
- GL(dl_tls_dtv_slotinfo_list)
- = calloc (1, (sizeof (struct dtv_slotinfo_list)
- + nelem * sizeof (struct dtv_slotinfo)));
- if (GL(dl_tls_dtv_slotinfo_list) == NULL)
- return -1;
- GL(dl_tls_dtv_slotinfo_list)->len = nelem;
-
- GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1;
-
- _dl_determine_tlsoffset ();
- return 0;
- }
- # endif
- static void *
- internal_function
- allocate_dtv (void *result)
- {
- dtv_t *dtv;
- size_t dtv_length;
-
- dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
- dtv = calloc (dtv_length + 2, sizeof (dtv_t));
- if (dtv != NULL)
- {
-
- dtv[0].counter = dtv_length;
-
-
- INSTALL_DTV (result, dtv);
- }
- else
- result = NULL;
- return result;
- }
- void
- internal_function
- _dl_get_tls_static_info (size_t *sizep, size_t *alignp)
- {
- *sizep = GL(dl_tls_static_size);
- *alignp = GL(dl_tls_static_align);
- }
- void *
- internal_function
- _dl_allocate_tls_storage (void)
- {
- void *result;
- size_t size = GL(dl_tls_static_size);
- # if defined(TLS_DTV_AT_TP)
-
- size += (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
- & ~(GL(dl_tls_static_align) - 1);
- # endif
-
- result = _dl_memalign (GL(dl_tls_static_align), size);
- if (__builtin_expect (result != NULL, 1))
- {
-
- void *allocated = result;
- # if defined(TLS_TCB_AT_TP)
-
- result = (char *) result + size - TLS_TCB_SIZE;
-
- _dl_memset (result, '\0', TLS_TCB_SIZE);
- # elif defined(TLS_DTV_AT_TP)
- result = (char *) result + size - GL(dl_tls_static_size);
-
- _dl_memset ((char *) result - TLS_PRE_TCB_SIZE, '\0',
- TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
- # endif
- result = allocate_dtv (result);
- if (result == NULL)
- free (allocated);
- }
- return result;
- }
- void *
- internal_function
- _dl_allocate_tls_init (void *result)
- {
- if (result == NULL)
-
- return NULL;
- dtv_t *dtv = GET_DTV (result);
- struct dtv_slotinfo_list *listp;
- size_t total = 0;
- size_t maxgen = 0;
-
- listp = GL(dl_tls_dtv_slotinfo_list);
- while (1)
- {
- size_t cnt;
- for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
- {
- struct link_map *map;
- void *dest;
-
- if (total + cnt > GL(dl_tls_max_dtv_idx))
- break;
- map = listp->slotinfo[cnt].map;
- if (map == NULL)
-
- continue;
-
- maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
- if (map->l_tls_offset == NO_TLS_OFFSET)
- {
-
- dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
- dtv[map->l_tls_modid].pointer.is_static = false;
- continue;
- }
- assert (map->l_tls_modid == cnt);
- assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
- # if defined(TLS_TCB_AT_TP)
- assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
- dest = (char *) result - map->l_tls_offset;
- # elif defined(TLS_DTV_AT_TP)
- dest = (char *) result + map->l_tls_offset;
- # else
- # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
- # endif
-
- dtv[map->l_tls_modid].pointer.val = dest;
- dtv[map->l_tls_modid].pointer.is_static = true;
- _dl_memset (_dl_mempcpy (dest, map->l_tls_initimage,
- map->l_tls_initimage_size), '\0',
- map->l_tls_blocksize - map->l_tls_initimage_size);
- }
- total += cnt;
- if (total >= GL(dl_tls_max_dtv_idx))
- break;
- listp = listp->next;
- assert (listp != NULL);
- }
-
- dtv[0].counter = maxgen;
- return result;
- }
- void *
- internal_function
- _dl_allocate_tls (void *mem)
- {
- return _dl_allocate_tls_init (mem == NULL
- ? _dl_allocate_tls_storage ()
- : allocate_dtv (mem));
- }
- void
- internal_function
- _dl_deallocate_tls (void *tcb, bool dealloc_tcb)
- {
- dtv_t *dtv = GET_DTV (tcb);
- size_t cnt;
-
- for (cnt = 0; cnt < dtv[-1].counter; ++cnt)
- if (! dtv[1 + cnt].pointer.is_static
- && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
- free (dtv[1 + cnt].pointer.val);
-
- #ifdef SHARED
- if (dtv != GL(dl_initial_dtv))
- #endif
- free (dtv - 1);
- if (dealloc_tcb)
- {
- # if defined(TLS_TCB_AT_TP)
-
- tcb -= GL(dl_tls_static_size) - TLS_TCB_SIZE;
- # elif defined(TLS_DTV_AT_TP)
-
- tcb -= (TLS_PRE_TCB_SIZE + GL(dl_tls_static_align) - 1)
- & ~(GL(dl_tls_static_align) - 1);
- # endif
- free (tcb);
- }
- }
- # ifdef SHARED
- # ifndef GET_ADDR_ARGS
- # define GET_ADDR_ARGS tls_index *ti
- # endif
- # ifndef GET_ADDR_MODULE
- # define GET_ADDR_MODULE ti->ti_module
- # endif
- # ifndef GET_ADDR_OFFSET
- # define GET_ADDR_OFFSET ti->ti_offset
- # endif
- static void *
- allocate_and_init (struct link_map *map)
- {
- void *newp;
- newp = _dl_memalign (map->l_tls_align, map->l_tls_blocksize);
- if (newp == NULL)
- oom ();
-
- _dl_memset (_dl_mempcpy (newp, map->l_tls_initimage, map->l_tls_initimage_size),
- '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
- return newp;
- }
- struct link_map *
- _dl_update_slotinfo (unsigned long int req_modid)
- {
- struct link_map *the_map = NULL;
- dtv_t *dtv = THREAD_DTV ();
-
- unsigned long int idx = req_modid;
- struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
- while (idx >= listp->len)
- {
- idx -= listp->len;
- listp = listp->next;
- }
- if (dtv[0].counter < listp->slotinfo[idx].gen)
- {
-
- size_t new_gen = listp->slotinfo[idx].gen;
- size_t total = 0;
-
- listp = GL(dl_tls_dtv_slotinfo_list);
- do
- {
- size_t cnt;
- for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
- {
- size_t gen = listp->slotinfo[cnt].gen;
- if (gen > new_gen)
-
- continue;
-
- if (gen <= dtv[0].counter)
- continue;
-
- struct link_map *map = listp->slotinfo[cnt].map;
- if (map == NULL)
- {
-
- if (! dtv[total + cnt].pointer.is_static
- && dtv[total + cnt].pointer.val != TLS_DTV_UNALLOCATED)
- {
- free (dtv[total + cnt].pointer.val);
- dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
- }
- continue;
- }
-
- size_t modid = map->l_tls_modid;
- assert (total + cnt == modid);
- if (dtv[-1].counter < modid)
- {
-
- dtv_t *newp;
- size_t newsize = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
- size_t oldsize = dtv[-1].counter;
- assert (map->l_tls_modid <= newsize);
- if (dtv == GL(dl_initial_dtv))
- {
-
- newp = malloc ((2 + newsize) * sizeof (dtv_t));
- if (newp == NULL)
- oom ();
- _dl_memcpy (newp, &dtv[-1], oldsize * sizeof (dtv_t));
- }
- else
- {
- newp = realloc (&dtv[-1],
- (2 + newsize) * sizeof (dtv_t));
- if (newp == NULL)
- oom ();
- }
- newp[0].counter = newsize;
-
- _dl_memset (newp + 2 + oldsize, '\0',
- (newsize - oldsize) * sizeof (dtv_t));
-
- dtv = &newp[1];
-
- INSTALL_NEW_DTV (dtv);
- }
-
-
- if (! dtv[modid].pointer.is_static
- && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED)
-
- free (dtv[modid].pointer.val);
-
- dtv[modid].pointer.is_static = false;
- dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
- if (modid == req_modid)
- the_map = map;
- }
- total += listp->len;
- }
- while ((listp = listp->next) != NULL);
-
- dtv[0].counter = new_gen;
- }
- return the_map;
- }
- void *
- __tls_get_addr (GET_ADDR_ARGS)
- {
- dtv_t *dtv = THREAD_DTV ();
- struct link_map *the_map = NULL;
- void *p;
- if (__builtin_expect (dtv[0].counter != GL(dl_tls_generation), 0))
- the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
- p = dtv[GET_ADDR_MODULE].pointer.val;
- if (__builtin_expect (p == TLS_DTV_UNALLOCATED, 0))
- {
-
- if (the_map == NULL)
- {
-
- size_t idx = GET_ADDR_MODULE;
- struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
- while (idx >= listp->len)
- {
- idx -= listp->len;
- listp = listp->next;
- }
- the_map = listp->slotinfo[idx].map;
- }
- p = dtv[GET_ADDR_MODULE].pointer.val = allocate_and_init (the_map);
- dtv[GET_ADDR_MODULE].pointer.is_static = false;
- }
- return (char *) p + GET_ADDR_OFFSET;
- }
- # endif
- void _dl_add_to_slotinfo (struct link_map *l);
- void
- _dl_add_to_slotinfo (struct link_map *l)
- {
-
- struct dtv_slotinfo_list *listp;
- struct dtv_slotinfo_list *prevp;
- size_t idx = l->l_tls_modid;
-
- listp = GL(dl_tls_dtv_slotinfo_list);
- prevp = NULL;
- do
- {
-
- if (idx < listp->len)
- break;
- idx -= listp->len;
- prevp = listp;
- listp = listp->next;
- }
- while (listp != NULL);
- if (listp == NULL)
- {
-
- assert (idx == 0);
- listp = prevp->next = (struct dtv_slotinfo_list *)
- malloc (sizeof (struct dtv_slotinfo_list)
- + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
- if (listp == NULL)
- {
-
-
- ++GL(dl_tls_generation);
- _dl_dprintf (_dl_debug_file,
- "cannot create TLS data structures: ABORT\n");
- _dl_exit (127);
- }
- listp->len = TLS_SLOTINFO_SURPLUS;
- listp->next = NULL;
- _dl_memset (listp->slotinfo, '\0',
- TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
- }
-
- listp->slotinfo[idx].map = l;
- listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
- }
|