123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726 |
- #include <errno.h>
- #include <sched.h>
- #include <time.h>
- #include <stdlib.h>
- #include <limits.h>
- #include "pthread.h"
- #include "internals.h"
- #include "spinlock.h"
- #include "restart.h"
- libpthread_hidden_proto(nanosleep)
- static void __pthread_acquire(int * spinlock);
- static __inline__ void __pthread_release(int * spinlock)
- {
- WRITE_MEMORY_BARRIER();
- *spinlock = __LT_SPINLOCK_INIT;
- __asm__ __volatile__ ("" : "=m" (*spinlock) : "m" (*spinlock));
- }
- void internal_function __pthread_lock(struct _pthread_fastlock * lock,
- pthread_descr self)
- {
- #if defined HAS_COMPARE_AND_SWAP
- long oldstatus, newstatus;
- int successful_seizure, spurious_wakeup_count;
- #endif
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- __pthread_acquire(&lock->__spinlock);
- return;
- }
- #endif
- #if defined HAS_COMPARE_AND_SWAP
-
- if (lock->__status == 0 && __compare_and_swap (&lock->__status, 0, 1))
- return;
- spurious_wakeup_count = 0;
-
- #if 0
- if (__pthread_smp_kernel) {
- int spin_count;
- int max_count = lock->__spinlock * 2 + 10;
- if (max_count > MAX_ADAPTIVE_SPIN_COUNT)
- max_count = MAX_ADAPTIVE_SPIN_COUNT;
- for (spin_count = 0; spin_count < max_count; spin_count++) {
- if (((oldstatus = lock->__status) & 1) == 0) {
- if(__compare_and_swap(&lock->__status, oldstatus, oldstatus | 1))
- {
- if (spin_count)
- lock->__spinlock += (spin_count - lock->__spinlock) / 8;
- READ_MEMORY_BARRIER();
- return;
- }
- }
- #ifdef BUSY_WAIT_NOP
- BUSY_WAIT_NOP;
- #endif
- __asm__ __volatile__ ("" : "=m" (lock->__status) : "m" (lock->__status));
- }
- lock->__spinlock += (spin_count - lock->__spinlock) / 8;
- }
- #endif
- again:
-
- do {
- oldstatus = lock->__status;
- successful_seizure = 0;
- if ((oldstatus & 1) == 0) {
- newstatus = oldstatus | 1;
- successful_seizure = 1;
- } else {
- if (self == NULL)
- self = thread_self();
- newstatus = (long) self | 1;
- }
- if (self != NULL) {
- THREAD_SETMEM(self, p_nextlock, (pthread_descr) (oldstatus));
-
- MEMORY_BARRIER();
- }
- } while(! __compare_and_swap(&lock->__status, oldstatus, newstatus));
-
- if (!successful_seizure) {
- for (;;) {
- suspend(self);
- if (self->p_nextlock != NULL) {
-
- spurious_wakeup_count++;
- continue;
- }
- break;
- }
- goto again;
- }
-
- while (spurious_wakeup_count--)
- restart(self);
- READ_MEMORY_BARRIER();
- #endif
- }
- int __pthread_unlock(struct _pthread_fastlock * lock)
- {
- #if defined HAS_COMPARE_AND_SWAP
- long oldstatus;
- pthread_descr thr, * ptr, * maxptr;
- int maxprio;
- #endif
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- __pthread_release(&lock->__spinlock);
- return 0;
- }
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- WRITE_MEMORY_BARRIER();
- again:
- oldstatus = lock->__status;
- if (oldstatus == 0 || oldstatus == 1) {
-
- if (! __compare_and_swap_with_release_semantics(&lock->__status,
- oldstatus, 0))
- goto again;
- return 0;
- }
-
- ptr = (pthread_descr *) &lock->__status;
- thr = (pthread_descr) (oldstatus & ~1L);
- maxprio = 0;
- maxptr = ptr;
-
- READ_MEMORY_BARRIER();
- while (thr != 0) {
- if (thr->p_priority >= maxprio) {
- maxptr = ptr;
- maxprio = thr->p_priority;
- }
- ptr = &(thr->p_nextlock);
- thr = (pthread_descr)((long)(thr->p_nextlock) & ~1L);
- }
-
- if (maxptr == (pthread_descr *) &lock->__status) {
-
- thr = (pthread_descr) (oldstatus & ~1L);
- if (! __compare_and_swap_with_release_semantics
- (&lock->__status, oldstatus, (long)(thr->p_nextlock) & ~1L))
- goto again;
- } else {
-
- thr = (pthread_descr)((long)*maxptr & ~1L);
- *maxptr = thr->p_nextlock;
-
- WRITE_MEMORY_BARRIER();
- do {
- oldstatus = lock->__status;
- } while (!__compare_and_swap_with_release_semantics(&lock->__status,
- oldstatus, oldstatus & ~1L));
- }
-
- thr->p_nextlock = NULL;
- restart(thr);
- return 0;
- #endif
- }
- struct wait_node {
- struct wait_node *next;
- pthread_descr thr;
- int abandoned;
- };
- static long wait_node_free_list;
- static int wait_node_free_list_spinlock;
- static struct wait_node *wait_node_alloc(void)
- {
- struct wait_node *new_node = 0;
- __pthread_acquire(&wait_node_free_list_spinlock);
- if (wait_node_free_list != 0) {
- new_node = (struct wait_node *) wait_node_free_list;
- wait_node_free_list = (long) new_node->next;
- }
- WRITE_MEMORY_BARRIER();
- __pthread_release(&wait_node_free_list_spinlock);
- if (new_node == 0)
- return malloc(sizeof *wait_node_alloc());
- return new_node;
- }
- static void wait_node_free(struct wait_node *wn)
- {
- __pthread_acquire(&wait_node_free_list_spinlock);
- wn->next = (struct wait_node *) wait_node_free_list;
- wait_node_free_list = (long) wn;
- WRITE_MEMORY_BARRIER();
- __pthread_release(&wait_node_free_list_spinlock);
- return;
- }
- #if defined HAS_COMPARE_AND_SWAP
- static void wait_node_dequeue(struct wait_node **pp_head,
- struct wait_node **pp_node,
- struct wait_node *p_node)
- {
-
- if (pp_node == pp_head) {
-
- long oldvalue = (long) p_node;
- long newvalue = (long) p_node->next;
- if (__compare_and_swap((long *) pp_node, oldvalue, newvalue))
- return;
-
- for (pp_node = pp_head; p_node != *pp_node; ) {
- pp_node = &(*pp_node)->next;
- READ_MEMORY_BARRIER();
- }
- }
- *pp_node = p_node->next;
- return;
- }
- #endif
- void __pthread_alt_lock(struct _pthread_fastlock * lock,
- pthread_descr self)
- {
- #if defined HAS_COMPARE_AND_SWAP
- long oldstatus, newstatus;
- #endif
- struct wait_node wait_node;
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- int suspend_needed = 0;
- __pthread_acquire(&lock->__spinlock);
- if (lock->__status == 0)
- lock->__status = 1;
- else {
- if (self == NULL)
- self = thread_self();
- wait_node.abandoned = 0;
- wait_node.next = (struct wait_node *) lock->__status;
- wait_node.thr = self;
- lock->__status = (long) &wait_node;
- suspend_needed = 1;
- }
- __pthread_release(&lock->__spinlock);
- if (suspend_needed)
- suspend (self);
- return;
- }
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- do {
- oldstatus = lock->__status;
- if (oldstatus == 0) {
- newstatus = 1;
- } else {
- if (self == NULL)
- self = thread_self();
- wait_node.thr = self;
- newstatus = (long) &wait_node;
- }
- wait_node.abandoned = 0;
- wait_node.next = (struct wait_node *) oldstatus;
-
- MEMORY_BARRIER();
- } while(! __compare_and_swap(&lock->__status, oldstatus, newstatus));
-
- if (oldstatus != 0)
- suspend(self);
- READ_MEMORY_BARRIER();
- #endif
- }
- int __pthread_alt_timedlock(struct _pthread_fastlock * lock,
- pthread_descr self, const struct timespec *abstime)
- {
- long oldstatus = 0;
- #if defined HAS_COMPARE_AND_SWAP
- long newstatus;
- #endif
- struct wait_node *p_wait_node = wait_node_alloc();
-
- if (p_wait_node == 0) {
- __pthread_alt_lock(lock, self);
- return 1;
- }
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- __pthread_acquire(&lock->__spinlock);
- if (lock->__status == 0)
- lock->__status = 1;
- else {
- if (self == NULL)
- self = thread_self();
- p_wait_node->abandoned = 0;
- p_wait_node->next = (struct wait_node *) lock->__status;
- p_wait_node->thr = self;
- lock->__status = (long) p_wait_node;
- oldstatus = 1;
- }
- __pthread_release(&lock->__spinlock);
- goto suspend;
- }
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- do {
- oldstatus = lock->__status;
- if (oldstatus == 0) {
- newstatus = 1;
- } else {
- if (self == NULL)
- self = thread_self();
- p_wait_node->thr = self;
- newstatus = (long) p_wait_node;
- }
- p_wait_node->abandoned = 0;
- p_wait_node->next = (struct wait_node *) oldstatus;
-
- MEMORY_BARRIER();
- } while(! __compare_and_swap(&lock->__status, oldstatus, newstatus));
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- suspend:
- #endif
-
- if (oldstatus != 0) {
- if (timedsuspend(self, abstime) == 0) {
- if (!testandset(&p_wait_node->abandoned))
- return 0;
-
- suspend(self);
- }
- }
- wait_node_free(p_wait_node);
- READ_MEMORY_BARRIER();
- return 1;
- }
- void __pthread_alt_unlock(struct _pthread_fastlock *lock)
- {
- struct wait_node *p_node, **pp_node, *p_max_prio, **pp_max_prio;
- struct wait_node ** const pp_head = (struct wait_node **) &lock->__status;
- int maxprio;
- WRITE_MEMORY_BARRIER();
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- __pthread_acquire(&lock->__spinlock);
- }
- #endif
- while (1) {
-
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- if (lock->__status == 0 || lock->__status == 1) {
- lock->__status = 0;
- break;
- }
- }
- #endif
- #if defined TEST_FOR_COMPARE_AND_SWAP
- else
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- {
- long oldstatus = lock->__status;
- if (oldstatus == 0 || oldstatus == 1) {
- if (__compare_and_swap_with_release_semantics (&lock->__status, oldstatus, 0))
- break;
- else
- continue;
- }
- }
- #endif
-
- pp_max_prio = pp_node = pp_head;
- p_max_prio = p_node = *pp_head;
- maxprio = INT_MIN;
- READ_MEMORY_BARRIER();
- while (p_node != (struct wait_node *) 1) {
- int prio;
- if (p_node->abandoned) {
-
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- *pp_node = p_node->next;
- #endif
- #if defined TEST_FOR_COMPARE_AND_SWAP
- else
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- wait_node_dequeue(pp_head, pp_node, p_node);
- #endif
- wait_node_free(p_node);
-
- p_node = *pp_node;
- if (pp_node == pp_head)
- READ_MEMORY_BARRIER();
- continue;
- } else if ((prio = p_node->thr->p_priority) >= maxprio) {
-
- maxprio = prio;
- pp_max_prio = pp_node;
- p_max_prio = p_node;
- }
-
- pp_node = &p_node->next;
- p_node = *pp_node;
- }
-
- if (maxprio == INT_MIN)
- continue;
-
- if (!testandset(&p_max_prio->abandoned)) {
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- *pp_max_prio = p_max_prio->next;
- #endif
- #if defined TEST_FOR_COMPARE_AND_SWAP
- else
- #endif
- #if defined HAS_COMPARE_AND_SWAP
- wait_node_dequeue(pp_head, pp_max_prio, p_max_prio);
- #endif
- restart(p_max_prio->thr);
- break;
- }
- }
- #if defined TEST_FOR_COMPARE_AND_SWAP
- if (!__pthread_has_cas)
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- {
- __pthread_release(&lock->__spinlock);
- }
- #endif
- }
- #ifdef TEST_FOR_COMPARE_AND_SWAP
- int __pthread_has_cas = 0;
- #endif
- #if !defined HAS_COMPARE_AND_SWAP || defined TEST_FOR_COMPARE_AND_SWAP
- int __pthread_compare_and_swap(long * ptr, long oldval, long newval,
- int * spinlock)
- {
- int res;
- __pthread_acquire(spinlock);
- if (*ptr == oldval) {
- *ptr = newval; res = 1;
- } else {
- res = 0;
- }
- __pthread_release(spinlock);
- return res;
- }
- #endif
- static void __pthread_acquire(int * spinlock)
- {
- int cnt = 0;
- struct timespec tm;
- READ_MEMORY_BARRIER();
- while (testandset(spinlock)) {
- if (cnt < MAX_SPIN_COUNT) {
- sched_yield();
- cnt++;
- } else {
- tm.tv_sec = 0;
- tm.tv_nsec = SPIN_SLEEP_DURATION;
- nanosleep(&tm, NULL);
- cnt = 0;
- }
- }
- }
|