Fix vDSO support for all supported architectures.

- Cleanup dl-vdso.c code.
- Pass `void *` as first arg to `load_vdso()`, using 32-bit type
  is completely wrong on 64bit architectures.
- Split libc code and vDSO-related code.
  Move arch-specific implementations into separate files.

The performance improvement is for example 50-60 times on ARMv7
and about 4 times on x86_64.

Signed-off-by: Dmitry Chestnykh <dm.chestnykh@gmail.com>

extra/Configs/Config.arm

@@ -13,6 +13,7 @@ config FORCE_OPTIONS_FOR_ARCH
 	select ARCH_ANY_ENDIAN
 	select ARCH_HAS_UCONTEXT
 	select ARCH_HAS_DEPRECATED_SYSCALLS
+	select ARCH_VDSO_SUPPORT
 
 config CONFIG_ARM_EABI
 	bool "Build for EABI"

ldso/include/dl-syscall.h

@@ -11,8 +11,11 @@
  * been dynamicly linked in yet. */
 #include "sys/syscall.h"
 extern int _dl_errno;
+
+#ifdef UCLIBC_LDSO
 #undef __set_errno
 #define __set_errno(X) {(_dl_errno) = (X);}
+#endif
 
 /* Pull in the arch specific syscall implementation */
 #include <dl-syscalls.h>

ldso/include/ldso.h

@@ -194,10 +194,6 @@ extern void *_dl_get_ready_to_run(struct elf_resolve *tpnt, DL_LOADADDR_TYPE loa
 #define AUX_MAX_AT_ID 40
 extern ElfW(auxv_t) _dl_auxvt[AUX_MAX_AT_ID];
 
-void load_vdso( uint32_t sys_info_ehdr, char **envp );
-
-#ifdef __VDSO_SUPPORT__
-extern void* _dl__vdso_gettimeofday;
-#endif
+void load_vdso(void *sys_info_ehdr, char **envp );
 
 #endif /* _LDSO_H */

ldso/ldso/arm/dl-syscalls.h

@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+    return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+    return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz)        __arm_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __arm_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */

ldso/ldso/dl-startup.c

@@ -373,7 +373,7 @@ DL_START(unsigned long args)
 					     DL_GET_READY_TO_RUN_EXTRA_ARGS);
 
 
-	load_vdso(_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp);
+	load_vdso((void *)_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp);
 
 	/* Transfer control to the application.  */
 	SEND_STDERR_DEBUG("transfering control to application @ ");

ldso/ldso/dl-vdso-calls.h

@@ -0,0 +1,68 @@
+#ifndef _DL_VDSO_CALLS_H
+#define _DL_VDSO_CALLS_H
+
+#include <sys/time.h>
+#include <sys/syscall.h>
+#include <errno.h>
+#include <time.h>
+
+void __attribute__((weak)) *_get__dl__vdso_clock_gettime(void);
+#if defined(__UCLIBC_USE_TIME64__)
+#include "internal/time64_helpers.h"
+void __attribute__((weak)) *_get__dl__vdso_clock_gettime64(void);
+typedef int (*clock_gettime_func)(clockid_t clock_id, struct __ts64_struct *tp);
+#else
+typedef int (*clock_gettime_func)(clockid_t clock_id, struct timespec *tp);
+#endif
+
+extern int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp);
+
+static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+    void *impl = NULL;
+#if defined(__UCLIBC_USE_TIME64__)
+    if (&_get__dl__vdso_clock_gettime64 && (impl = _get__dl__vdso_clock_gettime64())) {
+        struct __ts64_struct __ts64;
+        int __ret = ((clock_gettime_func)impl)(clock_id, &__ts64);
+        if (__ret != 0) {
+            __set_errno(-__ret);
+            return -1;
+        }
+
+        if (tp) {
+            tp->tv_sec = __ts64.tv_sec;
+            tp->tv_nsec = __ts64.tv_nsec;
+        }
+        return 0;
+    }
+
+    /* fallback to syscall */
+    return __libc_clock_gettime(clock_id, tp);
+#else
+    if (&_get__dl__vdso_clock_gettime && (impl = _get__dl__vdso_clock_gettime())) {
+        int __ret = ((clock_gettime_func)impl)(clock_id, tp);
+        if (__ret != 0) {
+            __set_errno(-__ret);
+            return -1;
+        }
+
+        return 0;
+    }
+
+    /* fallback to syscall */
+    return __libc_clock_gettime(clock_id, tp);
+#endif
+}
+
+static int __attribute__ ((used)) __generic_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+    struct timespec ts;
+    int __res = __generic_vdso_clock_gettime(CLOCK_REALTIME, &ts);
+    tv->tv_sec = ts.tv_sec;
+    tv->tv_usec = (suseconds_t)ts.tv_nsec / 1000;
+
+    return __res;
+}
+
+#endif /* _DL_VDSO_CALLS_H */

ldso/ldso/dl-vdso.c

@@ -1,11 +1,7 @@
 
 #include <elf.h>
-//#include <stdio.h>
 #include <string.h>
-
 #include "sys/auxv.h"
-//#include <linux/time.h>
-//#include <time.h>
 
 #include "ldso.h"
 #include "generated/autoconf.h"
@@ -24,7 +20,7 @@
 
 
 #ifndef __VDSO_SUPPORT__
-    void load_vdso( uint32_t sys_info_ehdr, char **envp ){
+    void load_vdso(void *sys_info_ehdr, char **envp ){
 #ifdef __SUPPORT_LD_DEBUG__
         if ( _dl_debug_vdso != 0 ){
             _dl_dprintf(2,"_dl_vdso support not enabled\n" );
@@ -34,21 +30,34 @@
     }
 #else
 
+void *_dl__vdso_gettimeofday  = 0;
+void *_dl__vdso_clock_gettime = 0;
 
+#if defined(__UCLIBC_USE_TIME64__)
+void *_dl__vdso_clock_gettime64 = 0;
+#endif
 
-   
-
-
-//typedef long (*gtod_t)(struct timeval *tv, struct timezone *tz);
-void* _dl__vdso_gettimeofday  = 0;
-
-
-//typedef long (*clock_gettime_t)(int clk_id, struct timespec *tp);
-void* _dl__vdso_clock_gettime = 0;
+void *_get__dl__vdso_clock_gettime(void);
+void *_get__dl__vdso_clock_gettime(void)
+{
+    return _dl__vdso_clock_gettime;
+}
 
+#if defined(__UCLIBC_USE_TIME64__)
+void *_get__dl__vdso_clock_gettime64(void);
+void *_get__dl__vdso_clock_gettime64(void)
+{
+    return _dl__vdso_clock_gettime64;
+}
+#endif
 
+void *_get__dl__vdso_gettimeofday(void);
+void *_get__dl__vdso_gettimeofday(void)
+{
+    return _dl__vdso_gettimeofday;
+}
 
-typedef struct{
+typedef struct {
 
     void* base_addr;
 
@@ -75,7 +84,7 @@ typedef struct{
     char* vers_strings[10];
 
 
-}elf_infos;
+} elf_infos;
 
 /*
  * the raise() dummy function is needed because of divisons in this code
@@ -111,13 +120,10 @@ static ELF(Shdr) *vdso_get_sec_header( elf_infos* elf, int index ){
     
 }
 
-
-void load_vdso( uint32_t sys_info_ehdr, char **envp ){
+void load_vdso(void *sys_info_ehdr, char **envp ){
 
     elf_infos vdso_infos;
-    
-    
-    
+
     if ( sys_info_ehdr == 0 ){
 #ifdef __SUPPORT_LD_DEBUG__
         if ( _dl_debug_vdso != 0 ){
@@ -137,22 +143,16 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
 #endif
         return;
     }
-    
-    
+
     _dl_memset( &vdso_infos, 0 , sizeof( elf_infos ) );
-    
-    
+
     vdso_infos.base_addr = (void*)sys_info_ehdr;
     vdso_infos.hdr = (ELF(Ehdr)*)vdso_infos.base_addr;
-    
-    //printf("base : %p\n",vdso_infos.base_addr);
-    
+
     if ( 0 != vdso_check_elf_header( &vdso_infos ) ){
         return;
     }
-    
-    
-    
+
     ELF(Shdr) *sec_header = vdso_get_sec_header( &vdso_infos, vdso_infos.hdr->e_shstrndx);
     vdso_infos.section_header_strtab = ( vdso_infos.base_addr + sec_header->sh_offset );
     
@@ -307,10 +307,7 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
         if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
             continue;
 
-
-
         char* name = vdso_infos.dynstr_table + sym->st_name;
-        char* vers = vdso_infos.vers_strings[ vdso_infos.versym_table[i] ];
         void* func_addr = (void*)( vdso_infos.base_addr + sym->st_value );
         
         // the function name is patched to zero if the kernel has no timer which is
@@ -324,9 +321,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
             continue;
         }
 
-        //printf("   %s@@%s\n", name , vers );
-        
-        //print_sym( sym );
         if ( 0 == _dl_strcmp( name, "__vdso_gettimeofday" ) ){
             _dl__vdso_gettimeofday = func_addr;
 #ifdef __SUPPORT_LD_DEBUG__
@@ -336,7 +330,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
 #endif
             continue;
         }
-        
         if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime" ) ){
             _dl__vdso_clock_gettime = func_addr;
 #ifdef __SUPPORT_LD_DEBUG__
@@ -346,17 +339,26 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
 #endif
             continue;
         }
+
+#if defined(__UCLIBC_USE_TIME64__)
+        if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime64" ) ){
+            _dl__vdso_clock_gettime64 = func_addr;
+#ifdef __SUPPORT_LD_DEBUG__
+            if ( _dl_debug_vdso != 0 ){
+                _dl_dprintf(2,"   %s at address %p\n", name, func_addr );
+            }
+#endif
+            continue;
+        }
+#endif  /* defined(__UCLIBC_USE_TIME64__) */
         
 #ifdef __SUPPORT_LD_DEBUG__
         if ( _dl_debug_vdso != 0 ){
             _dl_dprintf(2,"   <%s> not handled\n", name );
         }
 #endif
-        
-        
-    }
     
- 
+    }
 }
 
 #endif // __VDSO_SUPPORT__

ldso/ldso/mips/dl-syscalls.h

@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+    return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+    return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz)        __mips_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __mips_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */

ldso/ldso/x86_64/dl-syscalls.h

@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+    return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+    return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz)        __x86_64_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __x86_64_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */

libc/sysdeps/linux/common/clock_gettime.c

@@ -11,10 +11,14 @@
 #include <sys/syscall.h>
 #include <time.h>
 
+#ifdef __VDSO_SUPPORT__
+#include "ldso.h"
+#endif
+
 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_clock_gettime64)
 #include "internal/time64_helpers.h"
 
-int clock_gettime(clockid_t clock_id, struct timespec *tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp)
 {
 	struct __ts64_struct __ts64;
 	int __ret = INLINE_SYSCALL(clock_gettime64, 2, clock_id, &__ts64);
@@ -26,11 +30,14 @@ int clock_gettime(clockid_t clock_id, struct timespec *tp)
 	return __ret;
 }
 #elif defined(__NR_clock_gettime)
-_syscall2(int, clock_gettime, clockid_t, clock_id, struct timespec*, tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+	return INLINE_SYSCALL(clock_gettime, 2, clock_id, tp);
+}
 #else
 # include <sys/time.h>
 
-int clock_gettime(clockid_t clock_id, struct timespec* tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec* tp)
 {
 	struct timeval tv;
 	int retval = -1;
@@ -53,3 +60,12 @@ int clock_gettime(clockid_t clock_id, struct timespec* tp)
 	return retval;
 }
 #endif
+
+int clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME)
+	return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp);
+#else
+	return __libc_clock_gettime(clock_id, tp);
+#endif
+}

libc/sysdeps/linux/common/gettimeofday.c

@@ -14,30 +14,18 @@
 #include "ldso.h"
 #endif
 
-#ifdef __VDSO_SUPPORT__
-typedef int (*gettimeofday_func)(struct timeval * tv, __timezone_ptr_t tz);
-#endif
-
 int gettimeofday(struct timeval * tv, __timezone_ptr_t tz) {
-
-    #ifdef __VDSO_SUPPORT__
-        if ( _dl__vdso_gettimeofday != 0 ){
-            gettimeofday_func func= _dl__vdso_gettimeofday;
-            return func( tv, tz );
-
-        }else{
-            _syscall2_body(int, gettimeofday, struct timeval *, tv, __timezone_ptr_t, tz)
-        }
-    #else
-        if (!tv)
-            return 0;
-
-        struct timespec __ts;
-        int __ret = clock_gettime(CLOCK_REALTIME, &__ts);
-        tv->tv_sec = __ts.tv_sec;
-        tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000;
-        return __ret;
-    #endif
+    if (!tv)
+        return 0;
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_GETTIMEOFDAY)
+    return ARCH_VDSO_GETTIMEOFDAY(tv, tz);
+#else
+    struct timespec __ts;
+    int __ret = clock_gettime(CLOCK_REALTIME, &__ts);
+    tv->tv_sec = __ts.tv_sec;
+    tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000;
+    return __ret;
+#endif
 }
 
 

librt/clock_gettime.c

@@ -22,6 +22,10 @@
 #include <sys/time.h>
 #include "kernel-posix-cpu-timers.h"
 
+#ifdef __VDSO_SUPPORT__
+#include "ldso.h"
+#endif
+
 #if defined(__UCLIBC_USE_TIME64__)
 #include "internal/time64_helpers.h"
 #endif
@@ -68,9 +72,8 @@ realtime_gettime (struct timespec *tp)
   return retval;
 }
 
-/* Get current value of CLOCK and store it in TP.  */
 int
-clock_gettime (clockid_t clock_id, struct timespec *tp)
+__libc_clock_gettime (clockid_t clock_id, struct timespec *tp)
 {
   int retval = -1;
 #ifndef HANDLED_REALTIME
@@ -101,3 +104,14 @@ clock_gettime (clockid_t clock_id, struct timespec *tp)
 
   return retval;
 }
+
+/* Get current value of CLOCK and store it in TP.  */
+int
+clock_gettime (clockid_t clock_id, struct timespec *tp)
+{
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME)
+  return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp);
+#else
+  return __libc_clock_gettime(clock_id, tp);
+#endif
+}