superrob1500
/
mirror-linux
mirror of https://github.com/torvalds/linux
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update to the VDSO subsystem: 

- Make the handling of compat functions consistent and more robust
 
      - Rework the underlying data store so that it is dynamically
        allocated, which allows the conversion of the last holdout SPARC64
        to the generic VDSO implementation
 
      - Rework the SPARC64 VDSO to utilize the generic implementation
 
      - Mop up the left overs of the non-generic VDSO support in the core
        code.
 
      - Expand the VDSO selftest and make them more robust
 
      - Allow time namespaces to be enabled independently of the generic
        VDSO support, which was not possible before due to SPARC64 not
        using it.
 
      - Various cleanups and improvements in the related code.
 -----BEGIN PGP SIGNATURE-----
 
 iQJEBAABCgAuFiEEQp8+kY+LLUocC4bMphj1TA10mKEFAmnb0v8QHHRnbHhAa2Vy
 bmVsLm9yZwAKCRCmGPVMDXSYocfqD/9ywgnvwRH6B612mY4PI3qCbLHs6n9f78aH
 YwyXmmfBZ5vt1ZtptHD+BAxiIMm9GC+/exdj5zhcOWucnBVhorcloE6evxhkJAMn
 RhTQFKkEmcA/UV2Yfct9r+33kgZRyu4IIul4J7hgn2o5T1BqwZbOil0W/O5adr5P
 MDLxjT1OLV80ZZWI9qbWcR/aR7W7sHcdwfVPPqjhombRY7f391Mo3dZeM5C2y55x
 8TXCEqVpN1RJzFinWEgQN7QpP4OmF0rRuXSrDQpkH6pk/+RSqNlT/QGG7MJtmCQR
 E6CeBjNRUn318KiroaGyTKlM9xsL3gNoiCY24ZTwzZxx3g5gSAR3KTCTJhQU0hpu
 Svxj+ksqEAyW7fAOIsbce6W8fUPKC2KM+juXgPKcqZ5hjE2fALD+eEYMlq00jSiu
 sj71007cM9tZKOXPdWs3Fv7AY2Yj7iiRiRz9gv1wqS1z7ybxiaFjxjLYYakej0tr
 rmwBDEGhNow7msZZttr01BRZk9hDUWfIiJtL+0BrgRLNzst2A7WoagtZ2s0Z7Psl
 RjtWgYNBDJ878xK0J+Djqb9TyLraGWZShIIna9uYCAJX9i954xfKJ//NOnUkZhcl
 jslDLHhdttyJ+TmgIsc1ntUGvYvHqH5ywQpyDfWepMKyIYdaJLHOr2K6bwFnGHdw
 uocXvLrkXw==
 =8ixX
 -----END PGP SIGNATURE-----

Merge tag 'timers-vdso-2026-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull vdso updates from Thomas Gleixner:

 - Make the handling of compat functions consistent and more robust

 - Rework the underlying data store so that it is dynamically allocated,
   which allows the conversion of the last holdout SPARC64 to the
   generic VDSO implementation

 - Rework the SPARC64 VDSO to utilize the generic implementation

 - Mop up the left overs of the non-generic VDSO support in the core
   code

 - Expand the VDSO selftest and make them more robust

 - Allow time namespaces to be enabled independently of the generic VDSO
   support, which was not possible before due to SPARC64 not using it

 - Various cleanups and improvements in the related code

* tag 'timers-vdso-2026-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (51 commits)
  timens: Use task_lock guard in timens_get*()
  timens: Use mutex guard in proc_timens_set_offset()
  timens: Simplify some calls to put_time_ns()
  timens: Add a __free() wrapper for put_time_ns()
  timens: Remove dependency on the vDSO
  vdso/timens: Move functions to new file
  selftests: vDSO: vdso_test_correctness: Add a test for time()
  selftests: vDSO: vdso_test_correctness: Use facilities from parse_vdso.c
  selftests: vDSO: vdso_test_correctness: Handle different tv_usec types
  selftests: vDSO: vdso_test_correctness: Drop SYS_getcpu fallbacks
  selftests: vDSO: vdso_test_gettimeofday: Remove nolibc checks
  Revert "selftests: vDSO: parse_vdso: Use UAPI headers instead of libc headers"
  random: vDSO: Remove ifdeffery
  random: vDSO: Trim vDSO includes
  vdso/datapage: Trim down unnecessary includes
  vdso/datapage: Remove inclusion of gettimeofday.h
  vdso/helpers: Explicitly include vdso/processor.h
  vdso/gettimeofday: Add explicit includes
  random: vDSO: Add explicit includes
  MIPS: vdso: Explicitly include asm/vdso/vdso.h
  ...
master
Linus Torvalds 2026-04-14 10:53:44 -07:00
parent c1fe867b5b 7138a8698a
commit f21f7b5162
56 changed files with 848 additions and 1310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MAINTAINERS
View File

@ -10796,6 +10796,7 @@ S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/vdso T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/vdso
F: include/asm-generic/vdso/vsyscall.h F: include/asm-generic/vdso/vsyscall.h
F: include/vdso/ F: include/vdso/
F: kernel/time/namespace_vdso.c
F: kernel/time/vsyscall.c F: kernel/time/vsyscall.c
F: lib/vdso/ F: lib/vdso/
F: tools/testing/selftests/vDSO/ F: tools/testing/selftests/vDSO/
@ -21042,6 +21043,7 @@ F: include/trace/events/timer*
F: kernel/time/itimer.c F: kernel/time/itimer.c
F: kernel/time/posix-* F: kernel/time/posix-*
F: kernel/time/namespace.c F: kernel/time/namespace.c
F: kernel/time/namespace_vdso.c
POWER MANAGEMENT CORE POWER MANAGEMENT CORE
M: "Rafael J. Wysocki" <rafael@kernel.org> M: "Rafael J. Wysocki" <rafael@kernel.org>

2
arch/arm/include/asm/vdso/gettimeofday.h
View File

@ -11,6 +11,8 @@
#include <asm/errno.h> #include <asm/errno.h>
#include <asm/unistd.h> #include <asm/unistd.h>
#include <asm/vdso/cp15.h> #include <asm/vdso/cp15.h>
#include <vdso/clocksource.h>
#include <vdso/time32.h>
#include <uapi/linux/time.h> #include <uapi/linux/time.h>
#define VDSO_HAS_CLOCK_GETRES 1 #define VDSO_HAS_CLOCK_GETRES 1

3
arch/arm64/include/asm/vdso/compat_gettimeofday.h
View File

@ -7,6 +7,9 @@
#ifndef __ASSEMBLER__ #ifndef __ASSEMBLER__
#include <vdso/clocksource.h>
#include <vdso/time32.h>
#include <asm/barrier.h> #include <asm/barrier.h>
#include <asm/unistd_compat_32.h> #include <asm/unistd_compat_32.h>
#include <asm/errno.h> #include <asm/errno.h>

2
arch/arm64/include/asm/vdso/gettimeofday.h
View File

@ -9,6 +9,8 @@
#ifndef __ASSEMBLER__ #ifndef __ASSEMBLER__
#include <vdso/clocksource.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/arch_timer.h> #include <asm/arch_timer.h>
#include <asm/barrier.h> #include <asm/barrier.h>

1
arch/loongarch/kernel/process.c
View File

@ -52,6 +52,7 @@
#include <asm/switch_to.h> #include <asm/switch_to.h>
#include <asm/unwind.h> #include <asm/unwind.h>
#include <asm/vdso.h> #include <asm/vdso.h>
#include <asm/vdso/vdso.h>
#ifdef CONFIG_STACKPROTECTOR #ifdef CONFIG_STACKPROTECTOR
#include <linux/stackprotector.h> #include <linux/stackprotector.h>

1
arch/loongarch/kernel/vdso.c
View File

@ -18,6 +18,7 @@
#include <asm/page.h> #include <asm/page.h>
#include <asm/vdso.h> #include <asm/vdso.h>
#include <asm/vdso/vdso.h>
#include <vdso/helpers.h> #include <vdso/helpers.h>
#include <vdso/vsyscall.h> #include <vdso/vsyscall.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>

5
arch/mips/include/asm/vdso/vdso.h
View File

@ -4,6 +4,9 @@
* Author: Alex Smith <alex.smith@imgtec.com> * Author: Alex Smith <alex.smith@imgtec.com>
*/ */
#ifndef __ASM_VDSO_VDSO_H
#define __ASM_VDSO_VDSO_H
#include <asm/sgidefs.h> #include <asm/sgidefs.h>
#include <vdso/page.h> #include <vdso/page.h>
@ -70,3 +73,5 @@ static inline void __iomem *get_gic(const struct vdso_time_data *data)
#endif /* CONFIG_CLKSRC_MIPS_GIC */ #endif /* CONFIG_CLKSRC_MIPS_GIC */
#endif /* __ASSEMBLER__ */ #endif /* __ASSEMBLER__ */
#endif /* __ASM_VDSO_VDSO_H */

1
arch/mips/kernel/vdso.c
View File

@ -21,6 +21,7 @@
#include <asm/mips-cps.h> #include <asm/mips-cps.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/vdso.h> #include <asm/vdso.h>
#include <asm/vdso/vdso.h>
#include <vdso/helpers.h> #include <vdso/helpers.h>
#include <vdso/vsyscall.h> #include <vdso/vsyscall.h>

1
arch/powerpc/include/asm/vdso/gettimeofday.h
View File

@ -8,6 +8,7 @@
#include <asm/barrier.h> #include <asm/barrier.h>
#include <asm/unistd.h> #include <asm/unistd.h>
#include <uapi/linux/time.h> #include <uapi/linux/time.h>
#include <vdso/time32.h>
#define VDSO_HAS_CLOCK_GETRES 1 #define VDSO_HAS_CLOCK_GETRES 1

3
arch/powerpc/include/asm/vdso/processor.h
View File

@ -4,6 +4,9 @@
#ifndef __ASSEMBLER__ #ifndef __ASSEMBLER__
#include <asm/cputable.h>
#include <asm/feature-fixups.h>
/* Macros for adjusting thread priority (hardware multi-threading) */ /* Macros for adjusting thread priority (hardware multi-threading) */
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
#define HMT_very_low() asm volatile("or 31, 31, 31 # very low priority") #define HMT_very_low() asm volatile("or 31, 31, 31 # very low priority")

3
arch/powerpc/kernel/compat_audit.c
View File

@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
#undef __powerpc64__
#include <linux/audit_arch.h> #include <linux/audit_arch.h>
#include <asm/unistd.h> #include <asm/unistd_32.h>
#include "audit_32.h" #include "audit_32.h"

3
arch/s390/Makefile
View File

@ -12,8 +12,7 @@ LD_BFD := elf64-s390
KBUILD_LDFLAGS := -m elf64_s390 KBUILD_LDFLAGS := -m elf64_s390
KBUILD_AFLAGS_MODULE += -fPIC KBUILD_AFLAGS_MODULE += -fPIC
KBUILD_CFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC
KBUILD_AFLAGS += -m64 KBUILD_CPPFLAGS += -m64
KBUILD_CFLAGS += -m64
KBUILD_CFLAGS += -fPIC KBUILD_CFLAGS += -fPIC
LDFLAGS_vmlinux := $(call ld-option,-no-pie) LDFLAGS_vmlinux := $(call ld-option,-no-pie)
extra_tools := relocs extra_tools := relocs

3
arch/sparc/Kconfig
View File

@ -104,7 +104,6 @@ config SPARC64
select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_RWLOCKS
select ARCH_USE_QUEUED_SPINLOCKS select ARCH_USE_QUEUED_SPINLOCKS
select GENERIC_TIME_VSYSCALL select GENERIC_TIME_VSYSCALL
select ARCH_CLOCKSOURCE_DATA
select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_PTE_SPECIAL
select PCI_DOMAINS if PCI select PCI_DOMAINS if PCI
select ARCH_HAS_GIGANTIC_PAGE select ARCH_HAS_GIGANTIC_PAGE
@ -115,6 +114,8 @@ config SPARC64
select ARCH_SUPPORTS_SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_SMT if SMP
select ARCH_SUPPORTS_SCHED_MC if SMP select ARCH_SUPPORTS_SCHED_MC if SMP
select ARCH_HAS_LAZY_MMU_MODE select ARCH_HAS_LAZY_MMU_MODE
select HAVE_GENERIC_VDSO
select GENERIC_GETTIMEOFDAY
config ARCH_PROC_KCORE_TEXT config ARCH_PROC_KCORE_TEXT
def_bool y def_bool y

9
arch/sparc/include/asm/clocksource.h
View File

@ -5,13 +5,4 @@
#ifndef _ASM_SPARC_CLOCKSOURCE_H #ifndef _ASM_SPARC_CLOCKSOURCE_H
#define _ASM_SPARC_CLOCKSOURCE_H #define _ASM_SPARC_CLOCKSOURCE_H
/* VDSO clocksources */
#define VCLOCK_NONE 0 /* Nothing userspace can do. */
#define VCLOCK_TICK 1 /* Use %tick. */
#define VCLOCK_STICK 2 /* Use %stick. */
struct arch_clocksource_data {
int vclock_mode;
};
#endif /* _ASM_SPARC_CLOCKSOURCE_H */ #endif /* _ASM_SPARC_CLOCKSOURCE_H */

3
arch/sparc/include/asm/processor.h
View File

@ -1,6 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 */ /* SPDX-License-Identifier: GPL-2.0 */
#ifndef ___ASM_SPARC_PROCESSOR_H #ifndef ___ASM_SPARC_PROCESSOR_H
#define ___ASM_SPARC_PROCESSOR_H #define ___ASM_SPARC_PROCESSOR_H
#include <asm/vdso/processor.h>
#if defined(__sparc__) && defined(__arch64__) #if defined(__sparc__) && defined(__arch64__)
#include <asm/processor_64.h> #include <asm/processor_64.h>
#else #else

2
arch/sparc/include/asm/processor_32.h
View File

@ -91,8 +91,6 @@ unsigned long __get_wchan(struct task_struct *);
extern struct task_struct *last_task_used_math; extern struct task_struct *last_task_used_math;
int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); int do_mathemu(struct pt_regs *regs, struct task_struct *fpt);
#define cpu_relax() barrier()
extern void (*sparc_idle)(void); extern void (*sparc_idle)(void);
#endif #endif

25
arch/sparc/include/asm/processor_64.h
View File

@ -182,31 +182,6 @@ unsigned long __get_wchan(struct task_struct *task);
#define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc)
#define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP])
/* Please see the commentary in asm/backoff.h for a description of
* what these instructions are doing and how they have been chosen.
* To make a long story short, we are trying to yield the current cpu
* strand during busy loops.
*/
#ifdef BUILD_VDSO
#define cpu_relax() asm volatile("\n99:\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
::: "memory")
#else /* ! BUILD_VDSO */
#define cpu_relax() asm volatile("\n99:\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
".section .pause_3insn_patch,\"ax\"\n\t"\
".word 99b\n\t" \
"wr %%g0, 128, %%asr27\n\t" \
"nop\n\t" \
"nop\n\t" \
".previous" \
::: "memory")
#endif
/* Prefetch support. This is tuned for UltraSPARC-III and later. /* Prefetch support. This is tuned for UltraSPARC-III and later.
* UltraSPARC-I will treat these as nops, and UltraSPARC-II has * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
* a shallower prefetch queue than later chips. * a shallower prefetch queue than later chips.

2
arch/sparc/include/asm/vdso.h
View File

@ -8,8 +8,6 @@
struct vdso_image { struct vdso_image {
void *data; void *data;
unsigned long size; /* Always a multiple of PAGE_SIZE */ unsigned long size; /* Always a multiple of PAGE_SIZE */
long sym_vvar_start; /* Negative offset to the vvar area */
}; };
#ifdef CONFIG_SPARC64 #ifdef CONFIG_SPARC64

10
arch/sparc/include/asm/vdso/clocksource.h Normal file
View File

@ -0,0 +1,10 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_VDSO_CLOCKSOURCE_H
#define __ASM_VDSO_CLOCKSOURCE_H
/* VDSO clocksources */
#define VDSO_ARCH_CLOCKMODES \
VDSO_CLOCKMODE_TICK, \
VDSO_CLOCKMODE_STICK
#endif /* __ASM_VDSO_CLOCKSOURCE_H */

184
arch/sparc/include/asm/vdso/gettimeofday.h Normal file
View File

@ -0,0 +1,184 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright 2006 Andi Kleen, SUSE Labs.
*/
#ifndef _ASM_SPARC_VDSO_GETTIMEOFDAY_H
#define _ASM_SPARC_VDSO_GETTIMEOFDAY_H
#include <uapi/linux/time.h>
#include <uapi/linux/unistd.h>
#include <vdso/align.h>
#include <vdso/clocksource.h>
#include <vdso/datapage.h>
#include <vdso/page.h>
#include <linux/types.h>
#ifdef CONFIG_SPARC64
static __always_inline u64 vread_tick(void)
{
u64 ret;
__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
return ret;
}
static __always_inline u64 vread_tick_stick(void)
{
u64 ret;
__asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
return ret;
}
#else
static __always_inline u64 vdso_shift_ns(u64 val, u32 amt)
{
u64 ret;
__asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
"srl %L1, 0, %L1\n\t"
"or %%g1, %L1, %%g1\n\t"
"srlx %%g1, %2, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret)
: "r" (val), "r" (amt)
: "g1");
return ret;
}
#define vdso_shift_ns vdso_shift_ns
static __always_inline u64 vread_tick(void)
{
register unsigned long long ret asm("o4");
__asm__ __volatile__("rd %%tick, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret));
return ret;
}
static __always_inline u64 vread_tick_stick(void)
{
register unsigned long long ret asm("o4");
__asm__ __volatile__("rd %%asr24, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret));
return ret;
}
#endif
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd)
{
if (likely(clock_mode == VDSO_CLOCKMODE_STICK))
return vread_tick_stick();
else
return vread_tick();
}
#ifdef CONFIG_SPARC64
#define SYSCALL_STRING \
"ta 0x6d;" \
"bcs,a 1f;" \
" sub %%g0, %%o0, %%o0;" \
"1:"
#else
#define SYSCALL_STRING \
"ta 0x10;" \
"bcs,a 1f;" \
" sub %%g0, %%o0, %%o0;" \
"1:"
#endif
#define SYSCALL_CLOBBERS \
"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \
"f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \
"f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \
"f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \
"f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \
"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \
"cc", "memory"
#ifdef CONFIG_SPARC64
static __always_inline
long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts)
{
register long num __asm__("g1") = __NR_clock_gettime;
register long o0 __asm__("o0") = clock;
register long o1 __asm__("o1") = (long) ts;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
#else /* !CONFIG_SPARC64 */
static __always_inline
long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts)
{
register long num __asm__("g1") = __NR_clock_gettime64;
register long o0 __asm__("o0") = clock;
register long o1 __asm__("o1") = (long) ts;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
static __always_inline
long clock_gettime32_fallback(clockid_t clock, struct old_timespec32 *ts)
{
register long num __asm__("g1") = __NR_clock_gettime;
register long o0 __asm__("o0") = clock;
register long o1 __asm__("o1") = (long) ts;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
#endif /* CONFIG_SPARC64 */
static __always_inline
long gettimeofday_fallback(struct __kernel_old_timeval *tv, struct timezone *tz)
{
register long num __asm__("g1") = __NR_gettimeofday;
register long o0 __asm__("o0") = (long) tv;
register long o1 __asm__("o1") = (long) tz;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
{
unsigned long ret;
/*
* SPARC does not support native PC-relative code relocations.
* Calculate the address manually, works for 32 and 64 bit code.
*/
__asm__ __volatile__(
"1:\n"
"call 3f\n" // Jump over the embedded data and set up %o7
"nop\n" // Delay slot
"2:\n"
".word vdso_u_time_data - .\n" // Embedded offset to external symbol
"3:\n"
"add %%o7, 2b - 1b, %%o7\n" // Point %o7 to the embedded offset
"ldsw [%%o7], %0\n" // Load the offset
"add %0, %%o7, %0\n" // Calculate the absolute address
: "=r" (ret)
:
: "o7");
return (const struct vdso_time_data *)ret;
}
#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data
#endif /* _ASM_SPARC_VDSO_GETTIMEOFDAY_H */

41
arch/sparc/include/asm/vdso/processor.h Normal file
View File

@ -0,0 +1,41 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_SPARC_VDSO_PROCESSOR_H
#define _ASM_SPARC_VDSO_PROCESSOR_H
#include <linux/compiler.h>
#if defined(__arch64__)
/* Please see the commentary in asm/backoff.h for a description of
* what these instructions are doing and how they have been chosen.
* To make a long story short, we are trying to yield the current cpu
* strand during busy loops.
*/
#ifdef BUILD_VDSO
#define cpu_relax() asm volatile("\n99:\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
::: "memory")
#else /* ! BUILD_VDSO */
#define cpu_relax() asm volatile("\n99:\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
"rd %%ccr, %%g0\n\t" \
".section .pause_3insn_patch,\"ax\"\n\t"\
".word 99b\n\t" \
"wr %%g0, 128, %%asr27\n\t" \
"nop\n\t" \
"nop\n\t" \
".previous" \
::: "memory")
#endif /* BUILD_VDSO */
#else /* ! __arch64__ */
#define cpu_relax() barrier()
#endif /* __arch64__ */
#endif /* _ASM_SPARC_VDSO_PROCESSOR_H */

10
arch/sparc/include/asm/vdso/vsyscall.h Normal file
View File

@ -0,0 +1,10 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_SPARC_VDSO_VSYSCALL_H
#define _ASM_SPARC_VDSO_VSYSCALL_H
#define __VDSO_PAGES 4
#include <asm-generic/vdso/vsyscall.h>
#endif /* _ASM_SPARC_VDSO_VSYSCALL_H */

75
arch/sparc/include/asm/vvar.h
View File

@ -1,75 +0,0 @@
/*
* Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
*/
#ifndef _ASM_SPARC_VVAR_DATA_H
#define _ASM_SPARC_VVAR_DATA_H
#include <asm/clocksource.h>
#include <asm/processor.h>
#include <asm/barrier.h>
#include <linux/time.h>
#include <linux/types.h>
struct vvar_data {
unsigned int seq;
int vclock_mode;
struct { /* extract of a clocksource struct */
u64 cycle_last;
u64 mask;
int mult;
int shift;
} clock;
/* open coded 'struct timespec' */
u64 wall_time_sec;
u64 wall_time_snsec;
u64 monotonic_time_snsec;
u64 monotonic_time_sec;
u64 monotonic_time_coarse_sec;
u64 monotonic_time_coarse_nsec;
u64 wall_time_coarse_sec;
u64 wall_time_coarse_nsec;
int tz_minuteswest;
int tz_dsttime;
};
extern struct vvar_data *vvar_data;
extern int vdso_fix_stick;
static inline unsigned int vvar_read_begin(const struct vvar_data *s)
{
unsigned int ret;
repeat:
ret = READ_ONCE(s->seq);
if (unlikely(ret & 1)) {
cpu_relax();
goto repeat;
}
smp_rmb(); /* Finish all reads before we return seq */
return ret;
}
static inline int vvar_read_retry(const struct vvar_data *s,
unsigned int start)
{
smp_rmb(); /* Finish all reads before checking the value of seq */
return unlikely(s->seq != start);
}
static inline void vvar_write_begin(struct vvar_data *s)
{
++s->seq;
smp_wmb(); /* Makes sure that increment of seq is reflected */
}
static inline void vvar_write_end(struct vvar_data *s)
{
smp_wmb(); /* Makes the value of seq current before we increment */
++s->seq;
}
#endif /* _ASM_SPARC_VVAR_DATA_H */

1
arch/sparc/kernel/Makefile
View File

@ -41,7 +41,6 @@ obj-$(CONFIG_SPARC32) += systbls_32.o
obj-y += time_$(BITS).o obj-y += time_$(BITS).o
obj-$(CONFIG_SPARC32) += windows.o obj-$(CONFIG_SPARC32) += windows.o
obj-y += cpu.o obj-y += cpu.o
obj-$(CONFIG_SPARC64) += vdso.o
obj-$(CONFIG_SPARC32) += devices.o obj-$(CONFIG_SPARC32) += devices.o
obj-y += ptrace_$(BITS).o obj-y += ptrace_$(BITS).o
obj-y += unaligned_$(BITS).o obj-y += unaligned_$(BITS).o

6
arch/sparc/kernel/time_64.c
View File

@ -838,14 +838,14 @@ void __init time_init_early(void)
if (tlb_type == spitfire) { if (tlb_type == spitfire) {
if (is_hummingbird()) { if (is_hummingbird()) {
init_tick_ops(&hbtick_operations); init_tick_ops(&hbtick_operations);
clocksource_tick.archdata.vclock_mode = VCLOCK_NONE; clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
} else { } else {
init_tick_ops(&tick_operations); init_tick_ops(&tick_operations);
clocksource_tick.archdata.vclock_mode = VCLOCK_TICK; clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_TICK;
} }
} else { } else {
init_tick_ops(&stick_operations); init_tick_ops(&stick_operations);
clocksource_tick.archdata.vclock_mode = VCLOCK_STICK; clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_STICK;
} }
} }

69
arch/sparc/kernel/vdso.c
View File

@ -1,69 +0,0 @@
/*
* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
* Copyright 2003 Andi Kleen, SuSE Labs.
*
* Thanks to hpa@transmeta.com for some useful hint.
* Special thanks to Ingo Molnar for his early experience with
* a different vsyscall implementation for Linux/IA32 and for the name.
*/
#include <linux/time.h>
#include <linux/timekeeper_internal.h>
#include <asm/vvar.h>
void update_vsyscall_tz(void)
{
if (unlikely(vvar_data == NULL))
return;
vvar_data->tz_minuteswest = sys_tz.tz_minuteswest;
vvar_data->tz_dsttime = sys_tz.tz_dsttime;
}
void update_vsyscall(struct timekeeper *tk)
{
struct vvar_data *vdata = vvar_data;
if (unlikely(vdata == NULL))
return;
vvar_write_begin(vdata);
vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode;
vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
vdata->clock.mask = tk->tkr_mono.mask;
vdata->clock.mult = tk->tkr_mono.mult;
vdata->clock.shift = tk->tkr_mono.shift;
vdata->wall_time_sec = tk->xtime_sec;
vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
vdata->monotonic_time_sec = tk->xtime_sec +
tk->wall_to_monotonic.tv_sec;
vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec +
(tk->wall_to_monotonic.tv_nsec <<
tk->tkr_mono.shift);
while (vdata->monotonic_time_snsec >=
(((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
vdata->monotonic_time_snsec -=
((u64)NSEC_PER_SEC) << tk->tkr_mono.shift;
vdata->monotonic_time_sec++;
}
vdata->wall_time_coarse_sec = tk->xtime_sec;
vdata->wall_time_coarse_nsec =
(long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
vdata->monotonic_time_coarse_sec =
vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec;
vdata->monotonic_time_coarse_nsec =
vdata->wall_time_coarse_nsec + tk->wall_to_monotonic.tv_nsec;
while (vdata->monotonic_time_coarse_nsec >= NSEC_PER_SEC) {
vdata->monotonic_time_coarse_nsec -= NSEC_PER_SEC;
vdata->monotonic_time_coarse_sec++;
}
vvar_write_end(vdata);
}

11
arch/sparc/vdso/Makefile
View File

@ -3,6 +3,9 @@
# Building vDSO images for sparc. # Building vDSO images for sparc.
# #
# Include the generic Makefile to check the built vDSO:
include $(srctree)/lib/vdso/Makefile.include
# files to link into the vdso # files to link into the vdso
vobjs-y := vdso-note.o vclock_gettime.o vobjs-y := vdso-note.o vclock_gettime.o
@ -90,6 +93,9 @@ KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING
KBUILD_CFLAGS_32 += -mv8plus KBUILD_CFLAGS_32 += -mv8plus
$(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) $(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32)
CHECKFLAGS_32 := $(filter-out -m64 -D__sparc_v9__ -D__arch64__, $(CHECKFLAGS)) -m32
$(obj)/vdso32.so.dbg: CHECKFLAGS = $(CHECKFLAGS_32)
$(obj)/vdso32.so.dbg: FORCE \ $(obj)/vdso32.so.dbg: FORCE \
$(obj)/vdso32/vdso32.lds \ $(obj)/vdso32/vdso32.lds \
$(obj)/vdso32/vclock_gettime.o \ $(obj)/vdso32/vclock_gettime.o \
@ -102,6 +108,7 @@ $(obj)/vdso32.so.dbg: FORCE \
quiet_cmd_vdso = VDSO $@ quiet_cmd_vdso = VDSO $@
cmd_vdso = $(LD) -nostdlib -o $@ \ cmd_vdso = $(LD) -nostdlib -o $@ \
$(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
-T $(filter %.lds,$^) $(filter %.o,$^) -T $(filter %.lds,$^) $(filter %.o,$^); \
$(cmd_vdso_check)
VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined -z noexecstack

418
arch/sparc/vdso/vclock_gettime.c
View File

@ -12,382 +12,48 @@
* Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
*/ */
#include <linux/kernel.h> #include <linux/compiler.h>
#include <linux/time.h> #include <linux/types.h>
#include <linux/string.h>
#include <asm/io.h> #include <vdso/gettime.h>
#include <asm/unistd.h>
#include <asm/timex.h> #include <asm/vdso/gettimeofday.h>
#include <asm/clocksource.h>
#include <asm/vvar.h> #include "../../../../lib/vdso/gettimeofday.c"
int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
}
int gettimeofday(struct __kernel_old_timeval *, struct timezone *)
__weak __alias(__vdso_gettimeofday);
#if defined(CONFIG_SPARC64)
int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_gettime(clock, ts);
}
int clock_gettime(clockid_t, struct __kernel_timespec *)
__weak __alias(__vdso_clock_gettime);
#ifdef CONFIG_SPARC64
#define SYSCALL_STRING \
"ta 0x6d;" \
"bcs,a 1f;" \
" sub %%g0, %%o0, %%o0;" \
"1:"
#else #else
#define SYSCALL_STRING \
"ta 0x10;" \ int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
"bcs,a 1f;" \ {
" sub %%g0, %%o0, %%o0;" \ return __cvdso_clock_gettime32(clock, ts);
"1:" }
int clock_gettime(clockid_t, struct old_timespec32 *)
__weak __alias(__vdso_clock_gettime);
int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_gettime(clock, ts);
}
int clock_gettime64(clockid_t, struct __kernel_timespec *)
__weak __alias(__vdso_clock_gettime64);
#endif #endif
#define SYSCALL_CLOBBERS \
"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \
"f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \
"f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \
"f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \
"f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \
"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \
"cc", "memory"
/*
* Compute the vvar page's address in the process address space, and return it
* as a pointer to the vvar_data.
*/
notrace static __always_inline struct vvar_data *get_vvar_data(void)
{
unsigned long ret;
/*
* vdso data page is the first vDSO page so grab the PC
* and move up a page to get to the data page.
*/
__asm__("rd %%pc, %0" : "=r" (ret));
ret &= ~(8192 - 1);
ret -= 8192;
return (struct vvar_data *) ret;
}
notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
{
register long num __asm__("g1") = __NR_clock_gettime;
register long o0 __asm__("o0") = clock;
register long o1 __asm__("o1") = (long) ts;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
register long num __asm__("g1") = __NR_gettimeofday;
register long o0 __asm__("o0") = (long) tv;
register long o1 __asm__("o1") = (long) tz;
__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
"0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
return o0;
}
#ifdef CONFIG_SPARC64
notrace static __always_inline u64 __shr64(u64 val, int amt)
{
return val >> amt;
}
notrace static __always_inline u64 vread_tick(void)
{
u64 ret;
__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
return ret;
}
notrace static __always_inline u64 vread_tick_stick(void)
{
u64 ret;
__asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
return ret;
}
#else
notrace static __always_inline u64 __shr64(u64 val, int amt)
{
u64 ret;
__asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
"srl %L1, 0, %L1\n\t"
"or %%g1, %L1, %%g1\n\t"
"srlx %%g1, %2, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret)
: "r" (val), "r" (amt)
: "g1");
return ret;
}
notrace static __always_inline u64 vread_tick(void)
{
register unsigned long long ret asm("o4");
__asm__ __volatile__("rd %%tick, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret));
return ret;
}
notrace static __always_inline u64 vread_tick_stick(void)
{
register unsigned long long ret asm("o4");
__asm__ __volatile__("rd %%asr24, %L0\n\t"
"srlx %L0, 32, %H0"
: "=r" (ret));
return ret;
}
#endif
notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
{
u64 v;
u64 cycles;
cycles = vread_tick();
v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
return v * vvar->clock.mult;
}
notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
{
u64 v;
u64 cycles;
cycles = vread_tick_stick();
v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
return v * vvar->clock.mult;
}
notrace static __always_inline int do_realtime(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
u64 ns;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->wall_time_sec;
ns = vvar->wall_time_snsec;
ns += vgetsns(vvar);
ns = __shr64(ns, vvar->clock.shift);
} while (unlikely(vvar_read_retry(vvar, seq)));
ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return 0;
}
notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
u64 ns;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->wall_time_sec;
ns = vvar->wall_time_snsec;
ns += vgetsns_stick(vvar);
ns = __shr64(ns, vvar->clock.shift);
} while (unlikely(vvar_read_retry(vvar, seq)));
ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return 0;
}
notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
u64 ns;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->monotonic_time_sec;
ns = vvar->monotonic_time_snsec;
ns += vgetsns(vvar);
ns = __shr64(ns, vvar->clock.shift);
} while (unlikely(vvar_read_retry(vvar, seq)));
ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return 0;
}
notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
u64 ns;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->monotonic_time_sec;
ns = vvar->monotonic_time_snsec;
ns += vgetsns_stick(vvar);
ns = __shr64(ns, vvar->clock.shift);
} while (unlikely(vvar_read_retry(vvar, seq)));
ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
ts->tv_nsec = ns;
return 0;
}
notrace static int do_realtime_coarse(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->wall_time_coarse_sec;
ts->tv_nsec = vvar->wall_time_coarse_nsec;
} while (unlikely(vvar_read_retry(vvar, seq)));
return 0;
}
notrace static int do_monotonic_coarse(struct vvar_data *vvar,
struct __kernel_old_timespec *ts)
{
unsigned long seq;
do {
seq = vvar_read_begin(vvar);
ts->tv_sec = vvar->monotonic_time_coarse_sec;
ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
} while (unlikely(vvar_read_retry(vvar, seq)));
return 0;
}
notrace int
__vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
{
struct vvar_data *vvd = get_vvar_data();
switch (clock) {
case CLOCK_REALTIME:
if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
break;
return do_realtime(vvd, ts);
case CLOCK_MONOTONIC:
if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
break;
return do_monotonic(vvd, ts);
case CLOCK_REALTIME_COARSE:
return do_realtime_coarse(vvd, ts);
case CLOCK_MONOTONIC_COARSE:
return do_monotonic_coarse(vvd, ts);
}
/*
* Unknown clock ID ? Fall back to the syscall.
*/
return vdso_fallback_gettime(clock, ts);
}
int
clock_gettime(clockid_t, struct __kernel_old_timespec *)
__attribute__((weak, alias("__vdso_clock_gettime")));
notrace int
__vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
{
struct vvar_data *vvd = get_vvar_data();
switch (clock) {
case CLOCK_REALTIME:
if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
break;
return do_realtime_stick(vvd, ts);
case CLOCK_MONOTONIC:
if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
break;
return do_monotonic_stick(vvd, ts);
case CLOCK_REALTIME_COARSE:
return do_realtime_coarse(vvd, ts);
case CLOCK_MONOTONIC_COARSE:
return do_monotonic_coarse(vvd, ts);
}
/*
* Unknown clock ID ? Fall back to the syscall.
*/
return vdso_fallback_gettime(clock, ts);
}
notrace int
__vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
struct vvar_data *vvd = get_vvar_data();
if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
if (likely(tv != NULL)) {
union tstv_t {
struct __kernel_old_timespec ts;
struct __kernel_old_timeval tv;
} *tstv = (union tstv_t *) tv;
do_realtime(vvd, &tstv->ts);
/*
* Assign before dividing to ensure that the division is
* done in the type of tv_usec, not tv_nsec.
*
* There cannot be > 1 billion usec in a second:
* do_realtime() has already distributed such overflow
* into tv_sec. So we can assign it to an int safely.
*/
tstv->tv.tv_usec = tstv->ts.tv_nsec;
tstv->tv.tv_usec /= 1000;
}
if (unlikely(tz != NULL)) {
/* Avoid memcpy. Some old compilers fail to inline it */
tz->tz_minuteswest = vvd->tz_minuteswest;
tz->tz_dsttime = vvd->tz_dsttime;
}
return 0;
}
return vdso_fallback_gettimeofday(tv, tz);
}
int
gettimeofday(struct __kernel_old_timeval *, struct timezone *)
__attribute__((weak, alias("__vdso_gettimeofday")));
notrace int
__vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
{
struct vvar_data *vvd = get_vvar_data();
if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
if (likely(tv != NULL)) {
union tstv_t {
struct __kernel_old_timespec ts;
struct __kernel_old_timeval tv;
} *tstv = (union tstv_t *) tv;
do_realtime_stick(vvd, &tstv->ts);
/*
* Assign before dividing to ensure that the division is
* done in the type of tv_usec, not tv_nsec.
*
* There cannot be > 1 billion usec in a second:
* do_realtime() has already distributed such overflow
* into tv_sec. So we can assign it to an int safely.
*/
tstv->tv.tv_usec = tstv->ts.tv_nsec;
tstv->tv.tv_usec /= 1000;
}
if (unlikely(tz != NULL)) {
/* Avoid memcpy. Some old compilers fail to inline it */
tz->tz_minuteswest = vvd->tz_minuteswest;
tz->tz_dsttime = vvd->tz_dsttime;
}
return 0;
}
return vdso_fallback_gettimeofday(tv, tz);
}

26
arch/sparc/vdso/vdso-layout.lds.S
View File

@ -4,15 +4,9 @@
* This script controls its layout. * This script controls its layout.
*/ */
#if defined(BUILD_VDSO64) #include <vdso/datapage.h>
# define SHDR_SIZE 64 #include <vdso/page.h>
#elif defined(BUILD_VDSO32) #include <asm/vdso/vsyscall.h>
# define SHDR_SIZE 40
#else
# error unknown VDSO target
#endif
#define NUM_FAKE_SHDRS 7
SECTIONS SECTIONS
{ {
@ -23,8 +17,7 @@ SECTIONS
* segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries * segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries
*/ */
vvar_start = . -8192; VDSO_VVAR_SYMS
vvar_data = vvar_start;
. = SIZEOF_HEADERS; . = SIZEOF_HEADERS;
@ -47,19 +40,8 @@ SECTIONS
*(.bss*) *(.bss*)
*(.dynbss*) *(.dynbss*)
*(.gnu.linkonce.b.*) *(.gnu.linkonce.b.*)
/*
* Ideally this would live in a C file: kept in here for
* compatibility with x86-64.
*/
VDSO_FAKE_SECTION_TABLE_START = .;
. = . + NUM_FAKE_SHDRS * SHDR_SIZE;
VDSO_FAKE_SECTION_TABLE_END = .;
} :text } :text
.fake_shstrtab : { *(.fake_shstrtab) } :text
.note : { *(.note.*) } :text :note .note : { *(.note.*) } :text :note
.eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr

2
arch/sparc/vdso/vdso.lds.S
View File

@ -18,10 +18,8 @@ VERSION {
global: global:
clock_gettime; clock_gettime;
__vdso_clock_gettime; __vdso_clock_gettime;
__vdso_clock_gettime_stick;
gettimeofday; gettimeofday;
__vdso_gettimeofday; __vdso_gettimeofday;
__vdso_gettimeofday_stick;
local: *; local: *;
}; };
} }

24
arch/sparc/vdso/vdso2c.c
View File

@ -58,28 +58,6 @@
const char *outfilename; const char *outfilename;
/* Symbols that we need in vdso2c. */
enum {
sym_vvar_start,
sym_VDSO_FAKE_SECTION_TABLE_START,
sym_VDSO_FAKE_SECTION_TABLE_END,
};
struct vdso_sym {
const char *name;
int export;
};
struct vdso_sym required_syms[] = {
[sym_vvar_start] = {"vvar_start", 1},
[sym_VDSO_FAKE_SECTION_TABLE_START] = {
"VDSO_FAKE_SECTION_TABLE_START", 0
},
[sym_VDSO_FAKE_SECTION_TABLE_END] = {
"VDSO_FAKE_SECTION_TABLE_END", 0
},
};
__attribute__((format(printf, 1, 2))) __attribute__((noreturn)) __attribute__((format(printf, 1, 2))) __attribute__((noreturn))
static void fail(const char *format, ...) static void fail(const char *format, ...)
{ {
@ -119,8 +97,6 @@ static void fail(const char *format, ...)
#define PUT_BE(x, val) \ #define PUT_BE(x, val) \
PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val)))) PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val))))
#define NSYMS ARRAY_SIZE(required_syms)
#define BITSFUNC3(name, bits, suffix) name##bits##suffix #define BITSFUNC3(name, bits, suffix) name##bits##suffix
#define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix) #define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix)
#define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, ) #define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, )

45
arch/sparc/vdso/vdso2c.h
View File

@ -17,11 +17,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
unsigned long mapping_size; unsigned long mapping_size;
int i; int i;
unsigned long j; unsigned long j;
ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr; ELF(Shdr) *symtab_hdr = NULL;
ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr;
ELF(Dyn) *dyn = 0, *dyn_end = 0; ELF(Dyn) *dyn = 0, *dyn_end = 0;
INT_BITS syms[NSYMS] = {};
ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff)); ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff));
/* Walk the segment table. */ /* Walk the segment table. */
@ -72,42 +70,6 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
if (!symtab_hdr) if (!symtab_hdr)
fail("no symbol table\n"); fail("no symbol table\n");
strtab_hdr = raw_addr + GET_BE(&hdr->e_shoff) +
GET_BE(&hdr->e_shentsize) * GET_BE(&symtab_hdr->sh_link);
/* Walk the symbol table */
for (i = 0;
i < GET_BE(&symtab_hdr->sh_size) / GET_BE(&symtab_hdr->sh_entsize);
i++) {
int k;
ELF(Sym) *sym = raw_addr + GET_BE(&symtab_hdr->sh_offset) +
GET_BE(&symtab_hdr->sh_entsize) * i;
const char *name = raw_addr + GET_BE(&strtab_hdr->sh_offset) +
GET_BE(&sym->st_name);
for (k = 0; k < NSYMS; k++) {
if (!strcmp(name, required_syms[k].name)) {
if (syms[k]) {
fail("duplicate symbol %s\n",
required_syms[k].name);
}
/*
* Careful: we use negative addresses, but
* st_value is unsigned, so we rely
* on syms[k] being a signed type of the
* correct width.
*/
syms[k] = GET_BE(&sym->st_value);
}
}
}
/* Validate mapping addresses. */
if (syms[sym_vvar_start] % 8192)
fail("vvar_begin must be a multiple of 8192\n");
if (!name) { if (!name) {
fwrite(stripped_addr, stripped_len, 1, outfile); fwrite(stripped_addr, stripped_len, 1, outfile);
return; return;
@ -133,10 +95,5 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name); fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name);
fprintf(outfile, "\t.data = raw_data,\n"); fprintf(outfile, "\t.data = raw_data,\n");
fprintf(outfile, "\t.size = %lu,\n", mapping_size); fprintf(outfile, "\t.size = %lu,\n", mapping_size);
for (i = 0; i < NSYMS; i++) {
if (required_syms[i].export && syms[i])
fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n",
required_syms[i].name, (int64_t)syms[i]);
}
fprintf(outfile, "};\n"); fprintf(outfile, "};\n");
} }

4
arch/sparc/vdso/vdso32/vdso32.lds.S
View File

@ -17,10 +17,10 @@ VERSION {
global: global:
clock_gettime; clock_gettime;
__vdso_clock_gettime; __vdso_clock_gettime;
__vdso_clock_gettime_stick; clock_gettime64;
__vdso_clock_gettime64;
gettimeofday; gettimeofday;
__vdso_gettimeofday; __vdso_gettimeofday;
__vdso_gettimeofday_stick;
local: *; local: *;
}; };
} }

274
arch/sparc/vdso/vma.c
View File

@ -16,17 +16,16 @@
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/random.h> #include <linux/random.h>
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/vdso_datastore.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/spitfire.h> #include <asm/spitfire.h>
#include <asm/vdso.h> #include <asm/vdso.h>
#include <asm/vvar.h>
#include <asm/page.h> #include <asm/page.h>
unsigned int __read_mostly vdso_enabled = 1; #include <vdso/datapage.h>
#include <asm/vdso/vsyscall.h>
static struct vm_special_mapping vvar_mapping = { unsigned int __read_mostly vdso_enabled = 1;
.name = "[vvar]"
};
#ifdef CONFIG_SPARC64 #ifdef CONFIG_SPARC64
static struct vm_special_mapping vdso_mapping64 = { static struct vm_special_mapping vdso_mapping64 = {
@ -40,207 +39,8 @@ static struct vm_special_mapping vdso_mapping32 = {
}; };
#endif #endif
struct vvar_data *vvar_data;
struct vdso_elfinfo32 {
Elf32_Ehdr *hdr;
Elf32_Sym *dynsym;
unsigned long dynsymsize;
const char *dynstr;
unsigned long text;
};
struct vdso_elfinfo64 {
Elf64_Ehdr *hdr;
Elf64_Sym *dynsym;
unsigned long dynsymsize;
const char *dynstr;
unsigned long text;
};
struct vdso_elfinfo {
union {
struct vdso_elfinfo32 elf32;
struct vdso_elfinfo64 elf64;
} u;
};
static void *one_section64(struct vdso_elfinfo64 *e, const char *name,
unsigned long *size)
{
const char *snames;
Elf64_Shdr *shdrs;
unsigned int i;
shdrs = (void *)e->hdr + e->hdr->e_shoff;
snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
for (i = 1; i < e->hdr->e_shnum; i++) {
if (!strcmp(snames+shdrs[i].sh_name, name)) {
if (size)
*size = shdrs[i].sh_size;
return (void *)e->hdr + shdrs[i].sh_offset;
}
}
return NULL;
}
static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e)
{
struct vdso_elfinfo64 *e = &_e->u.elf64;
e->hdr = image->data;
e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize);
e->dynstr = one_section64(e, ".dynstr", NULL);
if (!e->dynsym || !e->dynstr) {
pr_err("VDSO64: Missing symbol sections.\n");
return -ENODEV;
}
return 0;
}
static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name)
{
unsigned int i;
for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) {
Elf64_Sym *s = &e->dynsym[i];
if (s->st_name == 0)
continue;
if (!strcmp(e->dynstr + s->st_name, name))
return s;
}
return NULL;
}
static int patchsym64(struct vdso_elfinfo *_e, const char *orig,
const char *new)
{
struct vdso_elfinfo64 *e = &_e->u.elf64;
Elf64_Sym *osym = find_sym64(e, orig);
Elf64_Sym *nsym = find_sym64(e, new);
if (!nsym || !osym) {
pr_err("VDSO64: Missing symbols.\n");
return -ENODEV;
}
osym->st_value = nsym->st_value;
osym->st_size = nsym->st_size;
osym->st_info = nsym->st_info;
osym->st_other = nsym->st_other;
osym->st_shndx = nsym->st_shndx;
return 0;
}
static void *one_section32(struct vdso_elfinfo32 *e, const char *name,
unsigned long *size)
{
const char *snames;
Elf32_Shdr *shdrs;
unsigned int i;
shdrs = (void *)e->hdr + e->hdr->e_shoff;
snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset;
for (i = 1; i < e->hdr->e_shnum; i++) {
if (!strcmp(snames+shdrs[i].sh_name, name)) {
if (size)
*size = shdrs[i].sh_size;
return (void *)e->hdr + shdrs[i].sh_offset;
}
}
return NULL;
}
static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e)
{
struct vdso_elfinfo32 *e = &_e->u.elf32;
e->hdr = image->data;
e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize);
e->dynstr = one_section32(e, ".dynstr", NULL);
if (!e->dynsym || !e->dynstr) {
pr_err("VDSO32: Missing symbol sections.\n");
return -ENODEV;
}
return 0;
}
static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name)
{
unsigned int i;
for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) {
Elf32_Sym *s = &e->dynsym[i];
if (s->st_name == 0)
continue;
if (!strcmp(e->dynstr + s->st_name, name))
return s;
}
return NULL;
}
static int patchsym32(struct vdso_elfinfo *_e, const char *orig,
const char *new)
{
struct vdso_elfinfo32 *e = &_e->u.elf32;
Elf32_Sym *osym = find_sym32(e, orig);
Elf32_Sym *nsym = find_sym32(e, new);
if (!nsym || !osym) {
pr_err("VDSO32: Missing symbols.\n");
return -ENODEV;
}
osym->st_value = nsym->st_value;
osym->st_size = nsym->st_size;
osym->st_info = nsym->st_info;
osym->st_other = nsym->st_other;
osym->st_shndx = nsym->st_shndx;
return 0;
}
static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e,
bool elf64)
{
if (elf64)
return find_sections64(image, e);
else
return find_sections32(image, e);
}
static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig,
const char *new_target, bool elf64)
{
if (elf64)
return patchsym64(e, orig, new_target);
else
return patchsym32(e, orig, new_target);
}
static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64)
{
int err;
err = find_sections(image, e, elf64);
if (err)
return err;
err = patch_one_symbol(e,
"__vdso_gettimeofday",
"__vdso_gettimeofday_stick", elf64);
if (err)
return err;
return patch_one_symbol(e,
"__vdso_clock_gettime",
"__vdso_clock_gettime_stick", elf64);
return 0;
}
/* /*
* Allocate pages for the vdso and vvar, and copy in the vdso text from the * Allocate pages for the vdso and copy in the vdso text from the
* kernel image. * kernel image.
*/ */
static int __init init_vdso_image(const struct vdso_image *image, static int __init init_vdso_image(const struct vdso_image *image,
@ -248,16 +48,8 @@ static int __init init_vdso_image(const struct vdso_image *image,
bool elf64) bool elf64)
{ {
int cnpages = (image->size) / PAGE_SIZE; int cnpages = (image->size) / PAGE_SIZE;
struct page *dp, **dpp = NULL;
struct page *cp, **cpp = NULL; struct page *cp, **cpp = NULL;
struct vdso_elfinfo ei; int i;
int i, dnpages = 0;
if (tlb_type != spitfire) {
int err = stick_patch(image, &ei, elf64);
if (err)
return err;
}
/* /*
* First, the vdso text. This is initialied data, an integral number of * First, the vdso text. This is initialied data, an integral number of
@ -280,31 +72,6 @@ static int __init init_vdso_image(const struct vdso_image *image,
copy_page(page_address(cp), image->data + i * PAGE_SIZE); copy_page(page_address(cp), image->data + i * PAGE_SIZE);
} }
/*
* Now the vvar page. This is uninitialized data.
*/
if (vvar_data == NULL) {
dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1;
if (WARN_ON(dnpages != 1))
goto oom;
dpp = kzalloc_objs(struct page *, dnpages);
vvar_mapping.pages = dpp;
if (!dpp)
goto oom;
dp = alloc_page(GFP_KERNEL);
if (!dp)
goto oom;
dpp[0] = dp;
vvar_data = page_address(dp);
memset(vvar_data, 0, PAGE_SIZE);
vvar_data->seq = 0;
}
return 0; return 0;
oom: oom:
if (cpp != NULL) { if (cpp != NULL) {
@ -316,15 +83,6 @@ static int __init init_vdso_image(const struct vdso_image *image,
vdso_mapping->pages = NULL; vdso_mapping->pages = NULL;
} }
if (dpp != NULL) {
for (i = 0; i < dnpages; i++) {
if (dpp[i] != NULL)
__free_page(dpp[i]);
}
kfree(dpp);
vvar_mapping.pages = NULL;
}
pr_warn("Cannot allocate vdso\n"); pr_warn("Cannot allocate vdso\n");
vdso_enabled = 0; vdso_enabled = 0;
return -ENOMEM; return -ENOMEM;
@ -359,9 +117,12 @@ static unsigned long vdso_addr(unsigned long start, unsigned int len)
return start + (offset << PAGE_SHIFT); return start + (offset << PAGE_SHIFT);
} }
static_assert(VDSO_NR_PAGES == __VDSO_PAGES);
static int map_vdso(const struct vdso_image *image, static int map_vdso(const struct vdso_image *image,
struct vm_special_mapping *vdso_mapping) struct vm_special_mapping *vdso_mapping)
{ {
const size_t area_size = image->size + VDSO_NR_PAGES * PAGE_SIZE;
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long text_start, addr = 0; unsigned long text_start, addr = 0;
@ -374,23 +135,20 @@ static int map_vdso(const struct vdso_image *image,
* region is free. * region is free.
*/ */
if (current->flags & PF_RANDOMIZE) { if (current->flags & PF_RANDOMIZE) {
addr = get_unmapped_area(NULL, 0, addr = get_unmapped_area(NULL, 0, area_size, 0, 0);
image->size - image->sym_vvar_start,
0, 0);
if (IS_ERR_VALUE(addr)) { if (IS_ERR_VALUE(addr)) {
ret = addr; ret = addr;
goto up_fail; goto up_fail;
} }
addr = vdso_addr(addr, image->size - image->sym_vvar_start); addr = vdso_addr(addr, area_size);
} }
addr = get_unmapped_area(NULL, addr, addr = get_unmapped_area(NULL, addr, area_size, 0, 0);
image->size - image->sym_vvar_start, 0, 0);
if (IS_ERR_VALUE(addr)) { if (IS_ERR_VALUE(addr)) {
ret = addr; ret = addr;
goto up_fail; goto up_fail;
} }
text_start = addr - image->sym_vvar_start; text_start = addr + VDSO_NR_PAGES * PAGE_SIZE;
current->mm->context.vdso = (void __user *)text_start; current->mm->context.vdso = (void __user *)text_start;
/* /*
@ -408,11 +166,7 @@ static int map_vdso(const struct vdso_image *image,
goto up_fail; goto up_fail;
} }
vma = _install_special_mapping(mm, vma = vdso_install_vvar_mapping(mm, addr);
addr,
-image->sym_vvar_start,
VM_READ|VM_MAYREAD,
&vvar_mapping);
if (IS_ERR(vma)) { if (IS_ERR(vma)) {
ret = PTR_ERR(vma); ret = PTR_ERR(vma);

4
arch/x86/entry/vdso/vdso32/Makefile
View File

@ -15,6 +15,10 @@ flags-y := -DBUILD_VDSO32 -m32 -mregparm=0
flags-$(CONFIG_X86_64) += -include $(src)/fake_32bit_build.h flags-$(CONFIG_X86_64) += -include $(src)/fake_32bit_build.h
flags-remove-y := -m64 flags-remove-y := -m64
# Checker flags
CHECKFLAGS := $(subst -m64,-m32,$(CHECKFLAGS))
CHECKFLAGS := $(subst -D__x86_64__,-D__i386__,$(CHECKFLAGS))
# The location of this include matters! # The location of this include matters!
include $(src)/../common/Makefile.include include $(src)/../common/Makefile.include

16
drivers/char/random.c
View File

@ -56,11 +56,7 @@
#include <linux/sched/isolation.h> #include <linux/sched/isolation.h>
#include <crypto/chacha.h> #include <crypto/chacha.h>
#include <crypto/blake2s.h> #include <crypto/blake2s.h>
#ifdef CONFIG_VDSO_GETRANDOM
#include <vdso/getrandom.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>
#include <vdso/vsyscall.h>
#endif
#include <asm/archrandom.h> #include <asm/archrandom.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/irq.h> #include <asm/irq.h>
@ -269,7 +265,7 @@ static void crng_reseed(struct work_struct *work)
if (next_gen == ULONG_MAX) if (next_gen == ULONG_MAX)
++next_gen; ++next_gen;
WRITE_ONCE(base_crng.generation, next_gen); WRITE_ONCE(base_crng.generation, next_gen);
#ifdef CONFIG_VDSO_GETRANDOM
/* base_crng.generation's invalid value is ULONG_MAX, while /* base_crng.generation's invalid value is ULONG_MAX, while
* vdso_k_rng_data->generation's invalid value is 0, so add one to the * vdso_k_rng_data->generation's invalid value is 0, so add one to the
* former to arrive at the latter. Use smp_store_release so that this * former to arrive at the latter. Use smp_store_release so that this
@ -283,8 +279,9 @@ static void crng_reseed(struct work_struct *work)
* because the vDSO side only checks whether the value changed, without * because the vDSO side only checks whether the value changed, without
* actually using or interpreting the value. * actually using or interpreting the value.
*/ */
smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1); if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
#endif smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
if (!static_branch_likely(&crng_is_ready)) if (!static_branch_likely(&crng_is_ready))
crng_init = CRNG_READY; crng_init = CRNG_READY;
spin_unlock_irqrestore(&base_crng.lock, flags); spin_unlock_irqrestore(&base_crng.lock, flags);
@ -734,9 +731,8 @@ static void __cold _credit_init_bits(size_t bits)
if (system_dfl_wq) if (system_dfl_wq)
queue_work(system_dfl_wq, &set_ready); queue_work(system_dfl_wq, &set_ready);
atomic_notifier_call_chain(&random_ready_notifier, 0, NULL); atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
#ifdef CONFIG_VDSO_GETRANDOM if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
WRITE_ONCE(vdso_k_rng_data->is_ready, true); WRITE_ONCE(vdso_k_rng_data->is_ready, true);
#endif
wake_up_interruptible(&crng_init_wait); wake_up_interruptible(&crng_init_wait);
kill_fasync(&fasync, SIGIO, POLL_IN); kill_fasync(&fasync, SIGIO, POLL_IN);
pr_notice("crng init done\n"); pr_notice("crng init done\n");

9
include/asm-generic/bitsperlong.h
View File

@ -19,6 +19,15 @@
#error Inconsistent word size. Check asm/bitsperlong.h #error Inconsistent word size. Check asm/bitsperlong.h
#endif #endif
#if __CHAR_BIT__ * __SIZEOF_LONG__ != __BITS_PER_LONG
#error Inconsistent word size. Check asm/bitsperlong.h
#endif
#ifndef __ASSEMBLER__
_Static_assert(sizeof(long) * 8 == __BITS_PER_LONG,
"Inconsistent word size. Check asm/bitsperlong.h");
#endif
#ifndef BITS_PER_LONG_LONG #ifndef BITS_PER_LONG_LONG
#define BITS_PER_LONG_LONG 64 #define BITS_PER_LONG_LONG 64
#endif #endif

6
include/linux/clocksource.h
View File

@ -25,8 +25,7 @@ struct clocksource_base;
struct clocksource; struct clocksource;
struct module; struct module;
#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ #if defined(CONFIG_GENERIC_GETTIMEOFDAY)
defined(CONFIG_GENERIC_GETTIMEOFDAY)
#include <asm/clocksource.h> #include <asm/clocksource.h>
#endif #endif
@ -103,9 +102,6 @@ struct clocksource {
u32 shift; u32 shift;
u64 max_idle_ns; u64 max_idle_ns;
u32 maxadj; u32 maxadj;
#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA
struct arch_clocksource_data archdata;
#endif
u64 max_cycles; u64 max_cycles;
u64 max_raw_delta; u64 max_raw_delta;
const char *name; const char *name;

39
include/linux/time_namespace.h
View File

@ -8,6 +8,7 @@
#include <linux/ns_common.h> #include <linux/ns_common.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/time64.h> #include <linux/time64.h>
#include <linux/cleanup.h>
struct user_namespace; struct user_namespace;
extern struct user_namespace init_user_ns; extern struct user_namespace init_user_ns;
@ -25,7 +26,9 @@ struct time_namespace {
struct ucounts *ucounts; struct ucounts *ucounts;
struct ns_common ns; struct ns_common ns;
struct timens_offsets offsets; struct timens_offsets offsets;
#ifdef CONFIG_TIME_NS_VDSO
struct page *vvar_page; struct page *vvar_page;
#endif
/* If set prevents changing offsets after any task joined namespace. */ /* If set prevents changing offsets after any task joined namespace. */
bool frozen_offsets; bool frozen_offsets;
} __randomize_layout; } __randomize_layout;
@ -38,9 +41,6 @@ static inline struct time_namespace *to_time_ns(struct ns_common *ns)
return container_of(ns, struct time_namespace, ns); return container_of(ns, struct time_namespace, ns);
} }
void __init time_ns_init(void); void __init time_ns_init(void);
extern int vdso_join_timens(struct task_struct *task,
struct time_namespace *ns);
extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns);
static inline struct time_namespace *get_time_ns(struct time_namespace *ns) static inline struct time_namespace *get_time_ns(struct time_namespace *ns)
{ {
@ -53,7 +53,6 @@ struct time_namespace *copy_time_ns(u64 flags,
struct time_namespace *old_ns); struct time_namespace *old_ns);
void free_time_ns(struct time_namespace *ns); void free_time_ns(struct time_namespace *ns);
void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk); void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk);
struct page *find_timens_vvar_page(struct vm_area_struct *vma);
static inline void put_time_ns(struct time_namespace *ns) static inline void put_time_ns(struct time_namespace *ns)
{ {
@ -117,17 +116,6 @@ static inline void __init time_ns_init(void)
{ {
} }
static inline int vdso_join_timens(struct task_struct *task,
struct time_namespace *ns)
{
return 0;
}
static inline void timens_commit(struct task_struct *tsk,
struct time_namespace *ns)
{
}
static inline struct time_namespace *get_time_ns(struct time_namespace *ns) static inline struct time_namespace *get_time_ns(struct time_namespace *ns)
{ {
return NULL; return NULL;
@ -154,11 +142,6 @@ static inline void timens_on_fork(struct nsproxy *nsproxy,
return; return;
} }
static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma)
{
return NULL;
}
static inline void timens_add_monotonic(struct timespec64 *ts) { } static inline void timens_add_monotonic(struct timespec64 *ts) { }
static inline void timens_add_boottime(struct timespec64 *ts) { } static inline void timens_add_boottime(struct timespec64 *ts) { }
@ -175,4 +158,20 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim)
} }
#endif #endif
#ifdef CONFIG_TIME_NS_VDSO
extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns);
struct page *find_timens_vvar_page(struct vm_area_struct *vma);
#else /* !CONFIG_TIME_NS_VDSO */
static inline void timens_commit(struct task_struct *tsk, struct time_namespace *ns)
{
}
static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma)
{
return NULL;
}
#endif /* CONFIG_TIME_NS_VDSO */
DEFINE_FREE(time_ns, struct time_namespace *, if (_T) put_time_ns(_T))
#endif /* _LINUX_TIMENS_H */ #endif /* _LINUX_TIMENS_H */

6
include/linux/vdso_datastore.h
View File

@ -2,9 +2,15 @@
#ifndef _LINUX_VDSO_DATASTORE_H #ifndef _LINUX_VDSO_DATASTORE_H
#define _LINUX_VDSO_DATASTORE_H #define _LINUX_VDSO_DATASTORE_H
#ifdef CONFIG_HAVE_GENERIC_VDSO
#include <linux/mm_types.h> #include <linux/mm_types.h>
extern const struct vm_special_mapping vdso_vvar_mapping; extern const struct vm_special_mapping vdso_vvar_mapping;
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr); struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
void __init vdso_setup_data_pages(void);
#else /* !CONFIG_HAVE_GENERIC_VDSO */
static inline void vdso_setup_data_pages(void) { }
#endif /* CONFIG_HAVE_GENERIC_VDSO */
#endif /* _LINUX_VDSO_DATASTORE_H */ #endif /* _LINUX_VDSO_DATASTORE_H */

27
include/vdso/datapage.h
View File

@ -4,24 +4,16 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/compiler.h> #include <linux/types.h>
#include <uapi/linux/bits.h> #include <uapi/linux/bits.h>
#include <uapi/linux/time.h> #include <uapi/linux/time.h>
#include <uapi/linux/types.h>
#include <uapi/asm-generic/errno-base.h>
#include <vdso/align.h> #include <vdso/align.h>
#include <vdso/bits.h> #include <vdso/bits.h>
#include <vdso/cache.h> #include <vdso/cache.h>
#include <vdso/clocksource.h>
#include <vdso/ktime.h>
#include <vdso/limits.h>
#include <vdso/math64.h>
#include <vdso/page.h> #include <vdso/page.h>
#include <vdso/processor.h>
#include <vdso/time.h> #include <vdso/time.h>
#include <vdso/time32.h>
#include <vdso/time64.h>
#ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA #ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA
#include <asm/vdso/time_data.h> #include <asm/vdso/time_data.h>
@ -80,8 +72,8 @@ struct vdso_timestamp {
* @mask: clocksource mask * @mask: clocksource mask
* @mult: clocksource multiplier * @mult: clocksource multiplier
* @shift: clocksource shift * @shift: clocksource shift
* @basetime[clock_id]: basetime per clock_id * @basetime: basetime per clock_id
* @offset[clock_id]: time namespace offset per clock_id * @offset: time namespace offset per clock_id
* *
* See also struct vdso_time_data for basic access and ordering information as * See also struct vdso_time_data for basic access and ordering information as
* struct vdso_clock is used there. * struct vdso_clock is used there.
@ -184,17 +176,6 @@ enum vdso_pages {
VDSO_NR_PAGES VDSO_NR_PAGES
}; };
/*
* The generic vDSO implementation requires that gettimeofday.h
* provides:
* - __arch_get_hw_counter(): to get the hw counter based on the
* clock_mode.
* - gettimeofday_fallback(): fallback for gettimeofday.
* - clock_gettime_fallback(): fallback for clock_gettime.
* - clock_getres_fallback(): fallback for clock_getres.
*/
#include <asm/vdso/gettimeofday.h>
#else /* !__ASSEMBLY__ */ #else /* !__ASSEMBLY__ */
#ifdef CONFIG_VDSO_GETRANDOM #ifdef CONFIG_VDSO_GETRANDOM

31
include/vdso/helpers.h
View File

@ -6,6 +6,13 @@
#include <asm/barrier.h> #include <asm/barrier.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>
#include <vdso/processor.h>
#include <vdso/clocksource.h>
static __always_inline bool vdso_is_timens_clock(const struct vdso_clock *vc)
{
return IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS;
}
static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
{ {
@ -18,6 +25,28 @@ static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
return seq; return seq;
} }
/*
* Variant of vdso_read_begin() to handle VDSO_CLOCKMODE_TIMENS.
*
* Time namespace enabled tasks have a special VVAR page installed which has
* vc->seq set to 1 and vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non
* time namespace affected tasks this does not affect performance because if
* vc->seq is odd, i.e. a concurrent update is in progress the extra check for
* vc->clock_mode is just a few extra instructions while spin waiting for
* vc->seq to become even again.
*/
static __always_inline bool vdso_read_begin_timens(const struct vdso_clock *vc, u32 *seq)
{
while (unlikely((*seq = READ_ONCE(vc->seq)) & 1)) {
if (vdso_is_timens_clock(vc))
return true;
cpu_relax();
}
smp_rmb();
return false;
}
static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
u32 start) u32 start)
{ {
@ -25,7 +54,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
smp_rmb(); smp_rmb();
seq = READ_ONCE(vc->seq); seq = READ_ONCE(vc->seq);
return seq != start; return unlikely(seq != start);
} }
static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc) static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc)

4
init/Kconfig
View File

@ -1400,12 +1400,14 @@ config UTS_NS
config TIME_NS config TIME_NS
bool "TIME namespace" bool "TIME namespace"
depends on GENERIC_GETTIMEOFDAY
default y default y
help help
In this namespace boottime and monotonic clocks can be set. In this namespace boottime and monotonic clocks can be set.
The time will keep going with the same pace. The time will keep going with the same pace.
config TIME_NS_VDSO
def_bool TIME_NS && GENERIC_GETTIMEOFDAY
config IPC_NS config IPC_NS
bool "IPC namespace" bool "IPC namespace"
depends on (SYSVIPC || POSIX_MQUEUE) depends on (SYSVIPC || POSIX_MQUEUE)

2
init/main.c
View File

@ -106,6 +106,7 @@
#include <linux/ptdump.h> #include <linux/ptdump.h>
#include <linux/time_namespace.h> #include <linux/time_namespace.h>
#include <linux/unaligned.h> #include <linux/unaligned.h>
#include <linux/vdso_datastore.h>
#include <net/net_namespace.h> #include <net/net_namespace.h>
#include <asm/io.h> #include <asm/io.h>
@ -1127,6 +1128,7 @@ void start_kernel(void)
srcu_init(); srcu_init();
hrtimers_init(); hrtimers_init();
softirq_init(); softirq_init();
vdso_setup_data_pages();
timekeeping_init(); timekeeping_init();
time_init(); time_init();

4
kernel/time/Kconfig
View File

@ -9,10 +9,6 @@
config CLOCKSOURCE_WATCHDOG config CLOCKSOURCE_WATCHDOG
bool bool
# Architecture has extra clocksource data
config ARCH_CLOCKSOURCE_DATA
bool
# Architecture has extra clocksource init called from registration # Architecture has extra clocksource init called from registration
config ARCH_CLOCKSOURCE_INIT config ARCH_CLOCKSOURCE_INIT
bool bool

1
kernel/time/Makefile
View File

@ -30,5 +30,6 @@ obj-$(CONFIG_GENERIC_GETTIMEOFDAY) += vsyscall.o
obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o
obj-$(CONFIG_TEST_UDELAY) += test_udelay.o obj-$(CONFIG_TEST_UDELAY) += test_udelay.o
obj-$(CONFIG_TIME_NS) += namespace.o obj-$(CONFIG_TIME_NS) += namespace.o
obj-$(CONFIG_TIME_NS_VDSO) += namespace_vdso.o
obj-$(CONFIG_TEST_CLOCKSOURCE_WATCHDOG) += clocksource-wdtest.o obj-$(CONFIG_TEST_CLOCKSOURCE_WATCHDOG) += clocksource-wdtest.o
obj-$(CONFIG_TIME_KUNIT_TEST) += time_test.o obj-$(CONFIG_TIME_KUNIT_TEST) += time_test.o

203
kernel/time/namespace.c
View File

@ -18,8 +18,9 @@
#include <linux/cred.h> #include <linux/cred.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/cleanup.h>
#include <vdso/datapage.h> #include "namespace_internal.h"
ktime_t do_timens_ktime_to_host(clockid_t clockid, ktime_t tim, ktime_t do_timens_ktime_to_host(clockid_t clockid, ktime_t tim,
struct timens_offsets *ns_offsets) struct timens_offsets *ns_offsets)
@ -93,8 +94,8 @@ static struct time_namespace *clone_time_ns(struct user_namespace *user_ns,
if (!ns) if (!ns)
goto fail_dec; goto fail_dec;
ns->vvar_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); err = timens_vdso_alloc_vvar_page(ns);
if (!ns->vvar_page) if (err)
goto fail_free; goto fail_free;
err = ns_common_init(ns); err = ns_common_init(ns);
@ -109,7 +110,7 @@ static struct time_namespace *clone_time_ns(struct user_namespace *user_ns,
return ns; return ns;
fail_free_page: fail_free_page:
__free_page(ns->vvar_page); timens_vdso_free_vvar_page(ns);
fail_free: fail_free:
kfree(ns); kfree(ns);
fail_dec: fail_dec:
@ -138,117 +139,7 @@ struct time_namespace *copy_time_ns(u64 flags,
return clone_time_ns(user_ns, old_ns); return clone_time_ns(user_ns, old_ns);
} }
static struct timens_offset offset_from_ts(struct timespec64 off) DEFINE_MUTEX(timens_offset_lock);
{
struct timens_offset ret;
ret.sec = off.tv_sec;
ret.nsec = off.tv_nsec;
return ret;
}
/*
* A time namespace VVAR page has the same layout as the VVAR page which
* contains the system wide VDSO data.
*
* For a normal task the VVAR pages are installed in the normal ordering:
* VVAR
* PVCLOCK
* HVCLOCK
* TIMENS <- Not really required
*
* Now for a timens task the pages are installed in the following order:
* TIMENS
* PVCLOCK
* HVCLOCK
* VVAR
*
* The check for vdso_clock->clock_mode is in the unlikely path of
* the seq begin magic. So for the non-timens case most of the time
* 'seq' is even, so the branch is not taken.
*
* If 'seq' is odd, i.e. a concurrent update is in progress, the extra check
* for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the
* update to finish and for 'seq' to become even anyway.
*
* Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which
* enforces the time namespace handling path.
*/
static void timens_setup_vdso_clock_data(struct vdso_clock *vc,
struct time_namespace *ns)
{
struct timens_offset *offset = vc->offset;
struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic);
struct timens_offset boottime = offset_from_ts(ns->offsets.boottime);
vc->seq = 1;
vc->clock_mode = VDSO_CLOCKMODE_TIMENS;
offset[CLOCK_MONOTONIC] = monotonic;
offset[CLOCK_MONOTONIC_RAW] = monotonic;
offset[CLOCK_MONOTONIC_COARSE] = monotonic;
offset[CLOCK_BOOTTIME] = boottime;
offset[CLOCK_BOOTTIME_ALARM] = boottime;
}
struct page *find_timens_vvar_page(struct vm_area_struct *vma)
{
if (likely(vma->vm_mm == current->mm))
return current->nsproxy->time_ns->vvar_page;
/*
* VM_PFNMAP | VM_IO protect .fault() handler from being called
* through interfaces like /proc/$pid/mem or
* process_vm_{readv,writev}() as long as there's no .access()
* in special_mapping_vmops().
* For more details check_vma_flags() and __access_remote_vm()
*/
WARN(1, "vvar_page accessed remotely");
return NULL;
}
/*
* Protects possibly multiple offsets writers racing each other
* and tasks entering the namespace.
*/
static DEFINE_MUTEX(offset_lock);
static void timens_set_vvar_page(struct task_struct *task,
struct time_namespace *ns)
{
struct vdso_time_data *vdata;
struct vdso_clock *vc;
unsigned int i;
if (ns == &init_time_ns)
return;
/* Fast-path, taken by every task in namespace except the first. */
if (likely(ns->frozen_offsets))
return;
mutex_lock(&offset_lock);
/* Nothing to-do: vvar_page has been already initialized. */
if (ns->frozen_offsets)
goto out;
ns->frozen_offsets = true;
vdata = page_address(ns->vvar_page);
vc = vdata->clock_data;
for (i = 0; i < CS_BASES; i++)
timens_setup_vdso_clock_data(&vc[i], ns);
if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) {
for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++)
timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns);
}
out:
mutex_unlock(&offset_lock);
}
void free_time_ns(struct time_namespace *ns) void free_time_ns(struct time_namespace *ns)
{ {
@ -256,41 +147,39 @@ void free_time_ns(struct time_namespace *ns)
dec_time_namespaces(ns->ucounts); dec_time_namespaces(ns->ucounts);
put_user_ns(ns->user_ns); put_user_ns(ns->user_ns);
ns_common_free(ns); ns_common_free(ns);
__free_page(ns->vvar_page); timens_vdso_free_vvar_page(ns);
/* Concurrent nstree traversal depends on a grace period. */ /* Concurrent nstree traversal depends on a grace period. */
kfree_rcu(ns, ns.ns_rcu); kfree_rcu(ns, ns.ns_rcu);
} }
static struct ns_common *timens_get(struct task_struct *task) static struct ns_common *timens_get(struct task_struct *task)
{ {
struct time_namespace *ns = NULL; struct time_namespace *ns;
struct nsproxy *nsproxy; struct nsproxy *nsproxy;
task_lock(task); guard(task_lock)(task);
nsproxy = task->nsproxy; nsproxy = task->nsproxy;
if (nsproxy) { if (!nsproxy)
ns = nsproxy->time_ns; return NULL;
get_time_ns(ns);
}
task_unlock(task);
return ns ? &ns->ns : NULL; ns = nsproxy->time_ns;
get_time_ns(ns);
return &ns->ns;
} }
static struct ns_common *timens_for_children_get(struct task_struct *task) static struct ns_common *timens_for_children_get(struct task_struct *task)
{ {
struct time_namespace *ns = NULL; struct time_namespace *ns;
struct nsproxy *nsproxy; struct nsproxy *nsproxy;
task_lock(task); guard(task_lock)(task);
nsproxy = task->nsproxy; nsproxy = task->nsproxy;
if (nsproxy) { if (!nsproxy)
ns = nsproxy->time_ns_for_children; return NULL;
get_time_ns(ns);
}
task_unlock(task);
return ns ? &ns->ns : NULL; ns = nsproxy->time_ns_for_children;
get_time_ns(ns);
return &ns->ns;
} }
static void timens_put(struct ns_common *ns) static void timens_put(struct ns_common *ns)
@ -298,12 +187,6 @@ static void timens_put(struct ns_common *ns)
put_time_ns(to_time_ns(ns)); put_time_ns(to_time_ns(ns));
} }
void timens_commit(struct task_struct *tsk, struct time_namespace *ns)
{
timens_set_vvar_page(tsk, ns);
vdso_join_timens(tsk, ns);
}
static int timens_install(struct nsset *nsset, struct ns_common *new) static int timens_install(struct nsset *nsset, struct ns_common *new)
{ {
struct nsproxy *nsproxy = nsset->nsproxy; struct nsproxy *nsproxy = nsset->nsproxy;
@ -367,36 +250,33 @@ static void show_offset(struct seq_file *m, int clockid, struct timespec64 *ts)
void proc_timens_show_offsets(struct task_struct *p, struct seq_file *m) void proc_timens_show_offsets(struct task_struct *p, struct seq_file *m)
{ {
struct ns_common *ns; struct time_namespace *time_ns __free(time_ns) = NULL;
struct time_namespace *time_ns; struct ns_common *ns = timens_for_children_get(p);
ns = timens_for_children_get(p);
if (!ns) if (!ns)
return; return;
time_ns = to_time_ns(ns); time_ns = to_time_ns(ns);
show_offset(m, CLOCK_MONOTONIC, &time_ns->offsets.monotonic); show_offset(m, CLOCK_MONOTONIC, &time_ns->offsets.monotonic);
show_offset(m, CLOCK_BOOTTIME, &time_ns->offsets.boottime); show_offset(m, CLOCK_BOOTTIME, &time_ns->offsets.boottime);
put_time_ns(time_ns);
} }
int proc_timens_set_offset(struct file *file, struct task_struct *p, int proc_timens_set_offset(struct file *file, struct task_struct *p,
struct proc_timens_offset *offsets, int noffsets) struct proc_timens_offset *offsets, int noffsets)
{ {
struct ns_common *ns; struct time_namespace *time_ns __free(time_ns) = NULL;
struct time_namespace *time_ns; struct ns_common *ns = timens_for_children_get(p);
struct timespec64 tp; struct timespec64 tp;
int i, err; int i;
ns = timens_for_children_get(p);
if (!ns) if (!ns)
return -ESRCH; return -ESRCH;
time_ns = to_time_ns(ns); time_ns = to_time_ns(ns);
if (!file_ns_capable(file, time_ns->user_ns, CAP_SYS_TIME)) { if (!file_ns_capable(file, time_ns->user_ns, CAP_SYS_TIME))
put_time_ns(time_ns);
return -EPERM; return -EPERM;
}
for (i = 0; i < noffsets; i++) { for (i = 0; i < noffsets; i++) {
struct proc_timens_offset *off = &offsets[i]; struct proc_timens_offset *off = &offsets[i];
@ -409,15 +289,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
ktime_get_boottime_ts64(&tp); ktime_get_boottime_ts64(&tp);
break; break;
default: default:
err = -EINVAL; return -EINVAL;
goto out;
} }
err = -ERANGE;
if (off->val.tv_sec > KTIME_SEC_MAX || if (off->val.tv_sec > KTIME_SEC_MAX ||
off->val.tv_sec < -KTIME_SEC_MAX) off->val.tv_sec < -KTIME_SEC_MAX)
goto out; return -ERANGE;
tp = timespec64_add(tp, off->val); tp = timespec64_add(tp, off->val);
/* /*
@ -425,16 +302,13 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
* still unreachable. * still unreachable.
*/ */
if (tp.tv_sec < 0 || tp.tv_sec > KTIME_SEC_MAX / 2) if (tp.tv_sec < 0 || tp.tv_sec > KTIME_SEC_MAX / 2)
goto out; return -ERANGE;
} }
mutex_lock(&offset_lock); guard(mutex)(&timens_offset_lock);
if (time_ns->frozen_offsets) { if (time_ns->frozen_offsets)
err = -EACCES; return -EACCES;
goto out_unlock;
}
err = 0;
/* Don't report errors after this line */ /* Don't report errors after this line */
for (i = 0; i < noffsets; i++) { for (i = 0; i < noffsets; i++) {
struct proc_timens_offset *off = &offsets[i]; struct proc_timens_offset *off = &offsets[i];
@ -452,12 +326,7 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
*offset = off->val; *offset = off->val;
} }
out_unlock: return 0;
mutex_unlock(&offset_lock);
out:
put_time_ns(time_ns);
return err;
} }
const struct proc_ns_operations timens_operations = { const struct proc_ns_operations timens_operations = {

28
kernel/time/namespace_internal.h Normal file
View File

@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _TIME_NAMESPACE_INTERNAL_H
#define _TIME_NAMESPACE_INTERNAL_H
#include <linux/mutex.h>
struct time_namespace;
/*
* Protects possibly multiple offsets writers racing each other
* and tasks entering the namespace.
*/
extern struct mutex timens_offset_lock;
#ifdef CONFIG_TIME_NS_VDSO
int timens_vdso_alloc_vvar_page(struct time_namespace *ns);
void timens_vdso_free_vvar_page(struct time_namespace *ns);
#else /* !CONFIG_TIME_NS_VDSO */
static inline int timens_vdso_alloc_vvar_page(struct time_namespace *ns)
{
return 0;
}
static inline void timens_vdso_free_vvar_page(struct time_namespace *ns)
{
}
#endif /* CONFIG_TIME_NS_VDSO */
#endif /* _TIME_NAMESPACE_INTERNAL_H */

160
kernel/time/namespace_vdso.c Normal file
View File

@ -0,0 +1,160 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Author: Andrei Vagin <avagin@openvz.org>
* Author: Dmitry Safonov <dima@arista.com>
*/
#include <linux/cleanup.h>
#include <linux/mm.h>
#include <linux/time_namespace.h>
#include <linux/time.h>
#include <linux/vdso_datastore.h>
#include <vdso/clocksource.h>
#include <vdso/datapage.h>
#include "namespace_internal.h"
static struct timens_offset offset_from_ts(struct timespec64 off)
{
struct timens_offset ret;
ret.sec = off.tv_sec;
ret.nsec = off.tv_nsec;
return ret;
}
/*
* A time namespace VVAR page has the same layout as the VVAR page which
* contains the system wide VDSO data.
*
* For a normal task the VVAR pages are installed in the normal ordering:
* VVAR
* PVCLOCK
* HVCLOCK
* TIMENS <- Not really required
*
* Now for a timens task the pages are installed in the following order:
* TIMENS
* PVCLOCK
* HVCLOCK
* VVAR
*
* The check for vdso_clock->clock_mode is in the unlikely path of
* the seq begin magic. So for the non-timens case most of the time
* 'seq' is even, so the branch is not taken.
*
* If 'seq' is odd, i.e. a concurrent update is in progress, the extra check
* for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the
* update to finish and for 'seq' to become even anyway.
*
* Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which
* enforces the time namespace handling path.
*/
static void timens_setup_vdso_clock_data(struct vdso_clock *vc,
struct time_namespace *ns)
{
struct timens_offset *offset = vc->offset;
struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic);
struct timens_offset boottime = offset_from_ts(ns->offsets.boottime);
vc->seq = 1;
vc->clock_mode = VDSO_CLOCKMODE_TIMENS;
offset[CLOCK_MONOTONIC] = monotonic;
offset[CLOCK_MONOTONIC_RAW] = monotonic;
offset[CLOCK_MONOTONIC_COARSE] = monotonic;
offset[CLOCK_BOOTTIME] = boottime;
offset[CLOCK_BOOTTIME_ALARM] = boottime;
}
struct page *find_timens_vvar_page(struct vm_area_struct *vma)
{
if (likely(vma->vm_mm == current->mm))
return current->nsproxy->time_ns->vvar_page;
/*
* VM_PFNMAP | VM_IO protect .fault() handler from being called
* through interfaces like /proc/$pid/mem or
* process_vm_{readv,writev}() as long as there's no .access()
* in special_mapping_vmops().
* For more details check_vma_flags() and __access_remote_vm()
*/
WARN(1, "vvar_page accessed remotely");
return NULL;
}
static void timens_set_vvar_page(struct task_struct *task,
struct time_namespace *ns)
{
struct vdso_time_data *vdata;
struct vdso_clock *vc;
unsigned int i;
if (ns == &init_time_ns)
return;
/* Fast-path, taken by every task in namespace except the first. */
if (likely(ns->frozen_offsets))
return;
guard(mutex)(&timens_offset_lock);
/* Nothing to-do: vvar_page has been already initialized. */
if (ns->frozen_offsets)
return;
ns->frozen_offsets = true;
vdata = page_address(ns->vvar_page);
vc = vdata->clock_data;
for (i = 0; i < CS_BASES; i++)
timens_setup_vdso_clock_data(&vc[i], ns);
if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) {
for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++)
timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns);
}
}
/*
* The vvar page layout depends on whether a task belongs to the root or
* non-root time namespace. Whenever a task changes its namespace, the VVAR
* page tables are cleared and then they will be re-faulted with a
* corresponding layout.
* See also the comment near timens_setup_vdso_clock_data() for details.
*/
static int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
guard(mmap_read_lock)(mm);
for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_vvar_mapping))
zap_vma_pages(vma);
}
return 0;
}
void timens_commit(struct task_struct *tsk, struct time_namespace *ns)
{
timens_set_vvar_page(tsk, ns);
vdso_join_timens(tsk, ns);
}
int timens_vdso_alloc_vvar_page(struct time_namespace *ns)
{
ns->vvar_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
if (!ns->vvar_page)
return -ENOMEM;
return 0;
}
void timens_vdso_free_vvar_page(struct time_namespace *ns)
{
__free_page(ns->vvar_page);
}

122
lib/vdso/datastore.c
View File

@ -1,64 +1,92 @@
// SPDX-License-Identifier: GPL-2.0-only // SPDX-License-Identifier: GPL-2.0-only
#include <linux/linkage.h> #include <linux/gfp.h>
#include <linux/mmap_lock.h> #include <linux/init.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/time_namespace.h> #include <linux/time_namespace.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/vdso_datastore.h> #include <linux/vdso_datastore.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>
/* static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
* The vDSO data page.
*/
#ifdef CONFIG_GENERIC_GETTIMEOFDAY #ifdef CONFIG_GENERIC_GETTIMEOFDAY
static union { struct vdso_time_data *vdso_k_time_data __refdata =
struct vdso_time_data data; (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
u8 page[PAGE_SIZE];
} vdso_time_data_store __page_aligned_data; static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
#ifdef CONFIG_VDSO_GETRANDOM #ifdef CONFIG_VDSO_GETRANDOM
static union { struct vdso_rng_data *vdso_k_rng_data __refdata =
struct vdso_rng_data data; (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
u8 page[PAGE_SIZE];
} vdso_rng_data_store __page_aligned_data; static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
#endif /* CONFIG_VDSO_GETRANDOM */ #endif /* CONFIG_VDSO_GETRANDOM */
#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
static union { struct vdso_arch_data *vdso_k_arch_data __refdata =
struct vdso_arch_data data; (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
u8 page[VDSO_ARCH_DATA_SIZE];
} vdso_arch_data_store __page_aligned_data;
struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
void __init vdso_setup_data_pages(void)
{
unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
struct page *pages;
/*
* Allocate the data pages dynamically. SPARC does not support mapping
* static pages to be mapped into userspace.
* It is also a requirement for mlockall() support.
*
* Do not use folios. In time namespaces the pages are mapped in a different order
* to userspace, which is not handled by the folio optimizations in finish_fault().
*/
pages = alloc_pages(GFP_KERNEL, order);
if (!pages)
panic("Unable to allocate VDSO storage pages");
/* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
split_page(pages, order);
/* Move the data already written by other subsystems to the new pages */
memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
}
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf) struct vm_area_struct *vma, struct vm_fault *vmf)
{ {
struct page *timens_page = find_timens_vvar_page(vma); struct page *page, *timens_page;
unsigned long addr, pfn;
vm_fault_t err; timens_page = find_timens_vvar_page(vma);
switch (vmf->pgoff) { switch (vmf->pgoff) {
case VDSO_TIME_PAGE_OFFSET: case VDSO_TIME_PAGE_OFFSET:
if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); page = virt_to_page(vdso_k_time_data);
if (timens_page) { if (timens_page) {
/* /*
* Fault in VVAR page too, since it will be accessed * Fault in VVAR page too, since it will be accessed
* to get clock data anyway. * to get clock data anyway.
*/ */
unsigned long addr;
vm_fault_t err;
addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
err = vmf_insert_pfn(vma, addr, pfn); err = vmf_insert_page(vma, addr, page);
if (unlikely(err & VM_FAULT_ERROR)) if (unlikely(err & VM_FAULT_ERROR))
return err; return err;
pfn = page_to_pfn(timens_page); page = timens_page;
} }
break; break;
case VDSO_TIMENS_PAGE_OFFSET: case VDSO_TIMENS_PAGE_OFFSET:
@ -71,24 +99,25 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
*/ */
if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); page = virt_to_page(vdso_k_time_data);
break; break;
case VDSO_RNG_PAGE_OFFSET: case VDSO_RNG_PAGE_OFFSET:
if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data)); page = virt_to_page(vdso_k_rng_data);
break; break;
case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) + page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
vmf->pgoff - VDSO_ARCH_PAGES_START;
break; break;
default: default:
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
} }
return vmf_insert_pfn(vma, vmf->address, pfn); get_page(page);
vmf->page = page;
return 0;
} }
const struct vm_special_mapping vdso_vvar_mapping = { const struct vm_special_mapping vdso_vvar_mapping = {
@ -100,31 +129,6 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned
{ {
return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
VM_PFNMAP | VM_SEALED_SYSMAP, VM_MIXEDMAP | VM_SEALED_SYSMAP,
&vdso_vvar_mapping); &vdso_vvar_mapping);
} }
#ifdef CONFIG_TIME_NS
/*
* The vvar page layout depends on whether a task belongs to the root or
* non-root time namespace. Whenever a task changes its namespace, the VVAR
* page tables are cleared and then they will be re-faulted with a
* corresponding layout.
* See also the comment near timens_setup_vdso_clock_data() for details.
*/
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_vvar_mapping))
zap_vma_pages(vma);
}
mmap_read_unlock(mm);
return 0;
}
#endif

3
lib/vdso/getrandom.c
View File

@ -7,8 +7,11 @@
#include <linux/minmax.h> #include <linux/minmax.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>
#include <vdso/getrandom.h> #include <vdso/getrandom.h>
#include <vdso/limits.h>
#include <vdso/unaligned.h> #include <vdso/unaligned.h>
#include <asm/barrier.h>
#include <asm/vdso/getrandom.h> #include <asm/vdso/getrandom.h>
#include <uapi/linux/errno.h>
#include <uapi/linux/mman.h> #include <uapi/linux/mman.h>
#include <uapi/linux/random.h> #include <uapi/linux/random.h>

99
lib/vdso/gettimeofday.c
View File

@ -3,8 +3,25 @@
* Generic userspace implementations of gettimeofday() and similar. * Generic userspace implementations of gettimeofday() and similar.
*/ */
#include <vdso/auxclock.h> #include <vdso/auxclock.h>
#include <vdso/clocksource.h>
#include <vdso/datapage.h> #include <vdso/datapage.h>
#include <vdso/helpers.h> #include <vdso/helpers.h>
#include <vdso/ktime.h>
#include <vdso/limits.h>
#include <vdso/math64.h>
#include <vdso/time32.h>
#include <vdso/time64.h>
/*
* The generic vDSO implementation requires that gettimeofday.h
* provides:
* - __arch_get_hw_counter(): to get the hw counter based on the
* clock_mode.
* - gettimeofday_fallback(): fallback for gettimeofday.
* - clock_gettime_fallback(): fallback for clock_gettime.
* - clock_getres_fallback(): fallback for clock_getres.
*/
#include <asm/vdso/gettimeofday.h>
/* Bring in default accessors */ /* Bring in default accessors */
#include <vdso/vsyscall.h> #include <vdso/vsyscall.h>
@ -135,7 +152,7 @@ bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *
if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
return false; return false;
} while (unlikely(vdso_read_retry(vc, seq))); } while (vdso_read_retry(vc, seq));
/* Add the namespace offset */ /* Add the namespace offset */
sec += offs->sec; sec += offs->sec;
@ -158,28 +175,12 @@ bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
return false; return false;
do { do {
/* if (vdso_read_begin_timens(vc, &seq))
* Open coded function vdso_read_begin() to handle return do_hres_timens(vd, vc, clk, ts);
* VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a
* special VVAR page installed which has vc->seq set to 1 and
* vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time
* namespace affected tasks this does not affect performance
* because if vc->seq is odd, i.e. a concurrent update is in
* progress the extra check for vc->clock_mode is just a few
* extra instructions while spin waiting for vc->seq to become
* even again.
*/
while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) {
if (IS_ENABLED(CONFIG_TIME_NS) &&
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
return do_hres_timens(vd, vc, clk, ts);
cpu_relax();
}
smp_rmb();
if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
return false; return false;
} while (unlikely(vdso_read_retry(vc, seq))); } while (vdso_read_retry(vc, seq));
vdso_set_timespec(ts, sec, ns); vdso_set_timespec(ts, sec, ns);
@ -204,7 +205,7 @@ bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock
seq = vdso_read_begin(vc); seq = vdso_read_begin(vc);
sec = vdso_ts->sec; sec = vdso_ts->sec;
nsec = vdso_ts->nsec; nsec = vdso_ts->nsec;
} while (unlikely(vdso_read_retry(vc, seq))); } while (vdso_read_retry(vc, seq));
/* Add the namespace offset */ /* Add the namespace offset */
sec += offs->sec; sec += offs->sec;
@ -223,21 +224,12 @@ bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
u32 seq; u32 seq;
do { do {
/* if (vdso_read_begin_timens(vc, &seq))
* Open coded function vdso_read_begin() to handle return do_coarse_timens(vd, vc, clk, ts);
* VDSO_CLOCK_TIMENS. See comment in do_hres().
*/
while ((seq = READ_ONCE(vc->seq)) & 1) {
if (IS_ENABLED(CONFIG_TIME_NS) &&
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
return do_coarse_timens(vd, vc, clk, ts);
cpu_relax();
}
smp_rmb();
ts->tv_sec = vdso_ts->sec; ts->tv_sec = vdso_ts->sec;
ts->tv_nsec = vdso_ts->nsec; ts->tv_nsec = vdso_ts->nsec;
} while (unlikely(vdso_read_retry(vc, seq))); } while (vdso_read_retry(vc, seq));
return true; return true;
} }
@ -256,20 +248,12 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti
vc = &vd->aux_clock_data[idx]; vc = &vd->aux_clock_data[idx];
do { do {
/* if (vdso_read_begin_timens(vc, &seq)) {
* Open coded function vdso_read_begin() to handle vd = __arch_get_vdso_u_timens_data(vd);
* VDSO_CLOCK_TIMENS. See comment in do_hres(). vc = &vd->aux_clock_data[idx];
*/ /* Re-read from the real time data page */
while ((seq = READ_ONCE(vc->seq)) & 1) { continue;
if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {
vd = __arch_get_vdso_u_timens_data(vd);
vc = &vd->aux_clock_data[idx];
/* Re-read from the real time data page */
continue;
}
cpu_relax();
} }
smp_rmb();
/* Auxclock disabled? */ /* Auxclock disabled? */
if (vc->clock_mode == VDSO_CLOCKMODE_NONE) if (vc->clock_mode == VDSO_CLOCKMODE_NONE)
@ -277,7 +261,7 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti
if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))
return false; return false;
} while (unlikely(vdso_read_retry(vc, seq))); } while (vdso_read_retry(vc, seq));
vdso_set_timespec(ts, sec, ns); vdso_set_timespec(ts, sec, ns);
@ -313,7 +297,7 @@ __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
return do_hres(vd, vc, clock, ts); return do_hres(vd, vc, clock, ts);
} }
static __maybe_unused int static int
__cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
struct __kernel_timespec *ts) struct __kernel_timespec *ts)
{ {
@ -333,7 +317,7 @@ __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
} }
#ifdef BUILD_VDSO32 #ifdef BUILD_VDSO32
static __maybe_unused int static int
__cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
struct old_timespec32 *res) struct old_timespec32 *res)
{ {
@ -359,7 +343,7 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
} }
#endif /* BUILD_VDSO32 */ #endif /* BUILD_VDSO32 */
static __maybe_unused int static int
__cvdso_gettimeofday_data(const struct vdso_time_data *vd, __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
struct __kernel_old_timeval *tv, struct timezone *tz) struct __kernel_old_timeval *tv, struct timezone *tz)
{ {
@ -376,8 +360,7 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
} }
if (unlikely(tz != NULL)) { if (unlikely(tz != NULL)) {
if (IS_ENABLED(CONFIG_TIME_NS) && if (vdso_is_timens_clock(vc))
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
vd = __arch_get_vdso_u_timens_data(vd); vd = __arch_get_vdso_u_timens_data(vd);
tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
@ -394,14 +377,13 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
} }
#ifdef VDSO_HAS_TIME #ifdef VDSO_HAS_TIME
static __maybe_unused __kernel_old_time_t static __kernel_old_time_t
__cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
{ {
const struct vdso_clock *vc = vd->clock_data; const struct vdso_clock *vc = vd->clock_data;
__kernel_old_time_t t; __kernel_old_time_t t;
if (IS_ENABLED(CONFIG_TIME_NS) && if (vdso_is_timens_clock(vc)) {
vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {
vd = __arch_get_vdso_u_timens_data(vd); vd = __arch_get_vdso_u_timens_data(vd);
vc = vd->clock_data; vc = vd->clock_data;
} }
@ -432,8 +414,7 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc
if (!vdso_clockid_valid(clock)) if (!vdso_clockid_valid(clock))
return false; return false;
if (IS_ENABLED(CONFIG_TIME_NS) && if (vdso_is_timens_clock(vc))
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
vd = __arch_get_vdso_u_timens_data(vd); vd = __arch_get_vdso_u_timens_data(vd);
/* /*
@ -464,7 +445,7 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc
return true; return true;
} }
static __maybe_unused static
int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
struct __kernel_timespec *res) struct __kernel_timespec *res)
{ {
@ -484,7 +465,7 @@ int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
} }
#ifdef BUILD_VDSO32 #ifdef BUILD_VDSO32
static __maybe_unused int static int
__cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
struct old_timespec32 *res) struct old_timespec32 *res)
{ {

6
tools/testing/selftests/vDSO/Makefile
View File

@ -19,8 +19,6 @@ endif
include ../lib.mk include ../lib.mk
CFLAGS += $(TOOLS_INCLUDES)
CFLAGS_NOLIBC := -nostdlib -nostdinc -ffreestanding -fno-asynchronous-unwind-tables \ CFLAGS_NOLIBC := -nostdlib -nostdinc -ffreestanding -fno-asynchronous-unwind-tables \
-fno-stack-protector -include $(top_srcdir)/tools/include/nolibc/nolibc.h \ -fno-stack-protector -include $(top_srcdir)/tools/include/nolibc/nolibc.h \
-I$(top_srcdir)/tools/include/nolibc/ $(KHDR_INCLUDES) -I$(top_srcdir)/tools/include/nolibc/ $(KHDR_INCLUDES)
@ -28,13 +26,11 @@ CFLAGS_NOLIBC := -nostdlib -nostdinc -ffreestanding -fno-asynchronous-unwind-tab
$(OUTPUT)/vdso_test_gettimeofday: parse_vdso.c vdso_test_gettimeofday.c $(OUTPUT)/vdso_test_gettimeofday: parse_vdso.c vdso_test_gettimeofday.c
$(OUTPUT)/vdso_test_getcpu: parse_vdso.c vdso_test_getcpu.c $(OUTPUT)/vdso_test_getcpu: parse_vdso.c vdso_test_getcpu.c
$(OUTPUT)/vdso_test_abi: parse_vdso.c vdso_test_abi.c $(OUTPUT)/vdso_test_abi: parse_vdso.c vdso_test_abi.c
$(OUTPUT)/vdso_test_correctness: parse_vdso.c vdso_test_correctness.c
$(OUTPUT)/vdso_standalone_test_x86: vdso_standalone_test_x86.c parse_vdso.c | headers $(OUTPUT)/vdso_standalone_test_x86: vdso_standalone_test_x86.c parse_vdso.c | headers
$(OUTPUT)/vdso_standalone_test_x86: CFLAGS:=$(CFLAGS_NOLIBC) $(CFLAGS) $(OUTPUT)/vdso_standalone_test_x86: CFLAGS:=$(CFLAGS_NOLIBC) $(CFLAGS)
$(OUTPUT)/vdso_test_correctness: vdso_test_correctness.c
$(OUTPUT)/vdso_test_correctness: LDFLAGS += -ldl
$(OUTPUT)/vdso_test_getrandom: parse_vdso.c $(OUTPUT)/vdso_test_getrandom: parse_vdso.c
$(OUTPUT)/vdso_test_getrandom: CFLAGS += -isystem $(top_srcdir)/tools/include \ $(OUTPUT)/vdso_test_getrandom: CFLAGS += -isystem $(top_srcdir)/tools/include \
$(KHDR_INCLUDES) \ $(KHDR_INCLUDES) \

3
tools/testing/selftests/vDSO/parse_vdso.c
View File

@ -19,8 +19,7 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#include <limits.h> #include <limits.h>
#include <linux/auxvec.h> #include <elf.h>
#include <linux/elf.h>
#include "parse_vdso.h" #include "parse_vdso.h"

112
tools/testing/selftests/vDSO/vdso_test_correctness.c
View File

@ -11,28 +11,22 @@
#include <time.h> #include <time.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <sys/auxv.h>
#include <sys/syscall.h> #include <sys/syscall.h>
#include <dlfcn.h>
#include <string.h> #include <string.h>
#include <errno.h> #include <errno.h>
#include <sched.h> #include <sched.h>
#include <stdbool.h> #include <stdbool.h>
#include <limits.h> #include <limits.h>
#include "parse_vdso.h"
#include "vdso_config.h" #include "vdso_config.h"
#include "vdso_call.h" #include "vdso_call.h"
#include "kselftest.h" #include "kselftest.h"
static const char *version;
static const char **name; static const char **name;
#ifndef SYS_getcpu
# ifdef __x86_64__
# define SYS_getcpu 309
# else
# define SYS_getcpu 318
# endif
#endif
#ifndef __NR_clock_gettime64 #ifndef __NR_clock_gettime64
#define __NR_clock_gettime64 403 #define __NR_clock_gettime64 403
#endif #endif
@ -61,6 +55,10 @@ typedef long (*vgtod_t)(struct timeval *tv, struct timezone *tz);
vgtod_t vdso_gettimeofday; vgtod_t vdso_gettimeofday;
typedef time_t (*vtime_t)(__kernel_time_t *tloc);
vtime_t vdso_time;
typedef long (*getcpu_t)(unsigned *, unsigned *, void *); typedef long (*getcpu_t)(unsigned *, unsigned *, void *);
getcpu_t vgetcpu; getcpu_t vgetcpu;
@ -110,42 +108,39 @@ static void *vsyscall_getcpu(void)
static void fill_function_pointers(void) static void fill_function_pointers(void)
{ {
void *vdso = dlopen("linux-vdso.so.1", unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
if (!vdso) if (!sysinfo_ehdr) {
vdso = dlopen("linux-gate.so.1",
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
if (!vdso)
vdso = dlopen("linux-vdso32.so.1",
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
if (!vdso)
vdso = dlopen("linux-vdso64.so.1",
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
if (!vdso) {
printf("[WARN]\tfailed to find vDSO\n"); printf("[WARN]\tfailed to find vDSO\n");
return; return;
} }
vdso_getcpu = (getcpu_t)dlsym(vdso, name[4]); vdso_init_from_sysinfo_ehdr(sysinfo_ehdr);
vdso_getcpu = (getcpu_t)vdso_sym(version, name[4]);
if (!vdso_getcpu) if (!vdso_getcpu)
printf("Warning: failed to find getcpu in vDSO\n"); printf("Warning: failed to find getcpu in vDSO\n");
vgetcpu = (getcpu_t) vsyscall_getcpu(); vgetcpu = (getcpu_t) vsyscall_getcpu();
vdso_clock_gettime = (vgettime_t)dlsym(vdso, name[1]); vdso_clock_gettime = (vgettime_t)vdso_sym(version, name[1]);
if (!vdso_clock_gettime) if (!vdso_clock_gettime)
printf("Warning: failed to find clock_gettime in vDSO\n"); printf("Warning: failed to find clock_gettime in vDSO\n");
#if defined(VDSO_32BIT) #if defined(VDSO_32BIT)
vdso_clock_gettime64 = (vgettime64_t)dlsym(vdso, name[5]); vdso_clock_gettime64 = (vgettime64_t)vdso_sym(version, name[5]);
if (!vdso_clock_gettime64) if (!vdso_clock_gettime64)
printf("Warning: failed to find clock_gettime64 in vDSO\n"); printf("Warning: failed to find clock_gettime64 in vDSO\n");
#endif #endif
vdso_gettimeofday = (vgtod_t)dlsym(vdso, name[0]); vdso_gettimeofday = (vgtod_t)vdso_sym(version, name[0]);
if (!vdso_gettimeofday) if (!vdso_gettimeofday)
printf("Warning: failed to find gettimeofday in vDSO\n"); printf("Warning: failed to find gettimeofday in vDSO\n");
vdso_time = (vtime_t)vdso_sym(version, name[2]);
if (!vdso_time)
printf("Warning: failed to find time in vDSO\n");
} }
static long sys_getcpu(unsigned * cpu, unsigned * node, static long sys_getcpu(unsigned * cpu, unsigned * node,
@ -169,6 +164,16 @@ static inline int sys_gettimeofday(struct timeval *tv, struct timezone *tz)
return syscall(__NR_gettimeofday, tv, tz); return syscall(__NR_gettimeofday, tv, tz);
} }
static inline __kernel_old_time_t sys_time(__kernel_old_time_t *tloc)
{
#ifdef __NR_time
return syscall(__NR_time, tloc);
#else
errno = ENOSYS;
return -1;
#endif
}
static void test_getcpu(void) static void test_getcpu(void)
{ {
printf("[RUN]\tTesting getcpu...\n"); printf("[RUN]\tTesting getcpu...\n");
@ -412,10 +417,10 @@ static void test_gettimeofday(void)
return; return;
} }
printf("\t%llu.%06ld %llu.%06ld %llu.%06ld\n", printf("\t%llu.%06lld %llu.%06lld %llu.%06lld\n",
(unsigned long long)start.tv_sec, start.tv_usec, (unsigned long long)start.tv_sec, (long long)start.tv_usec,
(unsigned long long)vdso.tv_sec, vdso.tv_usec, (unsigned long long)vdso.tv_sec, (long long)vdso.tv_usec,
(unsigned long long)end.tv_sec, end.tv_usec); (unsigned long long)end.tv_sec, (long long)end.tv_usec);
if (!tv_leq(&start, &vdso) || !tv_leq(&vdso, &end)) { if (!tv_leq(&start, &vdso) || !tv_leq(&vdso, &end)) {
printf("[FAIL]\tTimes are out of sequence\n"); printf("[FAIL]\tTimes are out of sequence\n");
@ -435,8 +440,56 @@ static void test_gettimeofday(void)
VDSO_CALL(vdso_gettimeofday, 2, &vdso, NULL); VDSO_CALL(vdso_gettimeofday, 2, &vdso, NULL);
} }
static void test_time(void)
{
__kernel_old_time_t start, end, vdso_ret, vdso_param;
if (!vdso_time)
return;
printf("[RUN]\tTesting time...\n");
if (sys_time(&start) < 0) {
if (errno == -ENOSYS) {
printf("[SKIP]\tNo time() support\n");
} else {
printf("[FAIL]\tsys_time failed (%d)\n", errno);
nerrs++;
}
return;
}
vdso_ret = VDSO_CALL(vdso_time, 1, &vdso_param);
end = sys_time(NULL);
if (vdso_ret < 0 || end < 0) {
printf("[FAIL]\tvDSO returned %d, syscall errno=%d\n",
(int)vdso_ret, errno);
nerrs++;
return;
}
printf("\t%lld %lld %lld\n",
(long long)start,
(long long)vdso_ret,
(long long)end);
if (vdso_ret != vdso_param) {
printf("[FAIL]\tinconsistent return values: %lld %lld\n",
(long long)vdso_ret, (long long)vdso_param);
nerrs++;
return;
}
if (!(start <= vdso_ret) || !(vdso_ret <= end)) {
printf("[FAIL]\tTimes are out of sequence\n");
nerrs++;
}
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
version = versions[VDSO_VERSION];
name = (const char **)&names[VDSO_NAMES]; name = (const char **)&names[VDSO_NAMES];
fill_function_pointers(); fill_function_pointers();
@ -444,6 +497,7 @@ int main(int argc, char **argv)
test_clock_gettime(); test_clock_gettime();
test_clock_gettime64(); test_clock_gettime64();
test_gettimeofday(); test_gettimeofday();
test_time();
/* /*
* Test getcpu() last so that, if something goes wrong setting affinity, * Test getcpu() last so that, if something goes wrong setting affinity,

2
tools/testing/selftests/vDSO/vdso_test_gettimeofday.c
View File

@ -11,10 +11,8 @@
*/ */
#include <stdio.h> #include <stdio.h>
#ifndef NOLIBC
#include <sys/auxv.h> #include <sys/auxv.h>
#include <sys/time.h> #include <sys/time.h>
#endif
#include "kselftest.h" #include "kselftest.h"
#include "parse_vdso.h" #include "parse_vdso.h"