From 242955ac56f175322af040df29ab8cfb54a3b4f7 Mon Sep 17 00:00:00 2001 From: Julien Grall Date: Thu, 20 Feb 2020 20:54:40 +0000 Subject: [PATCH] xen/rwlock: Add missing memory barrier in the unlock path of rwlock The rwlock unlock paths are using atomic_sub() to release the lock. However the implementation of atomic_sub() rightfully doesn't contain a memory barrier. On Arm, this means a processor is allowed to re-order the memory access with the preceeding access. In other words, the unlock may be seen by another processor before all the memory accesses within the "critical" section. The rwlock paths already contains barrier indirectly, but they are not very useful without the counterpart in the unlock paths. The memory barriers are not necessary on x86 because loads/stores are not re-ordered with lock instructions. So add arch_lock_release_barrier() in the unlock paths that will only add memory barrier on Arm. Take the opportunity to document each lock paths explaining why a barrier is not necessary. This is XSA-314. Signed-off-by: Julien Grall --- xen/include/xen/rwlock.h | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/xen/include/xen/rwlock.h b/xen/include/xen/rwlock.h index 4d1b48c722..49ea9c010a 100644 --- a/xen/include/xen/rwlock.h +++ b/xen/include/xen/rwlock.h @@ -60,6 +60,10 @@ static inline int _read_trylock(rwlock_t *lock) if ( likely(_can_read_lock(cnts)) ) { cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts); + /* + * atomic_add_return() is a full barrier so no need for an + * arch_lock_acquire_barrier(). + */ if ( likely(_can_read_lock(cnts)) ) return 1; atomic_sub(_QR_BIAS, &lock->cnts); @@ -78,11 +82,19 @@ static inline void _read_lock(rwlock_t *lock) preempt_disable(); cnts = atomic_add_return(_QR_BIAS, &lock->cnts); + /* + * atomic_add_return() is a full barrier so no need for an + * arch_lock_acquire_barrier(). + */ if ( likely(_can_read_lock(cnts)) ) return; /* The slowpath will decrement the reader count, if necessary. */ queue_read_lock_slowpath(lock); + /* + * queue_read_lock_slowpath() is using spinlock and therefore is a + * full barrier. So no need for an arch_lock_acquire_barrier(). + */ } static inline void _read_lock_irq(rwlock_t *lock) @@ -106,6 +118,7 @@ static inline unsigned long _read_lock_irqsave(rwlock_t *lock) */ static inline void _read_unlock(rwlock_t *lock) { + arch_lock_release_barrier(); /* * Atomically decrement the reader count */ @@ -141,12 +154,21 @@ static inline unsigned int _write_lock_val(void) */ static inline void _write_lock(rwlock_t *lock) { - /* Optimize for the unfair lock case where the fair flag is 0. */ preempt_disable(); + /* + * Optimize for the unfair lock case where the fair flag is 0. + * + * atomic_cmpxchg() is a full barrier so no need for an + * arch_lock_acquire_barrier(). + */ if ( atomic_cmpxchg(&lock->cnts, 0, _write_lock_val()) == 0 ) return; queue_write_lock_slowpath(lock); + /* + * queue_write_lock_slowpath() is using spinlock and therefore is a + * full barrier. So no need for an arch_lock_acquire_barrier(). + */ } static inline void _write_lock_irq(rwlock_t *lock) @@ -183,12 +205,17 @@ static inline int _write_trylock(rwlock_t *lock) return 0; } + /* + * atomic_cmpxchg() is a full barrier so no need for an + * arch_lock_acquire_barrier(). + */ return 1; } static inline void _write_unlock(rwlock_t *lock) { ASSERT(_is_write_locked_by_me(atomic_read(&lock->cnts))); + arch_lock_release_barrier(); atomic_and(~(_QW_CPUMASK | _QW_WMASK), &lock->cnts); preempt_enable(); } -- 2.17.1