DPDK  20.11.0
rte_mcslock.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Arm Limited
3  */
4 
5 #ifndef _RTE_MCSLOCK_H_
6 #define _RTE_MCSLOCK_H_
7 
22 #include <rte_lcore.h>
23 #include <rte_common.h>
24 #include <rte_pause.h>
25 
29 typedef struct rte_mcslock {
30  struct rte_mcslock *next;
31  int locked; /* 1 if the queue locked, 0 otherwise */
33 
45 static inline void
47 {
48  rte_mcslock_t *prev;
49 
50  /* Init me node */
51  __atomic_store_n(&me->locked, 1, __ATOMIC_RELAXED);
52  __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
53 
54  /* If the queue is empty, the exchange operation is enough to acquire
55  * the lock. Hence, the exchange operation requires acquire semantics.
56  * The store to me->next above should complete before the node is
57  * visible to other CPUs/threads. Hence, the exchange operation requires
58  * release semantics as well.
59  */
60  prev = __atomic_exchange_n(msl, me, __ATOMIC_ACQ_REL);
61  if (likely(prev == NULL)) {
62  /* Queue was empty, no further action required,
63  * proceed with lock taken.
64  */
65  return;
66  }
67  /* The store to me->next above should also complete before the node is
68  * visible to predecessor thread releasing the lock. Hence, the store
69  * prev->next also requires release semantics. Note that, for example,
70  * on ARM, the release semantics in the exchange operation is not
71  * strong as a release fence and is not sufficient to enforce the
72  * desired order here.
73  */
74  __atomic_store_n(&prev->next, me, __ATOMIC_RELEASE);
75 
76  /* The while-load of me->locked should not move above the previous
77  * store to prev->next. Otherwise it will cause a deadlock. Need a
78  * store-load barrier.
79  */
80  __atomic_thread_fence(__ATOMIC_ACQ_REL);
81  /* If the lock has already been acquired, it first atomically
82  * places the node at the end of the queue and then proceeds
83  * to spin on me->locked until the previous lock holder resets
84  * the me->locked using mcslock_unlock().
85  */
86  while (__atomic_load_n(&me->locked, __ATOMIC_ACQUIRE))
87  rte_pause();
88 }
89 
98 static inline void
100 {
101  /* Check if there are more nodes in the queue. */
102  if (likely(__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)) {
103  /* No, last member in the queue. */
104  rte_mcslock_t *save_me = __atomic_load_n(&me, __ATOMIC_RELAXED);
105 
106  /* Release the lock by setting it to NULL */
107  if (likely(__atomic_compare_exchange_n(msl, &save_me, NULL, 0,
108  __ATOMIC_RELEASE, __ATOMIC_RELAXED)))
109  return;
110 
111  /* Speculative execution would be allowed to read in the
112  * while-loop first. This has the potential to cause a
113  * deadlock. Need a load barrier.
114  */
115  __atomic_thread_fence(__ATOMIC_ACQUIRE);
116  /* More nodes added to the queue by other CPUs.
117  * Wait until the next pointer is set.
118  */
119  while (__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)
120  rte_pause();
121  }
122 
123  /* Pass lock to next waiter. */
124  __atomic_store_n(&me->next->locked, 0, __ATOMIC_RELEASE);
125 }
126 
137 static inline int
139 {
140  /* Init me node */
141  __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
142 
143  /* Try to lock */
144  rte_mcslock_t *expected = NULL;
145 
146  /* The lock can be taken only when the queue is empty. Hence,
147  * the compare-exchange operation requires acquire semantics.
148  * The store to me->next above should complete before the node
149  * is visible to other CPUs/threads. Hence, the compare-exchange
150  * operation requires release semantics as well.
151  */
152  return __atomic_compare_exchange_n(msl, &expected, me, 0,
153  __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
154 }
155 
164 static inline int
166 {
167  return (__atomic_load_n(&msl, __ATOMIC_RELAXED) != NULL);
168 }
169 
170 #endif /* _RTE_MCSLOCK_H_ */
#define likely(x)
static int rte_mcslock_trylock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:138
static int rte_mcslock_is_locked(rte_mcslock_t *msl)
Definition: rte_mcslock.h:165
struct rte_mcslock rte_mcslock_t
static void rte_mcslock_unlock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:99
static void rte_mcslock_lock(rte_mcslock_t **msl, rte_mcslock_t *me)
Definition: rte_mcslock.h:46
static void rte_pause(void)