Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2017 Cavium, Inc
3 : : * Copyright(c) 2019 Arm Limited
4 : : */
5 : :
6 : : #ifndef _RTE_PAUSE_H_
7 : : #define _RTE_PAUSE_H_
8 : :
9 : : /**
10 : : * @file
11 : : *
12 : : * CPU pause operation.
13 : : */
14 : :
15 : : #include <stdint.h>
16 : : #include <assert.h>
17 : :
18 : : #include <rte_common.h>
19 : : #include <rte_atomic.h>
20 : : #include <rte_stdatomic.h>
21 : :
22 : : /**
23 : : * Pause CPU execution for a short while
24 : : *
25 : : * This call is intended for tight loops which poll a shared resource or wait
26 : : * for an event. A short pause within the loop may reduce the power consumption.
27 : : */
28 : : static inline void rte_pause(void);
29 : :
30 : : /**
31 : : * Wait for *addr to be updated with a 16-bit expected value, with a relaxed
32 : : * memory ordering model meaning the loads around this API can be reordered.
33 : : *
34 : : * @param addr
35 : : * A pointer to the memory location.
36 : : * @param expected
37 : : * A 16-bit expected value to be in the memory location.
38 : : * @param memorder
39 : : * Two different memory orders that can be specified:
40 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
41 : : */
42 : : static __rte_always_inline void
43 : : rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
44 : : rte_memory_order memorder);
45 : :
46 : : /**
47 : : * Wait for *addr to be updated with a 32-bit expected value, with a relaxed
48 : : * memory ordering model meaning the loads around this API can be reordered.
49 : : *
50 : : * @param addr
51 : : * A pointer to the memory location.
52 : : * @param expected
53 : : * A 32-bit expected value to be in the memory location.
54 : : * @param memorder
55 : : * Two different memory orders that can be specified:
56 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
57 : : */
58 : : static __rte_always_inline void
59 : : rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
60 : : rte_memory_order memorder);
61 : :
62 : : /**
63 : : * Wait for *addr to be updated with a 64-bit expected value, with a relaxed
64 : : * memory ordering model meaning the loads around this API can be reordered.
65 : : *
66 : : * @param addr
67 : : * A pointer to the memory location.
68 : : * @param expected
69 : : * A 64-bit expected value to be in the memory location.
70 : : * @param memorder
71 : : * Two different memory orders that can be specified:
72 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
73 : : */
74 : : static __rte_always_inline void
75 : : rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
76 : : rte_memory_order memorder);
77 : :
78 : : #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
79 : : static __rte_always_inline void
80 : : rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
81 : : rte_memory_order memorder)
82 : : {
83 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
84 : :
85 : 680806 : while (rte_atomic_load_explicit((volatile __rte_atomic uint16_t *)addr, memorder)
86 [ + + + + : 680806 : != expected)
- + ]
87 : : rte_pause();
88 : : }
89 : :
90 : : static __rte_always_inline void
91 : : rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
92 : : rte_memory_order memorder)
93 : : {
94 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
95 : :
96 : 157462349 : while (rte_atomic_load_explicit((volatile __rte_atomic uint32_t *)addr, memorder)
97 [ + + + + : 157462349 : != expected)
+ + + + +
+ + + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ ]
98 : 0 : rte_pause();
99 : : }
100 : :
101 : : static __rte_always_inline void
102 : : rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
103 : : rte_memory_order memorder)
104 : : {
105 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
106 : :
107 : 0 : while (rte_atomic_load_explicit((volatile __rte_atomic uint64_t *)addr, memorder)
108 [ # # ]: 0 : != expected)
109 : : rte_pause();
110 : : }
111 : :
112 : : /*
113 : : * Wait until *addr & mask makes the condition true. With a relaxed memory
114 : : * ordering model, the loads around this helper can be reordered.
115 : : *
116 : : * @param addr
117 : : * A pointer to the memory location.
118 : : * @param mask
119 : : * A mask of value bits in interest.
120 : : * @param cond
121 : : * A symbol representing the condition.
122 : : * @param expected
123 : : * An expected value to be in the memory location.
124 : : * @param memorder
125 : : * Two different memory orders that can be specified:
126 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
127 : : */
128 : : #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
129 : : RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \
130 : : RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire && \
131 : : (memorder) != rte_memory_order_relaxed); \
132 : : typeof(*(addr)) expected_value = (expected); \
133 : : while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) \
134 : : cond expected_value)) \
135 : : rte_pause(); \
136 : : } while (0)
137 : : #endif /* ! RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */
138 : :
139 : : #endif /* _RTE_PAUSE_H_ */
|