Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2017 Cavium, Inc
3 : : * Copyright(c) 2019 Arm Limited
4 : : */
5 : :
6 : : #ifndef _RTE_PAUSE_H_
7 : : #define _RTE_PAUSE_H_
8 : :
9 : : /**
10 : : * @file
11 : : *
12 : : * CPU pause operation.
13 : : */
14 : :
15 : : #include <stdint.h>
16 : : #include <assert.h>
17 : :
18 : : #include <rte_common.h>
19 : : #include <rte_atomic.h>
20 : : #include <rte_stdatomic.h>
21 : :
22 : : #ifdef __cplusplus
23 : : extern "C" {
24 : : #endif
25 : :
26 : : /**
27 : : * Pause CPU execution for a short while
28 : : *
29 : : * This call is intended for tight loops which poll a shared resource or wait
30 : : * for an event. A short pause within the loop may reduce the power consumption.
31 : : */
32 : : static inline void rte_pause(void);
33 : :
34 : : /**
35 : : * Wait for *addr to be updated with a 16-bit expected value, with a relaxed
36 : : * memory ordering model meaning the loads around this API can be reordered.
37 : : *
38 : : * @param addr
39 : : * A pointer to the memory location.
40 : : * @param expected
41 : : * A 16-bit expected value to be in the memory location.
42 : : * @param memorder
43 : : * Two different memory orders that can be specified:
44 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
45 : : */
46 : : static __rte_always_inline void
47 : : rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
48 : : rte_memory_order memorder);
49 : :
50 : : /**
51 : : * Wait for *addr to be updated with a 32-bit expected value, with a relaxed
52 : : * memory ordering model meaning the loads around this API can be reordered.
53 : : *
54 : : * @param addr
55 : : * A pointer to the memory location.
56 : : * @param expected
57 : : * A 32-bit expected value to be in the memory location.
58 : : * @param memorder
59 : : * Two different memory orders that can be specified:
60 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
61 : : */
62 : : static __rte_always_inline void
63 : : rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
64 : : rte_memory_order memorder);
65 : :
66 : : /**
67 : : * Wait for *addr to be updated with a 64-bit expected value, with a relaxed
68 : : * memory ordering model meaning the loads around this API can be reordered.
69 : : *
70 : : * @param addr
71 : : * A pointer to the memory location.
72 : : * @param expected
73 : : * A 64-bit expected value to be in the memory location.
74 : : * @param memorder
75 : : * Two different memory orders that can be specified:
76 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
77 : : */
78 : : static __rte_always_inline void
79 : : rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
80 : : rte_memory_order memorder);
81 : :
82 : : #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
83 : : static __rte_always_inline void
84 : : rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
85 : : rte_memory_order memorder)
86 : : {
87 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
88 : :
89 : 1800434 : while (rte_atomic_load_explicit((volatile __rte_atomic uint16_t *)addr, memorder)
90 [ + + + + : 1800434 : != expected)
- + ]
91 : : rte_pause();
92 : : }
93 : :
94 : : static __rte_always_inline void
95 : : rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
96 : : rte_memory_order memorder)
97 : : {
98 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
99 : :
100 : 155345220 : while (rte_atomic_load_explicit((volatile __rte_atomic uint32_t *)addr, memorder)
101 [ + + + + : 155345220 : != expected)
+ + + + +
+ + + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
102 : 0 : rte_pause();
103 : : }
104 : :
105 : : static __rte_always_inline void
106 : : rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
107 : : rte_memory_order memorder)
108 : : {
109 : : assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
110 : :
111 : 0 : while (rte_atomic_load_explicit((volatile __rte_atomic uint64_t *)addr, memorder)
112 [ # # ]: 0 : != expected)
113 : : rte_pause();
114 : : }
115 : :
116 : : /*
117 : : * Wait until *addr & mask makes the condition true. With a relaxed memory
118 : : * ordering model, the loads around this helper can be reordered.
119 : : *
120 : : * @param addr
121 : : * A pointer to the memory location.
122 : : * @param mask
123 : : * A mask of value bits in interest.
124 : : * @param cond
125 : : * A symbol representing the condition.
126 : : * @param expected
127 : : * An expected value to be in the memory location.
128 : : * @param memorder
129 : : * Two different memory orders that can be specified:
130 : : * rte_memory_order_acquire and rte_memory_order_relaxed.
131 : : */
132 : : #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
133 : : RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \
134 : : RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire && \
135 : : (memorder) != rte_memory_order_relaxed); \
136 : : typeof(*(addr)) expected_value = (expected); \
137 : : while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) \
138 : : cond expected_value)) \
139 : : rte_pause(); \
140 : : } while (0)
141 : : #endif /* ! RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */
142 : :
143 : : #ifdef __cplusplus
144 : : }
145 : : #endif
146 : :
147 : : #endif /* _RTE_PAUSE_H_ */
|
