Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : *
3 : : * Copyright (c) 2010-2017 Intel Corporation
4 : : * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
5 : : * All rights reserved.
6 : : * Derived from FreeBSD's bufring.h
7 : : * Used as BSD-3 Licensed with permission from Kip Macy.
8 : : */
9 : :
10 : : #ifndef _RTE_RING_GENERIC_PVT_H_
11 : : #define _RTE_RING_GENERIC_PVT_H_
12 : :
13 : : static __rte_always_inline void
14 : : __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
15 : : uint32_t new_val, uint32_t single, uint32_t enqueue)
16 : : {
17 : : if (enqueue)
18 : 228408862 : rte_smp_wmb();
19 : : else
20 : 49161063 : rte_smp_rmb();
21 : : /*
22 : : * If there are other enqueues/dequeues in progress that preceded us,
23 : : * we need to wait for them to complete
24 : : */
25 : : if (!single)
26 : 89689773 : rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
27 : : rte_memory_order_relaxed);
28 : :
29 [ - + ]: 188132538 : ht->tail = new_val;
30 : 96445636 : }
31 : :
32 : : /**
33 : : * @internal This function updates the producer head for enqueue
34 : : *
35 : : * @param r
36 : : * A pointer to the ring structure
37 : : * @param is_sp
38 : : * Indicates whether multi-producer path is needed or not
39 : : * @param n
40 : : * The number of elements we will want to enqueue, i.e. how far should the
41 : : * head be moved
42 : : * @param behavior
43 : : * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
44 : : * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
45 : : * @param old_head
46 : : * Returns head value as it was before the move, i.e. where enqueue starts
47 : : * @param new_head
48 : : * Returns the current/new head value i.e. where enqueue finishes
49 : : * @param free_entries
50 : : * Returns the amount of free space in the ring BEFORE head was moved
51 : : * @return
52 : : * Actual number of objects enqueued.
53 : : * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
54 : : */
55 : : static __rte_always_inline unsigned int
56 : : __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,
57 : : unsigned int n, enum rte_ring_queue_behavior behavior,
58 : : uint32_t *old_head, uint32_t *new_head,
59 : : uint32_t *free_entries)
60 : : {
61 : 44726367 : const uint32_t capacity = r->capacity;
62 : : unsigned int max = n;
63 : : int success;
64 : :
65 : : do {
66 : : /* Reset n to the initial burst count */
67 : : n = max;
68 : :
69 : 228411675 : *old_head = r->prod.head;
70 : :
71 : : /* add rmb barrier to avoid load/load reorder in weak
72 : : * memory model. It is noop on x86
73 : : */
74 : 228411675 : rte_smp_rmb();
75 : :
76 : : /*
77 : : * The subtraction is done between two unsigned 32bits value
78 : : * (the result is always modulo 32 bits even if we have
79 : : * *old_head > cons_tail). So 'free_entries' is always between 0
80 : : * and capacity (which is < size).
81 : : */
82 : 228411675 : *free_entries = (capacity + r->cons.tail - *old_head);
83 : :
84 : : /* check that we have enough room in ring */
85 [ + + + + : 228411675 : if (unlikely(n > *free_entries))
+ + + + +
- + + + -
+ + + - +
+ - - + -
- - + + -
- - - - -
+ + - - -
- - - + -
- - - - -
- - + - -
- - - - -
- - - + +
- - + + +
+ + + - -
+ + + + +
+ - - + +
+ + + + -
- + + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
86 : : n = (behavior == RTE_RING_QUEUE_FIXED) ?
87 : : 0 : *free_entries;
88 : :
89 [ + + + + : 48488406 : if (n == 0)
+ - + + -
- + - - -
- + - - -
+ - - - +
- - + - -
- - - - -
- + - - -
- - - - -
- - + + -
- + + + +
+ - - - +
- + - + +
- - + + +
+ + - - -
+ - + - ]
90 : : return 0;
91 : :
92 : 228411622 : *new_head = *old_head + n;
93 : : if (is_sp) {
94 [ + + + - : 183685329 : r->prod.head = *new_head;
+ - + + +
+ + - + +
+ - + + -
- + + - -
- - + + -
- + + - -
+ + - - +
+ ]
95 : : success = 1;
96 : : } else
97 : 44726293 : success = rte_atomic32_cmpset((uint32_t *)(uintptr_t)&r->prod.head,
98 : : *old_head, *new_head);
99 [ - + - - : 44726293 : } while (unlikely(success == 0));
- - - - -
- - - - -
- - - - -
- - - - -
- + - + -
+ - + - +
- + - + -
+ ]
100 : : return n;
101 : : }
102 : :
103 : : /**
104 : : * @internal This function updates the consumer head for dequeue
105 : : *
106 : : * @param r
107 : : * A pointer to the ring structure
108 : : * @param is_sc
109 : : * Indicates whether multi-consumer path is needed or not
110 : : * @param n
111 : : * The number of elements we will want to dequeue, i.e. how far should the
112 : : * head be moved
113 : : * @param behavior
114 : : * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
115 : : * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
116 : : * @param old_head
117 : : * Returns head value as it was before the move, i.e. where dequeue starts
118 : : * @param new_head
119 : : * Returns the current/new head value i.e. where dequeue finishes
120 : : * @param entries
121 : : * Returns the number of entries in the ring BEFORE head was moved
122 : : * @return
123 : : * - Actual number of objects dequeued.
124 : : * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
125 : : */
126 : : static __rte_always_inline unsigned int
127 : : __rte_ring_move_cons_head(struct rte_ring *r, unsigned int is_sc,
128 : : unsigned int n, enum rte_ring_queue_behavior behavior,
129 : : uint32_t *old_head, uint32_t *new_head,
130 : : uint32_t *entries)
131 : : {
132 : : unsigned int max = n;
133 : : int success;
134 : :
135 : : /* move cons.head atomically */
136 : : do {
137 : : /* Restore n as it may change every loop */
138 : : n = max;
139 : :
140 : 66031851 : *old_head = r->cons.head;
141 : :
142 : : /* add rmb barrier to avoid load/load reorder in weak
143 : : * memory model. It is noop on x86
144 : : */
145 : 66031851 : rte_smp_rmb();
146 : :
147 : : /* The subtraction is done between two unsigned 32bits value
148 : : * (the result is always modulo 32 bits even if we have
149 : : * cons_head > prod_tail). So 'entries' is always between 0
150 : : * and size(ring)-1.
151 : : */
152 : 66031851 : *entries = (r->prod.tail - *old_head);
153 : :
154 : : /* Set the actual entries for dequeue */
155 [ + + + + : 45718833 : if (n > *entries)
+ + + + -
- + + - -
+ + - - +
- - - + +
- - - + -
- - + - -
- - - + -
- - + - +
- + - - -
+ - + # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
156 : : n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
157 : :
158 [ + + + + : 66031851 : if (unlikely(n == 0))
+ - + + -
- + + - -
+ + - - +
+ - - - +
- - + - -
- - - - -
+ - - - -
- - - + -
- - - - -
- + - - -
- - - - -
- - - + -
- - + - +
- + - - -
+ - + - +
- - - + -
+ - + - -
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
159 : : return 0;
160 : :
161 : 49163823 : *new_head = *old_head + n;
162 : : if (is_sc) {
163 : 4200343 : r->cons.head = *new_head;
164 : 4200343 : rte_smp_rmb();
165 : : success = 1;
166 : : } else {
167 : 44963480 : success = rte_atomic32_cmpset((uint32_t *)(uintptr_t)&r->cons.head,
168 : : *old_head, *new_head);
169 : : }
170 [ - + - - : 44963480 : } while (unlikely(success == 0));
- - - - -
- - - - -
- - - - -
- - - - -
- + - + -
+ - + - +
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
171 : : return n;
172 : : }
173 : :
174 : : #endif /* _RTE_RING_GENERIC_PVT_H_ */
|
