Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : *
3 : : * Copyright (c) 2010-2017 Intel Corporation
4 : : * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
5 : : * All rights reserved.
6 : : * Derived from FreeBSD's bufring.h
7 : : * Used as BSD-3 Licensed with permission from Kip Macy.
8 : : */
9 : :
10 : : #ifndef _RTE_RING_GCC_PVT_H_
11 : : #define _RTE_RING_GCC_PVT_H_
12 : :
13 : : /**
14 : : * @file rte_ring_gcc_pvt.h
15 : : * It is not recommended to include this file directly,
16 : : * include <rte_ring.h> instead.
17 : : * Contains internal helper functions for MP/SP and MC/SC ring modes.
18 : : * For more information please refer to <rte_ring.h>.
19 : : */
20 : :
21 : :
22 : : /**
23 : : * @internal This is a helper function that moves the producer/consumer head
24 : : * optimized for single threaded case
25 : : *
26 : : * @param d
27 : : * A pointer to the headtail structure with head value to be moved
28 : : * @param s
29 : : * A pointer to the counter-part headtail structure. Note that this
30 : : * function only reads tail value from it
31 : : * @param capacity
32 : : * Either ring capacity value (for producer), or zero (for consumer)
33 : : * @param n
34 : : * The number of elements we want to move head value on
35 : : * @param behavior
36 : : * RTE_RING_QUEUE_FIXED: Move on a fixed number of items
37 : : * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible
38 : : * @param old_head
39 : : * Returns head value as it was before the move
40 : : * @param new_head
41 : : * Returns the new head value
42 : : * @param entries
43 : : * Returns the number of ring entries available BEFORE head was moved
44 : : * @return
45 : : * Actual number of objects the head was moved on
46 : : * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
47 : : */
48 : : static __rte_always_inline unsigned int
49 : : __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,
50 : : const struct rte_ring_headtail *s, uint32_t capacity,
51 : : unsigned int n,
52 : : enum rte_ring_queue_behavior behavior,
53 : : uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
54 : : {
55 : :
56 : 213358363 : *old_head = d->head;
57 : :
58 : : /* add rmb barrier to avoid load/load reorder in weak
59 : : * memory model. It is noop on x86
60 : : */
61 : 213358363 : rte_smp_rmb();
62 : :
63 : : /*
64 : : * The subtraction is done between two unsigned 32bits value
65 : : * (the result is always modulo 32 bits even if we have
66 : : * *old_head > s->tail). So 'entries' is always between 0
67 : : * and capacity (which is < size).
68 : : */
69 : 213358363 : *entries = capacity + s->tail - *old_head;
70 : :
71 : : /* check that we have enough room in ring */
72 [ + + + + : 213358363 : if (unlikely(n > *entries))
+ + + + +
+ + + + +
+ + + + +
- + + + -
- + + - +
- - - + -
- - - + -
- - - - -
- - + + -
- + - - -
- + - - +
+ - - + -
- - - + -
- - - - -
- - - - +
+ - - + +
- - + - -
- + + - -
+ + - - +
+ - - + -
- - + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
73 [ + + - - : 3 : n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
+ - - - ]
74 : :
75 [ + + + + : 21809428 : if (n == 0)
+ + + + +
+ + + + +
+ - + + -
- - + - -
+ - - - -
+ - - - -
- - - - +
- - - - +
- - + - -
- - + - -
- - - - -
- - - - +
- - + + -
- + - - -
+ - - - -
+ - - + +
- - + - -
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
76 : : return 0;
77 : :
78 : 196555112 : *new_head = *old_head + n;
79 [ + + + + : 196549592 : d->head = *new_head;
+ + + + +
+ + + + +
+ - + + +
- + + + -
+ - + - +
- - - - +
- - + + -
- + + - -
+ + - - -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
80 : 5524 : return n;
81 : : }
82 : :
83 : : /**
84 : : * @internal This is a helper function that moves the producer/consumer head
85 : : * for use in multi-thread safe path
86 : : *
87 : : * @param d
88 : : * A pointer to the headtail structure with head value to be moved
89 : : * @param s
90 : : * A pointer to the counter-part headtail structure. Note that this
91 : : * function only reads tail value from it
92 : : * @param capacity
93 : : * Either ring capacity value (for producer), or zero (for consumer)
94 : : * @param n
95 : : * The number of elements we want to move head value on
96 : : * @param behavior
97 : : * RTE_RING_QUEUE_FIXED: Move on a fixed number of items
98 : : * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible
99 : : * @param old_head
100 : : * Returns head value as it was before the move
101 : : * @param new_head
102 : : * Returns the new head value
103 : : * @param entries
104 : : * Returns the number of ring entries available BEFORE head was moved
105 : : * @return
106 : : * Actual number of objects the head was moved on
107 : : * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
108 : : */
109 : : static __rte_always_inline unsigned int
110 : : __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,
111 : : const struct rte_ring_headtail *s, uint32_t capacity,
112 : : unsigned int n, enum rte_ring_queue_behavior behavior,
113 : : uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
114 : : {
115 : : unsigned int max = n;
116 : : bool success;
117 : :
118 : : do {
119 : : /* Reset n to the initial burst count */
120 : : n = max;
121 : :
122 : 90660751 : *old_head = d->head;
123 : :
124 : : /* add fence to avoid load/load reorder in weak
125 : : * memory model. It is noop on x86
126 : : */
127 : 90660751 : __atomic_thread_fence(__ATOMIC_ACQUIRE);
128 : :
129 : : /*
130 : : * The subtraction is done between two unsigned 32bits value
131 : : * (the result is always modulo 32 bits even if we have
132 : : * *old_head > s->tail). So 'entries' is always between 0
133 : : * and capacity (which is < size).
134 : : */
135 : 90660751 : *entries = (capacity + s->tail - *old_head);
136 : :
137 : : /* check that we have enough room in ring */
138 [ + + + + : 90660751 : if (unlikely(n > *entries))
+ + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - +
+ + + + +
+ + + - +
- + + + +
+ + + + +
+ + + + -
+ - + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
139 : : n = (behavior == RTE_RING_QUEUE_FIXED) ?
140 [ + - + + ]: 5 : 0 : *entries;
141 : :
142 [ + - + + : 90362409 : if (n == 0)
+ - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - + -
+ - + + +
+ + - + -
+ - + - +
- + - + +
+ + + - +
- + - +
- ]
143 : : return 0;
144 : :
145 : 90619456 : *new_head = *old_head + n;
146 : :
147 : 90619456 : success = __sync_bool_compare_and_swap(
148 : 90619456 : (uint32_t *)(uintptr_t)&d->head,
149 : : *old_head, *new_head);
150 [ - + - + : 90619456 : } while (unlikely(!success));
- + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
151 : :
152 : : return n;
153 : : }
154 : :
155 : : #endif /* _RTE_RING_GCC_PVT_H_ */
|
