Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : *
3 : : * Copyright (c) 2010-2017 Intel Corporation
4 : : * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
5 : : * All rights reserved.
6 : : * Derived from FreeBSD's bufring.h
7 : : * Used as BSD-3 Licensed with permission from Kip Macy.
8 : : */
9 : :
10 : : #ifndef _RTE_RING_GENERIC_PVT_H_
11 : : #define _RTE_RING_GENERIC_PVT_H_
12 : :
13 : : /**
14 : : * @file rte_ring_generic_pvt.h
15 : : * It is not recommended to include this file directly,
16 : : * include <rte_ring.h> instead.
17 : : * Contains internal helper functions for MP/SP and MC/SC ring modes.
18 : : * For more information please refer to <rte_ring.h>.
19 : : */
20 : :
21 : : /**
22 : : * @internal This function updates tail values.
23 : : */
24 : : static __rte_always_inline void
25 : : __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,
26 : : uint32_t new_val, uint32_t single, uint32_t enqueue)
27 : : {
28 : : if (enqueue)
29 : 228818383 : rte_smp_wmb();
30 : : else
31 : 49571320 : rte_smp_rmb();
32 : : /*
33 : : * If there are other enqueues/dequeues in progress that preceded us,
34 : : * we need to wait for them to complete
35 : : */
36 [ + - + - : 9 : if (!single)
+ - - - +
- + - - -
- - ]
37 : 89694714 : rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
38 : : rte_memory_order_relaxed);
39 : :
40 [ - + ]: 188952315 : ht->tail = new_val;
41 : 97297632 : }
42 : :
43 : : /**
44 : : * @internal This is a helper function that moves the producer/consumer head
45 : : *
46 : : * @param d
47 : : * A pointer to the headtail structure with head value to be moved
48 : : * @param s
49 : : * A pointer to the counter-part headtail structure. Note that this
50 : : * function only reads tail value from it
51 : : * @param capacity
52 : : * Either ring capacity value (for producer), or zero (for consumer)
53 : : * @param is_st
54 : : * Indicates whether multi-thread safe path is needed or not
55 : : * @param n
56 : : * The number of elements we want to move head value on
57 : : * @param behavior
58 : : * RTE_RING_QUEUE_FIXED: Move on a fixed number of items
59 : : * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible
60 : : * @param old_head
61 : : * Returns head value as it was before the move
62 : : * @param new_head
63 : : * Returns the new head value
64 : : * @param entries
65 : : * Returns the number of ring entries available BEFORE head was moved
66 : : * @return
67 : : * Actual number of objects the head was moved on
68 : : * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only
69 : : */
70 : : static __rte_always_inline unsigned int
71 : : __rte_ring_headtail_move_head(struct rte_ring_headtail *d,
72 : : const struct rte_ring_headtail *s, uint32_t capacity,
73 : : unsigned int is_st, unsigned int n,
74 : : enum rte_ring_queue_behavior behavior,
75 : : uint32_t *old_head, uint32_t *new_head, uint32_t *entries)
76 : : {
77 : : unsigned int max = n;
78 : : int success;
79 : :
80 : : do {
81 : : /* Reset n to the initial burst count */
82 : : n = max;
83 : :
84 : 295308558 : *old_head = d->head;
85 : :
86 : : /* add rmb barrier to avoid load/load reorder in weak
87 : : * memory model. It is noop on x86
88 : : */
89 : 295308558 : rte_smp_rmb();
90 : :
91 : : /*
92 : : * The subtraction is done between two unsigned 32bits value
93 : : * (the result is always modulo 32 bits even if we have
94 : : * *old_head > s->tail). So 'entries' is always between 0
95 : : * and capacity (which is < size).
96 : : */
97 : 295308558 : *entries = (capacity + s->tail - *old_head);
98 : :
99 : : /* check that we have enough room in ring */
100 [ + + + + : 295308558 : if (unlikely(n > *entries))
+ + + + +
+ + + + +
+ + + + +
+ + - + +
+ - + + -
- + + - -
+ + - - +
- - - + +
- - + - -
- + + - -
+ - - - -
+ - - - -
- - - - -
- - - + +
- - - - -
- + - - -
- - - - -
+ - - - -
- - + + -
- - - - -
+ - - - -
- - - - +
- - - - -
- - - - -
+ + - - +
+ + + + +
- - + + +
+ + - - -
+ - + - +
+ - - + +
+ + + + -
- + + + +
+ + - - +
+ + + + -
- - + - +
- + + - -
+ + + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
101 : : n = (behavior == RTE_RING_QUEUE_FIXED) ?
102 : : 0 : *entries;
103 : :
104 [ + + + + : 114582501 : if (n == 0)
+ + + + +
+ + + + -
+ + + - +
+ - - + +
- - - + -
- - - - -
+ + - - -
- - - - +
- - - - -
- - - - -
- - + - -
- - - - -
- + - - -
- - - + -
- - - - -
- - + - -
- - - - -
- - - + -
- - - + +
- + + - -
+ + + + +
- - - + -
+ - + - -
- + - + -
+ - - - -
+ + - + +
- - + + +
+ + - - -
+ - + - +
- - - + -
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
105 : : return 0;
106 : :
107 : 278395223 : *new_head = *old_head + n;
108 [ - + - - : 9 : if (is_st) {
- + - + -
+ - - - -
- - - + -
+ - - -
- ]
109 [ + + + + : 188700509 : d->head = *new_head;
+ + + + +
+ + + + +
+ - + + +
- + + + -
+ - + - +
- - - - +
- - + + -
- + + - -
+ + - - -
- + + - -
+ + - - +
+ - - + +
- - + + -
- + + - -
+ + - - +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
110 : : success = 1;
111 : : } else
112 : 89694714 : success = rte_atomic32_cmpset(
113 : : (uint32_t *)(uintptr_t)&d->head,
114 : : *old_head, *new_head);
115 [ - + - + : 89694714 : } while (unlikely(success == 0));
- + - + -
+ - - - -
- - - + -
+ - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
116 : : return n;
117 : : }
118 : :
119 : : #endif /* _RTE_RING_GENERIC_PVT_H_ */
|
