Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright 2014 6WIND S.A.
4 : : */
5 : :
6 : : #ifndef _RTE_MBUF_H_
7 : : #define _RTE_MBUF_H_
8 : :
9 : : /**
10 : : * @file
11 : : * RTE Mbuf
12 : : *
13 : : * The mbuf library provides the ability to create and destroy buffers
14 : : * that may be used by the RTE application to store message
15 : : * buffers. The message buffers are stored in a mempool, using the
16 : : * RTE mempool library.
17 : : *
18 : : * The preferred way to create a mbuf pool is to use
19 : : * rte_pktmbuf_pool_create(). However, in some situations, an
20 : : * application may want to have more control (ex: populate the pool with
21 : : * specific memory), in this case it is possible to use functions from
22 : : * rte_mempool. See how rte_pktmbuf_pool_create() is implemented for
23 : : * details.
24 : : *
25 : : * This library provides an API to allocate/free packet mbufs, which are
26 : : * used to carry network packets.
27 : : *
28 : : * To understand the concepts of packet buffers or mbufs, you
29 : : * should read "TCP/IP Illustrated, Volume 2: The Implementation,
30 : : * Addison-Wesley, 1995, ISBN 0-201-63354-X from Richard Stevens"
31 : : * http://www.kohala.com/start/tcpipiv2.html
32 : : */
33 : :
34 : : #include <stdint.h>
35 : :
36 : : #include <rte_common.h>
37 : : #include <rte_config.h>
38 : : #include <rte_mempool.h>
39 : : #include <rte_prefetch.h>
40 : : #include <rte_branch_prediction.h>
41 : : #include <rte_mbuf_ptype.h>
42 : : #include <rte_mbuf_core.h>
43 : :
44 : : #ifdef __cplusplus
45 : : extern "C" {
46 : : #endif
47 : :
48 : : /**
49 : : * Get the name of a RX offload flag
50 : : *
51 : : * @param mask
52 : : * The mask describing the flag.
53 : : * @return
54 : : * The name of this flag, or NULL if it's not a valid RX flag.
55 : : */
56 : : const char *rte_get_rx_ol_flag_name(uint64_t mask);
57 : :
58 : : /**
59 : : * Dump the list of RX offload flags in a buffer
60 : : *
61 : : * @param mask
62 : : * The mask describing the RX flags.
63 : : * @param buf
64 : : * The output buffer.
65 : : * @param buflen
66 : : * The length of the buffer.
67 : : * @return
68 : : * 0 on success, (-1) on error.
69 : : */
70 : : int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
71 : :
72 : : /**
73 : : * Get the name of a TX offload flag
74 : : *
75 : : * @param mask
76 : : * The mask describing the flag. Usually only one bit must be set.
77 : : * Several bits can be given if they belong to the same mask.
78 : : * Ex: RTE_MBUF_F_TX_L4_MASK.
79 : : * @return
80 : : * The name of this flag, or NULL if it's not a valid TX flag.
81 : : */
82 : : const char *rte_get_tx_ol_flag_name(uint64_t mask);
83 : :
84 : : /**
85 : : * Dump the list of TX offload flags in a buffer
86 : : *
87 : : * @param mask
88 : : * The mask describing the TX flags.
89 : : * @param buf
90 : : * The output buffer.
91 : : * @param buflen
92 : : * The length of the buffer.
93 : : * @return
94 : : * 0 on success, (-1) on error.
95 : : */
96 : : int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
97 : :
98 : : /**
99 : : * Prefetch the first part of the mbuf
100 : : *
101 : : * The first 64 bytes of the mbuf corresponds to fields that are used early
102 : : * in the receive path. If the cache line of the architecture is higher than
103 : : * 64B, the second part will also be prefetched.
104 : : *
105 : : * @param m
106 : : * The pointer to the mbuf.
107 : : */
108 : : static inline void
109 : : rte_mbuf_prefetch_part1(struct rte_mbuf *m)
110 : : {
111 : : rte_prefetch0(m);
112 : 0 : }
113 : :
114 : : /**
115 : : * Prefetch the second part of the mbuf
116 : : *
117 : : * The next 64 bytes of the mbuf corresponds to fields that are used in the
118 : : * transmit path. If the cache line of the architecture is higher than 64B,
119 : : * this function does nothing as it is expected that the full mbuf is
120 : : * already in cache.
121 : : *
122 : : * @param m
123 : : * The pointer to the mbuf.
124 : : */
125 : : static inline void
126 : : rte_mbuf_prefetch_part2(struct rte_mbuf *m)
127 : : {
128 : : #if RTE_CACHE_LINE_SIZE == 64
129 : 0 : rte_prefetch0(RTE_PTR_ADD(m, RTE_CACHE_LINE_MIN_SIZE));
130 : : #else
131 : : RTE_SET_USED(m);
132 : : #endif
133 : 0 : }
134 : :
135 : :
136 : : static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
137 : :
138 : : /**
139 : : * Get the IOVA address of the mbuf data buffer.
140 : : *
141 : : * @param m
142 : : * The pointer to the mbuf.
143 : : * @return
144 : : * The IOVA address of the mbuf.
145 : : */
146 : : static inline rte_iova_t
147 : 0 : rte_mbuf_iova_get(const struct rte_mbuf *m)
148 : : {
149 : : #if RTE_IOVA_IN_MBUF
150 [ + + - + : 421 : return m->buf_iova;
- + - + -
+ - - + +
+ + - + -
- - - - +
- + - - -
- ]
151 : : #else
152 : : return (rte_iova_t)m->buf_addr;
153 : : #endif
154 : : }
155 : :
156 : : /**
157 : : * Set the IOVA address of the mbuf data buffer.
158 : : *
159 : : * @param m
160 : : * The pointer to the mbuf.
161 : : * @param iova
162 : : * Value to set as IOVA address of the mbuf.
163 : : */
164 : : static inline void
165 : 0 : rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
166 : : {
167 : : #if RTE_IOVA_IN_MBUF
168 [ + + ]: 192 : m->buf_iova = iova;
169 : : #else
170 : : RTE_SET_USED(m);
171 : : RTE_SET_USED(iova);
172 : : #endif
173 : 0 : }
174 : :
175 : : /**
176 : : * Return the IO address of the beginning of the mbuf data
177 : : *
178 : : * @param mb
179 : : * The pointer to the mbuf.
180 : : * @return
181 : : * The IO address of the beginning of the mbuf data
182 : : */
183 : : static inline rte_iova_t
184 : : rte_mbuf_data_iova(const struct rte_mbuf *mb)
185 : : {
186 [ # # # # : 0 : return rte_mbuf_iova_get(mb) + mb->data_off;
# # # # ]
187 : : }
188 : :
189 : : /**
190 : : * Return the default IO address of the beginning of the mbuf data
191 : : *
192 : : * This function is used by drivers in their receive function, as it
193 : : * returns the location where data should be written by the NIC, taking
194 : : * the default headroom in account.
195 : : *
196 : : * @param mb
197 : : * The pointer to the mbuf.
198 : : * @return
199 : : * The IO address of the beginning of the mbuf data
200 : : */
201 : : static inline rte_iova_t
202 : : rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
203 : : {
204 [ # # # # : 0 : return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
# # # # ]
205 : : }
206 : :
207 : : /**
208 : : * Return the mbuf owning the data buffer address of an indirect mbuf.
209 : : *
210 : : * @param mi
211 : : * The pointer to the indirect mbuf.
212 : : * @return
213 : : * The address of the direct mbuf corresponding to buffer_addr.
214 : : */
215 : : static inline struct rte_mbuf *
216 : : rte_mbuf_from_indirect(struct rte_mbuf *mi)
217 : : {
218 : 58 : return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
219 : : }
220 : :
221 : : /**
222 : : * Return address of buffer embedded in the given mbuf.
223 : : *
224 : : * The return value shall be same as mb->buf_addr if the mbuf is already
225 : : * initialized and direct. However, this API is useful if mempool of the
226 : : * mbuf is already known because it doesn't need to access mbuf contents in
227 : : * order to get the mempool pointer.
228 : : *
229 : : * @param mb
230 : : * The pointer to the mbuf.
231 : : * @param mp
232 : : * The pointer to the mempool of the mbuf.
233 : : * @return
234 : : * The pointer of the mbuf buffer.
235 : : */
236 : : static inline char *
237 : : rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
238 : : {
239 : : return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
240 : : }
241 : :
242 : : /**
243 : : * Return the default address of the beginning of the mbuf data.
244 : : *
245 : : * @param mb
246 : : * The pointer to the mbuf.
247 : : * @return
248 : : * The pointer of the beginning of the mbuf data.
249 : : */
250 : : static inline char *
251 : : rte_mbuf_data_addr_default(struct rte_mbuf *mb)
252 : : {
253 : : return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
254 : : }
255 : :
256 : : /**
257 : : * Return address of buffer embedded in the given mbuf.
258 : : *
259 : : * @note: Accessing mempool pointer of a mbuf is expensive because the
260 : : * pointer is stored in the 2nd cache line of mbuf. If mempool is known, it
261 : : * is better not to reference the mempool pointer in mbuf but calling
262 : : * rte_mbuf_buf_addr() would be more efficient.
263 : : *
264 : : * @param md
265 : : * The pointer to the mbuf.
266 : : * @return
267 : : * The address of the data buffer owned by the mbuf.
268 : : */
269 : : static inline char *
270 : : rte_mbuf_to_baddr(struct rte_mbuf *md)
271 : : {
272 : : return rte_mbuf_buf_addr(md, md->pool);
273 : : }
274 : :
275 : : /**
276 : : * Return the starting address of the private data area embedded in
277 : : * the given mbuf.
278 : : *
279 : : * Note that no check is made to ensure that a private data area
280 : : * actually exists in the supplied mbuf.
281 : : *
282 : : * @param m
283 : : * The pointer to the mbuf.
284 : : * @return
285 : : * The starting address of the private data area of the given mbuf.
286 : : */
287 : : static inline void *
288 : 0 : rte_mbuf_to_priv(struct rte_mbuf *m)
289 : : {
290 [ # # # # ]: 0 : return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
291 : : }
292 : :
293 : : /**
294 : : * Private data in case of pktmbuf pool.
295 : : *
296 : : * A structure that contains some pktmbuf_pool-specific data that are
297 : : * appended after the mempool structure (in private data).
298 : : */
299 : : struct rte_pktmbuf_pool_private {
300 : : uint16_t mbuf_data_room_size; /**< Size of data space in each mbuf. */
301 : : uint16_t mbuf_priv_size; /**< Size of private area in each mbuf. */
302 : : uint32_t flags; /**< reserved for future use. */
303 : : };
304 : :
305 : : /**
306 : : * Return the flags from private data in an mempool structure.
307 : : *
308 : : * @param mp
309 : : * A pointer to the mempool structure.
310 : : * @return
311 : : * The flags from the private data structure.
312 : : */
313 : : static inline uint32_t
314 [ # # ]: 0 : rte_pktmbuf_priv_flags(struct rte_mempool *mp)
315 : : {
316 : : struct rte_pktmbuf_pool_private *mbp_priv;
317 : :
318 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
319 [ + + + + ]: 196 : return mbp_priv->flags;
320 : : }
321 : :
322 : : /**
323 : : * When set, pktmbuf mempool will hold only mbufs with pinned external
324 : : * buffer. The external buffer will be attached to the mbuf at the
325 : : * memory pool creation and will never be detached by the mbuf free calls.
326 : : * mbuf should not contain any room for data after the mbuf structure.
327 : : */
328 : : #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
329 : :
330 : : /**
331 : : * Returns non zero if given mbuf has a pinned external buffer, or zero
332 : : * otherwise. The pinned external buffer is allocated at pool creation
333 : : * time and should not be freed on mbuf freeing.
334 : : *
335 : : * External buffer is a user-provided anonymous buffer.
336 : : */
337 : : #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
338 : : (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
339 : :
340 : : #ifdef RTE_LIBRTE_MBUF_DEBUG
341 : :
342 : : /** check mbuf type in debug mode */
343 : : #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
344 : :
345 : : #else /* RTE_LIBRTE_MBUF_DEBUG */
346 : :
347 : : /** check mbuf type in debug mode */
348 : : #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
349 : :
350 : : #endif /* RTE_LIBRTE_MBUF_DEBUG */
351 : :
352 : : #ifdef RTE_MBUF_REFCNT_ATOMIC
353 : :
354 : : /**
355 : : * Reads the value of an mbuf's refcnt.
356 : : * @param m
357 : : * Mbuf to read
358 : : * @return
359 : : * Reference count number.
360 : : */
361 : : static inline uint16_t
362 : 2376637 : rte_mbuf_refcnt_read(const struct rte_mbuf *m)
363 : : {
364 [ + + - + : 2100626 : return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
- + - + -
+ # # # #
# # # # ]
365 : : }
366 : :
367 : : /**
368 : : * Sets an mbuf's refcnt to a defined value.
369 : : * @param m
370 : : * Mbuf to update
371 : : * @param new_value
372 : : * Value set
373 : : */
374 : : static inline void
375 : 0 : rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
376 : : {
377 [ + - ]: 89875 : rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
378 : 0 : }
379 : :
380 : : /* internal */
381 : : static inline uint16_t
382 : 260937 : __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
383 : : {
384 : 390552 : return rte_atomic_fetch_add_explicit(&m->refcnt, value,
385 : 390552 : rte_memory_order_acq_rel) + value;
386 : : }
387 : :
388 : : /**
389 : : * Adds given value to an mbuf's refcnt and returns its new value.
390 : : * @param m
391 : : * Mbuf to update
392 : : * @param value
393 : : * Value to add/subtract
394 : : * @return
395 : : * Updated value
396 : : */
397 : : static inline uint16_t
398 : 133734 : rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
399 : : {
400 : : /*
401 : : * The atomic_add is an expensive operation, so we don't want to
402 : : * call it in the case where we know we are the unique holder of
403 : : * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
404 : : * operation has to be used because concurrent accesses on the
405 : : * reference counter can occur.
406 : : */
407 [ + + ]: 133734 : if (likely(rte_mbuf_refcnt_read(m) == 1)) {
408 : 4119 : ++value;
409 : 4119 : rte_mbuf_refcnt_set(m, (uint16_t)value);
410 : 4119 : return (uint16_t)value;
411 : : }
412 : :
413 : 129615 : return __rte_mbuf_refcnt_update(m, value);
414 : : }
415 : :
416 : : #else /* ! RTE_MBUF_REFCNT_ATOMIC */
417 : :
418 : : /* internal */
419 : : static inline uint16_t
420 : : __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
421 : : {
422 : : m->refcnt = (uint16_t)(m->refcnt + value);
423 : : return m->refcnt;
424 : : }
425 : :
426 : : /**
427 : : * Adds given value to an mbuf's refcnt and returns its new value.
428 : : */
429 : : static inline uint16_t
430 : : rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
431 : : {
432 : : return __rte_mbuf_refcnt_update(m, value);
433 : : }
434 : :
435 : : /**
436 : : * Reads the value of an mbuf's refcnt.
437 : : */
438 : : static inline uint16_t
439 : : rte_mbuf_refcnt_read(const struct rte_mbuf *m)
440 : : {
441 : : return m->refcnt;
442 : : }
443 : :
444 : : /**
445 : : * Sets an mbuf's refcnt to the defined value.
446 : : */
447 : : static inline void
448 : : rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
449 : : {
450 : : m->refcnt = new_value;
451 : : }
452 : :
453 : : #endif /* RTE_MBUF_REFCNT_ATOMIC */
454 : :
455 : : /**
456 : : * Reads the refcnt of an external buffer.
457 : : *
458 : : * @param shinfo
459 : : * Shared data of the external buffer.
460 : : * @return
461 : : * Reference count number.
462 : : */
463 : : static inline uint16_t
464 : : rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
465 : : {
466 [ - + - + : 108 : return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
- + ]
467 : : }
468 : :
469 : : /**
470 : : * Set refcnt of an external buffer.
471 : : *
472 : : * @param shinfo
473 : : * Shared data of the external buffer.
474 : : * @param new_value
475 : : * Value set
476 : : */
477 : : static inline void
478 : 0 : rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
479 : : uint16_t new_value)
480 : : {
481 [ + + ]: 5 : rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
482 : 0 : }
483 : :
484 : : /**
485 : : * Add given value to refcnt of an external buffer and return its new
486 : : * value.
487 : : *
488 : : * @param shinfo
489 : : * Shared data of the external buffer.
490 : : * @param value
491 : : * Value to add/subtract
492 : : * @return
493 : : * Updated value
494 : : */
495 : : static inline uint16_t
496 : 18 : rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo,
497 : : int16_t value)
498 : : {
499 [ + + ]: 18 : if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
500 : 8 : ++value;
501 : 8 : rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
502 : 8 : return (uint16_t)value;
503 : : }
504 : :
505 : 10 : return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
506 : 10 : rte_memory_order_acq_rel) + value;
507 : : }
508 : :
509 : : /** Mbuf prefetch */
510 : : #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
511 : : if ((m) != NULL) \
512 : : rte_prefetch0(m); \
513 : : } while (0)
514 : :
515 : :
516 : : /**
517 : : * Sanity checks on an mbuf.
518 : : *
519 : : * Check the consistency of the given mbuf. The function will cause a
520 : : * panic if corruption is detected.
521 : : *
522 : : * @param m
523 : : * The mbuf to be checked.
524 : : * @param is_header
525 : : * True if the mbuf is a packet header, false if it is a sub-segment
526 : : * of a packet (in this case, some fields like nb_segs are not checked)
527 : : */
528 : : void
529 : : rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
530 : :
531 : : /**
532 : : * Sanity checks on a mbuf.
533 : : *
534 : : * Almost like rte_mbuf_sanity_check(), but this function gives the reason
535 : : * if corruption is detected rather than panic.
536 : : *
537 : : * @param m
538 : : * The mbuf to be checked.
539 : : * @param is_header
540 : : * True if the mbuf is a packet header, false if it is a sub-segment
541 : : * of a packet (in this case, some fields like nb_segs are not checked)
542 : : * @param reason
543 : : * A reference to a string pointer where to store the reason why a mbuf is
544 : : * considered invalid.
545 : : * @return
546 : : * - 0 if no issue has been found, reason is left untouched.
547 : : * - -1 if a problem is detected, reason then points to a string describing
548 : : * the reason why the mbuf is deemed invalid.
549 : : */
550 : : int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
551 : : const char **reason);
552 : :
553 : : /**
554 : : * Sanity checks on a reinitialized mbuf in debug mode.
555 : : *
556 : : * Check the consistency of the given reinitialized mbuf.
557 : : * The function will cause a panic if corruption is detected.
558 : : *
559 : : * Check that the mbuf is properly reinitialized (refcnt=1, next=NULL,
560 : : * nb_segs=1), as done by rte_pktmbuf_prefree_seg().
561 : : *
562 : : * @param m
563 : : * The mbuf to be checked.
564 : : */
565 : : static __rte_always_inline void
566 : : __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
567 : : {
568 : : RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
569 : : RTE_ASSERT(m->next == NULL);
570 : : RTE_ASSERT(m->nb_segs == 1);
571 : : RTE_ASSERT(!RTE_MBUF_CLONED(m));
572 : : RTE_ASSERT(!RTE_MBUF_HAS_EXTBUF(m) ||
573 : : (RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
574 : : rte_mbuf_ext_refcnt_read(m->shinfo) == 1));
575 : : __rte_mbuf_sanity_check(m, 0);
576 : : }
577 : :
578 : : /** For backwards compatibility. */
579 : : #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
580 : :
581 : : /**
582 : : * Allocate an uninitialized mbuf from mempool *mp*.
583 : : *
584 : : * This function can be used by PMDs (especially in RX functions) to
585 : : * allocate an uninitialized mbuf. The driver is responsible of
586 : : * initializing all the required fields. See rte_pktmbuf_reset().
587 : : * For standard needs, prefer rte_pktmbuf_alloc().
588 : : *
589 : : * The caller can expect that the following fields of the mbuf structure
590 : : * are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1,
591 : : * next=NULL, pool, priv_size. The other fields must be initialized
592 : : * by the caller.
593 : : *
594 : : * @param mp
595 : : * The mempool from which mbuf is allocated.
596 : : * @return
597 : : * - The pointer to the new mbuf on success.
598 : : * - NULL if allocation failed.
599 : : */
600 [ + + ]: 17527 : static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
601 : : {
602 : : union {
603 : : void *ptr;
604 : : struct rte_mbuf *m;
605 : : } ret;
606 : :
607 [ + + ]: 17493 : if (rte_mempool_get(mp, &ret.ptr) < 0)
608 : : return NULL;
609 : 17519 : __rte_mbuf_raw_sanity_check(ret.m);
610 : 17519 : return ret.m;
611 : : }
612 : :
613 : : /**
614 : : * @warning
615 : : * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
616 : : *
617 : : * Allocate a bulk of uninitialized mbufs from mempool *mp*.
618 : : *
619 : : * This function can be used by PMDs (especially in Rx functions)
620 : : * to allocate a bulk of uninitialized mbufs.
621 : : * The driver is responsible of initializing all the required fields.
622 : : * See rte_pktmbuf_reset().
623 : : * For standard needs, prefer rte_pktmbuf_alloc_bulk().
624 : : *
625 : : * The caller can expect that the following fields of the mbuf structure
626 : : * are initialized:
627 : : * buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, next=NULL, pool, priv_size.
628 : : * The other fields must be initialized by the caller.
629 : : *
630 : : * @param mp
631 : : * The mempool from which mbufs are allocated.
632 : : * @param mbufs
633 : : * Array of pointers to mbufs.
634 : : * @param count
635 : : * Array size.
636 : : * @return
637 : : * - 0: Success.
638 : : * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved.
639 : : */
640 : : __rte_experimental
641 : : static __rte_always_inline int
642 : : rte_mbuf_raw_alloc_bulk(struct rte_mempool *mp, struct rte_mbuf **mbufs, unsigned int count)
643 : : {
644 : : int rc = rte_mempool_get_bulk(mp, (void **)mbufs, count);
645 : : if (likely(rc == 0))
646 : : for (unsigned int idx = 0; idx < count; idx++)
647 : : __rte_mbuf_raw_sanity_check(mbufs[idx]);
648 : : return rc;
649 : : }
650 : :
651 : : /**
652 : : * Put mbuf back into its original mempool.
653 : : *
654 : : * The caller must ensure that the mbuf is direct and properly
655 : : * reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by
656 : : * rte_pktmbuf_prefree_seg().
657 : : *
658 : : * This function should be used with care, when optimization is
659 : : * required. For standard needs, prefer rte_pktmbuf_free() or
660 : : * rte_pktmbuf_free_seg().
661 : : *
662 : : * @param m
663 : : * The mbuf to be freed.
664 : : */
665 : : static __rte_always_inline void
666 : : rte_mbuf_raw_free(struct rte_mbuf *m)
667 : : {
668 : : __rte_mbuf_raw_sanity_check(m);
669 [ + + + + : 2106790 : rte_mempool_put(m->pool, m);
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
670 : 2106798 : }
671 : :
672 : : /**
673 : : * @warning
674 : : * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
675 : : *
676 : : * Put a bulk of mbufs allocated from the same mempool back into the mempool.
677 : : *
678 : : * The caller must ensure that the mbufs come from the specified mempool,
679 : : * are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1),
680 : : * as done by rte_pktmbuf_prefree_seg().
681 : : *
682 : : * This function should be used with care, when optimization is required.
683 : : * For standard needs, prefer rte_pktmbuf_free_bulk().
684 : : *
685 : : * @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE
686 : : *
687 : : * @param mp
688 : : * The mempool to which the mbufs belong.
689 : : * @param mbufs
690 : : * Array of pointers to packet mbufs.
691 : : * The array must not contain NULL pointers.
692 : : * @param count
693 : : * Array size.
694 : : */
695 : : __rte_experimental
696 : : static __rte_always_inline void
697 : : rte_mbuf_raw_free_bulk(struct rte_mempool *mp, struct rte_mbuf **mbufs, unsigned int count)
698 : : {
699 : : for (unsigned int idx = 0; idx < count; idx++) {
700 : : const struct rte_mbuf *m = mbufs[idx];
701 : : RTE_ASSERT(m != NULL);
702 : : RTE_ASSERT(m->pool == mp);
703 : : __rte_mbuf_raw_sanity_check(m);
704 : : }
705 : :
706 : : rte_mempool_put_bulk(mp, (void **)mbufs, count);
707 : : }
708 : :
709 : : /**
710 : : * The packet mbuf constructor.
711 : : *
712 : : * This function initializes some fields in the mbuf structure that are
713 : : * not modified by the user once created (origin pool, buffer start
714 : : * address, and so on). This function is given as a callback function to
715 : : * rte_mempool_obj_iter() or rte_mempool_create() at pool creation time.
716 : : *
717 : : * This function expects that the mempool private area was previously
718 : : * initialized with rte_pktmbuf_pool_init().
719 : : *
720 : : * @param mp
721 : : * The mempool from which mbufs originate.
722 : : * @param opaque_arg
723 : : * A pointer that can be used by the user to retrieve useful information
724 : : * for mbuf initialization. This pointer is the opaque argument passed to
725 : : * rte_mempool_obj_iter() or rte_mempool_create().
726 : : * @param m
727 : : * The mbuf to initialize.
728 : : * @param i
729 : : * The index of the mbuf in the pool table.
730 : : */
731 : : void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
732 : : void *m, unsigned i);
733 : :
734 : : /**
735 : : * A packet mbuf pool constructor.
736 : : *
737 : : * This function initializes the mempool private data in the case of a
738 : : * pktmbuf pool. This private data is needed by the driver. The
739 : : * function must be called on the mempool before it is used, or it
740 : : * can be given as a callback function to rte_mempool_create() at
741 : : * pool creation. It can be extended by the user, for example, to
742 : : * provide another packet size.
743 : : *
744 : : * The mempool private area size must be at least equal to
745 : : * sizeof(struct rte_pktmbuf_pool_private).
746 : : *
747 : : * @param mp
748 : : * The mempool from which mbufs originate.
749 : : * @param opaque_arg
750 : : * A pointer that can be used by the user to retrieve useful information
751 : : * for mbuf initialization. This pointer is the opaque argument passed to
752 : : * rte_mempool_create().
753 : : */
754 : : void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
755 : :
756 : : /**
757 : : * Create a mbuf pool.
758 : : *
759 : : * This function creates and initializes a packet mbuf pool. It is
760 : : * a wrapper to rte_mempool functions.
761 : : *
762 : : * @param name
763 : : * The name of the mbuf pool.
764 : : * @param n
765 : : * The number of elements in the mbuf pool. The optimum size (in terms
766 : : * of memory usage) for a mempool is when n is a power of two minus one:
767 : : * n = (2^q - 1).
768 : : * @param cache_size
769 : : * Size of the per-core object cache. See rte_mempool_create() for
770 : : * details.
771 : : * @param priv_size
772 : : * Size of application private are between the rte_mbuf structure
773 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
774 : : * @param data_room_size
775 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
776 : : * @param socket_id
777 : : * The socket identifier where the memory should be allocated. The
778 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
779 : : * reserved zone.
780 : : * @return
781 : : * The pointer to the new allocated mempool, on success. NULL on error
782 : : * with rte_errno set appropriately. Possible rte_errno values include:
783 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
784 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
785 : : * - ENOSPC - the maximum number of memzones has already been allocated
786 : : * - EEXIST - a memzone with the same name already exists
787 : : * - ENOMEM - no appropriate memory area found in which to create memzone
788 : : */
789 : : struct rte_mempool *
790 : : rte_pktmbuf_pool_create(const char *name, unsigned n,
791 : : unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
792 : : int socket_id);
793 : :
794 : : /**
795 : : * Create a mbuf pool with a given mempool ops name
796 : : *
797 : : * This function creates and initializes a packet mbuf pool. It is
798 : : * a wrapper to rte_mempool functions.
799 : : *
800 : : * @param name
801 : : * The name of the mbuf pool.
802 : : * @param n
803 : : * The number of elements in the mbuf pool. The optimum size (in terms
804 : : * of memory usage) for a mempool is when n is a power of two minus one:
805 : : * n = (2^q - 1).
806 : : * @param cache_size
807 : : * Size of the per-core object cache. See rte_mempool_create() for
808 : : * details.
809 : : * @param priv_size
810 : : * Size of application private are between the rte_mbuf structure
811 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
812 : : * @param data_room_size
813 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
814 : : * @param socket_id
815 : : * The socket identifier where the memory should be allocated. The
816 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
817 : : * reserved zone.
818 : : * @param ops_name
819 : : * The mempool ops name to be used for this mempool instead of
820 : : * default mempool. The value can be *NULL* to use default mempool.
821 : : * @return
822 : : * The pointer to the new allocated mempool, on success. NULL on error
823 : : * with rte_errno set appropriately. Possible rte_errno values include:
824 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
825 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
826 : : * - ENOSPC - the maximum number of memzones has already been allocated
827 : : * - EEXIST - a memzone with the same name already exists
828 : : * - ENOMEM - no appropriate memory area found in which to create memzone
829 : : */
830 : : struct rte_mempool *
831 : : rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
832 : : unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
833 : : int socket_id, const char *ops_name);
834 : :
835 : : /** A structure that describes the pinned external buffer segment. */
836 : : struct rte_pktmbuf_extmem {
837 : : void *buf_ptr; /**< The virtual address of data buffer. */
838 : : rte_iova_t buf_iova; /**< The IO address of the data buffer. */
839 : : size_t buf_len; /**< External buffer length in bytes. */
840 : : uint16_t elt_size; /**< mbuf element size in bytes. */
841 : : };
842 : :
843 : : /**
844 : : * Create a mbuf pool with external pinned data buffers.
845 : : *
846 : : * This function creates and initializes a packet mbuf pool that contains
847 : : * only mbufs with external buffer. It is a wrapper to rte_mempool functions.
848 : : *
849 : : * @param name
850 : : * The name of the mbuf pool.
851 : : * @param n
852 : : * The number of elements in the mbuf pool. The optimum size (in terms
853 : : * of memory usage) for a mempool is when n is a power of two minus one:
854 : : * n = (2^q - 1).
855 : : * @param cache_size
856 : : * Size of the per-core object cache. See rte_mempool_create() for
857 : : * details.
858 : : * @param priv_size
859 : : * Size of application private are between the rte_mbuf structure
860 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
861 : : * @param data_room_size
862 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
863 : : * @param socket_id
864 : : * The socket identifier where the memory should be allocated. The
865 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
866 : : * reserved zone.
867 : : * @param ext_mem
868 : : * Pointer to the array of structures describing the external memory
869 : : * for data buffers. It is caller responsibility to register this memory
870 : : * with rte_extmem_register() (if needed), map this memory to appropriate
871 : : * physical device, etc.
872 : : * @param ext_num
873 : : * Number of elements in the ext_mem array.
874 : : * @return
875 : : * The pointer to the new allocated mempool, on success. NULL on error
876 : : * with rte_errno set appropriately. Possible rte_errno values include:
877 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
878 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
879 : : * - ENOSPC - the maximum number of memzones has already been allocated
880 : : * - EEXIST - a memzone with the same name already exists
881 : : * - ENOMEM - no appropriate memory area found in which to create memzone
882 : : */
883 : : struct rte_mempool *
884 : : rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
885 : : unsigned int cache_size, uint16_t priv_size,
886 : : uint16_t data_room_size, int socket_id,
887 : : const struct rte_pktmbuf_extmem *ext_mem,
888 : : unsigned int ext_num);
889 : :
890 : : /**
891 : : * Get the data room size of mbufs stored in a pktmbuf_pool
892 : : *
893 : : * The data room size is the amount of data that can be stored in a
894 : : * mbuf including the headroom (RTE_PKTMBUF_HEADROOM).
895 : : *
896 : : * @param mp
897 : : * The packet mbuf pool.
898 : : * @return
899 : : * The data room size of mbufs stored in this mempool.
900 : : */
901 : : static inline uint16_t
902 : : rte_pktmbuf_data_room_size(struct rte_mempool *mp)
903 : : {
904 : : struct rte_pktmbuf_pool_private *mbp_priv;
905 : :
906 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
907 [ + + - + : 233 : return mbp_priv->mbuf_data_room_size;
- + ]
908 : : }
909 : :
910 : : /**
911 : : * Get the application private size of mbufs stored in a pktmbuf_pool
912 : : *
913 : : * The private size of mbuf is a zone located between the rte_mbuf
914 : : * structure and the data buffer where an application can store data
915 : : * associated to a packet.
916 : : *
917 : : * @param mp
918 : : * The packet mbuf pool.
919 : : * @return
920 : : * The private size of mbufs stored in this mempool.
921 : : */
922 : : static inline uint16_t
923 : : rte_pktmbuf_priv_size(struct rte_mempool *mp)
924 : : {
925 : : struct rte_pktmbuf_pool_private *mbp_priv;
926 : :
927 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
928 [ + + - + ]: 153 : return mbp_priv->mbuf_priv_size;
929 : : }
930 : :
931 : : /**
932 : : * Reset the data_off field of a packet mbuf to its default value.
933 : : *
934 : : * The given mbuf must have only one segment, which should be empty.
935 : : *
936 : : * @param m
937 : : * The packet mbuf's data_off field has to be reset.
938 : : */
939 : : static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
940 : : {
941 : 18778 : m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
942 : : (uint16_t)m->buf_len);
943 : : }
944 : :
945 : : /**
946 : : * Reset the fields of a packet mbuf to their default values.
947 : : *
948 : : * The given mbuf must have only one segment.
949 : : *
950 : : * @param m
951 : : * The packet mbuf to be reset.
952 : : */
953 : : static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
954 : : {
955 : 18895 : m->next = NULL;
956 : 18895 : m->pkt_len = 0;
957 : 18895 : m->tx_offload = 0;
958 : 18895 : m->vlan_tci = 0;
959 : 18895 : m->vlan_tci_outer = 0;
960 : 18895 : m->nb_segs = 1;
961 : 18895 : m->port = RTE_MBUF_PORT_INVALID;
962 : :
963 : 18895 : m->ol_flags &= RTE_MBUF_F_EXTERNAL;
964 : 18895 : m->packet_type = 0;
965 : : rte_pktmbuf_reset_headroom(m);
966 : :
967 [ + - ]: 18701 : m->data_len = 0;
968 : : __rte_mbuf_sanity_check(m, 1);
969 : 17516 : }
970 : :
971 : : /**
972 : : * Allocate a new mbuf from a mempool.
973 : : *
974 : : * This new mbuf contains one segment, which has a length of 0. The pointer
975 : : * to data is initialized to have some bytes of headroom in the buffer
976 : : * (if buffer size allows).
977 : : *
978 : : * @param mp
979 : : * The mempool from which the mbuf is allocated.
980 : : * @return
981 : : * - The pointer to the new mbuf on success.
982 : : * - NULL if allocation failed.
983 : : */
984 : 17524 : static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
985 : : {
986 : : struct rte_mbuf *m;
987 [ + + ]: 17524 : if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
988 : : rte_pktmbuf_reset(m);
989 : 17524 : return m;
990 : : }
991 : :
992 : : /**
993 : : * Allocate a bulk of mbufs, initialize refcnt and reset the fields to default
994 : : * values.
995 : : *
996 : : * @param pool
997 : : * The mempool from which mbufs are allocated.
998 : : * @param mbufs
999 : : * Array of pointers to mbufs
1000 : : * @param count
1001 : : * Array size
1002 : : * @return
1003 : : * - 0: Success
1004 : : * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved.
1005 : : */
1006 [ + + ]: 33 : static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
1007 : : struct rte_mbuf **mbufs, unsigned count)
1008 : : {
1009 : : unsigned idx = 0;
1010 : : int rc;
1011 : :
1012 : : rc = rte_mempool_get_bulk(pool, (void **)mbufs, count);
1013 [ + + ]: 33 : if (unlikely(rc))
1014 : : return rc;
1015 : :
1016 : : /* To understand duff's device on loop unwinding optimization, see
1017 : : * https://en.wikipedia.org/wiki/Duff's_device.
1018 : : * Here while() loop is used rather than do() while{} to avoid extra
1019 : : * check if count is zero.
1020 : : */
1021 [ + + + + ]: 25 : switch (count % 4) {
1022 : : case 0:
1023 [ + + ]: 291 : while (idx != count) {
1024 : 266 : __rte_mbuf_raw_sanity_check(mbufs[idx]);
1025 : : rte_pktmbuf_reset(mbufs[idx]);
1026 : 266 : idx++;
1027 : : /* fall-through */
1028 : 270 : case 3:
1029 : 270 : __rte_mbuf_raw_sanity_check(mbufs[idx]);
1030 : : rte_pktmbuf_reset(mbufs[idx]);
1031 : 270 : idx++;
1032 : : /* fall-through */
1033 : 273 : case 2:
1034 : 273 : __rte_mbuf_raw_sanity_check(mbufs[idx]);
1035 : : rte_pktmbuf_reset(mbufs[idx]);
1036 : 273 : idx++;
1037 : : /* fall-through */
1038 : 277 : case 1:
1039 : 277 : __rte_mbuf_raw_sanity_check(mbufs[idx]);
1040 : : rte_pktmbuf_reset(mbufs[idx]);
1041 : 277 : idx++;
1042 : : /* fall-through */
1043 : : }
1044 : : }
1045 : : return 0;
1046 : : }
1047 : :
1048 : : /**
1049 : : * Initialize shared data at the end of an external buffer before attaching
1050 : : * to a mbuf by ``rte_pktmbuf_attach_extbuf()``. This is not a mandatory
1051 : : * initialization but a helper function to simply spare a few bytes at the
1052 : : * end of the buffer for shared data. If shared data is allocated
1053 : : * separately, this should not be called but application has to properly
1054 : : * initialize the shared data according to its need.
1055 : : *
1056 : : * Free callback and its argument is saved and the refcnt is set to 1.
1057 : : *
1058 : : * @warning
1059 : : * The value of buf_len will be reduced to RTE_PTR_DIFF(shinfo, buf_addr)
1060 : : * after this initialization. This shall be used for
1061 : : * ``rte_pktmbuf_attach_extbuf()``
1062 : : *
1063 : : * @param buf_addr
1064 : : * The pointer to the external buffer.
1065 : : * @param [in,out] buf_len
1066 : : * The pointer to length of the external buffer. Input value must be
1067 : : * larger than the size of ``struct rte_mbuf_ext_shared_info`` and
1068 : : * padding for alignment. If not enough, this function will return NULL.
1069 : : * Adjusted buffer length will be returned through this pointer.
1070 : : * @param free_cb
1071 : : * Free callback function to call when the external buffer needs to be
1072 : : * freed.
1073 : : * @param fcb_opaque
1074 : : * Argument for the free callback function.
1075 : : *
1076 : : * @return
1077 : : * A pointer to the initialized shared data on success, return NULL
1078 : : * otherwise.
1079 : : */
1080 : : static inline struct rte_mbuf_ext_shared_info *
1081 : : rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
1082 : : rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
1083 : : {
1084 : : struct rte_mbuf_ext_shared_info *shinfo;
1085 : 3 : void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1086 : : void *addr;
1087 : :
1088 : 3 : addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1089 : : sizeof(uintptr_t));
1090 [ - + ]: 3 : if (addr <= buf_addr)
1091 : : return NULL;
1092 : :
1093 : : shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1094 : 3 : shinfo->free_cb = free_cb;
1095 [ - + ]: 3 : shinfo->fcb_opaque = fcb_opaque;
1096 : : rte_mbuf_ext_refcnt_set(shinfo, 1);
1097 : :
1098 : 2 : *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1099 : 0 : return shinfo;
1100 : : }
1101 : :
1102 : : /**
1103 : : * Attach an external buffer to a mbuf.
1104 : : *
1105 : : * User-managed anonymous buffer can be attached to an mbuf. When attaching
1106 : : * it, corresponding free callback function and its argument should be
1107 : : * provided via shinfo. This callback function will be called once all the
1108 : : * mbufs are detached from the buffer (refcnt becomes zero).
1109 : : *
1110 : : * The headroom length of the attaching mbuf will be set to zero and this
1111 : : * can be properly adjusted after attachment. For example, ``rte_pktmbuf_adj()``
1112 : : * or ``rte_pktmbuf_reset_headroom()`` might be used.
1113 : : *
1114 : : * Similarly, the packet length is initialized to 0. If the buffer contains
1115 : : * data, the user has to adjust ``data_len`` and the ``pkt_len`` field of
1116 : : * the mbuf accordingly.
1117 : : *
1118 : : * More mbufs can be attached to the same external buffer by
1119 : : * ``rte_pktmbuf_attach()`` once the external buffer has been attached by
1120 : : * this API.
1121 : : *
1122 : : * Detachment can be done by either ``rte_pktmbuf_detach_extbuf()`` or
1123 : : * ``rte_pktmbuf_detach()``.
1124 : : *
1125 : : * Memory for shared data must be provided and user must initialize all of
1126 : : * the content properly, especially free callback and refcnt. The pointer
1127 : : * of shared data will be stored in m->shinfo.
1128 : : * ``rte_pktmbuf_ext_shinfo_init_helper`` can help to simply spare a few
1129 : : * bytes at the end of buffer for the shared data, store free callback and
1130 : : * its argument and set the refcnt to 1. The following is an example:
1131 : : *
1132 : : * struct rte_mbuf_ext_shared_info *shinfo =
1133 : : * rte_pktmbuf_ext_shinfo_init_helper(buf_addr, &buf_len,
1134 : : * free_cb, fcb_arg);
1135 : : * rte_pktmbuf_attach_extbuf(m, buf_addr, buf_iova, buf_len, shinfo);
1136 : : * rte_pktmbuf_reset_headroom(m);
1137 : : * rte_pktmbuf_adj(m, data_len);
1138 : : *
1139 : : * Attaching an external buffer is quite similar to mbuf indirection in
1140 : : * replacing buffer addresses and length of a mbuf, but a few differences:
1141 : : * - When an indirect mbuf is attached, refcnt of the direct mbuf would be
1142 : : * 2 as long as the direct mbuf itself isn't freed after the attachment.
1143 : : * In such cases, the buffer area of a direct mbuf must be read-only. But
1144 : : * external buffer has its own refcnt and it starts from 1. Unless
1145 : : * multiple mbufs are attached to a mbuf having an external buffer, the
1146 : : * external buffer is writable.
1147 : : * - There's no need to allocate buffer from a mempool. Any buffer can be
1148 : : * attached with appropriate free callback and its IO address.
1149 : : * - Smaller metadata is required to maintain shared data such as refcnt.
1150 : : *
1151 : : * @param m
1152 : : * The pointer to the mbuf.
1153 : : * @param buf_addr
1154 : : * The pointer to the external buffer.
1155 : : * @param buf_iova
1156 : : * IO address of the external buffer.
1157 : : * @param buf_len
1158 : : * The size of the external buffer.
1159 : : * @param shinfo
1160 : : * User-provided memory for shared data of the external buffer.
1161 : : */
1162 : : static inline void
1163 : : rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1164 : : rte_iova_t buf_iova, uint16_t buf_len,
1165 : : struct rte_mbuf_ext_shared_info *shinfo)
1166 : : {
1167 : : /* mbuf should not be read-only */
1168 : : RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1169 : : RTE_ASSERT(shinfo->free_cb != NULL);
1170 : :
1171 : 3 : m->buf_addr = buf_addr;
1172 : : rte_mbuf_iova_set(m, buf_iova);
1173 : 3 : m->buf_len = buf_len;
1174 : :
1175 : 3 : m->data_len = 0;
1176 : 3 : m->data_off = 0;
1177 : :
1178 : 3 : m->ol_flags |= RTE_MBUF_F_EXTERNAL;
1179 [ - + ]: 3 : m->shinfo = shinfo;
1180 : : }
1181 : :
1182 : : /**
1183 : : * Detach the external buffer attached to a mbuf, same as
1184 : : * ``rte_pktmbuf_detach()``
1185 : : *
1186 : : * @param m
1187 : : * The mbuf having external buffer.
1188 : : */
1189 : : #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1190 : :
1191 : : /**
1192 : : * Copy dynamic fields from msrc to mdst.
1193 : : *
1194 : : * @param mdst
1195 : : * The destination mbuf.
1196 : : * @param msrc
1197 : : * The source mbuf.
1198 : : */
1199 : : static inline void
1200 : : rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1201 : : {
1202 : : #if !RTE_IOVA_IN_MBUF
1203 : : mdst->dynfield2 = msrc->dynfield2;
1204 : : #endif
1205 : 4146 : memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1206 : : }
1207 : :
1208 : : /* internal */
1209 : : static inline void
1210 : 4146 : __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1211 : : {
1212 : 4146 : mdst->port = msrc->port;
1213 : 4146 : mdst->vlan_tci = msrc->vlan_tci;
1214 : 4146 : mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1215 : 4146 : mdst->tx_offload = msrc->tx_offload;
1216 : 4146 : mdst->hash = msrc->hash;
1217 : 4146 : mdst->packet_type = msrc->packet_type;
1218 : : rte_mbuf_dynfield_copy(mdst, msrc);
1219 : 4146 : }
1220 : :
1221 : : /**
1222 : : * Attach packet mbuf to another packet mbuf.
1223 : : *
1224 : : * If the mbuf we are attaching to isn't a direct buffer and is attached to
1225 : : * an external buffer, the mbuf being attached will be attached to the
1226 : : * external buffer instead of mbuf indirection.
1227 : : *
1228 : : * Otherwise, the mbuf will be indirectly attached. After attachment we
1229 : : * refer the mbuf we attached as 'indirect', while mbuf we attached to as
1230 : : * 'direct'. The direct mbuf's reference counter is incremented.
1231 : : *
1232 : : * Right now, not supported:
1233 : : * - attachment for already indirect mbuf (e.g. - mi has to be direct).
1234 : : * - mbuf we trying to attach (mi) is used by someone else
1235 : : * e.g. it's reference counter is greater then 1.
1236 : : *
1237 : : * @param mi
1238 : : * The indirect packet mbuf.
1239 : : * @param m
1240 : : * The packet mbuf we're attaching to.
1241 : : */
1242 : 36 : static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1243 : : {
1244 : : RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1245 : : rte_mbuf_refcnt_read(mi) == 1);
1246 : :
1247 [ + + ]: 36 : if (RTE_MBUF_HAS_EXTBUF(m)) {
1248 : 7 : rte_mbuf_ext_refcnt_update(m->shinfo, 1);
1249 : 7 : mi->ol_flags = m->ol_flags;
1250 : 7 : mi->shinfo = m->shinfo;
1251 : : } else {
1252 : : /* if m is not direct, get the mbuf that embeds the data */
1253 : 29 : rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
1254 : 29 : mi->priv_size = m->priv_size;
1255 : 29 : mi->ol_flags = m->ol_flags | RTE_MBUF_F_INDIRECT;
1256 : : }
1257 : :
1258 : 36 : __rte_pktmbuf_copy_hdr(mi, m);
1259 : :
1260 : 36 : mi->data_off = m->data_off;
1261 : 36 : mi->data_len = m->data_len;
1262 : : rte_mbuf_iova_set(mi, rte_mbuf_iova_get(m));
1263 : 36 : mi->buf_addr = m->buf_addr;
1264 : 36 : mi->buf_len = m->buf_len;
1265 : :
1266 : 36 : mi->next = NULL;
1267 : 36 : mi->pkt_len = mi->data_len;
1268 : 36 : mi->nb_segs = 1;
1269 : :
1270 : : __rte_mbuf_sanity_check(mi, 1);
1271 : : __rte_mbuf_sanity_check(m, 0);
1272 : 36 : }
1273 : :
1274 : : /**
1275 : : * @internal used by rte_pktmbuf_detach().
1276 : : *
1277 : : * Decrement the reference counter of the external buffer. When the
1278 : : * reference counter becomes 0, the buffer is freed by pre-registered
1279 : : * callback.
1280 : : */
1281 : : static inline void
1282 : 10 : __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1283 : : {
1284 : : RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1285 : : RTE_ASSERT(m->shinfo != NULL);
1286 : :
1287 [ + + ]: 10 : if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1288 : 3 : m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1289 : 10 : }
1290 : :
1291 : : /**
1292 : : * @internal used by rte_pktmbuf_detach().
1293 : : *
1294 : : * Decrement the direct mbuf's reference counter. When the reference
1295 : : * counter becomes 0, the direct mbuf is freed.
1296 : : */
1297 : : static inline void
1298 : 29 : __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1299 : : {
1300 : : struct rte_mbuf *md;
1301 : :
1302 : : RTE_ASSERT(RTE_MBUF_CLONED(m));
1303 : :
1304 : : md = rte_mbuf_from_indirect(m);
1305 : :
1306 [ + + ]: 29 : if (rte_mbuf_refcnt_update(md, -1) == 0) {
1307 : 10 : md->next = NULL;
1308 [ + - ]: 10 : md->nb_segs = 1;
1309 : : rte_mbuf_refcnt_set(md, 1);
1310 : : rte_mbuf_raw_free(md);
1311 : : }
1312 : 29 : }
1313 : :
1314 : : /**
1315 : : * Detach a packet mbuf from external buffer or direct buffer.
1316 : : *
1317 : : * - decrement refcnt and free the external/direct buffer if refcnt
1318 : : * becomes zero.
1319 : : * - restore original mbuf address and length values.
1320 : : * - reset pktmbuf data and data_len to their default values.
1321 : : *
1322 : : * All other fields of the given packet mbuf will be left intact.
1323 : : *
1324 : : * If the packet mbuf was allocated from the pool with pinned
1325 : : * external buffers the rte_pktmbuf_detach does nothing with the
1326 : : * mbuf of this kind, because the pinned buffers are not supposed
1327 : : * to be detached.
1328 : : *
1329 : : * @param m
1330 : : * The indirect attached packet mbuf.
1331 : : */
1332 : 1128 : static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1333 : : {
1334 : 1128 : struct rte_mempool *mp = m->pool;
1335 : : uint32_t mbuf_size, buf_len;
1336 : : uint16_t priv_size;
1337 : :
1338 [ + + ]: 1128 : if (RTE_MBUF_HAS_EXTBUF(m)) {
1339 : : /*
1340 : : * The mbuf has the external attached buffer,
1341 : : * we should check the type of the memory pool where
1342 : : * the mbuf was allocated from to detect the pinned
1343 : : * external buffer.
1344 : : */
1345 : : uint32_t flags = rte_pktmbuf_priv_flags(mp);
1346 : :
1347 [ + + ]: 1099 : if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1348 : : /*
1349 : : * The pinned external buffer should not be
1350 : : * detached from its backing mbuf, just exit.
1351 : : */
1352 : : return;
1353 : : }
1354 : 10 : __rte_pktmbuf_free_extbuf(m);
1355 : : } else {
1356 : 29 : __rte_pktmbuf_free_direct(m);
1357 : : }
1358 : : priv_size = rte_pktmbuf_priv_size(mp);
1359 : 39 : mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1360 : : buf_len = rte_pktmbuf_data_room_size(mp);
1361 : :
1362 : 39 : m->priv_size = priv_size;
1363 : 39 : m->buf_addr = (char *)m + mbuf_size;
1364 : 39 : rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
1365 : 39 : m->buf_len = (uint16_t)buf_len;
1366 : : rte_pktmbuf_reset_headroom(m);
1367 : 39 : m->data_len = 0;
1368 : 39 : m->ol_flags = 0;
1369 : : }
1370 : :
1371 : : /**
1372 : : * @internal Handle the packet mbufs with attached pinned external buffer
1373 : : * on the mbuf freeing:
1374 : : *
1375 : : * - return zero if reference counter in shinfo is one. It means there is
1376 : : * no more reference to this pinned buffer and mbuf can be returned to
1377 : : * the pool
1378 : : *
1379 : : * - otherwise (if reference counter is not one), decrement reference
1380 : : * counter and return non-zero value to prevent freeing the backing mbuf.
1381 : : *
1382 : : * Returns non zero if mbuf should not be freed.
1383 : : */
1384 : 1089 : static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1385 : : {
1386 : : struct rte_mbuf_ext_shared_info *shinfo;
1387 : :
1388 : : /* Clear flags, mbuf is being freed. */
1389 : 1089 : m->ol_flags = RTE_MBUF_F_EXTERNAL;
1390 : 1089 : shinfo = m->shinfo;
1391 : :
1392 : : /* Optimize for performance - do not dec/reinit */
1393 [ + + ]: 1089 : if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1394 : : return 0;
1395 : :
1396 : : /*
1397 : : * Direct usage of add primitive to avoid
1398 : : * duplication of comparing with one.
1399 : : */
1400 [ - + ]: 2 : if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
1401 : : rte_memory_order_acq_rel) - 1))
1402 : : return 1;
1403 : :
1404 : : /* Reinitialize counter before mbuf freeing. */
1405 : : rte_mbuf_ext_refcnt_set(shinfo, 1);
1406 : 0 : return 0;
1407 : : }
1408 : :
1409 : : /**
1410 : : * Decrease reference counter and unlink a mbuf segment
1411 : : *
1412 : : * This function does the same than a free, except that it does not
1413 : : * return the segment to its pool.
1414 : : * It decreases the reference counter, and if it reaches 0, it is
1415 : : * detached from its parent for an indirect mbuf.
1416 : : *
1417 : : * @param m
1418 : : * The mbuf to be unlinked
1419 : : * @return
1420 : : * - (m) if it is the last reference. It can be recycled or freed.
1421 : : * - (NULL) if the mbuf still has remaining references on it.
1422 : : */
1423 : : static __rte_always_inline struct rte_mbuf *
1424 : : rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
1425 : : {
1426 : : __rte_mbuf_sanity_check(m, 0);
1427 : :
1428 [ + + + + : 2376657 : if (likely(rte_mbuf_refcnt_read(m) == 1)) {
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1429 : :
1430 [ + + + + : 2115528 : if (!RTE_MBUF_DIRECT(m)) {
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1431 : 1124 : rte_pktmbuf_detach(m);
1432 [ + + + + : 1124 : if (RTE_MBUF_HAS_EXTBUF(m) &&
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1433 [ + - + - : 3267 : RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ - + - +
- + + - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1434 : 1089 : __rte_pktmbuf_pinned_extbuf_decref(m))
1435 : : return NULL;
1436 : : }
1437 : :
1438 [ + + + + : 2115526 : if (m->next != NULL)
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1439 : 4724 : m->next = NULL;
1440 [ + + + + : 2115526 : if (m->nb_segs != 1)
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1441 : 4174 : m->nb_segs = 1;
1442 : :
1443 : : return m;
1444 : :
1445 [ - + - + : 261129 : } else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1446 : :
1447 [ # # # # : 0 : if (!RTE_MBUF_DIRECT(m)) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1448 : 0 : rte_pktmbuf_detach(m);
1449 [ # # # # : 0 : if (RTE_MBUF_HAS_EXTBUF(m) &&
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1450 [ # # # # : 0 : RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1451 : 0 : __rte_pktmbuf_pinned_extbuf_decref(m))
1452 : : return NULL;
1453 : : }
1454 : :
1455 [ # # # # : 0 : if (m->next != NULL)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1456 : 0 : m->next = NULL;
1457 [ # # # # : 0 : if (m->nb_segs != 1)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1458 : 0 : m->nb_segs = 1;
1459 : 0 : rte_mbuf_refcnt_set(m, 1);
1460 : :
1461 : 0 : return m;
1462 : : }
1463 : : return NULL;
1464 : : }
1465 : :
1466 : : /**
1467 : : * Free a segment of a packet mbuf into its original mempool.
1468 : : *
1469 : : * Free an mbuf, without parsing other segments in case of chained
1470 : : * buffers.
1471 : : *
1472 : : * @param m
1473 : : * The packet mbuf segment to be freed.
1474 : : */
1475 : : static __rte_always_inline void
1476 : : rte_pktmbuf_free_seg(struct rte_mbuf *m)
1477 : : {
1478 : : m = rte_pktmbuf_prefree_seg(m);
1479 [ + + + + : 2368392 : if (likely(m != NULL))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1480 : : rte_mbuf_raw_free(m);
1481 : : }
1482 : :
1483 : : /**
1484 : : * Free a packet mbuf back into its original mempool.
1485 : : *
1486 : : * Free an mbuf, and all its segments in case of chained buffers. Each
1487 : : * segment is added back into its original mempool.
1488 : : *
1489 : : * @param m
1490 : : * The packet mbuf to be freed. If NULL, the function does nothing.
1491 : : */
1492 : 2400590 : static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1493 : : {
1494 : : struct rte_mbuf *m_next;
1495 : :
1496 : : if (m != NULL)
1497 : : __rte_mbuf_sanity_check(m, 1);
1498 : :
1499 [ + + ]: 4768542 : while (m != NULL) {
1500 : 2367801 : m_next = m->next;
1501 : : rte_pktmbuf_free_seg(m);
1502 : : m = m_next;
1503 : : }
1504 : 2400741 : }
1505 : :
1506 : : /**
1507 : : * Free a bulk of packet mbufs back into their original mempools.
1508 : : *
1509 : : * Free a bulk of mbufs, and all their segments in case of chained buffers.
1510 : : * Each segment is added back into its original mempool.
1511 : : *
1512 : : * @param mbufs
1513 : : * Array of pointers to packet mbufs.
1514 : : * The array may contain NULL pointers.
1515 : : * @param count
1516 : : * Array size.
1517 : : */
1518 : : void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1519 : :
1520 : : /**
1521 : : * Create a "clone" of the given packet mbuf.
1522 : : *
1523 : : * Walks through all segments of the given packet mbuf, and for each of them:
1524 : : * - Creates a new packet mbuf from the given pool.
1525 : : * - Attaches newly created mbuf to the segment.
1526 : : * Then updates pkt_len and nb_segs of the "clone" packet mbuf to match values
1527 : : * from the original packet mbuf.
1528 : : *
1529 : : * @param md
1530 : : * The packet mbuf to be cloned.
1531 : : * @param mp
1532 : : * The mempool from which the "clone" mbufs are allocated.
1533 : : * @return
1534 : : * - The pointer to the new "clone" mbuf on success.
1535 : : * - NULL if allocation fails.
1536 : : */
1537 : : struct rte_mbuf *
1538 : : rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1539 : :
1540 : : /**
1541 : : * Create a full copy of a given packet mbuf.
1542 : : *
1543 : : * Copies all the data from a given packet mbuf to a newly allocated
1544 : : * set of mbufs. The private data are is not copied.
1545 : : *
1546 : : * @param m
1547 : : * The packet mbuf to be copied.
1548 : : * @param mp
1549 : : * The mempool from which the "clone" mbufs are allocated.
1550 : : * @param offset
1551 : : * The number of bytes to skip before copying.
1552 : : * If the mbuf does not have that many bytes, it is an error
1553 : : * and NULL is returned.
1554 : : * @param length
1555 : : * The upper limit on bytes to copy. Passing UINT32_MAX
1556 : : * means all data (after offset).
1557 : : * @return
1558 : : * - The pointer to the new "clone" mbuf on success.
1559 : : * - NULL if allocation fails.
1560 : : */
1561 : : struct rte_mbuf *
1562 : : rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1563 : : uint32_t offset, uint32_t length);
1564 : :
1565 : : /**
1566 : : * Adds given value to the refcnt of all packet mbuf segments.
1567 : : *
1568 : : * Walks through all segments of given packet mbuf and for each of them
1569 : : * invokes rte_mbuf_refcnt_update().
1570 : : *
1571 : : * @param m
1572 : : * The packet mbuf whose refcnt to be updated.
1573 : : * @param v
1574 : : * The value to add to the mbuf's segments refcnt.
1575 : : */
1576 : : static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1577 : : {
1578 : : __rte_mbuf_sanity_check(m, 1);
1579 : :
1580 : : do {
1581 : 133676 : rte_mbuf_refcnt_update(m, v);
1582 [ - + - + ]: 133676 : } while ((m = m->next) != NULL);
1583 : : }
1584 : :
1585 : : /**
1586 : : * Get the headroom in a packet mbuf.
1587 : : *
1588 : : * @param m
1589 : : * The packet mbuf.
1590 : : * @return
1591 : : * The length of the headroom.
1592 : : */
1593 : 0 : static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1594 : : {
1595 : : __rte_mbuf_sanity_check(m, 0);
1596 [ + + # # : 41203 : return m->data_off;
# # ]
1597 : : }
1598 : :
1599 : : /**
1600 : : * Get the tailroom of a packet mbuf.
1601 : : *
1602 : : * @param m
1603 : : * The packet mbuf.
1604 : : * @return
1605 : : * The length of the tailroom.
1606 : : */
1607 : 0 : static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1608 : : {
1609 : : __rte_mbuf_sanity_check(m, 0);
1610 [ + - ]: 5473 : return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1611 [ + + + + : 25346 : m->data_len);
+ + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - ]
1612 : : }
1613 : :
1614 : : /**
1615 : : * Get the last segment of the packet.
1616 : : *
1617 : : * @param m
1618 : : * The packet mbuf.
1619 : : * @return
1620 : : * The last segment of the given mbuf.
1621 : : */
1622 : 0 : static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1623 : : {
1624 : : __rte_mbuf_sanity_check(m, 1);
1625 [ + + + + : 24061 : while (m->next != NULL)
- + - + -
+ - + + +
- + + + -
+ + + + +
+ + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - - - -
- + - + -
- - - - -
- - - - -
- - - - -
- - - + -
+ - + - +
- + - + -
+ - + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
+ - + - -
- - - - -
- - + - +
+ + ]
1626 : : m = m->next;
1627 : 0 : return m;
1628 : : }
1629 : :
1630 : : /**
1631 : : * A macro that returns the length of the packet.
1632 : : *
1633 : : * The value can be read or assigned.
1634 : : *
1635 : : * @param m
1636 : : * The packet mbuf.
1637 : : */
1638 : : #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1639 : :
1640 : : /**
1641 : : * A macro that returns the length of the segment.
1642 : : *
1643 : : * The value can be read or assigned.
1644 : : *
1645 : : * @param m
1646 : : * The packet mbuf.
1647 : : */
1648 : : #define rte_pktmbuf_data_len(m) ((m)->data_len)
1649 : :
1650 : : /**
1651 : : * Prepend len bytes to an mbuf data area.
1652 : : *
1653 : : * Returns a pointer to the new
1654 : : * data start address. If there is not enough headroom in the first
1655 : : * segment, the function will return NULL, without modifying the mbuf.
1656 : : *
1657 : : * @param m
1658 : : * The pkt mbuf.
1659 : : * @param len
1660 : : * The amount of data to prepend (in bytes).
1661 : : * @return
1662 : : * A pointer to the start of the newly prepended data, or
1663 : : * NULL if there is not enough headroom space in the first segment
1664 : : */
1665 : 0 : static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1666 : : uint16_t len)
1667 : : {
1668 : : __rte_mbuf_sanity_check(m, 1);
1669 : :
1670 [ + - + - : 4111 : if (unlikely(len > rte_pktmbuf_headroom(m)))
+ - + - +
- # # ]
1671 : : return NULL;
1672 : :
1673 : : /* NB: elaborating the subtraction like this instead of using
1674 : : * -= allows us to ensure the result type is uint16_t
1675 : : * avoiding compiler warnings on gcc 8.1 at least */
1676 : 4109 : m->data_off = (uint16_t)(m->data_off - len);
1677 : 4109 : m->data_len = (uint16_t)(m->data_len + len);
1678 : 4101 : m->pkt_len = (m->pkt_len + len);
1679 : :
1680 [ - + - + : 4109 : return (char *)m->buf_addr + m->data_off;
- + - + ]
1681 : : }
1682 : :
1683 : : /**
1684 : : * Append len bytes to an mbuf.
1685 : : *
1686 : : * Append len bytes to an mbuf and return a pointer to the start address
1687 : : * of the added data. If there is not enough tailroom in the last
1688 : : * segment, the function will return NULL, without modifying the mbuf.
1689 : : *
1690 : : * @param m
1691 : : * The packet mbuf.
1692 : : * @param len
1693 : : * The amount of data to append (in bytes).
1694 : : * @return
1695 : : * A pointer to the start of the newly appended data, or
1696 : : * NULL if there is not enough tailroom space in the last segment
1697 : : */
1698 : 0 : static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1699 : : {
1700 : : void *tail;
1701 : : struct rte_mbuf *m_last;
1702 : :
1703 : : __rte_mbuf_sanity_check(m, 1);
1704 : :
1705 : : m_last = rte_pktmbuf_lastseg(m);
1706 [ + - + - : 5141 : if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
+ - + - +
- + - + -
+ - + - +
- + - + -
+ + + - +
- + - + -
+ - - - -
- - - - -
+ - + - +
- - - - -
+ - + - -
- - - - -
- - - - -
- - - - -
- - + - +
- + - + -
+ - + - +
- + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- + - + -
- - - - -
- + - +
- ]
1707 : : return NULL;
1708 : :
1709 : 5109 : tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1710 : 5139 : m_last->data_len = (uint16_t)(m_last->data_len + len);
1711 [ + + + + : 651 : m->pkt_len = (m->pkt_len + len);
- + - + -
+ - + - +
- + - - -
+ - + - +
- + - - -
- - - - -
- + - + ]
1712 : 4504 : return (char*) tail;
1713 : : }
1714 : :
1715 : : /**
1716 : : * Remove len bytes at the beginning of an mbuf.
1717 : : *
1718 : : * Returns a pointer to the start address of the new data area. If the
1719 : : * length is greater than the length of the first segment, then the
1720 : : * function will fail and return NULL, without modifying the mbuf.
1721 : : *
1722 : : * @param m
1723 : : * The packet mbuf.
1724 : : * @param len
1725 : : * The amount of data to remove (in bytes).
1726 : : * @return
1727 : : * A pointer to the new start of the data.
1728 : : */
1729 : : static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1730 : : {
1731 : : __rte_mbuf_sanity_check(m, 1);
1732 : :
1733 [ + - # # : 2 : if (unlikely(len > m->data_len))
# # # # #
# # # # #
# # ]
1734 : : return NULL;
1735 : :
1736 : : /* NB: elaborating the addition like this instead of using
1737 : : * += allows us to ensure the result type is uint16_t
1738 : : * avoiding compiler warnings on gcc 8.1 at least */
1739 : 2 : m->data_len = (uint16_t)(m->data_len - len);
1740 : 2 : m->data_off = (uint16_t)(m->data_off + len);
1741 : 2 : m->pkt_len = (m->pkt_len - len);
1742 : 2 : return (char *)m->buf_addr + m->data_off;
1743 : : }
1744 : :
1745 : : /**
1746 : : * Remove len bytes of data at the end of the mbuf.
1747 : : *
1748 : : * If the length is greater than the length of the last segment, the
1749 : : * function will fail and return -1 without modifying the mbuf.
1750 : : *
1751 : : * @param m
1752 : : * The packet mbuf.
1753 : : * @param len
1754 : : * The amount of data to remove (in bytes).
1755 : : * @return
1756 : : * - 0: On success.
1757 : : * - -1: On error.
1758 : : */
1759 : : static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1760 : : {
1761 : : struct rte_mbuf *m_last;
1762 : :
1763 : : __rte_mbuf_sanity_check(m, 1);
1764 : :
1765 : : m_last = rte_pktmbuf_lastseg(m);
1766 [ + - - + ]: 6 : if (unlikely(len > m_last->data_len))
1767 : : return -1;
1768 : :
1769 : 4 : m_last->data_len = (uint16_t)(m_last->data_len - len);
1770 [ - + ]: 4 : m->pkt_len = (m->pkt_len - len);
1771 : 2 : return 0;
1772 : : }
1773 : :
1774 : : /**
1775 : : * Test if mbuf data is contiguous.
1776 : : *
1777 : : * @param m
1778 : : * The packet mbuf.
1779 : : * @return
1780 : : * - 1, if all data is contiguous (one segment).
1781 : : * - 0, if there is several segments.
1782 : : */
1783 : : static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1784 : : {
1785 : : __rte_mbuf_sanity_check(m, 1);
1786 [ + + - + : 232 : return m->nb_segs == 1;
- + - + -
+ - + -
+ ]
1787 : : }
1788 : :
1789 : : /**
1790 : : * @internal used by rte_pktmbuf_read().
1791 : : */
1792 : : const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1793 : : uint32_t len, void *buf);
1794 : :
1795 : : /**
1796 : : * Read len data bytes in a mbuf at specified offset.
1797 : : *
1798 : : * If the data is contiguous, return the pointer in the mbuf data, else
1799 : : * copy the data in the buffer provided by the user and return its
1800 : : * pointer.
1801 : : *
1802 : : * @param m
1803 : : * The pointer to the mbuf.
1804 : : * @param off
1805 : : * The offset of the data in the mbuf.
1806 : : * @param len
1807 : : * The amount of bytes to read.
1808 : : * @param buf
1809 : : * The buffer where data is copied if it is not contiguous in mbuf
1810 : : * data. Its length should be at least equal to the len parameter.
1811 : : * @return
1812 : : * The pointer to the data, either in the mbuf if it is contiguous,
1813 : : * or in the user buffer. If mbuf is too small, NULL is returned.
1814 : : */
1815 : 0 : static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1816 : : uint32_t off, uint32_t len, void *buf)
1817 : : {
1818 [ + + + + : 207 : if (likely(off + len <= rte_pktmbuf_data_len(m)))
+ + + - +
+ + + + +
+ - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - -
- ]
1819 [ - + ]: 170 : return rte_pktmbuf_mtod_offset(m, char *, off);
1820 : : else
1821 : 45 : return __rte_pktmbuf_read(m, off, len, buf);
1822 : : }
1823 : :
1824 : : /**
1825 : : * Chain an mbuf to another, thereby creating a segmented packet.
1826 : : *
1827 : : * Note: The implementation will do a linear walk over the segments to find
1828 : : * the tail entry. For cases when there are many segments, it's better to
1829 : : * chain the entries manually.
1830 : : *
1831 : : * @param head
1832 : : * The head of the mbuf chain (the first packet)
1833 : : * @param tail
1834 : : * The mbuf to put last in the chain
1835 : : *
1836 : : * @return
1837 : : * - 0, on success.
1838 : : * - -EOVERFLOW, if the chain segment limit exceeded
1839 : : */
1840 : : static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1841 : : {
1842 : : struct rte_mbuf *cur_tail;
1843 : :
1844 : : /* Check for number-of-segments-overflow */
1845 [ + - + - : 665 : if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
+ - + - -
+ + - ]
1846 : : return -EOVERFLOW;
1847 : :
1848 : : /* Chain 'tail' onto the old tail */
1849 : : cur_tail = rte_pktmbuf_lastseg(head);
1850 : 665 : cur_tail->next = tail;
1851 : :
1852 : : /* accumulate number of segments and total length.
1853 : : * NB: elaborating the addition like this instead of using
1854 : : * -= allows us to ensure the result type is uint16_t
1855 : : * avoiding compiler warnings on gcc 8.1 at least */
1856 : 665 : head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1857 : 665 : head->pkt_len += tail->pkt_len;
1858 : :
1859 : : /* pkt_len is only set in the head */
1860 : 211 : tail->pkt_len = tail->data_len;
1861 : :
1862 : 538 : return 0;
1863 : : }
1864 : :
1865 : : /**
1866 : : * For given input values generate raw tx_offload value.
1867 : : * Note that it is caller responsibility to make sure that input parameters
1868 : : * don't exceed maximum bit-field values.
1869 : : * @param il2
1870 : : * l2_len value.
1871 : : * @param il3
1872 : : * l3_len value.
1873 : : * @param il4
1874 : : * l4_len value.
1875 : : * @param tso
1876 : : * tso_segsz value.
1877 : : * @param ol3
1878 : : * outer_l3_len value.
1879 : : * @param ol2
1880 : : * outer_l2_len value.
1881 : : * @param unused
1882 : : * unused value.
1883 : : * @return
1884 : : * raw tx_offload value.
1885 : : */
1886 : : static __rte_always_inline uint64_t
1887 : : rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1888 : : uint64_t ol3, uint64_t ol2, uint64_t unused)
1889 : : {
1890 : 65536 : return il2 << RTE_MBUF_L2_LEN_OFS |
1891 : 65536 : il3 << RTE_MBUF_L3_LEN_OFS |
1892 : 65536 : il4 << RTE_MBUF_L4_LEN_OFS |
1893 : 65536 : tso << RTE_MBUF_TSO_SEGSZ_OFS |
1894 : 65536 : ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1895 : 65536 : ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1896 : : unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1897 : : }
1898 : :
1899 : : /**
1900 : : * Validate general requirements for Tx offload in mbuf.
1901 : : *
1902 : : * This function checks correctness and completeness of Tx offload settings.
1903 : : *
1904 : : * @param m
1905 : : * The packet mbuf to be validated.
1906 : : * @return
1907 : : * 0 if packet is valid
1908 : : */
1909 : : static inline int
1910 : 11 : rte_validate_tx_offload(const struct rte_mbuf *m)
1911 : : {
1912 : 11 : uint64_t ol_flags = m->ol_flags;
1913 : :
1914 : : /* Does packet set any of available offloads? */
1915 [ + + ]: 11 : if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
1916 : : return 0;
1917 : :
1918 : : /* IP checksum can be counted only for IPv4 packet */
1919 [ + + ]: 10 : if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
1920 : : return -EINVAL;
1921 : :
1922 : : /* IP type not set when required */
1923 [ + + ]: 9 : if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
1924 [ + + ]: 7 : if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
1925 : : return -EINVAL;
1926 : :
1927 : : /* Check requirements for TSO packet */
1928 [ + + ]: 7 : if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1929 [ + + ]: 4 : if ((m->tso_segsz == 0) ||
1930 [ + + ]: 3 : ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
1931 : : !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1932 : : return -EINVAL;
1933 : :
1934 : : /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1935 [ + + ]: 5 : if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
1936 : : !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
1937 : 1 : return -EINVAL;
1938 : :
1939 : : return 0;
1940 : : }
1941 : :
1942 : : /**
1943 : : * @internal used by rte_pktmbuf_linearize().
1944 : : */
1945 : : int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1946 : :
1947 : : /**
1948 : : * Linearize data in mbuf.
1949 : : *
1950 : : * This function moves the mbuf data in the first segment if there is enough
1951 : : * tailroom. The subsequent segments are unchained and freed.
1952 : : *
1953 : : * @param mbuf
1954 : : * mbuf to linearize
1955 : : * @return
1956 : : * - 0, on success
1957 : : * - -1, on error
1958 : : */
1959 : : static inline int
1960 : : rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
1961 : : {
1962 [ + + ]: 10 : if (rte_pktmbuf_is_contiguous(mbuf))
1963 : : return 0;
1964 : 8 : return __rte_pktmbuf_linearize(mbuf);
1965 : : }
1966 : :
1967 : : /**
1968 : : * Dump an mbuf structure to a file.
1969 : : *
1970 : : * Dump all fields for the given packet mbuf and all its associated
1971 : : * segments (in the case of a chained buffer).
1972 : : *
1973 : : * @param f
1974 : : * A pointer to a file for output
1975 : : * @param m
1976 : : * The packet mbuf.
1977 : : * @param dump_len
1978 : : * If dump_len != 0, also dump the "dump_len" first data bytes of
1979 : : * the packet.
1980 : : */
1981 : : void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
1982 : :
1983 : : /**
1984 : : * Get the value of mbuf sched queue_id field.
1985 : : */
1986 : : static inline uint32_t
1987 : : rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
1988 : : {
1989 : 6 : return m->hash.sched.queue_id;
1990 : : }
1991 : :
1992 : : /**
1993 : : * Get the value of mbuf sched traffic_class field.
1994 : : */
1995 : : static inline uint8_t
1996 : : rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
1997 : : {
1998 : : return m->hash.sched.traffic_class;
1999 : : }
2000 : :
2001 : : /**
2002 : : * Get the value of mbuf sched color field.
2003 : : */
2004 : : static inline uint8_t
2005 : : rte_mbuf_sched_color_get(const struct rte_mbuf *m)
2006 : : {
2007 : 10 : return m->hash.sched.color;
2008 : : }
2009 : :
2010 : : /**
2011 : : * Get the values of mbuf sched queue_id, traffic_class and color.
2012 : : *
2013 : : * @param m
2014 : : * Mbuf to read
2015 : : * @param queue_id
2016 : : * Returns the queue id
2017 : : * @param traffic_class
2018 : : * Returns the traffic class id
2019 : : * @param color
2020 : : * Returns the colour id
2021 : : */
2022 : : static inline void
2023 : : rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
2024 : : uint8_t *traffic_class,
2025 : : uint8_t *color)
2026 : : {
2027 : : struct rte_mbuf_sched sched = m->hash.sched;
2028 : :
2029 : : *queue_id = sched.queue_id;
2030 : : *traffic_class = sched.traffic_class;
2031 : : *color = sched.color;
2032 : : }
2033 : :
2034 : : /**
2035 : : * Set the mbuf sched queue_id to the defined value.
2036 : : */
2037 : : static inline void
2038 : : rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
2039 : : {
2040 : : m->hash.sched.queue_id = queue_id;
2041 : : }
2042 : :
2043 : : /**
2044 : : * Set the mbuf sched traffic_class id to the defined value.
2045 : : */
2046 : : static inline void
2047 : : rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
2048 : : {
2049 : : m->hash.sched.traffic_class = traffic_class;
2050 : : }
2051 : :
2052 : : /**
2053 : : * Set the mbuf sched color id to the defined value.
2054 : : */
2055 : : static inline void
2056 : 0 : rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
2057 : : {
2058 : 0 : m->hash.sched.color = color;
2059 : 0 : }
2060 : :
2061 : : /**
2062 : : * Set the mbuf sched queue_id, traffic_class and color.
2063 : : *
2064 : : * @param m
2065 : : * Mbuf to set
2066 : : * @param queue_id
2067 : : * Queue id value to be set
2068 : : * @param traffic_class
2069 : : * Traffic class id value to be set
2070 : : * @param color
2071 : : * Color id to be set
2072 : : */
2073 : : static inline void
2074 : 0 : rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id,
2075 : : uint8_t traffic_class,
2076 : : uint8_t color)
2077 : : {
2078 : 10 : m->hash.sched = (struct rte_mbuf_sched){
2079 : : .queue_id = queue_id,
2080 : : .traffic_class = traffic_class,
2081 : : .color = color,
2082 : : .reserved = 0,
2083 : : };
2084 : 0 : }
2085 : :
2086 : : #ifdef __cplusplus
2087 : : }
2088 : : #endif
2089 : :
2090 : : #endif /* _RTE_MBUF_H_ */
|
