Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright 2014 6WIND S.A.
4 : : */
5 : :
6 : : #ifndef _RTE_MBUF_H_
7 : : #define _RTE_MBUF_H_
8 : :
9 : : /**
10 : : * @file
11 : : * RTE Mbuf
12 : : *
13 : : * The mbuf library provides the ability to create and destroy buffers
14 : : * that may be used by the RTE application to store message
15 : : * buffers. The message buffers are stored in a mempool, using the
16 : : * RTE mempool library.
17 : : *
18 : : * The preferred way to create a mbuf pool is to use
19 : : * rte_pktmbuf_pool_create(). However, in some situations, an
20 : : * application may want to have more control (ex: populate the pool with
21 : : * specific memory), in this case it is possible to use functions from
22 : : * rte_mempool. See how rte_pktmbuf_pool_create() is implemented for
23 : : * details.
24 : : *
25 : : * This library provides an API to allocate/free packet mbufs, which are
26 : : * used to carry network packets.
27 : : *
28 : : * To understand the concepts of packet buffers or mbufs, you
29 : : * should read "TCP/IP Illustrated, Volume 2: The Implementation,
30 : : * Addison-Wesley, 1995, ISBN 0-201-63354-X from Richard Stevens"
31 : : * http://www.kohala.com/start/tcpipiv2.html
32 : : */
33 : :
34 : : #include <stdbool.h>
35 : : #include <stdint.h>
36 : :
37 : : #include <rte_common.h>
38 : : #include <rte_config.h>
39 : : #include <rte_mempool.h>
40 : : #include <rte_prefetch.h>
41 : : #include <rte_branch_prediction.h>
42 : : #include <rte_mbuf_ptype.h>
43 : : #include <rte_mbuf_core.h>
44 : : #include <rte_mbuf_history.h>
45 : :
46 : : #ifdef __cplusplus
47 : : extern "C" {
48 : : #endif
49 : :
50 : : /**
51 : : * Get the name of a RX offload flag
52 : : *
53 : : * @param mask
54 : : * The mask describing the flag.
55 : : * @return
56 : : * The name of this flag, or NULL if it's not a valid RX flag.
57 : : */
58 : : const char *rte_get_rx_ol_flag_name(uint64_t mask);
59 : :
60 : : /**
61 : : * Dump the list of RX offload flags in a buffer
62 : : *
63 : : * @param mask
64 : : * The mask describing the RX flags.
65 : : * @param buf
66 : : * The output buffer.
67 : : * @param buflen
68 : : * The length of the buffer.
69 : : * @return
70 : : * 0 on success, (-1) on error.
71 : : */
72 : : int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
73 : :
74 : : /**
75 : : * Get the name of a TX offload flag
76 : : *
77 : : * @param mask
78 : : * The mask describing the flag. Usually only one bit must be set.
79 : : * Several bits can be given if they belong to the same mask.
80 : : * Ex: RTE_MBUF_F_TX_L4_MASK.
81 : : * @return
82 : : * The name of this flag, or NULL if it's not a valid TX flag.
83 : : */
84 : : const char *rte_get_tx_ol_flag_name(uint64_t mask);
85 : :
86 : : /**
87 : : * Dump the list of TX offload flags in a buffer
88 : : *
89 : : * @param mask
90 : : * The mask describing the TX flags.
91 : : * @param buf
92 : : * The output buffer.
93 : : * @param buflen
94 : : * The length of the buffer.
95 : : * @return
96 : : * 0 on success, (-1) on error.
97 : : */
98 : : int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
99 : :
100 : : /**
101 : : * Prefetch the first part of the mbuf
102 : : *
103 : : * The first 64 bytes of the mbuf corresponds to fields that are used early
104 : : * in the receive path. If the cache line of the architecture is higher than
105 : : * 64B, the second part will also be prefetched.
106 : : *
107 : : * @param m
108 : : * The pointer to the mbuf.
109 : : */
110 : : static inline void
111 : : rte_mbuf_prefetch_part1(struct rte_mbuf *m)
112 : : {
113 : : rte_prefetch0(m);
114 : 0 : }
115 : :
116 : : /**
117 : : * Prefetch the second part of the mbuf
118 : : *
119 : : * The next 64 bytes of the mbuf corresponds to fields that are used in the
120 : : * transmit path. If the cache line of the architecture is higher than 64B,
121 : : * this function does nothing as it is expected that the full mbuf is
122 : : * already in cache.
123 : : *
124 : : * @param m
125 : : * The pointer to the mbuf.
126 : : */
127 : : static inline void
128 : : rte_mbuf_prefetch_part2(struct rte_mbuf *m)
129 : : {
130 : : #if RTE_CACHE_LINE_SIZE == 64
131 : 0 : rte_prefetch0(RTE_PTR_ADD(m, RTE_CACHE_LINE_MIN_SIZE));
132 : : #else
133 : : RTE_SET_USED(m);
134 : : #endif
135 : 0 : }
136 : :
137 : :
138 : : static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp);
139 : :
140 : : /**
141 : : * Get the IOVA address of the mbuf data buffer.
142 : : *
143 : : * @param m
144 : : * The pointer to the mbuf.
145 : : * @return
146 : : * The IOVA address of the mbuf.
147 : : */
148 : : static inline rte_iova_t
149 : 0 : rte_mbuf_iova_get(const struct rte_mbuf *m)
150 : : {
151 : : #if RTE_IOVA_IN_MBUF
152 [ + + - + : 421 : return m->buf_iova;
- + - + -
+ - - + +
+ + - + -
- - - - +
- + - - -
- ]
153 : : #else
154 : : return (rte_iova_t)m->buf_addr;
155 : : #endif
156 : : }
157 : :
158 : : /**
159 : : * Set the IOVA address of the mbuf data buffer.
160 : : *
161 : : * @param m
162 : : * The pointer to the mbuf.
163 : : * @param iova
164 : : * Value to set as IOVA address of the mbuf.
165 : : */
166 : : static inline void
167 : 0 : rte_mbuf_iova_set(struct rte_mbuf *m, rte_iova_t iova)
168 : : {
169 : : #if RTE_IOVA_IN_MBUF
170 [ + + ]: 192 : m->buf_iova = iova;
171 : : #else
172 : : RTE_SET_USED(m);
173 : : RTE_SET_USED(iova);
174 : : #endif
175 : 0 : }
176 : :
177 : : /**
178 : : * Return the IO address of the beginning of the mbuf data
179 : : *
180 : : * @param mb
181 : : * The pointer to the mbuf.
182 : : * @return
183 : : * The IO address of the beginning of the mbuf data
184 : : */
185 : : static inline rte_iova_t
186 : 0 : rte_mbuf_data_iova(const struct rte_mbuf *mb)
187 : : {
188 [ # # # # : 0 : return rte_mbuf_iova_get(mb) + mb->data_off;
# # # # ]
189 : : }
190 : :
191 : : /**
192 : : * Return the default IO address of the beginning of the mbuf data
193 : : *
194 : : * This function is used by drivers in their receive function, as it
195 : : * returns the location where data should be written by the NIC, taking
196 : : * the default headroom in account.
197 : : *
198 : : * @param mb
199 : : * The pointer to the mbuf.
200 : : * @return
201 : : * The IO address of the beginning of the mbuf data
202 : : */
203 : : static inline rte_iova_t
204 : : rte_mbuf_data_iova_default(const struct rte_mbuf *mb)
205 : : {
206 [ # # # # : 0 : return rte_mbuf_iova_get(mb) + RTE_PKTMBUF_HEADROOM;
# # # # ]
207 : : }
208 : :
209 : : /**
210 : : * Return the mbuf owning the data buffer address of an indirect mbuf.
211 : : *
212 : : * @param mi
213 : : * The pointer to the indirect mbuf.
214 : : * @return
215 : : * The address of the direct mbuf corresponding to buffer_addr.
216 : : */
217 : : static inline struct rte_mbuf *
218 : : rte_mbuf_from_indirect(struct rte_mbuf *mi)
219 : : {
220 : 58 : return (struct rte_mbuf *)RTE_PTR_SUB(mi->buf_addr, sizeof(*mi) + mi->priv_size);
221 : : }
222 : :
223 : : /**
224 : : * Return address of buffer embedded in the given mbuf.
225 : : *
226 : : * The return value shall be same as mb->buf_addr if the mbuf is already
227 : : * initialized and direct. However, this API is useful if mempool of the
228 : : * mbuf is already known because it doesn't need to access mbuf contents in
229 : : * order to get the mempool pointer.
230 : : *
231 : : * @param mb
232 : : * The pointer to the mbuf.
233 : : * @param mp
234 : : * The pointer to the mempool of the mbuf.
235 : : * @return
236 : : * The pointer of the mbuf buffer.
237 : : */
238 : : static inline char *
239 : : rte_mbuf_buf_addr(struct rte_mbuf *mb, struct rte_mempool *mp)
240 : : {
241 : : return (char *)mb + sizeof(*mb) + rte_pktmbuf_priv_size(mp);
242 : : }
243 : :
244 : : /**
245 : : * Return the default address of the beginning of the mbuf data.
246 : : *
247 : : * @param mb
248 : : * The pointer to the mbuf.
249 : : * @return
250 : : * The pointer of the beginning of the mbuf data.
251 : : */
252 : : static inline char *
253 : : rte_mbuf_data_addr_default(struct rte_mbuf *mb)
254 : : {
255 : : return rte_mbuf_buf_addr(mb, mb->pool) + RTE_PKTMBUF_HEADROOM;
256 : : }
257 : :
258 : : /**
259 : : * Return address of buffer embedded in the given mbuf.
260 : : *
261 : : * @note: Accessing mempool pointer of a mbuf is expensive because the
262 : : * pointer is stored in the 2nd cache line of mbuf. If mempool is known, it
263 : : * is better not to reference the mempool pointer in mbuf but calling
264 : : * rte_mbuf_buf_addr() would be more efficient.
265 : : *
266 : : * @param md
267 : : * The pointer to the mbuf.
268 : : * @return
269 : : * The address of the data buffer owned by the mbuf.
270 : : */
271 : : static inline char *
272 : : rte_mbuf_to_baddr(struct rte_mbuf *md)
273 : : {
274 : : return rte_mbuf_buf_addr(md, md->pool);
275 : : }
276 : :
277 : : /**
278 : : * Return the starting address of the private data area embedded in
279 : : * the given mbuf.
280 : : *
281 : : * Note that no check is made to ensure that a private data area
282 : : * actually exists in the supplied mbuf.
283 : : *
284 : : * @param m
285 : : * The pointer to the mbuf.
286 : : * @return
287 : : * The starting address of the private data area of the given mbuf.
288 : : */
289 : : static inline void *
290 : 0 : rte_mbuf_to_priv(struct rte_mbuf *m)
291 : : {
292 : 0 : return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
293 : : }
294 : :
295 : : /**
296 : : * Private data in case of pktmbuf pool.
297 : : *
298 : : * A structure that contains some pktmbuf_pool-specific data that are
299 : : * appended after the mempool structure (in private data).
300 : : */
301 : : struct rte_pktmbuf_pool_private {
302 : : uint16_t mbuf_data_room_size; /**< Size of data space in each mbuf. */
303 : : uint16_t mbuf_priv_size; /**< Size of private area in each mbuf. */
304 : : uint32_t flags; /**< reserved for future use. */
305 : : };
306 : :
307 : : /**
308 : : * Return the flags from private data in an mempool structure.
309 : : *
310 : : * @param mp
311 : : * A pointer to the mempool structure.
312 : : * @return
313 : : * The flags from the private data structure.
314 : : */
315 : : static inline uint32_t
316 [ # # ]: 0 : rte_pktmbuf_priv_flags(struct rte_mempool *mp)
317 : : {
318 : : struct rte_pktmbuf_pool_private *mbp_priv;
319 : :
320 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
321 [ + + + + ]: 3401 : return mbp_priv->flags;
322 : : }
323 : :
324 : : /**
325 : : * When set, pktmbuf mempool will hold only mbufs with pinned external
326 : : * buffer. The external buffer will be attached to the mbuf at the
327 : : * memory pool creation and will never be detached by the mbuf free calls.
328 : : * mbuf should not contain any room for data after the mbuf structure.
329 : : */
330 : : #define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
331 : :
332 : : /**
333 : : * Returns non zero if given mbuf has a pinned external buffer, or zero
334 : : * otherwise. The pinned external buffer is allocated at pool creation
335 : : * time and should not be freed on mbuf freeing.
336 : : *
337 : : * External buffer is a user-provided anonymous buffer.
338 : : */
339 : : #define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
340 : : (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
341 : :
342 : : #if defined RTE_LIBRTE_MBUF_DEBUG || defined __DOXYGEN__
343 : :
344 : : /** Check reinitialized mbuf type in debug mode. */
345 : : #define __rte_mbuf_raw_sanity_check_mp(m, mp) rte_mbuf_raw_sanity_check(m, mp)
346 : : /** Check mbuf type in debug mode. */
347 : : #define __rte_mbuf_sanity_check(m, is_h) rte_mbuf_sanity_check(m, is_h)
348 : :
349 : : #else /* !RTE_LIBRTE_MBUF_DEBUG */
350 : :
351 : : #define __rte_mbuf_raw_sanity_check_mp(m, mp) do { } while (0)
352 : : #define __rte_mbuf_sanity_check(m, is_h) do { } while (0)
353 : :
354 : : #endif /* RTE_LIBRTE_MBUF_DEBUG */
355 : :
356 : : #ifdef RTE_MBUF_REFCNT_ATOMIC
357 : :
358 : : /**
359 : : * Reads the value of an mbuf's refcnt.
360 : : * @param m
361 : : * Mbuf to read
362 : : * @return
363 : : * Reference count number.
364 : : */
365 : : static inline uint16_t
366 : 2378168 : rte_mbuf_refcnt_read(const struct rte_mbuf *m)
367 : : {
368 [ + + - + : 2100626 : return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
- + - + -
+ # # # #
# # # # ]
369 : : }
370 : :
371 : : /**
372 : : * Sets an mbuf's refcnt to a defined value.
373 : : * @param m
374 : : * Mbuf to update
375 : : * @param new_value
376 : : * Value set
377 : : */
378 : : static inline void
379 : 0 : rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
380 : : {
381 [ + - # # ]: 89875 : rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
382 : 0 : }
383 : :
384 : : /* internal */
385 : : static inline uint16_t
386 : 262365 : __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
387 : : {
388 : 393716 : return rte_atomic_fetch_add_explicit(&m->refcnt, value,
389 : 393716 : rte_memory_order_acq_rel) + value;
390 : : }
391 : :
392 : : /**
393 : : * Adds given value to an mbuf's refcnt and returns its new value.
394 : : * @param m
395 : : * Mbuf to update
396 : : * @param value
397 : : * Value to add/subtract
398 : : * @return
399 : : * Updated value
400 : : */
401 : : static inline uint16_t
402 : 135470 : rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
403 : : {
404 : : /*
405 : : * The atomic_add is an expensive operation, so we don't want to
406 : : * call it in the case where we know we are the unique holder of
407 : : * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
408 : : * operation has to be used because concurrent accesses on the
409 : : * reference counter can occur.
410 : : */
411 [ + + ]: 135470 : if (likely(rte_mbuf_refcnt_read(m) == 1)) {
412 : 4119 : ++value;
413 : 4119 : rte_mbuf_refcnt_set(m, (uint16_t)value);
414 : 4119 : return (uint16_t)value;
415 : : }
416 : :
417 : 131351 : return __rte_mbuf_refcnt_update(m, value);
418 : : }
419 : :
420 : : #else /* ! RTE_MBUF_REFCNT_ATOMIC */
421 : :
422 : : /* internal */
423 : : static inline uint16_t
424 : : __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
425 : : {
426 : : m->refcnt = (uint16_t)(m->refcnt + value);
427 : : return m->refcnt;
428 : : }
429 : :
430 : : /**
431 : : * Adds given value to an mbuf's refcnt and returns its new value.
432 : : */
433 : : static inline uint16_t
434 : : rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
435 : : {
436 : : return __rte_mbuf_refcnt_update(m, value);
437 : : }
438 : :
439 : : /**
440 : : * Reads the value of an mbuf's refcnt.
441 : : */
442 : : static inline uint16_t
443 : : rte_mbuf_refcnt_read(const struct rte_mbuf *m)
444 : : {
445 : : return m->refcnt;
446 : : }
447 : :
448 : : /**
449 : : * Sets an mbuf's refcnt to the defined value.
450 : : */
451 : : static inline void
452 : : rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
453 : : {
454 : : m->refcnt = new_value;
455 : : }
456 : :
457 : : #endif /* RTE_MBUF_REFCNT_ATOMIC */
458 : :
459 : : /**
460 : : * Reads the refcnt of an external buffer.
461 : : *
462 : : * @param shinfo
463 : : * Shared data of the external buffer.
464 : : * @return
465 : : * Reference count number.
466 : : */
467 : : static inline uint16_t
468 : : rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
469 : : {
470 [ - + - + : 10 : return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
- + ]
471 : : }
472 : :
473 : : /**
474 : : * Set refcnt of an external buffer.
475 : : *
476 : : * @param shinfo
477 : : * Shared data of the external buffer.
478 : : * @param new_value
479 : : * Value set
480 : : */
481 : : static inline void
482 : 0 : rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
483 : : uint16_t new_value)
484 : : {
485 [ + + ]: 5 : rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
486 : 0 : }
487 : :
488 : : /**
489 : : * Add given value to refcnt of an external buffer and return its new
490 : : * value.
491 : : *
492 : : * @param shinfo
493 : : * Shared data of the external buffer.
494 : : * @param value
495 : : * Value to add/subtract
496 : : * @return
497 : : * Updated value
498 : : */
499 : : static inline uint16_t
500 : 18 : rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo,
501 : : int16_t value)
502 : : {
503 [ + + ]: 18 : if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) {
504 : 8 : ++value;
505 : 8 : rte_mbuf_ext_refcnt_set(shinfo, (uint16_t)value);
506 : 8 : return (uint16_t)value;
507 : : }
508 : :
509 : 10 : return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
510 : 10 : rte_memory_order_acq_rel) + value;
511 : : }
512 : :
513 : : /** Mbuf prefetch */
514 : : #define RTE_MBUF_PREFETCH_TO_FREE(m) do { \
515 : : if ((m) != NULL) \
516 : : rte_prefetch0(m); \
517 : : } while (0)
518 : :
519 : :
520 : : /**
521 : : * Sanity checks on a reinitialized mbuf.
522 : : *
523 : : * Check the consistency of the given reinitialized mbuf.
524 : : * The function will cause a panic if corruption is detected.
525 : : *
526 : : * Check that the mbuf is properly reinitialized (refcnt=1, next=NULL,
527 : : * nb_segs=1), as done by rte_pktmbuf_prefree_seg().
528 : : *
529 : : * @param m
530 : : * The mbuf to be checked.
531 : : * @param mp
532 : : * The mempool to which the mbuf belongs.
533 : : * NULL if unknown, not to be checked.
534 : : */
535 : : void
536 : : rte_mbuf_raw_sanity_check(const struct rte_mbuf *m, const struct rte_mempool *mp);
537 : :
538 : : /**
539 : : * Sanity checks on a reinitialized mbuf.
540 : : *
541 : : * Almost like rte_mbuf_raw_sanity_check(), but this function gives the reason
542 : : * if corruption is detected rather than panic.
543 : : *
544 : : * @param m
545 : : * The mbuf to be checked.
546 : : * @param mp
547 : : * The mempool to which the mbuf belongs.
548 : : * NULL if unknown, not to be checked.
549 : : * @param reason
550 : : * A reference to a string pointer where to store the reason
551 : : * why a mbuf is considered invalid.
552 : : * @return
553 : : * - 0 if no issue has been found, reason is left untouched.
554 : : * - -1 if a problem is detected, reason then points to a string
555 : : * describing the reason why the mbuf is deemed invalid.
556 : : */
557 : : int rte_mbuf_raw_check(const struct rte_mbuf *m, const struct rte_mempool *mp,
558 : : const char **reason);
559 : :
560 : : /**
561 : : * Sanity checks on an mbuf.
562 : : *
563 : : * Check the consistency of the given mbuf. The function will cause a
564 : : * panic if corruption is detected.
565 : : *
566 : : * @param m
567 : : * The mbuf to be checked.
568 : : * @param is_header
569 : : * True if the mbuf is a packet header, false if it is a sub-segment
570 : : * of a packet (in this case, some fields like nb_segs are not checked)
571 : : */
572 : : void
573 : : rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
574 : :
575 : : /**
576 : : * Sanity checks on a mbuf.
577 : : *
578 : : * Almost like rte_mbuf_sanity_check(), but this function gives the reason
579 : : * if corruption is detected rather than panic.
580 : : *
581 : : * @param m
582 : : * The mbuf to be checked.
583 : : * @param is_header
584 : : * True if the mbuf is a packet header, false if it is a sub-segment
585 : : * of a packet (in this case, some fields like nb_segs are not checked)
586 : : * @param reason
587 : : * A reference to a string pointer where to store the reason why a mbuf is
588 : : * considered invalid.
589 : : * @return
590 : : * - 0 if no issue has been found, reason is left untouched.
591 : : * - -1 if a problem is detected, reason then points to a string describing
592 : : * the reason why the mbuf is deemed invalid.
593 : : */
594 : : int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
595 : : const char **reason);
596 : :
597 : : /** For backwards compatibility. */
598 : : #define __rte_mbuf_raw_sanity_check(m) __rte_mbuf_raw_sanity_check_mp(m, NULL)
599 : :
600 : : /** For backwards compatibility. */
601 : : #define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check_mp(m, NULL)
602 : :
603 : : /**
604 : : * Allocate an uninitialized mbuf from mempool *mp*.
605 : : *
606 : : * This function can be used by PMDs (especially in RX functions) to
607 : : * allocate an uninitialized mbuf. The driver is responsible of
608 : : * initializing all the required fields. See rte_pktmbuf_reset().
609 : : * For standard needs, prefer rte_pktmbuf_alloc().
610 : : *
611 : : * The caller can expect that the following fields of the mbuf structure
612 : : * are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1,
613 : : * next=NULL, pool, priv_size. The other fields must be initialized
614 : : * by the caller.
615 : : *
616 : : * @param mp
617 : : * The mempool from which mbuf is allocated.
618 : : * @return
619 : : * - The pointer to the new mbuf on success.
620 : : * - NULL if allocation failed.
621 : : */
622 [ + + ]: 17529 : static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
623 : : {
624 : : union {
625 : : void *ptr;
626 : : struct rte_mbuf *m;
627 : : } ret;
628 : :
629 [ + + ]: 4616 : if (rte_mempool_get(mp, &ret.ptr) < 0)
630 : : return NULL;
631 : : __rte_mbuf_raw_sanity_check_mp(ret.m, mp);
632 : :
633 : 17521 : rte_mbuf_history_mark(ret.m, RTE_MBUF_HISTORY_OP_LIB_ALLOC);
634 : :
635 : 17521 : return ret.m;
636 : : }
637 : :
638 : : /**
639 : : * Allocate a bulk of uninitialized mbufs from mempool *mp*.
640 : : *
641 : : * This function can be used by PMDs (especially in Rx functions)
642 : : * to allocate a bulk of uninitialized mbufs.
643 : : * The driver is responsible of initializing all the required fields.
644 : : * See rte_pktmbuf_reset().
645 : : * For standard needs, prefer rte_pktmbuf_alloc_bulk().
646 : : *
647 : : * The caller can expect that the following fields of the mbuf structure
648 : : * are initialized:
649 : : * buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, next=NULL, pool, priv_size.
650 : : * The other fields must be initialized by the caller.
651 : : *
652 : : * @param mp
653 : : * The mempool from which mbufs are allocated.
654 : : * @param mbufs
655 : : * Array of pointers to mbufs.
656 : : * @param count
657 : : * Array size.
658 : : * @return
659 : : * - 0: Success.
660 : : * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved.
661 : : */
662 : : static __rte_always_inline int
663 : : rte_mbuf_raw_alloc_bulk(struct rte_mempool *mp, struct rte_mbuf **mbufs, unsigned int count)
664 : : {
665 : : int rc = rte_mempool_get_bulk(mp, (void **)mbufs, count);
666 : : if (likely(rc == 0)) {
667 : : for (unsigned int idx = 0; idx < count; idx++)
668 : : __rte_mbuf_raw_sanity_check_mp(mbufs[idx], mp);
669 : : }
670 : :
671 : : rte_mbuf_history_mark_bulk(mbufs, count, RTE_MBUF_HISTORY_OP_LIB_ALLOC);
672 : :
673 : : return rc;
674 : : }
675 : :
676 : : /**
677 : : * Put mbuf back into its original mempool.
678 : : *
679 : : * The caller must ensure that the mbuf is direct and properly
680 : : * reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by
681 : : * rte_pktmbuf_prefree_seg().
682 : : *
683 : : * This function should be used with care, when optimization is
684 : : * required. For standard needs, prefer rte_pktmbuf_free() or
685 : : * rte_pktmbuf_free_seg().
686 : : *
687 : : * @param m
688 : : * The mbuf to be freed.
689 : : */
690 : : static __rte_always_inline void
691 : : rte_mbuf_raw_free(struct rte_mbuf *m)
692 : : {
693 : : __rte_mbuf_raw_sanity_check_mp(m, NULL);
694 : : rte_mbuf_history_mark(m, RTE_MBUF_HISTORY_OP_LIB_FREE);
695 [ + + + + : 2106790 : rte_mempool_put(m->pool, m);
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
696 : 2106798 : }
697 : :
698 : : /**
699 : : * Put a bulk of mbufs allocated from the same mempool back into the mempool.
700 : : *
701 : : * The caller must ensure that the mbufs come from the specified mempool,
702 : : * are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1),
703 : : * as done by rte_pktmbuf_prefree_seg().
704 : : *
705 : : * This function should be used with care, when optimization is required.
706 : : * For standard needs, prefer rte_pktmbuf_free_bulk().
707 : : *
708 : : * @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE
709 : : *
710 : : * @param mp
711 : : * The mempool to which the mbufs belong.
712 : : * @param mbufs
713 : : * Array of pointers to packet mbufs.
714 : : * The array must not contain NULL pointers.
715 : : * @param count
716 : : * Array size.
717 : : */
718 : : static __rte_always_inline void
719 : : rte_mbuf_raw_free_bulk(struct rte_mempool *mp, struct rte_mbuf **mbufs, unsigned int count)
720 : : {
721 : : for (unsigned int idx = 0; idx < count; idx++)
722 : : __rte_mbuf_raw_sanity_check_mp(mbufs[idx], mp);
723 : : rte_mbuf_history_mark_bulk(mbufs, count, RTE_MBUF_HISTORY_OP_LIB_FREE);
724 : : rte_mempool_put_bulk(mp, (void **)mbufs, count);
725 : : }
726 : :
727 : : /**
728 : : * The packet mbuf constructor.
729 : : *
730 : : * This function initializes some fields in the mbuf structure that are
731 : : * not modified by the user once created (origin pool, buffer start
732 : : * address, and so on). This function is given as a callback function to
733 : : * rte_mempool_obj_iter() or rte_mempool_create() at pool creation time.
734 : : *
735 : : * This function expects that the mempool private area was previously
736 : : * initialized with rte_pktmbuf_pool_init().
737 : : *
738 : : * @param mp
739 : : * The mempool from which mbufs originate.
740 : : * @param opaque_arg
741 : : * A pointer that can be used by the user to retrieve useful information
742 : : * for mbuf initialization. This pointer is the opaque argument passed to
743 : : * rte_mempool_obj_iter() or rte_mempool_create().
744 : : * @param m
745 : : * The mbuf to initialize.
746 : : * @param i
747 : : * The index of the mbuf in the pool table.
748 : : */
749 : : void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
750 : : void *m, unsigned i);
751 : :
752 : : /**
753 : : * A packet mbuf pool constructor.
754 : : *
755 : : * This function initializes the mempool private data in the case of a
756 : : * pktmbuf pool. This private data is needed by the driver. The
757 : : * function must be called on the mempool before it is used, or it
758 : : * can be given as a callback function to rte_mempool_create() at
759 : : * pool creation. It can be extended by the user, for example, to
760 : : * provide another packet size.
761 : : *
762 : : * The mempool private area size must be at least equal to
763 : : * sizeof(struct rte_pktmbuf_pool_private).
764 : : *
765 : : * @param mp
766 : : * The mempool from which mbufs originate.
767 : : * @param opaque_arg
768 : : * A pointer that can be used by the user to retrieve useful information
769 : : * for mbuf initialization. This pointer is the opaque argument passed to
770 : : * rte_mempool_create().
771 : : */
772 : : void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
773 : :
774 : : /**
775 : : * Create a mbuf pool.
776 : : *
777 : : * This function creates and initializes a packet mbuf pool. It is
778 : : * a wrapper to rte_mempool functions.
779 : : *
780 : : * @param name
781 : : * The name of the mbuf pool.
782 : : * @param n
783 : : * The number of elements in the mbuf pool. The optimum size (in terms
784 : : * of memory usage) for a mempool is when n is a power of two minus one:
785 : : * n = (2^q - 1).
786 : : * @param cache_size
787 : : * Size of the per-core object cache. See rte_mempool_create() for
788 : : * details.
789 : : * @param priv_size
790 : : * Size of application private are between the rte_mbuf structure
791 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
792 : : * @param data_room_size
793 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
794 : : * @param socket_id
795 : : * The socket identifier where the memory should be allocated. The
796 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
797 : : * reserved zone.
798 : : * @return
799 : : * The pointer to the new allocated mempool, on success. NULL on error
800 : : * with rte_errno set appropriately. Possible rte_errno values include:
801 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
802 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
803 : : * - ENOSPC - the maximum number of memzones has already been allocated
804 : : * - EEXIST - a memzone with the same name already exists
805 : : * - ENOMEM - no appropriate memory area found in which to create memzone
806 : : */
807 : : struct rte_mempool *
808 : : rte_pktmbuf_pool_create(const char *name, unsigned n,
809 : : unsigned cache_size, uint16_t priv_size, uint16_t data_room_size,
810 : : int socket_id);
811 : :
812 : : /**
813 : : * Create a mbuf pool with a given mempool ops name
814 : : *
815 : : * This function creates and initializes a packet mbuf pool. It is
816 : : * a wrapper to rte_mempool functions.
817 : : *
818 : : * @param name
819 : : * The name of the mbuf pool.
820 : : * @param n
821 : : * The number of elements in the mbuf pool. The optimum size (in terms
822 : : * of memory usage) for a mempool is when n is a power of two minus one:
823 : : * n = (2^q - 1).
824 : : * @param cache_size
825 : : * Size of the per-core object cache. See rte_mempool_create() for
826 : : * details.
827 : : * @param priv_size
828 : : * Size of application private are between the rte_mbuf structure
829 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
830 : : * @param data_room_size
831 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
832 : : * @param socket_id
833 : : * The socket identifier where the memory should be allocated. The
834 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
835 : : * reserved zone.
836 : : * @param ops_name
837 : : * The mempool ops name to be used for this mempool instead of
838 : : * default mempool. The value can be *NULL* to use default mempool.
839 : : * @return
840 : : * The pointer to the new allocated mempool, on success. NULL on error
841 : : * with rte_errno set appropriately. Possible rte_errno values include:
842 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
843 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
844 : : * - ENOSPC - the maximum number of memzones has already been allocated
845 : : * - EEXIST - a memzone with the same name already exists
846 : : * - ENOMEM - no appropriate memory area found in which to create memzone
847 : : */
848 : : struct rte_mempool *
849 : : rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
850 : : unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
851 : : int socket_id, const char *ops_name);
852 : :
853 : : /** A structure that describes the pinned external buffer segment. */
854 : : struct rte_pktmbuf_extmem {
855 : : void *buf_ptr; /**< The virtual address of data buffer. */
856 : : rte_iova_t buf_iova; /**< The IO address of the data buffer. */
857 : : size_t buf_len; /**< External buffer length in bytes. */
858 : : uint16_t elt_size; /**< mbuf element size in bytes. */
859 : : };
860 : :
861 : : /**
862 : : * Create a mbuf pool with external pinned data buffers.
863 : : *
864 : : * This function creates and initializes a packet mbuf pool that contains
865 : : * only mbufs with external buffer. It is a wrapper to rte_mempool functions.
866 : : *
867 : : * @param name
868 : : * The name of the mbuf pool.
869 : : * @param n
870 : : * The number of elements in the mbuf pool. The optimum size (in terms
871 : : * of memory usage) for a mempool is when n is a power of two minus one:
872 : : * n = (2^q - 1).
873 : : * @param cache_size
874 : : * Size of the per-core object cache. See rte_mempool_create() for
875 : : * details.
876 : : * @param priv_size
877 : : * Size of application private are between the rte_mbuf structure
878 : : * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
879 : : * @param data_room_size
880 : : * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
881 : : * @param socket_id
882 : : * The socket identifier where the memory should be allocated. The
883 : : * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
884 : : * reserved zone.
885 : : * @param ext_mem
886 : : * Pointer to the array of structures describing the external memory
887 : : * for data buffers. It is caller responsibility to register this memory
888 : : * with rte_extmem_register() (if needed), map this memory to appropriate
889 : : * physical device, etc.
890 : : * @param ext_num
891 : : * Number of elements in the ext_mem array.
892 : : * @return
893 : : * The pointer to the new allocated mempool, on success. NULL on error
894 : : * with rte_errno set appropriately. Possible rte_errno values include:
895 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
896 : : * - EINVAL - cache size provided is too large, or priv_size is not aligned.
897 : : * - ENOSPC - the maximum number of memzones has already been allocated
898 : : * - EEXIST - a memzone with the same name already exists
899 : : * - ENOMEM - no appropriate memory area found in which to create memzone
900 : : */
901 : : struct rte_mempool *
902 : : rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
903 : : unsigned int cache_size, uint16_t priv_size,
904 : : uint16_t data_room_size, int socket_id,
905 : : const struct rte_pktmbuf_extmem *ext_mem,
906 : : unsigned int ext_num);
907 : :
908 : : /**
909 : : * Get the data room size of mbufs stored in a pktmbuf_pool
910 : : *
911 : : * The data room size is the amount of data that can be stored in a
912 : : * mbuf including the headroom (RTE_PKTMBUF_HEADROOM).
913 : : *
914 : : * @param mp
915 : : * The packet mbuf pool.
916 : : * @return
917 : : * The data room size of mbufs stored in this mempool.
918 : : */
919 : : static inline uint16_t
920 : : rte_pktmbuf_data_room_size(struct rte_mempool *mp)
921 : : {
922 : : struct rte_pktmbuf_pool_private *mbp_priv;
923 : :
924 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
925 [ + + - + : 3620 : return mbp_priv->mbuf_data_room_size;
- + ]
926 : : }
927 : :
928 : : /**
929 : : * Get the application private size of mbufs stored in a pktmbuf_pool
930 : : *
931 : : * The private size of mbuf is a zone located between the rte_mbuf
932 : : * structure and the data buffer where an application can store data
933 : : * associated to a packet.
934 : : *
935 : : * @param mp
936 : : * The packet mbuf pool.
937 : : * @return
938 : : * The private size of mbufs stored in this mempool.
939 : : */
940 : : static inline uint16_t
941 : : rte_pktmbuf_priv_size(struct rte_mempool *mp)
942 : : {
943 : : struct rte_pktmbuf_pool_private *mbp_priv;
944 : :
945 : : mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
946 [ + + - + ]: 153 : return mbp_priv->mbuf_priv_size;
947 : : }
948 : :
949 : : /**
950 : : * Reset the data_off field of a packet mbuf to its default value.
951 : : *
952 : : * The given mbuf must have only one segment, which should be empty.
953 : : *
954 : : * @param m
955 : : * The packet mbuf's data_off field has to be reset.
956 : : */
957 : : static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m)
958 : : {
959 : 330 : m->data_off = (uint16_t)RTE_MIN((uint16_t)RTE_PKTMBUF_HEADROOM,
960 : : (uint16_t)m->buf_len);
961 : : }
962 : :
963 : : /**
964 : : * Reset the fields of a bulk of packet mbufs to their default values.
965 : : *
966 : : * The caller must ensure that the mbufs come from the specified mempool,
967 : : * are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1),
968 : : * as done by rte_pktmbuf_prefree_seg().
969 : : *
970 : : * This function should be used with care, when optimization is required.
971 : : * For standard needs, prefer rte_pktmbuf_reset().
972 : : *
973 : : * @param mp
974 : : * The mempool to which the mbuf belongs.
975 : : * @param mbufs
976 : : * Array of pointers to packet mbufs.
977 : : * The array must not contain NULL pointers.
978 : : * @param count
979 : : * Array size.
980 : : */
981 : : static inline void
982 [ + + ]: 17543 : rte_mbuf_raw_reset_bulk(struct rte_mempool *mp, struct rte_mbuf **mbufs, unsigned int count)
983 : : {
984 : 17543 : uint64_t ol_flags = (rte_pktmbuf_priv_flags(mp) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) ?
985 [ + + ]: 17543 : RTE_MBUF_F_EXTERNAL : 0;
986 : : uint16_t data_off = RTE_MIN_T(RTE_PKTMBUF_HEADROOM, rte_pktmbuf_data_room_size(mp),
987 : : uint16_t);
988 : :
989 [ + + ]: 36147 : for (unsigned int idx = 0; idx < count; idx++) {
990 : 18604 : struct rte_mbuf *m = mbufs[idx];
991 : :
992 : 18604 : m->pkt_len = 0;
993 : 18604 : m->tx_offload = 0;
994 : 18604 : m->vlan_tci = 0;
995 : 18604 : m->vlan_tci_outer = 0;
996 : 18604 : m->port = RTE_MBUF_PORT_INVALID;
997 : :
998 : 18604 : m->ol_flags = ol_flags;
999 : 18604 : m->packet_type = 0;
1000 : 18604 : m->data_off = data_off;
1001 : :
1002 : 18604 : m->data_len = 0;
1003 : : __rte_mbuf_sanity_check(m, 1);
1004 : : }
1005 : 17543 : }
1006 : :
1007 : : /**
1008 : : * Reset the fields of a packet mbuf to their default values.
1009 : : *
1010 : : * The given mbuf must have only one segment.
1011 : : *
1012 : : * @param m
1013 : : * The packet mbuf to be reset.
1014 : : */
1015 : : static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
1016 : : {
1017 : 293 : m->next = NULL;
1018 : 293 : m->pkt_len = 0;
1019 : 293 : m->tx_offload = 0;
1020 : 293 : m->vlan_tci = 0;
1021 : 293 : m->vlan_tci_outer = 0;
1022 : 293 : m->nb_segs = 1;
1023 : 293 : m->port = RTE_MBUF_PORT_INVALID;
1024 : :
1025 : 293 : m->ol_flags &= RTE_MBUF_F_EXTERNAL;
1026 : 293 : m->packet_type = 0;
1027 : : rte_pktmbuf_reset_headroom(m);
1028 : :
1029 [ + - ]: 293 : m->data_len = 0;
1030 : : __rte_mbuf_sanity_check(m, 1);
1031 : : }
1032 : :
1033 : : /**
1034 : : * Allocate a new mbuf from a mempool.
1035 : : *
1036 : : * This new mbuf contains one segment, which has a length of 0. The pointer
1037 : : * to data is initialized to have some bytes of headroom in the buffer
1038 : : * (if buffer size allows).
1039 : : *
1040 : : * @param mp
1041 : : * The mempool from which the mbuf is allocated.
1042 : : * @return
1043 : : * - The pointer to the new mbuf on success.
1044 : : * - NULL if allocation failed.
1045 : : */
1046 : 17526 : static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp)
1047 : : {
1048 : : struct rte_mbuf *m;
1049 [ + + ]: 17526 : if ((m = rte_mbuf_raw_alloc(mp)) != NULL)
1050 : 17518 : rte_mbuf_raw_reset_bulk(mp, &m, 1);
1051 : 17526 : return m;
1052 : : }
1053 : :
1054 : : /**
1055 : : * Allocate a bulk of mbufs, initialize refcnt and reset the fields to default
1056 : : * values.
1057 : : *
1058 : : * @param pool
1059 : : * The mempool from which mbufs are allocated.
1060 : : * @param mbufs
1061 : : * Array of pointers to mbufs
1062 : : * @param count
1063 : : * Array size
1064 : : * @return
1065 : : * - 0: Success
1066 : : * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved.
1067 : : */
1068 [ + + ]: 33 : static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
1069 : : struct rte_mbuf **mbufs, unsigned count)
1070 : : {
1071 : : int rc;
1072 : :
1073 : : rc = rte_mbuf_raw_alloc_bulk(pool, mbufs, count);
1074 [ + + ]: 33 : if (unlikely(rc))
1075 : : return rc;
1076 : :
1077 : : rte_mbuf_history_mark_bulk(mbufs, count, RTE_MBUF_HISTORY_OP_LIB_ALLOC);
1078 : :
1079 : 25 : rte_mbuf_raw_reset_bulk(pool, mbufs, count);
1080 : :
1081 : 25 : return 0;
1082 : : }
1083 : :
1084 : : /**
1085 : : * Initialize shared data at the end of an external buffer before attaching
1086 : : * to a mbuf by ``rte_pktmbuf_attach_extbuf()``. This is not a mandatory
1087 : : * initialization but a helper function to simply spare a few bytes at the
1088 : : * end of the buffer for shared data. If shared data is allocated
1089 : : * separately, this should not be called but application has to properly
1090 : : * initialize the shared data according to its need.
1091 : : *
1092 : : * Free callback and its argument is saved and the refcnt is set to 1.
1093 : : *
1094 : : * @warning
1095 : : * The value of buf_len will be reduced to RTE_PTR_DIFF(shinfo, buf_addr)
1096 : : * after this initialization. This shall be used for
1097 : : * ``rte_pktmbuf_attach_extbuf()``
1098 : : *
1099 : : * @param buf_addr
1100 : : * The pointer to the external buffer.
1101 : : * @param [in,out] buf_len
1102 : : * The pointer to length of the external buffer. Input value must be
1103 : : * larger than the size of ``struct rte_mbuf_ext_shared_info`` and
1104 : : * padding for alignment. If not enough, this function will return NULL.
1105 : : * Adjusted buffer length will be returned through this pointer.
1106 : : * @param free_cb
1107 : : * Free callback function to call when the external buffer needs to be
1108 : : * freed.
1109 : : * @param fcb_opaque
1110 : : * Argument for the free callback function.
1111 : : *
1112 : : * @return
1113 : : * A pointer to the initialized shared data on success, return NULL
1114 : : * otherwise.
1115 : : */
1116 : : static inline struct rte_mbuf_ext_shared_info *
1117 : : rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len,
1118 : : rte_mbuf_extbuf_free_callback_t free_cb, void *fcb_opaque)
1119 : : {
1120 : : struct rte_mbuf_ext_shared_info *shinfo;
1121 : 3 : void *buf_end = RTE_PTR_ADD(buf_addr, *buf_len);
1122 : : void *addr;
1123 : :
1124 : 3 : addr = RTE_PTR_ALIGN_FLOOR(RTE_PTR_SUB(buf_end, sizeof(*shinfo)),
1125 : : sizeof(uintptr_t));
1126 [ - + ]: 3 : if (addr <= buf_addr)
1127 : : return NULL;
1128 : :
1129 : : shinfo = (struct rte_mbuf_ext_shared_info *)addr;
1130 : 3 : shinfo->free_cb = free_cb;
1131 [ - + ]: 3 : shinfo->fcb_opaque = fcb_opaque;
1132 : : rte_mbuf_ext_refcnt_set(shinfo, 1);
1133 : :
1134 : 2 : *buf_len = (uint16_t)RTE_PTR_DIFF(shinfo, buf_addr);
1135 : 0 : return shinfo;
1136 : : }
1137 : :
1138 : : /**
1139 : : * Attach an external buffer to a mbuf.
1140 : : *
1141 : : * User-managed anonymous buffer can be attached to an mbuf. When attaching
1142 : : * it, corresponding free callback function and its argument should be
1143 : : * provided via shinfo. This callback function will be called once all the
1144 : : * mbufs are detached from the buffer (refcnt becomes zero).
1145 : : *
1146 : : * The headroom length of the attaching mbuf will be set to zero and this
1147 : : * can be properly adjusted after attachment. For example, ``rte_pktmbuf_adj()``
1148 : : * or ``rte_pktmbuf_reset_headroom()`` might be used.
1149 : : *
1150 : : * Similarly, the packet length is initialized to 0. If the buffer contains
1151 : : * data, the user has to adjust ``data_len`` and the ``pkt_len`` field of
1152 : : * the mbuf accordingly.
1153 : : *
1154 : : * More mbufs can be attached to the same external buffer by
1155 : : * ``rte_pktmbuf_attach()`` once the external buffer has been attached by
1156 : : * this API.
1157 : : *
1158 : : * Detachment can be done by either ``rte_pktmbuf_detach_extbuf()`` or
1159 : : * ``rte_pktmbuf_detach()``.
1160 : : *
1161 : : * Memory for shared data must be provided and user must initialize all of
1162 : : * the content properly, especially free callback and refcnt. The pointer
1163 : : * of shared data will be stored in m->shinfo.
1164 : : * ``rte_pktmbuf_ext_shinfo_init_helper`` can help to simply spare a few
1165 : : * bytes at the end of buffer for the shared data, store free callback and
1166 : : * its argument and set the refcnt to 1. The following is an example:
1167 : : *
1168 : : * struct rte_mbuf_ext_shared_info *shinfo =
1169 : : * rte_pktmbuf_ext_shinfo_init_helper(buf_addr, &buf_len,
1170 : : * free_cb, fcb_arg);
1171 : : * rte_pktmbuf_attach_extbuf(m, buf_addr, buf_iova, buf_len, shinfo);
1172 : : * rte_pktmbuf_reset_headroom(m);
1173 : : * rte_pktmbuf_adj(m, data_len);
1174 : : *
1175 : : * Attaching an external buffer is quite similar to mbuf indirection in
1176 : : * replacing buffer addresses and length of a mbuf, but a few differences:
1177 : : * - When an indirect mbuf is attached, refcnt of the direct mbuf would be
1178 : : * 2 as long as the direct mbuf itself isn't freed after the attachment.
1179 : : * In such cases, the buffer area of a direct mbuf must be read-only. But
1180 : : * external buffer has its own refcnt and it starts from 1. Unless
1181 : : * multiple mbufs are attached to a mbuf having an external buffer, the
1182 : : * external buffer is writable.
1183 : : * - There's no need to allocate buffer from a mempool. Any buffer can be
1184 : : * attached with appropriate free callback and its IO address.
1185 : : * - Smaller metadata is required to maintain shared data such as refcnt.
1186 : : *
1187 : : * @param m
1188 : : * The pointer to the mbuf.
1189 : : * @param buf_addr
1190 : : * The pointer to the external buffer.
1191 : : * @param buf_iova
1192 : : * IO address of the external buffer.
1193 : : * @param buf_len
1194 : : * The size of the external buffer.
1195 : : * @param shinfo
1196 : : * User-provided memory for shared data of the external buffer.
1197 : : */
1198 : : static inline void
1199 : : rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr,
1200 : : rte_iova_t buf_iova, uint16_t buf_len,
1201 : : struct rte_mbuf_ext_shared_info *shinfo)
1202 : : {
1203 : : /* mbuf should not be read-only */
1204 : : RTE_ASSERT(RTE_MBUF_DIRECT(m) && rte_mbuf_refcnt_read(m) == 1);
1205 : : RTE_ASSERT(shinfo->free_cb != NULL);
1206 : :
1207 : 3 : m->buf_addr = buf_addr;
1208 : : rte_mbuf_iova_set(m, buf_iova);
1209 : 3 : m->buf_len = buf_len;
1210 : :
1211 : 3 : m->data_len = 0;
1212 : 3 : m->data_off = 0;
1213 : :
1214 : 3 : m->ol_flags |= RTE_MBUF_F_EXTERNAL;
1215 [ - + ]: 3 : m->shinfo = shinfo;
1216 : : }
1217 : :
1218 : : /**
1219 : : * Detach the external buffer attached to a mbuf, same as
1220 : : * ``rte_pktmbuf_detach()``
1221 : : *
1222 : : * @param m
1223 : : * The mbuf having external buffer.
1224 : : */
1225 : : #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m)
1226 : :
1227 : : /**
1228 : : * Copy dynamic fields from msrc to mdst.
1229 : : *
1230 : : * @param mdst
1231 : : * The destination mbuf.
1232 : : * @param msrc
1233 : : * The source mbuf.
1234 : : */
1235 : : static inline void
1236 : : rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1237 : : {
1238 : : #if !RTE_IOVA_IN_MBUF
1239 : : mdst->dynfield2 = msrc->dynfield2;
1240 : : #endif
1241 : 4147 : memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1));
1242 : : }
1243 : :
1244 : : /* internal */
1245 : : static inline void
1246 : 4147 : __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc)
1247 : : {
1248 : 4147 : mdst->port = msrc->port;
1249 : 4147 : mdst->vlan_tci = msrc->vlan_tci;
1250 : 4147 : mdst->vlan_tci_outer = msrc->vlan_tci_outer;
1251 : 4147 : mdst->tx_offload = msrc->tx_offload;
1252 : 4147 : mdst->hash = msrc->hash;
1253 : 4147 : mdst->packet_type = msrc->packet_type;
1254 : : rte_mbuf_dynfield_copy(mdst, msrc);
1255 : 4147 : }
1256 : :
1257 : : /**
1258 : : * Attach packet mbuf to another packet mbuf.
1259 : : *
1260 : : * If the mbuf we are attaching to isn't a direct buffer and is attached to
1261 : : * an external buffer, the mbuf being attached will be attached to the
1262 : : * external buffer instead of mbuf indirection.
1263 : : *
1264 : : * Otherwise, the mbuf will be indirectly attached. After attachment we
1265 : : * refer the mbuf we attached as 'indirect', while mbuf we attached to as
1266 : : * 'direct'. The direct mbuf's reference counter is incremented.
1267 : : *
1268 : : * Right now, not supported:
1269 : : * - attachment for already indirect mbuf (e.g. - mi has to be direct).
1270 : : * - mbuf we trying to attach (mi) is used by someone else
1271 : : * e.g. it's reference counter is greater then 1.
1272 : : *
1273 : : * @param mi
1274 : : * The indirect packet mbuf.
1275 : : * @param m
1276 : : * The packet mbuf we're attaching to.
1277 : : */
1278 : 36 : static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
1279 : : {
1280 : : RTE_ASSERT(RTE_MBUF_DIRECT(mi) &&
1281 : : rte_mbuf_refcnt_read(mi) == 1);
1282 : :
1283 [ + + ]: 36 : if (RTE_MBUF_HAS_EXTBUF(m)) {
1284 : 7 : rte_mbuf_ext_refcnt_update(m->shinfo, 1);
1285 : 7 : mi->ol_flags = m->ol_flags;
1286 : 7 : mi->shinfo = m->shinfo;
1287 : : } else {
1288 : : /* if m is not direct, get the mbuf that embeds the data */
1289 : 29 : rte_mbuf_refcnt_update(rte_mbuf_from_indirect(m), 1);
1290 : 29 : mi->priv_size = m->priv_size;
1291 : 29 : mi->ol_flags = m->ol_flags | RTE_MBUF_F_INDIRECT;
1292 : : }
1293 : :
1294 : 36 : __rte_pktmbuf_copy_hdr(mi, m);
1295 : :
1296 : 36 : mi->data_off = m->data_off;
1297 : 36 : mi->data_len = m->data_len;
1298 : : rte_mbuf_iova_set(mi, rte_mbuf_iova_get(m));
1299 : 36 : mi->buf_addr = m->buf_addr;
1300 : 36 : mi->buf_len = m->buf_len;
1301 : :
1302 : 36 : mi->next = NULL;
1303 : 36 : mi->pkt_len = mi->data_len;
1304 : 36 : mi->nb_segs = 1;
1305 : :
1306 : : __rte_mbuf_sanity_check(mi, 1);
1307 : : __rte_mbuf_sanity_check(m, 0);
1308 : 36 : }
1309 : :
1310 : : /**
1311 : : * @internal used by rte_pktmbuf_detach().
1312 : : *
1313 : : * Decrement the reference counter of the external buffer. When the
1314 : : * reference counter becomes 0, the buffer is freed by pre-registered
1315 : : * callback.
1316 : : */
1317 : : static inline void
1318 : 10 : __rte_pktmbuf_free_extbuf(struct rte_mbuf *m)
1319 : : {
1320 : : RTE_ASSERT(RTE_MBUF_HAS_EXTBUF(m));
1321 : : RTE_ASSERT(m->shinfo != NULL);
1322 : :
1323 [ + + ]: 10 : if (rte_mbuf_ext_refcnt_update(m->shinfo, -1) == 0)
1324 : 3 : m->shinfo->free_cb(m->buf_addr, m->shinfo->fcb_opaque);
1325 : 10 : }
1326 : :
1327 : : /**
1328 : : * @internal used by rte_pktmbuf_detach().
1329 : : *
1330 : : * Decrement the direct mbuf's reference counter. When the reference
1331 : : * counter becomes 0, the direct mbuf is freed.
1332 : : */
1333 : : static inline void
1334 : 29 : __rte_pktmbuf_free_direct(struct rte_mbuf *m)
1335 : : {
1336 : : struct rte_mbuf *md;
1337 : :
1338 : : RTE_ASSERT(RTE_MBUF_CLONED(m));
1339 : :
1340 : : md = rte_mbuf_from_indirect(m);
1341 : :
1342 [ + + ]: 29 : if (rte_mbuf_refcnt_update(md, -1) == 0) {
1343 : 10 : md->next = NULL;
1344 [ + - ]: 10 : md->nb_segs = 1;
1345 : : rte_mbuf_refcnt_set(md, 1);
1346 : : rte_mbuf_raw_free(md);
1347 : : }
1348 : 29 : }
1349 : :
1350 : : /**
1351 : : * Detach a packet mbuf from external buffer or direct buffer.
1352 : : *
1353 : : * - decrement refcnt and free the external/direct buffer if refcnt
1354 : : * becomes zero.
1355 : : * - restore original mbuf address and length values.
1356 : : * - reset pktmbuf data and data_len to their default values.
1357 : : *
1358 : : * All other fields of the given packet mbuf will be left intact.
1359 : : *
1360 : : * If the packet mbuf was allocated from the pool with pinned
1361 : : * external buffers the rte_pktmbuf_detach does nothing with the
1362 : : * mbuf of this kind, because the pinned buffers are not supposed
1363 : : * to be detached.
1364 : : *
1365 : : * @param m
1366 : : * The indirect attached packet mbuf.
1367 : : */
1368 : 1141 : static inline void rte_pktmbuf_detach(struct rte_mbuf *m)
1369 : : {
1370 : 1141 : struct rte_mempool *mp = m->pool;
1371 : : uint32_t mbuf_size, buf_len;
1372 : : uint16_t priv_size;
1373 : :
1374 [ + + ]: 1141 : if (RTE_MBUF_HAS_EXTBUF(m)) {
1375 : : /*
1376 : : * The mbuf has the external attached buffer,
1377 : : * we should check the type of the memory pool where
1378 : : * the mbuf was allocated from to detect the pinned
1379 : : * external buffer.
1380 : : */
1381 : : uint32_t flags = rte_pktmbuf_priv_flags(mp);
1382 : :
1383 [ + + ]: 1112 : if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
1384 : : /*
1385 : : * The pinned external buffer should not be
1386 : : * detached from its backing mbuf, just exit.
1387 : : */
1388 : : return;
1389 : : }
1390 : 10 : __rte_pktmbuf_free_extbuf(m);
1391 : : } else {
1392 : 29 : __rte_pktmbuf_free_direct(m);
1393 : : }
1394 : : priv_size = rte_pktmbuf_priv_size(mp);
1395 : 39 : mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
1396 : : buf_len = rte_pktmbuf_data_room_size(mp);
1397 : :
1398 : 39 : m->priv_size = priv_size;
1399 : 39 : m->buf_addr = (char *)m + mbuf_size;
1400 : 39 : rte_mbuf_iova_set(m, rte_mempool_virt2iova(m) + mbuf_size);
1401 : 39 : m->buf_len = (uint16_t)buf_len;
1402 : : rte_pktmbuf_reset_headroom(m);
1403 : 39 : m->data_len = 0;
1404 : 39 : m->ol_flags = 0;
1405 : : }
1406 : :
1407 : : /**
1408 : : * @internal Handle the packet mbufs with attached pinned external buffer
1409 : : * on the mbuf freeing:
1410 : : *
1411 : : * - return zero if reference counter in shinfo is one. It means there is
1412 : : * no more reference to this pinned buffer and mbuf can be returned to
1413 : : * the pool
1414 : : *
1415 : : * - otherwise (if reference counter is not one), decrement reference
1416 : : * counter and return non-zero value to prevent freeing the backing mbuf.
1417 : : *
1418 : : * Returns non zero if mbuf should not be freed.
1419 : : */
1420 : 1102 : static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
1421 : : {
1422 : : struct rte_mbuf_ext_shared_info *shinfo;
1423 : :
1424 : : /* Clear flags, mbuf is being freed. */
1425 : 1102 : m->ol_flags = RTE_MBUF_F_EXTERNAL;
1426 : 1102 : shinfo = m->shinfo;
1427 : :
1428 : : /* Optimize for performance - do not dec/reinit */
1429 [ + + ]: 1102 : if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
1430 : : return 0;
1431 : :
1432 : : /*
1433 : : * Direct usage of add primitive to avoid
1434 : : * duplication of comparing with one.
1435 : : */
1436 [ - + ]: 2 : if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
1437 : : rte_memory_order_acq_rel) - 1))
1438 : : return 1;
1439 : :
1440 : : /* Reinitialize counter before mbuf freeing. */
1441 : : rte_mbuf_ext_refcnt_set(shinfo, 1);
1442 : 0 : return 0;
1443 : : }
1444 : :
1445 : : /**
1446 : : * Decrease reference counter and unlink a mbuf segment
1447 : : *
1448 : : * This function does the same than a free, except that it does not
1449 : : * return the segment to its pool.
1450 : : * It decreases the reference counter, and if it reaches 0, it is
1451 : : * detached from its parent for an indirect mbuf.
1452 : : *
1453 : : * @param m
1454 : : * The mbuf to be unlinked
1455 : : * @return
1456 : : * - (m) if it is the last reference. It can be recycled or freed.
1457 : : * - (NULL) if the mbuf still has remaining references on it.
1458 : : */
1459 : : static __rte_always_inline struct rte_mbuf *
1460 : : rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
1461 : : {
1462 : : bool refcnt_not_one;
1463 : :
1464 : : __rte_mbuf_sanity_check(m, 0);
1465 : :
1466 : 4210010 : refcnt_not_one = unlikely(rte_mbuf_refcnt_read(m) != 1);
1467 [ + + - + : 2378112 : if (refcnt_not_one && __rte_mbuf_refcnt_update(m, -1) != 0)
+ + - + -
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1468 : : return NULL;
1469 : :
1470 [ + + + + : 2115465 : if (unlikely(!RTE_MBUF_DIRECT(m))) {
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1471 : 1137 : rte_pktmbuf_detach(m);
1472 [ + + + + : 1137 : if (RTE_MBUF_HAS_EXTBUF(m) &&
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1473 [ + - + - : 3306 : RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ - + - +
- + + - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1474 : 1102 : __rte_pktmbuf_pinned_extbuf_decref(m))
1475 : : return NULL;
1476 : : }
1477 : :
1478 [ - + - + : 2115463 : if (refcnt_not_one)
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1479 : 0 : rte_mbuf_refcnt_set(m, 1);
1480 [ + + + + : 2115463 : if (m->nb_segs != 1)
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1481 : 4175 : m->nb_segs = 1;
1482 [ + + + + : 2115463 : if (m->next != NULL)
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1483 : 4725 : m->next = NULL;
1484 : :
1485 : : return m;
1486 : : }
1487 : :
1488 : : /**
1489 : : * Free a segment of a packet mbuf into its original mempool.
1490 : : *
1491 : : * Free an mbuf, without parsing other segments in case of chained
1492 : : * buffers.
1493 : : *
1494 : : * @param m
1495 : : * The packet mbuf segment to be freed.
1496 : : */
1497 : : static __rte_always_inline void
1498 : : rte_pktmbuf_free_seg(struct rte_mbuf *m)
1499 : : {
1500 : : m = rte_pktmbuf_prefree_seg(m);
1501 [ + + + + : 2370083 : if (likely(m != NULL))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1502 : : rte_mbuf_raw_free(m);
1503 : : }
1504 : :
1505 : : /**
1506 : : * Free a packet mbuf back into its original mempool.
1507 : : *
1508 : : * Free an mbuf, and all its segments in case of chained buffers. Each
1509 : : * segment is added back into its original mempool.
1510 : : *
1511 : : * @param m
1512 : : * The packet mbuf to be freed. If NULL, the function does nothing.
1513 : : */
1514 : 2402351 : static inline void rte_pktmbuf_free(struct rte_mbuf *m)
1515 : : {
1516 : : struct rte_mbuf *m_next;
1517 : :
1518 : : if (m != NULL)
1519 : : __rte_mbuf_sanity_check(m, 1);
1520 : :
1521 [ + + ]: 4771994 : while (m != NULL) {
1522 : 2369654 : m_next = m->next;
1523 : : rte_pktmbuf_free_seg(m);
1524 : : m = m_next;
1525 : : }
1526 : 2402340 : }
1527 : :
1528 : : /**
1529 : : * Free a bulk of packet mbufs back into their original mempools.
1530 : : *
1531 : : * Free a bulk of mbufs, and all their segments in case of chained buffers.
1532 : : * Each segment is added back into its original mempool.
1533 : : *
1534 : : * @param mbufs
1535 : : * Array of pointers to packet mbufs.
1536 : : * The array may contain NULL pointers.
1537 : : * @param count
1538 : : * Array size.
1539 : : */
1540 : : void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count);
1541 : :
1542 : : /**
1543 : : * Create a "clone" of the given packet mbuf.
1544 : : *
1545 : : * Walks through all segments of the given packet mbuf, and for each of them:
1546 : : * - Creates a new packet mbuf from the given pool.
1547 : : * - Attaches newly created mbuf to the segment.
1548 : : * Then updates pkt_len and nb_segs of the "clone" packet mbuf to match values
1549 : : * from the original packet mbuf.
1550 : : *
1551 : : * @param md
1552 : : * The packet mbuf to be cloned.
1553 : : * @param mp
1554 : : * The mempool from which the "clone" mbufs are allocated.
1555 : : * @return
1556 : : * - The pointer to the new "clone" mbuf on success.
1557 : : * - NULL if allocation fails.
1558 : : */
1559 : : struct rte_mbuf *
1560 : : rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp);
1561 : :
1562 : : /**
1563 : : * Create a full copy of a given packet mbuf.
1564 : : *
1565 : : * Copies all the data from a given packet mbuf to a newly allocated
1566 : : * set of mbufs. The private data are is not copied.
1567 : : *
1568 : : * @param m
1569 : : * The packet mbuf to be copied.
1570 : : * @param mp
1571 : : * The mempool from which the "clone" mbufs are allocated.
1572 : : * @param offset
1573 : : * The number of bytes to skip before copying.
1574 : : * If the mbuf does not have that many bytes, it is an error
1575 : : * and NULL is returned.
1576 : : * @param length
1577 : : * The upper limit on bytes to copy. Passing UINT32_MAX
1578 : : * means all data (after offset).
1579 : : * @return
1580 : : * - The pointer to the new "clone" mbuf on success.
1581 : : * - NULL if allocation fails.
1582 : : */
1583 : : struct rte_mbuf *
1584 : : rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
1585 : : uint32_t offset, uint32_t length);
1586 : :
1587 : : /**
1588 : : * Adds given value to the refcnt of all packet mbuf segments.
1589 : : *
1590 : : * Walks through all segments of given packet mbuf and for each of them
1591 : : * invokes rte_mbuf_refcnt_update().
1592 : : *
1593 : : * @param m
1594 : : * The packet mbuf whose refcnt to be updated.
1595 : : * @param v
1596 : : * The value to add to the mbuf's segments refcnt.
1597 : : */
1598 : : static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
1599 : : {
1600 : : __rte_mbuf_sanity_check(m, 1);
1601 : :
1602 : : do {
1603 : 135412 : rte_mbuf_refcnt_update(m, v);
1604 [ - + - + ]: 135412 : } while ((m = m->next) != NULL);
1605 : : }
1606 : :
1607 : : /**
1608 : : * Get the headroom in a packet mbuf.
1609 : : *
1610 : : * @param m
1611 : : * The packet mbuf.
1612 : : * @return
1613 : : * The length of the headroom.
1614 : : */
1615 : 0 : static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m)
1616 : : {
1617 : : __rte_mbuf_sanity_check(m, 0);
1618 [ + + - - : 41213 : return m->data_off;
- - ]
1619 : : }
1620 : :
1621 : : /**
1622 : : * Get the tailroom of a packet mbuf.
1623 : : *
1624 : : * @param m
1625 : : * The packet mbuf.
1626 : : * @return
1627 : : * The length of the tailroom.
1628 : : */
1629 : 0 : static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
1630 : : {
1631 : : __rte_mbuf_sanity_check(m, 0);
1632 [ + - ]: 5474 : return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) -
1633 [ + + + + : 25352 : m->data_len);
+ + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - -
- ]
1634 : : }
1635 : :
1636 : : /**
1637 : : * Get the last segment of the packet.
1638 : : *
1639 : : * @param m
1640 : : * The packet mbuf.
1641 : : * @return
1642 : : * The last segment of the given mbuf.
1643 : : */
1644 : 0 : static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
1645 : : {
1646 : : __rte_mbuf_sanity_check(m, 1);
1647 [ + + + + : 24078 : while (m->next != NULL)
- + + + -
+ - + + +
- + + + -
+ + + + +
+ + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - - - -
- + - + -
- - - - -
- - - - -
- - - - -
- - - + -
+ - + - +
- + - + -
+ - + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - + - +
- - - - -
- - - - +
- + + + ]
1648 : : m = m->next;
1649 : 0 : return m;
1650 : : }
1651 : :
1652 : : /**
1653 : : * A macro that returns the length of the packet.
1654 : : *
1655 : : * The value can be read or assigned.
1656 : : *
1657 : : * @param m
1658 : : * The packet mbuf.
1659 : : */
1660 : : #define rte_pktmbuf_pkt_len(m) ((m)->pkt_len)
1661 : :
1662 : : /**
1663 : : * A macro that returns the length of the segment.
1664 : : *
1665 : : * The value can be read or assigned.
1666 : : *
1667 : : * @param m
1668 : : * The packet mbuf.
1669 : : */
1670 : : #define rte_pktmbuf_data_len(m) ((m)->data_len)
1671 : :
1672 : : /**
1673 : : * Prepend len bytes to an mbuf data area.
1674 : : *
1675 : : * Returns a pointer to the new
1676 : : * data start address. If there is not enough headroom in the first
1677 : : * segment, the function will return NULL, without modifying the mbuf.
1678 : : *
1679 : : * @param m
1680 : : * The pkt mbuf.
1681 : : * @param len
1682 : : * The amount of data to prepend (in bytes).
1683 : : * @return
1684 : : * A pointer to the start of the newly prepended data, or
1685 : : * NULL if there is not enough headroom space in the first segment
1686 : : */
1687 : 0 : static inline char *rte_pktmbuf_prepend(struct rte_mbuf *m,
1688 : : uint16_t len)
1689 : : {
1690 : : __rte_mbuf_sanity_check(m, 1);
1691 : :
1692 [ + - + - : 4112 : if (unlikely(len > rte_pktmbuf_headroom(m)))
+ - + - +
- # # ]
1693 : : return NULL;
1694 : :
1695 : : /* NB: elaborating the subtraction like this instead of using
1696 : : * -= allows us to ensure the result type is uint16_t
1697 : : * avoiding compiler warnings on gcc 8.1 at least */
1698 : 4110 : m->data_off = (uint16_t)(m->data_off - len);
1699 : 4110 : m->data_len = (uint16_t)(m->data_len + len);
1700 : 4102 : m->pkt_len = (m->pkt_len + len);
1701 : :
1702 [ - + - + : 4110 : return (char *)m->buf_addr + m->data_off;
- + - + ]
1703 : : }
1704 : :
1705 : : /**
1706 : : * Append len bytes to an mbuf.
1707 : : *
1708 : : * Append len bytes to an mbuf and return a pointer to the start address
1709 : : * of the added data. If there is not enough tailroom in the last
1710 : : * segment, the function will return NULL, without modifying the mbuf.
1711 : : *
1712 : : * @param m
1713 : : * The packet mbuf.
1714 : : * @param len
1715 : : * The amount of data to append (in bytes).
1716 : : * @return
1717 : : * A pointer to the start of the newly appended data, or
1718 : : * NULL if there is not enough tailroom space in the last segment
1719 : : */
1720 : 0 : static inline char *rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)
1721 : : {
1722 : : void *tail;
1723 : : struct rte_mbuf *m_last;
1724 : :
1725 : : __rte_mbuf_sanity_check(m, 1);
1726 : :
1727 : : m_last = rte_pktmbuf_lastseg(m);
1728 [ + - + - : 5142 : if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
+ - + - +
- + - + -
+ - + - +
- + - + -
+ + + - +
- + - + -
+ - - - -
- - - - -
+ - + - +
- - - - -
+ - + - -
- - - - -
- - - - -
- - - - -
- - + - +
- + - + -
+ - + - +
- + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- + - + -
- - - - -
- + - +
- ]
1729 : : return NULL;
1730 : :
1731 : 5110 : tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
1732 : 5140 : m_last->data_len = (uint16_t)(m_last->data_len + len);
1733 [ + + + + : 656 : m->pkt_len = (m->pkt_len + len);
- + - + -
+ - + - +
- + - + -
+ - + - -
- - - - -
+ - + - -
- - - - -
- - + - +
- + - + -
- - - - -
- - - + -
+ ]
1734 : 4500 : return (char*) tail;
1735 : : }
1736 : :
1737 : : /**
1738 : : * Remove len bytes at the beginning of an mbuf.
1739 : : *
1740 : : * Returns a pointer to the start address of the new data area. If the
1741 : : * length is greater than the length of the first segment, then the
1742 : : * function will fail and return NULL, without modifying the mbuf.
1743 : : *
1744 : : * @param m
1745 : : * The packet mbuf.
1746 : : * @param len
1747 : : * The amount of data to remove (in bytes).
1748 : : * @return
1749 : : * A pointer to the new start of the data.
1750 : : */
1751 : : static inline char *rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)
1752 : : {
1753 : : __rte_mbuf_sanity_check(m, 1);
1754 : :
1755 [ + - # # : 2 : if (unlikely(len > m->data_len))
# # # # #
# # # # #
# # ]
1756 : : return NULL;
1757 : :
1758 : : /* NB: elaborating the addition like this instead of using
1759 : : * += allows us to ensure the result type is uint16_t
1760 : : * avoiding compiler warnings on gcc 8.1 at least */
1761 : 2 : m->data_len = (uint16_t)(m->data_len - len);
1762 : 2 : m->data_off = (uint16_t)(m->data_off + len);
1763 : 2 : m->pkt_len = (m->pkt_len - len);
1764 : 2 : return (char *)m->buf_addr + m->data_off;
1765 : : }
1766 : :
1767 : : /**
1768 : : * Remove len bytes of data at the end of the mbuf.
1769 : : *
1770 : : * If the length is greater than the length of the last segment, the
1771 : : * function will fail and return -1 without modifying the mbuf.
1772 : : *
1773 : : * @param m
1774 : : * The packet mbuf.
1775 : : * @param len
1776 : : * The amount of data to remove (in bytes).
1777 : : * @return
1778 : : * - 0: On success.
1779 : : * - -1: On error.
1780 : : */
1781 : : static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)
1782 : : {
1783 : : struct rte_mbuf *m_last;
1784 : :
1785 : : __rte_mbuf_sanity_check(m, 1);
1786 : :
1787 : : m_last = rte_pktmbuf_lastseg(m);
1788 [ + - - + ]: 6 : if (unlikely(len > m_last->data_len))
1789 : : return -1;
1790 : :
1791 : 4 : m_last->data_len = (uint16_t)(m_last->data_len - len);
1792 [ - + ]: 4 : m->pkt_len = (m->pkt_len - len);
1793 : 2 : return 0;
1794 : : }
1795 : :
1796 : : /**
1797 : : * Test if mbuf data is contiguous.
1798 : : *
1799 : : * @param m
1800 : : * The packet mbuf.
1801 : : * @return
1802 : : * - 1, if all data is contiguous (one segment).
1803 : : * - 0, if there is several segments.
1804 : : */
1805 : : static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
1806 : : {
1807 : : __rte_mbuf_sanity_check(m, 1);
1808 [ + + - + : 232 : return m->nb_segs == 1;
- + - + -
+ - + -
+ ]
1809 : : }
1810 : :
1811 : : /**
1812 : : * @internal used by rte_pktmbuf_read().
1813 : : */
1814 : : const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off,
1815 : : uint32_t len, void *buf);
1816 : :
1817 : : /**
1818 : : * Read len data bytes in a mbuf at specified offset.
1819 : : *
1820 : : * If the data is contiguous, return the pointer in the mbuf data, else
1821 : : * copy the data in the buffer provided by the user and return its
1822 : : * pointer.
1823 : : *
1824 : : * @param m
1825 : : * The pointer to the mbuf.
1826 : : * @param off
1827 : : * The offset of the data in the mbuf.
1828 : : * @param len
1829 : : * The amount of bytes to read.
1830 : : * @param buf
1831 : : * The buffer where data is copied if it is not contiguous in mbuf
1832 : : * data. Its length should be at least equal to the len parameter.
1833 : : * @return
1834 : : * The pointer to the data, either in the mbuf if it is contiguous,
1835 : : * or in the user buffer. If mbuf is too small, NULL is returned.
1836 : : */
1837 : 0 : static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m,
1838 : : uint32_t off, uint32_t len, void *buf)
1839 : : {
1840 [ + + + + : 210 : if (likely(off + len <= rte_pktmbuf_data_len(m)))
+ + + - +
+ + + + +
+ - - - -
- - - - -
- - - - -
- + - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - -
- ]
1841 [ - + ]: 173 : return rte_pktmbuf_mtod_offset(m, char *, off);
1842 : : else
1843 : 45 : return __rte_pktmbuf_read(m, off, len, buf);
1844 : : }
1845 : :
1846 : : /**
1847 : : * Chain an mbuf to another, thereby creating a segmented packet.
1848 : : *
1849 : : * Note: The implementation will do a linear walk over the segments to find
1850 : : * the tail entry. For cases when there are many segments, it's better to
1851 : : * chain the entries manually.
1852 : : *
1853 : : * @param head
1854 : : * The head of the mbuf chain (the first packet)
1855 : : * @param tail
1856 : : * The mbuf to put last in the chain
1857 : : *
1858 : : * @return
1859 : : * - 0, on success.
1860 : : * - -EOVERFLOW, if the chain segment limit exceeded
1861 : : */
1862 : : static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail)
1863 : : {
1864 : : struct rte_mbuf *cur_tail;
1865 : :
1866 : : /* Check for number-of-segments-overflow */
1867 [ + - + - : 665 : if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS)
+ - + - -
+ + - ]
1868 : : return -EOVERFLOW;
1869 : :
1870 : : /* Chain 'tail' onto the old tail */
1871 : : cur_tail = rte_pktmbuf_lastseg(head);
1872 : 665 : cur_tail->next = tail;
1873 : :
1874 : : /* accumulate number of segments and total length.
1875 : : * NB: elaborating the addition like this instead of using
1876 : : * -= allows us to ensure the result type is uint16_t
1877 : : * avoiding compiler warnings on gcc 8.1 at least */
1878 : 665 : head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
1879 : 665 : head->pkt_len += tail->pkt_len;
1880 : :
1881 : : /* pkt_len is only set in the head */
1882 : 211 : tail->pkt_len = tail->data_len;
1883 : :
1884 : 538 : return 0;
1885 : : }
1886 : :
1887 : : /**
1888 : : * For given input values generate raw tx_offload value.
1889 : : * Note that it is caller responsibility to make sure that input parameters
1890 : : * don't exceed maximum bit-field values.
1891 : : * @param il2
1892 : : * l2_len value.
1893 : : * @param il3
1894 : : * l3_len value.
1895 : : * @param il4
1896 : : * l4_len value.
1897 : : * @param tso
1898 : : * tso_segsz value.
1899 : : * @param ol3
1900 : : * outer_l3_len value.
1901 : : * @param ol2
1902 : : * outer_l2_len value.
1903 : : * @param unused
1904 : : * unused value.
1905 : : * @return
1906 : : * raw tx_offload value.
1907 : : */
1908 : : static __rte_always_inline uint64_t
1909 : : rte_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
1910 : : uint64_t ol3, uint64_t ol2, uint64_t unused)
1911 : : {
1912 : 65536 : return il2 << RTE_MBUF_L2_LEN_OFS |
1913 : 65536 : il3 << RTE_MBUF_L3_LEN_OFS |
1914 : 65536 : il4 << RTE_MBUF_L4_LEN_OFS |
1915 : 65536 : tso << RTE_MBUF_TSO_SEGSZ_OFS |
1916 : 65536 : ol3 << RTE_MBUF_OUTL3_LEN_OFS |
1917 : 65536 : ol2 << RTE_MBUF_OUTL2_LEN_OFS |
1918 : : unused << RTE_MBUF_TXOFLD_UNUSED_OFS;
1919 : : }
1920 : :
1921 : : /**
1922 : : * Validate general requirements for Tx offload in mbuf.
1923 : : *
1924 : : * This function checks correctness and completeness of Tx offload settings.
1925 : : *
1926 : : * @param m
1927 : : * The packet mbuf to be validated.
1928 : : * @return
1929 : : * 0 if packet is valid
1930 : : */
1931 : : static inline int
1932 : 11 : rte_validate_tx_offload(const struct rte_mbuf *m)
1933 : : {
1934 : 11 : uint64_t ol_flags = m->ol_flags;
1935 : :
1936 : : /* Does packet set any of available offloads? */
1937 [ + + ]: 11 : if (!(ol_flags & RTE_MBUF_F_TX_OFFLOAD_MASK))
1938 : : return 0;
1939 : :
1940 : : /* IP checksum can be counted only for IPv4 packet */
1941 [ + + ]: 10 : if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && (ol_flags & RTE_MBUF_F_TX_IPV6))
1942 : : return -EINVAL;
1943 : :
1944 : : /* IP type not set when required */
1945 [ + + ]: 9 : if (ol_flags & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_TCP_SEG))
1946 [ + + ]: 7 : if (!(ol_flags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)))
1947 : : return -EINVAL;
1948 : :
1949 : : /* Check requirements for TSO packet */
1950 [ + + ]: 7 : if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1951 [ + + ]: 4 : if ((m->tso_segsz == 0) ||
1952 [ + + ]: 3 : ((ol_flags & RTE_MBUF_F_TX_IPV4) &&
1953 : : !(ol_flags & RTE_MBUF_F_TX_IP_CKSUM)))
1954 : : return -EINVAL;
1955 : :
1956 : : /* RTE_MBUF_F_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */
1957 [ + + ]: 5 : if ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) &&
1958 : : !(ol_flags & RTE_MBUF_F_TX_OUTER_IPV4))
1959 : 1 : return -EINVAL;
1960 : :
1961 : : return 0;
1962 : : }
1963 : :
1964 : : /**
1965 : : * @internal used by rte_pktmbuf_linearize().
1966 : : */
1967 : : int __rte_pktmbuf_linearize(struct rte_mbuf *mbuf);
1968 : :
1969 : : /**
1970 : : * Linearize data in mbuf.
1971 : : *
1972 : : * This function moves the mbuf data in the first segment if there is enough
1973 : : * tailroom. The subsequent segments are unchained and freed.
1974 : : *
1975 : : * @param mbuf
1976 : : * mbuf to linearize
1977 : : * @return
1978 : : * - 0, on success
1979 : : * - -1, on error
1980 : : */
1981 : : static inline int
1982 : : rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
1983 : : {
1984 [ + + ]: 10 : if (rte_pktmbuf_is_contiguous(mbuf))
1985 : : return 0;
1986 : 8 : return __rte_pktmbuf_linearize(mbuf);
1987 : : }
1988 : :
1989 : : /**
1990 : : * Dump an mbuf structure to a file.
1991 : : *
1992 : : * Dump all fields for the given packet mbuf and all its associated
1993 : : * segments (in the case of a chained buffer).
1994 : : *
1995 : : * @param f
1996 : : * A pointer to a file for output
1997 : : * @param m
1998 : : * The packet mbuf.
1999 : : * @param dump_len
2000 : : * If dump_len != 0, also dump the "dump_len" first data bytes of
2001 : : * the packet.
2002 : : */
2003 : : void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len);
2004 : :
2005 : : /**
2006 : : * Get the value of mbuf sched queue_id field.
2007 : : */
2008 : : static inline uint32_t
2009 : : rte_mbuf_sched_queue_get(const struct rte_mbuf *m)
2010 : : {
2011 : 6 : return m->hash.sched.queue_id;
2012 : : }
2013 : :
2014 : : /**
2015 : : * Get the value of mbuf sched traffic_class field.
2016 : : */
2017 : : static inline uint8_t
2018 : : rte_mbuf_sched_traffic_class_get(const struct rte_mbuf *m)
2019 : : {
2020 : : return m->hash.sched.traffic_class;
2021 : : }
2022 : :
2023 : : /**
2024 : : * Get the value of mbuf sched color field.
2025 : : */
2026 : : static inline uint8_t
2027 : : rte_mbuf_sched_color_get(const struct rte_mbuf *m)
2028 : : {
2029 : 10 : return m->hash.sched.color;
2030 : : }
2031 : :
2032 : : /**
2033 : : * Get the values of mbuf sched queue_id, traffic_class and color.
2034 : : *
2035 : : * @param m
2036 : : * Mbuf to read
2037 : : * @param queue_id
2038 : : * Returns the queue id
2039 : : * @param traffic_class
2040 : : * Returns the traffic class id
2041 : : * @param color
2042 : : * Returns the colour id
2043 : : */
2044 : : static inline void
2045 : : rte_mbuf_sched_get(const struct rte_mbuf *m, uint32_t *queue_id,
2046 : : uint8_t *traffic_class,
2047 : : uint8_t *color)
2048 : : {
2049 : : struct rte_mbuf_sched sched = m->hash.sched;
2050 : :
2051 : : *queue_id = sched.queue_id;
2052 : : *traffic_class = sched.traffic_class;
2053 : : *color = sched.color;
2054 : : }
2055 : :
2056 : : /**
2057 : : * Set the mbuf sched queue_id to the defined value.
2058 : : */
2059 : : static inline void
2060 : : rte_mbuf_sched_queue_set(struct rte_mbuf *m, uint32_t queue_id)
2061 : : {
2062 : : m->hash.sched.queue_id = queue_id;
2063 : : }
2064 : :
2065 : : /**
2066 : : * Set the mbuf sched traffic_class id to the defined value.
2067 : : */
2068 : : static inline void
2069 : : rte_mbuf_sched_traffic_class_set(struct rte_mbuf *m, uint8_t traffic_class)
2070 : : {
2071 : : m->hash.sched.traffic_class = traffic_class;
2072 : : }
2073 : :
2074 : : /**
2075 : : * Set the mbuf sched color id to the defined value.
2076 : : */
2077 : : static inline void
2078 : 0 : rte_mbuf_sched_color_set(struct rte_mbuf *m, uint8_t color)
2079 : : {
2080 : 0 : m->hash.sched.color = color;
2081 : 0 : }
2082 : :
2083 : : /**
2084 : : * Set the mbuf sched queue_id, traffic_class and color.
2085 : : *
2086 : : * @param m
2087 : : * Mbuf to set
2088 : : * @param queue_id
2089 : : * Queue id value to be set
2090 : : * @param traffic_class
2091 : : * Traffic class id value to be set
2092 : : * @param color
2093 : : * Color id to be set
2094 : : */
2095 : : static inline void
2096 : 0 : rte_mbuf_sched_set(struct rte_mbuf *m, uint32_t queue_id,
2097 : : uint8_t traffic_class,
2098 : : uint8_t color)
2099 : : {
2100 : 10 : m->hash.sched = (struct rte_mbuf_sched){
2101 : : .queue_id = queue_id,
2102 : : .traffic_class = traffic_class,
2103 : : .color = color,
2104 : : .reserved = 0,
2105 : : };
2106 : 0 : }
2107 : :
2108 : : #ifdef __cplusplus
2109 : : }
2110 : : #endif
2111 : :
2112 : : #endif /* _RTE_MBUF_H_ */
|
