Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright(c) 2016 6WIND S.A.
4 : : * Copyright(c) 2022 SmartShare Systems
5 : : */
6 : :
7 : : #ifndef _RTE_MEMPOOL_H_
8 : : #define _RTE_MEMPOOL_H_
9 : :
10 : : /**
11 : : * @file
12 : : * RTE Mempool.
13 : : *
14 : : * A memory pool is an allocator of fixed-size object. It is
15 : : * identified by its name, and uses a ring to store free objects. It
16 : : * provides some other optional services, like a per-core object
17 : : * cache, and an alignment helper to ensure that objects are padded
18 : : * to spread them equally on all RAM channels, ranks, and so on.
19 : : *
20 : : * Objects owned by a mempool should never be added in another
21 : : * mempool. When an object is freed using rte_mempool_put() or
22 : : * equivalent, the object data is not modified; the user can save some
23 : : * meta-data in the object data and retrieve them when allocating a
24 : : * new object.
25 : : *
26 : : * Note: the mempool implementation is not preemptible. An lcore must not be
27 : : * interrupted by another task that uses the same mempool (because it uses a
28 : : * ring which is not preemptible). Also, usual mempool functions like
29 : : * rte_mempool_get() or rte_mempool_put() are designed to be called from an EAL
30 : : * thread due to the internal per-lcore cache. Due to the lack of caching,
31 : : * rte_mempool_get() or rte_mempool_put() performance will suffer when called
32 : : * by unregistered non-EAL threads. Instead, unregistered non-EAL threads
33 : : * should call rte_mempool_generic_get() or rte_mempool_generic_put() with a
34 : : * user cache created with rte_mempool_cache_create().
35 : : */
36 : :
37 : : #include <stdalign.h>
38 : : #include <stdio.h>
39 : : #include <stdint.h>
40 : : #include <inttypes.h>
41 : :
42 : : #include <rte_compat.h>
43 : : #include <rte_config.h>
44 : : #include <rte_spinlock.h>
45 : : #include <rte_debug.h>
46 : : #include <rte_lcore.h>
47 : : #include <rte_log.h>
48 : : #include <rte_branch_prediction.h>
49 : : #include <rte_ring.h>
50 : : #include <rte_memcpy.h>
51 : : #include <rte_common.h>
52 : :
53 : : #include "rte_mempool_trace_fp.h"
54 : :
55 : : #ifdef __cplusplus
56 : : extern "C" {
57 : : #endif
58 : :
59 : : #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL /**< Header cookie. */
60 : : #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL /**< Header cookie. */
61 : : #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
62 : :
63 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
64 : : /**
65 : : * A structure that stores the mempool statistics (per-lcore).
66 : : * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
67 : : * captured since they can be calculated from other stats.
68 : : * For example: put_cache_objs = put_objs - put_common_pool_objs.
69 : : */
70 : : struct __rte_cache_aligned rte_mempool_debug_stats {
71 : : uint64_t put_bulk; /**< Number of puts. */
72 : : uint64_t put_objs; /**< Number of objects successfully put. */
73 : : uint64_t put_common_pool_bulk; /**< Number of bulks enqueued in common pool. */
74 : : uint64_t put_common_pool_objs; /**< Number of objects enqueued in common pool. */
75 : : uint64_t get_common_pool_bulk; /**< Number of bulks dequeued from common pool. */
76 : : uint64_t get_common_pool_objs; /**< Number of objects dequeued from common pool. */
77 : : uint64_t get_success_bulk; /**< Successful allocation number. */
78 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
79 : : uint64_t get_fail_bulk; /**< Failed allocation number. */
80 : : uint64_t get_fail_objs; /**< Objects that failed to be allocated. */
81 : : uint64_t get_success_blks; /**< Successful allocation number of contiguous blocks. */
82 : : uint64_t get_fail_blks; /**< Failed allocation number of contiguous blocks. */
83 : : RTE_CACHE_GUARD;
84 : : };
85 : : #endif
86 : :
87 : : /**
88 : : * A structure that stores a per-core object cache.
89 : : */
90 : : struct __rte_cache_aligned rte_mempool_cache {
91 : : uint32_t size; /**< Size of the cache */
92 : : uint32_t flushthresh; /**< Obsolete; for API/ABI compatibility purposes only */
93 : : uint32_t len; /**< Current cache count */
94 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
95 : : uint32_t unused;
96 : : /*
97 : : * Alternative location for the most frequently updated mempool statistics (per-lcore),
98 : : * providing faster update access when using a mempool cache.
99 : : */
100 : : struct {
101 : : uint64_t put_bulk; /**< Number of puts. */
102 : : uint64_t put_objs; /**< Number of objects successfully put. */
103 : : uint64_t get_success_bulk; /**< Successful allocation number. */
104 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
105 : : } stats; /**< Statistics */
106 : : #endif
107 : : /**
108 : : * Cache objects
109 : : *
110 : : * Note:
111 : : * Cache is allocated at double size for API/ABI compatibility purposes only.
112 : : * When reducing its size at an API/ABI breaking release,
113 : : * remember to add a cache guard after it.
114 : : */
115 : : alignas(RTE_CACHE_LINE_SIZE) void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2];
116 : : };
117 : :
118 : : /**
119 : : * A structure that stores the size of mempool elements.
120 : : */
121 : : struct rte_mempool_objsz {
122 : : uint32_t elt_size; /**< Size of an element. */
123 : : uint32_t header_size; /**< Size of header (before elt). */
124 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
125 : : uint32_t total_size;
126 : : /**< Total size of an object (header + elt + trailer). */
127 : : };
128 : :
129 : : /**< Maximum length of a memory pool's name. */
130 : : #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
131 : : sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
132 : : #define RTE_MEMPOOL_MZ_PREFIX "MP_"
133 : :
134 : : /* "MP_<name>" */
135 : : #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
136 : :
137 : : #ifndef RTE_MEMPOOL_ALIGN
138 : : /**
139 : : * Alignment of elements inside mempool.
140 : : */
141 : : #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
142 : : #endif
143 : :
144 : : #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
145 : :
146 : : /**
147 : : * Mempool object header structure
148 : : *
149 : : * Each object stored in mempools are prefixed by this header structure,
150 : : * it allows retrieving the mempool pointer from the object and
151 : : * iterate on all objects attached to a mempool. When debug is enabled,
152 : : * a cookie is also added in this structure preventing corruptions and
153 : : * double-frees.
154 : : */
155 : : struct rte_mempool_objhdr {
156 : : RTE_STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */
157 : : struct rte_mempool *mp; /**< The mempool owning the object. */
158 : : rte_iova_t iova; /**< IO address of the object. */
159 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
160 : : uint64_t cookie; /**< Debug cookie. */
161 : : #endif
162 : : };
163 : :
164 : : /**
165 : : * A list of object headers type
166 : : */
167 : : RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
168 : :
169 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
170 : :
171 : : /**
172 : : * Mempool object trailer structure
173 : : *
174 : : * In debug mode, each object stored in mempools are suffixed by this
175 : : * trailer structure containing a cookie preventing memory corruptions.
176 : : */
177 : : struct rte_mempool_objtlr {
178 : : uint64_t cookie; /**< Debug cookie. */
179 : : };
180 : :
181 : : #endif
182 : :
183 : : /**
184 : : * @internal Logtype used for mempool related messages.
185 : : */
186 : : extern int rte_mempool_logtype;
187 : : #define RTE_LOGTYPE_MEMPOOL rte_mempool_logtype
188 : : #define RTE_MEMPOOL_LOG(level, ...) \
189 : : RTE_LOG_LINE(level, MEMPOOL, "" __VA_ARGS__)
190 : :
191 : : /**
192 : : * A list of memory where objects are stored
193 : : */
194 : : RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
195 : :
196 : : /**
197 : : * Callback used to free a memory chunk
198 : : */
199 : : typedef void (rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr,
200 : : void *opaque);
201 : :
202 : : /**
203 : : * Mempool objects memory header structure
204 : : *
205 : : * The memory chunks where objects are stored. Each chunk is virtually
206 : : * and physically contiguous.
207 : : */
208 : : struct rte_mempool_memhdr {
209 : : RTE_STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */
210 : : struct rte_mempool *mp; /**< The mempool owning the chunk */
211 : : void *addr; /**< Virtual address of the chunk */
212 : : rte_iova_t iova; /**< IO address of the chunk */
213 : : size_t len; /**< length of the chunk */
214 : : rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */
215 : : void *opaque; /**< Argument passed to the free callback */
216 : : };
217 : :
218 : : /**
219 : : * Additional information about the mempool
220 : : *
221 : : * The structure is cache-line aligned to avoid ABI breakages in
222 : : * a number of cases when something small is added.
223 : : */
224 : : struct __rte_cache_aligned rte_mempool_info {
225 : : /** Number of objects in the contiguous block */
226 : : unsigned int contig_block_size;
227 : : };
228 : :
229 : : /**
230 : : * The RTE mempool structure.
231 : : */
232 : : struct __rte_cache_aligned rte_mempool {
233 : : char name[RTE_MEMPOOL_NAMESIZE]; /**< Name of mempool. */
234 : : union {
235 : : void *pool_data; /**< Ring or pool to store objects. */
236 : : uint64_t pool_id; /**< External mempool identifier. */
237 : : };
238 : : void *pool_config; /**< optional args for ops alloc. */
239 : : const struct rte_memzone *mz; /**< Memzone where pool is alloc'd. */
240 : : unsigned int flags; /**< Flags of the mempool. */
241 : : int socket_id; /**< Socket id passed at create. */
242 : : uint32_t size; /**< Max size of the mempool. */
243 : : uint32_t cache_size;
244 : : /**< Size of per-lcore default local cache. */
245 : :
246 : : uint32_t elt_size; /**< Size of an element. */
247 : : uint32_t header_size; /**< Size of header (before elt). */
248 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
249 : :
250 : : unsigned private_data_size; /**< Size of private data. */
251 : : /**
252 : : * Index into rte_mempool_ops_table array of mempool ops
253 : : * structs, which contain callback function pointers.
254 : : * We're using an index here rather than pointers to the callbacks
255 : : * to facilitate any secondary processes that may want to use
256 : : * this mempool.
257 : : */
258 : : int32_t ops_index;
259 : :
260 : : struct rte_mempool_cache *local_cache; /**< Per-lcore local cache */
261 : :
262 : : uint32_t populated_size; /**< Number of populated objects. */
263 : : struct rte_mempool_objhdr_list elt_list; /**< List of objects in pool */
264 : : uint32_t nb_mem_chunks; /**< Number of memory chunks */
265 : : struct rte_mempool_memhdr_list mem_list; /**< List of memory chunks */
266 : :
267 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
268 : : /** Per-lcore statistics.
269 : : *
270 : : * Plus one, for unregistered non-EAL threads.
271 : : */
272 : : struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
273 : : #endif
274 : : };
275 : :
276 : : /** Spreading among memory channels not required. */
277 : : #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
278 : : /**
279 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_SPREAD.
280 : : * To be deprecated.
281 : : */
282 : : #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
283 : : /** Do not align objects on cache lines. */
284 : : #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
285 : : /**
286 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_CACHE_ALIGN.
287 : : * To be deprecated.
288 : : */
289 : : #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
290 : : /** Default put is "single-producer". */
291 : : #define RTE_MEMPOOL_F_SP_PUT 0x0004
292 : : /**
293 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SP_PUT.
294 : : * To be deprecated.
295 : : */
296 : : #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
297 : : /** Default get is "single-consumer". */
298 : : #define RTE_MEMPOOL_F_SC_GET 0x0008
299 : : /**
300 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SC_GET.
301 : : * To be deprecated.
302 : : */
303 : : #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
304 : : /** Internal: pool is created. */
305 : : #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
306 : : /** Don't need IOVA contiguous objects. */
307 : : #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
308 : : /**
309 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_IOVA_CONTIG.
310 : : * To be deprecated.
311 : : */
312 : : #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
313 : : /** Internal: no object from the pool can be used for device IO (DMA). */
314 : : #define RTE_MEMPOOL_F_NON_IO 0x0040
315 : :
316 : : /**
317 : : * This macro lists all the mempool flags an application may request.
318 : : */
319 : : #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
320 : : | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
321 : : | RTE_MEMPOOL_F_SP_PUT \
322 : : | RTE_MEMPOOL_F_SC_GET \
323 : : | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
324 : : )
325 : :
326 : : /**
327 : : * @internal When stats is enabled, store some statistics.
328 : : *
329 : : * @param mp
330 : : * Pointer to the memory pool.
331 : : * @param name
332 : : * Name of the statistics field to increment in the memory pool.
333 : : * @param n
334 : : * Number to add to the statistics.
335 : : */
336 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
337 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
338 : : unsigned int __lcore_id = rte_lcore_id(); \
339 : : if (likely(__lcore_id != LCORE_ID_ANY)) \
340 : : (mp)->stats[__lcore_id].name += (n); \
341 : : else \
342 : : rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name), \
343 : : (n), rte_memory_order_relaxed); \
344 : : } while (0)
345 : : #else
346 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
347 : : #endif
348 : :
349 : : /**
350 : : * @internal When stats is enabled, store some statistics.
351 : : *
352 : : * @param cache
353 : : * Pointer to the memory pool cache.
354 : : * @param name
355 : : * Name of the statistics field to increment in the memory pool cache.
356 : : * @param n
357 : : * Number to add to the statistics.
358 : : */
359 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
360 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
361 : : #else
362 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
363 : : #endif
364 : :
365 : : /**
366 : : * @internal Calculate the size of the mempool header.
367 : : *
368 : : * @param mp
369 : : * Pointer to the memory pool.
370 : : * @param cs
371 : : * Size of the per-lcore cache.
372 : : */
373 : : #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
374 : : (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
375 : : (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
376 : :
377 : : /* return the header of a mempool object (internal) */
378 : : static inline struct rte_mempool_objhdr *
379 : : rte_mempool_get_header(void *obj)
380 : : {
381 [ # # ]: 243989 : return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
382 : : sizeof(struct rte_mempool_objhdr));
383 : : }
384 : :
385 : : /**
386 : : * Return a pointer to the mempool owning this object.
387 : : *
388 : : * @param obj
389 : : * An object that is owned by a pool. If this is not the case,
390 : : * the behavior is undefined.
391 : : * @return
392 : : * A pointer to the mempool structure.
393 : : */
394 : : static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
395 : : {
396 : : struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
397 [ - + + - ]: 243989 : return hdr->mp;
398 : : }
399 : :
400 : : /* return the trailer of a mempool object (internal) */
401 : : static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
402 : : {
403 : : struct rte_mempool *mp = rte_mempool_from_obj(obj);
404 : : return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
405 : : }
406 : :
407 : : /**
408 : : * @internal Check and update cookies or panic.
409 : : *
410 : : * @param mp
411 : : * Pointer to the memory pool.
412 : : * @param obj_table_const
413 : : * Pointer to a table of void * pointers (objects).
414 : : * @param n
415 : : * Index of object in object table.
416 : : * @param free
417 : : * - 0: object is supposed to be allocated, mark it as free
418 : : * - 1: object is supposed to be free, mark it as allocated
419 : : * - 2: just check that cookie is valid (free or allocated)
420 : : */
421 : : void rte_mempool_check_cookies(const struct rte_mempool *mp,
422 : : void * const *obj_table_const, unsigned n, int free);
423 : :
424 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
425 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
426 : : rte_mempool_check_cookies(mp, obj_table_const, n, free)
427 : : #else
428 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
429 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
430 : :
431 : : /**
432 : : * @internal Check contiguous object blocks and update cookies or panic.
433 : : *
434 : : * @param mp
435 : : * Pointer to the memory pool.
436 : : * @param first_obj_table_const
437 : : * Pointer to a table of void * pointers (first object of the contiguous
438 : : * object blocks).
439 : : * @param n
440 : : * Number of contiguous object blocks.
441 : : * @param free
442 : : * - 0: object is supposed to be allocated, mark it as free
443 : : * - 1: object is supposed to be free, mark it as allocated
444 : : * - 2: just check that cookie is valid (free or allocated)
445 : : */
446 : : void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
447 : : void * const *first_obj_table_const, unsigned int n, int free);
448 : :
449 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
450 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
451 : : free) \
452 : : rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
453 : : free)
454 : : #else
455 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
456 : : free) \
457 : : do {} while (0)
458 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
459 : :
460 : : #define RTE_MEMPOOL_OPS_NAMESIZE 32 /**< Max length of ops struct name. */
461 : :
462 : : /**
463 : : * Prototype for implementation specific data provisioning function.
464 : : *
465 : : * The function should provide the implementation specific memory for
466 : : * use by the other mempool ops functions in a given mempool ops struct.
467 : : * E.g. the default ops provides an instance of the rte_ring for this purpose.
468 : : * it will most likely point to a different type of data structure, and
469 : : * will be transparent to the application programmer.
470 : : * This function should set mp->pool_data.
471 : : */
472 : : typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
473 : :
474 : : /**
475 : : * Free the opaque private data pointed to by mp->pool_data pointer.
476 : : */
477 : : typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
478 : :
479 : : /**
480 : : * Enqueue 'n' objects into the external pool.
481 : : * @return
482 : : * - 0: Success
483 : : * - <0: Error
484 : : */
485 : : typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
486 : : void * const *obj_table, unsigned int n);
487 : :
488 : : /**
489 : : * Dequeue 'n' objects from the external pool.
490 : : * @return
491 : : * - 0: Success
492 : : * - <0: Error
493 : : */
494 : : typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
495 : : void **obj_table, unsigned int n);
496 : :
497 : : /**
498 : : * Dequeue a number of contiguous object blocks from the external pool.
499 : : */
500 : : typedef int (*rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp,
501 : : void **first_obj_table, unsigned int n);
502 : :
503 : : /**
504 : : * Return the number of available objects in the external pool.
505 : : */
506 : : typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
507 : :
508 : : /**
509 : : * Calculate memory size required to store given number of objects.
510 : : *
511 : : * If mempool objects are not required to be IOVA-contiguous
512 : : * (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is set), min_chunk_size defines
513 : : * virtually contiguous chunk size. Otherwise, if mempool objects must
514 : : * be IOVA-contiguous (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is clear),
515 : : * min_chunk_size defines IOVA-contiguous chunk size.
516 : : *
517 : : * @param[in] mp
518 : : * Pointer to the memory pool.
519 : : * @param[in] obj_num
520 : : * Number of objects.
521 : : * @param[in] pg_shift
522 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
523 : : * @param[out] min_chunk_size
524 : : * Location for minimum size of the memory chunk which may be used to
525 : : * store memory pool objects.
526 : : * @param[out] align
527 : : * Location for required memory chunk alignment.
528 : : * @return
529 : : * Required memory size.
530 : : */
531 : : typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
532 : : uint32_t obj_num, uint32_t pg_shift,
533 : : size_t *min_chunk_size, size_t *align);
534 : :
535 : : /**
536 : : * @internal Helper to calculate memory size required to store given
537 : : * number of objects.
538 : : *
539 : : * This function is internal to mempool library and mempool drivers.
540 : : *
541 : : * If page boundaries may be ignored, it is just a product of total
542 : : * object size including header and trailer and number of objects.
543 : : * Otherwise, it is a number of pages required to store given number of
544 : : * objects without crossing page boundary.
545 : : *
546 : : * Note that if object size is bigger than page size, then it assumes
547 : : * that pages are grouped in subsets of physically continuous pages big
548 : : * enough to store at least one object.
549 : : *
550 : : * Minimum size of memory chunk is the total element size.
551 : : * Required memory chunk alignment is the cache line size.
552 : : *
553 : : * @param[in] mp
554 : : * A pointer to the mempool structure.
555 : : * @param[in] obj_num
556 : : * Number of objects to be added in mempool.
557 : : * @param[in] pg_shift
558 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
559 : : * @param[in] chunk_reserve
560 : : * Amount of memory that must be reserved at the beginning of each page,
561 : : * or at the beginning of the memory area if pg_shift is 0.
562 : : * @param[out] min_chunk_size
563 : : * Location for minimum size of the memory chunk which may be used to
564 : : * store memory pool objects.
565 : : * @param[out] align
566 : : * Location for required memory chunk alignment.
567 : : * @return
568 : : * Required memory size.
569 : : */
570 : : ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
571 : : uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
572 : : size_t *min_chunk_size, size_t *align);
573 : :
574 : : /**
575 : : * Default way to calculate memory size required to store given number of
576 : : * objects.
577 : : *
578 : : * Equivalent to rte_mempool_op_calc_mem_size_helper(mp, obj_num, pg_shift,
579 : : * 0, min_chunk_size, align).
580 : : */
581 : : ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
582 : : uint32_t obj_num, uint32_t pg_shift,
583 : : size_t *min_chunk_size, size_t *align);
584 : :
585 : : /**
586 : : * Function to be called for each populated object.
587 : : *
588 : : * @param[in] mp
589 : : * A pointer to the mempool structure.
590 : : * @param[in] opaque
591 : : * An opaque pointer passed to iterator.
592 : : * @param[in] vaddr
593 : : * Object virtual address.
594 : : * @param[in] iova
595 : : * Input/output virtual address of the object or RTE_BAD_IOVA.
596 : : */
597 : : typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
598 : : void *opaque, void *vaddr, rte_iova_t iova);
599 : :
600 : : /**
601 : : * Populate memory pool objects using provided memory chunk.
602 : : *
603 : : * Populated objects should be enqueued to the pool, e.g. using
604 : : * rte_mempool_ops_enqueue_bulk().
605 : : *
606 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
607 : : * the chunk doesn't need to be physically contiguous (only virtually),
608 : : * and allocated objects may span two pages.
609 : : *
610 : : * @param[in] mp
611 : : * A pointer to the mempool structure.
612 : : * @param[in] max_objs
613 : : * Maximum number of objects to be populated.
614 : : * @param[in] vaddr
615 : : * The virtual address of memory that should be used to store objects.
616 : : * @param[in] iova
617 : : * The IO address
618 : : * @param[in] len
619 : : * The length of memory in bytes.
620 : : * @param[in] obj_cb
621 : : * Callback function to be executed for each populated object.
622 : : * @param[in] obj_cb_arg
623 : : * An opaque pointer passed to the callback function.
624 : : * @return
625 : : * The number of objects added on success.
626 : : * On error, no objects are populated and a negative errno is returned.
627 : : */
628 : : typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
629 : : unsigned int max_objs,
630 : : void *vaddr, rte_iova_t iova, size_t len,
631 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
632 : :
633 : : /**
634 : : * Align objects on addresses multiple of total_elt_sz.
635 : : */
636 : : #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
637 : :
638 : : /**
639 : : * @internal Helper to populate memory pool object using provided memory
640 : : * chunk: just slice objects one by one, taking care of not
641 : : * crossing page boundaries.
642 : : *
643 : : * If RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ is set in flags, the addresses
644 : : * of object headers will be aligned on a multiple of total_elt_sz.
645 : : * This feature is used by octeontx hardware.
646 : : *
647 : : * This function is internal to mempool library and mempool drivers.
648 : : *
649 : : * @param[in] mp
650 : : * A pointer to the mempool structure.
651 : : * @param[in] flags
652 : : * Logical OR of following flags:
653 : : * - RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ: align objects on addresses
654 : : * multiple of total_elt_sz.
655 : : * @param[in] max_objs
656 : : * Maximum number of objects to be added in mempool.
657 : : * @param[in] vaddr
658 : : * The virtual address of memory that should be used to store objects.
659 : : * @param[in] iova
660 : : * The IO address corresponding to vaddr, or RTE_BAD_IOVA.
661 : : * @param[in] len
662 : : * The length of memory in bytes.
663 : : * @param[in] obj_cb
664 : : * Callback function to be executed for each populated object.
665 : : * @param[in] obj_cb_arg
666 : : * An opaque pointer passed to the callback function.
667 : : * @return
668 : : * The number of objects added in mempool.
669 : : */
670 : : int rte_mempool_op_populate_helper(struct rte_mempool *mp,
671 : : unsigned int flags, unsigned int max_objs,
672 : : void *vaddr, rte_iova_t iova, size_t len,
673 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
674 : :
675 : : /**
676 : : * Default way to populate memory pool object using provided memory chunk.
677 : : *
678 : : * Equivalent to rte_mempool_op_populate_helper(mp, 0, max_objs, vaddr, iova,
679 : : * len, obj_cb, obj_cb_arg).
680 : : */
681 : : int rte_mempool_op_populate_default(struct rte_mempool *mp,
682 : : unsigned int max_objs,
683 : : void *vaddr, rte_iova_t iova, size_t len,
684 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
685 : :
686 : : /**
687 : : * Get some additional information about a mempool.
688 : : */
689 : : typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
690 : : struct rte_mempool_info *info);
691 : :
692 : :
693 : : /** Structure defining mempool operations structure */
694 : : struct __rte_cache_aligned rte_mempool_ops {
695 : : char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */
696 : : rte_mempool_alloc_t alloc; /**< Allocate private data. */
697 : : rte_mempool_free_t free; /**< Free the external pool. */
698 : : rte_mempool_enqueue_t enqueue; /**< Enqueue an object. */
699 : : rte_mempool_dequeue_t dequeue; /**< Dequeue an object. */
700 : : rte_mempool_get_count get_count; /**< Get qty of available objs. */
701 : : /**
702 : : * Optional callback to calculate memory size required to
703 : : * store specified number of objects.
704 : : */
705 : : rte_mempool_calc_mem_size_t calc_mem_size;
706 : : /**
707 : : * Optional callback to populate mempool objects using
708 : : * provided memory chunk.
709 : : */
710 : : rte_mempool_populate_t populate;
711 : : /**
712 : : * Get mempool info
713 : : */
714 : : rte_mempool_get_info_t get_info;
715 : : /**
716 : : * Dequeue a number of contiguous object blocks.
717 : : */
718 : : rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks;
719 : : };
720 : :
721 : : #define RTE_MEMPOOL_MAX_OPS_IDX 16 /**< Max registered ops structs */
722 : :
723 : : /**
724 : : * Structure storing the table of registered ops structs, each of which contain
725 : : * the function pointers for the mempool ops functions.
726 : : * Each process has its own storage for this ops struct array so that
727 : : * the mempools can be shared across primary and secondary processes.
728 : : * The indices used to access the array are valid across processes, whereas
729 : : * any function pointers stored directly in the mempool struct would not be.
730 : : * This results in us simply having "ops_index" in the mempool struct.
731 : : */
732 : : struct __rte_cache_aligned rte_mempool_ops_table {
733 : : rte_spinlock_t sl; /**< Spinlock for add/delete. */
734 : : uint32_t num_ops; /**< Number of used ops structs in the table. */
735 : : /**
736 : : * Storage for all possible ops structs.
737 : : */
738 : : struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX];
739 : : };
740 : :
741 : : /** Array of registered ops structs. */
742 : : extern struct rte_mempool_ops_table rte_mempool_ops_table;
743 : :
744 : : /**
745 : : * @internal Get the mempool ops struct from its index.
746 : : *
747 : : * @param ops_index
748 : : * The index of the ops struct in the ops struct table. It must be a valid
749 : : * index: (0 <= idx < num_ops).
750 : : * @return
751 : : * The pointer to the ops struct in the table.
752 : : */
753 : : static inline struct rte_mempool_ops *
754 : 0 : rte_mempool_get_ops(int ops_index)
755 : : {
756 : 0 : RTE_ASSERT((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
757 : :
758 : 0 : return &rte_mempool_ops_table.ops[ops_index];
759 : : }
760 : :
761 : : /**
762 : : * @internal Wrapper for mempool_ops alloc callback.
763 : : *
764 : : * @param mp
765 : : * Pointer to the memory pool.
766 : : * @return
767 : : * - 0: Success; successfully allocated mempool pool_data.
768 : : * - <0: Error; code of alloc function.
769 : : */
770 : : int
771 : : rte_mempool_ops_alloc(struct rte_mempool *mp);
772 : :
773 : : /**
774 : : * @internal Wrapper for mempool_ops dequeue callback.
775 : : *
776 : : * @param mp
777 : : * Pointer to the memory pool.
778 : : * @param obj_table
779 : : * Pointer to a table of void * pointers (objects).
780 : : * @param n
781 : : * Number of objects to get.
782 : : * @return
783 : : * - 0: Success; got n objects.
784 : : * - <0: Error; code of dequeue function.
785 : : */
786 : : static inline int
787 : 11340 : rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
788 : : void **obj_table, unsigned n)
789 : : {
790 : 0 : struct rte_mempool_ops *ops;
791 : 0 : int ret;
792 : :
793 : 0 : rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
794 : 45390556 : ops = rte_mempool_get_ops(mp->ops_index);
795 : 45390527 : ret = ops->dequeue(mp, obj_table, n);
796 : 0 : RTE_ASSERT(ret <= 0);
797 : 0 : if (likely(ret == 0)) {
798 : 0 : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
799 : 0 : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
800 : : }
801 : 11340 : return ret;
802 : : }
803 : :
804 : : /**
805 : : * @internal Wrapper for mempool_ops dequeue_contig_blocks callback.
806 : : *
807 : : * @param[in] mp
808 : : * Pointer to the memory pool.
809 : : * @param[out] first_obj_table
810 : : * Pointer to a table of void * pointers (first objects).
811 : : * @param[in] n
812 : : * Number of blocks to get.
813 : : * @return
814 : : * - 0: Success; got n objects.
815 : : * - <0: Error; code of dequeue function.
816 : : */
817 : : static inline int
818 : : rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
819 : : void **first_obj_table, unsigned int n)
820 : : {
821 : : struct rte_mempool_ops *ops;
822 : : int ret;
823 : :
824 : 0 : ops = rte_mempool_get_ops(mp->ops_index);
825 : : RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
826 : : rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
827 : 0 : ret = ops->dequeue_contig_blocks(mp, first_obj_table, n);
828 : : RTE_ASSERT(ret <= 0);
829 : : return ret;
830 : : }
831 : :
832 : : /**
833 : : * @internal wrapper for mempool_ops enqueue callback.
834 : : *
835 : : * @param mp
836 : : * Pointer to the memory pool.
837 : : * @param obj_table
838 : : * Pointer to a table of void * pointers (objects).
839 : : * @param n
840 : : * Number of objects to put.
841 : : * @return
842 : : * - 0: Success; n objects supplied.
843 : : * - <0: Error; code of enqueue function.
844 : : */
845 : : static inline int
846 : 0 : rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
847 : : unsigned n)
848 : : {
849 : 0 : struct rte_mempool_ops *ops;
850 : 0 : int ret;
851 : :
852 : 0 : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
853 : 0 : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
854 : 0 : rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
855 : 45458750 : ops = rte_mempool_get_ops(mp->ops_index);
856 : 45458750 : ret = ops->enqueue(mp, obj_table, n);
857 : 0 : RTE_ASSERT(ret <= 0);
858 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
859 : : if (unlikely(ret < 0))
860 : : RTE_MEMPOOL_LOG(CRIT, "cannot enqueue %u objects to mempool %s",
861 : : n, mp->name);
862 : : #endif
863 : 253023 : return ret;
864 : : }
865 : :
866 : : /**
867 : : * @internal wrapper for mempool_ops get_count callback.
868 : : *
869 : : * @param mp
870 : : * Pointer to the memory pool.
871 : : * @return
872 : : * The number of available objects in the external pool.
873 : : */
874 : : unsigned
875 : : rte_mempool_ops_get_count(const struct rte_mempool *mp);
876 : :
877 : : /**
878 : : * @internal wrapper for mempool_ops calc_mem_size callback.
879 : : * API to calculate size of memory required to store specified number of
880 : : * object.
881 : : *
882 : : * @param[in] mp
883 : : * Pointer to the memory pool.
884 : : * @param[in] obj_num
885 : : * Number of objects.
886 : : * @param[in] pg_shift
887 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
888 : : * @param[out] min_chunk_size
889 : : * Location for minimum size of the memory chunk which may be used to
890 : : * store memory pool objects.
891 : : * @param[out] align
892 : : * Location for required memory chunk alignment.
893 : : * @return
894 : : * Required memory size aligned at page boundary.
895 : : */
896 : : ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
897 : : uint32_t obj_num, uint32_t pg_shift,
898 : : size_t *min_chunk_size, size_t *align);
899 : :
900 : : /**
901 : : * @internal wrapper for mempool_ops populate callback.
902 : : *
903 : : * Populate memory pool objects using provided memory chunk.
904 : : *
905 : : * @param[in] mp
906 : : * A pointer to the mempool structure.
907 : : * @param[in] max_objs
908 : : * Maximum number of objects to be populated.
909 : : * @param[in] vaddr
910 : : * The virtual address of memory that should be used to store objects.
911 : : * @param[in] iova
912 : : * The IO address
913 : : * @param[in] len
914 : : * The length of memory in bytes.
915 : : * @param[in] obj_cb
916 : : * Callback function to be executed for each populated object.
917 : : * @param[in] obj_cb_arg
918 : : * An opaque pointer passed to the callback function.
919 : : * @return
920 : : * The number of objects added on success.
921 : : * On error, no objects are populated and a negative errno is returned.
922 : : */
923 : : int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
924 : : void *vaddr, rte_iova_t iova, size_t len,
925 : : rte_mempool_populate_obj_cb_t *obj_cb,
926 : : void *obj_cb_arg);
927 : :
928 : : /**
929 : : * Wrapper for mempool_ops get_info callback.
930 : : *
931 : : * @param[in] mp
932 : : * Pointer to the memory pool.
933 : : * @param[out] info
934 : : * Pointer to the rte_mempool_info structure
935 : : * @return
936 : : * - 0: Success; The mempool driver supports retrieving supplementary
937 : : * mempool information
938 : : * - -ENOTSUP - doesn't support get_info ops (valid case).
939 : : */
940 : : int rte_mempool_ops_get_info(const struct rte_mempool *mp,
941 : : struct rte_mempool_info *info);
942 : :
943 : : /**
944 : : * @internal wrapper for mempool_ops free callback.
945 : : *
946 : : * @param mp
947 : : * Pointer to the memory pool.
948 : : */
949 : : void
950 : : rte_mempool_ops_free(struct rte_mempool *mp);
951 : :
952 : : /**
953 : : * Set the ops of a mempool.
954 : : *
955 : : * This can only be done on a mempool that is not populated, i.e. just after
956 : : * a call to rte_mempool_create_empty().
957 : : *
958 : : * @param mp
959 : : * Pointer to the memory pool.
960 : : * @param name
961 : : * Name of the ops structure to use for this mempool.
962 : : * @param pool_config
963 : : * Opaque data that can be passed by the application to the ops functions.
964 : : * @return
965 : : * - 0: Success; the mempool is now using the requested ops functions.
966 : : * - -EINVAL - Invalid ops struct name provided.
967 : : * - -EEXIST - mempool already has an ops struct assigned.
968 : : */
969 : : int
970 : : rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
971 : : void *pool_config);
972 : :
973 : : /**
974 : : * Register mempool operations.
975 : : *
976 : : * @param ops
977 : : * Pointer to an ops structure to register.
978 : : * @return
979 : : * - >=0: Success; return the index of the ops struct in the table.
980 : : * - -EINVAL - some missing callbacks while registering ops struct.
981 : : * - -ENOSPC - the maximum number of ops structs has been reached.
982 : : */
983 : : int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
984 : :
985 : : /**
986 : : * Macro to statically register the ops of a mempool handler.
987 : : * Note that the rte_mempool_register_ops fails silently here when
988 : : * more than RTE_MEMPOOL_MAX_OPS_IDX is registered.
989 : : */
990 : : #define RTE_MEMPOOL_REGISTER_OPS(ops) \
991 : : RTE_INIT(mp_hdlr_init_##ops) \
992 : : { \
993 : : rte_mempool_register_ops(&ops); \
994 : : }
995 : :
996 : : /**
997 : : * An object callback function for mempool.
998 : : *
999 : : * Used by rte_mempool_create() and rte_mempool_obj_iter().
1000 : : */
1001 : : typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
1002 : : void *opaque, void *obj, unsigned obj_idx);
1003 : : typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
1004 : :
1005 : : /**
1006 : : * A memory callback function for mempool.
1007 : : *
1008 : : * Used by rte_mempool_mem_iter().
1009 : : */
1010 : : typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1011 : : void *opaque, struct rte_mempool_memhdr *memhdr,
1012 : : unsigned mem_idx);
1013 : :
1014 : : /**
1015 : : * A mempool constructor callback function.
1016 : : *
1017 : : * Arguments are the mempool and the opaque pointer given by the user in
1018 : : * rte_mempool_create().
1019 : : */
1020 : : typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
1021 : :
1022 : : /**
1023 : : * Free a mempool
1024 : : *
1025 : : * Unlink the mempool from global list, free the memory chunks, and all
1026 : : * memory referenced by the mempool. The objects must not be used by
1027 : : * other cores as they will be freed.
1028 : : *
1029 : : * @param mp
1030 : : * A pointer to the mempool structure.
1031 : : * If NULL then, the function does nothing.
1032 : : */
1033 : : void
1034 : : rte_mempool_free(struct rte_mempool *mp);
1035 : :
1036 : : /**
1037 : : * Create a new mempool named *name* in memory.
1038 : : *
1039 : : * This function uses ``rte_memzone_reserve()`` to allocate memory. The
1040 : : * pool contains n elements of elt_size. Its size is set to n.
1041 : : *
1042 : : * @param name
1043 : : * The name of the mempool.
1044 : : * @param n
1045 : : * The number of elements in the mempool.
1046 : : * @param elt_size
1047 : : * The size of each element.
1048 : : * @param cache_size
1049 : : * If cache_size is non-zero, the rte_mempool library will try to
1050 : : * limit the accesses to the common lockless pool, by maintaining a
1051 : : * per-lcore object cache. This argument must be lower or equal to
1052 : : * RTE_MEMPOOL_CACHE_MAX_SIZE and n.
1053 : : * The access to the per-lcore table is of course
1054 : : * faster than the multi-producer/consumer pool. The cache can be
1055 : : * disabled if the cache_size argument is set to 0; it can be useful to
1056 : : * avoid losing objects in cache.
1057 : : * Note:
1058 : : * Mempool put/get requests of more than cache_size / 2 objects may be
1059 : : * partially or fully served directly by the multi-producer/consumer
1060 : : * pool, to avoid the overhead of copying the objects twice (instead of
1061 : : * once) when using the cache as a bounce buffer.
1062 : : * @param private_data_size
1063 : : * The size of the private data appended after the mempool
1064 : : * structure. This is useful for storing some private data after the
1065 : : * mempool structure, as is done for rte_mbuf_pool for example.
1066 : : * @param mp_init
1067 : : * A function pointer that is called for initialization of the pool,
1068 : : * before object initialization. The user can initialize the private
1069 : : * data in this function if needed. This parameter can be NULL if
1070 : : * not needed.
1071 : : * @param mp_init_arg
1072 : : * An opaque pointer to data that can be used in the mempool
1073 : : * constructor function.
1074 : : * @param obj_init
1075 : : * A function pointer that is called for each object at
1076 : : * initialization of the pool. The user can set some meta data in
1077 : : * objects if needed. This parameter can be NULL if not needed.
1078 : : * The obj_init() function takes the mempool pointer, the init_arg,
1079 : : * the object pointer and the object number as parameters.
1080 : : * @param obj_init_arg
1081 : : * An opaque pointer to data that can be used as an argument for
1082 : : * each call to the object constructor function.
1083 : : * @param socket_id
1084 : : * The *socket_id* argument is the socket identifier in the case of
1085 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1086 : : * constraint for the reserved zone.
1087 : : * @param flags
1088 : : * The *flags* arguments is an OR of following flags:
1089 : : * - RTE_MEMPOOL_F_NO_SPREAD: By default, objects addresses are spread
1090 : : * between channels in RAM: the pool allocator will add padding
1091 : : * between objects depending on the hardware configuration. See
1092 : : * Memory alignment constraints for details. If this flag is set,
1093 : : * the allocator will just align them to a cache line.
1094 : : * - RTE_MEMPOOL_F_NO_CACHE_ALIGN: By default, the returned objects are
1095 : : * cache-aligned. This flag removes this constraint, and no
1096 : : * padding will be present between objects. This flag implies
1097 : : * RTE_MEMPOOL_F_NO_SPREAD.
1098 : : * - RTE_MEMPOOL_F_SP_PUT: If this flag is set, the default behavior
1099 : : * when using rte_mempool_put() or rte_mempool_put_bulk() is
1100 : : * "single-producer". Otherwise, it is "multi-producers".
1101 : : * - RTE_MEMPOOL_F_SC_GET: If this flag is set, the default behavior
1102 : : * when using rte_mempool_get() or rte_mempool_get_bulk() is
1103 : : * "single-consumer". Otherwise, it is "multi-consumers".
1104 : : * - RTE_MEMPOOL_F_NO_IOVA_CONTIG: If set, allocated objects won't
1105 : : * necessarily be contiguous in IO memory.
1106 : : * @return
1107 : : * The pointer to the new allocated mempool, on success. NULL on error
1108 : : * with rte_errno set appropriately. Possible rte_errno values include:
1109 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
1110 : : * - EINVAL - cache size provided is too large or an unknown flag was passed
1111 : : * - ENOSPC - the maximum number of memzones has already been allocated
1112 : : * - EEXIST - a memzone with the same name already exists
1113 : : * - ENOMEM - no appropriate memory area found in which to create memzone
1114 : : */
1115 : : struct rte_mempool *
1116 : : rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1117 : : unsigned cache_size, unsigned private_data_size,
1118 : : rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1119 : : rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1120 : : int socket_id, unsigned int flags)
1121 : : __rte_malloc __rte_dealloc(rte_mempool_free, 1);
1122 : :
1123 : : /**
1124 : : * Create an empty mempool
1125 : : *
1126 : : * The mempool is allocated and initialized, but it is not populated: no
1127 : : * memory is allocated for the mempool elements. The user has to call
1128 : : * rte_mempool_populate_*() to add memory chunks to the pool. Once
1129 : : * populated, the user may also want to initialize each object with
1130 : : * rte_mempool_obj_iter().
1131 : : *
1132 : : * @param name
1133 : : * The name of the mempool.
1134 : : * @param n
1135 : : * The maximum number of elements that can be added in the mempool.
1136 : : * @param elt_size
1137 : : * The size of each element.
1138 : : * @param cache_size
1139 : : * Size of the cache. See rte_mempool_create() for details.
1140 : : * @param private_data_size
1141 : : * The size of the private data appended after the mempool
1142 : : * structure. This is useful for storing some private data after the
1143 : : * mempool structure, as is done for rte_mbuf_pool for example.
1144 : : * @param socket_id
1145 : : * The *socket_id* argument is the socket identifier in the case of
1146 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1147 : : * constraint for the reserved zone.
1148 : : * @param flags
1149 : : * Flags controlling the behavior of the mempool. See
1150 : : * rte_mempool_create() for details.
1151 : : * @return
1152 : : * The pointer to the new allocated mempool, on success. NULL on error
1153 : : * with rte_errno set appropriately. See rte_mempool_create() for details.
1154 : : */
1155 : : struct rte_mempool *
1156 : : rte_mempool_create_empty(const char *name, unsigned int n, unsigned int elt_size,
1157 : : unsigned int cache_size, unsigned int private_data_size,
1158 : : int socket_id, unsigned int flags)
1159 : : __rte_malloc __rte_dealloc(rte_mempool_free, 1);
1160 : :
1161 : : /**
1162 : : * Add physically contiguous memory for objects in the pool at init
1163 : : *
1164 : : * Add a virtually and physically contiguous memory chunk in the pool
1165 : : * where objects can be instantiated.
1166 : : *
1167 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
1168 : : * the chunk doesn't need to be physically contiguous (only virtually),
1169 : : * and allocated objects may span two pages.
1170 : : *
1171 : : * @param mp
1172 : : * A pointer to the mempool structure.
1173 : : * @param vaddr
1174 : : * The virtual address of memory that should be used to store objects.
1175 : : * @param iova
1176 : : * The IO address
1177 : : * @param len
1178 : : * The length of memory in bytes.
1179 : : * @param free_cb
1180 : : * The callback used to free this chunk when destroying the mempool.
1181 : : * @param opaque
1182 : : * An opaque argument passed to free_cb.
1183 : : * @return
1184 : : * The number of objects added on success (strictly positive).
1185 : : * On error, the chunk is not added in the memory list of the
1186 : : * mempool the following code is returned:
1187 : : * (0): not enough room in chunk for one object.
1188 : : * (-ENOSPC): mempool is already populated.
1189 : : * (-ENOMEM): allocation failure.
1190 : : */
1191 : : int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1192 : : rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1193 : : void *opaque);
1194 : :
1195 : : /**
1196 : : * Add virtually contiguous memory for objects in the pool at init
1197 : : *
1198 : : * Add a virtually contiguous memory chunk in the pool where objects can
1199 : : * be instantiated.
1200 : : *
1201 : : * @param mp
1202 : : * A pointer to the mempool structure.
1203 : : * @param addr
1204 : : * The virtual address of memory that should be used to store objects.
1205 : : * @param len
1206 : : * The length of memory in bytes.
1207 : : * @param pg_sz
1208 : : * The size of memory pages in this virtual area.
1209 : : * @param free_cb
1210 : : * The callback used to free this chunk when destroying the mempool.
1211 : : * @param opaque
1212 : : * An opaque argument passed to free_cb.
1213 : : * @return
1214 : : * The number of objects added on success (strictly positive).
1215 : : * On error, the chunk is not added in the memory list of the
1216 : : * mempool the following code is returned:
1217 : : * (0): not enough room in chunk for one object.
1218 : : * (-ENOSPC): mempool is already populated.
1219 : : * (-ENOMEM): allocation failure.
1220 : : */
1221 : : int
1222 : : rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1223 : : size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1224 : : void *opaque);
1225 : :
1226 : : /**
1227 : : * Add memory for objects in the pool at init
1228 : : *
1229 : : * This is the default function used by rte_mempool_create() to populate
1230 : : * the mempool. It adds memory allocated using rte_memzone_reserve().
1231 : : *
1232 : : * @param mp
1233 : : * A pointer to the mempool structure.
1234 : : * @return
1235 : : * The number of objects added on success.
1236 : : * On error, the chunk is not added in the memory list of the
1237 : : * mempool and a negative errno is returned.
1238 : : */
1239 : : int rte_mempool_populate_default(struct rte_mempool *mp);
1240 : :
1241 : : /**
1242 : : * Add memory from anonymous mapping for objects in the pool at init
1243 : : *
1244 : : * This function mmap an anonymous memory zone that is locked in
1245 : : * memory to store the objects of the mempool.
1246 : : *
1247 : : * @param mp
1248 : : * A pointer to the mempool structure.
1249 : : * @return
1250 : : * The number of objects added on success.
1251 : : * On error, 0 is returned, rte_errno is set, and the chunk is not added in
1252 : : * the memory list of the mempool.
1253 : : */
1254 : : int rte_mempool_populate_anon(struct rte_mempool *mp);
1255 : :
1256 : : /**
1257 : : * Call a function for each mempool element
1258 : : *
1259 : : * Iterate across all objects attached to a rte_mempool and call the
1260 : : * callback function on it.
1261 : : *
1262 : : * @param mp
1263 : : * A pointer to an initialized mempool.
1264 : : * @param obj_cb
1265 : : * A function pointer that is called for each object.
1266 : : * @param obj_cb_arg
1267 : : * An opaque pointer passed to the callback function.
1268 : : * @return
1269 : : * Number of objects iterated.
1270 : : */
1271 : : uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1272 : : rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1273 : :
1274 : : /**
1275 : : * Call a function for each mempool memory chunk
1276 : : *
1277 : : * Iterate across all memory chunks attached to a rte_mempool and call
1278 : : * the callback function on it.
1279 : : *
1280 : : * @param mp
1281 : : * A pointer to an initialized mempool.
1282 : : * @param mem_cb
1283 : : * A function pointer that is called for each memory chunk.
1284 : : * @param mem_cb_arg
1285 : : * An opaque pointer passed to the callback function.
1286 : : * @return
1287 : : * Number of memory chunks iterated.
1288 : : */
1289 : : uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1290 : : rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1291 : :
1292 : : /**
1293 : : * @warning
1294 : : * @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
1295 : : *
1296 : : * Reset the statistics of a mempool.
1297 : : *
1298 : : * This function is intended for use when analyzing mempool statistics
1299 : : * without counting any mempool operations performed during application
1300 : : * initialization.
1301 : : * For example, populating the mempool counts as put operations into the
1302 : : * common pool, and setting up ethdev Rx queues counts as get operations.
1303 : : *
1304 : : * This function should only be called after application initialization,
1305 : : * before the data path is started; otherwise, the mempool statistics may
1306 : : * become inconsistent.
1307 : : *
1308 : : * For a perfectly clean slate, the local caches of the mempools used
1309 : : * during application initialization should be flushed before resetting
1310 : : * the mempool statistics.
1311 : : * For example, mbuf pools used by ethdev Rx queues.
1312 : : *
1313 : : * @see rte_mempool_cache_flush()
1314 : : *
1315 : : * @param mp
1316 : : * A pointer to the mempool structure.
1317 : : */
1318 : : __rte_experimental
1319 : : void rte_mempool_stats_reset(struct rte_mempool *mp);
1320 : :
1321 : : /**
1322 : : * Dump the status of the mempool to a file.
1323 : : *
1324 : : * @param f
1325 : : * A pointer to a file for output
1326 : : * @param mp
1327 : : * A pointer to the mempool structure.
1328 : : */
1329 : : void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1330 : :
1331 : : /**
1332 : : * Create a user-owned mempool cache.
1333 : : *
1334 : : * This can be used by unregistered non-EAL threads to enable caching when they
1335 : : * interact with a mempool.
1336 : : *
1337 : : * @param size
1338 : : * The size of the mempool cache. See rte_mempool_create()'s cache_size
1339 : : * parameter description for more information. The same limits and
1340 : : * considerations apply here too.
1341 : : * @param socket_id
1342 : : * The socket identifier in the case of NUMA. The value can be
1343 : : * SOCKET_ID_ANY if there is no NUMA constraint for the reserved zone.
1344 : : */
1345 : : struct rte_mempool_cache *
1346 : : rte_mempool_cache_create(uint32_t size, int socket_id);
1347 : :
1348 : : /**
1349 : : * Free a user-owned mempool cache.
1350 : : *
1351 : : * @param cache
1352 : : * A pointer to the mempool cache.
1353 : : */
1354 : : void
1355 : : rte_mempool_cache_free(struct rte_mempool_cache *cache);
1356 : :
1357 : : /**
1358 : : * Get a pointer to the per-lcore default mempool cache.
1359 : : *
1360 : : * @param mp
1361 : : * A pointer to the mempool structure.
1362 : : * @param lcore_id
1363 : : * The logical core id.
1364 : : * @return
1365 : : * A pointer to the mempool cache or NULL if disabled or unregistered non-EAL
1366 : : * thread.
1367 : : */
1368 : : static __rte_always_inline struct rte_mempool_cache *
1369 : 0 : rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1370 : : {
1371 [ + + + + : 2687156 : if (unlikely(mp->cache_size == 0))
+ + + + +
+ + + + +
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1372 : : return NULL;
1373 : :
1374 [ + + + - : 2179006 : if (unlikely(lcore_id == LCORE_ID_ANY))
+ + + - +
- + - + -
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1375 : : return NULL;
1376 : :
1377 : 0 : rte_mempool_trace_default_cache(mp, lcore_id,
1378 : 2178994 : &mp->local_cache[lcore_id]);
1379 : 2178995 : return &mp->local_cache[lcore_id];
1380 : : }
1381 : :
1382 : : /**
1383 : : * Flush a user-owned mempool cache to the specified mempool.
1384 : : *
1385 : : * @param cache
1386 : : * A pointer to the mempool cache.
1387 : : * @param mp
1388 : : * A pointer to the mempool.
1389 : : */
1390 : : static __rte_always_inline void
1391 : : rte_mempool_cache_flush(struct rte_mempool_cache *cache,
1392 : : struct rte_mempool *mp)
1393 : : {
1394 [ - + ]: 3 : if (cache == NULL)
1395 [ # # ]: 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
1396 [ + - + - ]: 3 : if (cache == NULL || cache->len == 0)
1397 : : return;
1398 : : rte_mempool_trace_cache_flush(cache, mp);
1399 : 3 : rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1400 : 3 : cache->len = 0;
1401 : : }
1402 : :
1403 : : /**
1404 : : * @internal Put several objects back in the mempool; used internally.
1405 : : * @param mp
1406 : : * A pointer to the mempool structure.
1407 : : * @param obj_table
1408 : : * A pointer to a table of void * pointers (objects).
1409 : : * @param n
1410 : : * The number of objects to store back in the mempool, must be strictly
1411 : : * positive.
1412 : : * @param cache
1413 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1414 : : */
1415 : : static __rte_always_inline void
1416 : 0 : rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1417 : : unsigned int n, struct rte_mempool_cache *cache)
1418 : : {
1419 : 0 : void **cache_objs;
1420 : :
1421 : : /* No cache provided? */
1422 [ + + + + : 2362681 : if (unlikely(cache == NULL))
+ + - + +
+ + + + +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1423 : 253017 : goto driver_enqueue;
1424 : :
1425 : : /* Increment stats now, adding in mempool always succeeds. */
1426 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1427 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1428 : :
1429 [ - + - + : 2109664 : __rte_assume(cache->size <= RTE_MEMPOOL_CACHE_MAX_SIZE);
- + - + -
+ - + - +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1430 : 0 : __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
1431 [ - + - + : 2109664 : __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE);
- + - + -
+ - + - +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1432 [ - + - + : 2109664 : __rte_assume(cache->len <= cache->size);
- + - + -
+ - + - +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1433 [ + + + + : 2109662 : if (likely(cache->len + n <= cache->size)) {
+ + + - +
- + - + +
- - - - +
- + + - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1434 : : /* Sufficient room in the cache for the objects. */
1435 : 1978407 : cache_objs = &cache->objs[cache->len];
1436 [ # # # # ]: 1978407 : cache->len += n;
1437 [ + - + + : 131255 : } else if (n <= cache->size / 2) {
- + - - -
- - - + -
- - - - -
- + - - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1438 : : /*
1439 : : * The number of objects is within the cache bounce buffer limit,
1440 : : * but - as detected by the comparison above - the cache has
1441 : : * insufficient room for them.
1442 : : * Flush the cache to the backend to make room for the objects;
1443 : : * flush (size / 2) objects from the bottom of the cache, where
1444 : : * objects are less hot, and move down the remaining objects, which
1445 : : * are more hot, from the upper half of the cache.
1446 : : */
1447 [ - + - + : 131249 : __rte_assume(cache->len > cache->size / 2);
- - - - -
- - - - +
- - - - -
- - + - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1448 : 131249 : rte_mempool_ops_enqueue_bulk(mp, &cache->objs[0], cache->size / 2);
1449 : 131249 : rte_memcpy(&cache->objs[0], &cache->objs[cache->size / 2],
1450 [ - + - + : 131249 : sizeof(void *) * (cache->len - cache->size / 2));
- - - - -
- - - - +
- - - - -
- - + - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1451 : 131249 : cache_objs = &cache->objs[cache->len - cache->size / 2];
1452 : 131249 : cache->len = cache->len - cache->size / 2 + n;
1453 : : } else {
1454 : : /* The request itself is too big for the cache. */
1455 : 8 : goto driver_enqueue_stats_incremented;
1456 : : }
1457 : :
1458 : : /* Add the objects to the cache. */
1459 [ + + - - : 68 : rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1460 : :
1461 : : return;
1462 : :
1463 : 0 : driver_enqueue:
1464 : :
1465 : : /* increment stat now, adding in mempool always success */
1466 : 0 : RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1467 : 0 : RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1468 : :
1469 : 253025 : driver_enqueue_stats_incremented:
1470 : :
1471 : : /* push objects to the backend */
1472 [ # # # # ]: 0 : rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1473 : : }
1474 : :
1475 : :
1476 : : /**
1477 : : * Put several objects back in the mempool.
1478 : : *
1479 : : * @param mp
1480 : : * A pointer to the mempool structure.
1481 : : * @param obj_table
1482 : : * A pointer to a table of void * pointers (objects).
1483 : : * @param n
1484 : : * The number of objects to add in the mempool from the obj_table.
1485 : : * @param cache
1486 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1487 : : */
1488 : : static __rte_always_inline void
1489 : 0 : rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1490 : : unsigned int n, struct rte_mempool_cache *cache)
1491 : : {
1492 : 0 : rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1493 : 0 : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1494 : 0 : rte_mempool_do_generic_put(mp, obj_table, n, cache);
1495 : : }
1496 : :
1497 : : /**
1498 : : * Put several objects back in the mempool.
1499 : : *
1500 : : * This function calls the multi-producer or the single-producer
1501 : : * version depending on the default behavior that was specified at
1502 : : * mempool creation time (see flags).
1503 : : *
1504 : : * @param mp
1505 : : * A pointer to the mempool structure.
1506 : : * @param obj_table
1507 : : * A pointer to a table of void * pointers (objects).
1508 : : * @param n
1509 : : * The number of objects to add in the mempool from obj_table.
1510 : : */
1511 : : static __rte_always_inline void
1512 : 0 : rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1513 : : unsigned int n)
1514 : : {
1515 : 0 : struct rte_mempool_cache *cache;
1516 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1517 : 0 : rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1518 [ # # # # : 0 : rte_mempool_generic_put(mp, obj_table, n, cache);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1519 : : }
1520 : :
1521 : : /**
1522 : : * Put one object back in the mempool.
1523 : : *
1524 : : * This function calls the multi-producer or the single-producer
1525 : : * version depending on the default behavior that was specified at
1526 : : * mempool creation time (see flags).
1527 : : *
1528 : : * @param mp
1529 : : * A pointer to the mempool structure.
1530 : : * @param obj
1531 : : * A pointer to the object to be added.
1532 : : */
1533 : : static __rte_always_inline void
1534 : 0 : rte_mempool_put(struct rte_mempool *mp, void *obj)
1535 : : {
1536 [ # # # # : 0 : rte_mempool_put_bulk(mp, &obj, 1);
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1537 : : }
1538 : :
1539 : : /**
1540 : : * @internal Get several objects from the mempool; used internally.
1541 : : * @param mp
1542 : : * A pointer to the mempool structure.
1543 : : * @param obj_table
1544 : : * A pointer to a table of void * pointers (objects).
1545 : : * @param n
1546 : : * The number of objects to get, must be strictly positive.
1547 : : * @param cache
1548 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1549 : : * @return
1550 : : * - 0: Success.
1551 : : * - <0: Error; code of driver dequeue function.
1552 : : */
1553 : : static __rte_always_inline int
1554 : 0 : rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1555 : : unsigned int n, struct rte_mempool_cache *cache)
1556 : : {
1557 : 0 : int ret;
1558 : 0 : unsigned int remaining;
1559 : 0 : uint32_t index, len;
1560 : 0 : void **cache_objs;
1561 : :
1562 : : /* No cache provided? */
1563 [ + + + + : 335053 : if (unlikely(cache == NULL)) {
+ - + + +
+ + + + +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1564 : 0 : remaining = n;
1565 : 257259 : goto driver_dequeue;
1566 : : }
1567 : :
1568 : : /* The cache is a stack, so copy will be in reverse order. */
1569 : 77794 : cache_objs = &cache->objs[cache->len];
1570 : :
1571 [ - + - + : 77794 : __rte_assume(cache->len <= RTE_MEMPOOL_CACHE_MAX_SIZE);
- - - + -
+ - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1572 [ + + + + : 77792 : if (likely(n <= cache->len)) {
- - - + +
- + + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1573 : : /* The entire request can be satisfied from the cache. */
1574 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1575 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1576 : :
1577 : : /*
1578 : : * If the request size is known at build time,
1579 : : * the compiler unrolls the fixed length copy loop.
1580 : : */
1581 : 11893 : cache->len -= n;
1582 [ + + + + : 24197 : for (index = 0; index < n; index++)
- - - - +
+ + + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1583 : 12304 : *obj_table++ = *--cache_objs;
1584 : :
1585 : : return 0;
1586 : : }
1587 : :
1588 : : /* Use the cache as much as we have to return hot objects first. */
1589 : 0 : len = cache->len;
1590 : 65901 : remaining = n - len;
1591 : 65901 : cache->len = 0;
1592 [ + + - + : 66529 : for (index = 0; index < len; index++)
- - - + -
- + + + +
+ + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1593 : 628 : *obj_table++ = *--cache_objs;
1594 : :
1595 : : /* Dequeue below would exceed the cache bounce buffer limit? */
1596 [ - + - + : 65901 : __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
- - - + -
- - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1597 [ + + - + : 65899 : if (unlikely(remaining > cache->size / 2))
- - - + -
- + + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1598 : 65558 : goto driver_dequeue;
1599 : :
1600 : : /* Fill the cache from the backend; fetch (size / 2) objects. */
1601 : 343 : ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs, cache->size / 2);
1602 [ + + + + : 343 : if (unlikely(ret < 0)) {
- - - + -
- - + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1603 : : /*
1604 : : * We are buffer constrained, and not able to fetch all that.
1605 : : * Do not fill the cache, just satisfy the remaining part of
1606 : : * the request directly from the backend.
1607 : : */
1608 : 132 : goto driver_dequeue;
1609 : : }
1610 : :
1611 : : /* Satisfy the remaining part of the request from the filled cache. */
1612 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1613 : 0 : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1614 : :
1615 [ - + - + : 211 : __rte_assume(cache->size / 2 <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
- - - + -
- - + - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1616 [ - + - + : 211 : __rte_assume(remaining <= RTE_MEMPOOL_CACHE_MAX_SIZE / 2);
- - - + -
- - + - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1617 [ - + - + : 211 : __rte_assume(remaining <= cache->size / 2);
- - - + -
- - + - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1618 : 211 : cache_objs = &cache->objs[cache->size / 2];
1619 : 211 : cache->len = cache->size / 2 - remaining;
1620 [ + + + + : 515 : for (index = 0; index < remaining; index++)
- - + + -
- + + + +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1621 : 304 : *obj_table++ = *--cache_objs;
1622 : :
1623 : : return 0;
1624 : :
1625 : 318581 : driver_dequeue:
1626 : :
1627 : : /* Get remaining objects directly from the backend. */
1628 : 11149 : ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1629 : :
1630 [ + + + + : 322949 : if (unlikely(ret < 0)) {
- + - + -
+ - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1631 [ + + + + : 18 : if (likely(cache != NULL)) {
- - - - -
- - - - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1632 : 16 : cache->len = n - remaining;
1633 : : /*
1634 : : * No further action is required to roll the first part
1635 : : * of the request back into the cache, as objects in
1636 : : * the cache are intact.
1637 : : */
1638 : : }
1639 : :
1640 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1641 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1642 : : } else {
1643 : 0 : if (likely(cache != NULL)) {
1644 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1645 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1646 : : } else {
1647 : 0 : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1648 : 0 : RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1649 : : }
1650 [ - + - + : 322931 : __rte_assume(ret == 0);
- + - + -
+ + + + +
+ - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1651 : : }
1652 : :
1653 : : return ret;
1654 : : }
1655 : :
1656 : : /**
1657 : : * Get several objects from the mempool.
1658 : : *
1659 : : * If cache is enabled, objects will be retrieved first from cache,
1660 : : * subsequently from the common pool. Note that it can return -ENOENT when
1661 : : * the local cache and common pool are empty, even if cache from other
1662 : : * lcores are full.
1663 : : *
1664 : : * @param mp
1665 : : * A pointer to the mempool structure.
1666 : : * @param obj_table
1667 : : * A pointer to a table of void * pointers (objects) that will be filled.
1668 : : * @param n
1669 : : * The number of objects to get from mempool to obj_table.
1670 : : * @param cache
1671 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1672 : : * @return
1673 : : * - 0: Success; objects taken.
1674 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1675 : : */
1676 : : static __rte_always_inline int
1677 : 0 : rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1678 : : unsigned int n, struct rte_mempool_cache *cache)
1679 : : {
1680 : 0 : int ret;
1681 : 0 : ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1682 : 0 : if (likely(ret == 0))
1683 : 0 : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1684 : 0 : rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1685 : 0 : return ret;
1686 : : }
1687 : :
1688 : : /**
1689 : : * Get several objects from the mempool.
1690 : : *
1691 : : * This function calls the multi-consumers or the single-consumer
1692 : : * version, depending on the default behaviour that was specified at
1693 : : * mempool creation time (see flags).
1694 : : *
1695 : : * If cache is enabled, objects will be retrieved first from cache,
1696 : : * subsequently from the common pool. Note that it can return -ENOENT when
1697 : : * the local cache and common pool are empty, even if cache from other
1698 : : * lcores are full.
1699 : : *
1700 : : * @param mp
1701 : : * A pointer to the mempool structure.
1702 : : * @param obj_table
1703 : : * A pointer to a table of void * pointers (objects) that will be filled.
1704 : : * @param n
1705 : : * The number of objects to get from the mempool to obj_table.
1706 : : * @return
1707 : : * - 0: Success; objects taken
1708 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1709 : : */
1710 : : static __rte_always_inline int
1711 : 0 : rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1712 : : {
1713 : 0 : struct rte_mempool_cache *cache;
1714 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1715 : 0 : rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1716 [ # # # # : 0 : return rte_mempool_generic_get(mp, obj_table, n, cache);
# # ]
1717 : : }
1718 : :
1719 : : /**
1720 : : * Get one object from the mempool.
1721 : : *
1722 : : * This function calls the multi-consumers or the single-consumer
1723 : : * version, depending on the default behavior that was specified at
1724 : : * mempool creation (see flags).
1725 : : *
1726 : : * If cache is enabled, objects will be retrieved first from cache,
1727 : : * subsequently from the common pool. Note that it can return -ENOENT when
1728 : : * the local cache and common pool are empty, even if cache from other
1729 : : * lcores are full.
1730 : : *
1731 : : * @param mp
1732 : : * A pointer to the mempool structure.
1733 : : * @param obj_p
1734 : : * A pointer to a void * pointer (object) that will be filled.
1735 : : * @return
1736 : : * - 0: Success; objects taken.
1737 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1738 : : */
1739 : : static __rte_always_inline int
1740 : 0 : rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1741 : : {
1742 [ # # # # ]: 0 : return rte_mempool_get_bulk(mp, obj_p, 1);
1743 : : }
1744 : :
1745 : : /**
1746 : : * Get a contiguous blocks of objects from the mempool.
1747 : : *
1748 : : * If cache is enabled, consider to flush it first, to reuse objects
1749 : : * as soon as possible.
1750 : : *
1751 : : * The application should check that the driver supports the operation
1752 : : * by calling rte_mempool_ops_get_info() and checking that `contig_block_size`
1753 : : * is not zero.
1754 : : *
1755 : : * @param mp
1756 : : * A pointer to the mempool structure.
1757 : : * @param first_obj_table
1758 : : * A pointer to a pointer to the first object in each block.
1759 : : * @param n
1760 : : * The number of blocks to get from mempool.
1761 : : * @return
1762 : : * - 0: Success; blocks taken.
1763 : : * - -ENOBUFS: Not enough entries in the mempool; no object is retrieved.
1764 : : * - -EOPNOTSUPP: The mempool driver does not support block dequeue
1765 : : */
1766 : : static __rte_always_inline int
1767 : : rte_mempool_get_contig_blocks(struct rte_mempool *mp,
1768 : : void **first_obj_table, unsigned int n)
1769 : : {
1770 : : int ret;
1771 : :
1772 : : ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1773 : : if (likely(ret == 0)) {
1774 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1775 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1776 : : RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1777 : : 1);
1778 : : } else {
1779 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1780 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1781 : : }
1782 : :
1783 : : rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1784 : : return ret;
1785 : : }
1786 : :
1787 : : /**
1788 : : * Return the number of entries in the mempool.
1789 : : *
1790 : : * When cache is enabled, this function has to browse the length of
1791 : : * all lcores, so it should not be used in a data path, but only for
1792 : : * debug purposes. User-owned mempool caches are not accounted for.
1793 : : *
1794 : : * @param mp
1795 : : * A pointer to the mempool structure.
1796 : : * @return
1797 : : * The number of entries in the mempool.
1798 : : */
1799 : : unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1800 : :
1801 : : /**
1802 : : * Return the number of elements which have been allocated from the mempool
1803 : : *
1804 : : * When cache is enabled, this function has to browse the length of
1805 : : * all lcores, so it should not be used in a data path, but only for
1806 : : * debug purposes.
1807 : : *
1808 : : * @param mp
1809 : : * A pointer to the mempool structure.
1810 : : * @return
1811 : : * The number of free entries in the mempool.
1812 : : */
1813 : : unsigned int
1814 : : rte_mempool_in_use_count(const struct rte_mempool *mp);
1815 : :
1816 : : /**
1817 : : * Test if the mempool is full.
1818 : : *
1819 : : * When cache is enabled, this function has to browse the length of all
1820 : : * lcores, so it should not be used in a data path, but only for debug
1821 : : * purposes. User-owned mempool caches are not accounted for.
1822 : : *
1823 : : * @param mp
1824 : : * A pointer to the mempool structure.
1825 : : * @return
1826 : : * - 1: The mempool is full.
1827 : : * - 0: The mempool is not full.
1828 : : */
1829 : : static inline int
1830 : : rte_mempool_full(const struct rte_mempool *mp)
1831 : : {
1832 [ + + - + : 6 : return rte_mempool_avail_count(mp) == mp->size;
- + - + +
- - + ]
1833 : : }
1834 : :
1835 : : /**
1836 : : * Test if the mempool is empty.
1837 : : *
1838 : : * When cache is enabled, this function has to browse the length of all
1839 : : * lcores, so it should not be used in a data path, but only for debug
1840 : : * purposes. User-owned mempool caches are not accounted for.
1841 : : *
1842 : : * @param mp
1843 : : * A pointer to the mempool structure.
1844 : : * @return
1845 : : * - 1: The mempool is empty.
1846 : : * - 0: The mempool is not empty.
1847 : : */
1848 : : static inline int
1849 : : rte_mempool_empty(const struct rte_mempool *mp)
1850 : : {
1851 : 5 : return rte_mempool_avail_count(mp) == 0;
1852 : : }
1853 : :
1854 : : /**
1855 : : * Return the IO address of elt, which is an element of the pool mp.
1856 : : *
1857 : : * @param elt
1858 : : * A pointer (virtual address) to the element of the pool.
1859 : : * @return
1860 : : * The IO address of the elt element.
1861 : : * If the mempool was created with RTE_MEMPOOL_F_NO_IOVA_CONTIG, the
1862 : : * returned value is RTE_BAD_IOVA.
1863 : : */
1864 : : static inline rte_iova_t
1865 : 0 : rte_mempool_virt2iova(const void *elt)
1866 : : {
1867 : 0 : const struct rte_mempool_objhdr *hdr;
1868 : 104982 : hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1869 : : sizeof(*hdr));
1870 [ - + ]: 363 : return hdr->iova;
1871 : : }
1872 : :
1873 : : /**
1874 : : * Check the consistency of mempool objects.
1875 : : *
1876 : : * Verify the coherency of fields in the mempool structure. Also check
1877 : : * that the cookies of mempool objects (even the ones that are not
1878 : : * present in pool) have a correct value. If not, a panic will occur.
1879 : : *
1880 : : * @param mp
1881 : : * A pointer to the mempool structure.
1882 : : */
1883 : : void rte_mempool_audit(struct rte_mempool *mp);
1884 : :
1885 : : /**
1886 : : * Return a pointer to the private data in an mempool structure.
1887 : : *
1888 : : * @param mp
1889 : : * A pointer to the mempool structure.
1890 : : * @return
1891 : : * A pointer to the private data.
1892 : : */
1893 : 0 : static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1894 : : {
1895 [ + + - + : 23724 : return (char *)mp +
+ + + + +
+ + + + +
+ + + - #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1896 [ + + + + : 120175 : RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
+ + + + +
+ + + + +
+ + + + +
+ # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1897 : : }
1898 : :
1899 : : /**
1900 : : * Dump the status of all mempools on the console
1901 : : *
1902 : : * @param f
1903 : : * A pointer to a file for output
1904 : : */
1905 : : void rte_mempool_list_dump(FILE *f);
1906 : :
1907 : : /**
1908 : : * Search a mempool from its name
1909 : : *
1910 : : * @param name
1911 : : * The name of the mempool.
1912 : : * @return
1913 : : * The pointer to the mempool matching the name, or NULL if not found.
1914 : : * NULL on error
1915 : : * with rte_errno set appropriately. Possible rte_errno values include:
1916 : : * - ENOENT - required entry not available to return.
1917 : : */
1918 : : struct rte_mempool *rte_mempool_lookup(const char *name);
1919 : :
1920 : : /**
1921 : : * Get the header, trailer and total size of a mempool element.
1922 : : *
1923 : : * Given a desired size of the mempool element and mempool flags,
1924 : : * calculates header, trailer, body and total sizes of the mempool object.
1925 : : *
1926 : : * @param elt_size
1927 : : * The size of each element, without header and trailer.
1928 : : * @param flags
1929 : : * The flags used for the mempool creation.
1930 : : * Consult rte_mempool_create() for more information about possible values.
1931 : : * @param sz
1932 : : * The calculated detailed size the mempool object. May be NULL.
1933 : : * @return
1934 : : * Total size of the mempool object.
1935 : : */
1936 : : uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1937 : : struct rte_mempool_objsz *sz);
1938 : :
1939 : : /**
1940 : : * Walk list of all memory pools
1941 : : *
1942 : : * @param func
1943 : : * Iterator function
1944 : : * @param arg
1945 : : * Argument passed to iterator
1946 : : */
1947 : : void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1948 : : void *arg);
1949 : :
1950 : : /**
1951 : : * A structure used to retrieve information about the memory range
1952 : : * of the mempool.
1953 : : */
1954 : : struct rte_mempool_mem_range_info {
1955 : : /** Start of the memory range used by mempool objects */
1956 : : void *start;
1957 : : /** Length of the memory range used by mempool objects */
1958 : : size_t length;
1959 : : /** Are all memory addresses used by mempool objects contiguous */
1960 : : bool is_contiguous;
1961 : : };
1962 : :
1963 : : /**
1964 : : * @warning
1965 : : * @b EXPERIMENTAL: this API may change without prior notice.
1966 : : *
1967 : : * Get information about the memory range used to store objects in the mempool.
1968 : : *
1969 : : * @param[in] mp
1970 : : * Pointer to an initialized mempool.
1971 : : * @param[out] mem_range
1972 : : * Pointer to struct which is used to return lowest address,
1973 : : * length of the memory range containing all the addresses,
1974 : : * and whether these addresses are contiguous.
1975 : : * @return
1976 : : * 0 on success, -EINVAL if mempool is not valid or mem_range is NULL.
1977 : : */
1978 : : __rte_experimental
1979 : : int
1980 : : rte_mempool_get_mem_range(const struct rte_mempool *mp,
1981 : : struct rte_mempool_mem_range_info *mem_range);
1982 : :
1983 : : /**
1984 : : * @warning
1985 : : * @b EXPERIMENTAL: this API may change without prior notice.
1986 : : *
1987 : : * Return alignment of objects stored in the mempool.
1988 : : *
1989 : : * @param[in] mp
1990 : : * Pointer to a mempool.
1991 : : * @return
1992 : : * Object alignment if mp is valid. 0 if mp is NULL.
1993 : : */
1994 : : __rte_experimental
1995 : : size_t
1996 : : rte_mempool_get_obj_alignment(const struct rte_mempool *mp);
1997 : :
1998 : : /**
1999 : : * @internal Get page size used for mempool object allocation.
2000 : : * This function is internal to mempool library and mempool drivers.
2001 : : */
2002 : : int
2003 : : rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
2004 : :
2005 : : /**
2006 : : * Mempool event type.
2007 : : * @internal
2008 : : */
2009 : : enum rte_mempool_event {
2010 : : /** Occurs after a mempool is fully populated. */
2011 : : RTE_MEMPOOL_EVENT_READY = 0,
2012 : : /** Occurs before the destruction of a mempool begins. */
2013 : : RTE_MEMPOOL_EVENT_DESTROY = 1,
2014 : : };
2015 : :
2016 : : /**
2017 : : * @internal
2018 : : * Mempool event callback.
2019 : : *
2020 : : * rte_mempool_event_callback_register() may be called from within the callback,
2021 : : * but the callbacks registered this way will not be invoked for the same event.
2022 : : * rte_mempool_event_callback_unregister() may only be safely called
2023 : : * to remove the running callback.
2024 : : */
2025 : : typedef void (rte_mempool_event_callback)(
2026 : : enum rte_mempool_event event,
2027 : : struct rte_mempool *mp,
2028 : : void *user_data);
2029 : :
2030 : : /**
2031 : : * @internal
2032 : : * Register a callback function invoked on mempool life cycle event.
2033 : : * The function will be invoked in the process
2034 : : * that performs an action which triggers the callback.
2035 : : * Registration is process-private,
2036 : : * i.e. each process must manage callbacks on its own if needed.
2037 : : *
2038 : : * @param func
2039 : : * Callback function.
2040 : : * @param user_data
2041 : : * User data.
2042 : : *
2043 : : * @return
2044 : : * 0 on success, negative on failure and rte_errno is set.
2045 : : */
2046 : : __rte_internal
2047 : : int
2048 : : rte_mempool_event_callback_register(rte_mempool_event_callback *func,
2049 : : void *user_data);
2050 : :
2051 : : /**
2052 : : * @internal
2053 : : * Unregister a callback added with rte_mempool_event_callback_register().
2054 : : * @p func and @p user_data must exactly match registration parameters.
2055 : : *
2056 : : * @param func
2057 : : * Callback function.
2058 : : * @param user_data
2059 : : * User data.
2060 : : *
2061 : : * @return
2062 : : * 0 on success, negative on failure and rte_errno is set.
2063 : : */
2064 : : __rte_internal
2065 : : int
2066 : : rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
2067 : : void *user_data);
2068 : :
2069 : : #ifdef __cplusplus
2070 : : }
2071 : : #endif
2072 : :
2073 : : #endif /* _RTE_MEMPOOL_H_ */
|
