Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright(c) 2016 6WIND S.A.
4 : : * Copyright(c) 2022 SmartShare Systems
5 : : */
6 : :
7 : : #ifndef _RTE_MEMPOOL_H_
8 : : #define _RTE_MEMPOOL_H_
9 : :
10 : : /**
11 : : * @file
12 : : * RTE Mempool.
13 : : *
14 : : * A memory pool is an allocator of fixed-size object. It is
15 : : * identified by its name, and uses a ring to store free objects. It
16 : : * provides some other optional services, like a per-core object
17 : : * cache, and an alignment helper to ensure that objects are padded
18 : : * to spread them equally on all RAM channels, ranks, and so on.
19 : : *
20 : : * Objects owned by a mempool should never be added in another
21 : : * mempool. When an object is freed using rte_mempool_put() or
22 : : * equivalent, the object data is not modified; the user can save some
23 : : * meta-data in the object data and retrieve them when allocating a
24 : : * new object.
25 : : *
26 : : * Note: the mempool implementation is not preemptible. An lcore must not be
27 : : * interrupted by another task that uses the same mempool (because it uses a
28 : : * ring which is not preemptible). Also, usual mempool functions like
29 : : * rte_mempool_get() or rte_mempool_put() are designed to be called from an EAL
30 : : * thread due to the internal per-lcore cache. Due to the lack of caching,
31 : : * rte_mempool_get() or rte_mempool_put() performance will suffer when called
32 : : * by unregistered non-EAL threads. Instead, unregistered non-EAL threads
33 : : * should call rte_mempool_generic_get() or rte_mempool_generic_put() with a
34 : : * user cache created with rte_mempool_cache_create().
35 : : */
36 : :
37 : : #include <stdalign.h>
38 : : #include <stdio.h>
39 : : #include <stdint.h>
40 : : #include <inttypes.h>
41 : :
42 : : #include <rte_compat.h>
43 : : #include <rte_config.h>
44 : : #include <rte_spinlock.h>
45 : : #include <rte_debug.h>
46 : : #include <rte_lcore.h>
47 : : #include <rte_log.h>
48 : : #include <rte_branch_prediction.h>
49 : : #include <rte_ring.h>
50 : : #include <rte_memcpy.h>
51 : : #include <rte_common.h>
52 : :
53 : : #include "rte_mempool_trace_fp.h"
54 : :
55 : : #ifdef __cplusplus
56 : : extern "C" {
57 : : #endif
58 : :
59 : : #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL /**< Header cookie. */
60 : : #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL /**< Header cookie. */
61 : : #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
62 : :
63 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
64 : : /**
65 : : * A structure that stores the mempool statistics (per-lcore).
66 : : * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
67 : : * captured since they can be calculated from other stats.
68 : : * For example: put_cache_objs = put_objs - put_common_pool_objs.
69 : : */
70 : : struct __rte_cache_aligned rte_mempool_debug_stats {
71 : : uint64_t put_bulk; /**< Number of puts. */
72 : : uint64_t put_objs; /**< Number of objects successfully put. */
73 : : uint64_t put_common_pool_bulk; /**< Number of bulks enqueued in common pool. */
74 : : uint64_t put_common_pool_objs; /**< Number of objects enqueued in common pool. */
75 : : uint64_t get_common_pool_bulk; /**< Number of bulks dequeued from common pool. */
76 : : uint64_t get_common_pool_objs; /**< Number of objects dequeued from common pool. */
77 : : uint64_t get_success_bulk; /**< Successful allocation number. */
78 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
79 : : uint64_t get_fail_bulk; /**< Failed allocation number. */
80 : : uint64_t get_fail_objs; /**< Objects that failed to be allocated. */
81 : : uint64_t get_success_blks; /**< Successful allocation number of contiguous blocks. */
82 : : uint64_t get_fail_blks; /**< Failed allocation number of contiguous blocks. */
83 : : RTE_CACHE_GUARD;
84 : : };
85 : : #endif
86 : :
87 : : /**
88 : : * A structure that stores a per-core object cache.
89 : : */
90 : : struct __rte_cache_aligned rte_mempool_cache {
91 : : uint32_t size; /**< Size of the cache */
92 : : uint32_t flushthresh; /**< Threshold before we flush excess elements */
93 : : uint32_t len; /**< Current cache count */
94 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
95 : : uint32_t unused;
96 : : /*
97 : : * Alternative location for the most frequently updated mempool statistics (per-lcore),
98 : : * providing faster update access when using a mempool cache.
99 : : */
100 : : struct {
101 : : uint64_t put_bulk; /**< Number of puts. */
102 : : uint64_t put_objs; /**< Number of objects successfully put. */
103 : : uint64_t get_success_bulk; /**< Successful allocation number. */
104 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
105 : : } stats; /**< Statistics */
106 : : #endif
107 : : /**
108 : : * Cache objects
109 : : *
110 : : * Cache is allocated to this size to allow it to overflow in certain
111 : : * cases to avoid needless emptying of cache.
112 : : */
113 : : alignas(RTE_CACHE_LINE_SIZE) void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2];
114 : : };
115 : :
116 : : /**
117 : : * A structure that stores the size of mempool elements.
118 : : */
119 : : struct rte_mempool_objsz {
120 : : uint32_t elt_size; /**< Size of an element. */
121 : : uint32_t header_size; /**< Size of header (before elt). */
122 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
123 : : uint32_t total_size;
124 : : /**< Total size of an object (header + elt + trailer). */
125 : : };
126 : :
127 : : /**< Maximum length of a memory pool's name. */
128 : : #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
129 : : sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
130 : : #define RTE_MEMPOOL_MZ_PREFIX "MP_"
131 : :
132 : : /* "MP_<name>" */
133 : : #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
134 : :
135 : : #ifndef RTE_MEMPOOL_ALIGN
136 : : /**
137 : : * Alignment of elements inside mempool.
138 : : */
139 : : #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
140 : : #endif
141 : :
142 : : #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
143 : :
144 : : /**
145 : : * Mempool object header structure
146 : : *
147 : : * Each object stored in mempools are prefixed by this header structure,
148 : : * it allows to retrieve the mempool pointer from the object and to
149 : : * iterate on all objects attached to a mempool. When debug is enabled,
150 : : * a cookie is also added in this structure preventing corruptions and
151 : : * double-frees.
152 : : */
153 : : struct rte_mempool_objhdr {
154 : : RTE_STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */
155 : : struct rte_mempool *mp; /**< The mempool owning the object. */
156 : : rte_iova_t iova; /**< IO address of the object. */
157 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
158 : : uint64_t cookie; /**< Debug cookie. */
159 : : #endif
160 : : };
161 : :
162 : : /**
163 : : * A list of object headers type
164 : : */
165 : : RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
166 : :
167 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
168 : :
169 : : /**
170 : : * Mempool object trailer structure
171 : : *
172 : : * In debug mode, each object stored in mempools are suffixed by this
173 : : * trailer structure containing a cookie preventing memory corruptions.
174 : : */
175 : : struct rte_mempool_objtlr {
176 : : uint64_t cookie; /**< Debug cookie. */
177 : : };
178 : :
179 : : #endif
180 : :
181 : : /**
182 : : * @internal Logtype used for mempool related messages.
183 : : */
184 : : extern int rte_mempool_logtype;
185 : : #define RTE_LOGTYPE_MEMPOOL rte_mempool_logtype
186 : : #define RTE_MEMPOOL_LOG(level, ...) \
187 : : RTE_LOG_LINE(level, MEMPOOL, "" __VA_ARGS__)
188 : :
189 : : /**
190 : : * A list of memory where objects are stored
191 : : */
192 : : RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
193 : :
194 : : /**
195 : : * Callback used to free a memory chunk
196 : : */
197 : : typedef void (rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr,
198 : : void *opaque);
199 : :
200 : : /**
201 : : * Mempool objects memory header structure
202 : : *
203 : : * The memory chunks where objects are stored. Each chunk is virtually
204 : : * and physically contiguous.
205 : : */
206 : : struct rte_mempool_memhdr {
207 : : RTE_STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */
208 : : struct rte_mempool *mp; /**< The mempool owning the chunk */
209 : : void *addr; /**< Virtual address of the chunk */
210 : : rte_iova_t iova; /**< IO address of the chunk */
211 : : size_t len; /**< length of the chunk */
212 : : rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */
213 : : void *opaque; /**< Argument passed to the free callback */
214 : : };
215 : :
216 : : /**
217 : : * Additional information about the mempool
218 : : *
219 : : * The structure is cache-line aligned to avoid ABI breakages in
220 : : * a number of cases when something small is added.
221 : : */
222 : : struct __rte_cache_aligned rte_mempool_info {
223 : : /** Number of objects in the contiguous block */
224 : : unsigned int contig_block_size;
225 : : };
226 : :
227 : : /**
228 : : * The RTE mempool structure.
229 : : */
230 : : struct __rte_cache_aligned rte_mempool {
231 : : char name[RTE_MEMPOOL_NAMESIZE]; /**< Name of mempool. */
232 : : union {
233 : : void *pool_data; /**< Ring or pool to store objects. */
234 : : uint64_t pool_id; /**< External mempool identifier. */
235 : : };
236 : : void *pool_config; /**< optional args for ops alloc. */
237 : : const struct rte_memzone *mz; /**< Memzone where pool is alloc'd. */
238 : : unsigned int flags; /**< Flags of the mempool. */
239 : : int socket_id; /**< Socket id passed at create. */
240 : : uint32_t size; /**< Max size of the mempool. */
241 : : uint32_t cache_size;
242 : : /**< Size of per-lcore default local cache. */
243 : :
244 : : uint32_t elt_size; /**< Size of an element. */
245 : : uint32_t header_size; /**< Size of header (before elt). */
246 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
247 : :
248 : : unsigned private_data_size; /**< Size of private data. */
249 : : /**
250 : : * Index into rte_mempool_ops_table array of mempool ops
251 : : * structs, which contain callback function pointers.
252 : : * We're using an index here rather than pointers to the callbacks
253 : : * to facilitate any secondary processes that may want to use
254 : : * this mempool.
255 : : */
256 : : int32_t ops_index;
257 : :
258 : : struct rte_mempool_cache *local_cache; /**< Per-lcore local cache */
259 : :
260 : : uint32_t populated_size; /**< Number of populated objects. */
261 : : struct rte_mempool_objhdr_list elt_list; /**< List of objects in pool */
262 : : uint32_t nb_mem_chunks; /**< Number of memory chunks */
263 : : struct rte_mempool_memhdr_list mem_list; /**< List of memory chunks */
264 : :
265 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
266 : : /** Per-lcore statistics.
267 : : *
268 : : * Plus one, for unregistered non-EAL threads.
269 : : */
270 : : struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
271 : : #endif
272 : : };
273 : :
274 : : /** Spreading among memory channels not required. */
275 : : #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
276 : : /**
277 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_SPREAD.
278 : : * To be deprecated.
279 : : */
280 : : #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
281 : : /** Do not align objects on cache lines. */
282 : : #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
283 : : /**
284 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_CACHE_ALIGN.
285 : : * To be deprecated.
286 : : */
287 : : #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
288 : : /** Default put is "single-producer". */
289 : : #define RTE_MEMPOOL_F_SP_PUT 0x0004
290 : : /**
291 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SP_PUT.
292 : : * To be deprecated.
293 : : */
294 : : #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
295 : : /** Default get is "single-consumer". */
296 : : #define RTE_MEMPOOL_F_SC_GET 0x0008
297 : : /**
298 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SC_GET.
299 : : * To be deprecated.
300 : : */
301 : : #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
302 : : /** Internal: pool is created. */
303 : : #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
304 : : /** Don't need IOVA contiguous objects. */
305 : : #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
306 : : /**
307 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_IOVA_CONTIG.
308 : : * To be deprecated.
309 : : */
310 : : #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
311 : : /** Internal: no object from the pool can be used for device IO (DMA). */
312 : : #define RTE_MEMPOOL_F_NON_IO 0x0040
313 : :
314 : : /**
315 : : * This macro lists all the mempool flags an application may request.
316 : : */
317 : : #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
318 : : | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
319 : : | RTE_MEMPOOL_F_SP_PUT \
320 : : | RTE_MEMPOOL_F_SC_GET \
321 : : | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
322 : : )
323 : :
324 : : /**
325 : : * @internal When stats is enabled, store some statistics.
326 : : *
327 : : * @param mp
328 : : * Pointer to the memory pool.
329 : : * @param name
330 : : * Name of the statistics field to increment in the memory pool.
331 : : * @param n
332 : : * Number to add to the statistics.
333 : : */
334 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
335 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
336 : : unsigned int __lcore_id = rte_lcore_id(); \
337 : : if (likely(__lcore_id < RTE_MAX_LCORE)) \
338 : : (mp)->stats[__lcore_id].name += (n); \
339 : : else \
340 : : rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name), \
341 : : (n), rte_memory_order_relaxed); \
342 : : } while (0)
343 : : #else
344 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
345 : : #endif
346 : :
347 : : /**
348 : : * @internal When stats is enabled, store some statistics.
349 : : *
350 : : * @param cache
351 : : * Pointer to the memory pool cache.
352 : : * @param name
353 : : * Name of the statistics field to increment in the memory pool cache.
354 : : * @param n
355 : : * Number to add to the statistics.
356 : : */
357 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
358 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
359 : : #else
360 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
361 : : #endif
362 : :
363 : : /**
364 : : * @internal Calculate the size of the mempool header.
365 : : *
366 : : * @param mp
367 : : * Pointer to the memory pool.
368 : : * @param cs
369 : : * Size of the per-lcore cache.
370 : : */
371 : : #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
372 : : (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
373 : : (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
374 : :
375 : : /* return the header of a mempool object (internal) */
376 : : static inline struct rte_mempool_objhdr *
377 : : rte_mempool_get_header(void *obj)
378 : : {
379 [ # # ]: 255680 : return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
380 : : sizeof(struct rte_mempool_objhdr));
381 : : }
382 : :
383 : : /**
384 : : * Return a pointer to the mempool owning this object.
385 : : *
386 : : * @param obj
387 : : * An object that is owned by a pool. If this is not the case,
388 : : * the behavior is undefined.
389 : : * @return
390 : : * A pointer to the mempool structure.
391 : : */
392 : : static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
393 : : {
394 : : struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
395 [ - + + - ]: 255680 : return hdr->mp;
396 : : }
397 : :
398 : : /* return the trailer of a mempool object (internal) */
399 : : static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
400 : : {
401 : : struct rte_mempool *mp = rte_mempool_from_obj(obj);
402 : : return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
403 : : }
404 : :
405 : : /**
406 : : * @internal Check and update cookies or panic.
407 : : *
408 : : * @param mp
409 : : * Pointer to the memory pool.
410 : : * @param obj_table_const
411 : : * Pointer to a table of void * pointers (objects).
412 : : * @param n
413 : : * Index of object in object table.
414 : : * @param free
415 : : * - 0: object is supposed to be allocated, mark it as free
416 : : * - 1: object is supposed to be free, mark it as allocated
417 : : * - 2: just check that cookie is valid (free or allocated)
418 : : */
419 : : void rte_mempool_check_cookies(const struct rte_mempool *mp,
420 : : void * const *obj_table_const, unsigned n, int free);
421 : :
422 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
423 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
424 : : rte_mempool_check_cookies(mp, obj_table_const, n, free)
425 : : #else
426 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
427 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
428 : :
429 : : /**
430 : : * @internal Check contiguous object blocks and update cookies or panic.
431 : : *
432 : : * @param mp
433 : : * Pointer to the memory pool.
434 : : * @param first_obj_table_const
435 : : * Pointer to a table of void * pointers (first object of the contiguous
436 : : * object blocks).
437 : : * @param n
438 : : * Number of contiguous object blocks.
439 : : * @param free
440 : : * - 0: object is supposed to be allocated, mark it as free
441 : : * - 1: object is supposed to be free, mark it as allocated
442 : : * - 2: just check that cookie is valid (free or allocated)
443 : : */
444 : : void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
445 : : void * const *first_obj_table_const, unsigned int n, int free);
446 : :
447 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
448 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
449 : : free) \
450 : : rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
451 : : free)
452 : : #else
453 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
454 : : free) \
455 : : do {} while (0)
456 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
457 : :
458 : : #define RTE_MEMPOOL_OPS_NAMESIZE 32 /**< Max length of ops struct name. */
459 : :
460 : : /**
461 : : * Prototype for implementation specific data provisioning function.
462 : : *
463 : : * The function should provide the implementation specific memory for
464 : : * use by the other mempool ops functions in a given mempool ops struct.
465 : : * E.g. the default ops provides an instance of the rte_ring for this purpose.
466 : : * it will most likely point to a different type of data structure, and
467 : : * will be transparent to the application programmer.
468 : : * This function should set mp->pool_data.
469 : : */
470 : : typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
471 : :
472 : : /**
473 : : * Free the opaque private data pointed to by mp->pool_data pointer.
474 : : */
475 : : typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
476 : :
477 : : /**
478 : : * Enqueue 'n' objects into the external pool.
479 : : * @return
480 : : * - 0: Success
481 : : * - <0: Error
482 : : */
483 : : typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
484 : : void * const *obj_table, unsigned int n);
485 : :
486 : : /**
487 : : * Dequeue 'n' objects from the external pool.
488 : : * @return
489 : : * - 0: Success
490 : : * - <0: Error
491 : : */
492 : : typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
493 : : void **obj_table, unsigned int n);
494 : :
495 : : /**
496 : : * Dequeue a number of contiguous object blocks from the external pool.
497 : : */
498 : : typedef int (*rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp,
499 : : void **first_obj_table, unsigned int n);
500 : :
501 : : /**
502 : : * Return the number of available objects in the external pool.
503 : : */
504 : : typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
505 : :
506 : : /**
507 : : * Calculate memory size required to store given number of objects.
508 : : *
509 : : * If mempool objects are not required to be IOVA-contiguous
510 : : * (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is set), min_chunk_size defines
511 : : * virtually contiguous chunk size. Otherwise, if mempool objects must
512 : : * be IOVA-contiguous (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is clear),
513 : : * min_chunk_size defines IOVA-contiguous chunk size.
514 : : *
515 : : * @param[in] mp
516 : : * Pointer to the memory pool.
517 : : * @param[in] obj_num
518 : : * Number of objects.
519 : : * @param[in] pg_shift
520 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
521 : : * @param[out] min_chunk_size
522 : : * Location for minimum size of the memory chunk which may be used to
523 : : * store memory pool objects.
524 : : * @param[out] align
525 : : * Location for required memory chunk alignment.
526 : : * @return
527 : : * Required memory size.
528 : : */
529 : : typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
530 : : uint32_t obj_num, uint32_t pg_shift,
531 : : size_t *min_chunk_size, size_t *align);
532 : :
533 : : /**
534 : : * @internal Helper to calculate memory size required to store given
535 : : * number of objects.
536 : : *
537 : : * This function is internal to mempool library and mempool drivers.
538 : : *
539 : : * If page boundaries may be ignored, it is just a product of total
540 : : * object size including header and trailer and number of objects.
541 : : * Otherwise, it is a number of pages required to store given number of
542 : : * objects without crossing page boundary.
543 : : *
544 : : * Note that if object size is bigger than page size, then it assumes
545 : : * that pages are grouped in subsets of physically continuous pages big
546 : : * enough to store at least one object.
547 : : *
548 : : * Minimum size of memory chunk is the total element size.
549 : : * Required memory chunk alignment is the cache line size.
550 : : *
551 : : * @param[in] mp
552 : : * A pointer to the mempool structure.
553 : : * @param[in] obj_num
554 : : * Number of objects to be added in mempool.
555 : : * @param[in] pg_shift
556 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
557 : : * @param[in] chunk_reserve
558 : : * Amount of memory that must be reserved at the beginning of each page,
559 : : * or at the beginning of the memory area if pg_shift is 0.
560 : : * @param[out] min_chunk_size
561 : : * Location for minimum size of the memory chunk which may be used to
562 : : * store memory pool objects.
563 : : * @param[out] align
564 : : * Location for required memory chunk alignment.
565 : : * @return
566 : : * Required memory size.
567 : : */
568 : : ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
569 : : uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
570 : : size_t *min_chunk_size, size_t *align);
571 : :
572 : : /**
573 : : * Default way to calculate memory size required to store given number of
574 : : * objects.
575 : : *
576 : : * Equivalent to rte_mempool_op_calc_mem_size_helper(mp, obj_num, pg_shift,
577 : : * 0, min_chunk_size, align).
578 : : */
579 : : ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
580 : : uint32_t obj_num, uint32_t pg_shift,
581 : : size_t *min_chunk_size, size_t *align);
582 : :
583 : : /**
584 : : * Function to be called for each populated object.
585 : : *
586 : : * @param[in] mp
587 : : * A pointer to the mempool structure.
588 : : * @param[in] opaque
589 : : * An opaque pointer passed to iterator.
590 : : * @param[in] vaddr
591 : : * Object virtual address.
592 : : * @param[in] iova
593 : : * Input/output virtual address of the object or RTE_BAD_IOVA.
594 : : */
595 : : typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
596 : : void *opaque, void *vaddr, rte_iova_t iova);
597 : :
598 : : /**
599 : : * Populate memory pool objects using provided memory chunk.
600 : : *
601 : : * Populated objects should be enqueued to the pool, e.g. using
602 : : * rte_mempool_ops_enqueue_bulk().
603 : : *
604 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
605 : : * the chunk doesn't need to be physically contiguous (only virtually),
606 : : * and allocated objects may span two pages.
607 : : *
608 : : * @param[in] mp
609 : : * A pointer to the mempool structure.
610 : : * @param[in] max_objs
611 : : * Maximum number of objects to be populated.
612 : : * @param[in] vaddr
613 : : * The virtual address of memory that should be used to store objects.
614 : : * @param[in] iova
615 : : * The IO address
616 : : * @param[in] len
617 : : * The length of memory in bytes.
618 : : * @param[in] obj_cb
619 : : * Callback function to be executed for each populated object.
620 : : * @param[in] obj_cb_arg
621 : : * An opaque pointer passed to the callback function.
622 : : * @return
623 : : * The number of objects added on success.
624 : : * On error, no objects are populated and a negative errno is returned.
625 : : */
626 : : typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
627 : : unsigned int max_objs,
628 : : void *vaddr, rte_iova_t iova, size_t len,
629 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
630 : :
631 : : /**
632 : : * Align objects on addresses multiple of total_elt_sz.
633 : : */
634 : : #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
635 : :
636 : : /**
637 : : * @internal Helper to populate memory pool object using provided memory
638 : : * chunk: just slice objects one by one, taking care of not
639 : : * crossing page boundaries.
640 : : *
641 : : * If RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ is set in flags, the addresses
642 : : * of object headers will be aligned on a multiple of total_elt_sz.
643 : : * This feature is used by octeontx hardware.
644 : : *
645 : : * This function is internal to mempool library and mempool drivers.
646 : : *
647 : : * @param[in] mp
648 : : * A pointer to the mempool structure.
649 : : * @param[in] flags
650 : : * Logical OR of following flags:
651 : : * - RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ: align objects on addresses
652 : : * multiple of total_elt_sz.
653 : : * @param[in] max_objs
654 : : * Maximum number of objects to be added in mempool.
655 : : * @param[in] vaddr
656 : : * The virtual address of memory that should be used to store objects.
657 : : * @param[in] iova
658 : : * The IO address corresponding to vaddr, or RTE_BAD_IOVA.
659 : : * @param[in] len
660 : : * The length of memory in bytes.
661 : : * @param[in] obj_cb
662 : : * Callback function to be executed for each populated object.
663 : : * @param[in] obj_cb_arg
664 : : * An opaque pointer passed to the callback function.
665 : : * @return
666 : : * The number of objects added in mempool.
667 : : */
668 : : int rte_mempool_op_populate_helper(struct rte_mempool *mp,
669 : : unsigned int flags, unsigned int max_objs,
670 : : void *vaddr, rte_iova_t iova, size_t len,
671 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
672 : :
673 : : /**
674 : : * Default way to populate memory pool object using provided memory chunk.
675 : : *
676 : : * Equivalent to rte_mempool_op_populate_helper(mp, 0, max_objs, vaddr, iova,
677 : : * len, obj_cb, obj_cb_arg).
678 : : */
679 : : int rte_mempool_op_populate_default(struct rte_mempool *mp,
680 : : unsigned int max_objs,
681 : : void *vaddr, rte_iova_t iova, size_t len,
682 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
683 : :
684 : : /**
685 : : * Get some additional information about a mempool.
686 : : */
687 : : typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
688 : : struct rte_mempool_info *info);
689 : :
690 : :
691 : : /** Structure defining mempool operations structure */
692 : : struct __rte_cache_aligned rte_mempool_ops {
693 : : char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */
694 : : rte_mempool_alloc_t alloc; /**< Allocate private data. */
695 : : rte_mempool_free_t free; /**< Free the external pool. */
696 : : rte_mempool_enqueue_t enqueue; /**< Enqueue an object. */
697 : : rte_mempool_dequeue_t dequeue; /**< Dequeue an object. */
698 : : rte_mempool_get_count get_count; /**< Get qty of available objs. */
699 : : /**
700 : : * Optional callback to calculate memory size required to
701 : : * store specified number of objects.
702 : : */
703 : : rte_mempool_calc_mem_size_t calc_mem_size;
704 : : /**
705 : : * Optional callback to populate mempool objects using
706 : : * provided memory chunk.
707 : : */
708 : : rte_mempool_populate_t populate;
709 : : /**
710 : : * Get mempool info
711 : : */
712 : : rte_mempool_get_info_t get_info;
713 : : /**
714 : : * Dequeue a number of contiguous object blocks.
715 : : */
716 : : rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks;
717 : : };
718 : :
719 : : #define RTE_MEMPOOL_MAX_OPS_IDX 16 /**< Max registered ops structs */
720 : :
721 : : /**
722 : : * Structure storing the table of registered ops structs, each of which contain
723 : : * the function pointers for the mempool ops functions.
724 : : * Each process has its own storage for this ops struct array so that
725 : : * the mempools can be shared across primary and secondary processes.
726 : : * The indices used to access the array are valid across processes, whereas
727 : : * any function pointers stored directly in the mempool struct would not be.
728 : : * This results in us simply having "ops_index" in the mempool struct.
729 : : */
730 : : struct __rte_cache_aligned rte_mempool_ops_table {
731 : : rte_spinlock_t sl; /**< Spinlock for add/delete. */
732 : : uint32_t num_ops; /**< Number of used ops structs in the table. */
733 : : /**
734 : : * Storage for all possible ops structs.
735 : : */
736 : : struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX];
737 : : };
738 : :
739 : : /** Array of registered ops structs. */
740 : : extern struct rte_mempool_ops_table rte_mempool_ops_table;
741 : :
742 : : /**
743 : : * @internal Get the mempool ops struct from its index.
744 : : *
745 : : * @param ops_index
746 : : * The index of the ops struct in the ops struct table. It must be a valid
747 : : * index: (0 <= idx < num_ops).
748 : : * @return
749 : : * The pointer to the ops struct in the table.
750 : : */
751 : : static inline struct rte_mempool_ops *
752 : : rte_mempool_get_ops(int ops_index)
753 : : {
754 [ - + - + : 2214 : RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
- + - + -
+ - + ]
755 : :
756 : : return &rte_mempool_ops_table.ops[ops_index];
757 : : }
758 : :
759 : : /**
760 : : * @internal Wrapper for mempool_ops alloc callback.
761 : : *
762 : : * @param mp
763 : : * Pointer to the memory pool.
764 : : * @return
765 : : * - 0: Success; successfully allocated mempool pool_data.
766 : : * - <0: Error; code of alloc function.
767 : : */
768 : : int
769 : : rte_mempool_ops_alloc(struct rte_mempool *mp);
770 : :
771 : : /**
772 : : * @internal Wrapper for mempool_ops dequeue callback.
773 : : *
774 : : * @param mp
775 : : * Pointer to the memory pool.
776 : : * @param obj_table
777 : : * Pointer to a table of void * pointers (objects).
778 : : * @param n
779 : : * Number of objects to get.
780 : : * @return
781 : : * - 0: Success; got n objects.
782 : : * - <0: Error; code of dequeue function.
783 : : */
784 : : static inline int
785 : 44969306 : rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
786 : : void **obj_table, unsigned n)
787 : : {
788 : : struct rte_mempool_ops *ops;
789 : : int ret;
790 : :
791 : : rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
792 [ - + ]: 44969306 : ops = rte_mempool_get_ops(mp->ops_index);
793 : 44969306 : ret = ops->dequeue(mp, obj_table, n);
794 : : if (ret == 0) {
795 : : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
796 : : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
797 : : }
798 : 44969306 : return ret;
799 : : }
800 : :
801 : : /**
802 : : * @internal Wrapper for mempool_ops dequeue_contig_blocks callback.
803 : : *
804 : : * @param[in] mp
805 : : * Pointer to the memory pool.
806 : : * @param[out] first_obj_table
807 : : * Pointer to a table of void * pointers (first objects).
808 : : * @param[in] n
809 : : * Number of blocks to get.
810 : : * @return
811 : : * - 0: Success; got n objects.
812 : : * - <0: Error; code of dequeue function.
813 : : */
814 : : static inline int
815 : 0 : rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
816 : : void **first_obj_table, unsigned int n)
817 : : {
818 : : struct rte_mempool_ops *ops;
819 : :
820 [ # # ]: 0 : ops = rte_mempool_get_ops(mp->ops_index);
821 : : RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
822 : : rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
823 : 0 : return ops->dequeue_contig_blocks(mp, first_obj_table, n);
824 : : }
825 : :
826 : : /**
827 : : * @internal wrapper for mempool_ops enqueue callback.
828 : : *
829 : : * @param mp
830 : : * Pointer to the memory pool.
831 : : * @param obj_table
832 : : * Pointer to a table of void * pointers (objects).
833 : : * @param n
834 : : * Number of objects to put.
835 : : * @return
836 : : * - 0: Success; n objects supplied.
837 : : * - <0: Error; code of enqueue function.
838 : : */
839 : : static inline int
840 : 44978299 : rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
841 : : unsigned n)
842 : : {
843 : : struct rte_mempool_ops *ops;
844 : : int ret;
845 : :
846 : : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
847 : : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
848 : : rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
849 [ - + ]: 44978299 : ops = rte_mempool_get_ops(mp->ops_index);
850 : 44978299 : ret = ops->enqueue(mp, obj_table, n);
851 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
852 : : if (unlikely(ret < 0))
853 : : RTE_MEMPOOL_LOG(CRIT, "cannot enqueue %u objects to mempool %s",
854 : : n, mp->name);
855 : : #endif
856 : 44978299 : return ret;
857 : : }
858 : :
859 : : /**
860 : : * @internal wrapper for mempool_ops get_count callback.
861 : : *
862 : : * @param mp
863 : : * Pointer to the memory pool.
864 : : * @return
865 : : * The number of available objects in the external pool.
866 : : */
867 : : unsigned
868 : : rte_mempool_ops_get_count(const struct rte_mempool *mp);
869 : :
870 : : /**
871 : : * @internal wrapper for mempool_ops calc_mem_size callback.
872 : : * API to calculate size of memory required to store specified number of
873 : : * object.
874 : : *
875 : : * @param[in] mp
876 : : * Pointer to the memory pool.
877 : : * @param[in] obj_num
878 : : * Number of objects.
879 : : * @param[in] pg_shift
880 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
881 : : * @param[out] min_chunk_size
882 : : * Location for minimum size of the memory chunk which may be used to
883 : : * store memory pool objects.
884 : : * @param[out] align
885 : : * Location for required memory chunk alignment.
886 : : * @return
887 : : * Required memory size aligned at page boundary.
888 : : */
889 : : ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
890 : : uint32_t obj_num, uint32_t pg_shift,
891 : : size_t *min_chunk_size, size_t *align);
892 : :
893 : : /**
894 : : * @internal wrapper for mempool_ops populate callback.
895 : : *
896 : : * Populate memory pool objects using provided memory chunk.
897 : : *
898 : : * @param[in] mp
899 : : * A pointer to the mempool structure.
900 : : * @param[in] max_objs
901 : : * Maximum number of objects to be populated.
902 : : * @param[in] vaddr
903 : : * The virtual address of memory that should be used to store objects.
904 : : * @param[in] iova
905 : : * The IO address
906 : : * @param[in] len
907 : : * The length of memory in bytes.
908 : : * @param[in] obj_cb
909 : : * Callback function to be executed for each populated object.
910 : : * @param[in] obj_cb_arg
911 : : * An opaque pointer passed to the callback function.
912 : : * @return
913 : : * The number of objects added on success.
914 : : * On error, no objects are populated and a negative errno is returned.
915 : : */
916 : : int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
917 : : void *vaddr, rte_iova_t iova, size_t len,
918 : : rte_mempool_populate_obj_cb_t *obj_cb,
919 : : void *obj_cb_arg);
920 : :
921 : : /**
922 : : * Wrapper for mempool_ops get_info callback.
923 : : *
924 : : * @param[in] mp
925 : : * Pointer to the memory pool.
926 : : * @param[out] info
927 : : * Pointer to the rte_mempool_info structure
928 : : * @return
929 : : * - 0: Success; The mempool driver supports retrieving supplementary
930 : : * mempool information
931 : : * - -ENOTSUP - doesn't support get_info ops (valid case).
932 : : */
933 : : int rte_mempool_ops_get_info(const struct rte_mempool *mp,
934 : : struct rte_mempool_info *info);
935 : :
936 : : /**
937 : : * @internal wrapper for mempool_ops free callback.
938 : : *
939 : : * @param mp
940 : : * Pointer to the memory pool.
941 : : */
942 : : void
943 : : rte_mempool_ops_free(struct rte_mempool *mp);
944 : :
945 : : /**
946 : : * Set the ops of a mempool.
947 : : *
948 : : * This can only be done on a mempool that is not populated, i.e. just after
949 : : * a call to rte_mempool_create_empty().
950 : : *
951 : : * @param mp
952 : : * Pointer to the memory pool.
953 : : * @param name
954 : : * Name of the ops structure to use for this mempool.
955 : : * @param pool_config
956 : : * Opaque data that can be passed by the application to the ops functions.
957 : : * @return
958 : : * - 0: Success; the mempool is now using the requested ops functions.
959 : : * - -EINVAL - Invalid ops struct name provided.
960 : : * - -EEXIST - mempool already has an ops struct assigned.
961 : : */
962 : : int
963 : : rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
964 : : void *pool_config);
965 : :
966 : : /**
967 : : * Register mempool operations.
968 : : *
969 : : * @param ops
970 : : * Pointer to an ops structure to register.
971 : : * @return
972 : : * - >=0: Success; return the index of the ops struct in the table.
973 : : * - -EINVAL - some missing callbacks while registering ops struct.
974 : : * - -ENOSPC - the maximum number of ops structs has been reached.
975 : : */
976 : : int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
977 : :
978 : : /**
979 : : * Macro to statically register the ops of a mempool handler.
980 : : * Note that the rte_mempool_register_ops fails silently here when
981 : : * more than RTE_MEMPOOL_MAX_OPS_IDX is registered.
982 : : */
983 : : #define RTE_MEMPOOL_REGISTER_OPS(ops) \
984 : : RTE_INIT(mp_hdlr_init_##ops) \
985 : : { \
986 : : rte_mempool_register_ops(&ops); \
987 : : }
988 : :
989 : : /**
990 : : * An object callback function for mempool.
991 : : *
992 : : * Used by rte_mempool_create() and rte_mempool_obj_iter().
993 : : */
994 : : typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
995 : : void *opaque, void *obj, unsigned obj_idx);
996 : : typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
997 : :
998 : : /**
999 : : * A memory callback function for mempool.
1000 : : *
1001 : : * Used by rte_mempool_mem_iter().
1002 : : */
1003 : : typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1004 : : void *opaque, struct rte_mempool_memhdr *memhdr,
1005 : : unsigned mem_idx);
1006 : :
1007 : : /**
1008 : : * A mempool constructor callback function.
1009 : : *
1010 : : * Arguments are the mempool and the opaque pointer given by the user in
1011 : : * rte_mempool_create().
1012 : : */
1013 : : typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
1014 : :
1015 : : /**
1016 : : * Create a new mempool named *name* in memory.
1017 : : *
1018 : : * This function uses ``rte_memzone_reserve()`` to allocate memory. The
1019 : : * pool contains n elements of elt_size. Its size is set to n.
1020 : : *
1021 : : * @param name
1022 : : * The name of the mempool.
1023 : : * @param n
1024 : : * The number of elements in the mempool. The optimum size (in terms of
1025 : : * memory usage) for a mempool is when n is a power of two minus one:
1026 : : * n = (2^q - 1).
1027 : : * @param elt_size
1028 : : * The size of each element.
1029 : : * @param cache_size
1030 : : * If cache_size is non-zero, the rte_mempool library will try to
1031 : : * limit the accesses to the common lockless pool, by maintaining a
1032 : : * per-lcore object cache. This argument must be lower or equal to
1033 : : * RTE_MEMPOOL_CACHE_MAX_SIZE and n / 1.5. It is advised to choose
1034 : : * cache_size to have "n modulo cache_size == 0": if this is
1035 : : * not the case, some elements will always stay in the pool and will
1036 : : * never be used. The access to the per-lcore table is of course
1037 : : * faster than the multi-producer/consumer pool. The cache can be
1038 : : * disabled if the cache_size argument is set to 0; it can be useful to
1039 : : * avoid losing objects in cache.
1040 : : * @param private_data_size
1041 : : * The size of the private data appended after the mempool
1042 : : * structure. This is useful for storing some private data after the
1043 : : * mempool structure, as is done for rte_mbuf_pool for example.
1044 : : * @param mp_init
1045 : : * A function pointer that is called for initialization of the pool,
1046 : : * before object initialization. The user can initialize the private
1047 : : * data in this function if needed. This parameter can be NULL if
1048 : : * not needed.
1049 : : * @param mp_init_arg
1050 : : * An opaque pointer to data that can be used in the mempool
1051 : : * constructor function.
1052 : : * @param obj_init
1053 : : * A function pointer that is called for each object at
1054 : : * initialization of the pool. The user can set some meta data in
1055 : : * objects if needed. This parameter can be NULL if not needed.
1056 : : * The obj_init() function takes the mempool pointer, the init_arg,
1057 : : * the object pointer and the object number as parameters.
1058 : : * @param obj_init_arg
1059 : : * An opaque pointer to data that can be used as an argument for
1060 : : * each call to the object constructor function.
1061 : : * @param socket_id
1062 : : * The *socket_id* argument is the socket identifier in the case of
1063 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1064 : : * constraint for the reserved zone.
1065 : : * @param flags
1066 : : * The *flags* arguments is an OR of following flags:
1067 : : * - RTE_MEMPOOL_F_NO_SPREAD: By default, objects addresses are spread
1068 : : * between channels in RAM: the pool allocator will add padding
1069 : : * between objects depending on the hardware configuration. See
1070 : : * Memory alignment constraints for details. If this flag is set,
1071 : : * the allocator will just align them to a cache line.
1072 : : * - RTE_MEMPOOL_F_NO_CACHE_ALIGN: By default, the returned objects are
1073 : : * cache-aligned. This flag removes this constraint, and no
1074 : : * padding will be present between objects. This flag implies
1075 : : * RTE_MEMPOOL_F_NO_SPREAD.
1076 : : * - RTE_MEMPOOL_F_SP_PUT: If this flag is set, the default behavior
1077 : : * when using rte_mempool_put() or rte_mempool_put_bulk() is
1078 : : * "single-producer". Otherwise, it is "multi-producers".
1079 : : * - RTE_MEMPOOL_F_SC_GET: If this flag is set, the default behavior
1080 : : * when using rte_mempool_get() or rte_mempool_get_bulk() is
1081 : : * "single-consumer". Otherwise, it is "multi-consumers".
1082 : : * - RTE_MEMPOOL_F_NO_IOVA_CONTIG: If set, allocated objects won't
1083 : : * necessarily be contiguous in IO memory.
1084 : : * @return
1085 : : * The pointer to the new allocated mempool, on success. NULL on error
1086 : : * with rte_errno set appropriately. Possible rte_errno values include:
1087 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
1088 : : * - EINVAL - cache size provided is too large or an unknown flag was passed
1089 : : * - ENOSPC - the maximum number of memzones has already been allocated
1090 : : * - EEXIST - a memzone with the same name already exists
1091 : : * - ENOMEM - no appropriate memory area found in which to create memzone
1092 : : */
1093 : : struct rte_mempool *
1094 : : rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1095 : : unsigned cache_size, unsigned private_data_size,
1096 : : rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1097 : : rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1098 : : int socket_id, unsigned flags);
1099 : :
1100 : : /**
1101 : : * Create an empty mempool
1102 : : *
1103 : : * The mempool is allocated and initialized, but it is not populated: no
1104 : : * memory is allocated for the mempool elements. The user has to call
1105 : : * rte_mempool_populate_*() to add memory chunks to the pool. Once
1106 : : * populated, the user may also want to initialize each object with
1107 : : * rte_mempool_obj_iter().
1108 : : *
1109 : : * @param name
1110 : : * The name of the mempool.
1111 : : * @param n
1112 : : * The maximum number of elements that can be added in the mempool.
1113 : : * The optimum size (in terms of memory usage) for a mempool is when n
1114 : : * is a power of two minus one: n = (2^q - 1).
1115 : : * @param elt_size
1116 : : * The size of each element.
1117 : : * @param cache_size
1118 : : * Size of the cache. See rte_mempool_create() for details.
1119 : : * @param private_data_size
1120 : : * The size of the private data appended after the mempool
1121 : : * structure. This is useful for storing some private data after the
1122 : : * mempool structure, as is done for rte_mbuf_pool for example.
1123 : : * @param socket_id
1124 : : * The *socket_id* argument is the socket identifier in the case of
1125 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1126 : : * constraint for the reserved zone.
1127 : : * @param flags
1128 : : * Flags controlling the behavior of the mempool. See
1129 : : * rte_mempool_create() for details.
1130 : : * @return
1131 : : * The pointer to the new allocated mempool, on success. NULL on error
1132 : : * with rte_errno set appropriately. See rte_mempool_create() for details.
1133 : : */
1134 : : struct rte_mempool *
1135 : : rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1136 : : unsigned cache_size, unsigned private_data_size,
1137 : : int socket_id, unsigned flags);
1138 : : /**
1139 : : * Free a mempool
1140 : : *
1141 : : * Unlink the mempool from global list, free the memory chunks, and all
1142 : : * memory referenced by the mempool. The objects must not be used by
1143 : : * other cores as they will be freed.
1144 : : *
1145 : : * @param mp
1146 : : * A pointer to the mempool structure.
1147 : : * If NULL then, the function does nothing.
1148 : : */
1149 : : void
1150 : : rte_mempool_free(struct rte_mempool *mp);
1151 : :
1152 : : /**
1153 : : * Add physically contiguous memory for objects in the pool at init
1154 : : *
1155 : : * Add a virtually and physically contiguous memory chunk in the pool
1156 : : * where objects can be instantiated.
1157 : : *
1158 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
1159 : : * the chunk doesn't need to be physically contiguous (only virtually),
1160 : : * and allocated objects may span two pages.
1161 : : *
1162 : : * @param mp
1163 : : * A pointer to the mempool structure.
1164 : : * @param vaddr
1165 : : * The virtual address of memory that should be used to store objects.
1166 : : * @param iova
1167 : : * The IO address
1168 : : * @param len
1169 : : * The length of memory in bytes.
1170 : : * @param free_cb
1171 : : * The callback used to free this chunk when destroying the mempool.
1172 : : * @param opaque
1173 : : * An opaque argument passed to free_cb.
1174 : : * @return
1175 : : * The number of objects added on success (strictly positive).
1176 : : * On error, the chunk is not added in the memory list of the
1177 : : * mempool the following code is returned:
1178 : : * (0): not enough room in chunk for one object.
1179 : : * (-ENOSPC): mempool is already populated.
1180 : : * (-ENOMEM): allocation failure.
1181 : : */
1182 : : int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1183 : : rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1184 : : void *opaque);
1185 : :
1186 : : /**
1187 : : * Add virtually contiguous memory for objects in the pool at init
1188 : : *
1189 : : * Add a virtually contiguous memory chunk in the pool where objects can
1190 : : * be instantiated.
1191 : : *
1192 : : * @param mp
1193 : : * A pointer to the mempool structure.
1194 : : * @param addr
1195 : : * The virtual address of memory that should be used to store objects.
1196 : : * @param len
1197 : : * The length of memory in bytes.
1198 : : * @param pg_sz
1199 : : * The size of memory pages in this virtual area.
1200 : : * @param free_cb
1201 : : * The callback used to free this chunk when destroying the mempool.
1202 : : * @param opaque
1203 : : * An opaque argument passed to free_cb.
1204 : : * @return
1205 : : * The number of objects added on success (strictly positive).
1206 : : * On error, the chunk is not added in the memory list of the
1207 : : * mempool the following code is returned:
1208 : : * (0): not enough room in chunk for one object.
1209 : : * (-ENOSPC): mempool is already populated.
1210 : : * (-ENOMEM): allocation failure.
1211 : : */
1212 : : int
1213 : : rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1214 : : size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1215 : : void *opaque);
1216 : :
1217 : : /**
1218 : : * Add memory for objects in the pool at init
1219 : : *
1220 : : * This is the default function used by rte_mempool_create() to populate
1221 : : * the mempool. It adds memory allocated using rte_memzone_reserve().
1222 : : *
1223 : : * @param mp
1224 : : * A pointer to the mempool structure.
1225 : : * @return
1226 : : * The number of objects added on success.
1227 : : * On error, the chunk is not added in the memory list of the
1228 : : * mempool and a negative errno is returned.
1229 : : */
1230 : : int rte_mempool_populate_default(struct rte_mempool *mp);
1231 : :
1232 : : /**
1233 : : * Add memory from anonymous mapping for objects in the pool at init
1234 : : *
1235 : : * This function mmap an anonymous memory zone that is locked in
1236 : : * memory to store the objects of the mempool.
1237 : : *
1238 : : * @param mp
1239 : : * A pointer to the mempool structure.
1240 : : * @return
1241 : : * The number of objects added on success.
1242 : : * On error, 0 is returned, rte_errno is set, and the chunk is not added in
1243 : : * the memory list of the mempool.
1244 : : */
1245 : : int rte_mempool_populate_anon(struct rte_mempool *mp);
1246 : :
1247 : : /**
1248 : : * Call a function for each mempool element
1249 : : *
1250 : : * Iterate across all objects attached to a rte_mempool and call the
1251 : : * callback function on it.
1252 : : *
1253 : : * @param mp
1254 : : * A pointer to an initialized mempool.
1255 : : * @param obj_cb
1256 : : * A function pointer that is called for each object.
1257 : : * @param obj_cb_arg
1258 : : * An opaque pointer passed to the callback function.
1259 : : * @return
1260 : : * Number of objects iterated.
1261 : : */
1262 : : uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1263 : : rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1264 : :
1265 : : /**
1266 : : * Call a function for each mempool memory chunk
1267 : : *
1268 : : * Iterate across all memory chunks attached to a rte_mempool and call
1269 : : * the callback function on it.
1270 : : *
1271 : : * @param mp
1272 : : * A pointer to an initialized mempool.
1273 : : * @param mem_cb
1274 : : * A function pointer that is called for each memory chunk.
1275 : : * @param mem_cb_arg
1276 : : * An opaque pointer passed to the callback function.
1277 : : * @return
1278 : : * Number of memory chunks iterated.
1279 : : */
1280 : : uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1281 : : rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1282 : :
1283 : : /**
1284 : : * Dump the status of the mempool to a file.
1285 : : *
1286 : : * @param f
1287 : : * A pointer to a file for output
1288 : : * @param mp
1289 : : * A pointer to the mempool structure.
1290 : : */
1291 : : void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1292 : :
1293 : : /**
1294 : : * Create a user-owned mempool cache.
1295 : : *
1296 : : * This can be used by unregistered non-EAL threads to enable caching when they
1297 : : * interact with a mempool.
1298 : : *
1299 : : * @param size
1300 : : * The size of the mempool cache. See rte_mempool_create()'s cache_size
1301 : : * parameter description for more information. The same limits and
1302 : : * considerations apply here too.
1303 : : * @param socket_id
1304 : : * The socket identifier in the case of NUMA. The value can be
1305 : : * SOCKET_ID_ANY if there is no NUMA constraint for the reserved zone.
1306 : : */
1307 : : struct rte_mempool_cache *
1308 : : rte_mempool_cache_create(uint32_t size, int socket_id);
1309 : :
1310 : : /**
1311 : : * Free a user-owned mempool cache.
1312 : : *
1313 : : * @param cache
1314 : : * A pointer to the mempool cache.
1315 : : */
1316 : : void
1317 : : rte_mempool_cache_free(struct rte_mempool_cache *cache);
1318 : :
1319 : : /**
1320 : : * Get a pointer to the per-lcore default mempool cache.
1321 : : *
1322 : : * @param mp
1323 : : * A pointer to the mempool structure.
1324 : : * @param lcore_id
1325 : : * The logical core id.
1326 : : * @return
1327 : : * A pointer to the mempool cache or NULL if disabled or unregistered non-EAL
1328 : : * thread.
1329 : : */
1330 : : static __rte_always_inline struct rte_mempool_cache *
1331 : : rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1332 : : {
1333 [ + + + + : 2708081 : if (mp->cache_size == 0)
+ + + + +
+ + + + +
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1334 : : return NULL;
1335 : :
1336 [ + + + - : 2173699 : if (lcore_id >= RTE_MAX_LCORE)
+ + + - +
- + - + -
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1337 : : return NULL;
1338 : :
1339 : : rte_mempool_trace_default_cache(mp, lcore_id,
1340 [ # # # # : 2173687 : &mp->local_cache[lcore_id]);
# # # # ]
1341 : 2173688 : return &mp->local_cache[lcore_id];
1342 : : }
1343 : :
1344 : : /**
1345 : : * Flush a user-owned mempool cache to the specified mempool.
1346 : : *
1347 : : * @param cache
1348 : : * A pointer to the mempool cache.
1349 : : * @param mp
1350 : : * A pointer to the mempool.
1351 : : */
1352 : : static __rte_always_inline void
1353 : : rte_mempool_cache_flush(struct rte_mempool_cache *cache,
1354 : : struct rte_mempool *mp)
1355 : : {
1356 [ - + ]: 3 : if (cache == NULL)
1357 : : cache = rte_mempool_default_cache(mp, rte_lcore_id());
1358 [ + - + - ]: 3 : if (cache == NULL || cache->len == 0)
1359 : : return;
1360 : : rte_mempool_trace_cache_flush(cache, mp);
1361 : 3 : rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1362 : 3 : cache->len = 0;
1363 : : }
1364 : :
1365 : : /**
1366 : : * @internal Put several objects back in the mempool; used internally.
1367 : : * @param mp
1368 : : * A pointer to the mempool structure.
1369 : : * @param obj_table
1370 : : * A pointer to a table of void * pointers (objects).
1371 : : * @param n
1372 : : * The number of objects to store back in the mempool, must be strictly
1373 : : * positive.
1374 : : * @param cache
1375 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1376 : : */
1377 : : static __rte_always_inline void
1378 : : rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1379 : : unsigned int n, struct rte_mempool_cache *cache)
1380 : : {
1381 : : void **cache_objs;
1382 : :
1383 : : /* No cache provided */
1384 [ + + + + : 2371278 : if (unlikely(cache == NULL))
+ + - + +
+ + + + +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1385 : 264120 : goto driver_enqueue;
1386 : :
1387 : : /* increment stat now, adding in mempool always success */
1388 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1389 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1390 : :
1391 : : /* The request itself is too big for the cache */
1392 [ - + - + : 2107158 : if (unlikely(n > cache->flushthresh))
- + - + -
+ - + - +
- - - - -
+ - - - -
+ - - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1393 : 2 : goto driver_enqueue_stats_incremented;
1394 : :
1395 : : /*
1396 : : * The cache follows the following algorithm:
1397 : : * 1. If the objects cannot be added to the cache without crossing
1398 : : * the flush threshold, flush the cache to the backend.
1399 : : * 2. Add the objects to the cache.
1400 : : */
1401 : :
1402 [ + + + + : 2107156 : if (cache->len + n <= cache->flushthresh) {
+ - + - +
- + - + +
- - - - +
+ - - - -
- - + + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1403 : 2063423 : cache_objs = &cache->objs[cache->len];
1404 : 2063423 : cache->len += n;
1405 : : } else {
1406 : 43733 : cache_objs = &cache->objs[0];
1407 : 43733 : rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
1408 : 43733 : cache->len = n;
1409 : : }
1410 : :
1411 : : /* Add the objects to the cache. */
1412 [ + + # # : 4 : rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
# # # # #
# # # # #
# # ]
1413 : :
1414 : : return;
1415 : :
1416 : : driver_enqueue:
1417 : :
1418 : : /* increment stat now, adding in mempool always success */
1419 : : RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1420 : : RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1421 : :
1422 : 264122 : driver_enqueue_stats_incremented:
1423 : :
1424 : : /* push objects to the backend */
1425 : 264122 : rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1426 : : }
1427 : :
1428 : :
1429 : : /**
1430 : : * Put several objects back in the mempool.
1431 : : *
1432 : : * @param mp
1433 : : * A pointer to the mempool structure.
1434 : : * @param obj_table
1435 : : * A pointer to a table of void * pointers (objects).
1436 : : * @param n
1437 : : * The number of objects to add in the mempool from the obj_table.
1438 : : * @param cache
1439 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1440 : : */
1441 : : static __rte_always_inline void
1442 : : rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1443 : : unsigned int n, struct rte_mempool_cache *cache)
1444 : : {
1445 : : rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1446 : : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1447 : : rte_mempool_do_generic_put(mp, obj_table, n, cache);
1448 : : }
1449 : :
1450 : : /**
1451 : : * Put several objects back in the mempool.
1452 : : *
1453 : : * This function calls the multi-producer or the single-producer
1454 : : * version depending on the default behavior that was specified at
1455 : : * mempool creation time (see flags).
1456 : : *
1457 : : * @param mp
1458 : : * A pointer to the mempool structure.
1459 : : * @param obj_table
1460 : : * A pointer to a table of void * pointers (objects).
1461 : : * @param n
1462 : : * The number of objects to add in the mempool from obj_table.
1463 : : */
1464 : : static __rte_always_inline void
1465 : : rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1466 : : unsigned int n)
1467 : : {
1468 : : struct rte_mempool_cache *cache;
1469 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1470 : : rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1471 : : rte_mempool_generic_put(mp, obj_table, n, cache);
1472 : : }
1473 : :
1474 : : /**
1475 : : * Put one object back in the mempool.
1476 : : *
1477 : : * This function calls the multi-producer or the single-producer
1478 : : * version depending on the default behavior that was specified at
1479 : : * mempool creation time (see flags).
1480 : : *
1481 : : * @param mp
1482 : : * A pointer to the mempool structure.
1483 : : * @param obj
1484 : : * A pointer to the object to be added.
1485 : : */
1486 : : static __rte_always_inline void
1487 : : rte_mempool_put(struct rte_mempool *mp, void *obj)
1488 : : {
1489 : : rte_mempool_put_bulk(mp, &obj, 1);
1490 : : }
1491 : :
1492 : : /**
1493 : : * @internal Get several objects from the mempool; used internally.
1494 : : * @param mp
1495 : : * A pointer to the mempool structure.
1496 : : * @param obj_table
1497 : : * A pointer to a table of void * pointers (objects).
1498 : : * @param n
1499 : : * The number of objects to get, must be strictly positive.
1500 : : * @param cache
1501 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1502 : : * @return
1503 : : * - 0: Success.
1504 : : * - <0: Error; code of driver dequeue function.
1505 : : */
1506 : : static __rte_always_inline int
1507 : : rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1508 : : unsigned int n, struct rte_mempool_cache *cache)
1509 : : {
1510 : : int ret;
1511 : : unsigned int remaining;
1512 : : uint32_t index, len;
1513 : : void **cache_objs;
1514 : :
1515 : : /* No cache provided */
1516 [ + + + + : 347381 : if (unlikely(cache == NULL)) {
+ - + + +
+ + + + +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1517 : : remaining = n;
1518 : 272388 : goto driver_dequeue;
1519 : : }
1520 : :
1521 : : /* The cache is a stack, so copy will be in reverse order. */
1522 : 74993 : cache_objs = &cache->objs[cache->len];
1523 : :
1524 [ + + + + : 74993 : if (__rte_constant(n) && n <= cache->len) {
+ + - + +
- + - + +
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1525 : : /*
1526 : : * The request size is known at build time, and
1527 : : * the entire request can be satisfied from the cache,
1528 : : * so let the compiler unroll the fixed length copy loop.
1529 : : */
1530 : 41727 : cache->len -= n;
1531 [ + + + + : 1099293 : for (index = 0; index < n; index++)
- - - - +
+ + + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1532 : 1057566 : *obj_table++ = *--cache_objs;
1533 : :
1534 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1535 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1536 : :
1537 : : return 0;
1538 : : }
1539 : :
1540 : : /*
1541 : : * Use the cache as much as we have to return hot objects first.
1542 : : * If the request size 'n' is known at build time, the above comparison
1543 : : * ensures that n > cache->len here, so omit RTE_MIN().
1544 : : */
1545 [ + + # # : 31 : len = __rte_constant(n) ? cache->len : RTE_MIN(n, cache->len);
# # # # ]
1546 : 33266 : cache->len -= len;
1547 : 32802 : remaining = n - len;
1548 [ + + - - : 33350 : for (index = 0; index < len; index++)
- - - - -
- + + - +
+ + # # #
# # # #
# ]
1549 : 548 : *obj_table++ = *--cache_objs;
1550 : :
1551 : : /*
1552 : : * If the request size 'n' is known at build time, the case
1553 : : * where the entire request can be satisfied from the cache
1554 : : * has already been handled above, so omit handling it here.
1555 : : */
1556 [ + + + + : 31 : if (!__rte_constant(n) && remaining == 0) {
# # # # #
# # # # #
# # ]
1557 : : /* The entire request is satisfied from the cache. */
1558 : :
1559 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1560 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1561 : :
1562 : : return 0;
1563 : : }
1564 : :
1565 : : /* if dequeue below would overflow mem allocated for cache */
1566 [ + + + - : 29 : if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
# # # # ]
1567 : 8 : goto driver_dequeue;
1568 : :
1569 : : /* Fill the cache from the backend; fetch size + remaining objects. */
1570 : 33254 : ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1571 : 33254 : cache->size + remaining);
1572 [ + + + + : 33254 : if (unlikely(ret < 0)) {
- - - + -
- - + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1573 : : /*
1574 : : * We are buffer constrained, and not able to allocate
1575 : : * cache + remaining.
1576 : : * Do not fill the cache, just satisfy the remaining part of
1577 : : * the request directly from the backend.
1578 : : */
1579 : 424 : goto driver_dequeue;
1580 : : }
1581 : :
1582 : : /* Satisfy the remaining part of the request from the filled cache. */
1583 : 32830 : cache_objs = &cache->objs[cache->size + remaining];
1584 [ + + + + : 1081821 : for (index = 0; index < remaining; index++)
- - + + -
- + + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1585 : 1048991 : *obj_table++ = *--cache_objs;
1586 : :
1587 : 32816 : cache->len = cache->size;
1588 : :
1589 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1590 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1591 : :
1592 : 16 : return 0;
1593 : :
1594 : 272820 : driver_dequeue:
1595 : :
1596 : : /* Get remaining objects directly from the backend. */
1597 : 272820 : ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1598 : :
1599 [ + + + + : 272820 : if (ret < 0) {
- + - + -
+ - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1600 [ + + + + : 18 : if (likely(cache != NULL)) {
- - - - -
- - - - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1601 : 16 : cache->len = n - remaining;
1602 : : /*
1603 : : * No further action is required to roll the first part
1604 : : * of the request back into the cache, as objects in
1605 : : * the cache are intact.
1606 : : */
1607 : : }
1608 : :
1609 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1610 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1611 : : } else {
1612 : : if (likely(cache != NULL)) {
1613 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1614 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1615 : : } else {
1616 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1617 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1618 : : }
1619 : : }
1620 : :
1621 : : return ret;
1622 : : }
1623 : :
1624 : : /**
1625 : : * Get several objects from the mempool.
1626 : : *
1627 : : * If cache is enabled, objects will be retrieved first from cache,
1628 : : * subsequently from the common pool. Note that it can return -ENOENT when
1629 : : * the local cache and common pool are empty, even if cache from other
1630 : : * lcores are full.
1631 : : *
1632 : : * @param mp
1633 : : * A pointer to the mempool structure.
1634 : : * @param obj_table
1635 : : * A pointer to a table of void * pointers (objects) that will be filled.
1636 : : * @param n
1637 : : * The number of objects to get from mempool to obj_table.
1638 : : * @param cache
1639 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1640 : : * @return
1641 : : * - 0: Success; objects taken.
1642 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1643 : : */
1644 : : static __rte_always_inline int
1645 : : rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1646 : : unsigned int n, struct rte_mempool_cache *cache)
1647 : : {
1648 : : int ret;
1649 : : ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1650 : : if (ret == 0)
1651 : : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1652 : : rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1653 : : return ret;
1654 : : }
1655 : :
1656 : : /**
1657 : : * Get several objects from the mempool.
1658 : : *
1659 : : * This function calls the multi-consumers or the single-consumer
1660 : : * version, depending on the default behaviour that was specified at
1661 : : * mempool creation time (see flags).
1662 : : *
1663 : : * If cache is enabled, objects will be retrieved first from cache,
1664 : : * subsequently from the common pool. Note that it can return -ENOENT when
1665 : : * the local cache and common pool are empty, even if cache from other
1666 : : * lcores are full.
1667 : : *
1668 : : * @param mp
1669 : : * A pointer to the mempool structure.
1670 : : * @param obj_table
1671 : : * A pointer to a table of void * pointers (objects) that will be filled.
1672 : : * @param n
1673 : : * The number of objects to get from the mempool to obj_table.
1674 : : * @return
1675 : : * - 0: Success; objects taken
1676 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1677 : : */
1678 : : static __rte_always_inline int
1679 : : rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1680 : : {
1681 : : struct rte_mempool_cache *cache;
1682 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1683 : : rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1684 : : return rte_mempool_generic_get(mp, obj_table, n, cache);
1685 : : }
1686 : :
1687 : : /**
1688 : : * Get one object from the mempool.
1689 : : *
1690 : : * This function calls the multi-consumers or the single-consumer
1691 : : * version, depending on the default behavior that was specified at
1692 : : * mempool creation (see flags).
1693 : : *
1694 : : * If cache is enabled, objects will be retrieved first from cache,
1695 : : * subsequently from the common pool. Note that it can return -ENOENT when
1696 : : * the local cache and common pool are empty, even if cache from other
1697 : : * lcores are full.
1698 : : *
1699 : : * @param mp
1700 : : * A pointer to the mempool structure.
1701 : : * @param obj_p
1702 : : * A pointer to a void * pointer (object) that will be filled.
1703 : : * @return
1704 : : * - 0: Success; objects taken.
1705 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1706 : : */
1707 : : static __rte_always_inline int
1708 : : rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1709 : : {
1710 : : return rte_mempool_get_bulk(mp, obj_p, 1);
1711 : : }
1712 : :
1713 : : /**
1714 : : * Get a contiguous blocks of objects from the mempool.
1715 : : *
1716 : : * If cache is enabled, consider to flush it first, to reuse objects
1717 : : * as soon as possible.
1718 : : *
1719 : : * The application should check that the driver supports the operation
1720 : : * by calling rte_mempool_ops_get_info() and checking that `contig_block_size`
1721 : : * is not zero.
1722 : : *
1723 : : * @param mp
1724 : : * A pointer to the mempool structure.
1725 : : * @param first_obj_table
1726 : : * A pointer to a pointer to the first object in each block.
1727 : : * @param n
1728 : : * The number of blocks to get from mempool.
1729 : : * @return
1730 : : * - 0: Success; blocks taken.
1731 : : * - -ENOBUFS: Not enough entries in the mempool; no object is retrieved.
1732 : : * - -EOPNOTSUPP: The mempool driver does not support block dequeue
1733 : : */
1734 : : static __rte_always_inline int
1735 : : rte_mempool_get_contig_blocks(struct rte_mempool *mp,
1736 : : void **first_obj_table, unsigned int n)
1737 : : {
1738 : : int ret;
1739 : :
1740 : 0 : ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1741 : : if (ret == 0) {
1742 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1743 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1744 : : RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1745 : : 1);
1746 : : } else {
1747 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1748 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1749 : : }
1750 : :
1751 : : rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1752 : : return ret;
1753 : : }
1754 : :
1755 : : /**
1756 : : * Return the number of entries in the mempool.
1757 : : *
1758 : : * When cache is enabled, this function has to browse the length of
1759 : : * all lcores, so it should not be used in a data path, but only for
1760 : : * debug purposes. User-owned mempool caches are not accounted for.
1761 : : *
1762 : : * @param mp
1763 : : * A pointer to the mempool structure.
1764 : : * @return
1765 : : * The number of entries in the mempool.
1766 : : */
1767 : : unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1768 : :
1769 : : /**
1770 : : * Return the number of elements which have been allocated from the mempool
1771 : : *
1772 : : * When cache is enabled, this function has to browse the length of
1773 : : * all lcores, so it should not be used in a data path, but only for
1774 : : * debug purposes.
1775 : : *
1776 : : * @param mp
1777 : : * A pointer to the mempool structure.
1778 : : * @return
1779 : : * The number of free entries in the mempool.
1780 : : */
1781 : : unsigned int
1782 : : rte_mempool_in_use_count(const struct rte_mempool *mp);
1783 : :
1784 : : /**
1785 : : * Test if the mempool is full.
1786 : : *
1787 : : * When cache is enabled, this function has to browse the length of all
1788 : : * lcores, so it should not be used in a data path, but only for debug
1789 : : * purposes. User-owned mempool caches are not accounted for.
1790 : : *
1791 : : * @param mp
1792 : : * A pointer to the mempool structure.
1793 : : * @return
1794 : : * - 1: The mempool is full.
1795 : : * - 0: The mempool is not full.
1796 : : */
1797 : : static inline int
1798 : : rte_mempool_full(const struct rte_mempool *mp)
1799 : : {
1800 [ + + - + : 6 : return rte_mempool_avail_count(mp) == mp->size;
- + - + +
- - + ]
1801 : : }
1802 : :
1803 : : /**
1804 : : * Test if the mempool is empty.
1805 : : *
1806 : : * When cache is enabled, this function has to browse the length of all
1807 : : * lcores, so it should not be used in a data path, but only for debug
1808 : : * purposes. User-owned mempool caches are not accounted for.
1809 : : *
1810 : : * @param mp
1811 : : * A pointer to the mempool structure.
1812 : : * @return
1813 : : * - 1: The mempool is empty.
1814 : : * - 0: The mempool is not empty.
1815 : : */
1816 : : static inline int
1817 : : rte_mempool_empty(const struct rte_mempool *mp)
1818 : : {
1819 : 5 : return rte_mempool_avail_count(mp) == 0;
1820 : : }
1821 : :
1822 : : /**
1823 : : * Return the IO address of elt, which is an element of the pool mp.
1824 : : *
1825 : : * @param elt
1826 : : * A pointer (virtual address) to the element of the pool.
1827 : : * @return
1828 : : * The IO address of the elt element.
1829 : : * If the mempool was created with RTE_MEMPOOL_F_NO_IOVA_CONTIG, the
1830 : : * returned value is RTE_BAD_IOVA.
1831 : : */
1832 : : static inline rte_iova_t
1833 : 0 : rte_mempool_virt2iova(const void *elt)
1834 : : {
1835 : : const struct rte_mempool_objhdr *hdr;
1836 : 94136 : hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1837 : : sizeof(*hdr));
1838 [ - + ]: 333 : return hdr->iova;
1839 : : }
1840 : :
1841 : : /**
1842 : : * Check the consistency of mempool objects.
1843 : : *
1844 : : * Verify the coherency of fields in the mempool structure. Also check
1845 : : * that the cookies of mempool objects (even the ones that are not
1846 : : * present in pool) have a correct value. If not, a panic will occur.
1847 : : *
1848 : : * @param mp
1849 : : * A pointer to the mempool structure.
1850 : : */
1851 : : void rte_mempool_audit(struct rte_mempool *mp);
1852 : :
1853 : : /**
1854 : : * Return a pointer to the private data in an mempool structure.
1855 : : *
1856 : : * @param mp
1857 : : * A pointer to the mempool structure.
1858 : : * @return
1859 : : * A pointer to the private data.
1860 : : */
1861 : : static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1862 : : {
1863 [ + + - + : 7017 : return (char *)mp +
- + - + +
+ + + + +
- + + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1864 [ + + + + : 92643 : RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
+ + + + +
+ + + + +
- + + + +
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1865 : : }
1866 : :
1867 : : /**
1868 : : * Dump the status of all mempools on the console
1869 : : *
1870 : : * @param f
1871 : : * A pointer to a file for output
1872 : : */
1873 : : void rte_mempool_list_dump(FILE *f);
1874 : :
1875 : : /**
1876 : : * Search a mempool from its name
1877 : : *
1878 : : * @param name
1879 : : * The name of the mempool.
1880 : : * @return
1881 : : * The pointer to the mempool matching the name, or NULL if not found.
1882 : : * NULL on error
1883 : : * with rte_errno set appropriately. Possible rte_errno values include:
1884 : : * - ENOENT - required entry not available to return.
1885 : : */
1886 : : struct rte_mempool *rte_mempool_lookup(const char *name);
1887 : :
1888 : : /**
1889 : : * Get the header, trailer and total size of a mempool element.
1890 : : *
1891 : : * Given a desired size of the mempool element and mempool flags,
1892 : : * calculates header, trailer, body and total sizes of the mempool object.
1893 : : *
1894 : : * @param elt_size
1895 : : * The size of each element, without header and trailer.
1896 : : * @param flags
1897 : : * The flags used for the mempool creation.
1898 : : * Consult rte_mempool_create() for more information about possible values.
1899 : : * The size of each element.
1900 : : * @param sz
1901 : : * The calculated detailed size the mempool object. May be NULL.
1902 : : * @return
1903 : : * Total size of the mempool object.
1904 : : */
1905 : : uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1906 : : struct rte_mempool_objsz *sz);
1907 : :
1908 : : /**
1909 : : * Walk list of all memory pools
1910 : : *
1911 : : * @param func
1912 : : * Iterator function
1913 : : * @param arg
1914 : : * Argument passed to iterator
1915 : : */
1916 : : void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1917 : : void *arg);
1918 : :
1919 : : /**
1920 : : * A structure used to retrieve information about the memory range
1921 : : * of the mempool.
1922 : : */
1923 : : struct rte_mempool_mem_range_info {
1924 : : /** Start of the memory range used by mempool objects */
1925 : : void *start;
1926 : : /** Length of the memory range used by mempool objects */
1927 : : size_t length;
1928 : : /** Are all memory addresses used by mempool objects contiguous */
1929 : : bool is_contiguous;
1930 : : };
1931 : :
1932 : : /**
1933 : : * @warning
1934 : : * @b EXPERIMENTAL: this API may change without prior notice.
1935 : : *
1936 : : * Get information about the memory range used to store objects in the mempool.
1937 : : *
1938 : : * @param[in] mp
1939 : : * Pointer to an initialized mempool.
1940 : : * @param[out] mem_range
1941 : : * Pointer to struct which is used to return lowest address,
1942 : : * length of the memory range containing all the addresses,
1943 : : * and whether these addresses are contiguous.
1944 : : * @return
1945 : : * 0 on success, -EINVAL if mempool is not valid or mem_range is NULL.
1946 : : **/
1947 : : __rte_experimental
1948 : : int
1949 : : rte_mempool_get_mem_range(const struct rte_mempool *mp,
1950 : : struct rte_mempool_mem_range_info *mem_range);
1951 : :
1952 : : /**
1953 : : * @warning
1954 : : * @b EXPERIMENTAL: this API may change without prior notice.
1955 : : *
1956 : : * Return alignment of objects stored in the mempool.
1957 : : *
1958 : : * @param[in] mp
1959 : : * Pointer to a mempool.
1960 : : * @return
1961 : : * Object alignment if mp is valid. 0 if mp is NULL.
1962 : : *
1963 : : **/
1964 : : __rte_experimental
1965 : : size_t
1966 : : rte_mempool_get_obj_alignment(const struct rte_mempool *mp);
1967 : :
1968 : : /**
1969 : : * @internal Get page size used for mempool object allocation.
1970 : : * This function is internal to mempool library and mempool drivers.
1971 : : */
1972 : : int
1973 : : rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1974 : :
1975 : : /**
1976 : : * Mempool event type.
1977 : : * @internal
1978 : : */
1979 : : enum rte_mempool_event {
1980 : : /** Occurs after a mempool is fully populated. */
1981 : : RTE_MEMPOOL_EVENT_READY = 0,
1982 : : /** Occurs before the destruction of a mempool begins. */
1983 : : RTE_MEMPOOL_EVENT_DESTROY = 1,
1984 : : };
1985 : :
1986 : : /**
1987 : : * @internal
1988 : : * Mempool event callback.
1989 : : *
1990 : : * rte_mempool_event_callback_register() may be called from within the callback,
1991 : : * but the callbacks registered this way will not be invoked for the same event.
1992 : : * rte_mempool_event_callback_unregister() may only be safely called
1993 : : * to remove the running callback.
1994 : : */
1995 : : typedef void (rte_mempool_event_callback)(
1996 : : enum rte_mempool_event event,
1997 : : struct rte_mempool *mp,
1998 : : void *user_data);
1999 : :
2000 : : /**
2001 : : * @internal
2002 : : * Register a callback function invoked on mempool life cycle event.
2003 : : * The function will be invoked in the process
2004 : : * that performs an action which triggers the callback.
2005 : : * Registration is process-private,
2006 : : * i.e. each process must manage callbacks on its own if needed.
2007 : : *
2008 : : * @param func
2009 : : * Callback function.
2010 : : * @param user_data
2011 : : * User data.
2012 : : *
2013 : : * @return
2014 : : * 0 on success, negative on failure and rte_errno is set.
2015 : : */
2016 : : __rte_internal
2017 : : int
2018 : : rte_mempool_event_callback_register(rte_mempool_event_callback *func,
2019 : : void *user_data);
2020 : :
2021 : : /**
2022 : : * @internal
2023 : : * Unregister a callback added with rte_mempool_event_callback_register().
2024 : : * @p func and @p user_data must exactly match registration parameters.
2025 : : *
2026 : : * @param func
2027 : : * Callback function.
2028 : : * @param user_data
2029 : : * User data.
2030 : : *
2031 : : * @return
2032 : : * 0 on success, negative on failure and rte_errno is set.
2033 : : */
2034 : : __rte_internal
2035 : : int
2036 : : rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
2037 : : void *user_data);
2038 : :
2039 : : #ifdef __cplusplus
2040 : : }
2041 : : #endif
2042 : :
2043 : : #endif /* _RTE_MEMPOOL_H_ */
|
