Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright(c) 2016 6WIND S.A.
4 : : * Copyright(c) 2022 SmartShare Systems
5 : : */
6 : :
7 : : #ifndef _RTE_MEMPOOL_H_
8 : : #define _RTE_MEMPOOL_H_
9 : :
10 : : /**
11 : : * @file
12 : : * RTE Mempool.
13 : : *
14 : : * A memory pool is an allocator of fixed-size object. It is
15 : : * identified by its name, and uses a ring to store free objects. It
16 : : * provides some other optional services, like a per-core object
17 : : * cache, and an alignment helper to ensure that objects are padded
18 : : * to spread them equally on all RAM channels, ranks, and so on.
19 : : *
20 : : * Objects owned by a mempool should never be added in another
21 : : * mempool. When an object is freed using rte_mempool_put() or
22 : : * equivalent, the object data is not modified; the user can save some
23 : : * meta-data in the object data and retrieve them when allocating a
24 : : * new object.
25 : : *
26 : : * Note: the mempool implementation is not preemptible. An lcore must not be
27 : : * interrupted by another task that uses the same mempool (because it uses a
28 : : * ring which is not preemptible). Also, usual mempool functions like
29 : : * rte_mempool_get() or rte_mempool_put() are designed to be called from an EAL
30 : : * thread due to the internal per-lcore cache. Due to the lack of caching,
31 : : * rte_mempool_get() or rte_mempool_put() performance will suffer when called
32 : : * by unregistered non-EAL threads. Instead, unregistered non-EAL threads
33 : : * should call rte_mempool_generic_get() or rte_mempool_generic_put() with a
34 : : * user cache created with rte_mempool_cache_create().
35 : : */
36 : :
37 : : #include <stdio.h>
38 : : #include <stdint.h>
39 : : #include <inttypes.h>
40 : :
41 : : #include <rte_compat.h>
42 : : #include <rte_config.h>
43 : : #include <rte_spinlock.h>
44 : : #include <rte_debug.h>
45 : : #include <rte_lcore.h>
46 : : #include <rte_log.h>
47 : : #include <rte_branch_prediction.h>
48 : : #include <rte_ring.h>
49 : : #include <rte_memcpy.h>
50 : : #include <rte_common.h>
51 : :
52 : : #include "rte_mempool_trace_fp.h"
53 : :
54 : : #ifdef __cplusplus
55 : : extern "C" {
56 : : #endif
57 : :
58 : : #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL /**< Header cookie. */
59 : : #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL /**< Header cookie. */
60 : : #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL /**< Trailer cookie.*/
61 : :
62 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
63 : : /**
64 : : * A structure that stores the mempool statistics (per-lcore).
65 : : * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
66 : : * captured since they can be calculated from other stats.
67 : : * For example: put_cache_objs = put_objs - put_common_pool_objs.
68 : : */
69 : : struct rte_mempool_debug_stats {
70 : : uint64_t put_bulk; /**< Number of puts. */
71 : : uint64_t put_objs; /**< Number of objects successfully put. */
72 : : uint64_t put_common_pool_bulk; /**< Number of bulks enqueued in common pool. */
73 : : uint64_t put_common_pool_objs; /**< Number of objects enqueued in common pool. */
74 : : uint64_t get_common_pool_bulk; /**< Number of bulks dequeued from common pool. */
75 : : uint64_t get_common_pool_objs; /**< Number of objects dequeued from common pool. */
76 : : uint64_t get_success_bulk; /**< Successful allocation number. */
77 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
78 : : uint64_t get_fail_bulk; /**< Failed allocation number. */
79 : : uint64_t get_fail_objs; /**< Objects that failed to be allocated. */
80 : : uint64_t get_success_blks; /**< Successful allocation number of contiguous blocks. */
81 : : uint64_t get_fail_blks; /**< Failed allocation number of contiguous blocks. */
82 : : RTE_CACHE_GUARD;
83 : : } __rte_cache_aligned;
84 : : #endif
85 : :
86 : : /**
87 : : * A structure that stores a per-core object cache.
88 : : */
89 : : struct rte_mempool_cache {
90 : : uint32_t size; /**< Size of the cache */
91 : : uint32_t flushthresh; /**< Threshold before we flush excess elements */
92 : : uint32_t len; /**< Current cache count */
93 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
94 : : uint32_t unused;
95 : : /*
96 : : * Alternative location for the most frequently updated mempool statistics (per-lcore),
97 : : * providing faster update access when using a mempool cache.
98 : : */
99 : : struct {
100 : : uint64_t put_bulk; /**< Number of puts. */
101 : : uint64_t put_objs; /**< Number of objects successfully put. */
102 : : uint64_t get_success_bulk; /**< Successful allocation number. */
103 : : uint64_t get_success_objs; /**< Objects successfully allocated. */
104 : : } stats; /**< Statistics */
105 : : #endif
106 : : /**
107 : : * Cache objects
108 : : *
109 : : * Cache is allocated to this size to allow it to overflow in certain
110 : : * cases to avoid needless emptying of cache.
111 : : */
112 : : void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 2] __rte_cache_aligned;
113 : : } __rte_cache_aligned;
114 : :
115 : : /**
116 : : * A structure that stores the size of mempool elements.
117 : : */
118 : : struct rte_mempool_objsz {
119 : : uint32_t elt_size; /**< Size of an element. */
120 : : uint32_t header_size; /**< Size of header (before elt). */
121 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
122 : : uint32_t total_size;
123 : : /**< Total size of an object (header + elt + trailer). */
124 : : };
125 : :
126 : : /**< Maximum length of a memory pool's name. */
127 : : #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
128 : : sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
129 : : #define RTE_MEMPOOL_MZ_PREFIX "MP_"
130 : :
131 : : /* "MP_<name>" */
132 : : #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
133 : :
134 : : #ifndef RTE_MEMPOOL_ALIGN
135 : : /**
136 : : * Alignment of elements inside mempool.
137 : : */
138 : : #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
139 : : #endif
140 : :
141 : : #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
142 : :
143 : : /**
144 : : * Mempool object header structure
145 : : *
146 : : * Each object stored in mempools are prefixed by this header structure,
147 : : * it allows to retrieve the mempool pointer from the object and to
148 : : * iterate on all objects attached to a mempool. When debug is enabled,
149 : : * a cookie is also added in this structure preventing corruptions and
150 : : * double-frees.
151 : : */
152 : : struct rte_mempool_objhdr {
153 : : RTE_STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */
154 : : struct rte_mempool *mp; /**< The mempool owning the object. */
155 : : rte_iova_t iova; /**< IO address of the object. */
156 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
157 : : uint64_t cookie; /**< Debug cookie. */
158 : : #endif
159 : : };
160 : :
161 : : /**
162 : : * A list of object headers type
163 : : */
164 : : RTE_STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
165 : :
166 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
167 : :
168 : : /**
169 : : * Mempool object trailer structure
170 : : *
171 : : * In debug mode, each object stored in mempools are suffixed by this
172 : : * trailer structure containing a cookie preventing memory corruptions.
173 : : */
174 : : struct rte_mempool_objtlr {
175 : : uint64_t cookie; /**< Debug cookie. */
176 : : };
177 : :
178 : : #endif
179 : :
180 : : /**
181 : : * @internal Logtype used for mempool related messages.
182 : : */
183 : : extern int rte_mempool_logtype;
184 : : #define RTE_LOGTYPE_MEMPOOL rte_mempool_logtype
185 : : #define RTE_MEMPOOL_LOG(level, ...) \
186 : : RTE_LOG_LINE(level, MEMPOOL, "" __VA_ARGS__)
187 : :
188 : : /**
189 : : * A list of memory where objects are stored
190 : : */
191 : : RTE_STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
192 : :
193 : : /**
194 : : * Callback used to free a memory chunk
195 : : */
196 : : typedef void (rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr,
197 : : void *opaque);
198 : :
199 : : /**
200 : : * Mempool objects memory header structure
201 : : *
202 : : * The memory chunks where objects are stored. Each chunk is virtually
203 : : * and physically contiguous.
204 : : */
205 : : struct rte_mempool_memhdr {
206 : : RTE_STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */
207 : : struct rte_mempool *mp; /**< The mempool owning the chunk */
208 : : void *addr; /**< Virtual address of the chunk */
209 : : rte_iova_t iova; /**< IO address of the chunk */
210 : : size_t len; /**< length of the chunk */
211 : : rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */
212 : : void *opaque; /**< Argument passed to the free callback */
213 : : };
214 : :
215 : : /**
216 : : * Additional information about the mempool
217 : : *
218 : : * The structure is cache-line aligned to avoid ABI breakages in
219 : : * a number of cases when something small is added.
220 : : */
221 : : struct rte_mempool_info {
222 : : /** Number of objects in the contiguous block */
223 : : unsigned int contig_block_size;
224 : : } __rte_cache_aligned;
225 : :
226 : : /**
227 : : * The RTE mempool structure.
228 : : */
229 : : struct rte_mempool {
230 : : char name[RTE_MEMPOOL_NAMESIZE]; /**< Name of mempool. */
231 : : union {
232 : : void *pool_data; /**< Ring or pool to store objects. */
233 : : uint64_t pool_id; /**< External mempool identifier. */
234 : : };
235 : : void *pool_config; /**< optional args for ops alloc. */
236 : : const struct rte_memzone *mz; /**< Memzone where pool is alloc'd. */
237 : : unsigned int flags; /**< Flags of the mempool. */
238 : : int socket_id; /**< Socket id passed at create. */
239 : : uint32_t size; /**< Max size of the mempool. */
240 : : uint32_t cache_size;
241 : : /**< Size of per-lcore default local cache. */
242 : :
243 : : uint32_t elt_size; /**< Size of an element. */
244 : : uint32_t header_size; /**< Size of header (before elt). */
245 : : uint32_t trailer_size; /**< Size of trailer (after elt). */
246 : :
247 : : unsigned private_data_size; /**< Size of private data. */
248 : : /**
249 : : * Index into rte_mempool_ops_table array of mempool ops
250 : : * structs, which contain callback function pointers.
251 : : * We're using an index here rather than pointers to the callbacks
252 : : * to facilitate any secondary processes that may want to use
253 : : * this mempool.
254 : : */
255 : : int32_t ops_index;
256 : :
257 : : struct rte_mempool_cache *local_cache; /**< Per-lcore local cache */
258 : :
259 : : uint32_t populated_size; /**< Number of populated objects. */
260 : : struct rte_mempool_objhdr_list elt_list; /**< List of objects in pool */
261 : : uint32_t nb_mem_chunks; /**< Number of memory chunks */
262 : : struct rte_mempool_memhdr_list mem_list; /**< List of memory chunks */
263 : :
264 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
265 : : /** Per-lcore statistics.
266 : : *
267 : : * Plus one, for unregistered non-EAL threads.
268 : : */
269 : : struct rte_mempool_debug_stats stats[RTE_MAX_LCORE + 1];
270 : : #endif
271 : : } __rte_cache_aligned;
272 : :
273 : : /** Spreading among memory channels not required. */
274 : : #define RTE_MEMPOOL_F_NO_SPREAD 0x0001
275 : : /**
276 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_SPREAD.
277 : : * To be deprecated.
278 : : */
279 : : #define MEMPOOL_F_NO_SPREAD RTE_MEMPOOL_F_NO_SPREAD
280 : : /** Do not align objects on cache lines. */
281 : : #define RTE_MEMPOOL_F_NO_CACHE_ALIGN 0x0002
282 : : /**
283 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_CACHE_ALIGN.
284 : : * To be deprecated.
285 : : */
286 : : #define MEMPOOL_F_NO_CACHE_ALIGN RTE_MEMPOOL_F_NO_CACHE_ALIGN
287 : : /** Default put is "single-producer". */
288 : : #define RTE_MEMPOOL_F_SP_PUT 0x0004
289 : : /**
290 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SP_PUT.
291 : : * To be deprecated.
292 : : */
293 : : #define MEMPOOL_F_SP_PUT RTE_MEMPOOL_F_SP_PUT
294 : : /** Default get is "single-consumer". */
295 : : #define RTE_MEMPOOL_F_SC_GET 0x0008
296 : : /**
297 : : * Backward compatibility synonym for RTE_MEMPOOL_F_SC_GET.
298 : : * To be deprecated.
299 : : */
300 : : #define MEMPOOL_F_SC_GET RTE_MEMPOOL_F_SC_GET
301 : : /** Internal: pool is created. */
302 : : #define RTE_MEMPOOL_F_POOL_CREATED 0x0010
303 : : /** Don't need IOVA contiguous objects. */
304 : : #define RTE_MEMPOOL_F_NO_IOVA_CONTIG 0x0020
305 : : /**
306 : : * Backward compatibility synonym for RTE_MEMPOOL_F_NO_IOVA_CONTIG.
307 : : * To be deprecated.
308 : : */
309 : : #define MEMPOOL_F_NO_IOVA_CONTIG RTE_MEMPOOL_F_NO_IOVA_CONTIG
310 : : /** Internal: no object from the pool can be used for device IO (DMA). */
311 : : #define RTE_MEMPOOL_F_NON_IO 0x0040
312 : :
313 : : /**
314 : : * This macro lists all the mempool flags an application may request.
315 : : */
316 : : #define RTE_MEMPOOL_VALID_USER_FLAGS (RTE_MEMPOOL_F_NO_SPREAD \
317 : : | RTE_MEMPOOL_F_NO_CACHE_ALIGN \
318 : : | RTE_MEMPOOL_F_SP_PUT \
319 : : | RTE_MEMPOOL_F_SC_GET \
320 : : | RTE_MEMPOOL_F_NO_IOVA_CONTIG \
321 : : )
322 : :
323 : : /**
324 : : * @internal When stats is enabled, store some statistics.
325 : : *
326 : : * @param mp
327 : : * Pointer to the memory pool.
328 : : * @param name
329 : : * Name of the statistics field to increment in the memory pool.
330 : : * @param n
331 : : * Number to add to the statistics.
332 : : */
333 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
334 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
335 : : unsigned int __lcore_id = rte_lcore_id(); \
336 : : if (likely(__lcore_id < RTE_MAX_LCORE)) \
337 : : (mp)->stats[__lcore_id].name += (n); \
338 : : else \
339 : : rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name), \
340 : : (n), rte_memory_order_relaxed); \
341 : : } while (0)
342 : : #else
343 : : #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
344 : : #endif
345 : :
346 : : /**
347 : : * @internal When stats is enabled, store some statistics.
348 : : *
349 : : * @param cache
350 : : * Pointer to the memory pool cache.
351 : : * @param name
352 : : * Name of the statistics field to increment in the memory pool cache.
353 : : * @param n
354 : : * Number to add to the statistics.
355 : : */
356 : : #ifdef RTE_LIBRTE_MEMPOOL_STATS
357 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n))
358 : : #else
359 : : #define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0)
360 : : #endif
361 : :
362 : : /**
363 : : * @internal Calculate the size of the mempool header.
364 : : *
365 : : * @param mp
366 : : * Pointer to the memory pool.
367 : : * @param cs
368 : : * Size of the per-lcore cache.
369 : : */
370 : : #define RTE_MEMPOOL_HEADER_SIZE(mp, cs) \
371 : : (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
372 : : (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
373 : :
374 : : /* return the header of a mempool object (internal) */
375 : : static inline struct rte_mempool_objhdr *
376 : : rte_mempool_get_header(void *obj)
377 : : {
378 : 276530 : return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
379 : : sizeof(struct rte_mempool_objhdr));
380 : : }
381 : :
382 : : /**
383 : : * Return a pointer to the mempool owning this object.
384 : : *
385 : : * @param obj
386 : : * An object that is owned by a pool. If this is not the case,
387 : : * the behavior is undefined.
388 : : * @return
389 : : * A pointer to the mempool structure.
390 : : */
391 : : static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
392 : : {
393 : : struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
394 [ - + # # ]: 276530 : return hdr->mp;
395 : : }
396 : :
397 : : /* return the trailer of a mempool object (internal) */
398 : : static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
399 : : {
400 : : struct rte_mempool *mp = rte_mempool_from_obj(obj);
401 : : return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
402 : : }
403 : :
404 : : /**
405 : : * @internal Check and update cookies or panic.
406 : : *
407 : : * @param mp
408 : : * Pointer to the memory pool.
409 : : * @param obj_table_const
410 : : * Pointer to a table of void * pointers (objects).
411 : : * @param n
412 : : * Index of object in object table.
413 : : * @param free
414 : : * - 0: object is supposed to be allocated, mark it as free
415 : : * - 1: object is supposed to be free, mark it as allocated
416 : : * - 2: just check that cookie is valid (free or allocated)
417 : : */
418 : : void rte_mempool_check_cookies(const struct rte_mempool *mp,
419 : : void * const *obj_table_const, unsigned n, int free);
420 : :
421 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
422 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
423 : : rte_mempool_check_cookies(mp, obj_table_const, n, free)
424 : : #else
425 : : #define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
426 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
427 : :
428 : : /**
429 : : * @internal Check contiguous object blocks and update cookies or panic.
430 : : *
431 : : * @param mp
432 : : * Pointer to the memory pool.
433 : : * @param first_obj_table_const
434 : : * Pointer to a table of void * pointers (first object of the contiguous
435 : : * object blocks).
436 : : * @param n
437 : : * Number of contiguous object blocks.
438 : : * @param free
439 : : * - 0: object is supposed to be allocated, mark it as free
440 : : * - 1: object is supposed to be free, mark it as allocated
441 : : * - 2: just check that cookie is valid (free or allocated)
442 : : */
443 : : void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
444 : : void * const *first_obj_table_const, unsigned int n, int free);
445 : :
446 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
447 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
448 : : free) \
449 : : rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
450 : : free)
451 : : #else
452 : : #define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
453 : : free) \
454 : : do {} while (0)
455 : : #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
456 : :
457 : : #define RTE_MEMPOOL_OPS_NAMESIZE 32 /**< Max length of ops struct name. */
458 : :
459 : : /**
460 : : * Prototype for implementation specific data provisioning function.
461 : : *
462 : : * The function should provide the implementation specific memory for
463 : : * use by the other mempool ops functions in a given mempool ops struct.
464 : : * E.g. the default ops provides an instance of the rte_ring for this purpose.
465 : : * it will most likely point to a different type of data structure, and
466 : : * will be transparent to the application programmer.
467 : : * This function should set mp->pool_data.
468 : : */
469 : : typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
470 : :
471 : : /**
472 : : * Free the opaque private data pointed to by mp->pool_data pointer.
473 : : */
474 : : typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
475 : :
476 : : /**
477 : : * Enqueue 'n' objects into the external pool.
478 : : * @return
479 : : * - 0: Success
480 : : * - <0: Error
481 : : */
482 : : typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
483 : : void * const *obj_table, unsigned int n);
484 : :
485 : : /**
486 : : * Dequeue 'n' objects from the external pool.
487 : : * @return
488 : : * - 0: Success
489 : : * - <0: Error
490 : : */
491 : : typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
492 : : void **obj_table, unsigned int n);
493 : :
494 : : /**
495 : : * Dequeue a number of contiguous object blocks from the external pool.
496 : : */
497 : : typedef int (*rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp,
498 : : void **first_obj_table, unsigned int n);
499 : :
500 : : /**
501 : : * Return the number of available objects in the external pool.
502 : : */
503 : : typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
504 : :
505 : : /**
506 : : * Calculate memory size required to store given number of objects.
507 : : *
508 : : * If mempool objects are not required to be IOVA-contiguous
509 : : * (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is set), min_chunk_size defines
510 : : * virtually contiguous chunk size. Otherwise, if mempool objects must
511 : : * be IOVA-contiguous (the flag RTE_MEMPOOL_F_NO_IOVA_CONTIG is clear),
512 : : * min_chunk_size defines IOVA-contiguous chunk size.
513 : : *
514 : : * @param[in] mp
515 : : * Pointer to the memory pool.
516 : : * @param[in] obj_num
517 : : * Number of objects.
518 : : * @param[in] pg_shift
519 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
520 : : * @param[out] min_chunk_size
521 : : * Location for minimum size of the memory chunk which may be used to
522 : : * store memory pool objects.
523 : : * @param[out] align
524 : : * Location for required memory chunk alignment.
525 : : * @return
526 : : * Required memory size.
527 : : */
528 : : typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
529 : : uint32_t obj_num, uint32_t pg_shift,
530 : : size_t *min_chunk_size, size_t *align);
531 : :
532 : : /**
533 : : * @internal Helper to calculate memory size required to store given
534 : : * number of objects.
535 : : *
536 : : * This function is internal to mempool library and mempool drivers.
537 : : *
538 : : * If page boundaries may be ignored, it is just a product of total
539 : : * object size including header and trailer and number of objects.
540 : : * Otherwise, it is a number of pages required to store given number of
541 : : * objects without crossing page boundary.
542 : : *
543 : : * Note that if object size is bigger than page size, then it assumes
544 : : * that pages are grouped in subsets of physically continuous pages big
545 : : * enough to store at least one object.
546 : : *
547 : : * Minimum size of memory chunk is the total element size.
548 : : * Required memory chunk alignment is the cache line size.
549 : : *
550 : : * @param[in] mp
551 : : * A pointer to the mempool structure.
552 : : * @param[in] obj_num
553 : : * Number of objects to be added in mempool.
554 : : * @param[in] pg_shift
555 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
556 : : * @param[in] chunk_reserve
557 : : * Amount of memory that must be reserved at the beginning of each page,
558 : : * or at the beginning of the memory area if pg_shift is 0.
559 : : * @param[out] min_chunk_size
560 : : * Location for minimum size of the memory chunk which may be used to
561 : : * store memory pool objects.
562 : : * @param[out] align
563 : : * Location for required memory chunk alignment.
564 : : * @return
565 : : * Required memory size.
566 : : */
567 : : ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
568 : : uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
569 : : size_t *min_chunk_size, size_t *align);
570 : :
571 : : /**
572 : : * Default way to calculate memory size required to store given number of
573 : : * objects.
574 : : *
575 : : * Equivalent to rte_mempool_op_calc_mem_size_helper(mp, obj_num, pg_shift,
576 : : * 0, min_chunk_size, align).
577 : : */
578 : : ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
579 : : uint32_t obj_num, uint32_t pg_shift,
580 : : size_t *min_chunk_size, size_t *align);
581 : :
582 : : /**
583 : : * Function to be called for each populated object.
584 : : *
585 : : * @param[in] mp
586 : : * A pointer to the mempool structure.
587 : : * @param[in] opaque
588 : : * An opaque pointer passed to iterator.
589 : : * @param[in] vaddr
590 : : * Object virtual address.
591 : : * @param[in] iova
592 : : * Input/output virtual address of the object or RTE_BAD_IOVA.
593 : : */
594 : : typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
595 : : void *opaque, void *vaddr, rte_iova_t iova);
596 : :
597 : : /**
598 : : * Populate memory pool objects using provided memory chunk.
599 : : *
600 : : * Populated objects should be enqueued to the pool, e.g. using
601 : : * rte_mempool_ops_enqueue_bulk().
602 : : *
603 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
604 : : * the chunk doesn't need to be physically contiguous (only virtually),
605 : : * and allocated objects may span two pages.
606 : : *
607 : : * @param[in] mp
608 : : * A pointer to the mempool structure.
609 : : * @param[in] max_objs
610 : : * Maximum number of objects to be populated.
611 : : * @param[in] vaddr
612 : : * The virtual address of memory that should be used to store objects.
613 : : * @param[in] iova
614 : : * The IO address
615 : : * @param[in] len
616 : : * The length of memory in bytes.
617 : : * @param[in] obj_cb
618 : : * Callback function to be executed for each populated object.
619 : : * @param[in] obj_cb_arg
620 : : * An opaque pointer passed to the callback function.
621 : : * @return
622 : : * The number of objects added on success.
623 : : * On error, no objects are populated and a negative errno is returned.
624 : : */
625 : : typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
626 : : unsigned int max_objs,
627 : : void *vaddr, rte_iova_t iova, size_t len,
628 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
629 : :
630 : : /**
631 : : * Align objects on addresses multiple of total_elt_sz.
632 : : */
633 : : #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
634 : :
635 : : /**
636 : : * @internal Helper to populate memory pool object using provided memory
637 : : * chunk: just slice objects one by one, taking care of not
638 : : * crossing page boundaries.
639 : : *
640 : : * If RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ is set in flags, the addresses
641 : : * of object headers will be aligned on a multiple of total_elt_sz.
642 : : * This feature is used by octeontx hardware.
643 : : *
644 : : * This function is internal to mempool library and mempool drivers.
645 : : *
646 : : * @param[in] mp
647 : : * A pointer to the mempool structure.
648 : : * @param[in] flags
649 : : * Logical OR of following flags:
650 : : * - RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ: align objects on addresses
651 : : * multiple of total_elt_sz.
652 : : * @param[in] max_objs
653 : : * Maximum number of objects to be added in mempool.
654 : : * @param[in] vaddr
655 : : * The virtual address of memory that should be used to store objects.
656 : : * @param[in] iova
657 : : * The IO address corresponding to vaddr, or RTE_BAD_IOVA.
658 : : * @param[in] len
659 : : * The length of memory in bytes.
660 : : * @param[in] obj_cb
661 : : * Callback function to be executed for each populated object.
662 : : * @param[in] obj_cb_arg
663 : : * An opaque pointer passed to the callback function.
664 : : * @return
665 : : * The number of objects added in mempool.
666 : : */
667 : : int rte_mempool_op_populate_helper(struct rte_mempool *mp,
668 : : unsigned int flags, unsigned int max_objs,
669 : : void *vaddr, rte_iova_t iova, size_t len,
670 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
671 : :
672 : : /**
673 : : * Default way to populate memory pool object using provided memory chunk.
674 : : *
675 : : * Equivalent to rte_mempool_op_populate_helper(mp, 0, max_objs, vaddr, iova,
676 : : * len, obj_cb, obj_cb_arg).
677 : : */
678 : : int rte_mempool_op_populate_default(struct rte_mempool *mp,
679 : : unsigned int max_objs,
680 : : void *vaddr, rte_iova_t iova, size_t len,
681 : : rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
682 : :
683 : : /**
684 : : * Get some additional information about a mempool.
685 : : */
686 : : typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
687 : : struct rte_mempool_info *info);
688 : :
689 : :
690 : : /** Structure defining mempool operations structure */
691 : : struct rte_mempool_ops {
692 : : char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */
693 : : rte_mempool_alloc_t alloc; /**< Allocate private data. */
694 : : rte_mempool_free_t free; /**< Free the external pool. */
695 : : rte_mempool_enqueue_t enqueue; /**< Enqueue an object. */
696 : : rte_mempool_dequeue_t dequeue; /**< Dequeue an object. */
697 : : rte_mempool_get_count get_count; /**< Get qty of available objs. */
698 : : /**
699 : : * Optional callback to calculate memory size required to
700 : : * store specified number of objects.
701 : : */
702 : : rte_mempool_calc_mem_size_t calc_mem_size;
703 : : /**
704 : : * Optional callback to populate mempool objects using
705 : : * provided memory chunk.
706 : : */
707 : : rte_mempool_populate_t populate;
708 : : /**
709 : : * Get mempool info
710 : : */
711 : : rte_mempool_get_info_t get_info;
712 : : /**
713 : : * Dequeue a number of contiguous object blocks.
714 : : */
715 : : rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks;
716 : : } __rte_cache_aligned;
717 : :
718 : : #define RTE_MEMPOOL_MAX_OPS_IDX 16 /**< Max registered ops structs */
719 : :
720 : : /**
721 : : * Structure storing the table of registered ops structs, each of which contain
722 : : * the function pointers for the mempool ops functions.
723 : : * Each process has its own storage for this ops struct array so that
724 : : * the mempools can be shared across primary and secondary processes.
725 : : * The indices used to access the array are valid across processes, whereas
726 : : * any function pointers stored directly in the mempool struct would not be.
727 : : * This results in us simply having "ops_index" in the mempool struct.
728 : : */
729 : : struct rte_mempool_ops_table {
730 : : rte_spinlock_t sl; /**< Spinlock for add/delete. */
731 : : uint32_t num_ops; /**< Number of used ops structs in the table. */
732 : : /**
733 : : * Storage for all possible ops structs.
734 : : */
735 : : struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX];
736 : : } __rte_cache_aligned;
737 : :
738 : : /** Array of registered ops structs. */
739 : : extern struct rte_mempool_ops_table rte_mempool_ops_table;
740 : :
741 : : /**
742 : : * @internal Get the mempool ops struct from its index.
743 : : *
744 : : * @param ops_index
745 : : * The index of the ops struct in the ops struct table. It must be a valid
746 : : * index: (0 <= idx < num_ops).
747 : : * @return
748 : : * The pointer to the ops struct in the table.
749 : : */
750 : : static inline struct rte_mempool_ops *
751 : : rte_mempool_get_ops(int ops_index)
752 : : {
753 [ - + - + : 1597 : RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
- + - + -
+ - + ]
754 : :
755 : : return &rte_mempool_ops_table.ops[ops_index];
756 : : }
757 : :
758 : : /**
759 : : * @internal Wrapper for mempool_ops alloc callback.
760 : : *
761 : : * @param mp
762 : : * Pointer to the memory pool.
763 : : * @return
764 : : * - 0: Success; successfully allocated mempool pool_data.
765 : : * - <0: Error; code of alloc function.
766 : : */
767 : : int
768 : : rte_mempool_ops_alloc(struct rte_mempool *mp);
769 : :
770 : : /**
771 : : * @internal Wrapper for mempool_ops dequeue callback.
772 : : *
773 : : * @param mp
774 : : * Pointer to the memory pool.
775 : : * @param obj_table
776 : : * Pointer to a table of void * pointers (objects).
777 : : * @param n
778 : : * Number of objects to get.
779 : : * @return
780 : : * - 0: Success; got n objects.
781 : : * - <0: Error; code of dequeue function.
782 : : */
783 : : static inline int
784 : 44569657 : rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
785 : : void **obj_table, unsigned n)
786 : : {
787 : : struct rte_mempool_ops *ops;
788 : : int ret;
789 : :
790 : : rte_mempool_trace_ops_dequeue_bulk(mp, obj_table, n);
791 [ - + ]: 44569657 : ops = rte_mempool_get_ops(mp->ops_index);
792 : 44569657 : ret = ops->dequeue(mp, obj_table, n);
793 : : if (ret == 0) {
794 : : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
795 : : RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
796 : : }
797 : 44569657 : return ret;
798 : : }
799 : :
800 : : /**
801 : : * @internal Wrapper for mempool_ops dequeue_contig_blocks callback.
802 : : *
803 : : * @param[in] mp
804 : : * Pointer to the memory pool.
805 : : * @param[out] first_obj_table
806 : : * Pointer to a table of void * pointers (first objects).
807 : : * @param[in] n
808 : : * Number of blocks to get.
809 : : * @return
810 : : * - 0: Success; got n objects.
811 : : * - <0: Error; code of dequeue function.
812 : : */
813 : : static inline int
814 : 0 : rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
815 : : void **first_obj_table, unsigned int n)
816 : : {
817 : : struct rte_mempool_ops *ops;
818 : :
819 [ # # ]: 0 : ops = rte_mempool_get_ops(mp->ops_index);
820 : : RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
821 : : rte_mempool_trace_ops_dequeue_contig_blocks(mp, first_obj_table, n);
822 : 0 : return ops->dequeue_contig_blocks(mp, first_obj_table, n);
823 : : }
824 : :
825 : : /**
826 : : * @internal wrapper for mempool_ops enqueue callback.
827 : : *
828 : : * @param mp
829 : : * Pointer to the memory pool.
830 : : * @param obj_table
831 : : * Pointer to a table of void * pointers (objects).
832 : : * @param n
833 : : * Number of objects to put.
834 : : * @return
835 : : * - 0: Success; n objects supplied.
836 : : * - <0: Error; code of enqueue function.
837 : : */
838 : : static inline int
839 : 44586709 : rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
840 : : unsigned n)
841 : : {
842 : : struct rte_mempool_ops *ops;
843 : : int ret;
844 : :
845 : : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
846 : : RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
847 : : rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
848 [ - + ]: 44586709 : ops = rte_mempool_get_ops(mp->ops_index);
849 : 44586709 : ret = ops->enqueue(mp, obj_table, n);
850 : : #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
851 : : if (unlikely(ret < 0))
852 : : RTE_MEMPOOL_LOG(CRIT, "cannot enqueue %u objects to mempool %s",
853 : : n, mp->name);
854 : : #endif
855 : 44586709 : return ret;
856 : : }
857 : :
858 : : /**
859 : : * @internal wrapper for mempool_ops get_count callback.
860 : : *
861 : : * @param mp
862 : : * Pointer to the memory pool.
863 : : * @return
864 : : * The number of available objects in the external pool.
865 : : */
866 : : unsigned
867 : : rte_mempool_ops_get_count(const struct rte_mempool *mp);
868 : :
869 : : /**
870 : : * @internal wrapper for mempool_ops calc_mem_size callback.
871 : : * API to calculate size of memory required to store specified number of
872 : : * object.
873 : : *
874 : : * @param[in] mp
875 : : * Pointer to the memory pool.
876 : : * @param[in] obj_num
877 : : * Number of objects.
878 : : * @param[in] pg_shift
879 : : * LOG2 of the physical pages size. If set to 0, ignore page boundaries.
880 : : * @param[out] min_chunk_size
881 : : * Location for minimum size of the memory chunk which may be used to
882 : : * store memory pool objects.
883 : : * @param[out] align
884 : : * Location for required memory chunk alignment.
885 : : * @return
886 : : * Required memory size aligned at page boundary.
887 : : */
888 : : ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
889 : : uint32_t obj_num, uint32_t pg_shift,
890 : : size_t *min_chunk_size, size_t *align);
891 : :
892 : : /**
893 : : * @internal wrapper for mempool_ops populate callback.
894 : : *
895 : : * Populate memory pool objects using provided memory chunk.
896 : : *
897 : : * @param[in] mp
898 : : * A pointer to the mempool structure.
899 : : * @param[in] max_objs
900 : : * Maximum number of objects to be populated.
901 : : * @param[in] vaddr
902 : : * The virtual address of memory that should be used to store objects.
903 : : * @param[in] iova
904 : : * The IO address
905 : : * @param[in] len
906 : : * The length of memory in bytes.
907 : : * @param[in] obj_cb
908 : : * Callback function to be executed for each populated object.
909 : : * @param[in] obj_cb_arg
910 : : * An opaque pointer passed to the callback function.
911 : : * @return
912 : : * The number of objects added on success.
913 : : * On error, no objects are populated and a negative errno is returned.
914 : : */
915 : : int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
916 : : void *vaddr, rte_iova_t iova, size_t len,
917 : : rte_mempool_populate_obj_cb_t *obj_cb,
918 : : void *obj_cb_arg);
919 : :
920 : : /**
921 : : * Wrapper for mempool_ops get_info callback.
922 : : *
923 : : * @param[in] mp
924 : : * Pointer to the memory pool.
925 : : * @param[out] info
926 : : * Pointer to the rte_mempool_info structure
927 : : * @return
928 : : * - 0: Success; The mempool driver supports retrieving supplementary
929 : : * mempool information
930 : : * - -ENOTSUP - doesn't support get_info ops (valid case).
931 : : */
932 : : int rte_mempool_ops_get_info(const struct rte_mempool *mp,
933 : : struct rte_mempool_info *info);
934 : :
935 : : /**
936 : : * @internal wrapper for mempool_ops free callback.
937 : : *
938 : : * @param mp
939 : : * Pointer to the memory pool.
940 : : */
941 : : void
942 : : rte_mempool_ops_free(struct rte_mempool *mp);
943 : :
944 : : /**
945 : : * Set the ops of a mempool.
946 : : *
947 : : * This can only be done on a mempool that is not populated, i.e. just after
948 : : * a call to rte_mempool_create_empty().
949 : : *
950 : : * @param mp
951 : : * Pointer to the memory pool.
952 : : * @param name
953 : : * Name of the ops structure to use for this mempool.
954 : : * @param pool_config
955 : : * Opaque data that can be passed by the application to the ops functions.
956 : : * @return
957 : : * - 0: Success; the mempool is now using the requested ops functions.
958 : : * - -EINVAL - Invalid ops struct name provided.
959 : : * - -EEXIST - mempool already has an ops struct assigned.
960 : : */
961 : : int
962 : : rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
963 : : void *pool_config);
964 : :
965 : : /**
966 : : * Register mempool operations.
967 : : *
968 : : * @param ops
969 : : * Pointer to an ops structure to register.
970 : : * @return
971 : : * - >=0: Success; return the index of the ops struct in the table.
972 : : * - -EINVAL - some missing callbacks while registering ops struct.
973 : : * - -ENOSPC - the maximum number of ops structs has been reached.
974 : : */
975 : : int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
976 : :
977 : : /**
978 : : * Macro to statically register the ops of a mempool handler.
979 : : * Note that the rte_mempool_register_ops fails silently here when
980 : : * more than RTE_MEMPOOL_MAX_OPS_IDX is registered.
981 : : */
982 : : #define RTE_MEMPOOL_REGISTER_OPS(ops) \
983 : : RTE_INIT(mp_hdlr_init_##ops) \
984 : : { \
985 : : rte_mempool_register_ops(&ops); \
986 : : }
987 : :
988 : : /**
989 : : * An object callback function for mempool.
990 : : *
991 : : * Used by rte_mempool_create() and rte_mempool_obj_iter().
992 : : */
993 : : typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
994 : : void *opaque, void *obj, unsigned obj_idx);
995 : : typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
996 : :
997 : : /**
998 : : * A memory callback function for mempool.
999 : : *
1000 : : * Used by rte_mempool_mem_iter().
1001 : : */
1002 : : typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
1003 : : void *opaque, struct rte_mempool_memhdr *memhdr,
1004 : : unsigned mem_idx);
1005 : :
1006 : : /**
1007 : : * A mempool constructor callback function.
1008 : : *
1009 : : * Arguments are the mempool and the opaque pointer given by the user in
1010 : : * rte_mempool_create().
1011 : : */
1012 : : typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
1013 : :
1014 : : /**
1015 : : * Create a new mempool named *name* in memory.
1016 : : *
1017 : : * This function uses ``rte_memzone_reserve()`` to allocate memory. The
1018 : : * pool contains n elements of elt_size. Its size is set to n.
1019 : : *
1020 : : * @param name
1021 : : * The name of the mempool.
1022 : : * @param n
1023 : : * The number of elements in the mempool. The optimum size (in terms of
1024 : : * memory usage) for a mempool is when n is a power of two minus one:
1025 : : * n = (2^q - 1).
1026 : : * @param elt_size
1027 : : * The size of each element.
1028 : : * @param cache_size
1029 : : * If cache_size is non-zero, the rte_mempool library will try to
1030 : : * limit the accesses to the common lockless pool, by maintaining a
1031 : : * per-lcore object cache. This argument must be lower or equal to
1032 : : * RTE_MEMPOOL_CACHE_MAX_SIZE and n / 1.5. It is advised to choose
1033 : : * cache_size to have "n modulo cache_size == 0": if this is
1034 : : * not the case, some elements will always stay in the pool and will
1035 : : * never be used. The access to the per-lcore table is of course
1036 : : * faster than the multi-producer/consumer pool. The cache can be
1037 : : * disabled if the cache_size argument is set to 0; it can be useful to
1038 : : * avoid losing objects in cache.
1039 : : * @param private_data_size
1040 : : * The size of the private data appended after the mempool
1041 : : * structure. This is useful for storing some private data after the
1042 : : * mempool structure, as is done for rte_mbuf_pool for example.
1043 : : * @param mp_init
1044 : : * A function pointer that is called for initialization of the pool,
1045 : : * before object initialization. The user can initialize the private
1046 : : * data in this function if needed. This parameter can be NULL if
1047 : : * not needed.
1048 : : * @param mp_init_arg
1049 : : * An opaque pointer to data that can be used in the mempool
1050 : : * constructor function.
1051 : : * @param obj_init
1052 : : * A function pointer that is called for each object at
1053 : : * initialization of the pool. The user can set some meta data in
1054 : : * objects if needed. This parameter can be NULL if not needed.
1055 : : * The obj_init() function takes the mempool pointer, the init_arg,
1056 : : * the object pointer and the object number as parameters.
1057 : : * @param obj_init_arg
1058 : : * An opaque pointer to data that can be used as an argument for
1059 : : * each call to the object constructor function.
1060 : : * @param socket_id
1061 : : * The *socket_id* argument is the socket identifier in the case of
1062 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1063 : : * constraint for the reserved zone.
1064 : : * @param flags
1065 : : * The *flags* arguments is an OR of following flags:
1066 : : * - RTE_MEMPOOL_F_NO_SPREAD: By default, objects addresses are spread
1067 : : * between channels in RAM: the pool allocator will add padding
1068 : : * between objects depending on the hardware configuration. See
1069 : : * Memory alignment constraints for details. If this flag is set,
1070 : : * the allocator will just align them to a cache line.
1071 : : * - RTE_MEMPOOL_F_NO_CACHE_ALIGN: By default, the returned objects are
1072 : : * cache-aligned. This flag removes this constraint, and no
1073 : : * padding will be present between objects. This flag implies
1074 : : * RTE_MEMPOOL_F_NO_SPREAD.
1075 : : * - RTE_MEMPOOL_F_SP_PUT: If this flag is set, the default behavior
1076 : : * when using rte_mempool_put() or rte_mempool_put_bulk() is
1077 : : * "single-producer". Otherwise, it is "multi-producers".
1078 : : * - RTE_MEMPOOL_F_SC_GET: If this flag is set, the default behavior
1079 : : * when using rte_mempool_get() or rte_mempool_get_bulk() is
1080 : : * "single-consumer". Otherwise, it is "multi-consumers".
1081 : : * - RTE_MEMPOOL_F_NO_IOVA_CONTIG: If set, allocated objects won't
1082 : : * necessarily be contiguous in IO memory.
1083 : : * @return
1084 : : * The pointer to the new allocated mempool, on success. NULL on error
1085 : : * with rte_errno set appropriately. Possible rte_errno values include:
1086 : : * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
1087 : : * - EINVAL - cache size provided is too large or an unknown flag was passed
1088 : : * - ENOSPC - the maximum number of memzones has already been allocated
1089 : : * - EEXIST - a memzone with the same name already exists
1090 : : * - ENOMEM - no appropriate memory area found in which to create memzone
1091 : : */
1092 : : struct rte_mempool *
1093 : : rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1094 : : unsigned cache_size, unsigned private_data_size,
1095 : : rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1096 : : rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1097 : : int socket_id, unsigned flags);
1098 : :
1099 : : /**
1100 : : * Create an empty mempool
1101 : : *
1102 : : * The mempool is allocated and initialized, but it is not populated: no
1103 : : * memory is allocated for the mempool elements. The user has to call
1104 : : * rte_mempool_populate_*() to add memory chunks to the pool. Once
1105 : : * populated, the user may also want to initialize each object with
1106 : : * rte_mempool_obj_iter().
1107 : : *
1108 : : * @param name
1109 : : * The name of the mempool.
1110 : : * @param n
1111 : : * The maximum number of elements that can be added in the mempool.
1112 : : * The optimum size (in terms of memory usage) for a mempool is when n
1113 : : * is a power of two minus one: n = (2^q - 1).
1114 : : * @param elt_size
1115 : : * The size of each element.
1116 : : * @param cache_size
1117 : : * Size of the cache. See rte_mempool_create() for details.
1118 : : * @param private_data_size
1119 : : * The size of the private data appended after the mempool
1120 : : * structure. This is useful for storing some private data after the
1121 : : * mempool structure, as is done for rte_mbuf_pool for example.
1122 : : * @param socket_id
1123 : : * The *socket_id* argument is the socket identifier in the case of
1124 : : * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
1125 : : * constraint for the reserved zone.
1126 : : * @param flags
1127 : : * Flags controlling the behavior of the mempool. See
1128 : : * rte_mempool_create() for details.
1129 : : * @return
1130 : : * The pointer to the new allocated mempool, on success. NULL on error
1131 : : * with rte_errno set appropriately. See rte_mempool_create() for details.
1132 : : */
1133 : : struct rte_mempool *
1134 : : rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1135 : : unsigned cache_size, unsigned private_data_size,
1136 : : int socket_id, unsigned flags);
1137 : : /**
1138 : : * Free a mempool
1139 : : *
1140 : : * Unlink the mempool from global list, free the memory chunks, and all
1141 : : * memory referenced by the mempool. The objects must not be used by
1142 : : * other cores as they will be freed.
1143 : : *
1144 : : * @param mp
1145 : : * A pointer to the mempool structure.
1146 : : * If NULL then, the function does nothing.
1147 : : */
1148 : : void
1149 : : rte_mempool_free(struct rte_mempool *mp);
1150 : :
1151 : : /**
1152 : : * Add physically contiguous memory for objects in the pool at init
1153 : : *
1154 : : * Add a virtually and physically contiguous memory chunk in the pool
1155 : : * where objects can be instantiated.
1156 : : *
1157 : : * If the given IO address is unknown (iova = RTE_BAD_IOVA),
1158 : : * the chunk doesn't need to be physically contiguous (only virtually),
1159 : : * and allocated objects may span two pages.
1160 : : *
1161 : : * @param mp
1162 : : * A pointer to the mempool structure.
1163 : : * @param vaddr
1164 : : * The virtual address of memory that should be used to store objects.
1165 : : * @param iova
1166 : : * The IO address
1167 : : * @param len
1168 : : * The length of memory in bytes.
1169 : : * @param free_cb
1170 : : * The callback used to free this chunk when destroying the mempool.
1171 : : * @param opaque
1172 : : * An opaque argument passed to free_cb.
1173 : : * @return
1174 : : * The number of objects added on success (strictly positive).
1175 : : * On error, the chunk is not added in the memory list of the
1176 : : * mempool the following code is returned:
1177 : : * (0): not enough room in chunk for one object.
1178 : : * (-ENOSPC): mempool is already populated.
1179 : : * (-ENOMEM): allocation failure.
1180 : : */
1181 : : int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1182 : : rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1183 : : void *opaque);
1184 : :
1185 : : /**
1186 : : * Add virtually contiguous memory for objects in the pool at init
1187 : : *
1188 : : * Add a virtually contiguous memory chunk in the pool where objects can
1189 : : * be instantiated.
1190 : : *
1191 : : * @param mp
1192 : : * A pointer to the mempool structure.
1193 : : * @param addr
1194 : : * The virtual address of memory that should be used to store objects.
1195 : : * @param len
1196 : : * The length of memory in bytes.
1197 : : * @param pg_sz
1198 : : * The size of memory pages in this virtual area.
1199 : : * @param free_cb
1200 : : * The callback used to free this chunk when destroying the mempool.
1201 : : * @param opaque
1202 : : * An opaque argument passed to free_cb.
1203 : : * @return
1204 : : * The number of objects added on success (strictly positive).
1205 : : * On error, the chunk is not added in the memory list of the
1206 : : * mempool the following code is returned:
1207 : : * (0): not enough room in chunk for one object.
1208 : : * (-ENOSPC): mempool is already populated.
1209 : : * (-ENOMEM): allocation failure.
1210 : : */
1211 : : int
1212 : : rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1213 : : size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1214 : : void *opaque);
1215 : :
1216 : : /**
1217 : : * Add memory for objects in the pool at init
1218 : : *
1219 : : * This is the default function used by rte_mempool_create() to populate
1220 : : * the mempool. It adds memory allocated using rte_memzone_reserve().
1221 : : *
1222 : : * @param mp
1223 : : * A pointer to the mempool structure.
1224 : : * @return
1225 : : * The number of objects added on success.
1226 : : * On error, the chunk is not added in the memory list of the
1227 : : * mempool and a negative errno is returned.
1228 : : */
1229 : : int rte_mempool_populate_default(struct rte_mempool *mp);
1230 : :
1231 : : /**
1232 : : * Add memory from anonymous mapping for objects in the pool at init
1233 : : *
1234 : : * This function mmap an anonymous memory zone that is locked in
1235 : : * memory to store the objects of the mempool.
1236 : : *
1237 : : * @param mp
1238 : : * A pointer to the mempool structure.
1239 : : * @return
1240 : : * The number of objects added on success.
1241 : : * On error, 0 is returned, rte_errno is set, and the chunk is not added in
1242 : : * the memory list of the mempool.
1243 : : */
1244 : : int rte_mempool_populate_anon(struct rte_mempool *mp);
1245 : :
1246 : : /**
1247 : : * Call a function for each mempool element
1248 : : *
1249 : : * Iterate across all objects attached to a rte_mempool and call the
1250 : : * callback function on it.
1251 : : *
1252 : : * @param mp
1253 : : * A pointer to an initialized mempool.
1254 : : * @param obj_cb
1255 : : * A function pointer that is called for each object.
1256 : : * @param obj_cb_arg
1257 : : * An opaque pointer passed to the callback function.
1258 : : * @return
1259 : : * Number of objects iterated.
1260 : : */
1261 : : uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1262 : : rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1263 : :
1264 : : /**
1265 : : * Call a function for each mempool memory chunk
1266 : : *
1267 : : * Iterate across all memory chunks attached to a rte_mempool and call
1268 : : * the callback function on it.
1269 : : *
1270 : : * @param mp
1271 : : * A pointer to an initialized mempool.
1272 : : * @param mem_cb
1273 : : * A function pointer that is called for each memory chunk.
1274 : : * @param mem_cb_arg
1275 : : * An opaque pointer passed to the callback function.
1276 : : * @return
1277 : : * Number of memory chunks iterated.
1278 : : */
1279 : : uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1280 : : rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1281 : :
1282 : : /**
1283 : : * Dump the status of the mempool to a file.
1284 : : *
1285 : : * @param f
1286 : : * A pointer to a file for output
1287 : : * @param mp
1288 : : * A pointer to the mempool structure.
1289 : : */
1290 : : void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1291 : :
1292 : : /**
1293 : : * Create a user-owned mempool cache.
1294 : : *
1295 : : * This can be used by unregistered non-EAL threads to enable caching when they
1296 : : * interact with a mempool.
1297 : : *
1298 : : * @param size
1299 : : * The size of the mempool cache. See rte_mempool_create()'s cache_size
1300 : : * parameter description for more information. The same limits and
1301 : : * considerations apply here too.
1302 : : * @param socket_id
1303 : : * The socket identifier in the case of NUMA. The value can be
1304 : : * SOCKET_ID_ANY if there is no NUMA constraint for the reserved zone.
1305 : : */
1306 : : struct rte_mempool_cache *
1307 : : rte_mempool_cache_create(uint32_t size, int socket_id);
1308 : :
1309 : : /**
1310 : : * Free a user-owned mempool cache.
1311 : : *
1312 : : * @param cache
1313 : : * A pointer to the mempool cache.
1314 : : */
1315 : : void
1316 : : rte_mempool_cache_free(struct rte_mempool_cache *cache);
1317 : :
1318 : : /**
1319 : : * Get a pointer to the per-lcore default mempool cache.
1320 : : *
1321 : : * @param mp
1322 : : * A pointer to the mempool structure.
1323 : : * @param lcore_id
1324 : : * The logical core id.
1325 : : * @return
1326 : : * A pointer to the mempool cache or NULL if disabled or unregistered non-EAL
1327 : : * thread.
1328 : : */
1329 : : static __rte_always_inline struct rte_mempool_cache *
1330 : : rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1331 : : {
1332 [ + + + + : 2739022 : if (mp->cache_size == 0)
+ + + + +
+ + + + -
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1333 : : return NULL;
1334 : :
1335 [ + + + - : 2172555 : if (lcore_id >= RTE_MAX_LCORE)
+ + + - +
- + - + -
- - - - -
- - - + -
+ - - - -
- - - + -
+ - - - -
- + - + -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
1336 : : return NULL;
1337 : :
1338 : : rte_mempool_trace_default_cache(mp, lcore_id,
1339 [ # # # # : 2172543 : &mp->local_cache[lcore_id]);
# # ]
1340 : 2172544 : return &mp->local_cache[lcore_id];
1341 : : }
1342 : :
1343 : : /**
1344 : : * Flush a user-owned mempool cache to the specified mempool.
1345 : : *
1346 : : * @param cache
1347 : : * A pointer to the mempool cache.
1348 : : * @param mp
1349 : : * A pointer to the mempool.
1350 : : */
1351 : : static __rte_always_inline void
1352 : : rte_mempool_cache_flush(struct rte_mempool_cache *cache,
1353 : : struct rte_mempool *mp)
1354 : : {
1355 [ - + ]: 3 : if (cache == NULL)
1356 : : cache = rte_mempool_default_cache(mp, rte_lcore_id());
1357 [ + - + - ]: 3 : if (cache == NULL || cache->len == 0)
1358 : : return;
1359 : : rte_mempool_trace_cache_flush(cache, mp);
1360 : 3 : rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1361 : 3 : cache->len = 0;
1362 : : }
1363 : :
1364 : : /**
1365 : : * @internal Put several objects back in the mempool; used internally.
1366 : : * @param mp
1367 : : * A pointer to the mempool structure.
1368 : : * @param obj_table
1369 : : * A pointer to a table of void * pointers (objects).
1370 : : * @param n
1371 : : * The number of objects to store back in the mempool, must be strictly
1372 : : * positive.
1373 : : * @param cache
1374 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1375 : : */
1376 : : static __rte_always_inline void
1377 : : rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
1378 : : unsigned int n, struct rte_mempool_cache *cache)
1379 : : {
1380 : : void **cache_objs;
1381 : :
1382 : : /* No cache provided */
1383 [ + + + + : 2390795 : if (unlikely(cache == NULL))
+ + - + +
+ + + + +
- - - - -
+ - - - -
- + - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1384 : 284196 : goto driver_enqueue;
1385 : :
1386 : : /* increment stat now, adding in mempool always success */
1387 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_bulk, 1);
1388 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, put_objs, n);
1389 : :
1390 : : /* The request itself is too big for the cache */
1391 [ - + - + : 2106599 : if (unlikely(n > cache->flushthresh))
- + - + -
+ - + - +
- - - - -
+ - - - -
+ - - + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1392 : 2 : goto driver_enqueue_stats_incremented;
1393 : :
1394 : : /*
1395 : : * The cache follows the following algorithm:
1396 : : * 1. If the objects cannot be added to the cache without crossing
1397 : : * the flush threshold, flush the cache to the backend.
1398 : : * 2. Add the objects to the cache.
1399 : : */
1400 : :
1401 [ + + + + : 2106597 : if (cache->len + n <= cache->flushthresh) {
+ - + - +
- + - + +
- - - - +
+ - - - -
- - + + -
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1402 : 2062864 : cache_objs = &cache->objs[cache->len];
1403 : 2062864 : cache->len += n;
1404 : : } else {
1405 : 43733 : cache_objs = &cache->objs[0];
1406 : 43733 : rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
1407 : 43733 : cache->len = n;
1408 : : }
1409 : :
1410 : : /* Add the objects to the cache. */
1411 [ + + # # : 4 : rte_memcpy(cache_objs, obj_table, sizeof(void *) * n);
# # # # #
# # # ]
1412 : :
1413 : : return;
1414 : :
1415 : : driver_enqueue:
1416 : :
1417 : : /* increment stat now, adding in mempool always success */
1418 : : RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
1419 : : RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
1420 : :
1421 : 284198 : driver_enqueue_stats_incremented:
1422 : :
1423 : : /* push objects to the backend */
1424 : 284198 : rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1425 : : }
1426 : :
1427 : :
1428 : : /**
1429 : : * Put several objects back in the mempool.
1430 : : *
1431 : : * @param mp
1432 : : * A pointer to the mempool structure.
1433 : : * @param obj_table
1434 : : * A pointer to a table of void * pointers (objects).
1435 : : * @param n
1436 : : * The number of objects to add in the mempool from the obj_table.
1437 : : * @param cache
1438 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1439 : : */
1440 : : static __rte_always_inline void
1441 : : rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1442 : : unsigned int n, struct rte_mempool_cache *cache)
1443 : : {
1444 : : rte_mempool_trace_generic_put(mp, obj_table, n, cache);
1445 : : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
1446 : : rte_mempool_do_generic_put(mp, obj_table, n, cache);
1447 : : }
1448 : :
1449 : : /**
1450 : : * Put several objects back in the mempool.
1451 : : *
1452 : : * This function calls the multi-producer or the single-producer
1453 : : * version depending on the default behavior that was specified at
1454 : : * mempool creation time (see flags).
1455 : : *
1456 : : * @param mp
1457 : : * A pointer to the mempool structure.
1458 : : * @param obj_table
1459 : : * A pointer to a table of void * pointers (objects).
1460 : : * @param n
1461 : : * The number of objects to add in the mempool from obj_table.
1462 : : */
1463 : : static __rte_always_inline void
1464 : : rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1465 : : unsigned int n)
1466 : : {
1467 : : struct rte_mempool_cache *cache;
1468 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
1469 : : rte_mempool_trace_put_bulk(mp, obj_table, n, cache);
1470 : : rte_mempool_generic_put(mp, obj_table, n, cache);
1471 : : }
1472 : :
1473 : : /**
1474 : : * Put one object back in the mempool.
1475 : : *
1476 : : * This function calls the multi-producer or the single-producer
1477 : : * version depending on the default behavior that was specified at
1478 : : * mempool creation time (see flags).
1479 : : *
1480 : : * @param mp
1481 : : * A pointer to the mempool structure.
1482 : : * @param obj
1483 : : * A pointer to the object to be added.
1484 : : */
1485 : : static __rte_always_inline void
1486 : : rte_mempool_put(struct rte_mempool *mp, void *obj)
1487 : : {
1488 : : rte_mempool_put_bulk(mp, &obj, 1);
1489 : : }
1490 : :
1491 : : /**
1492 : : * @internal Get several objects from the mempool; used internally.
1493 : : * @param mp
1494 : : * A pointer to the mempool structure.
1495 : : * @param obj_table
1496 : : * A pointer to a table of void * pointers (objects).
1497 : : * @param n
1498 : : * The number of objects to get, must be strictly positive.
1499 : : * @param cache
1500 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1501 : : * @return
1502 : : * - 0: Success.
1503 : : * - <0: Error; code of driver dequeue function.
1504 : : */
1505 : : static __rte_always_inline int
1506 : : rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
1507 : : unsigned int n, struct rte_mempool_cache *cache)
1508 : : {
1509 : : int ret;
1510 : : unsigned int remaining;
1511 : : uint32_t index, len;
1512 : : void **cache_objs;
1513 : :
1514 : : /* No cache provided */
1515 [ + + + + : 358805 : if (unlikely(cache == NULL)) {
+ - + + +
+ + + + +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1516 : : remaining = n;
1517 : 284397 : goto driver_dequeue;
1518 : : }
1519 : :
1520 : : /* The cache is a stack, so copy will be in reverse order. */
1521 : 74408 : cache_objs = &cache->objs[cache->len];
1522 : :
1523 [ + + - + : 74408 : if (__extension__(__builtin_constant_p(n)) && n <= cache->len) {
+ + - + +
- + - + +
+ + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
1524 : : /*
1525 : : * The request size is known at build time, and
1526 : : * the entire request can be satisfied from the cache,
1527 : : * so let the compiler unroll the fixed length copy loop.
1528 : : */
1529 : 41144 : cache->len -= n;
1530 [ + + + + : 1098127 : for (index = 0; index < n; index++)
- - - - +
+ + + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1531 : 1056983 : *obj_table++ = *--cache_objs;
1532 : :
1533 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1534 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1535 : :
1536 : : return 0;
1537 : : }
1538 : :
1539 : : /*
1540 : : * Use the cache as much as we have to return hot objects first.
1541 : : * If the request size 'n' is known at build time, the above comparison
1542 : : * ensures that n > cache->len here, so omit RTE_MIN().
1543 : : */
1544 [ + + # # : 30 : len = __extension__(__builtin_constant_p(n)) ? cache->len :
# # # # ]
1545 : 28 : RTE_MIN(n, cache->len);
1546 : 33264 : cache->len -= len;
1547 : 32801 : remaining = n - len;
1548 [ + + - - : 33349 : for (index = 0; index < len; index++)
- - - - -
- + + - +
+ + # # #
# # # #
# ]
1549 : 548 : *obj_table++ = *--cache_objs;
1550 : :
1551 : : /*
1552 : : * If the request size 'n' is known at build time, the case
1553 : : * where the entire request can be satisfied from the cache
1554 : : * has already been handled above, so omit handling it here.
1555 : : */
1556 [ + + + + : 30 : if (!__extension__(__builtin_constant_p(n)) && remaining == 0) {
# # # # #
# # # # #
# # ]
1557 : : /* The entire request is satisfied from the cache. */
1558 : :
1559 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1560 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1561 : :
1562 : : return 0;
1563 : : }
1564 : :
1565 : : /* if dequeue below would overflow mem allocated for cache */
1566 [ + + + - : 28 : if (unlikely(remaining > RTE_MEMPOOL_CACHE_MAX_SIZE))
# # # # ]
1567 : 8 : goto driver_dequeue;
1568 : :
1569 : : /* Fill the cache from the backend; fetch size + remaining objects. */
1570 : 33252 : ret = rte_mempool_ops_dequeue_bulk(mp, cache->objs,
1571 : 33252 : cache->size + remaining);
1572 [ + + + + : 33252 : if (unlikely(ret < 0)) {
- - - + -
- - + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1573 : : /*
1574 : : * We are buffer constrained, and not able to allocate
1575 : : * cache + remaining.
1576 : : * Do not fill the cache, just satisfy the remaining part of
1577 : : * the request directly from the backend.
1578 : : */
1579 : 424 : goto driver_dequeue;
1580 : : }
1581 : :
1582 : : /* Satisfy the remaining part of the request from the filled cache. */
1583 : 32828 : cache_objs = &cache->objs[cache->size + remaining];
1584 [ + + + + : 1081817 : for (index = 0; index < remaining; index++)
- - + + -
- + + + +
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1585 : 1048989 : *obj_table++ = *--cache_objs;
1586 : :
1587 : 32814 : cache->len = cache->size;
1588 : :
1589 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1590 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1591 : :
1592 : 16 : return 0;
1593 : :
1594 : 284829 : driver_dequeue:
1595 : :
1596 : : /* Get remaining objects directly from the backend. */
1597 : 284829 : ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, remaining);
1598 : :
1599 [ + + + + : 284829 : if (ret < 0) {
- + - + -
+ - + - +
- + # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1600 [ + + + + : 18 : if (likely(cache != NULL)) {
- - - - -
- - - - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1601 : 16 : cache->len = n - remaining;
1602 : : /*
1603 : : * No further action is required to roll the first part
1604 : : * of the request back into the cache, as objects in
1605 : : * the cache are intact.
1606 : : */
1607 : : }
1608 : :
1609 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1610 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
1611 : : } else {
1612 : : if (likely(cache != NULL)) {
1613 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_bulk, 1);
1614 : : RTE_MEMPOOL_CACHE_STAT_ADD(cache, get_success_objs, n);
1615 : : } else {
1616 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1617 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
1618 : : }
1619 : : }
1620 : :
1621 : : return ret;
1622 : : }
1623 : :
1624 : : /**
1625 : : * Get several objects from the mempool.
1626 : : *
1627 : : * If cache is enabled, objects will be retrieved first from cache,
1628 : : * subsequently from the common pool. Note that it can return -ENOENT when
1629 : : * the local cache and common pool are empty, even if cache from other
1630 : : * lcores are full.
1631 : : *
1632 : : * @param mp
1633 : : * A pointer to the mempool structure.
1634 : : * @param obj_table
1635 : : * A pointer to a table of void * pointers (objects) that will be filled.
1636 : : * @param n
1637 : : * The number of objects to get from mempool to obj_table.
1638 : : * @param cache
1639 : : * A pointer to a mempool cache structure. May be NULL if not needed.
1640 : : * @return
1641 : : * - 0: Success; objects taken.
1642 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1643 : : */
1644 : : static __rte_always_inline int
1645 : : rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1646 : : unsigned int n, struct rte_mempool_cache *cache)
1647 : : {
1648 : : int ret;
1649 : : ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
1650 : : if (ret == 0)
1651 : : RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
1652 : : rte_mempool_trace_generic_get(mp, obj_table, n, cache);
1653 : : return ret;
1654 : : }
1655 : :
1656 : : /**
1657 : : * Get several objects from the mempool.
1658 : : *
1659 : : * This function calls the multi-consumers or the single-consumer
1660 : : * version, depending on the default behaviour that was specified at
1661 : : * mempool creation time (see flags).
1662 : : *
1663 : : * If cache is enabled, objects will be retrieved first from cache,
1664 : : * subsequently from the common pool. Note that it can return -ENOENT when
1665 : : * the local cache and common pool are empty, even if cache from other
1666 : : * lcores are full.
1667 : : *
1668 : : * @param mp
1669 : : * A pointer to the mempool structure.
1670 : : * @param obj_table
1671 : : * A pointer to a table of void * pointers (objects) that will be filled.
1672 : : * @param n
1673 : : * The number of objects to get from the mempool to obj_table.
1674 : : * @return
1675 : : * - 0: Success; objects taken
1676 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1677 : : */
1678 : : static __rte_always_inline int
1679 : : rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1680 : : {
1681 : : struct rte_mempool_cache *cache;
1682 [ # # # # : 0 : cache = rte_mempool_default_cache(mp, rte_lcore_id());
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1683 : : rte_mempool_trace_get_bulk(mp, obj_table, n, cache);
1684 : : return rte_mempool_generic_get(mp, obj_table, n, cache);
1685 : : }
1686 : :
1687 : : /**
1688 : : * Get one object from the mempool.
1689 : : *
1690 : : * This function calls the multi-consumers or the single-consumer
1691 : : * version, depending on the default behavior that was specified at
1692 : : * mempool creation (see flags).
1693 : : *
1694 : : * If cache is enabled, objects will be retrieved first from cache,
1695 : : * subsequently from the common pool. Note that it can return -ENOENT when
1696 : : * the local cache and common pool are empty, even if cache from other
1697 : : * lcores are full.
1698 : : *
1699 : : * @param mp
1700 : : * A pointer to the mempool structure.
1701 : : * @param obj_p
1702 : : * A pointer to a void * pointer (object) that will be filled.
1703 : : * @return
1704 : : * - 0: Success; objects taken.
1705 : : * - -ENOENT: Not enough entries in the mempool; no object is retrieved.
1706 : : */
1707 : : static __rte_always_inline int
1708 : : rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1709 : : {
1710 : : return rte_mempool_get_bulk(mp, obj_p, 1);
1711 : : }
1712 : :
1713 : : /**
1714 : : * Get a contiguous blocks of objects from the mempool.
1715 : : *
1716 : : * If cache is enabled, consider to flush it first, to reuse objects
1717 : : * as soon as possible.
1718 : : *
1719 : : * The application should check that the driver supports the operation
1720 : : * by calling rte_mempool_ops_get_info() and checking that `contig_block_size`
1721 : : * is not zero.
1722 : : *
1723 : : * @param mp
1724 : : * A pointer to the mempool structure.
1725 : : * @param first_obj_table
1726 : : * A pointer to a pointer to the first object in each block.
1727 : : * @param n
1728 : : * The number of blocks to get from mempool.
1729 : : * @return
1730 : : * - 0: Success; blocks taken.
1731 : : * - -ENOBUFS: Not enough entries in the mempool; no object is retrieved.
1732 : : * - -EOPNOTSUPP: The mempool driver does not support block dequeue
1733 : : */
1734 : : static __rte_always_inline int
1735 : : rte_mempool_get_contig_blocks(struct rte_mempool *mp,
1736 : : void **first_obj_table, unsigned int n)
1737 : : {
1738 : : int ret;
1739 : :
1740 : 0 : ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1741 : : if (ret == 0) {
1742 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
1743 : : RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
1744 : : RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1745 : : 1);
1746 : : } else {
1747 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
1748 : : RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
1749 : : }
1750 : :
1751 : : rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);
1752 : : return ret;
1753 : : }
1754 : :
1755 : : /**
1756 : : * Return the number of entries in the mempool.
1757 : : *
1758 : : * When cache is enabled, this function has to browse the length of
1759 : : * all lcores, so it should not be used in a data path, but only for
1760 : : * debug purposes. User-owned mempool caches are not accounted for.
1761 : : *
1762 : : * @param mp
1763 : : * A pointer to the mempool structure.
1764 : : * @return
1765 : : * The number of entries in the mempool.
1766 : : */
1767 : : unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1768 : :
1769 : : /**
1770 : : * Return the number of elements which have been allocated from the mempool
1771 : : *
1772 : : * When cache is enabled, this function has to browse the length of
1773 : : * all lcores, so it should not be used in a data path, but only for
1774 : : * debug purposes.
1775 : : *
1776 : : * @param mp
1777 : : * A pointer to the mempool structure.
1778 : : * @return
1779 : : * The number of free entries in the mempool.
1780 : : */
1781 : : unsigned int
1782 : : rte_mempool_in_use_count(const struct rte_mempool *mp);
1783 : :
1784 : : /**
1785 : : * Test if the mempool is full.
1786 : : *
1787 : : * When cache is enabled, this function has to browse the length of all
1788 : : * lcores, so it should not be used in a data path, but only for debug
1789 : : * purposes. User-owned mempool caches are not accounted for.
1790 : : *
1791 : : * @param mp
1792 : : * A pointer to the mempool structure.
1793 : : * @return
1794 : : * - 1: The mempool is full.
1795 : : * - 0: The mempool is not full.
1796 : : */
1797 : : static inline int
1798 : : rte_mempool_full(const struct rte_mempool *mp)
1799 : : {
1800 [ + + - + : 6 : return rte_mempool_avail_count(mp) == mp->size;
- + - + +
- - + ]
1801 : : }
1802 : :
1803 : : /**
1804 : : * Test if the mempool is empty.
1805 : : *
1806 : : * When cache is enabled, this function has to browse the length of all
1807 : : * lcores, so it should not be used in a data path, but only for debug
1808 : : * purposes. User-owned mempool caches are not accounted for.
1809 : : *
1810 : : * @param mp
1811 : : * A pointer to the mempool structure.
1812 : : * @return
1813 : : * - 1: The mempool is empty.
1814 : : * - 0: The mempool is not empty.
1815 : : */
1816 : : static inline int
1817 : : rte_mempool_empty(const struct rte_mempool *mp)
1818 : : {
1819 : 5 : return rte_mempool_avail_count(mp) == 0;
1820 : : }
1821 : :
1822 : : /**
1823 : : * Return the IO address of elt, which is an element of the pool mp.
1824 : : *
1825 : : * @param elt
1826 : : * A pointer (virtual address) to the element of the pool.
1827 : : * @return
1828 : : * The IO address of the elt element.
1829 : : * If the mempool was created with RTE_MEMPOOL_F_NO_IOVA_CONTIG, the
1830 : : * returned value is RTE_BAD_IOVA.
1831 : : */
1832 : : static inline rte_iova_t
1833 : 0 : rte_mempool_virt2iova(const void *elt)
1834 : : {
1835 : : const struct rte_mempool_objhdr *hdr;
1836 : 75457 : hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1837 : : sizeof(*hdr));
1838 [ # # ]: 95 : return hdr->iova;
1839 : : }
1840 : :
1841 : : /**
1842 : : * Check the consistency of mempool objects.
1843 : : *
1844 : : * Verify the coherency of fields in the mempool structure. Also check
1845 : : * that the cookies of mempool objects (even the ones that are not
1846 : : * present in pool) have a correct value. If not, a panic will occur.
1847 : : *
1848 : : * @param mp
1849 : : * A pointer to the mempool structure.
1850 : : */
1851 : : void rte_mempool_audit(struct rte_mempool *mp);
1852 : :
1853 : : /**
1854 : : * Return a pointer to the private data in an mempool structure.
1855 : : *
1856 : : * @param mp
1857 : : * A pointer to the mempool structure.
1858 : : * @return
1859 : : * A pointer to the private data.
1860 : : */
1861 : : static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1862 : : {
1863 [ + + - - : 2445 : return (char *)mp +
- + - + +
+ + + + -
- + + + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
1864 [ + - - - : 77828 : RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size);
+ - + - +
- + + + +
- - + - +
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
1865 : : }
1866 : :
1867 : : /**
1868 : : * Dump the status of all mempools on the console
1869 : : *
1870 : : * @param f
1871 : : * A pointer to a file for output
1872 : : */
1873 : : void rte_mempool_list_dump(FILE *f);
1874 : :
1875 : : /**
1876 : : * Search a mempool from its name
1877 : : *
1878 : : * @param name
1879 : : * The name of the mempool.
1880 : : * @return
1881 : : * The pointer to the mempool matching the name, or NULL if not found.
1882 : : * NULL on error
1883 : : * with rte_errno set appropriately. Possible rte_errno values include:
1884 : : * - ENOENT - required entry not available to return.
1885 : : */
1886 : : struct rte_mempool *rte_mempool_lookup(const char *name);
1887 : :
1888 : : /**
1889 : : * Get the header, trailer and total size of a mempool element.
1890 : : *
1891 : : * Given a desired size of the mempool element and mempool flags,
1892 : : * calculates header, trailer, body and total sizes of the mempool object.
1893 : : *
1894 : : * @param elt_size
1895 : : * The size of each element, without header and trailer.
1896 : : * @param flags
1897 : : * The flags used for the mempool creation.
1898 : : * Consult rte_mempool_create() for more information about possible values.
1899 : : * The size of each element.
1900 : : * @param sz
1901 : : * The calculated detailed size the mempool object. May be NULL.
1902 : : * @return
1903 : : * Total size of the mempool object.
1904 : : */
1905 : : uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1906 : : struct rte_mempool_objsz *sz);
1907 : :
1908 : : /**
1909 : : * Walk list of all memory pools
1910 : : *
1911 : : * @param func
1912 : : * Iterator function
1913 : : * @param arg
1914 : : * Argument passed to iterator
1915 : : */
1916 : : void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1917 : : void *arg);
1918 : :
1919 : : /**
1920 : : * @internal Get page size used for mempool object allocation.
1921 : : * This function is internal to mempool library and mempool drivers.
1922 : : */
1923 : : int
1924 : : rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1925 : :
1926 : : /**
1927 : : * Mempool event type.
1928 : : * @internal
1929 : : */
1930 : : enum rte_mempool_event {
1931 : : /** Occurs after a mempool is fully populated. */
1932 : : RTE_MEMPOOL_EVENT_READY = 0,
1933 : : /** Occurs before the destruction of a mempool begins. */
1934 : : RTE_MEMPOOL_EVENT_DESTROY = 1,
1935 : : };
1936 : :
1937 : : /**
1938 : : * @internal
1939 : : * Mempool event callback.
1940 : : *
1941 : : * rte_mempool_event_callback_register() may be called from within the callback,
1942 : : * but the callbacks registered this way will not be invoked for the same event.
1943 : : * rte_mempool_event_callback_unregister() may only be safely called
1944 : : * to remove the running callback.
1945 : : */
1946 : : typedef void (rte_mempool_event_callback)(
1947 : : enum rte_mempool_event event,
1948 : : struct rte_mempool *mp,
1949 : : void *user_data);
1950 : :
1951 : : /**
1952 : : * @internal
1953 : : * Register a callback function invoked on mempool life cycle event.
1954 : : * The function will be invoked in the process
1955 : : * that performs an action which triggers the callback.
1956 : : * Registration is process-private,
1957 : : * i.e. each process must manage callbacks on its own if needed.
1958 : : *
1959 : : * @param func
1960 : : * Callback function.
1961 : : * @param user_data
1962 : : * User data.
1963 : : *
1964 : : * @return
1965 : : * 0 on success, negative on failure and rte_errno is set.
1966 : : */
1967 : : __rte_internal
1968 : : int
1969 : : rte_mempool_event_callback_register(rte_mempool_event_callback *func,
1970 : : void *user_data);
1971 : :
1972 : : /**
1973 : : * @internal
1974 : : * Unregister a callback added with rte_mempool_event_callback_register().
1975 : : * @p func and @p user_data must exactly match registration parameters.
1976 : : *
1977 : : * @param func
1978 : : * Callback function.
1979 : : * @param user_data
1980 : : * User data.
1981 : : *
1982 : : * @return
1983 : : * 0 on success, negative on failure and rte_errno is set.
1984 : : */
1985 : : __rte_internal
1986 : : int
1987 : : rte_mempool_event_callback_unregister(rte_mempool_event_callback *func,
1988 : : void *user_data);
1989 : :
1990 : : #ifdef __cplusplus
1991 : : }
1992 : : #endif
1993 : :
1994 : : #endif /* _RTE_MEMPOOL_H_ */
|