Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation.
3 : : * Copyright(c) 2013 6WIND S.A.
4 : : */
5 : :
6 : : #include <inttypes.h>
7 : : #include <stdlib.h>
8 : : #include <string.h>
9 : :
10 : : #include <rte_log.h>
11 : : #include <rte_string_fns.h>
12 : :
13 : : #include "eal_internal_cfg.h"
14 : : #include "eal_memalloc.h"
15 : : #include "eal_memcfg.h"
16 : : #include "eal_private.h"
17 : :
18 : : /** @file Functions common to EALs that support dynamic memory allocation. */
19 : :
20 : : int
21 : 191 : eal_dynmem_memseg_lists_init(void)
22 : : {
23 : 191 : struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
24 : : struct memtype {
25 : : uint64_t page_sz;
26 : : int socket_id;
27 : : unsigned int hpi_idx;
28 : : unsigned int n_segs;
29 : : size_t mem_sz;
30 : : size_t va_offset;
31 : 191 : } memtypes[RTE_MAX_MEMSEG_LISTS] = {0};
32 : : int i, hpi_idx, msl_idx, ret = -1; /* fail unless told to succeed */
33 : : struct rte_memseg_list *msl;
34 : : size_t mem_va_len, mem_va_page_sz;
35 : : unsigned int n_memtypes, cur_type;
36 : : void *mem_va_addr = NULL;
37 : : struct internal_config *internal_conf =
38 : 191 : eal_get_internal_configuration();
39 : :
40 : : /* no-huge does not need this at all */
41 [ + + ]: 191 : if (internal_conf->no_hugetlbfs)
42 : : return 0;
43 : :
44 : : /*
45 : : * figuring out amount of memory we're going to have is a long and very
46 : : * involved process. the basic element we're operating with is a memory
47 : : * type, defined as a combination of NUMA node ID and page size (so that
48 : : * e.g. 2 sockets with 2 page sizes yield 4 memory types in total).
49 : : *
50 : : * deciding amount of memory going towards each memory type is a
51 : : * balancing act between maximum segments per type, maximum memory per
52 : : * type, and number of detected NUMA nodes.
53 : : *
54 : : * the total amount of memory per type is limited by per-page-size
55 : : * memory values in internal config. each memory type is allotted one
56 : : * memseg list.
57 : : */
58 : :
59 : : /* maximum number of memtypes we're ever going to get */
60 : 63 : n_memtypes = internal_conf->num_hugepage_sizes * rte_socket_count();
61 : :
62 : : /* can we fit all memtypes into the memseg lists? */
63 [ - + ]: 63 : if (n_memtypes > RTE_MAX_MEMSEG_LISTS) {
64 : 0 : EAL_LOG(ERR, "Too many memory types detected: %u. Please increase RTE_MAX_MEMSEG_LISTS in configuration.",
65 : : n_memtypes);
66 : 0 : return -1;
67 : : }
68 : :
69 : : /* populate mem types */
70 : : cur_type = 0;
71 [ + + ]: 132 : for (hpi_idx = 0; hpi_idx < (int) internal_conf->num_hugepage_sizes;
72 : 69 : hpi_idx++) {
73 : : struct hugepage_info *hpi;
74 : : uint64_t hugepage_sz;
75 : :
76 : : hpi = &internal_conf->hugepage_info[hpi_idx];
77 : 69 : hugepage_sz = hpi->hugepage_sz;
78 : :
79 [ + + ]: 207 : for (i = 0; i < (int) rte_socket_count(); i++, cur_type++) {
80 : 138 : int socket_id = rte_socket_id_by_idx(i);
81 : :
82 : : #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
83 : : /* we can still sort pages by socket in legacy mode */
84 : : if (!internal_conf->legacy_mem && socket_id > 0)
85 : : break;
86 : : #endif
87 : 138 : memtypes[cur_type].page_sz = hugepage_sz;
88 : 138 : memtypes[cur_type].socket_id = socket_id;
89 : 138 : memtypes[cur_type].hpi_idx = hpi_idx;
90 : :
91 : 138 : EAL_LOG(DEBUG, "Detected memory type: "
92 : : "socket_id:%u hugepage_sz:%" PRIu64,
93 : : socket_id, hugepage_sz);
94 : : }
95 : : }
96 : : /* number of memtypes could have been lower due to no NUMA support */
97 : : n_memtypes = cur_type;
98 : :
99 : 63 : mem_va_len = 0;
100 : : mem_va_page_sz = 0;
101 : :
102 : : /* calculate total VA space and offsets for all mem types */
103 [ + + ]: 201 : for (cur_type = 0; cur_type < n_memtypes; cur_type++) {
104 : : unsigned int n_segs;
105 : : struct memtype *type = &memtypes[cur_type];
106 : : uint64_t max_mem_per_type;
107 : : uint64_t pagesz;
108 : :
109 : 138 : pagesz = type->page_sz;
110 : 138 : max_mem_per_type =
111 : 138 : internal_conf->hugepage_mem_sz_limits[type->hpi_idx];
112 : :
113 : : /*
114 : : * we need to create a segment list for this type. we must take
115 : : * into account the following things:
116 : : *
117 : : * 1. total amount of memory to use for this memory type
118 : : * 2. total amount of memory allowed per type
119 : : * 3. number of segments needed to fit the amount of memory
120 : : */
121 : 138 : n_segs = max_mem_per_type / pagesz;
122 : 138 : type->n_segs = n_segs;
123 : 138 : type->mem_sz = (size_t)pagesz * type->n_segs;
124 : 138 : mem_va_page_sz = RTE_MAX(mem_va_page_sz, (size_t)pagesz);
125 : 138 : mem_va_len = RTE_ALIGN_CEIL(mem_va_len, pagesz);
126 : 138 : type->va_offset = mem_va_len;
127 : 138 : mem_va_len += type->mem_sz;
128 : : }
129 : :
130 [ - + ]: 63 : if (mem_va_len == 0) {
131 : 0 : EAL_LOG(ERR, "No virtual memory will be reserved");
132 : 0 : goto out;
133 : : }
134 : :
135 : 63 : mem_va_addr = eal_get_virtual_area(NULL, &mem_va_len,
136 : : mem_va_page_sz, 0, 0);
137 [ - + ]: 63 : if (mem_va_addr == NULL) {
138 : 0 : EAL_LOG(ERR, "Cannot reserve VA space for memseg lists");
139 : 0 : goto out;
140 : : }
141 : :
142 : : /* go through all mem types and create segment lists */
143 : : msl_idx = 0;
144 [ + + ]: 201 : for (cur_type = 0; cur_type < n_memtypes; cur_type++) {
145 : : struct memtype *type = &memtypes[cur_type];
146 : : uint64_t pagesz;
147 : : int socket_id;
148 : :
149 : : /* skip page sizes with zero memory limit */
150 [ - + ]: 138 : if (type->n_segs == 0)
151 : 0 : continue;
152 : :
153 : 138 : pagesz = type->page_sz;
154 : 138 : socket_id = type->socket_id;
155 : :
156 : 138 : EAL_LOG(DEBUG, "Creating segment list: n_segs:%u socket_id:%i hugepage_sz:%" PRIu64,
157 : : type->n_segs, socket_id, pagesz);
158 : :
159 : 138 : msl = &mcfg->memsegs[msl_idx];
160 : :
161 [ - + ]: 138 : if (eal_memseg_list_init(msl, pagesz, type->n_segs, socket_id, msl_idx, true))
162 : 0 : goto out;
163 : :
164 [ - + ]: 138 : if (eal_memseg_list_assign(msl,
165 : 138 : RTE_PTR_ADD(mem_va_addr, type->va_offset))) {
166 : 0 : EAL_LOG(ERR, "Cannot assign VA space for memseg list");
167 : 0 : goto out;
168 : : }
169 : 138 : msl_idx++;
170 : : }
171 : : /* we're successful */
172 : : ret = 0;
173 : 63 : out:
174 : : if (ret != 0) {
175 [ # # ]: 0 : if (mem_va_addr != NULL)
176 : 0 : eal_mem_free(mem_va_addr, mem_va_len);
177 : : } else {
178 : : /* store the VA space data in shared config */
179 : 63 : mcfg->mem_va_addr = (uintptr_t)mem_va_addr;
180 : 63 : mcfg->mem_va_len = mem_va_len;
181 : 63 : mcfg->mem_va_page_sz = mem_va_page_sz;
182 : : }
183 : : return ret;
184 : : }
185 : :
186 : : static int __rte_unused
187 : : hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
188 : : {
189 : : struct hugepage_info *hpi = arg;
190 : :
191 : : if (msl->page_sz != hpi->hugepage_sz)
192 : : return 0;
193 : :
194 : : hpi->num_pages[msl->socket_id] += msl->memseg_arr.len;
195 : : return 0;
196 : : }
197 : :
198 : : static int
199 : 0 : limits_callback(int socket_id, size_t cur_limit, size_t new_len)
200 : : {
201 : : RTE_SET_USED(socket_id);
202 : : RTE_SET_USED(cur_limit);
203 : : RTE_SET_USED(new_len);
204 : 0 : return -1;
205 : : }
206 : :
207 : : int
208 : 61 : eal_dynmem_hugepage_init(void)
209 : : {
210 : : struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
211 : : uint64_t memory[RTE_MAX_NUMA_NODES];
212 : : int hp_sz_idx, socket_id;
213 : : struct internal_config *internal_conf =
214 : 61 : eal_get_internal_configuration();
215 : :
216 : : memset(used_hp, 0, sizeof(used_hp));
217 : :
218 : 61 : for (hp_sz_idx = 0;
219 [ + + ]: 128 : hp_sz_idx < (int) internal_conf->num_hugepage_sizes;
220 : 67 : hp_sz_idx++) {
221 : : #ifndef RTE_ARCH_64
222 : : struct hugepage_info dummy;
223 : : unsigned int i;
224 : : #endif
225 : : /* also initialize used_hp hugepage sizes in used_hp */
226 : : struct hugepage_info *hpi;
227 : : hpi = &internal_conf->hugepage_info[hp_sz_idx];
228 : 67 : used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz;
229 : :
230 : : #ifndef RTE_ARCH_64
231 : : /* for 32-bit, limit number of pages on socket to whatever we've
232 : : * preallocated, as we cannot allocate more.
233 : : */
234 : : memset(&dummy, 0, sizeof(dummy));
235 : : dummy.hugepage_sz = hpi->hugepage_sz;
236 : : /* memory_hotplug_lock is held during initialization, so it's
237 : : * safe to call thread-unsafe version.
238 : : */
239 : : if (rte_memseg_list_walk_thread_unsafe(hugepage_count_walk, &dummy) < 0)
240 : : return -1;
241 : :
242 : : for (i = 0; i < RTE_DIM(dummy.num_pages); i++) {
243 : : hpi->num_pages[i] = RTE_MIN(hpi->num_pages[i],
244 : : dummy.num_pages[i]);
245 : : }
246 : : #endif
247 : : }
248 : :
249 : : /* make a copy of numa_mem, needed for balanced allocation. */
250 [ + + ]: 2013 : for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES; hp_sz_idx++)
251 : 1952 : memory[hp_sz_idx] = internal_conf->numa_mem[hp_sz_idx];
252 : :
253 : : /* calculate final number of pages */
254 [ + + ]: 61 : if (eal_dynmem_calc_num_pages_per_socket(memory,
255 : 61 : internal_conf->hugepage_info, used_hp,
256 : : internal_conf->num_hugepage_sizes) < 0)
257 : : return -1;
258 : :
259 : : for (hp_sz_idx = 0;
260 [ + + ]: 126 : hp_sz_idx < (int)internal_conf->num_hugepage_sizes;
261 : 66 : hp_sz_idx++) {
262 [ + + ]: 2178 : for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES;
263 : 2112 : socket_id++) {
264 : : struct rte_memseg **pages;
265 : : struct hugepage_info *hpi = &used_hp[hp_sz_idx];
266 : 2112 : unsigned int num_pages = hpi->num_pages[socket_id];
267 : : unsigned int num_pages_alloc;
268 : :
269 [ + + ]: 2112 : if (num_pages == 0)
270 : 2100 : continue;
271 : :
272 : 12 : EAL_LOG(DEBUG,
273 : : "Allocating %u pages of size %" PRIu64 "M "
274 : : "on socket %i",
275 : : num_pages, hpi->hugepage_sz >> 20, socket_id);
276 : :
277 : : /* we may not be able to allocate all pages in one go,
278 : : * because we break up our memory map into multiple
279 : : * memseg lists. therefore, try allocating multiple
280 : : * times and see if we can get the desired number of
281 : : * pages from multiple allocations.
282 : : */
283 : :
284 : : num_pages_alloc = 0;
285 : : do {
286 : : int i, cur_pages, needed;
287 : :
288 : 12 : needed = num_pages - num_pages_alloc;
289 : :
290 : 12 : pages = malloc(sizeof(*pages) * needed);
291 [ - + ]: 12 : if (pages == NULL) {
292 : 0 : EAL_LOG(ERR, "Failed to malloc pages");
293 : 0 : return -1;
294 : : }
295 : :
296 : : /* do not request exact number of pages */
297 : 12 : cur_pages = eal_memalloc_alloc_seg_bulk(pages,
298 : : needed, hpi->hugepage_sz,
299 : : socket_id, false);
300 [ - + ]: 12 : if (cur_pages <= 0) {
301 : 0 : free(pages);
302 : 0 : return -1;
303 : : }
304 : :
305 : : /* mark preallocated pages as unfreeable */
306 [ + + ]: 104 : for (i = 0; i < cur_pages; i++) {
307 : 92 : struct rte_memseg *ms = pages[i];
308 : 92 : ms->flags |=
309 : : RTE_MEMSEG_FLAG_DO_NOT_FREE;
310 : : }
311 : 12 : free(pages);
312 : :
313 : 12 : num_pages_alloc += cur_pages;
314 [ - + ]: 12 : } while (num_pages_alloc != num_pages);
315 : : }
316 : : }
317 : :
318 : : /* if socket limits were specified, set them */
319 [ - + ]: 60 : if (internal_conf->force_numa_limits) {
320 : : unsigned int i;
321 [ # # ]: 0 : for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
322 : 0 : uint64_t limit = internal_conf->numa_limit[i];
323 [ # # ]: 0 : if (limit == 0)
324 : 0 : continue;
325 [ # # ]: 0 : if (rte_mem_alloc_validator_register("socket-limit",
326 : : limits_callback, i, limit))
327 : 0 : EAL_LOG(ERR, "Failed to register socket limits validator callback");
328 : : }
329 : : }
330 : : return 0;
331 : : }
332 : :
333 : : __rte_unused /* function is unused on 32-bit builds */
334 : : static inline uint64_t
335 : 12 : get_socket_mem_size(int socket)
336 : : {
337 : : uint64_t size = 0;
338 : : unsigned int i;
339 : : struct internal_config *internal_conf =
340 : 12 : eal_get_internal_configuration();
341 : :
342 [ + + ]: 27 : for (i = 0; i < internal_conf->num_hugepage_sizes; i++) {
343 : : struct hugepage_info *hpi = &internal_conf->hugepage_info[i];
344 : 15 : size += hpi->hugepage_sz * hpi->num_pages[socket];
345 : : }
346 : :
347 : 12 : return size;
348 : : }
349 : :
350 : : int
351 : 63 : eal_dynmem_calc_num_pages_per_socket(
352 : : uint64_t *memory, struct hugepage_info *hp_info,
353 : : struct hugepage_info *hp_used, unsigned int num_hp_info)
354 : : {
355 : : unsigned int socket, j, i = 0;
356 : : unsigned int requested, available;
357 : : int total_num_pages = 0;
358 : : uint64_t remaining_mem, cur_mem;
359 : : const struct internal_config *internal_conf =
360 : 63 : eal_get_internal_configuration();
361 : 63 : uint64_t total_mem = internal_conf->memory;
362 : :
363 [ + - ]: 63 : if (num_hp_info == 0)
364 : : return -1;
365 : :
366 : : /* if specific memory amounts per socket weren't requested */
367 [ + + ]: 63 : if (internal_conf->force_numa == 0) {
368 : : size_t total_size;
369 : : #ifdef RTE_ARCH_64
370 : : int cpu_per_socket[RTE_MAX_NUMA_NODES];
371 : : size_t default_size;
372 : : unsigned int lcore_id;
373 : :
374 : : /* Compute number of cores per socket */
375 : : memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
376 [ + + ]: 159 : RTE_LCORE_FOREACH(lcore_id) {
377 : 99 : cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
378 : : }
379 : :
380 : : /*
381 : : * Automatically spread requested memory amongst detected
382 : : * sockets according to number of cores from CPU mask present
383 : : * on each socket.
384 : : */
385 : 60 : total_size = internal_conf->memory;
386 [ + + ]: 72 : for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
387 : 12 : socket++) {
388 : :
389 : : /* Set memory amount per socket */
390 : 24 : default_size = internal_conf->memory *
391 : 12 : cpu_per_socket[socket] / rte_lcore_count();
392 : :
393 : : /* Limit to maximum available memory on socket */
394 : 12 : default_size = RTE_MIN(
395 : : default_size, get_socket_mem_size(socket));
396 : :
397 : : /* Update sizes */
398 : 12 : memory[socket] = default_size;
399 : 12 : total_size -= default_size;
400 : : }
401 : :
402 : : /*
403 : : * If some memory is remaining, try to allocate it by getting
404 : : * all available memory from sockets, one after the other.
405 : : */
406 [ - + ]: 60 : for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
407 : 0 : socket++) {
408 : : /* take whatever is available */
409 : 0 : default_size = RTE_MIN(
410 : : get_socket_mem_size(socket) - memory[socket],
411 : : total_size);
412 : :
413 : : /* Update sizes */
414 : 0 : memory[socket] += default_size;
415 : 0 : total_size -= default_size;
416 : : }
417 : : #else
418 : : /* in 32-bit mode, allocate all of the memory only on main
419 : : * lcore socket
420 : : */
421 : : total_size = internal_conf->memory;
422 : : for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
423 : : socket++) {
424 : : struct rte_config *cfg = rte_eal_get_configuration();
425 : : unsigned int main_lcore_socket;
426 : :
427 : : main_lcore_socket =
428 : : rte_lcore_to_socket_id(cfg->main_lcore);
429 : :
430 : : if (main_lcore_socket != socket)
431 : : continue;
432 : :
433 : : /* Update sizes */
434 : : memory[socket] = total_size;
435 : : break;
436 : : }
437 : : #endif
438 : : }
439 : :
440 [ + + ]: 79 : for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem != 0;
441 : 16 : socket++) {
442 : : /* skips if the memory on specific socket wasn't requested */
443 [ + + + - ]: 21 : for (i = 0; i < num_hp_info && memory[socket] != 0; i++) {
444 : 20 : rte_strscpy(hp_used[i].hugedir, hp_info[i].hugedir,
445 : : sizeof(hp_used[i].hugedir));
446 : 20 : hp_used[i].num_pages[socket] = RTE_MIN(
447 : : memory[socket] / hp_info[i].hugepage_sz,
448 : : hp_info[i].num_pages[socket]);
449 : :
450 : 20 : cur_mem = hp_used[i].num_pages[socket] *
451 : 20 : hp_used[i].hugepage_sz;
452 : :
453 : 20 : memory[socket] -= cur_mem;
454 : 20 : total_mem -= cur_mem;
455 : :
456 : 20 : total_num_pages += hp_used[i].num_pages[socket];
457 : :
458 : : /* check if we have met all memory requests */
459 [ + + ]: 20 : if (memory[socket] == 0)
460 : : break;
461 : :
462 : : /* Check if we have any more pages left at this size,
463 : : * if so, move on to next size.
464 : : */
465 : 8 : if (hp_used[i].num_pages[socket] ==
466 [ + - ]: 4 : hp_info[i].num_pages[socket])
467 : 4 : continue;
468 : : /* At this point we know that there are more pages
469 : : * available that are bigger than the memory we want,
470 : : * so lets see if we can get enough from other page
471 : : * sizes.
472 : : */
473 : : remaining_mem = 0;
474 [ # # ]: 0 : for (j = i+1; j < num_hp_info; j++)
475 : 0 : remaining_mem += hp_info[j].hugepage_sz *
476 : 0 : hp_info[j].num_pages[socket];
477 : :
478 : : /* Is there enough other memory?
479 : : * If not, allocate another page and quit.
480 : : */
481 [ # # ]: 0 : if (remaining_mem < memory[socket]) {
482 : 0 : cur_mem = RTE_MIN(
483 : : memory[socket], hp_info[i].hugepage_sz);
484 : 0 : memory[socket] -= cur_mem;
485 : 0 : total_mem -= cur_mem;
486 : 0 : hp_used[i].num_pages[socket]++;
487 : 0 : total_num_pages++;
488 : 0 : break; /* we are done with this socket*/
489 : : }
490 : : }
491 : :
492 : : /* if we didn't satisfy all memory requirements per socket */
493 [ + + ]: 17 : if (memory[socket] > 0 &&
494 [ + - ]: 1 : internal_conf->numa_mem[socket] != 0) {
495 : 1 : requested = internal_conf->numa_mem[socket] / 0x100000;
496 : 1 : available = requested - (memory[socket] / 0x100000);
497 : 1 : EAL_LOG(ERR, "Not enough memory available on socket %u! Requested: %uMB, available: %uMB",
498 : : socket, requested, available);
499 : 1 : return -1;
500 : : }
501 : : }
502 : :
503 : : /* if we didn't satisfy total memory requirements */
504 [ - + ]: 62 : if (total_mem > 0) {
505 : 0 : requested = internal_conf->memory / 0x100000;
506 : 0 : available = requested - (total_mem / 0x100000);
507 : 0 : EAL_LOG(ERR, "Not enough memory available! Requested: %uMB, available: %uMB",
508 : : requested, available);
509 : 0 : return -1;
510 : : }
511 : : return total_num_pages;
512 : : }
|
