Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2010-2014 Intel Corporation
3 : : */
4 : :
5 : : #ifndef _RTE_MEMCPY_X86_64_H_
6 : : #define _RTE_MEMCPY_X86_64_H_
7 : :
8 : : /**
9 : : * @file
10 : : *
11 : : * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
12 : : */
13 : :
14 : : #include <stdio.h>
15 : : #include <stdint.h>
16 : : #include <string.h>
17 : : #include <rte_vect.h>
18 : : #include <rte_common.h>
19 : : #include <rte_config.h>
20 : :
21 : : #ifdef __cplusplus
22 : : extern "C" {
23 : : #endif
24 : :
25 : : /*
26 : : * GCC older than version 11 doesn't compile AVX properly, so use SSE instead.
27 : : * There are no problems with AVX2.
28 : : */
29 : : #if defined __AVX2__
30 : : #define RTE_MEMCPY_AVX
31 : : #elif defined __AVX__ && !(defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 110000))
32 : : #define RTE_MEMCPY_AVX
33 : : #endif
34 : :
35 : : /**
36 : : * Copy bytes from one location to another. The locations must not overlap.
37 : : *
38 : : * @param dst
39 : : * Pointer to the destination of the data.
40 : : * @param src
41 : : * Pointer to the source data.
42 : : * @param n
43 : : * Number of bytes to copy.
44 : : * @return
45 : : * Pointer to the destination data.
46 : : */
47 : : static __rte_always_inline void *
48 : : rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n);
49 : :
50 : : /**
51 : : * Copy bytes from one location to another,
52 : : * locations must not overlap.
53 : : * Use with n <= 15.
54 : : */
55 : : static __rte_always_inline void *
56 : 0 : rte_mov15_or_less(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
57 : : {
58 : : /**
59 : : * Use the following structs to avoid violating C standard
60 : : * alignment requirements and to avoid strict aliasing bugs
61 : : */
62 : 0 : struct __rte_packed_begin rte_uint64_alias {
63 : : uint64_t val;
64 : : } __rte_packed_end __rte_may_alias;
65 : 0 : struct __rte_packed_begin rte_uint32_alias {
66 : : uint32_t val;
67 : : } __rte_packed_end __rte_may_alias;
68 : 0 : struct __rte_packed_begin rte_uint16_alias {
69 : : uint16_t val;
70 : : } __rte_packed_end __rte_may_alias;
71 : :
72 : 0 : void *ret = dst;
73 [ + + + + : 41749 : if (n & 8) {
+ + + + +
+ + - - -
- + - - +
- - + - -
- - + + +
- + + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - + -
- + + - -
- - - - -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
74 [ # # # # ]: 2151127 : ((struct rte_uint64_alias *)dst)->val =
75 [ # # ]: 2150157 : ((const struct rte_uint64_alias *)src)->val;
76 : 913 : src = (const uint64_t *)src + 1;
77 : 966 : dst = (uint64_t *)dst + 1;
78 : : }
79 [ + + + + : 1369 : if (n & 4) {
+ + + + -
+ + - - -
+ - - - +
- - + - -
- - + - +
- + + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - +
+ - - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
80 [ # # ]: 2000713 : ((struct rte_uint32_alias *)dst)->val =
81 [ # # ]: 545 : ((const struct rte_uint32_alias *)src)->val;
82 : 545 : src = (const uint32_t *)src + 1;
83 : 545 : dst = (uint32_t *)dst + 1;
84 : : }
85 [ + + + + : 1369 : if (n & 2) {
+ + + + -
+ - + - -
+ - - - -
+ + - - -
- - + + -
+ - + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - +
- + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
86 [ # # # # : 2000464 : ((struct rte_uint16_alias *)dst)->val =
# # # # #
# ]
87 [ # # # # : 451 : ((const struct rte_uint16_alias *)src)->val;
# # ]
88 : 451 : src = (const uint16_t *)src + 1;
89 : 451 : dst = (uint16_t *)dst + 1;
90 : : }
91 [ + + + + : 1369 : if (n & 1)
+ + - + -
+ - + - -
+ - - - -
+ - + - -
- - + - -
+ - + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - +
- + - + #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
92 [ # # # # ]: 410 : *(uint8_t *)dst = *(const uint8_t *)src;
93 : : return ret;
94 : : }
95 : :
96 : : /**
97 : : * Copy 16 bytes from one location to another,
98 : : * locations must not overlap.
99 : : */
100 : : static __rte_always_inline void
101 : 0 : rte_mov16(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
102 : : {
103 : 0 : __m128i xmm0;
104 : :
105 : 0 : xmm0 = _mm_loadu_si128((const __m128i *)(const void *)src);
106 [ # # # # : 0 : _mm_storeu_si128((__m128i *)(void *)dst, xmm0);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
107 : 0 : }
108 : :
109 : : /**
110 : : * Copy 32 bytes from one location to another,
111 : : * locations must not overlap.
112 : : */
113 : : static __rte_always_inline void
114 : 0 : rte_mov32(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
115 : : {
116 : : #if defined RTE_MEMCPY_AVX
117 : 0 : __m256i ymm0;
118 : :
119 : 0 : ymm0 = _mm256_loadu_si256((const __m256i *)(const void *)src);
120 : 0 : _mm256_storeu_si256((__m256i *)(void *)dst, ymm0);
121 : : #else /* SSE implementation */
122 : : rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16);
123 : : rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16);
124 : : #endif
125 : 3238644 : }
126 : :
127 : : /**
128 : : * Copy 48 bytes from one location to another,
129 : : * locations must not overlap.
130 : : */
131 : : static __rte_always_inline void
132 : : rte_mov48(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
133 : : {
134 : : #if defined RTE_MEMCPY_AVX
135 : : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
136 : : rte_mov16((uint8_t *)dst + 32, (const uint8_t *)src + 32);
137 : : #else /* SSE implementation */
138 : : rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16);
139 : : rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16);
140 : : rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16);
141 : : #endif
142 : : }
143 : :
144 : : /**
145 : : * Copy 64 bytes from one location to another,
146 : : * locations must not overlap.
147 : : */
148 : : static __rte_always_inline void
149 : 0 : rte_mov64(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
150 : : {
151 : : #if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
152 : : __m512i zmm0;
153 : :
154 : : zmm0 = _mm512_loadu_si512((const void *)src);
155 : : _mm512_storeu_si512((void *)dst, zmm0);
156 : : #else /* AVX2, AVX & SSE implementation */
157 : 0 : rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32);
158 : 0 : rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32);
159 : : #endif
160 : : }
161 : :
162 : : /**
163 : : * Copy 128 bytes from one location to another,
164 : : * locations must not overlap.
165 : : */
166 : : static __rte_always_inline void
167 : 0 : rte_mov128(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
168 : : {
169 : 0 : rte_mov64(dst + 0 * 64, src + 0 * 64);
170 : 0 : rte_mov64(dst + 1 * 64, src + 1 * 64);
171 : : }
172 : :
173 : : /**
174 : : * Copy 256 bytes from one location to another,
175 : : * locations must not overlap.
176 : : */
177 : : static __rte_always_inline void
178 : : rte_mov256(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src)
179 : : {
180 : : rte_mov128(dst + 0 * 128, src + 0 * 128);
181 : : rte_mov128(dst + 1 * 128, src + 1 * 128);
182 : : }
183 : :
184 : : #if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
185 : :
186 : : /**
187 : : * AVX512 implementation below
188 : : */
189 : :
190 : : #define ALIGNMENT_MASK 0x3F
191 : :
192 : : /**
193 : : * Copy 128-byte blocks from one location to another,
194 : : * locations must not overlap.
195 : : */
196 : : static __rte_always_inline void
197 : : rte_mov128blocks(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src, size_t n)
198 : : {
199 : : __m512i zmm0, zmm1;
200 : :
201 : : while (n >= 128) {
202 : : zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64));
203 : : n -= 128;
204 : : zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64));
205 : : src = src + 128;
206 : : _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0);
207 : : _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1);
208 : : dst = dst + 128;
209 : : }
210 : : }
211 : :
212 : : /**
213 : : * Copy 512-byte blocks from one location to another,
214 : : * locations must not overlap.
215 : : */
216 : : static inline void
217 : : rte_mov512blocks(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src, size_t n)
218 : : {
219 : : __m512i zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7;
220 : :
221 : : while (n >= 512) {
222 : : zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64));
223 : : n -= 512;
224 : : zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64));
225 : : zmm2 = _mm512_loadu_si512((const void *)(src + 2 * 64));
226 : : zmm3 = _mm512_loadu_si512((const void *)(src + 3 * 64));
227 : : zmm4 = _mm512_loadu_si512((const void *)(src + 4 * 64));
228 : : zmm5 = _mm512_loadu_si512((const void *)(src + 5 * 64));
229 : : zmm6 = _mm512_loadu_si512((const void *)(src + 6 * 64));
230 : : zmm7 = _mm512_loadu_si512((const void *)(src + 7 * 64));
231 : : src = src + 512;
232 : : _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0);
233 : : _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1);
234 : : _mm512_storeu_si512((void *)(dst + 2 * 64), zmm2);
235 : : _mm512_storeu_si512((void *)(dst + 3 * 64), zmm3);
236 : : _mm512_storeu_si512((void *)(dst + 4 * 64), zmm4);
237 : : _mm512_storeu_si512((void *)(dst + 5 * 64), zmm5);
238 : : _mm512_storeu_si512((void *)(dst + 6 * 64), zmm6);
239 : : _mm512_storeu_si512((void *)(dst + 7 * 64), zmm7);
240 : : dst = dst + 512;
241 : : }
242 : : }
243 : :
244 : : /**
245 : : * Copy bytes from one location to another,
246 : : * locations must not overlap.
247 : : * Use with n > 64.
248 : : */
249 : : static __rte_always_inline void *
250 : : rte_memcpy_generic_more_than_64(void *__rte_restrict dst, const void *__rte_restrict src,
251 : : size_t n)
252 : : {
253 : : void *ret = dst;
254 : : size_t dstofss;
255 : : size_t bits;
256 : :
257 : : /**
258 : : * Fast way when copy size doesn't exceed 512 bytes
259 : : */
260 : : if (n <= 512) {
261 : : if (n >= 256) {
262 : : n -= 256;
263 : : rte_mov256((uint8_t *)dst, (const uint8_t *)src);
264 : : src = (const uint8_t *)src + 256;
265 : : dst = (uint8_t *)dst + 256;
266 : : }
267 : : if (n >= 128) {
268 : : n -= 128;
269 : : rte_mov128((uint8_t *)dst, (const uint8_t *)src);
270 : : src = (const uint8_t *)src + 128;
271 : : dst = (uint8_t *)dst + 128;
272 : : }
273 : : COPY_BLOCK_128_BACK63:
274 : : if (n > 64) {
275 : : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
276 : : rte_mov64((uint8_t *)dst - 64 + n,
277 : : (const uint8_t *)src - 64 + n);
278 : : return ret;
279 : : }
280 : : if (n > 0)
281 : : rte_mov64((uint8_t *)dst - 64 + n,
282 : : (const uint8_t *)src - 64 + n);
283 : : return ret;
284 : : }
285 : :
286 : : /**
287 : : * Make store aligned when copy size exceeds 512 bytes
288 : : */
289 : : dstofss = ((uintptr_t)dst & 0x3F);
290 : : if (dstofss > 0) {
291 : : dstofss = 64 - dstofss;
292 : : n -= dstofss;
293 : : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
294 : : src = (const uint8_t *)src + dstofss;
295 : : dst = (uint8_t *)dst + dstofss;
296 : : }
297 : :
298 : : /**
299 : : * Copy 512-byte blocks.
300 : : * Use copy block function for better instruction order control,
301 : : * which is important when load is unaligned.
302 : : */
303 : : rte_mov512blocks((uint8_t *)dst, (const uint8_t *)src, n);
304 : : bits = n;
305 : : n = n & 511;
306 : : bits -= n;
307 : : src = (const uint8_t *)src + bits;
308 : : dst = (uint8_t *)dst + bits;
309 : :
310 : : /**
311 : : * Copy 128-byte blocks.
312 : : * Use copy block function for better instruction order control,
313 : : * which is important when load is unaligned.
314 : : */
315 : : if (n >= 128) {
316 : : rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n);
317 : : bits = n;
318 : : n = n & 127;
319 : : bits -= n;
320 : : src = (const uint8_t *)src + bits;
321 : : dst = (uint8_t *)dst + bits;
322 : : }
323 : :
324 : : /**
325 : : * Copy whatever left
326 : : */
327 : : goto COPY_BLOCK_128_BACK63;
328 : : }
329 : :
330 : : #elif defined RTE_MEMCPY_AVX
331 : :
332 : : /**
333 : : * AVX implementation below
334 : : */
335 : :
336 : : #define ALIGNMENT_MASK 0x1F
337 : :
338 : : /**
339 : : * Copy 128-byte blocks from one location to another,
340 : : * locations must not overlap.
341 : : */
342 : : static __rte_always_inline void
343 : 0 : rte_mov128blocks(uint8_t *__rte_restrict dst, const uint8_t *__rte_restrict src, size_t n)
344 : : {
345 : 0 : __m256i ymm0, ymm1, ymm2, ymm3;
346 : :
347 [ + + + + : 59207 : while (n >= 128) {
- - - - -
- + + + +
- - - - -
- - - - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
348 : 0 : ymm0 = _mm256_loadu_si256((const __m256i *)(const void *)
349 : : ((const uint8_t *)src + 0 * 32));
350 : 54378 : n -= 128;
351 : 0 : ymm1 = _mm256_loadu_si256((const __m256i *)(const void *)
352 : : ((const uint8_t *)src + 1 * 32));
353 : 0 : ymm2 = _mm256_loadu_si256((const __m256i *)(const void *)
354 : : ((const uint8_t *)src + 2 * 32));
355 : 0 : ymm3 = _mm256_loadu_si256((const __m256i *)(const void *)
356 : : ((const uint8_t *)src + 3 * 32));
357 : 54378 : src = (const uint8_t *)src + 128;
358 : 0 : _mm256_storeu_si256((__m256i *)(void *)
359 : : ((uint8_t *)dst + 0 * 32), ymm0);
360 : 0 : _mm256_storeu_si256((__m256i *)(void *)
361 : : ((uint8_t *)dst + 1 * 32), ymm1);
362 : 0 : _mm256_storeu_si256((__m256i *)(void *)
363 : : ((uint8_t *)dst + 2 * 32), ymm2);
364 : 0 : _mm256_storeu_si256((__m256i *)(void *)
365 : : ((uint8_t *)dst + 3 * 32), ymm3);
366 : 54378 : dst = (uint8_t *)dst + 128;
367 : : }
368 : : }
369 : :
370 : : /**
371 : : * Copy bytes from one location to another,
372 : : * locations must not overlap.
373 : : * Use with n > 64.
374 : : */
375 : : static __rte_always_inline void *
376 : 0 : rte_memcpy_generic_more_than_64(void *__rte_restrict dst, const void *__rte_restrict src,
377 : : size_t n)
378 : : {
379 : 0 : void *ret = dst;
380 : 0 : size_t dstofss;
381 : 0 : size_t bits;
382 : :
383 : : /**
384 : : * Fast way when copy size doesn't exceed 256 bytes
385 : : */
386 [ + + + + : 7417132 : if (n <= 256) {
- - - - -
- + + - +
- - - - -
- + - - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
387 [ + + + + : 7412303 : if (n >= 128) {
- - - - -
- - + - -
- - - - -
- - + - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
388 [ # # ]: 782826 : n -= 128;
389 : 0 : rte_mov128((uint8_t *)dst, (const uint8_t *)src);
390 : 782768 : src = (const uint8_t *)src + 128;
391 : 782826 : dst = (uint8_t *)dst + 128;
392 : : }
393 : 6629477 : COPY_BLOCK_128_BACK31:
394 [ + + + + : 7417132 : if (n >= 64) {
- - - - -
- + + - +
- - - - -
- - + - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
395 : 6632292 : n -= 64;
396 : 0 : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
397 : 6632292 : src = (const uint8_t *)src + 64;
398 : 6632292 : dst = (uint8_t *)dst + 64;
399 : : }
400 [ + + + + : 7417132 : if (n > 32) {
- - - - -
- + + + -
- - - - -
- + - - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
401 : 0 : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
402 : 3396016 : rte_mov32((uint8_t *)dst - 32 + n,
403 : 3396016 : (const uint8_t *)src - 32 + n);
404 : 3396016 : return ret;
405 : : }
406 [ + + + + : 4021116 : if (n > 0) {
- - - - -
- - + - -
- - - - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
407 : 3238644 : rte_mov32((uint8_t *)dst - 32 + n,
408 : 3238644 : (const uint8_t *)src - 32 + n);
409 : : }
410 : : return ret;
411 : : }
412 : :
413 : : /**
414 : : * Make store aligned when copy size exceeds 256 bytes
415 : : */
416 : 4829 : dstofss = (uintptr_t)dst & 0x1F;
417 [ + + + - : 4829 : if (dstofss > 0) {
- - - - -
- + - + -
- - - - -
- - - - -
- - - - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
418 : 4715 : dstofss = 32 - dstofss;
419 : 4715 : n -= dstofss;
420 : 0 : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
421 : 4715 : src = (const uint8_t *)src + dstofss;
422 : 4715 : dst = (uint8_t *)dst + dstofss;
423 : : }
424 : :
425 : : /**
426 : : * Copy 128-byte blocks
427 : : */
428 : 0 : rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n);
429 : 0 : bits = n;
430 : 4829 : n = n & 127;
431 : 4829 : bits -= n;
432 : 4829 : src = (const uint8_t *)src + bits;
433 : 4829 : dst = (uint8_t *)dst + bits;
434 : :
435 : : /**
436 : : * Copy whatever left
437 : : */
438 [ # # # # : 4829 : goto COPY_BLOCK_128_BACK31;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
439 : : }
440 : :
441 : : #else /* __AVX512F__ */
442 : :
443 : : /**
444 : : * SSE implementation below
445 : : */
446 : :
447 : : #define ALIGNMENT_MASK 0x0F
448 : :
449 : : /**
450 : : * Macro for copying unaligned block from one location to another with constant load offset,
451 : : * 47 bytes leftover maximum,
452 : : * locations must not overlap.
453 : : * Requirements:
454 : : * - Store is aligned
455 : : * - Load offset is <offset>, which must be immediate value within [1, 15]
456 : : * - For <src>, make sure <offset> bit backwards & <16 - offset> bit forwards are available for loading
457 : : * - <dst>, <src>, <len> must be variables
458 : : * - __m128i <xmm0> ~ <xmm8> must be pre-defined
459 : : */
460 : : #define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \
461 : : { \
462 : : size_t tmp; \
463 : : while (len >= 128 + 16 - offset) { \
464 : : xmm0 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 0 * 16)); \
465 : : len -= 128; \
466 : : xmm1 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 1 * 16)); \
467 : : xmm2 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 2 * 16)); \
468 : : xmm3 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 3 * 16)); \
469 : : xmm4 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 4 * 16)); \
470 : : xmm5 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 5 * 16)); \
471 : : xmm6 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 6 * 16)); \
472 : : xmm7 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 7 * 16)); \
473 : : xmm8 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 8 * 16)); \
474 : : src = (const uint8_t *)src + 128; \
475 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \
476 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \
477 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 2 * 16), _mm_alignr_epi8(xmm3, xmm2, offset)); \
478 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 3 * 16), _mm_alignr_epi8(xmm4, xmm3, offset)); \
479 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 4 * 16), _mm_alignr_epi8(xmm5, xmm4, offset)); \
480 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 5 * 16), _mm_alignr_epi8(xmm6, xmm5, offset)); \
481 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 6 * 16), _mm_alignr_epi8(xmm7, xmm6, offset)); \
482 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 7 * 16), _mm_alignr_epi8(xmm8, xmm7, offset)); \
483 : : dst = (uint8_t *)dst + 128; \
484 : : } \
485 : : tmp = len; \
486 : : len = ((len - 16 + offset) & 127) + 16 - offset; \
487 : : tmp -= len; \
488 : : src = (const uint8_t *)src + tmp; \
489 : : dst = (uint8_t *)dst + tmp; \
490 : : if (len >= 32 + 16 - offset) { \
491 : : while (len >= 32 + 16 - offset) { \
492 : : xmm0 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 0 * 16)); \
493 : : len -= 32; \
494 : : xmm1 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 1 * 16)); \
495 : : xmm2 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 2 * 16)); \
496 : : src = (const uint8_t *)src + 32; \
497 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \
498 : : _mm_storeu_si128((__m128i *)(void *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \
499 : : dst = (uint8_t *)dst + 32; \
500 : : } \
501 : : tmp = len; \
502 : : len = ((len - 16 + offset) & 31) + 16 - offset; \
503 : : tmp -= len; \
504 : : src = (const uint8_t *)src + tmp; \
505 : : dst = (uint8_t *)dst + tmp; \
506 : : } \
507 : : }
508 : :
509 : : /**
510 : : * Macro for copying unaligned block from one location to another,
511 : : * 47 bytes leftover maximum,
512 : : * locations must not overlap.
513 : : * Use switch here because the aligning instruction requires immediate value for shift count.
514 : : * Requirements:
515 : : * - Store is aligned
516 : : * - Load offset is <offset>, which must be within [1, 15]
517 : : * - For <src>, make sure <offset> bit backwards & <16 - offset> bit forwards are available for loading
518 : : * - <dst>, <src>, <len> must be variables
519 : : * - __m128i <xmm0> ~ <xmm8> used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined
520 : : */
521 : : #define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \
522 : : { \
523 : : switch (offset) { \
524 : : case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \
525 : : case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \
526 : : case 0x03: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x03); break; \
527 : : case 0x04: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x04); break; \
528 : : case 0x05: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x05); break; \
529 : : case 0x06: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x06); break; \
530 : : case 0x07: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x07); break; \
531 : : case 0x08: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x08); break; \
532 : : case 0x09: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x09); break; \
533 : : case 0x0A: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0A); break; \
534 : : case 0x0B: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0B); break; \
535 : : case 0x0C: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0C); break; \
536 : : case 0x0D: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0D); break; \
537 : : case 0x0E: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0E); break; \
538 : : case 0x0F: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0F); break; \
539 : : default:; \
540 : : } \
541 : : }
542 : :
543 : : /**
544 : : * Copy bytes from one location to another,
545 : : * locations must not overlap.
546 : : * Use with n > 64.
547 : : */
548 : : static __rte_always_inline void *
549 : : rte_memcpy_generic_more_than_64(void *__rte_restrict dst, const void *__rte_restrict src,
550 : : size_t n)
551 : : {
552 : : __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8;
553 : : void *ret = dst;
554 : : size_t dstofss;
555 : : size_t srcofs;
556 : :
557 : : /**
558 : : * Fast way when copy size doesn't exceed 512 bytes
559 : : */
560 : : if (n <= 128) {
561 : : goto COPY_BLOCK_128_BACK15;
562 : : }
563 : : if (n <= 512) {
564 : : if (n >= 256) {
565 : : n -= 256;
566 : : rte_mov128((uint8_t *)dst, (const uint8_t *)src);
567 : : rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128);
568 : : src = (const uint8_t *)src + 256;
569 : : dst = (uint8_t *)dst + 256;
570 : : }
571 : : COPY_BLOCK_255_BACK15:
572 : : if (n >= 128) {
573 : : n -= 128;
574 : : rte_mov128((uint8_t *)dst, (const uint8_t *)src);
575 : : src = (const uint8_t *)src + 128;
576 : : dst = (uint8_t *)dst + 128;
577 : : }
578 : : COPY_BLOCK_128_BACK15:
579 : : if (n >= 64) {
580 : : n -= 64;
581 : : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
582 : : src = (const uint8_t *)src + 64;
583 : : dst = (uint8_t *)dst + 64;
584 : : }
585 : : COPY_BLOCK_64_BACK15:
586 : : if (n >= 32) {
587 : : n -= 32;
588 : : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
589 : : src = (const uint8_t *)src + 32;
590 : : dst = (uint8_t *)dst + 32;
591 : : }
592 : : if (n > 16) {
593 : : rte_mov16((uint8_t *)dst, (const uint8_t *)src);
594 : : rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
595 : : return ret;
596 : : }
597 : : if (n > 0) {
598 : : rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
599 : : }
600 : : return ret;
601 : : }
602 : :
603 : : /**
604 : : * Make store aligned when copy size exceeds 512 bytes,
605 : : * and make sure the first 15 bytes are copied, because
606 : : * unaligned copy functions require up to 15 bytes
607 : : * backwards access.
608 : : */
609 : : dstofss = (uintptr_t)dst & 0x0F;
610 : : if (dstofss > 0) {
611 : : dstofss = 16 - dstofss + 16;
612 : : n -= dstofss;
613 : : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
614 : : src = (const uint8_t *)src + dstofss;
615 : : dst = (uint8_t *)dst + dstofss;
616 : : }
617 : : srcofs = ((uintptr_t)src & 0x0F);
618 : :
619 : : /**
620 : : * For aligned copy
621 : : */
622 : : if (srcofs == 0) {
623 : : /**
624 : : * Copy 256-byte blocks
625 : : */
626 : : for (; n >= 256; n -= 256) {
627 : : rte_mov256((uint8_t *)dst, (const uint8_t *)src);
628 : : dst = (uint8_t *)dst + 256;
629 : : src = (const uint8_t *)src + 256;
630 : : }
631 : :
632 : : /**
633 : : * Copy whatever left
634 : : */
635 : : goto COPY_BLOCK_255_BACK15;
636 : : }
637 : :
638 : : /**
639 : : * For copy with unaligned load
640 : : */
641 : : MOVEUNALIGNED_LEFT47(dst, src, n, srcofs);
642 : :
643 : : /**
644 : : * Copy whatever left
645 : : */
646 : : goto COPY_BLOCK_64_BACK15;
647 : : }
648 : :
649 : : #endif /* __AVX512F__ */
650 : :
651 : : /**
652 : : * Copy bytes from one vector register size aligned location to another,
653 : : * locations must not overlap.
654 : : * Use with n > 64.
655 : : */
656 : : static __rte_always_inline void *
657 : : rte_memcpy_aligned_more_than_64(void *__rte_restrict dst, const void *__rte_restrict src,
658 : : size_t n)
659 : : {
660 : : void *ret = dst;
661 : :
662 : : /* Copy 64 bytes blocks */
663 [ + + + + : 271492 : for (; n > 64; n -= 64) {
+ + + + +
+ - - + +
+ + + + -
- + + - -
- - - - -
- - - - -
- - - - +
+ + + - -
- - - - -
- - - - -
- - - - -
- + + - -
- - - - -
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
664 : 0 : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
665 : 139792 : dst = (uint8_t *)dst + 64;
666 : 139792 : src = (const uint8_t *)src + 64;
667 : : }
668 : :
669 : : /* Copy whatever left */
670 : 131700 : rte_mov64((uint8_t *)dst - 64 + n,
671 : 131700 : (const uint8_t *)src - 64 + n);
672 : :
673 : 0 : return ret;
674 : : }
675 : :
676 : : static __rte_always_inline void *
677 : 0 : rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
678 : : {
679 : : /* Fast way when copy size doesn't exceed 64 bytes. */
680 [ + + + + : 10183271 : if (n < 16)
+ + + + +
+ + + + +
+ + - + -
+ + + - +
- - - - +
- + - + +
- + - + -
- - - - -
- - - - -
- - - - -
- - - - -
- - + - -
+ - + - +
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
681 [ # # # # : 0 : return rte_mov15_or_less(dst, src, n);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
682 [ + + + + : 10141490 : if (n <= 32) {
+ + + + +
+ + + - +
+ + - + +
- - + + +
- - - - -
- - - + -
+ - + - -
- - - - -
- - - - -
- - - - -
- - - - -
- - + - -
- - - - -
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
683 [ + + - + : 612447 : if (__rte_constant(n) && n == 32) {
- + - - -
+ - - - +
- - - + -
- - + - -
- - - - -
+ - - - -
- - - + -
- - - - -
- + - - -
- - - - -
- - - - -
- - - - -
- + - - -
+ - - - +
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
684 : 0 : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
685 : 0 : return dst;
686 : : }
687 [ # # # # : 0 : rte_mov16((uint8_t *)dst, (const uint8_t *)src);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
688 [ + + + - : 612447 : if (__rte_constant(n) && n == 16)
- + - - -
+ - - - +
- - - + -
- - + - -
- - - - -
+ - - - -
- - - + -
- - - - -
- + - - -
- - - - -
- - - - -
- - - - -
- + - - -
+ - - - +
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
689 : : return dst; /* avoid (harmless) duplicate copy */
690 : 612447 : rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
691 : 612447 : return dst;
692 : : }
693 [ + + + + : 9529043 : if (n <= 64) {
+ + + + -
+ - + - +
+ + - + -
- - + - +
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - + - -
- - - - -
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
694 [ - + - - : 1980213 : if (__rte_constant(n) && n == 64) {
- + - - -
+ - - - +
- - - - -
- - - - -
- - - - -
+ - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
695 : 0 : rte_mov64((uint8_t *)dst, (const uint8_t *)src);
696 : 0 : return dst;
697 : : }
698 : : #if defined RTE_MEMCPY_AVX
699 : 0 : rte_mov32((uint8_t *)dst, (const uint8_t *)src);
700 [ # # # # ]: 1980213 : rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n);
701 : : #else /* SSE implementation */
702 : : rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16);
703 : : rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16);
704 : : if (n > 48)
705 : : rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16);
706 : : rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
707 : : #endif
708 : 1980214 : return dst;
709 : : }
710 : :
711 : : /* Implementation for size > 64 bytes depends on alignment with vector register size. */
712 [ + + + + : 7548837 : if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
+ - + - +
- - + + +
+ - + - -
- + + - -
- - - - -
- - - - -
- - - - +
- + - - -
- - - - -
- - - - -
- - - - -
- + - - -
- - - - -
- - - - -
- - # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
713 : 131700 : return rte_memcpy_aligned_more_than_64(dst, src, n);
714 : : else
715 [ # # # # : 5 : return rte_memcpy_generic_more_than_64(dst, src, n);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
716 : : }
717 : :
718 : : #undef ALIGNMENT_MASK
719 : :
720 : : #ifdef __cplusplus
721 : : }
722 : : #endif
723 : :
724 : : #endif /* _RTE_MEMCPY_X86_64_H_ */
|
