Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2021 HiSilicon Limited
3 : : * Copyright(c) 2021 Intel Corporation
4 : : * Copyright(c) 2021 Marvell International Ltd
5 : : * Copyright(c) 2021 SmartShare Systems
6 : : */
7 : :
8 : : #ifndef RTE_DMADEV_H
9 : : #define RTE_DMADEV_H
10 : :
11 : : /**
12 : : * @file rte_dmadev.h
13 : : *
14 : : * DMA (Direct Memory Access) device API.
15 : : *
16 : : * The DMA framework is built on the following model:
17 : : *
18 : : * --------------- --------------- ---------------
19 : : * | virtual DMA | | virtual DMA | | virtual DMA |
20 : : * | channel | | channel | | channel |
21 : : * --------------- --------------- ---------------
22 : : * | | |
23 : : * ------------------ |
24 : : * | |
25 : : * ------------ ------------
26 : : * | dmadev | | dmadev |
27 : : * ------------ ------------
28 : : * | |
29 : : * ------------------ ------------------
30 : : * | HW DMA channel | | HW DMA channel |
31 : : * ------------------ ------------------
32 : : * | |
33 : : * --------------------------------
34 : : * |
35 : : * ---------------------
36 : : * | HW DMA Controller |
37 : : * ---------------------
38 : : *
39 : : * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues),
40 : : * each HW-DMA-channel should be represented by a dmadev.
41 : : *
42 : : * The dmadev could create multiple virtual DMA channels, each virtual DMA
43 : : * channel represents a different transfer context. The DMA operation request
44 : : * must be submitted to the virtual DMA channel. e.g. Application could create
45 : : * virtual DMA channel 0 for memory-to-memory transfer scenario, and create
46 : : * virtual DMA channel 1 for memory-to-device transfer scenario.
47 : : *
48 : : * This framework uses 'int16_t dev_id' as the device identifier of a dmadev,
49 : : * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev.
50 : : *
51 : : * The functions exported by the dmadev API to setup a device designated by its
52 : : * device identifier must be invoked in the following order:
53 : : * - rte_dma_configure()
54 : : * - rte_dma_vchan_setup()
55 : : * - rte_dma_start()
56 : : *
57 : : * Then, the application can invoke dataplane functions to process jobs.
58 : : *
59 : : * If the application wants to change the configuration (i.e. invoke
60 : : * rte_dma_configure() or rte_dma_vchan_setup()), it must invoke
61 : : * rte_dma_stop() first to stop the device and then do the reconfiguration
62 : : * before invoking rte_dma_start() again. The dataplane functions should not
63 : : * be invoked when the device is stopped.
64 : : *
65 : : * Finally, an application can close a dmadev by invoking the rte_dma_close()
66 : : * function.
67 : : *
68 : : * The dataplane APIs include two parts:
69 : : * The first part is the submission of operation requests:
70 : : * - rte_dma_copy()
71 : : * - rte_dma_copy_sg()
72 : : * - rte_dma_fill()
73 : : * - rte_dma_submit()
74 : : *
75 : : * These APIs could work with different virtual DMA channels which have
76 : : * different contexts.
77 : : *
78 : : * The first three APIs are used to submit the operation request to the virtual
79 : : * DMA channel, if the submission is successful, a positive
80 : : * ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned.
81 : : *
82 : : * The last API is used to issue doorbell to hardware, and also there are flags
83 : : * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the
84 : : * same work.
85 : : * @note When enqueuing a set of jobs to the device, having a separate submit
86 : : * outside a loop makes for clearer code than having a check for the last
87 : : * iteration inside the loop to set a special submit flag. However, for cases
88 : : * where one item alone is to be submitted or there is a small set of jobs to
89 : : * be submitted sequentially, having a submit flag provides a lower-overhead
90 : : * way of doing the submission while still keeping the code clean.
91 : : *
92 : : * The second part is to obtain the result of requests:
93 : : * - rte_dma_completed()
94 : : * - return the number of operation requests completed successfully.
95 : : * - rte_dma_completed_status()
96 : : * - return the number of operation requests completed.
97 : : *
98 : : * @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT),
99 : : * application does not invoke the above two completed APIs.
100 : : *
101 : : * About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill())
102 : : * return, the rules are as follows:
103 : : * - ring_idx for each virtual DMA channel are independent.
104 : : * - For a virtual DMA channel, the ring_idx is monotonically incremented,
105 : : * when it reach UINT16_MAX, it wraps back to zero.
106 : : * - This ring_idx can be used by applications to track per-operation
107 : : * metadata in an application-defined circular ring.
108 : : * - The initial ring_idx of a virtual DMA channel is zero, after the
109 : : * device is stopped, the ring_idx needs to be reset to zero.
110 : : *
111 : : * One example:
112 : : * - step-1: start one dmadev
113 : : * - step-2: enqueue a copy operation, the ring_idx return is 0
114 : : * - step-3: enqueue a copy operation again, the ring_idx return is 1
115 : : * - ...
116 : : * - step-101: stop the dmadev
117 : : * - step-102: start the dmadev
118 : : * - step-103: enqueue a copy operation, the ring_idx return is 0
119 : : * - ...
120 : : * - step-x+0: enqueue a fill operation, the ring_idx return is 65535
121 : : * - step-x+1: enqueue a copy operation, the ring_idx return is 0
122 : : * - ...
123 : : *
124 : : * The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(),
125 : : * rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type.
126 : : *
127 : : * The dmadev supports two types of address: memory address and device address.
128 : : *
129 : : * - memory address: the source and destination address of the memory-to-memory
130 : : * transfer type, or the source address of the memory-to-device transfer type,
131 : : * or the destination address of the device-to-memory transfer type.
132 : : * @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address
133 : : * can be any VA address, otherwise it must be an IOVA address.
134 : : *
135 : : * - device address: the source and destination address of the device-to-device
136 : : * transfer type, or the source address of the device-to-memory transfer type,
137 : : * or the destination address of the memory-to-device transfer type.
138 : : *
139 : : * About MT-safe, all the functions of the dmadev API implemented by a PMD are
140 : : * lock-free functions which assume to not be invoked in parallel on different
141 : : * logical cores to work on the same target dmadev object.
142 : : * @note Different virtual DMA channels on the same dmadev *DO NOT* support
143 : : * parallel invocation because these virtual DMA channels share the same
144 : : * HW-DMA-channel.
145 : : */
146 : :
147 : : #include <stdint.h>
148 : :
149 : : #include <rte_bitops.h>
150 : : #include <rte_common.h>
151 : :
152 : : #ifdef __cplusplus
153 : : extern "C" {
154 : : #endif
155 : :
156 : : /** Maximum number of devices if rte_dma_dev_max() is not called. */
157 : : #define RTE_DMADEV_DEFAULT_MAX 64
158 : :
159 : : /**
160 : : * Configure the maximum number of dmadevs.
161 : : * @note This function can be invoked before the primary process rte_eal_init()
162 : : * to change the maximum number of dmadevs. If not invoked, the maximum number
163 : : * of dmadevs is @see RTE_DMADEV_DEFAULT_MAX
164 : : *
165 : : * @param dev_max
166 : : * maximum number of dmadevs.
167 : : *
168 : : * @return
169 : : * 0 on success. Otherwise negative value is returned.
170 : : */
171 : : int rte_dma_dev_max(size_t dev_max);
172 : :
173 : : /**
174 : : * Get the device identifier for the named DMA device.
175 : : *
176 : : * @param name
177 : : * DMA device name.
178 : : *
179 : : * @return
180 : : * Returns DMA device identifier on success.
181 : : * - <0: Failure to find named DMA device.
182 : : */
183 : : int rte_dma_get_dev_id_by_name(const char *name);
184 : :
185 : : /**
186 : : * Check whether the dev_id is valid.
187 : : *
188 : : * @param dev_id
189 : : * DMA device index.
190 : : *
191 : : * @return
192 : : * - If the device index is valid (true) or not (false).
193 : : */
194 : : bool rte_dma_is_valid(int16_t dev_id);
195 : :
196 : : /**
197 : : * Get the total number of DMA devices that have been successfully
198 : : * initialised.
199 : : *
200 : : * @return
201 : : * The total number of usable DMA devices.
202 : : */
203 : : uint16_t rte_dma_count_avail(void);
204 : :
205 : : /**
206 : : * Iterates over valid dmadev instances.
207 : : *
208 : : * @param start_dev_id
209 : : * The id of the next possible dmadev.
210 : : * @return
211 : : * Next valid dmadev, UINT16_MAX if there is none.
212 : : */
213 : : int16_t rte_dma_next_dev(int16_t start_dev_id);
214 : :
215 : : /** Utility macro to iterate over all available dmadevs */
216 : : #define RTE_DMA_FOREACH_DEV(p) \
217 : : for (p = rte_dma_next_dev(0); \
218 : : p != -1; \
219 : : p = rte_dma_next_dev(p + 1))
220 : :
221 : :
222 : : /**@{@name DMA capability
223 : : * @see struct rte_dma_info::dev_capa
224 : : */
225 : : /** Support memory-to-memory transfer */
226 : : #define RTE_DMA_CAPA_MEM_TO_MEM RTE_BIT64(0)
227 : : /** Support memory-to-device transfer. */
228 : : #define RTE_DMA_CAPA_MEM_TO_DEV RTE_BIT64(1)
229 : : /** Support device-to-memory transfer. */
230 : : #define RTE_DMA_CAPA_DEV_TO_MEM RTE_BIT64(2)
231 : : /** Support device-to-device transfer. */
232 : : #define RTE_DMA_CAPA_DEV_TO_DEV RTE_BIT64(3)
233 : : /** Support SVA which could use VA as DMA address.
234 : : * If device support SVA then application could pass any VA address like memory
235 : : * from rte_malloc(), rte_memzone(), malloc, stack memory.
236 : : * If device don't support SVA, then application should pass IOVA address which
237 : : * from rte_malloc(), rte_memzone().
238 : : */
239 : : #define RTE_DMA_CAPA_SVA RTE_BIT64(4)
240 : : /** Support work in silent mode.
241 : : * In this mode, application don't required to invoke rte_dma_completed*()
242 : : * API.
243 : : * @see struct rte_dma_conf::silent_mode
244 : : */
245 : : #define RTE_DMA_CAPA_SILENT RTE_BIT64(5)
246 : : /** Supports error handling
247 : : *
248 : : * With this bit set, invalid input addresses will be reported as operation failures
249 : : * to the user but other operations can continue.
250 : : * Without this bit set, invalid data is not handled by either HW or driver, so user
251 : : * must ensure that all memory addresses are valid and accessible by HW.
252 : : */
253 : : #define RTE_DMA_CAPA_HANDLES_ERRORS RTE_BIT64(6)
254 : : /** Support auto free for source buffer once mem to dev transfer is completed.
255 : : *
256 : : * @note Even though the DMA driver has this capability, it may not support all
257 : : * mempool drivers. If the mempool is not supported by the DMA driver,
258 : : * rte_dma_vchan_setup() will fail.
259 : : */
260 : : #define RTE_DMA_CAPA_M2D_AUTO_FREE RTE_BIT64(7)
261 : : /** Support strict priority scheduling.
262 : : *
263 : : * Application could assign fixed priority to the DMA device using 'priority'
264 : : * field in struct rte_dma_conf. Number of supported priority levels will be
265 : : * known from 'nb_priorities' field in struct rte_dma_info.
266 : : */
267 : : #define RTE_DMA_CAPA_PRI_POLICY_SP RTE_BIT64(8)
268 : :
269 : : /** Support copy operation.
270 : : * This capability start with index of 32, so that it could leave gap between
271 : : * normal capability and ops capability.
272 : : */
273 : : #define RTE_DMA_CAPA_OPS_COPY RTE_BIT64(32)
274 : : /** Support scatter-gather list copy operation. */
275 : : #define RTE_DMA_CAPA_OPS_COPY_SG RTE_BIT64(33)
276 : : /** Support fill operation. */
277 : : #define RTE_DMA_CAPA_OPS_FILL RTE_BIT64(34)
278 : : /**@}*/
279 : :
280 : : /**
281 : : * A structure used to retrieve the information of a DMA device.
282 : : *
283 : : * @see rte_dma_info_get
284 : : */
285 : : struct rte_dma_info {
286 : : const char *dev_name; /**< Unique device name. */
287 : : /** Device capabilities (RTE_DMA_CAPA_*). */
288 : : uint64_t dev_capa;
289 : : /** Maximum number of virtual DMA channels supported. */
290 : : uint16_t max_vchans;
291 : : /** Maximum allowed number of virtual DMA channel descriptors. */
292 : : uint16_t max_desc;
293 : : /** Minimum allowed number of virtual DMA channel descriptors. */
294 : : uint16_t min_desc;
295 : : /** Maximum number of source or destination scatter-gather entry
296 : : * supported.
297 : : * If the device does not support COPY_SG capability, this value can be
298 : : * zero.
299 : : * If the device supports COPY_SG capability, then rte_dma_copy_sg()
300 : : * parameter nb_src/nb_dst should not exceed this value.
301 : : */
302 : : uint16_t max_sges;
303 : : /** NUMA node connection, -1 if unknown. */
304 : : int16_t numa_node;
305 : : /** Number of virtual DMA channel configured. */
306 : : uint16_t nb_vchans;
307 : : /** Number of priority levels (must be > 1) if priority scheduling is supported,
308 : : * 0 otherwise.
309 : : */
310 : : uint16_t nb_priorities;
311 : : };
312 : :
313 : : /**
314 : : * Retrieve information of a DMA device.
315 : : *
316 : : * @param dev_id
317 : : * The identifier of the device.
318 : : * @param[out] dev_info
319 : : * A pointer to a structure of type *rte_dma_info* to be filled with the
320 : : * information of the device.
321 : : *
322 : : * @return
323 : : * 0 on success. Otherwise negative value is returned.
324 : : */
325 : : int rte_dma_info_get(int16_t dev_id, struct rte_dma_info *dev_info);
326 : :
327 : : /**
328 : : * A structure used to configure a DMA device.
329 : : *
330 : : * @see rte_dma_configure
331 : : */
332 : : struct rte_dma_conf {
333 : : /** The number of virtual DMA channels to set up for the DMA device.
334 : : * This value cannot be greater than the field 'max_vchans' of struct
335 : : * rte_dma_info which get from rte_dma_info_get().
336 : : */
337 : : uint16_t nb_vchans;
338 : : /** Indicates whether to enable silent mode.
339 : : * false-default mode, true-silent mode.
340 : : * This value can be set to true only when the SILENT capability is
341 : : * supported.
342 : : *
343 : : * @see RTE_DMA_CAPA_SILENT
344 : : */
345 : : bool enable_silent;
346 : : /* The priority of the DMA device.
347 : : * This value should be lower than the field 'nb_priorities' of struct
348 : : * rte_dma_info which get from rte_dma_info_get(). If the DMA device
349 : : * does not support priority scheduling, this value should be zero.
350 : : *
351 : : * Lowest value indicates higher priority and vice-versa.
352 : : */
353 : : uint16_t priority;
354 : : };
355 : :
356 : : /**
357 : : * Configure a DMA device.
358 : : *
359 : : * This function must be invoked first before any other function in the
360 : : * API. This function can also be re-invoked when a device is in the
361 : : * stopped state.
362 : : *
363 : : * @param dev_id
364 : : * The identifier of the device to configure.
365 : : * @param dev_conf
366 : : * The DMA device configuration structure encapsulated into rte_dma_conf
367 : : * object.
368 : : *
369 : : * @return
370 : : * 0 on success. Otherwise negative value is returned.
371 : : */
372 : : int rte_dma_configure(int16_t dev_id, const struct rte_dma_conf *dev_conf);
373 : :
374 : : /**
375 : : * Start a DMA device.
376 : : *
377 : : * The device start step is the last one and consists of setting the DMA
378 : : * to start accepting jobs.
379 : : *
380 : : * @param dev_id
381 : : * The identifier of the device.
382 : : *
383 : : * @return
384 : : * 0 on success. Otherwise negative value is returned.
385 : : */
386 : : int rte_dma_start(int16_t dev_id);
387 : :
388 : : /**
389 : : * Stop a DMA device.
390 : : *
391 : : * The device can be restarted with a call to rte_dma_start().
392 : : *
393 : : * @param dev_id
394 : : * The identifier of the device.
395 : : *
396 : : * @return
397 : : * 0 on success. Otherwise negative value is returned.
398 : : */
399 : : int rte_dma_stop(int16_t dev_id);
400 : :
401 : : /**
402 : : * Close a DMA device.
403 : : *
404 : : * The device cannot be restarted after this call.
405 : : *
406 : : * @param dev_id
407 : : * The identifier of the device.
408 : : *
409 : : * @return
410 : : * 0 on success. Otherwise negative value is returned.
411 : : */
412 : : int rte_dma_close(int16_t dev_id);
413 : :
414 : : /**
415 : : * DMA transfer direction defines.
416 : : *
417 : : * @see struct rte_dma_vchan_conf::direction
418 : : */
419 : : enum rte_dma_direction {
420 : : /** DMA transfer direction - from memory to memory.
421 : : *
422 : : * @see struct rte_dma_vchan_conf::direction
423 : : */
424 : : RTE_DMA_DIR_MEM_TO_MEM,
425 : : /** DMA transfer direction - from memory to device.
426 : : * In a typical scenario, the SoCs are installed on host servers as
427 : : * iNICs through the PCIe interface. In this case, the SoCs works in
428 : : * EP(endpoint) mode, it could initiate a DMA move request from memory
429 : : * (which is SoCs memory) to device (which is host memory).
430 : : *
431 : : * @see struct rte_dma_vchan_conf::direction
432 : : */
433 : : RTE_DMA_DIR_MEM_TO_DEV,
434 : : /** DMA transfer direction - from device to memory.
435 : : * In a typical scenario, the SoCs are installed on host servers as
436 : : * iNICs through the PCIe interface. In this case, the SoCs works in
437 : : * EP(endpoint) mode, it could initiate a DMA move request from device
438 : : * (which is host memory) to memory (which is SoCs memory).
439 : : *
440 : : * @see struct rte_dma_vchan_conf::direction
441 : : */
442 : : RTE_DMA_DIR_DEV_TO_MEM,
443 : : /** DMA transfer direction - from device to device.
444 : : * In a typical scenario, the SoCs are installed on host servers as
445 : : * iNICs through the PCIe interface. In this case, the SoCs works in
446 : : * EP(endpoint) mode, it could initiate a DMA move request from device
447 : : * (which is host memory) to the device (which is another host memory).
448 : : *
449 : : * @see struct rte_dma_vchan_conf::direction
450 : : */
451 : : RTE_DMA_DIR_DEV_TO_DEV,
452 : : };
453 : :
454 : : /**
455 : : * DMA access port type defines.
456 : : *
457 : : * @see struct rte_dma_port_param::port_type
458 : : */
459 : : enum rte_dma_port_type {
460 : : RTE_DMA_PORT_NONE,
461 : : RTE_DMA_PORT_PCIE, /**< The DMA access port is PCIe. */
462 : : };
463 : :
464 : : /**
465 : : * A structure used to descript DMA access port parameters.
466 : : *
467 : : * @see struct rte_dma_vchan_conf::src_port
468 : : * @see struct rte_dma_vchan_conf::dst_port
469 : : */
470 : : struct rte_dma_port_param {
471 : : /** The device access port type.
472 : : *
473 : : * @see enum rte_dma_port_type
474 : : */
475 : : enum rte_dma_port_type port_type;
476 : : union {
477 : : /** PCIe access port parameters.
478 : : *
479 : : * The following model shows SoC's PCIe module connects to
480 : : * multiple PCIe hosts and multiple endpoints. The PCIe module
481 : : * has an integrated DMA controller.
482 : : *
483 : : * If the DMA wants to access the memory of host A, it can be
484 : : * initiated by PF1 in core0, or by VF0 of PF0 in core0.
485 : : *
486 : : * \code{.unparsed}
487 : : * System Bus
488 : : * | ----------PCIe module----------
489 : : * | Bus
490 : : * | Interface
491 : : * | ----- ------------------
492 : : * | | | | PCIe Core0 |
493 : : * | | | | | -----------
494 : : * | | | | PF-0 -- VF-0 | | Host A |
495 : : * | | |--------| |- VF-1 |--------| Root |
496 : : * | | | | PF-1 | | Complex |
497 : : * | | | | PF-2 | -----------
498 : : * | | | ------------------
499 : : * | | |
500 : : * | | | ------------------
501 : : * | | | | PCIe Core1 |
502 : : * | | | | | -----------
503 : : * | | | | PF-0 -- VF-0 | | Host B |
504 : : * |-----| |--------| PF-1 -- VF-0 |--------| Root |
505 : : * | | | | |- VF-1 | | Complex |
506 : : * | | | | PF-2 | -----------
507 : : * | | | ------------------
508 : : * | | |
509 : : * | | | ------------------
510 : : * | |DMA| | | ------
511 : : * | | | | |--------| EP |
512 : : * | | |--------| PCIe Core2 | ------
513 : : * | | | | | ------
514 : : * | | | | |--------| EP |
515 : : * | | | | | ------
516 : : * | ----- ------------------
517 : : *
518 : : * \endcode
519 : : *
520 : : * @note If some fields can not be supported by the
521 : : * hardware/driver, then the driver ignores those fields.
522 : : * Please check driver-specific documentation for limitations
523 : : * and capabilities.
524 : : */
525 : : __extension__
526 : : struct {
527 : : uint64_t coreid : 4; /**< PCIe core id used. */
528 : : uint64_t pfid : 8; /**< PF id used. */
529 : : uint64_t vfen : 1; /**< VF enable bit. */
530 : : uint64_t vfid : 16; /**< VF id used. */
531 : : /** The pasid filed in TLP packet. */
532 : : uint64_t pasid : 20;
533 : : /** The attributes filed in TLP packet. */
534 : : uint64_t attr : 3;
535 : : /** The processing hint filed in TLP packet. */
536 : : uint64_t ph : 2;
537 : : /** The steering tag filed in TLP packet. */
538 : : uint64_t st : 16;
539 : : } pcie;
540 : : };
541 : : uint64_t reserved[2]; /**< Reserved for future fields. */
542 : : };
543 : :
544 : : /**
545 : : * A structure used for offload auto free params.
546 : : */
547 : : struct rte_dma_auto_free_param {
548 : : union {
549 : : struct {
550 : : /**
551 : : * Mempool from which buffer is allocated. Mempool info
552 : : * is used for freeing buffer by hardware.
553 : : *
554 : : * @note If the mempool is not supported by the DMA device,
555 : : * rte_dma_vchan_setup() will fail.
556 : : */
557 : : struct rte_mempool *pool;
558 : : } m2d;
559 : : };
560 : : /** Reserved for future fields. */
561 : : uint64_t reserved[2];
562 : : };
563 : :
564 : : /**
565 : : * A structure used to configure a virtual DMA channel.
566 : : *
567 : : * @see rte_dma_vchan_setup
568 : : */
569 : : struct rte_dma_vchan_conf {
570 : : /** Transfer direction
571 : : *
572 : : * @see enum rte_dma_direction
573 : : */
574 : : enum rte_dma_direction direction;
575 : : /** Number of descriptor for the virtual DMA channel */
576 : : uint16_t nb_desc;
577 : : /** 1) Used to describes the device access port parameter in the
578 : : * device-to-memory transfer scenario.
579 : : * 2) Used to describes the source device access port parameter in the
580 : : * device-to-device transfer scenario.
581 : : *
582 : : * @see struct rte_dma_port_param
583 : : */
584 : : struct rte_dma_port_param src_port;
585 : : /** 1) Used to describes the device access port parameter in the
586 : : * memory-to-device transfer scenario.
587 : : * 2) Used to describes the destination device access port parameter in
588 : : * the device-to-device transfer scenario.
589 : : *
590 : : * @see struct rte_dma_port_param
591 : : */
592 : : struct rte_dma_port_param dst_port;
593 : : /** Buffer params to auto free buffer by hardware. To free the buffer
594 : : * by hardware, RTE_DMA_OP_FLAG_AUTO_FREE must be set while calling
595 : : * rte_dma_copy and rte_dma_copy_sg().
596 : : *
597 : : * @see RTE_DMA_OP_FLAG_AUTO_FREE
598 : : * @see struct rte_dma_auto_free_param
599 : : */
600 : : struct rte_dma_auto_free_param auto_free;
601 : : };
602 : :
603 : : /**
604 : : * Allocate and set up a virtual DMA channel.
605 : : *
606 : : * @param dev_id
607 : : * The identifier of the device.
608 : : * @param vchan
609 : : * The identifier of virtual DMA channel. The value must be in the range
610 : : * [0, nb_vchans - 1] previously supplied to rte_dma_configure().
611 : : * @param conf
612 : : * The virtual DMA channel configuration structure encapsulated into
613 : : * rte_dma_vchan_conf object.
614 : : *
615 : : * @return
616 : : * 0 on success. Otherwise negative value is returned.
617 : : */
618 : : int rte_dma_vchan_setup(int16_t dev_id, uint16_t vchan,
619 : : const struct rte_dma_vchan_conf *conf);
620 : :
621 : : /**
622 : : * A structure used to retrieve statistics.
623 : : *
624 : : * @see rte_dma_stats_get
625 : : */
626 : : struct rte_dma_stats {
627 : : /** Count of operations which were submitted to hardware. */
628 : : uint64_t submitted;
629 : : /** Count of operations which were completed, including successful and
630 : : * failed completions.
631 : : */
632 : : uint64_t completed;
633 : : /** Count of operations which failed to complete. */
634 : : uint64_t errors;
635 : : };
636 : :
637 : : /**
638 : : * Special ID, which is used to represent all virtual DMA channels.
639 : : *
640 : : * @see rte_dma_stats_get
641 : : * @see rte_dma_stats_reset
642 : : */
643 : : #define RTE_DMA_ALL_VCHAN 0xFFFFu
644 : :
645 : : /**
646 : : * Retrieve basic statistics of a or all virtual DMA channel(s).
647 : : *
648 : : * @param dev_id
649 : : * The identifier of the device.
650 : : * @param vchan
651 : : * The identifier of virtual DMA channel.
652 : : * If equal RTE_DMA_ALL_VCHAN means all channels.
653 : : * @param[out] stats
654 : : * The basic statistics structure encapsulated into rte_dma_stats
655 : : * object.
656 : : *
657 : : * @return
658 : : * 0 on success. Otherwise negative value is returned.
659 : : */
660 : : int rte_dma_stats_get(int16_t dev_id, uint16_t vchan,
661 : : struct rte_dma_stats *stats);
662 : :
663 : : /**
664 : : * Reset basic statistics of a or all virtual DMA channel(s).
665 : : *
666 : : * @param dev_id
667 : : * The identifier of the device.
668 : : * @param vchan
669 : : * The identifier of virtual DMA channel.
670 : : * If equal RTE_DMA_ALL_VCHAN means all channels.
671 : : *
672 : : * @return
673 : : * 0 on success. Otherwise negative value is returned.
674 : : */
675 : : int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);
676 : :
677 : : /**
678 : : * device vchannel status
679 : : *
680 : : * Enum with the options for the channel status, either idle, active or halted due to error
681 : : * @see rte_dma_vchan_status
682 : : */
683 : : enum rte_dma_vchan_status {
684 : : RTE_DMA_VCHAN_IDLE, /**< not processing, awaiting ops */
685 : : RTE_DMA_VCHAN_ACTIVE, /**< currently processing jobs */
686 : : RTE_DMA_VCHAN_HALTED_ERROR, /**< not processing due to error, cannot accept new ops */
687 : : };
688 : :
689 : : /**
690 : : * Determine if all jobs have completed on a device channel.
691 : : * This function is primarily designed for testing use, as it allows a process to check if
692 : : * all jobs are completed, without actually gathering completions from those jobs.
693 : : *
694 : : * @param dev_id
695 : : * The identifier of the device.
696 : : * @param vchan
697 : : * The identifier of virtual DMA channel.
698 : : * @param[out] status
699 : : * The vchan status
700 : : * @return
701 : : * 0 - call completed successfully
702 : : * < 0 - error code indicating there was a problem calling the API
703 : : */
704 : : int
705 : : rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *status);
706 : :
707 : : /**
708 : : * Dump DMA device info.
709 : : *
710 : : * @param dev_id
711 : : * The identifier of the device.
712 : : * @param f
713 : : * The file to write the output to.
714 : : *
715 : : * @return
716 : : * 0 on success. Otherwise negative value is returned.
717 : : */
718 : : int rte_dma_dump(int16_t dev_id, FILE *f);
719 : :
720 : : /**
721 : : * DMA transfer result status code defines.
722 : : *
723 : : * @see rte_dma_completed_status
724 : : */
725 : : enum rte_dma_status_code {
726 : : /** The operation completed successfully. */
727 : : RTE_DMA_STATUS_SUCCESSFUL,
728 : : /** The operation failed to complete due abort by user.
729 : : * This is mainly used when processing dev_stop, user could modify the
730 : : * descriptors (e.g. change one bit to tell hardware abort this job),
731 : : * it allows outstanding requests to be complete as much as possible,
732 : : * so reduce the time to stop the device.
733 : : */
734 : : RTE_DMA_STATUS_USER_ABORT,
735 : : /** The operation failed to complete due to following scenarios:
736 : : * The jobs in a particular batch are not attempted because they
737 : : * appeared after a fence where a previous job failed. In some HW
738 : : * implementation it's possible for jobs from later batches would be
739 : : * completed, though, so report the status from the not attempted jobs
740 : : * before reporting those newer completed jobs.
741 : : */
742 : : RTE_DMA_STATUS_NOT_ATTEMPTED,
743 : : /** The operation failed to complete due invalid source address. */
744 : : RTE_DMA_STATUS_INVALID_SRC_ADDR,
745 : : /** The operation failed to complete due invalid destination address. */
746 : : RTE_DMA_STATUS_INVALID_DST_ADDR,
747 : : /** The operation failed to complete due invalid source or destination
748 : : * address, cover the case that only knows the address error, but not
749 : : * sure which address error.
750 : : */
751 : : RTE_DMA_STATUS_INVALID_ADDR,
752 : : /** The operation failed to complete due invalid length. */
753 : : RTE_DMA_STATUS_INVALID_LENGTH,
754 : : /** The operation failed to complete due invalid opcode.
755 : : * The DMA descriptor could have multiple format, which are
756 : : * distinguished by the opcode field.
757 : : */
758 : : RTE_DMA_STATUS_INVALID_OPCODE,
759 : : /** The operation failed to complete due bus read error. */
760 : : RTE_DMA_STATUS_BUS_READ_ERROR,
761 : : /** The operation failed to complete due bus write error. */
762 : : RTE_DMA_STATUS_BUS_WRITE_ERROR,
763 : : /** The operation failed to complete due bus error, cover the case that
764 : : * only knows the bus error, but not sure which direction error.
765 : : */
766 : : RTE_DMA_STATUS_BUS_ERROR,
767 : : /** The operation failed to complete due data poison. */
768 : : RTE_DMA_STATUS_DATA_POISION,
769 : : /** The operation failed to complete due descriptor read error. */
770 : : RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
771 : : /** The operation failed to complete due device link error.
772 : : * Used to indicates that the link error in the memory-to-device/
773 : : * device-to-memory/device-to-device transfer scenario.
774 : : */
775 : : RTE_DMA_STATUS_DEV_LINK_ERROR,
776 : : /** The operation failed to complete due lookup page fault. */
777 : : RTE_DMA_STATUS_PAGE_FAULT,
778 : : /** The operation failed to complete due unknown reason.
779 : : * The initial value is 256, which reserves space for future errors.
780 : : */
781 : : RTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,
782 : : };
783 : :
784 : : /**
785 : : * A structure used to hold scatter-gather DMA operation request entry.
786 : : *
787 : : * @see rte_dma_copy_sg
788 : : */
789 : : struct rte_dma_sge {
790 : : rte_iova_t addr; /**< The DMA operation address. */
791 : : uint32_t length; /**< The DMA operation length. */
792 : : };
793 : :
794 : : #ifdef __cplusplus
795 : : }
796 : : #endif
797 : :
798 : : #include "rte_dmadev_core.h"
799 : : #include "rte_dmadev_trace_fp.h"
800 : :
801 : : #ifdef __cplusplus
802 : : extern "C" {
803 : : #endif
804 : :
805 : : /**@{@name DMA operation flag
806 : : * @see rte_dma_copy()
807 : : * @see rte_dma_copy_sg()
808 : : * @see rte_dma_fill()
809 : : */
810 : : /** Fence flag.
811 : : * It means the operation with this flag must be processed only after all
812 : : * previous operations are completed.
813 : : * If the specify DMA HW works in-order (it means it has default fence between
814 : : * operations), this flag could be NOP.
815 : : */
816 : : #define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0)
817 : : /** Submit flag.
818 : : * It means the operation with this flag must issue doorbell to hardware after
819 : : * enqueued jobs.
820 : : */
821 : : #define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1)
822 : : /** Write data to low level cache hint.
823 : : * Used for performance optimization, this is just a hint, and there is no
824 : : * capability bit for this, driver should not return error if this flag was set.
825 : : */
826 : : #define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2)
827 : : /** Auto free buffer flag.
828 : : * Operation with this flag must issue command to hardware to free the DMA
829 : : * buffer after DMA transfer is completed.
830 : : *
831 : : * @see struct rte_dma_vchan_conf::auto_free
832 : : */
833 : : #define RTE_DMA_OP_FLAG_AUTO_FREE RTE_BIT64(3)
834 : : /**@}*/
835 : :
836 : : /**
837 : : * Enqueue a copy operation onto the virtual DMA channel.
838 : : *
839 : : * This queues up a copy operation to be performed by hardware, if the 'flags'
840 : : * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
841 : : * this operation, otherwise do not trigger doorbell.
842 : : *
843 : : * @param dev_id
844 : : * The identifier of the device.
845 : : * @param vchan
846 : : * The identifier of virtual DMA channel.
847 : : * @param src
848 : : * The address of the source buffer.
849 : : * @param dst
850 : : * The address of the destination buffer.
851 : : * @param length
852 : : * The length of the data to be copied.
853 : : * @param flags
854 : : * An flags for this operation.
855 : : * @see RTE_DMA_OP_FLAG_*
856 : : *
857 : : * @return
858 : : * - 0..UINT16_MAX: index of enqueued job.
859 : : * - -ENOSPC: if no space left to enqueue.
860 : : * - other values < 0 on failure.
861 : : */
862 : : static inline int
863 : : rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
864 : : uint32_t length, uint64_t flags)
865 : : {
866 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
867 : : int ret;
868 : :
869 : : #ifdef RTE_DMADEV_DEBUG
870 : : if (!rte_dma_is_valid(dev_id) || length == 0)
871 : : return -EINVAL;
872 : : if (*obj->copy == NULL)
873 : : return -ENOTSUP;
874 : : #endif
875 : :
876 : 0 : ret = (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);
877 : : rte_dma_trace_copy(dev_id, vchan, src, dst, length, flags, ret);
878 : :
879 : : return ret;
880 : : }
881 : :
882 : : /**
883 : : * Enqueue a scatter-gather list copy operation onto the virtual DMA channel.
884 : : *
885 : : * This queues up a scatter-gather list copy operation to be performed by
886 : : * hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then
887 : : * trigger doorbell to begin this operation, otherwise do not trigger doorbell.
888 : : *
889 : : * @param dev_id
890 : : * The identifier of the device.
891 : : * @param vchan
892 : : * The identifier of virtual DMA channel.
893 : : * @param src
894 : : * The pointer of source scatter-gather entry array.
895 : : * @param dst
896 : : * The pointer of destination scatter-gather entry array.
897 : : * @param nb_src
898 : : * The number of source scatter-gather entry.
899 : : * @see struct rte_dma_info::max_sges
900 : : * @param nb_dst
901 : : * The number of destination scatter-gather entry.
902 : : * @see struct rte_dma_info::max_sges
903 : : * @param flags
904 : : * An flags for this operation.
905 : : * @see RTE_DMA_OP_FLAG_*
906 : : *
907 : : * @return
908 : : * - 0..UINT16_MAX: index of enqueued job.
909 : : * - -ENOSPC: if no space left to enqueue.
910 : : * - other values < 0 on failure.
911 : : */
912 : : static inline int
913 : : rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,
914 : : struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,
915 : : uint64_t flags)
916 : : {
917 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
918 : : int ret;
919 : :
920 : : #ifdef RTE_DMADEV_DEBUG
921 : : if (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||
922 : : nb_src == 0 || nb_dst == 0)
923 : : return -EINVAL;
924 : : if (*obj->copy_sg == NULL)
925 : : return -ENOTSUP;
926 : : #endif
927 : :
928 : 0 : ret = (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,
929 : : nb_dst, flags);
930 : : rte_dma_trace_copy_sg(dev_id, vchan, src, dst, nb_src, nb_dst, flags,
931 : : ret);
932 : :
933 : : return ret;
934 : : }
935 : :
936 : : /**
937 : : * Enqueue a fill operation onto the virtual DMA channel.
938 : : *
939 : : * This queues up a fill operation to be performed by hardware, if the 'flags'
940 : : * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
941 : : * this operation, otherwise do not trigger doorbell.
942 : : *
943 : : * @param dev_id
944 : : * The identifier of the device.
945 : : * @param vchan
946 : : * The identifier of virtual DMA channel.
947 : : * @param pattern
948 : : * The pattern to populate the destination buffer with.
949 : : * @param dst
950 : : * The address of the destination buffer.
951 : : * @param length
952 : : * The length of the destination buffer.
953 : : * @param flags
954 : : * An flags for this operation.
955 : : * @see RTE_DMA_OP_FLAG_*
956 : : *
957 : : * @return
958 : : * - 0..UINT16_MAX: index of enqueued job.
959 : : * - -ENOSPC: if no space left to enqueue.
960 : : * - other values < 0 on failure.
961 : : */
962 : : static inline int
963 : : rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,
964 : : rte_iova_t dst, uint32_t length, uint64_t flags)
965 : : {
966 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
967 : : int ret;
968 : :
969 : : #ifdef RTE_DMADEV_DEBUG
970 : : if (!rte_dma_is_valid(dev_id) || length == 0)
971 : : return -EINVAL;
972 : : if (*obj->fill == NULL)
973 : : return -ENOTSUP;
974 : : #endif
975 : :
976 : 0 : ret = (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,
977 : : flags);
978 : : rte_dma_trace_fill(dev_id, vchan, pattern, dst, length, flags, ret);
979 : :
980 : : return ret;
981 : : }
982 : :
983 : : /**
984 : : * Trigger hardware to begin performing enqueued operations.
985 : : *
986 : : * Writes the "doorbell" to the hardware to trigger it
987 : : * to begin the operations previously enqueued by rte_dma_copy/fill().
988 : : *
989 : : * @param dev_id
990 : : * The identifier of the device.
991 : : * @param vchan
992 : : * The identifier of virtual DMA channel.
993 : : *
994 : : * @return
995 : : * 0 on success. Otherwise negative value is returned.
996 : : */
997 : : static inline int
998 : : rte_dma_submit(int16_t dev_id, uint16_t vchan)
999 : : {
1000 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1001 : : int ret;
1002 : :
1003 : : #ifdef RTE_DMADEV_DEBUG
1004 : : if (!rte_dma_is_valid(dev_id))
1005 : : return -EINVAL;
1006 : : if (*obj->submit == NULL)
1007 : : return -ENOTSUP;
1008 : : #endif
1009 : :
1010 : 0 : ret = (*obj->submit)(obj->dev_private, vchan);
1011 : : rte_dma_trace_submit(dev_id, vchan, ret);
1012 : :
1013 : 0 : return ret;
1014 : : }
1015 : :
1016 : : /**
1017 : : * Return the number of operations that have been successfully completed.
1018 : : * Once an operation has been reported as completed, the results of that
1019 : : * operation will be visible to all cores on the system.
1020 : : *
1021 : : * @param dev_id
1022 : : * The identifier of the device.
1023 : : * @param vchan
1024 : : * The identifier of virtual DMA channel.
1025 : : * @param nb_cpls
1026 : : * The maximum number of completed operations that can be processed.
1027 : : * @param[out] last_idx
1028 : : * The last completed operation's ring_idx.
1029 : : * If not required, NULL can be passed in.
1030 : : * @param[out] has_error
1031 : : * Indicates if there are transfer error.
1032 : : * If not required, NULL can be passed in.
1033 : : *
1034 : : * @return
1035 : : * The number of operations that successfully completed. This return value
1036 : : * must be less than or equal to the value of nb_cpls.
1037 : : */
1038 : : static inline uint16_t
1039 : : rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
1040 : : uint16_t *last_idx, bool *has_error)
1041 : : {
1042 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1043 : : uint16_t idx, ret;
1044 : : bool err;
1045 : :
1046 : : #ifdef RTE_DMADEV_DEBUG
1047 : : if (!rte_dma_is_valid(dev_id) || nb_cpls == 0)
1048 : : return 0;
1049 : : if (*obj->completed == NULL)
1050 : : return 0;
1051 : : #endif
1052 : :
1053 : : /* Ensure the pointer values are non-null to simplify drivers.
1054 : : * In most cases these should be compile time evaluated, since this is
1055 : : * an inline function.
1056 : : * - If NULL is explicitly passed as parameter, then compiler knows the
1057 : : * value is NULL
1058 : : * - If address of local variable is passed as parameter, then compiler
1059 : : * can know it's non-NULL.
1060 : : */
1061 : : if (last_idx == NULL)
1062 : : last_idx = &idx;
1063 : : if (has_error == NULL)
1064 : : has_error = &err;
1065 : :
1066 : 0 : *has_error = false;
1067 : 0 : ret = (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,
1068 : : has_error);
1069 : : rte_dma_trace_completed(dev_id, vchan, nb_cpls, last_idx, has_error,
1070 : : ret);
1071 : :
1072 : : return ret;
1073 : : }
1074 : :
1075 : : /**
1076 : : * Return the number of operations that have been completed, and the operations
1077 : : * result may succeed or fail.
1078 : : * Once an operation has been reported as completed successfully, the results of that
1079 : : * operation will be visible to all cores on the system.
1080 : : *
1081 : : * @param dev_id
1082 : : * The identifier of the device.
1083 : : * @param vchan
1084 : : * The identifier of virtual DMA channel.
1085 : : * @param nb_cpls
1086 : : * Indicates the size of status array.
1087 : : * @param[out] last_idx
1088 : : * The last completed operation's ring_idx.
1089 : : * If not required, NULL can be passed in.
1090 : : * @param[out] status
1091 : : * This is a pointer to an array of length 'nb_cpls' that holds the completion
1092 : : * status code of each operation.
1093 : : * @see enum rte_dma_status_code
1094 : : *
1095 : : * @return
1096 : : * The number of operations that completed. This return value must be less
1097 : : * than or equal to the value of nb_cpls.
1098 : : * If this number is greater than zero (assuming n), then n values in the
1099 : : * status array are also set.
1100 : : */
1101 : : static inline uint16_t
1102 : : rte_dma_completed_status(int16_t dev_id, uint16_t vchan,
1103 : : const uint16_t nb_cpls, uint16_t *last_idx,
1104 : : enum rte_dma_status_code *status)
1105 : : {
1106 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1107 : : uint16_t idx, ret;
1108 : :
1109 : : #ifdef RTE_DMADEV_DEBUG
1110 : : if (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)
1111 : : return 0;
1112 : : if (*obj->completed_status == NULL)
1113 : : return 0;
1114 : : #endif
1115 : :
1116 : : if (last_idx == NULL)
1117 : : last_idx = &idx;
1118 : :
1119 : 0 : ret = (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,
1120 : : last_idx, status);
1121 : : rte_dma_trace_completed_status(dev_id, vchan, nb_cpls, last_idx, status,
1122 : : ret);
1123 : :
1124 : : return ret;
1125 : : }
1126 : :
1127 : : /**
1128 : : * Check remaining capacity in descriptor ring for the current burst.
1129 : : *
1130 : : * @param dev_id
1131 : : * The identifier of the device.
1132 : : * @param vchan
1133 : : * The identifier of virtual DMA channel.
1134 : : *
1135 : : * @return
1136 : : * - Remaining space in the descriptor ring for the current burst.
1137 : : * - 0 on error
1138 : : */
1139 : : static inline uint16_t
1140 : : rte_dma_burst_capacity(int16_t dev_id, uint16_t vchan)
1141 : : {
1142 : 0 : struct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];
1143 : : uint16_t ret;
1144 : :
1145 : : #ifdef RTE_DMADEV_DEBUG
1146 : : if (!rte_dma_is_valid(dev_id))
1147 : : return 0;
1148 : : if (*obj->burst_capacity == NULL)
1149 : : return 0;
1150 : : #endif
1151 : 0 : ret = (*obj->burst_capacity)(obj->dev_private, vchan);
1152 : : rte_dma_trace_burst_capacity(dev_id, vchan, ret);
1153 : :
1154 : : return ret;
1155 : : }
1156 : :
1157 : : #ifdef __cplusplus
1158 : : }
1159 : : #endif
1160 : :
1161 : : #endif /* RTE_DMADEV_H */
|
