Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(C) 2022 Marvell.
3 : : */
4 : :
5 : : #ifndef _CNXK_SG_H_
6 : : #define _CNXK_SG_H_
7 : :
8 : : #include "roc_cpt_sg.h"
9 : : #include "roc_se.h"
10 : :
11 : : static __rte_always_inline uint32_t
12 : : fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
13 : : {
14 : : struct roc_sglist_comp *to = &list[i >> 2];
15 : :
16 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
17 [ # # # # : 0 : to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
18 : : return ++i;
19 : : }
20 : :
21 : : static __rte_always_inline uint32_t
22 : : fill_sg_comp_from_buf(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from)
23 : : {
24 : 0 : struct roc_sglist_comp *to = &list[i >> 2];
25 : :
26 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # #
# ]
27 : 0 : to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
28 : 0 : return ++i;
29 : : }
30 : :
31 : : static __rte_always_inline uint32_t
32 : : fill_sg_comp_from_buf_min(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from,
33 : : uint32_t *psize)
34 : : {
35 : : struct roc_sglist_comp *to = &list[i >> 2];
36 : : uint32_t size = *psize;
37 : : uint32_t e_len;
38 : :
39 : 0 : e_len = RTE_MIN(from->size, size);
40 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
41 [ # # # # : 0 : to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
# # ]
42 : 0 : *psize -= e_len;
43 : 0 : return ++i;
44 : : }
45 : :
46 : : /*
47 : : * This fills the MC expected SGIO list
48 : : * from IOV given by user.
49 : : */
50 : : static __rte_always_inline uint32_t
51 : : fill_sg_comp_from_iov(struct roc_sglist_comp *list, uint32_t i, struct roc_se_iov_ptr *from,
52 : : uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
53 : : uint32_t extra_offset)
54 : : {
55 [ # # # # : 0 : uint32_t extra_len = extra_buf ? extra_buf->size : 0;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
56 : : uint32_t size = *psize;
57 : : int32_t j;
58 : :
59 [ # # # # : 0 : for (j = 0; j < from->buf_cnt; j++) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
60 : 0 : struct roc_sglist_comp *to = &list[i >> 2];
61 : 0 : uint32_t buf_sz = from->bufs[j].size;
62 : 0 : void *vaddr = from->bufs[j].vaddr;
63 : : uint64_t e_vaddr;
64 : : uint32_t e_len;
65 : :
66 : : if (unlikely(from_offset)) {
67 : : if (from_offset >= buf_sz) {
68 : : from_offset -= buf_sz;
69 : : continue;
70 : : }
71 : : e_vaddr = (uint64_t)vaddr + from_offset;
72 : : e_len = RTE_MIN((buf_sz - from_offset), size);
73 : : from_offset = 0;
74 : : } else {
75 : 0 : e_vaddr = (uint64_t)vaddr;
76 : 0 : e_len = RTE_MIN(buf_sz, size);
77 : : }
78 : :
79 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
80 [ # # # # : 0 : to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
81 : :
82 [ # # # # : 0 : if (extra_len && (e_len >= extra_offset)) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
83 : : /* Break the data at given offset */
84 : 0 : uint32_t next_len = e_len - extra_offset;
85 : 0 : uint64_t next_vaddr = e_vaddr + extra_offset;
86 : :
87 [ # # # # : 0 : if (!extra_offset) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
88 : 0 : i--;
89 : : } else {
90 : : e_len = extra_offset;
91 : 0 : size -= e_len;
92 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
93 : : }
94 : :
95 : 0 : extra_len = RTE_MIN(extra_len, size);
96 : : /* Insert extra data ptr */
97 [ # # # # : 0 : if (extra_len) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
98 : 0 : i++;
99 : 0 : to = &list[i >> 2];
100 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(extra_len);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
101 : 0 : to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)extra_buf->vaddr);
102 : 0 : size -= extra_len;
103 : : }
104 : :
105 : 0 : next_len = RTE_MIN(next_len, size);
106 : : /* insert the rest of the data */
107 [ # # # # : 0 : if (next_len) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
108 : 0 : i++;
109 : 0 : to = &list[i >> 2];
110 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
111 [ # # # # : 0 : to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
112 : 0 : size -= next_len;
113 : : }
114 : : extra_len = 0;
115 : :
116 : : } else {
117 : 0 : size -= e_len;
118 : : }
119 [ # # # # : 0 : if (extra_offset)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
120 : 0 : extra_offset -= size;
121 : 0 : i++;
122 : :
123 [ # # # # : 0 : if (unlikely(!size))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
124 : : break;
125 : : }
126 : :
127 : : *psize = size;
128 : : return (uint32_t)i;
129 : : }
130 : :
131 : : static __rte_always_inline uint32_t
132 : : fill_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte_mbuf *pkt)
133 : : {
134 : : uint32_t buf_sz;
135 : : void *vaddr;
136 : :
137 [ # # # # : 0 : while (unlikely(pkt != NULL)) {
# # # # #
# # # # #
# # ]
138 : 0 : struct roc_sglist_comp *to = &list[i >> 2];
139 : 0 : buf_sz = pkt->data_len;
140 : 0 : vaddr = rte_pktmbuf_mtod(pkt, void *);
141 : :
142 [ # # # # : 0 : to->u.s.len[i % 4] = rte_cpu_to_be_16(buf_sz);
# # # # #
# # # # #
# # ]
143 : 0 : to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)vaddr);
144 : :
145 : 0 : pkt = pkt->next;
146 : 0 : i++;
147 : : }
148 : :
149 : : return i;
150 : : }
151 : :
152 : : static __rte_always_inline uint32_t
153 : : fill_sg2_comp_from_pkt(struct roc_sg2list_comp *list, uint32_t i, struct rte_mbuf *pkt)
154 : : {
155 : : uint32_t buf_sz;
156 : : void *vaddr;
157 : :
158 [ # # # # : 0 : while (unlikely(pkt != NULL)) {
# # # # #
# # # # #
# # ]
159 : 0 : struct roc_sg2list_comp *to = &list[i / 3];
160 : 0 : buf_sz = pkt->data_len;
161 : 0 : vaddr = rte_pktmbuf_mtod(pkt, void *);
162 : :
163 : 0 : to->u.s.len[i % 3] = buf_sz;
164 : 0 : to->ptr[i % 3] = (uint64_t)vaddr;
165 : 0 : to->u.s.valid_segs = (i % 3) + 1;
166 : :
167 : 0 : pkt = pkt->next;
168 : 0 : i++;
169 : : }
170 : :
171 : : return i;
172 : : }
173 : :
174 : : static __rte_always_inline uint32_t
175 : : fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
176 : : {
177 : : struct roc_sg2list_comp *to = &list[i / 3];
178 : :
179 : 0 : to->u.s.len[i % 3] = (size);
180 : 0 : to->ptr[i % 3] = (dma_addr);
181 [ # # # # : 0 : to->u.s.valid_segs = (i % 3) + 1;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
182 : : return ++i;
183 : : }
184 : :
185 : : static __rte_always_inline uint32_t
186 : : fill_sg2_comp_from_buf(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from)
187 : : {
188 : 0 : struct roc_sg2list_comp *to = &list[i / 3];
189 : :
190 : 0 : to->u.s.len[i % 3] = (from->size);
191 : 0 : to->ptr[i % 3] = ((uint64_t)from->vaddr);
192 : 0 : to->u.s.valid_segs = (i % 3) + 1;
193 : 0 : return ++i;
194 : : }
195 : :
196 : : static __rte_always_inline uint32_t
197 : : fill_sg2_comp_from_buf_min(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from,
198 : : uint32_t *psize)
199 : : {
200 : : struct roc_sg2list_comp *to = &list[i / 3];
201 : : uint32_t size = *psize;
202 : : uint32_t e_len;
203 : :
204 : 0 : e_len = RTE_MIN(from->size, size);
205 : 0 : to->u.s.len[i % 3] = (e_len);
206 : 0 : to->ptr[i % 3] = ((uint64_t)from->vaddr);
207 : 0 : to->u.s.valid_segs = (i % 3) + 1;
208 [ # # # # : 0 : *psize -= e_len;
# # ]
209 : 0 : return ++i;
210 : : }
211 : :
212 : : static __rte_always_inline uint32_t
213 : : fill_sg2_comp_from_iov(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from,
214 : : uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
215 : : uint32_t extra_offset)
216 : : {
217 [ # # # # : 0 : uint32_t extra_len = extra_buf ? extra_buf->size : 0;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
218 : : uint32_t size = *psize;
219 : : int32_t j;
220 : :
221 : 0 : rte_prefetch2(psize);
222 : :
223 [ # # # # : 0 : for (j = 0; j < from->buf_cnt; j++) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
224 : 0 : struct roc_sg2list_comp *to = &list[i / 3];
225 : 0 : uint32_t buf_sz = from->bufs[j].size;
226 : 0 : void *vaddr = from->bufs[j].vaddr;
227 : : uint64_t e_vaddr;
228 : : uint32_t e_len;
229 : :
230 : : if (unlikely(from_offset)) {
231 : : if (from_offset >= buf_sz) {
232 : : from_offset -= buf_sz;
233 : : continue;
234 : : }
235 : : e_vaddr = (uint64_t)vaddr + from_offset;
236 : : e_len = RTE_MIN((buf_sz - from_offset), size);
237 : : from_offset = 0;
238 : : } else {
239 : 0 : e_vaddr = (uint64_t)vaddr;
240 : 0 : e_len = RTE_MIN(buf_sz, size);
241 : : }
242 : :
243 : 0 : to->u.s.len[i % 3] = (e_len);
244 : 0 : to->ptr[i % 3] = (e_vaddr);
245 : 0 : to->u.s.valid_segs = (i % 3) + 1;
246 : :
247 [ # # # # : 0 : if (extra_len && (e_len >= extra_offset)) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
248 : : /* Break the data at given offset */
249 : 0 : uint32_t next_len = e_len - extra_offset;
250 : 0 : uint64_t next_vaddr = e_vaddr + extra_offset;
251 : :
252 [ # # # # : 0 : if (!extra_offset) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
253 : 0 : i--;
254 : : } else {
255 : : e_len = extra_offset;
256 : 0 : size -= e_len;
257 : 0 : to->u.s.len[i % 3] = (e_len);
258 : : }
259 : :
260 : 0 : extra_len = RTE_MIN(extra_len, size);
261 : : /* Insert extra data ptr */
262 [ # # # # : 0 : if (extra_len) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
263 : 0 : i++;
264 : 0 : to = &list[i / 3];
265 : 0 : to->u.s.len[i % 3] = (extra_len);
266 : 0 : to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);
267 : 0 : to->u.s.valid_segs = (i % 3) + 1;
268 : 0 : size -= extra_len;
269 : : }
270 : :
271 : 0 : next_len = RTE_MIN(next_len, size);
272 : : /* insert the rest of the data */
273 [ # # # # : 0 : if (next_len) {
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
274 : 0 : i++;
275 : 0 : to = &list[i / 3];
276 : 0 : to->u.s.len[i % 3] = (next_len);
277 : 0 : to->ptr[i % 3] = (next_vaddr);
278 : 0 : to->u.s.valid_segs = (i % 3) + 1;
279 : 0 : size -= next_len;
280 : : }
281 : : extra_len = 0;
282 : :
283 : : } else {
284 : 0 : size -= e_len;
285 : : }
286 [ # # # # : 0 : if (extra_offset)
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
287 : 0 : extra_offset -= size;
288 : 0 : i++;
289 : :
290 [ # # # # : 0 : if (unlikely(!size))
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# ]
291 : : break;
292 : : }
293 : :
294 [ # # # # : 0 : *psize = size;
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
295 : : return (uint32_t)i;
296 : : }
297 : :
298 : : #endif /*_CNXK_SG_H_ */
|
