Branch data     Line data    Source code

       1                 :            : /* SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
       3                 :            :  */
       4                 :            : 
       5                 :            : #ifndef _VIRTIO_RING_H_
       6                 :            : #define _VIRTIO_RING_H_
       7                 :            : 
       8                 :            : #include <stdint.h>
       9                 :            : 
      10                 :            : #include <rte_common.h>
      11                 :            : 
      12                 :            : /* This marks a buffer as continuing via the next field. */
      13                 :            : #define VRING_DESC_F_NEXT       1
      14                 :            : /* This marks a buffer as write-only (otherwise read-only). */
      15                 :            : #define VRING_DESC_F_WRITE      2
      16                 :            : /* This means the buffer contains a list of buffer descriptors. */
      17                 :            : #define VRING_DESC_F_INDIRECT   4
      18                 :            : 
      19                 :            : /* This flag means the descriptor was made available by the driver */
      20                 :            : #define VRING_PACKED_DESC_F_AVAIL       (1 << 7)
      21                 :            : /* This flag means the descriptor was used by the device */
      22                 :            : #define VRING_PACKED_DESC_F_USED        (1 << 15)
      23                 :            : 
      24                 :            : /* Frequently used combinations */
      25                 :            : #define VRING_PACKED_DESC_F_AVAIL_USED  (VRING_PACKED_DESC_F_AVAIL | \
      26                 :            :                                          VRING_PACKED_DESC_F_USED)
      27                 :            : 
      28                 :            : /* The Host uses this in used->flags to advise the Guest: don't kick me
      29                 :            :  * when you add a buffer.  It's unreliable, so it's simply an
      30                 :            :  * optimization.  Guest will still kick if it's out of buffers.
      31                 :            :  */
      32                 :            : #define VRING_USED_F_NO_NOTIFY  1
      33                 :            : /* The Guest uses this in avail->flags to advise the Host: don't
      34                 :            :  * interrupt me when you consume a buffer.  It's unreliable, so it's
      35                 :            :  * simply an optimization.
      36                 :            :  */
      37                 :            : #define VRING_AVAIL_F_NO_INTERRUPT  1
      38                 :            : 
      39                 :            : /* VirtIO ring descriptors: 16 bytes.
      40                 :            :  * These can chain together via "next".
      41                 :            :  */
      42                 :            : struct vring_desc {
      43                 :            :         uint64_t addr;  /*  Address (guest-physical). */
      44                 :            :         uint32_t len;   /* Length. */
      45                 :            :         uint16_t flags; /* The flags as indicated above. */
      46                 :            :         uint16_t next;  /* We chain unused descriptors via this. */
      47                 :            : };
      48                 :            : 
      49                 :            : struct vring_avail {
      50                 :            :         uint16_t flags;
      51                 :            :         uint16_t idx;
      52                 :            :         uint16_t ring[];
      53                 :            : };
      54                 :            : 
      55                 :            : /* id is a 16bit index. uint32_t is used here for ids for padding reasons. */
      56                 :            : struct vring_used_elem {
      57                 :            :         /* Index of start of used descriptor chain. */
      58                 :            :         uint32_t id;
      59                 :            :         /* Total length of the descriptor chain which was written to. */
      60                 :            :         uint32_t len;
      61                 :            : };
      62                 :            : 
      63                 :            : struct vring_used {
      64                 :            :         uint16_t flags;
      65                 :            :         volatile uint16_t idx;
      66                 :            :         struct vring_used_elem ring[];
      67                 :            : };
      68                 :            : 
      69                 :            : /* For support of packed virtqueues in Virtio 1.1 the format of descriptors
      70                 :            :  * looks like this.
      71                 :            :  */
      72                 :            : struct vring_packed_desc {
      73                 :            :         uint64_t addr;
      74                 :            :         uint32_t len;
      75                 :            :         uint16_t id;
      76                 :            :         uint16_t flags;
      77                 :            : };
      78                 :            : 
      79                 :            : #define RING_EVENT_FLAGS_ENABLE 0x0
      80                 :            : #define RING_EVENT_FLAGS_DISABLE 0x1
      81                 :            : #define RING_EVENT_FLAGS_DESC 0x2
      82                 :            : struct vring_packed_desc_event {
      83                 :            :         uint16_t desc_event_off_wrap;
      84                 :            :         uint16_t desc_event_flags;
      85                 :            : };
      86                 :            : 
      87                 :            : struct vring_packed {
      88                 :            :         unsigned int num;
      89                 :            :         rte_iova_t desc_iova;
      90                 :            :         struct vring_packed_desc *desc;
      91                 :            :         struct vring_packed_desc_event *driver;
      92                 :            :         struct vring_packed_desc_event *device;
      93                 :            : };
      94                 :            : 
      95                 :            : struct vring {
      96                 :            :         unsigned int num;
      97                 :            :         rte_iova_t desc_iova;
      98                 :            :         struct vring_desc  *desc;
      99                 :            :         struct vring_avail *avail;
     100                 :            :         struct vring_used  *used;
     101                 :            : };
     102                 :            : 
     103                 :            : /* The standard layout for the ring is a continuous chunk of memory which
     104                 :            :  * looks like this.  We assume num is a power of 2.
     105                 :            :  *
     106                 :            :  * struct vring {
     107                 :            :  *      // The actual descriptors (16 bytes each)
     108                 :            :  *      struct vring_desc desc[num];
     109                 :            :  *
     110                 :            :  *      // A ring of available descriptor heads with free-running index.
     111                 :            :  *      __u16 avail_flags;
     112                 :            :  *      __u16 avail_idx;
     113                 :            :  *      __u16 available[num];
     114                 :            :  *      __u16 used_event_idx;
     115                 :            :  *
     116                 :            :  *      // Padding to the next align boundary.
     117                 :            :  *      char pad[];
     118                 :            :  *
     119                 :            :  *      // A ring of used descriptor heads with free-running index.
     120                 :            :  *      __u16 used_flags;
     121                 :            :  *      __u16 used_idx;
     122                 :            :  *      struct vring_used_elem used[num];
     123                 :            :  *      __u16 avail_event_idx;
     124                 :            :  * };
     125                 :            :  *
     126                 :            :  * NOTE: for VirtIO PCI, align is 4096.
     127                 :            :  */
     128                 :            : 
     129                 :            : /*
     130                 :            :  * We publish the used event index at the end of the available ring, and vice
     131                 :            :  * versa. They are at the end for backwards compatibility.
     132                 :            :  */
     133                 :            : #define vring_used_event(vr)  ((vr)->avail->ring[(vr)->num])
     134                 :            : #define vring_avail_event(vr) (*(uint16_t *)&(vr)->used->ring[(vr)->num])
     135                 :            : 
     136                 :            : static inline size_t
     137         [ #  # ]:          0 : vring_size(struct virtio_crypto_hw *hw, unsigned int num, unsigned long align)
     138                 :            : {
     139                 :            :         size_t size;
     140                 :            : 
     141         [ #  # ]:          0 :         if (vtpci_with_packed_queue(hw)) {
     142                 :          0 :                 size = num * sizeof(struct vring_packed_desc);
     143                 :          0 :                 size += sizeof(struct vring_packed_desc_event);
     144                 :          0 :                 size = RTE_ALIGN_CEIL(size, align);
     145                 :          0 :                 size += sizeof(struct vring_packed_desc_event);
     146                 :          0 :                 return size;
     147                 :            :         }
     148                 :            : 
     149                 :          0 :         size = num * sizeof(struct vring_desc);
     150                 :          0 :         size += sizeof(struct vring_avail) + (num * sizeof(uint16_t));
     151                 :          0 :         size = RTE_ALIGN_CEIL(size, align);
     152                 :          0 :         size += sizeof(struct vring_used) +
     153                 :          0 :                 (num * sizeof(struct vring_used_elem));
     154                 :          0 :         return size;
     155                 :            : }
     156                 :            : 
     157                 :            : static inline void
     158                 :            : vring_init_split(struct vring *vr, uint8_t *p, rte_iova_t iova,
     159                 :            :                  unsigned long align, unsigned int num)
     160                 :            : {
     161                 :          0 :         vr->num = num;
     162                 :          0 :         vr->desc = (struct vring_desc *) p;
     163                 :          0 :         vr->desc_iova = iova;
     164                 :          0 :         vr->avail = (struct vring_avail *) (p +
     165                 :          0 :                 num * sizeof(struct vring_desc));
     166                 :          0 :         vr->used = (void *)
     167                 :          0 :                 RTE_ALIGN_CEIL((uintptr_t)(&vr->avail->ring[num]), align);
     168                 :            : }
     169                 :            : 
     170                 :            : static inline void
     171                 :            : vring_init_packed(struct vring_packed *vr, uint8_t *p, rte_iova_t iova,
     172                 :            :                   unsigned long align, unsigned int num)
     173                 :            : {
     174                 :          0 :         vr->num = num;
     175                 :          0 :         vr->desc = (struct vring_packed_desc *)p;
     176                 :          0 :         vr->desc_iova = iova;
     177                 :          0 :         vr->driver = (struct vring_packed_desc_event *)(p +
     178                 :          0 :                         vr->num * sizeof(struct vring_packed_desc));
     179                 :          0 :         vr->device = (struct vring_packed_desc_event *)
     180                 :          0 :                 RTE_ALIGN_CEIL(((uintptr_t)vr->driver +
     181                 :            :                                 sizeof(struct vring_packed_desc_event)), align);
     182                 :          0 : }
     183                 :            : 
     184                 :            : /*
     185                 :            :  * The following is used with VIRTIO_RING_F_EVENT_IDX.
     186                 :            :  * Assuming a given event_idx value from the other size, if we have
     187                 :            :  * just incremented index from old to new_idx, should we trigger an
     188                 :            :  * event?
     189                 :            :  */
     190                 :            : static inline int
     191                 :            : vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
     192                 :            : {
     193                 :            :         return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old);
     194                 :            : }
     195                 :            : 
     196                 :            : #endif /* _VIRTIO_RING_H_ */