Branch data     Line data    Source code

       1                 :            : /* SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  * Copyright(c) 2010-2015 Intel Corporation
       3                 :            :  */
       4                 :            : 
       5                 :            : #ifndef _RTE_PREFETCH_H_
       6                 :            : #define _RTE_PREFETCH_H_
       7                 :            : 
       8                 :            : #include <rte_compat.h>
       9                 :            : 
      10                 :            : #ifdef __cplusplus
      11                 :            : extern "C" {
      12                 :            : #endif
      13                 :            : 
      14                 :            : /**
      15                 :            :  * @file
      16                 :            :  *
      17                 :            :  * Prefetch operations.
      18                 :            :  *
      19                 :            :  * This file defines an API for prefetch macros / inline-functions,
      20                 :            :  * which are architecture-dependent. Prefetching occurs when a
      21                 :            :  * processor requests an instruction or data from memory to cache
      22                 :            :  * before it is actually needed, potentially speeding up the execution of the
      23                 :            :  * program.
      24                 :            :  */
      25                 :            : 
      26                 :            : /**
      27                 :            :  * Prefetch a cache line into all cache levels.
      28                 :            :  * @param p
      29                 :            :  *   Address to prefetch
      30                 :            :  */
      31                 :            : static inline void rte_prefetch0(const volatile void *p);
      32                 :            : 
      33                 :            : /**
      34                 :            :  * Prefetch a cache line into all cache levels except the 0th cache level.
      35                 :            :  * @param p
      36                 :            :  *   Address to prefetch
      37                 :            :  */
      38                 :            : static inline void rte_prefetch1(const volatile void *p);
      39                 :            : 
      40                 :            : /**
      41                 :            :  * Prefetch a cache line into all cache levels except the 0th and 1th cache
      42                 :            :  * levels.
      43                 :            :  * @param p
      44                 :            :  *   Address to prefetch
      45                 :            :  */
      46                 :            : static inline void rte_prefetch2(const volatile void *p);
      47                 :            : 
      48                 :            : /**
      49                 :            :  * Prefetch a cache line into all cache levels (non-temporal/transient version)
      50                 :            :  *
      51                 :            :  * The non-temporal prefetch is intended as a prefetch hint that processor will
      52                 :            :  * use the prefetched data only once or short period, unlike the
      53                 :            :  * rte_prefetch0() function which imply that prefetched data to use repeatedly.
      54                 :            :  *
      55                 :            :  * @param p
      56                 :            :  *   Address to prefetch
      57                 :            :  */
      58                 :            : static inline void rte_prefetch_non_temporal(const volatile void *p);
      59                 :            : 
      60                 :            : /**
      61                 :            :  * @warning
      62                 :            :  * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
      63                 :            :  *
      64                 :            :  * Prefetch a cache line into all cache levels, with intention to write. This
      65                 :            :  * prefetch variant hints to the CPU that the program is expecting to write to
      66                 :            :  * the cache line being prefetched.
      67                 :            :  *
      68                 :            :  * @param p Address to prefetch
      69                 :            :  */
      70                 :            : __rte_experimental
      71                 :            : static inline void
      72                 :          0 : rte_prefetch0_write(const void *p)
      73                 :            : {
      74                 :            :         /* 1 indicates intention to write, 3 sets target cache level to L1. See
      75                 :            :          * GCC docs where these integer constants are described in more detail:
      76                 :            :          *  https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
      77                 :            :          */
      78                 :            : #ifdef RTE_TOOLCHAIN_MSVC
      79                 :            :         rte_prefetch0(p);
      80                 :            : #else
      81                 :          1 :         __builtin_prefetch(p, 1, 3);
      82                 :            : #endif
      83                 :          0 : }
      84                 :            : 
      85                 :            : /**
      86                 :            :  * @warning
      87                 :            :  * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
      88                 :            :  *
      89                 :            :  * Prefetch a cache line into all cache levels, except the 0th, with intention
      90                 :            :  * to write. This prefetch variant hints to the CPU that the program is
      91                 :            :  * expecting to write to the cache line being prefetched.
      92                 :            :  *
      93                 :            :  * @param p Address to prefetch
      94                 :            :  */
      95                 :            : __rte_experimental
      96                 :            : static inline void
      97                 :            : rte_prefetch1_write(const void *p)
      98                 :            : {
      99                 :            :         /* 1 indicates intention to write, 2 sets target cache level to L2. See
     100                 :            :          * GCC docs where these integer constants are described in more detail:
     101                 :            :          *  https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
     102                 :            :          */
     103                 :            : #ifdef RTE_TOOLCHAIN_MSVC
     104                 :            :         rte_prefetch1(p);
     105                 :            : #else
     106                 :          1 :         __builtin_prefetch(p, 1, 2);
     107                 :            : #endif
     108                 :            : }
     109                 :            : 
     110                 :            : /**
     111                 :            :  * @warning
     112                 :            :  * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
     113                 :            :  *
     114                 :            :  * Prefetch a cache line into all cache levels, except the 0th and 1st, with
     115                 :            :  * intention to write. This prefetch variant hints to the CPU that the program
     116                 :            :  * is expecting to write to the cache line being prefetched.
     117                 :            :  *
     118                 :            :  * @param p Address to prefetch
     119                 :            :  */
     120                 :            : __rte_experimental
     121                 :            : static inline void
     122                 :            : rte_prefetch2_write(const void *p)
     123                 :            : {
     124                 :            :         /* 1 indicates intention to write, 1 sets target cache level to L3. See
     125                 :            :          * GCC docs where these integer constants are described in more detail:
     126                 :            :          *  https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
     127                 :            :          */
     128                 :            : #ifdef RTE_TOOLCHAIN_MSVC
     129                 :            :         rte_prefetch2(p);
     130                 :            : #else
     131                 :          1 :         __builtin_prefetch(p, 1, 1);
     132                 :            : #endif
     133                 :            : }
     134                 :            : 
     135                 :            : /**
     136                 :            :  * @warning
     137                 :            :  * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
     138                 :            :  *
     139                 :            :  * Demote a cache line to a more distant level of cache from the processor.
     140                 :            :  * CLDEMOTE hints to hardware to move (demote) a cache line from the closest to
     141                 :            :  * the processor to a level more distant from the processor. It is a hint and
     142                 :            :  * not guaranteed. rte_cldemote is intended to move the cache line to the more
     143                 :            :  * remote cache, where it expects sharing to be efficient and to indicate that
     144                 :            :  * a line may be accessed by a different core in the future.
     145                 :            :  *
     146                 :            :  * @param p
     147                 :            :  *   Address to demote
     148                 :            :  */
     149                 :            : __rte_experimental
     150                 :            : static inline void
     151                 :            : rte_cldemote(const volatile void *p);
     152                 :            : 
     153                 :            : #ifdef __cplusplus
     154                 :            : }
     155                 :            : #endif
     156                 :            : 
     157                 :            : #endif /* _RTE_PREFETCH_H_ */