include/linux/context_tracking.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_CONTEXT_TRACKING_H
   3 #define _LINUX_CONTEXT_TRACKING_H
   4
   5 #include <linux/sched.h>
   6 #include <linux/vtime.h>
   7 #include <linux/context_tracking_state.h>
   8 #include <linux/instrumentation.h>
   9
  10 #include <asm/ptrace.h>
  11
  12
  13 #ifdef CONFIG_CONTEXT_TRACKING_USER
  14 extern void ct_cpu_track_user(int cpu);
  15
  16 /* Called with interrupts disabled.  */
  17 extern void __ct_user_enter(enum ctx_state state);
  18 extern void __ct_user_exit(enum ctx_state state);
  19
  20 extern void ct_user_enter(enum ctx_state state);
  21 extern void ct_user_exit(enum ctx_state state);
  22
  23 extern void user_enter_callable(void);
  24 extern void user_exit_callable(void);
  25
  26 static inline void user_enter(void)
  27 {
  28         if (context_tracking_enabled())
  29                 ct_user_enter(CT_STATE_USER);
  30
  31 }
  32 static inline void user_exit(void)
  33 {
  34         if (context_tracking_enabled())
  35                 ct_user_exit(CT_STATE_USER);
  36 }
  37
  38 /* Called with interrupts disabled.  */
  39 static __always_inline void user_enter_irqoff(void)
  40 {
  41         if (context_tracking_enabled())
  42                 __ct_user_enter(CT_STATE_USER);
  43
  44 }
  45 static __always_inline void user_exit_irqoff(void)
  46 {
  47         if (context_tracking_enabled())
  48                 __ct_user_exit(CT_STATE_USER);
  49 }
  50
  51 static inline enum ctx_state exception_enter(void)
  52 {
  53         enum ctx_state prev_ctx;
  54
  55         if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) ||
  56             !context_tracking_enabled())
  57                 return 0;
  58
  59         prev_ctx = __ct_state();
  60         if (prev_ctx != CT_STATE_KERNEL)
  61                 ct_user_exit(prev_ctx);
  62
  63         return prev_ctx;
  64 }
  65
  66 static inline void exception_exit(enum ctx_state prev_ctx)
  67 {
  68         if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) &&
  69             context_tracking_enabled()) {
  70                 if (prev_ctx != CT_STATE_KERNEL)
  71                         ct_user_enter(prev_ctx);
  72         }
  73 }
  74
  75 static __always_inline bool context_tracking_guest_enter(void)
  76 {
  77         if (context_tracking_enabled())
  78                 __ct_user_enter(CT_STATE_GUEST);
  79
  80         return context_tracking_enabled_this_cpu();
  81 }
  82
  83 static __always_inline void context_tracking_guest_exit(void)
  84 {
  85         if (context_tracking_enabled())
  86                 __ct_user_exit(CT_STATE_GUEST);
  87 }
  88
  89 #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond))
  90
  91 #else
  92 static inline void user_enter(void) { }
  93 static inline void user_exit(void) { }
  94 static inline void user_enter_irqoff(void) { }
  95 static inline void user_exit_irqoff(void) { }
  96 static inline int exception_enter(void) { return 0; }
  97 static inline void exception_exit(enum ctx_state prev_ctx) { }
  98 static inline int ct_state(void) { return -1; }
  99 static inline int __ct_state(void) { return -1; }
 100 static __always_inline bool context_tracking_guest_enter(void) { return false; }
 101 static __always_inline void context_tracking_guest_exit(void) { }
 102 #define CT_WARN_ON(cond) do { } while (0)
 103 #endif /* !CONFIG_CONTEXT_TRACKING_USER */
 104
 105 #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE
 106 extern void context_tracking_init(void);
 107 #else
 108 static inline void context_tracking_init(void) { }
 109 #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */
 110
 111 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
 112 extern void ct_idle_enter(void);
 113 extern void ct_idle_exit(void);
 114
 115 /*
 116  * Is RCU watching the current CPU (IOW, it is not in an extended quiescent state)?
 117  *
 118  * Note that this returns the actual boolean data (watching / not watching),
 119  * whereas ct_rcu_watching() returns the RCU_WATCHING subvariable of
 120  * context_tracking.state.
 121  *
 122  * No ordering, as we are sampling CPU-local information.
 123  */
 124 static __always_inline bool rcu_is_watching_curr_cpu(void)
 125 {
 126         return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_RCU_WATCHING;
 127 }
 128
 129 /*
 130  * Increment the current CPU's context_tracking structure's ->state field
 131  * with ordering.  Return the new value.
 132  */
 133 static __always_inline unsigned long ct_state_inc(int incby)
 134 {
 135         return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
 136 }
 137
 138 static __always_inline bool warn_rcu_enter(void)
 139 {
 140         bool ret = false;
 141
 142         /*
 143          * Horrible hack to shut up recursive RCU isn't watching fail since
 144          * lots of the actual reporting also relies on RCU.
 145          */
 146         preempt_disable_notrace();
 147         if (!rcu_is_watching_curr_cpu()) {
 148                 ret = true;
 149                 ct_state_inc(CT_RCU_WATCHING);
 150         }
 151
 152         return ret;
 153 }
 154
 155 static __always_inline void warn_rcu_exit(bool rcu)
 156 {
 157         if (rcu)
 158                 ct_state_inc(CT_RCU_WATCHING);
 159         preempt_enable_notrace();
 160 }
 161
 162 #else
 163 static inline void ct_idle_enter(void) { }
 164 static inline void ct_idle_exit(void) { }
 165
 166 static __always_inline bool warn_rcu_enter(void) { return false; }
 167 static __always_inline void warn_rcu_exit(bool rcu) { }
 168 #endif /* !CONFIG_CONTEXT_TRACKING_IDLE */
 169
 170 #endif