drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /*
   3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
  26 #define KFD_DEVICE_QUEUE_MANAGER_H_
  27
  28 #include <linux/rwsem.h>
  29 #include <linux/list.h>
  30 #include <linux/mutex.h>
  31 #include <linux/sched/mm.h>
  32 #include "kfd_priv.h"
  33 #include "kfd_mqd_manager.h"
  34
  35
  36 #define VMID_NUM 16
  37
  38 #define KFD_MES_PROCESS_QUANTUM         100000
  39 #define KFD_MES_GANG_QUANTUM            10000
  40 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
  41
  42 struct device_process_node {
  43         struct qcm_process_device *qpd;
  44         struct list_head list;
  45 };
  46
  47 union SQ_CMD_BITS {
  48         struct {
  49                 uint32_t cmd:3;
  50                 uint32_t:1;
  51                 uint32_t mode:3;
  52                 uint32_t check_vmid:1;
  53                 uint32_t trap_id:3;
  54                 uint32_t:5;
  55                 uint32_t wave_id:4;
  56                 uint32_t simd_id:2;
  57                 uint32_t:2;
  58                 uint32_t queue_id:3;
  59                 uint32_t:1;
  60                 uint32_t vm_id:4;
  61         } bitfields, bits;
  62         uint32_t u32All;
  63         signed int i32All;
  64         float f32All;
  65 };
  66
  67 union GRBM_GFX_INDEX_BITS {
  68         struct {
  69                 uint32_t instance_index:8;
  70                 uint32_t sh_index:8;
  71                 uint32_t se_index:8;
  72                 uint32_t:5;
  73                 uint32_t sh_broadcast_writes:1;
  74                 uint32_t instance_broadcast_writes:1;
  75                 uint32_t se_broadcast_writes:1;
  76         } bitfields, bits;
  77         uint32_t u32All;
  78         signed int i32All;
  79         float f32All;
  80 };
  81
  82 /**
  83  * struct device_queue_manager_ops
  84  *
  85  * @create_queue: Queue creation routine.
  86  *
  87  * @destroy_queue: Queue destruction routine.
  88  *
  89  * @update_queue: Queue update routine.
  90  *
  91  * @exeute_queues: Dispatches the queues list to the H/W.
  92  *
  93  * @register_process: This routine associates a specific process with device.
  94  *
  95  * @unregister_process: destroys the associations between process to device.
  96  *
  97  * @initialize: Initializes the pipelines and memory module for that device.
  98  *
  99  * @start: Initializes the resources/modules the device needs for queues
 100  * execution. This function is called on device initialization and after the
 101  * system woke up after suspension.
 102  *
 103  * @stop: This routine stops execution of all the active queue running on the
 104  * H/W and basically this function called on system suspend.
 105  *
 106  * @uninitialize: Destroys all the device queue manager resources allocated in
 107  * initialize routine.
 108  *
 109  * @halt: This routine unmaps queues from runlist and set halt status to true
 110  * so no more queues will be mapped to runlist until unhalt.
 111  *
 112  * @unhalt: This routine unset halt status to flase and maps queues back to
 113  * runlist.
 114  *
 115  * @create_kernel_queue: Creates kernel queue. Used for debug queue.
 116  *
 117  * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
 118  *
 119  * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
 120  * memory apertures.
 121  *
 122  * @process_termination: Clears all process queues belongs to that device.
 123  *
 124  * @evict_process_queues: Evict all active queues of a process
 125  *
 126  * @restore_process_queues: Restore all evicted queues of a process
 127  *
 128  * @get_wave_state: Retrieves context save state and optionally copies the
 129  * control stack, if kept in the MQD, to the given userspace address.
 130  *
 131  * @reset_queues: reset queues which consume RAS poison
 132  * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
 133  *
 134  * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
 135  */
 136
 137 struct device_queue_manager_ops {
 138         int     (*create_queue)(struct device_queue_manager *dqm,
 139                                 struct queue *q,
 140                                 struct qcm_process_device *qpd,
 141                                 const struct kfd_criu_queue_priv_data *qd,
 142                                 const void *restore_mqd,
 143                                 const void *restore_ctl_stack);
 144
 145         int     (*destroy_queue)(struct device_queue_manager *dqm,
 146                                 struct qcm_process_device *qpd,
 147                                 struct queue *q);
 148
 149         int     (*update_queue)(struct device_queue_manager *dqm,
 150                                 struct queue *q, struct mqd_update_info *minfo);
 151
 152         int     (*register_process)(struct device_queue_manager *dqm,
 153                                         struct qcm_process_device *qpd);
 154
 155         int     (*unregister_process)(struct device_queue_manager *dqm,
 156                                         struct qcm_process_device *qpd);
 157
 158         int     (*initialize)(struct device_queue_manager *dqm);
 159         int     (*start)(struct device_queue_manager *dqm);
 160         int     (*stop)(struct device_queue_manager *dqm);
 161         void    (*uninitialize)(struct device_queue_manager *dqm);
 162         int     (*halt)(struct device_queue_manager *dqm);
 163         int     (*unhalt)(struct device_queue_manager *dqm);
 164         int     (*create_kernel_queue)(struct device_queue_manager *dqm,
 165                                         struct kernel_queue *kq,
 166                                         struct qcm_process_device *qpd);
 167
 168         void    (*destroy_kernel_queue)(struct device_queue_manager *dqm,
 169                                         struct kernel_queue *kq,
 170                                         struct qcm_process_device *qpd);
 171
 172         bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 173                                            struct qcm_process_device *qpd,
 174                                            enum cache_policy default_policy,
 175                                            enum cache_policy alternate_policy,
 176                                            void __user *alternate_aperture_base,
 177                                            uint64_t alternate_aperture_size);
 178
 179         int (*process_termination)(struct device_queue_manager *dqm,
 180                         struct qcm_process_device *qpd);
 181
 182         int (*evict_process_queues)(struct device_queue_manager *dqm,
 183                                     struct qcm_process_device *qpd);
 184         int (*restore_process_queues)(struct device_queue_manager *dqm,
 185                                       struct qcm_process_device *qpd);
 186
 187         int     (*get_wave_state)(struct device_queue_manager *dqm,
 188                                   struct queue *q,
 189                                   void __user *ctl_stack,
 190                                   u32 *ctl_stack_used_size,
 191                                   u32 *save_area_used_size);
 192
 193         int (*reset_queues)(struct device_queue_manager *dqm,
 194                                         uint16_t pasid);
 195         void    (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
 196                                   const struct queue *q, u32 *mqd_size,
 197                                   u32 *ctl_stack_size);
 198
 199         int     (*checkpoint_mqd)(struct device_queue_manager *dqm,
 200                                   const struct queue *q,
 201                                   void *mqd,
 202                                   void *ctl_stack);
 203 };
 204
 205 struct device_queue_manager_asic_ops {
 206         int     (*update_qpd)(struct device_queue_manager *dqm,
 207                                         struct qcm_process_device *qpd);
 208         bool    (*set_cache_memory_policy)(struct device_queue_manager *dqm,
 209                                            struct qcm_process_device *qpd,
 210                                            enum cache_policy default_policy,
 211                                            enum cache_policy alternate_policy,
 212                                            void __user *alternate_aperture_base,
 213                                            uint64_t alternate_aperture_size);
 214         void    (*init_sdma_vm)(struct device_queue_manager *dqm,
 215                                 struct queue *q,
 216                                 struct qcm_process_device *qpd);
 217         struct mqd_manager *    (*mqd_manager_init)(enum KFD_MQD_TYPE type,
 218                                  struct kfd_node *dev);
 219 };
 220
 221 struct dqm_detect_hang_info {
 222         int pipe_id;
 223         int queue_id;
 224         int xcc_id;
 225         uint64_t queue_address;
 226 };
 227
 228 /**
 229  * struct device_queue_manager
 230  *
 231  * This struct is a base class for the kfd queues scheduler in the
 232  * device level. The device base class should expose the basic operations
 233  * for queue creation and queue destruction. This base class hides the
 234  * scheduling mode of the driver and the specific implementation of the
 235  * concrete device. This class is the only class in the queues scheduler
 236  * that configures the H/W.
 237  *
 238  */
 239
 240 struct device_queue_manager {
 241         struct device_queue_manager_ops ops;
 242         struct device_queue_manager_asic_ops asic_ops;
 243
 244         struct mqd_manager      *mqd_mgrs[KFD_MQD_TYPE_MAX];
 245         struct packet_manager   packet_mgr;
 246         struct kfd_node         *dev;
 247         struct mutex            lock_hidden; /* use dqm_lock/unlock(dqm) */
 248         struct list_head        queues;
 249         unsigned int            saved_flags;
 250         unsigned int            processes_count;
 251         unsigned int            active_queue_count;
 252         unsigned int            active_cp_queue_count;
 253         unsigned int            gws_queue_count;
 254         unsigned int            total_queue_count;
 255         unsigned int            next_pipe_to_allocate;
 256         unsigned int            *allocated_queues;
 257         DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 258         DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 259         /* the pasid mapping for each kfd vmid */
 260         uint16_t                vmid_pasid[VMID_NUM];
 261         uint64_t                pipelines_addr;
 262         uint64_t                fence_gpu_addr;
 263         uint64_t                *fence_addr;
 264         struct kfd_mem_obj      *fence_mem;
 265         bool                    active_runlist;
 266         int                     sched_policy;
 267         uint32_t                trap_debug_vmid;
 268
 269         /* hw exception  */
 270         bool                    is_hws_hang;
 271         bool                    is_resetting;
 272         struct work_struct      hw_exception_work;
 273         struct kfd_mem_obj      hiq_sdma_mqd;
 274         bool                    sched_running;
 275         bool                    sched_halt;
 276
 277         /* used for GFX 9.4.3 only */
 278         uint32_t                current_logical_xcc_start;
 279
 280         uint32_t                wait_times;
 281
 282         wait_queue_head_t       destroy_wait;
 283
 284         /* for per-queue reset support */
 285         struct dqm_detect_hang_info *detect_hang_info;
 286         size_t detect_hang_info_size;
 287         int detect_hang_count;
 288 };
 289
 290 void device_queue_manager_init_cik(
 291                 struct device_queue_manager_asic_ops *asic_ops);
 292 void device_queue_manager_init_vi(
 293                 struct device_queue_manager_asic_ops *asic_ops);
 294 void device_queue_manager_init_v9(
 295                 struct device_queue_manager_asic_ops *asic_ops);
 296 void device_queue_manager_init_v10(
 297                 struct device_queue_manager_asic_ops *asic_ops);
 298 void device_queue_manager_init_v11(
 299                 struct device_queue_manager_asic_ops *asic_ops);
 300 void device_queue_manager_init_v12(
 301                 struct device_queue_manager_asic_ops *asic_ops);
 302 void program_sh_mem_settings(struct device_queue_manager *dqm,
 303                                         struct qcm_process_device *qpd);
 304 unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
 305 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
 306 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
 307 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
 308 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
 309 int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
 310                         struct qcm_process_device *qpd);
 311 int release_debug_trap_vmid(struct device_queue_manager *dqm,
 312                         struct qcm_process_device *qpd);
 313 int suspend_queues(struct kfd_process *p,
 314                         uint32_t num_queues,
 315                         uint32_t grace_period,
 316                         uint64_t exception_clear_mask,
 317                         uint32_t *usr_queue_id_array);
 318 int resume_queues(struct kfd_process *p,
 319                 uint32_t num_queues,
 320                 uint32_t *usr_queue_id_array);
 321 void set_queue_snapshot_entry(struct queue *q,
 322                               uint64_t exception_clear_mask,
 323                               struct kfd_queue_snapshot_entry *qss_entry);
 324 int debug_lock_and_unmap(struct device_queue_manager *dqm);
 325 int debug_map_and_unlock(struct device_queue_manager *dqm);
 326 int debug_refresh_runlist(struct device_queue_manager *dqm);
 327 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
 328                                  struct qcm_process_device *qpd,
 329                                  int doorbell_off, u32 *queue_format);
 330
 331 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 332 {
 333         return (pdd->lds_base >> 16) & 0xFF;
 334 }
 335
 336 static inline unsigned int
 337 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 338 {
 339         return (pdd->lds_base >> 60) & 0x0E;
 340 }
 341
 342 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
 343  * happens while holding this lock anywhere to prevent deadlocks when
 344  * an MMU notifier runs in reclaim-FS context.
 345  */
 346 static inline void dqm_lock(struct device_queue_manager *dqm)
 347 {
 348         mutex_lock(&dqm->lock_hidden);
 349         dqm->saved_flags = memalloc_noreclaim_save();
 350 }
 351 static inline void dqm_unlock(struct device_queue_manager *dqm)
 352 {
 353         memalloc_noreclaim_restore(dqm->saved_flags);
 354         mutex_unlock(&dqm->lock_hidden);
 355 }
 356
 357 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
 358 {
 359         /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
 360         return get_user(*val, q_rptr + 1);
 361 }
 362 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */