drivers/gpu/drm/i915/gt/selftest_engine_pm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/sort.h>
   7
   8 #include "gt/intel_gt_print.h"
   9 #include "i915_selftest.h"
  10 #include "intel_engine_regs.h"
  11 #include "intel_gpu_commands.h"
  12 #include "intel_gt_clock_utils.h"
  13 #include "selftest_engine.h"
  14 #include "selftest_engine_heartbeat.h"
  15 #include "selftests/igt_atomic.h"
  16 #include "selftests/igt_flush_test.h"
  17 #include "selftests/igt_spinner.h"
  18
  19 #define COUNT 5
  20
  21 static int cmp_u64(const void *A, const void *B)
  22 {
  23         const u64 *a = A, *b = B;
  24
  25         return *a - *b;
  26 }
  27
  28 static u64 trifilter(u64 *a)
  29 {
  30         sort(a, COUNT, sizeof(*a), cmp_u64, NULL);
  31         return (a[1] + 2 * a[2] + a[3]) >> 2;
  32 }
  33
  34 static u32 *emit_wait(u32 *cs, u32 offset, int op, u32 value)
  35 {
  36         *cs++ = MI_SEMAPHORE_WAIT |
  37                 MI_SEMAPHORE_GLOBAL_GTT |
  38                 MI_SEMAPHORE_POLL |
  39                 op;
  40         *cs++ = value;
  41         *cs++ = offset;
  42         *cs++ = 0;
  43
  44         return cs;
  45 }
  46
  47 static u32 *emit_store(u32 *cs, u32 offset, u32 value)
  48 {
  49         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  50         *cs++ = offset;
  51         *cs++ = 0;
  52         *cs++ = value;
  53
  54         return cs;
  55 }
  56
  57 static u32 *emit_srm(u32 *cs, i915_reg_t reg, u32 offset)
  58 {
  59         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
  60         *cs++ = i915_mmio_reg_offset(reg);
  61         *cs++ = offset;
  62         *cs++ = 0;
  63
  64         return cs;
  65 }
  66
  67 static void write_semaphore(u32 *x, u32 value)
  68 {
  69         WRITE_ONCE(*x, value);
  70         wmb();
  71 }
  72
  73 static int __measure_timestamps(struct intel_context *ce,
  74                                 u64 *dt, u64 *d_ring, u64 *d_ctx)
  75 {
  76         struct intel_engine_cs *engine = ce->engine;
  77         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 5);
  78         u32 offset = i915_ggtt_offset(engine->status_page.vma);
  79         struct i915_request *rq;
  80         u32 *cs;
  81
  82         rq = intel_context_create_request(ce);
  83         if (IS_ERR(rq))
  84                 return PTR_ERR(rq);
  85
  86         cs = intel_ring_begin(rq, 28);
  87         if (IS_ERR(cs)) {
  88                 i915_request_add(rq);
  89                 return PTR_ERR(cs);
  90         }
  91
  92         /* Signal & wait for start */
  93         cs = emit_store(cs, offset + 4008, 1);
  94         cs = emit_wait(cs, offset + 4008, MI_SEMAPHORE_SAD_NEQ_SDD, 1);
  95
  96         cs = emit_srm(cs, RING_TIMESTAMP(engine->mmio_base), offset + 4000);
  97         cs = emit_srm(cs, RING_CTX_TIMESTAMP(engine->mmio_base), offset + 4004);
  98
  99         /* Busy wait */
 100         cs = emit_wait(cs, offset + 4008, MI_SEMAPHORE_SAD_EQ_SDD, 1);
 101
 102         cs = emit_srm(cs, RING_TIMESTAMP(engine->mmio_base), offset + 4016);
 103         cs = emit_srm(cs, RING_CTX_TIMESTAMP(engine->mmio_base), offset + 4012);
 104
 105         intel_ring_advance(rq, cs);
 106         i915_request_get(rq);
 107         i915_request_add(rq);
 108         intel_engine_flush_submission(engine);
 109
 110         /* Wait for the request to start executing, that then waits for us */
 111         while (READ_ONCE(sema[2]) == 0)
 112                 cpu_relax();
 113
 114         /* Run the request for a 100us, sampling timestamps before/after */
 115         local_irq_disable();
 116         write_semaphore(&sema[2], 0);
 117         while (READ_ONCE(sema[1]) == 0) /* wait for the gpu to catch up */
 118                 cpu_relax();
 119         *dt = local_clock();
 120         udelay(100);
 121         *dt = local_clock() - *dt;
 122         write_semaphore(&sema[2], 1);
 123         local_irq_enable();
 124
 125         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
 126                 i915_request_put(rq);
 127                 return -ETIME;
 128         }
 129         i915_request_put(rq);
 130
 131         pr_debug("%s CTX_TIMESTAMP: [%x, %x], RING_TIMESTAMP: [%x, %x]\n",
 132                  engine->name, sema[1], sema[3], sema[0], sema[4]);
 133
 134         *d_ctx = sema[3] - sema[1];
 135         *d_ring = sema[4] - sema[0];
 136         return 0;
 137 }
 138
 139 static int __live_engine_timestamps(struct intel_engine_cs *engine)
 140 {
 141         u64 s_ring[COUNT], s_ctx[COUNT], st[COUNT], d_ring, d_ctx, dt;
 142         struct intel_context *ce;
 143         int i, err = 0;
 144
 145         ce = intel_context_create(engine);
 146         if (IS_ERR(ce))
 147                 return PTR_ERR(ce);
 148
 149         for (i = 0; i < COUNT; i++) {
 150                 err = __measure_timestamps(ce, &st[i], &s_ring[i], &s_ctx[i]);
 151                 if (err)
 152                         break;
 153         }
 154         intel_context_put(ce);
 155         if (err)
 156                 return err;
 157
 158         dt = trifilter(st);
 159         d_ring = trifilter(s_ring);
 160         d_ctx = trifilter(s_ctx);
 161
 162         pr_info("%s elapsed:%lldns, CTX_TIMESTAMP:%lldns, RING_TIMESTAMP:%lldns\n",
 163                 engine->name, dt,
 164                 intel_gt_clock_interval_to_ns(engine->gt, d_ctx),
 165                 intel_gt_clock_interval_to_ns(engine->gt, d_ring));
 166
 167         d_ring = intel_gt_clock_interval_to_ns(engine->gt, d_ring);
 168         if (3 * dt > 4 * d_ring || 4 * dt < 3 * d_ring) {
 169                 pr_err("%s Mismatch between ring timestamp and walltime!\n",
 170                        engine->name);
 171                 return -EINVAL;
 172         }
 173
 174         d_ring = trifilter(s_ring);
 175         d_ctx = trifilter(s_ctx);
 176
 177         d_ctx *= engine->gt->clock_frequency;
 178         if (GRAPHICS_VER(engine->i915) == 11)
 179                 d_ring *= 12500000; /* Fixed 80ns for GEN11 ctx timestamp? */
 180         else
 181                 d_ring *= engine->gt->clock_frequency;
 182
 183         if (3 * d_ctx > 4 * d_ring || 4 * d_ctx < 3 * d_ring) {
 184                 pr_err("%s Mismatch between ring and context timestamps!\n",
 185                        engine->name);
 186                 return -EINVAL;
 187         }
 188
 189         return 0;
 190 }
 191
 192 static int live_engine_timestamps(void *arg)
 193 {
 194         struct intel_gt *gt = arg;
 195         struct intel_engine_cs *engine;
 196         enum intel_engine_id id;
 197
 198         /*
 199          * Check that CS_TIMESTAMP / CTX_TIMESTAMP are in sync, i.e. share
 200          * the same CS clock.
 201          */
 202
 203         if (GRAPHICS_VER(gt->i915) < 8)
 204                 return 0;
 205
 206         for_each_engine(engine, gt, id) {
 207                 int err;
 208
 209                 st_engine_heartbeat_disable(engine);
 210                 err = __live_engine_timestamps(engine);
 211                 st_engine_heartbeat_enable(engine);
 212                 if (err)
 213                         return err;
 214         }
 215
 216         return 0;
 217 }
 218
 219 static int __spin_until_busier(struct intel_engine_cs *engine, ktime_t busyness)
 220 {
 221         ktime_t start, unused, dt;
 222
 223         if (!intel_engine_uses_guc(engine))
 224                 return 0;
 225
 226         /*
 227          * In GuC mode of submission, the busyness stats may get updated after
 228          * the batch starts running. Poll for a change in busyness and timeout
 229          * after 500 us.
 230          */
 231         start = ktime_get();
 232         while (intel_engine_get_busy_time(engine, &unused) == busyness) {
 233                 dt = ktime_get() - start;
 234                 if (dt > 10000000) {
 235                         pr_err("active wait timed out %lld\n", dt);
 236                         ENGINE_TRACE(engine, "active wait time out %lld\n", dt);
 237                         return -ETIME;
 238                 }
 239         }
 240
 241         return 0;
 242 }
 243
 244 static int live_engine_busy_stats(void *arg)
 245 {
 246         struct intel_gt *gt = arg;
 247         struct intel_engine_cs *engine;
 248         enum intel_engine_id id;
 249         struct igt_spinner spin;
 250         int err = 0;
 251
 252         /*
 253          * Check that if an engine supports busy-stats, they tell the truth.
 254          */
 255
 256         if (igt_spinner_init(&spin, gt))
 257                 return -ENOMEM;
 258
 259         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 260         for_each_engine(engine, gt, id) {
 261                 struct i915_request *rq;
 262                 ktime_t busyness, dummy;
 263                 ktime_t de, dt;
 264                 ktime_t t[2];
 265
 266                 if (!intel_engine_supports_stats(engine))
 267                         continue;
 268
 269                 if (!intel_engine_can_store_dword(engine))
 270                         continue;
 271
 272                 if (intel_gt_pm_wait_for_idle(gt)) {
 273                         err = -EBUSY;
 274                         break;
 275                 }
 276
 277                 st_engine_heartbeat_disable(engine);
 278
 279                 ENGINE_TRACE(engine, "measuring idle time\n");
 280                 preempt_disable();
 281                 de = intel_engine_get_busy_time(engine, &t[0]);
 282                 udelay(100);
 283                 de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
 284                 preempt_enable();
 285                 dt = ktime_sub(t[1], t[0]);
 286                 if (de < 0 || de > 10) {
 287                         pr_err("%s: reported %lldns [%d%%] busyness while sleeping [for %lldns]\n",
 288                                engine->name,
 289                                de, (int)div64_u64(100 * de, dt), dt);
 290                         GEM_TRACE_DUMP();
 291                         err = -EINVAL;
 292                         goto end;
 293                 }
 294
 295                 /* 100% busy */
 296                 rq = igt_spinner_create_request(&spin,
 297                                                 engine->kernel_context,
 298                                                 MI_NOOP);
 299                 if (IS_ERR(rq)) {
 300                         err = PTR_ERR(rq);
 301                         goto end;
 302                 }
 303                 i915_request_add(rq);
 304
 305                 busyness = intel_engine_get_busy_time(engine, &dummy);
 306                 if (!igt_wait_for_spinner(&spin, rq)) {
 307                         intel_gt_set_wedged(engine->gt);
 308                         err = -ETIME;
 309                         goto end;
 310                 }
 311
 312                 err = __spin_until_busier(engine, busyness);
 313                 if (err) {
 314                         GEM_TRACE_DUMP();
 315                         goto end;
 316                 }
 317
 318                 ENGINE_TRACE(engine, "measuring busy time\n");
 319                 preempt_disable();
 320                 de = intel_engine_get_busy_time(engine, &t[0]);
 321                 mdelay(100);
 322                 de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
 323                 preempt_enable();
 324                 dt = ktime_sub(t[1], t[0]);
 325                 if (100 * de < 95 * dt || 95 * de > 100 * dt) {
 326                         pr_err("%s: reported %lldns [%d%%] busyness while spinning [for %lldns]\n",
 327                                engine->name,
 328                                de, (int)div64_u64(100 * de, dt), dt);
 329                         GEM_TRACE_DUMP();
 330                         err = -EINVAL;
 331                         goto end;
 332                 }
 333
 334 end:
 335                 st_engine_heartbeat_enable(engine);
 336                 igt_spinner_end(&spin);
 337                 if (igt_flush_test(gt->i915))
 338                         err = -EIO;
 339                 if (err)
 340                         break;
 341         }
 342
 343         igt_spinner_fini(&spin);
 344         if (igt_flush_test(gt->i915))
 345                 err = -EIO;
 346         return err;
 347 }
 348
 349 static int live_engine_pm(void *arg)
 350 {
 351         struct intel_gt *gt = arg;
 352         struct intel_engine_cs *engine;
 353         enum intel_engine_id id;
 354
 355         /*
 356          * Check we can call intel_engine_pm_put from any context. No
 357          * failures are reported directly, but if we mess up lockdep should
 358          * tell us.
 359          */
 360         if (intel_gt_pm_wait_for_idle(gt)) {
 361                 pr_err("Unable to flush GT pm before test\n");
 362                 return -EBUSY;
 363         }
 364
 365         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 366         for_each_engine(engine, gt, id) {
 367                 const typeof(*igt_atomic_phases) *p;
 368
 369                 for (p = igt_atomic_phases; p->name; p++) {
 370                         /*
 371                          * Acquisition is always synchronous, except if we
 372                          * know that the engine is already awake, in which
 373                          * case we should use intel_engine_pm_get_if_awake()
 374                          * to atomically grab the wakeref.
 375                          *
 376                          * In practice,
 377                          *    intel_engine_pm_get();
 378                          *    intel_engine_pm_put();
 379                          * occurs in one thread, while simultaneously
 380                          *    intel_engine_pm_get_if_awake();
 381                          *    intel_engine_pm_put();
 382                          * occurs from atomic context in another.
 383                          */
 384                         GEM_BUG_ON(intel_engine_pm_is_awake(engine));
 385                         intel_engine_pm_get(engine);
 386
 387                         p->critical_section_begin();
 388                         if (!intel_engine_pm_get_if_awake(engine))
 389                                 pr_err("intel_engine_pm_get_if_awake(%s) failed under %s\n",
 390                                        engine->name, p->name);
 391                         else
 392                                 intel_engine_pm_put_async(engine);
 393                         intel_engine_pm_put_async(engine);
 394                         p->critical_section_end();
 395
 396                         intel_engine_pm_flush(engine);
 397
 398                         if (intel_engine_pm_is_awake(engine)) {
 399                                 pr_err("%s is still awake after flushing pm\n",
 400                                        engine->name);
 401                                 return -EINVAL;
 402                         }
 403
 404                         /* gt wakeref is async (deferred to workqueue) */
 405                         if (intel_gt_pm_wait_for_idle(gt)) {
 406                                 gt_err(gt, "GT failed to idle\n");
 407                                 return -EINVAL;
 408                         }
 409                 }
 410         }
 411
 412         return 0;
 413 }
 414
 415 int live_engine_pm_selftests(struct intel_gt *gt)
 416 {
 417         static const struct i915_subtest tests[] = {
 418                 SUBTEST(live_engine_timestamps),
 419                 SUBTEST(live_engine_busy_stats),
 420                 SUBTEST(live_engine_pm),
 421         };
 422
 423         return intel_gt_live_subtests(tests, gt);
 424 }