drivers/gpu/drm/i915/gt/selftest_engine_pm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include <linux/sort.h>
   7
   8 #include "i915_selftest.h"
   9 #include "intel_engine_regs.h"
  10 #include "intel_gpu_commands.h"
  11 #include "intel_gt_clock_utils.h"
  12 #include "selftest_engine.h"
  13 #include "selftest_engine_heartbeat.h"
  14 #include "selftests/igt_atomic.h"
  15 #include "selftests/igt_flush_test.h"
  16 #include "selftests/igt_spinner.h"
  17
  18 #define COUNT 5
  19
  20 static int cmp_u64(const void *A, const void *B)
  21 {
  22         const u64 *a = A, *b = B;
  23
  24         return *a - *b;
  25 }
  26
  27 static u64 trifilter(u64 *a)
  28 {
  29         sort(a, COUNT, sizeof(*a), cmp_u64, NULL);
  30         return (a[1] + 2 * a[2] + a[3]) >> 2;
  31 }
  32
  33 static u32 *emit_wait(u32 *cs, u32 offset, int op, u32 value)
  34 {
  35         *cs++ = MI_SEMAPHORE_WAIT |
  36                 MI_SEMAPHORE_GLOBAL_GTT |
  37                 MI_SEMAPHORE_POLL |
  38                 op;
  39         *cs++ = value;
  40         *cs++ = offset;
  41         *cs++ = 0;
  42
  43         return cs;
  44 }
  45
  46 static u32 *emit_store(u32 *cs, u32 offset, u32 value)
  47 {
  48         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  49         *cs++ = offset;
  50         *cs++ = 0;
  51         *cs++ = value;
  52
  53         return cs;
  54 }
  55
  56 static u32 *emit_srm(u32 *cs, i915_reg_t reg, u32 offset)
  57 {
  58         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
  59         *cs++ = i915_mmio_reg_offset(reg);
  60         *cs++ = offset;
  61         *cs++ = 0;
  62
  63         return cs;
  64 }
  65
  66 static void write_semaphore(u32 *x, u32 value)
  67 {
  68         WRITE_ONCE(*x, value);
  69         wmb();
  70 }
  71
  72 static int __measure_timestamps(struct intel_context *ce,
  73                                 u64 *dt, u64 *d_ring, u64 *d_ctx)
  74 {
  75         struct intel_engine_cs *engine = ce->engine;
  76         u32 *sema = memset32(engine->status_page.addr + 1000, 0, 5);
  77         u32 offset = i915_ggtt_offset(engine->status_page.vma);
  78         struct i915_request *rq;
  79         u32 *cs;
  80
  81         rq = intel_context_create_request(ce);
  82         if (IS_ERR(rq))
  83                 return PTR_ERR(rq);
  84
  85         cs = intel_ring_begin(rq, 28);
  86         if (IS_ERR(cs)) {
  87                 i915_request_add(rq);
  88                 return PTR_ERR(cs);
  89         }
  90
  91         /* Signal & wait for start */
  92         cs = emit_store(cs, offset + 4008, 1);
  93         cs = emit_wait(cs, offset + 4008, MI_SEMAPHORE_SAD_NEQ_SDD, 1);
  94
  95         cs = emit_srm(cs, RING_TIMESTAMP(engine->mmio_base), offset + 4000);
  96         cs = emit_srm(cs, RING_CTX_TIMESTAMP(engine->mmio_base), offset + 4004);
  97
  98         /* Busy wait */
  99         cs = emit_wait(cs, offset + 4008, MI_SEMAPHORE_SAD_EQ_SDD, 1);
 100
 101         cs = emit_srm(cs, RING_TIMESTAMP(engine->mmio_base), offset + 4016);
 102         cs = emit_srm(cs, RING_CTX_TIMESTAMP(engine->mmio_base), offset + 4012);
 103
 104         intel_ring_advance(rq, cs);
 105         i915_request_get(rq);
 106         i915_request_add(rq);
 107         intel_engine_flush_submission(engine);
 108
 109         /* Wait for the request to start executing, that then waits for us */
 110         while (READ_ONCE(sema[2]) == 0)
 111                 cpu_relax();
 112
 113         /* Run the request for a 100us, sampling timestamps before/after */
 114         local_irq_disable();
 115         write_semaphore(&sema[2], 0);
 116         while (READ_ONCE(sema[1]) == 0) /* wait for the gpu to catch up */
 117                 cpu_relax();
 118         *dt = local_clock();
 119         udelay(100);
 120         *dt = local_clock() - *dt;
 121         write_semaphore(&sema[2], 1);
 122         local_irq_enable();
 123
 124         if (i915_request_wait(rq, 0, HZ / 2) < 0) {
 125                 i915_request_put(rq);
 126                 return -ETIME;
 127         }
 128         i915_request_put(rq);
 129
 130         pr_debug("%s CTX_TIMESTAMP: [%x, %x], RING_TIMESTAMP: [%x, %x]\n",
 131                  engine->name, sema[1], sema[3], sema[0], sema[4]);
 132
 133         *d_ctx = sema[3] - sema[1];
 134         *d_ring = sema[4] - sema[0];
 135         return 0;
 136 }
 137
 138 static int __live_engine_timestamps(struct intel_engine_cs *engine)
 139 {
 140         u64 s_ring[COUNT], s_ctx[COUNT], st[COUNT], d_ring, d_ctx, dt;
 141         struct intel_context *ce;
 142         int i, err = 0;
 143
 144         ce = intel_context_create(engine);
 145         if (IS_ERR(ce))
 146                 return PTR_ERR(ce);
 147
 148         for (i = 0; i < COUNT; i++) {
 149                 err = __measure_timestamps(ce, &st[i], &s_ring[i], &s_ctx[i]);
 150                 if (err)
 151                         break;
 152         }
 153         intel_context_put(ce);
 154         if (err)
 155                 return err;
 156
 157         dt = trifilter(st);
 158         d_ring = trifilter(s_ring);
 159         d_ctx = trifilter(s_ctx);
 160
 161         pr_info("%s elapsed:%lldns, CTX_TIMESTAMP:%lldns, RING_TIMESTAMP:%lldns\n",
 162                 engine->name, dt,
 163                 intel_gt_clock_interval_to_ns(engine->gt, d_ctx),
 164                 intel_gt_clock_interval_to_ns(engine->gt, d_ring));
 165
 166         d_ring = intel_gt_clock_interval_to_ns(engine->gt, d_ring);
 167         if (3 * dt > 4 * d_ring || 4 * dt < 3 * d_ring) {
 168                 pr_err("%s Mismatch between ring timestamp and walltime!\n",
 169                        engine->name);
 170                 return -EINVAL;
 171         }
 172
 173         d_ring = trifilter(s_ring);
 174         d_ctx = trifilter(s_ctx);
 175
 176         d_ctx *= engine->gt->clock_frequency;
 177         if (GRAPHICS_VER(engine->i915) == 11)
 178                 d_ring *= 12500000; /* Fixed 80ns for GEN11 ctx timestamp? */
 179         else
 180                 d_ring *= engine->gt->clock_frequency;
 181
 182         if (3 * d_ctx > 4 * d_ring || 4 * d_ctx < 3 * d_ring) {
 183                 pr_err("%s Mismatch between ring and context timestamps!\n",
 184                        engine->name);
 185                 return -EINVAL;
 186         }
 187
 188         return 0;
 189 }
 190
 191 static int live_engine_timestamps(void *arg)
 192 {
 193         struct intel_gt *gt = arg;
 194         struct intel_engine_cs *engine;
 195         enum intel_engine_id id;
 196
 197         /*
 198          * Check that CS_TIMESTAMP / CTX_TIMESTAMP are in sync, i.e. share
 199          * the same CS clock.
 200          */
 201
 202         if (GRAPHICS_VER(gt->i915) < 8)
 203                 return 0;
 204
 205         for_each_engine(engine, gt, id) {
 206                 int err;
 207
 208                 st_engine_heartbeat_disable(engine);
 209                 err = __live_engine_timestamps(engine);
 210                 st_engine_heartbeat_enable(engine);
 211                 if (err)
 212                         return err;
 213         }
 214
 215         return 0;
 216 }
 217
 218 static int __spin_until_busier(struct intel_engine_cs *engine, ktime_t busyness)
 219 {
 220         ktime_t start, unused, dt;
 221
 222         if (!intel_engine_uses_guc(engine))
 223                 return 0;
 224
 225         /*
 226          * In GuC mode of submission, the busyness stats may get updated after
 227          * the batch starts running. Poll for a change in busyness and timeout
 228          * after 500 us.
 229          */
 230         start = ktime_get();
 231         while (intel_engine_get_busy_time(engine, &unused) == busyness) {
 232                 dt = ktime_get() - start;
 233                 if (dt > 10000000) {
 234                         pr_err("active wait timed out %lld\n", dt);
 235                         ENGINE_TRACE(engine, "active wait time out %lld\n", dt);
 236                         return -ETIME;
 237                 }
 238         }
 239
 240         return 0;
 241 }
 242
 243 static int live_engine_busy_stats(void *arg)
 244 {
 245         struct intel_gt *gt = arg;
 246         struct intel_engine_cs *engine;
 247         enum intel_engine_id id;
 248         struct igt_spinner spin;
 249         int err = 0;
 250
 251         /*
 252          * Check that if an engine supports busy-stats, they tell the truth.
 253          */
 254
 255         if (igt_spinner_init(&spin, gt))
 256                 return -ENOMEM;
 257
 258         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 259         for_each_engine(engine, gt, id) {
 260                 struct i915_request *rq;
 261                 ktime_t busyness, dummy;
 262                 ktime_t de, dt;
 263                 ktime_t t[2];
 264
 265                 if (!intel_engine_supports_stats(engine))
 266                         continue;
 267
 268                 if (!intel_engine_can_store_dword(engine))
 269                         continue;
 270
 271                 if (intel_gt_pm_wait_for_idle(gt)) {
 272                         err = -EBUSY;
 273                         break;
 274                 }
 275
 276                 st_engine_heartbeat_disable(engine);
 277
 278                 ENGINE_TRACE(engine, "measuring idle time\n");
 279                 preempt_disable();
 280                 de = intel_engine_get_busy_time(engine, &t[0]);
 281                 udelay(100);
 282                 de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
 283                 preempt_enable();
 284                 dt = ktime_sub(t[1], t[0]);
 285                 if (de < 0 || de > 10) {
 286                         pr_err("%s: reported %lldns [%d%%] busyness while sleeping [for %lldns]\n",
 287                                engine->name,
 288                                de, (int)div64_u64(100 * de, dt), dt);
 289                         GEM_TRACE_DUMP();
 290                         err = -EINVAL;
 291                         goto end;
 292                 }
 293
 294                 /* 100% busy */
 295                 rq = igt_spinner_create_request(&spin,
 296                                                 engine->kernel_context,
 297                                                 MI_NOOP);
 298                 if (IS_ERR(rq)) {
 299                         err = PTR_ERR(rq);
 300                         goto end;
 301                 }
 302                 i915_request_add(rq);
 303
 304                 busyness = intel_engine_get_busy_time(engine, &dummy);
 305                 if (!igt_wait_for_spinner(&spin, rq)) {
 306                         intel_gt_set_wedged(engine->gt);
 307                         err = -ETIME;
 308                         goto end;
 309                 }
 310
 311                 err = __spin_until_busier(engine, busyness);
 312                 if (err) {
 313                         GEM_TRACE_DUMP();
 314                         goto end;
 315                 }
 316
 317                 ENGINE_TRACE(engine, "measuring busy time\n");
 318                 preempt_disable();
 319                 de = intel_engine_get_busy_time(engine, &t[0]);
 320                 mdelay(10);
 321                 de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
 322                 preempt_enable();
 323                 dt = ktime_sub(t[1], t[0]);
 324                 if (100 * de < 95 * dt || 95 * de > 100 * dt) {
 325                         pr_err("%s: reported %lldns [%d%%] busyness while spinning [for %lldns]\n",
 326                                engine->name,
 327                                de, (int)div64_u64(100 * de, dt), dt);
 328                         GEM_TRACE_DUMP();
 329                         err = -EINVAL;
 330                         goto end;
 331                 }
 332
 333 end:
 334                 st_engine_heartbeat_enable(engine);
 335                 igt_spinner_end(&spin);
 336                 if (igt_flush_test(gt->i915))
 337                         err = -EIO;
 338                 if (err)
 339                         break;
 340         }
 341
 342         igt_spinner_fini(&spin);
 343         if (igt_flush_test(gt->i915))
 344                 err = -EIO;
 345         return err;
 346 }
 347
 348 static int live_engine_pm(void *arg)
 349 {
 350         struct intel_gt *gt = arg;
 351         struct intel_engine_cs *engine;
 352         enum intel_engine_id id;
 353
 354         /*
 355          * Check we can call intel_engine_pm_put from any context. No
 356          * failures are reported directly, but if we mess up lockdep should
 357          * tell us.
 358          */
 359         if (intel_gt_pm_wait_for_idle(gt)) {
 360                 pr_err("Unable to flush GT pm before test\n");
 361                 return -EBUSY;
 362         }
 363
 364         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 365         for_each_engine(engine, gt, id) {
 366                 const typeof(*igt_atomic_phases) *p;
 367
 368                 for (p = igt_atomic_phases; p->name; p++) {
 369                         /*
 370                          * Acquisition is always synchronous, except if we
 371                          * know that the engine is already awake, in which
 372                          * case we should use intel_engine_pm_get_if_awake()
 373                          * to atomically grab the wakeref.
 374                          *
 375                          * In practice,
 376                          *    intel_engine_pm_get();
 377                          *    intel_engine_pm_put();
 378                          * occurs in one thread, while simultaneously
 379                          *    intel_engine_pm_get_if_awake();
 380                          *    intel_engine_pm_put();
 381                          * occurs from atomic context in another.
 382                          */
 383                         GEM_BUG_ON(intel_engine_pm_is_awake(engine));
 384                         intel_engine_pm_get(engine);
 385
 386                         p->critical_section_begin();
 387                         if (!intel_engine_pm_get_if_awake(engine))
 388                                 pr_err("intel_engine_pm_get_if_awake(%s) failed under %s\n",
 389                                        engine->name, p->name);
 390                         else
 391                                 intel_engine_pm_put_async(engine);
 392                         intel_engine_pm_put_async(engine);
 393                         p->critical_section_end();
 394
 395                         intel_engine_pm_flush(engine);
 396
 397                         if (intel_engine_pm_is_awake(engine)) {
 398                                 pr_err("%s is still awake after flushing pm\n",
 399                                        engine->name);
 400                                 return -EINVAL;
 401                         }
 402
 403                         /* gt wakeref is async (deferred to workqueue) */
 404                         if (intel_gt_pm_wait_for_idle(gt)) {
 405                                 pr_err("GT failed to idle\n");
 406                                 return -EINVAL;
 407                         }
 408                 }
 409         }
 410
 411         return 0;
 412 }
 413
 414 int live_engine_pm_selftests(struct intel_gt *gt)
 415 {
 416         static const struct i915_subtest tests[] = {
 417                 SUBTEST(live_engine_timestamps),
 418                 SUBTEST(live_engine_busy_stats),
 419                 SUBTEST(live_engine_pm),
 420         };
 421
 422         return intel_gt_live_subtests(tests, gt);
 423 }