drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2019 Intel Corporation
   4  */
   5
   6 #include <linux/bsearch.h>
   7
   8 #include "gt/intel_engine_regs.h"
   9 #include "gt/intel_gt.h"
  10 #include "gt/intel_gt_regs.h"
  11 #include "gt/intel_lrc.h"
  12 #include "gt/shmem_utils.h"
  13 #include "intel_guc_ads.h"
  14 #include "intel_guc_capture.h"
  15 #include "intel_guc_fwif.h"
  16 #include "intel_uc.h"
  17 #include "i915_drv.h"
  18
  19 /*
  20  * The Additional Data Struct (ADS) has pointers for different buffers used by
  21  * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
  22  * all the extra buffers indirectly linked via the ADS struct's entries.
  23  *
  24  * Layout of the ADS blob allocated for the GuC:
  25  *
  26  *      +---------------------------------------+ <== base
  27  *      | guc_ads                               |
  28  *      +---------------------------------------+
  29  *      | guc_policies                          |
  30  *      +---------------------------------------+
  31  *      | guc_gt_system_info                    |
  32  *      +---------------------------------------+
  33  *      | guc_engine_usage                      |
  34  *      +---------------------------------------+ <== static
  35  *      | guc_mmio_reg[countA] (engine 0.0)     |
  36  *      | guc_mmio_reg[countB] (engine 0.1)     |
  37  *      | guc_mmio_reg[countC] (engine 1.0)     |
  38  *      |   ...                                 |
  39  *      +---------------------------------------+ <== dynamic
  40  *      | padding                               |
  41  *      +---------------------------------------+ <== 4K aligned
  42  *      | golden contexts                       |
  43  *      +---------------------------------------+
  44  *      | padding                               |
  45  *      +---------------------------------------+ <== 4K aligned
  46  *      | capture lists                         |
  47  *      +---------------------------------------+
  48  *      | padding                               |
  49  *      +---------------------------------------+ <== 4K aligned
  50  *      | private data                          |
  51  *      +---------------------------------------+
  52  *      | padding                               |
  53  *      +---------------------------------------+ <== 4K aligned
  54  */
  55 struct __guc_ads_blob {
  56         struct guc_ads ads;
  57         struct guc_policies policies;
  58         struct guc_gt_system_info system_info;
  59         struct guc_engine_usage engine_usage;
  60         /* From here on, location is dynamic! Refer to above diagram. */
  61         struct guc_mmio_reg regset[];
  62 } __packed;
  63
  64 #define ads_blob_read(guc_, field_)                                     \
  65         iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
  66
  67 #define ads_blob_write(guc_, field_, val_)                              \
  68         iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob,  \
  69                            field_, val_)
  70
  71 #define info_map_write(map_, field_, val_) \
  72         iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
  73
  74 #define info_map_read(map_, field_) \
  75         iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
  76
  77 static u32 guc_ads_regset_size(struct intel_guc *guc)
  78 {
  79         GEM_BUG_ON(!guc->ads_regset_size);
  80         return guc->ads_regset_size;
  81 }
  82
  83 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
  84 {
  85         return PAGE_ALIGN(guc->ads_golden_ctxt_size);
  86 }
  87
  88 static u32 guc_ads_capture_size(struct intel_guc *guc)
  89 {
  90         return PAGE_ALIGN(guc->ads_capture_size);
  91 }
  92
  93 static u32 guc_ads_private_data_size(struct intel_guc *guc)
  94 {
  95         return PAGE_ALIGN(guc->fw.private_data_size);
  96 }
  97
  98 static u32 guc_ads_regset_offset(struct intel_guc *guc)
  99 {
 100         return offsetof(struct __guc_ads_blob, regset);
 101 }
 102
 103 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
 104 {
 105         u32 offset;
 106
 107         offset = guc_ads_regset_offset(guc) +
 108                  guc_ads_regset_size(guc);
 109
 110         return PAGE_ALIGN(offset);
 111 }
 112
 113 static u32 guc_ads_capture_offset(struct intel_guc *guc)
 114 {
 115         u32 offset;
 116
 117         offset = guc_ads_golden_ctxt_offset(guc) +
 118                  guc_ads_golden_ctxt_size(guc);
 119
 120         return PAGE_ALIGN(offset);
 121 }
 122
 123 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
 124 {
 125         u32 offset;
 126
 127         offset = guc_ads_capture_offset(guc) +
 128                  guc_ads_capture_size(guc);
 129
 130         return PAGE_ALIGN(offset);
 131 }
 132
 133 static u32 guc_ads_blob_size(struct intel_guc *guc)
 134 {
 135         return guc_ads_private_data_offset(guc) +
 136                guc_ads_private_data_size(guc);
 137 }
 138
 139 static void guc_policies_init(struct intel_guc *guc)
 140 {
 141         struct intel_gt *gt = guc_to_gt(guc);
 142         struct drm_i915_private *i915 = gt->i915;
 143         u32 global_flags = 0;
 144
 145         ads_blob_write(guc, policies.dpc_promote_time,
 146                        GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
 147         ads_blob_write(guc, policies.max_num_work_items,
 148                        GLOBAL_POLICY_MAX_NUM_WI);
 149
 150         if (i915->params.reset < 2)
 151                 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
 152
 153         ads_blob_write(guc, policies.global_flags, global_flags);
 154         ads_blob_write(guc, policies.is_valid, 1);
 155 }
 156
 157 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
 158                                      struct drm_printer *dp)
 159 {
 160         if (unlikely(iosys_map_is_null(&guc->ads_map)))
 161                 return;
 162
 163         drm_printf(dp, "Global scheduling policies:\n");
 164         drm_printf(dp, "  DPC promote time   = %u\n",
 165                    ads_blob_read(guc, policies.dpc_promote_time));
 166         drm_printf(dp, "  Max num work items = %u\n",
 167                    ads_blob_read(guc, policies.max_num_work_items));
 168         drm_printf(dp, "  Flags              = %u\n",
 169                    ads_blob_read(guc, policies.global_flags));
 170 }
 171
 172 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
 173 {
 174         u32 action[] = {
 175                 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
 176                 policy_offset
 177         };
 178
 179         return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
 180 }
 181
 182 int intel_guc_global_policies_update(struct intel_guc *guc)
 183 {
 184         struct intel_gt *gt = guc_to_gt(guc);
 185         u32 scheduler_policies;
 186         intel_wakeref_t wakeref;
 187         int ret;
 188
 189         if (iosys_map_is_null(&guc->ads_map))
 190                 return -EOPNOTSUPP;
 191
 192         scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
 193         GEM_BUG_ON(!scheduler_policies);
 194
 195         guc_policies_init(guc);
 196
 197         if (!intel_guc_is_ready(guc))
 198                 return 0;
 199
 200         with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
 201                 ret = guc_action_policies_update(guc, scheduler_policies);
 202
 203         return ret;
 204 }
 205
 206 static void guc_mapping_table_init(struct intel_gt *gt,
 207                                    struct iosys_map *info_map)
 208 {
 209         unsigned int i, j;
 210         struct intel_engine_cs *engine;
 211         enum intel_engine_id id;
 212
 213         /* Table must be set to invalid values for entries not used */
 214         for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
 215                 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
 216                         info_map_write(info_map, mapping_table[i][j],
 217                                        GUC_MAX_INSTANCES_PER_CLASS);
 218
 219         for_each_engine(engine, gt, id) {
 220                 u8 guc_class = engine_class_to_guc_class(engine->class);
 221
 222                 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
 223                                engine->instance);
 224         }
 225 }
 226
 227 /*
 228  * The save/restore register list must be pre-calculated to a temporary
 229  * buffer before it can be copied inside the ADS.
 230  */
 231 struct temp_regset {
 232         /*
 233          * ptr to the section of the storage for the engine currently being
 234          * worked on
 235          */
 236         struct guc_mmio_reg *registers;
 237         /* ptr to the base of the allocated storage for all engines */
 238         struct guc_mmio_reg *storage;
 239         u32 storage_used;
 240         u32 storage_max;
 241 };
 242
 243 static int guc_mmio_reg_cmp(const void *a, const void *b)
 244 {
 245         const struct guc_mmio_reg *ra = a;
 246         const struct guc_mmio_reg *rb = b;
 247
 248         return (int)ra->offset - (int)rb->offset;
 249 }
 250
 251 static struct guc_mmio_reg * __must_check
 252 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
 253 {
 254         u32 pos = regset->storage_used;
 255         struct guc_mmio_reg *slot;
 256
 257         if (pos >= regset->storage_max) {
 258                 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
 259                 struct guc_mmio_reg *r = krealloc(regset->storage,
 260                                                   size, GFP_KERNEL);
 261                 if (!r) {
 262                         WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
 263                                   -ENOMEM);
 264                         return ERR_PTR(-ENOMEM);
 265                 }
 266
 267                 regset->registers = r + (regset->registers - regset->storage);
 268                 regset->storage = r;
 269                 regset->storage_max = size / sizeof(*slot);
 270         }
 271
 272         slot = &regset->storage[pos];
 273         regset->storage_used++;
 274         *slot = *reg;
 275
 276         return slot;
 277 }
 278
 279 #define GUC_REGSET_STEERING(group, instance) ( \
 280         FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
 281         FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
 282         GUC_REGSET_NEEDS_STEERING \
 283 )
 284
 285 static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
 286                                           struct temp_regset *regset,
 287                                           i915_reg_t reg, u32 flags)
 288 {
 289         u32 count = regset->storage_used - (regset->registers - regset->storage);
 290         u32 offset = i915_mmio_reg_offset(reg);
 291         struct guc_mmio_reg entry = {
 292                 .offset = offset,
 293                 .flags = flags,
 294         };
 295         struct guc_mmio_reg *slot;
 296         u8 group, inst;
 297
 298         /*
 299          * The mmio list is built using separate lists within the driver.
 300          * It's possible that at some point we may attempt to add the same
 301          * register more than once. Do not consider this an error; silently
 302          * move on if the register is already in the list.
 303          */
 304         if (bsearch(&entry, regset->registers, count,
 305                     sizeof(entry), guc_mmio_reg_cmp))
 306                 return 0;
 307
 308         /*
 309          * The GuC doesn't have a default steering, so we need to explicitly
 310          * steer all registers that need steering. However, we do not keep track
 311          * of all the steering ranges, only of those that have a chance of using
 312          * a non-default steering from the i915 pov. Instead of adding such
 313          * tracking, it is easier to just program the default steering for all
 314          * regs that don't need a non-default one.
 315          */
 316         intel_gt_get_valid_steering_for_reg(gt, reg, &group, &inst);
 317         entry.flags |= GUC_REGSET_STEERING(group, inst);
 318
 319         slot = __mmio_reg_add(regset, &entry);
 320         if (IS_ERR(slot))
 321                 return PTR_ERR(slot);
 322
 323         while (slot-- > regset->registers) {
 324                 GEM_BUG_ON(slot[0].offset == slot[1].offset);
 325                 if (slot[1].offset > slot[0].offset)
 326                         break;
 327
 328                 swap(slot[1], slot[0]);
 329         }
 330
 331         return 0;
 332 }
 333
 334 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
 335         guc_mmio_reg_add(gt, \
 336                          regset, \
 337                          (reg), \
 338                          (masked) ? GUC_REGSET_MASKED : 0)
 339
 340 static int guc_mmio_regset_init(struct temp_regset *regset,
 341                                 struct intel_engine_cs *engine)
 342 {
 343         struct intel_gt *gt = engine->gt;
 344         const u32 base = engine->mmio_base;
 345         struct i915_wa_list *wal = &engine->wa_list;
 346         struct i915_wa *wa;
 347         unsigned int i;
 348         int ret = 0;
 349
 350         /*
 351          * Each engine's registers point to a new start relative to
 352          * storage
 353          */
 354         regset->registers = regset->storage + regset->storage_used;
 355
 356         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
 357         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
 358         ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
 359
 360         if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
 361             CCS_MASK(engine->gt))
 362                 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
 363
 364         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 365                 ret |= GUC_MMIO_REG_ADD(gt, regset, wa->reg, wa->masked_reg);
 366
 367         /* Be extra paranoid and include all whitelist registers. */
 368         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
 369                 ret |= GUC_MMIO_REG_ADD(gt, regset,
 370                                         RING_FORCE_TO_NONPRIV(base, i),
 371                                         false);
 372
 373         /* add in local MOCS registers */
 374         for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++)
 375                 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
 376
 377         return ret ? -1 : 0;
 378 }
 379
 380 static long guc_mmio_reg_state_create(struct intel_guc *guc)
 381 {
 382         struct intel_gt *gt = guc_to_gt(guc);
 383         struct intel_engine_cs *engine;
 384         enum intel_engine_id id;
 385         struct temp_regset temp_set = {};
 386         long total = 0;
 387         long ret;
 388
 389         for_each_engine(engine, gt, id) {
 390                 u32 used = temp_set.storage_used;
 391
 392                 ret = guc_mmio_regset_init(&temp_set, engine);
 393                 if (ret < 0)
 394                         goto fail_regset_init;
 395
 396                 guc->ads_regset_count[id] = temp_set.storage_used - used;
 397                 total += guc->ads_regset_count[id];
 398         }
 399
 400         guc->ads_regset = temp_set.storage;
 401
 402         drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n",
 403                 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
 404
 405         return total * sizeof(struct guc_mmio_reg);
 406
 407 fail_regset_init:
 408         kfree(temp_set.storage);
 409         return ret;
 410 }
 411
 412 static void guc_mmio_reg_state_init(struct intel_guc *guc)
 413 {
 414         struct intel_gt *gt = guc_to_gt(guc);
 415         struct intel_engine_cs *engine;
 416         enum intel_engine_id id;
 417         u32 addr_ggtt, offset;
 418
 419         offset = guc_ads_regset_offset(guc);
 420         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 421
 422         iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
 423                             guc->ads_regset_size);
 424
 425         for_each_engine(engine, gt, id) {
 426                 u32 count = guc->ads_regset_count[id];
 427                 u8 guc_class;
 428
 429                 /* Class index is checked in class converter */
 430                 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
 431
 432                 guc_class = engine_class_to_guc_class(engine->class);
 433
 434                 if (!count) {
 435                         ads_blob_write(guc,
 436                                        ads.reg_state_list[guc_class][engine->instance].address,
 437                                        0);
 438                         ads_blob_write(guc,
 439                                        ads.reg_state_list[guc_class][engine->instance].count,
 440                                        0);
 441                         continue;
 442                 }
 443
 444                 ads_blob_write(guc,
 445                                ads.reg_state_list[guc_class][engine->instance].address,
 446                                addr_ggtt);
 447                 ads_blob_write(guc,
 448                                ads.reg_state_list[guc_class][engine->instance].count,
 449                                count);
 450
 451                 addr_ggtt += count * sizeof(struct guc_mmio_reg);
 452         }
 453 }
 454
 455 static void fill_engine_enable_masks(struct intel_gt *gt,
 456                                      struct iosys_map *info_map)
 457 {
 458         info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
 459         info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
 460         info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 1);
 461         info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
 462         info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
 463 }
 464
 465 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
 466 #define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE)
 467 static int guc_prep_golden_context(struct intel_guc *guc)
 468 {
 469         struct intel_gt *gt = guc_to_gt(guc);
 470         u32 addr_ggtt, offset;
 471         u32 total_size = 0, alloc_size, real_size;
 472         u8 engine_class, guc_class;
 473         struct guc_gt_system_info local_info;
 474         struct iosys_map info_map;
 475
 476         /*
 477          * Reserve the memory for the golden contexts and point GuC at it but
 478          * leave it empty for now. The context data will be filled in later
 479          * once there is something available to put there.
 480          *
 481          * Note that the HWSP and ring context are not included.
 482          *
 483          * Note also that the storage must be pinned in the GGTT, so that the
 484          * address won't change after GuC has been told where to find it. The
 485          * GuC will also validate that the LRC base + size fall within the
 486          * allowed GGTT range.
 487          */
 488         if (!iosys_map_is_null(&guc->ads_map)) {
 489                 offset = guc_ads_golden_ctxt_offset(guc);
 490                 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 491                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 492                                                  offsetof(struct __guc_ads_blob, system_info));
 493         } else {
 494                 memset(&local_info, 0, sizeof(local_info));
 495                 iosys_map_set_vaddr(&info_map, &local_info);
 496                 fill_engine_enable_masks(gt, &info_map);
 497         }
 498
 499         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 500                 if (engine_class == OTHER_CLASS)
 501                         continue;
 502
 503                 guc_class = engine_class_to_guc_class(engine_class);
 504
 505                 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
 506                         continue;
 507
 508                 real_size = intel_engine_context_size(gt, engine_class);
 509                 alloc_size = PAGE_ALIGN(real_size);
 510                 total_size += alloc_size;
 511
 512                 if (iosys_map_is_null(&guc->ads_map))
 513                         continue;
 514
 515                 /*
 516                  * This interface is slightly confusing. We need to pass the
 517                  * base address of the full golden context and the size of just
 518                  * the engine state, which is the section of the context image
 519                  * that starts after the execlists context. This is required to
 520                  * allow the GuC to restore just the engine state when a
 521                  * watchdog reset occurs.
 522                  * We calculate the engine state size by removing the size of
 523                  * what comes before it in the context image (which is identical
 524                  * on all engines).
 525                  */
 526                 ads_blob_write(guc, ads.eng_state_size[guc_class],
 527                                real_size - LRC_SKIP_SIZE);
 528                 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
 529                                addr_ggtt);
 530
 531                 addr_ggtt += alloc_size;
 532         }
 533
 534         /* Make sure current size matches what we calculated previously */
 535         if (guc->ads_golden_ctxt_size)
 536                 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 537
 538         return total_size;
 539 }
 540
 541 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
 542 {
 543         struct intel_engine_cs *engine;
 544         enum intel_engine_id id;
 545
 546         for_each_engine(engine, gt, id) {
 547                 if (engine->class != engine_class)
 548                         continue;
 549
 550                 if (!engine->default_state)
 551                         continue;
 552
 553                 return engine;
 554         }
 555
 556         return NULL;
 557 }
 558
 559 static void guc_init_golden_context(struct intel_guc *guc)
 560 {
 561         struct intel_engine_cs *engine;
 562         struct intel_gt *gt = guc_to_gt(guc);
 563         unsigned long offset;
 564         u32 addr_ggtt, total_size = 0, alloc_size, real_size;
 565         u8 engine_class, guc_class;
 566
 567         if (!intel_uc_uses_guc_submission(&gt->uc))
 568                 return;
 569
 570         GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
 571
 572         /*
 573          * Go back and fill in the golden context data now that it is
 574          * available.
 575          */
 576         offset = guc_ads_golden_ctxt_offset(guc);
 577         addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
 578
 579         for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
 580                 if (engine_class == OTHER_CLASS)
 581                         continue;
 582
 583                 guc_class = engine_class_to_guc_class(engine_class);
 584                 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
 585                         continue;
 586
 587                 real_size = intel_engine_context_size(gt, engine_class);
 588                 alloc_size = PAGE_ALIGN(real_size);
 589                 total_size += alloc_size;
 590
 591                 engine = find_engine_state(gt, engine_class);
 592                 if (!engine) {
 593                         drm_err(&gt->i915->drm, "No engine state recorded for class %d!\n",
 594                                 engine_class);
 595                         ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
 596                         ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
 597                         continue;
 598                 }
 599
 600                 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
 601                            real_size - LRC_SKIP_SIZE);
 602                 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
 603
 604                 addr_ggtt += alloc_size;
 605
 606                 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
 607                                         offset, real_size);
 608                 offset += alloc_size;
 609         }
 610
 611         GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
 612 }
 613
 614 static int
 615 guc_capture_prep_lists(struct intel_guc *guc)
 616 {
 617         struct intel_gt *gt = guc_to_gt(guc);
 618         struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
 619         u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
 620         struct guc_gt_system_info local_info;
 621         struct iosys_map info_map;
 622         bool ads_is_mapped;
 623         size_t size = 0;
 624         void *ptr;
 625         int i, j;
 626
 627         ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
 628         if (ads_is_mapped) {
 629                 capture_offset = guc_ads_capture_offset(guc);
 630                 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
 631                 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 632                                                  offsetof(struct __guc_ads_blob, system_info));
 633         } else {
 634                 memset(&local_info, 0, sizeof(local_info));
 635                 iosys_map_set_vaddr(&info_map, &local_info);
 636                 fill_engine_enable_masks(gt, &info_map);
 637         }
 638
 639         /* first, set aside the first page for a capture_list with zero descriptors */
 640         total_size = PAGE_SIZE;
 641         if (ads_is_mapped) {
 642                 if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
 643                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 644                 null_ggtt = ads_ggtt + capture_offset;
 645                 capture_offset += PAGE_SIZE;
 646         }
 647
 648         for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
 649                 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
 650
 651                         /* null list if we dont have said engine or list */
 652                         if (!info_map_read(&info_map, engine_enabled_masks[j])) {
 653                                 if (ads_is_mapped) {
 654                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 655                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 656                                 }
 657                                 continue;
 658                         }
 659                         if (intel_guc_capture_getlistsize(guc, i,
 660                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 661                                                           j, &size)) {
 662                                 if (ads_is_mapped)
 663                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 664                                 goto engine_instance_list;
 665                         }
 666                         total_size += size;
 667                         if (ads_is_mapped) {
 668                                 if (total_size > guc->ads_capture_size ||
 669                                     intel_guc_capture_getlist(guc, i,
 670                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
 671                                                               j, &ptr)) {
 672                                         ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
 673                                         continue;
 674                                 }
 675                                 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
 676                                                capture_offset);
 677                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 678                                 capture_offset += size;
 679                         }
 680 engine_instance_list:
 681                         if (intel_guc_capture_getlistsize(guc, i,
 682                                                           GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 683                                                           j, &size)) {
 684                                 if (ads_is_mapped)
 685                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 686                                 continue;
 687                         }
 688                         total_size += size;
 689                         if (ads_is_mapped) {
 690                                 if (total_size > guc->ads_capture_size ||
 691                                     intel_guc_capture_getlist(guc, i,
 692                                                               GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
 693                                                               j, &ptr)) {
 694                                         ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
 695                                         continue;
 696                                 }
 697                                 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
 698                                                capture_offset);
 699                                 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 700                                 capture_offset += size;
 701                         }
 702                 }
 703                 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
 704                         if (ads_is_mapped)
 705                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 706                         continue;
 707                 }
 708                 total_size += size;
 709                 if (ads_is_mapped) {
 710                         if (total_size > guc->ads_capture_size ||
 711                             intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
 712                                                       &ptr)) {
 713                                 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
 714                                 continue;
 715                         }
 716                         ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
 717                         iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
 718                         capture_offset += size;
 719                 }
 720         }
 721
 722         if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
 723                 drm_warn(&i915->drm, "GuC->ADS->Capture alloc size changed from %d to %d\n",
 724                          guc->ads_capture_size, PAGE_ALIGN(total_size));
 725
 726         return PAGE_ALIGN(total_size);
 727 }
 728
 729 static void __guc_ads_init(struct intel_guc *guc)
 730 {
 731         struct intel_gt *gt = guc_to_gt(guc);
 732         struct drm_i915_private *i915 = gt->i915;
 733         struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
 734                         offsetof(struct __guc_ads_blob, system_info));
 735         u32 base;
 736
 737         /* GuC scheduling policies */
 738         guc_policies_init(guc);
 739
 740         /* System info */
 741         fill_engine_enable_masks(gt, &info_map);
 742
 743         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
 744                        hweight8(gt->info.sseu.slice_mask));
 745         ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
 746                        gt->info.vdbox_sfc_access);
 747
 748         if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
 749                 u32 distdbreg = intel_uncore_read(gt->uncore,
 750                                                   GEN12_DIST_DBS_POPULATED);
 751                 ads_blob_write(guc,
 752                                system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
 753                                ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
 754                                 & GEN12_DOORBELLS_PER_SQIDI) + 1);
 755         }
 756
 757         /* Golden contexts for re-initialising after a watchdog reset */
 758         guc_prep_golden_context(guc);
 759
 760         guc_mapping_table_init(guc_to_gt(guc), &info_map);
 761
 762         base = intel_guc_ggtt_offset(guc, guc->ads_vma);
 763
 764         /* Lists for error capture debug */
 765         guc_capture_prep_lists(guc);
 766
 767         /* ADS */
 768         ads_blob_write(guc, ads.scheduler_policies, base +
 769                        offsetof(struct __guc_ads_blob, policies));
 770         ads_blob_write(guc, ads.gt_system_info, base +
 771                        offsetof(struct __guc_ads_blob, system_info));
 772
 773         /* MMIO save/restore list */
 774         guc_mmio_reg_state_init(guc);
 775
 776         /* Private Data */
 777         ads_blob_write(guc, ads.private_data, base +
 778                        guc_ads_private_data_offset(guc));
 779
 780         i915_gem_object_flush_map(guc->ads_vma->obj);
 781 }
 782
 783 /**
 784  * intel_guc_ads_create() - allocates and initializes GuC ADS.
 785  * @guc: intel_guc struct
 786  *
 787  * GuC needs memory block (Additional Data Struct), where it will store
 788  * some data. Allocate and initialize such memory block for GuC use.
 789  */
 790 int intel_guc_ads_create(struct intel_guc *guc)
 791 {
 792         void *ads_blob;
 793         u32 size;
 794         int ret;
 795
 796         GEM_BUG_ON(guc->ads_vma);
 797
 798         /*
 799          * Create reg state size dynamically on system memory to be copied to
 800          * the final ads blob on gt init/reset
 801          */
 802         ret = guc_mmio_reg_state_create(guc);
 803         if (ret < 0)
 804                 return ret;
 805         guc->ads_regset_size = ret;
 806
 807         /* Likewise the golden contexts: */
 808         ret = guc_prep_golden_context(guc);
 809         if (ret < 0)
 810                 return ret;
 811         guc->ads_golden_ctxt_size = ret;
 812
 813         /* Likewise the capture lists: */
 814         ret = guc_capture_prep_lists(guc);
 815         if (ret < 0)
 816                 return ret;
 817         guc->ads_capture_size = ret;
 818
 819         /* Now the total size can be determined: */
 820         size = guc_ads_blob_size(guc);
 821
 822         ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
 823                                              &ads_blob);
 824         if (ret)
 825                 return ret;
 826
 827         if (i915_gem_object_is_lmem(guc->ads_vma->obj))
 828                 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
 829         else
 830                 iosys_map_set_vaddr(&guc->ads_map, ads_blob);
 831
 832         __guc_ads_init(guc);
 833
 834         return 0;
 835 }
 836
 837 void intel_guc_ads_init_late(struct intel_guc *guc)
 838 {
 839         /*
 840          * The golden context setup requires the saved engine state from
 841          * __engines_record_defaults(). However, that requires engines to be
 842          * operational which means the ADS must already have been configured.
 843          * Fortunately, the golden context state is not needed until a hang
 844          * occurs, so it can be filled in during this late init phase.
 845          */
 846         guc_init_golden_context(guc);
 847 }
 848
 849 void intel_guc_ads_destroy(struct intel_guc *guc)
 850 {
 851         i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
 852         iosys_map_clear(&guc->ads_map);
 853         kfree(guc->ads_regset);
 854 }
 855
 856 static void guc_ads_private_data_reset(struct intel_guc *guc)
 857 {
 858         u32 size;
 859
 860         size = guc_ads_private_data_size(guc);
 861         if (!size)
 862                 return;
 863
 864         iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
 865                          0, size);
 866 }
 867
 868 /**
 869  * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
 870  * @guc: intel_guc struct
 871  *
 872  * GuC stores some data in ADS, which might be stale after a reset.
 873  * Reinitialize whole ADS in case any part of it was corrupted during
 874  * previous GuC run.
 875  */
 876 void intel_guc_ads_reset(struct intel_guc *guc)
 877 {
 878         if (!guc->ads_vma)
 879                 return;
 880
 881         __guc_ads_init(guc);
 882
 883         guc_ads_private_data_reset(guc);
 884 }
 885
 886 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
 887 {
 888         return intel_guc_ggtt_offset(guc, guc->ads_vma) +
 889                 offsetof(struct __guc_ads_blob, engine_usage);
 890 }
 891
 892 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
 893 {
 894         struct intel_guc *guc = &engine->gt->uc.guc;
 895         u8 guc_class = engine_class_to_guc_class(engine->class);
 896         size_t offset = offsetof(struct __guc_ads_blob,
 897                                  engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
 898
 899         return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
 900 }