drivers/gpu/drm/i915/gt/intel_rps.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_gt.h"
   9 #include "intel_gt_irq.h"
  10 #include "intel_gt_pm_irq.h"
  11 #include "intel_rps.h"
  12 #include "intel_sideband.h"
  13 #include "../../../platform/x86/intel_ips.h"
  14
  15 /*
  16  * Lock protecting IPS related data structures
  17  */
  18 static DEFINE_SPINLOCK(mchdev_lock);
  19
  20 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  21 {
  22         return container_of(rps, struct intel_gt, rps);
  23 }
  24
  25 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  26 {
  27         return rps_to_gt(rps)->i915;
  28 }
  29
  30 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  31 {
  32         return rps_to_gt(rps)->uncore;
  33 }
  34
  35 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  36 {
  37         return mask & ~rps->pm_intrmsk_mbz;
  38 }
  39
  40 static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  41 {
  42         intel_uncore_write_fw(uncore, reg, val);
  43 }
  44
  45 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
  46 {
  47         u32 mask = 0;
  48
  49         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
  50         if (val > rps->min_freq_softlimit)
  51                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
  52                          GEN6_PM_RP_DOWN_THRESHOLD |
  53                          GEN6_PM_RP_DOWN_TIMEOUT);
  54
  55         if (val < rps->max_freq_softlimit)
  56                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
  57
  58         mask &= rps->pm_events;
  59
  60         return rps_pm_sanitize_mask(rps, ~mask);
  61 }
  62
  63 static void rps_reset_ei(struct intel_rps *rps)
  64 {
  65         memset(&rps->ei, 0, sizeof(rps->ei));
  66 }
  67
  68 static void rps_enable_interrupts(struct intel_rps *rps)
  69 {
  70         struct intel_gt *gt = rps_to_gt(rps);
  71
  72         rps_reset_ei(rps);
  73
  74         if (IS_VALLEYVIEW(gt->i915))
  75                 /* WaGsvRC0ResidencyMethod:vlv */
  76                 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
  77         else
  78                 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
  79                                   GEN6_PM_RP_DOWN_THRESHOLD |
  80                                   GEN6_PM_RP_DOWN_TIMEOUT);
  81
  82         spin_lock_irq(&gt->irq_lock);
  83         gen6_gt_pm_enable_irq(gt, rps->pm_events);
  84         spin_unlock_irq(&gt->irq_lock);
  85
  86         set(gt->uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, rps->cur_freq));
  87 }
  88
  89 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
  90 {
  91         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
  92 }
  93
  94 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
  95 {
  96         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
  97                 ;
  98 }
  99
 100 static void rps_reset_interrupts(struct intel_rps *rps)
 101 {
 102         struct intel_gt *gt = rps_to_gt(rps);
 103
 104         spin_lock_irq(&gt->irq_lock);
 105         if (INTEL_GEN(gt->i915) >= 11)
 106                 gen11_rps_reset_interrupts(rps);
 107         else
 108                 gen6_rps_reset_interrupts(rps);
 109
 110         rps->pm_iir = 0;
 111         spin_unlock_irq(&gt->irq_lock);
 112 }
 113
 114 static void rps_disable_interrupts(struct intel_rps *rps)
 115 {
 116         struct intel_gt *gt = rps_to_gt(rps);
 117
 118         rps->pm_events = 0;
 119
 120         set(gt->uncore, GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 121
 122         spin_lock_irq(&gt->irq_lock);
 123         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 124         spin_unlock_irq(&gt->irq_lock);
 125
 126         intel_synchronize_irq(gt->i915);
 127
 128         /*
 129          * Now that we will not be generating any more work, flush any
 130          * outstanding tasks. As we are called on the RPS idle path,
 131          * we will reset the GPU to minimum frequencies, so the current
 132          * state of the worker can be discarded.
 133          */
 134         cancel_work_sync(&rps->work);
 135
 136         rps_reset_interrupts(rps);
 137 }
 138
 139 static const struct cparams {
 140         u16 i;
 141         u16 t;
 142         u16 m;
 143         u16 c;
 144 } cparams[] = {
 145         { 1, 1333, 301, 28664 },
 146         { 1, 1066, 294, 24460 },
 147         { 1, 800, 294, 25192 },
 148         { 0, 1333, 276, 27605 },
 149         { 0, 1066, 276, 27605 },
 150         { 0, 800, 231, 23784 },
 151 };
 152
 153 static void gen5_rps_init(struct intel_rps *rps)
 154 {
 155         struct drm_i915_private *i915 = rps_to_i915(rps);
 156         struct intel_uncore *uncore = rps_to_uncore(rps);
 157         u8 fmax, fmin, fstart;
 158         u32 rgvmodectl;
 159         int c_m, i;
 160
 161         if (i915->fsb_freq <= 3200)
 162                 c_m = 0;
 163         else if (i915->fsb_freq <= 4800)
 164                 c_m = 1;
 165         else
 166                 c_m = 2;
 167
 168         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 169                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 170                         rps->ips.m = cparams[i].m;
 171                         rps->ips.c = cparams[i].c;
 172                         break;
 173                 }
 174         }
 175
 176         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 177
 178         /* Set up min, max, and cur for interrupt handling */
 179         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 180         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 181         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 182                 MEMMODE_FSTART_SHIFT;
 183         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
 184                          fmax, fmin, fstart);
 185
 186         rps->min_freq = fmax;
 187         rps->max_freq = fmin;
 188
 189         rps->idle_freq = rps->min_freq;
 190         rps->cur_freq = rps->idle_freq;
 191 }
 192
 193 static unsigned long
 194 __ips_chipset_val(struct intel_ips *ips)
 195 {
 196         struct intel_uncore *uncore =
 197                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 198         unsigned long now = jiffies_to_msecs(jiffies), dt;
 199         unsigned long result;
 200         u64 total, delta;
 201
 202         lockdep_assert_held(&mchdev_lock);
 203
 204         /*
 205          * Prevent division-by-zero if we are asking too fast.
 206          * Also, we don't get interesting results if we are polling
 207          * faster than once in 10ms, so just return the saved value
 208          * in such cases.
 209          */
 210         dt = now - ips->last_time1;
 211         if (dt <= 10)
 212                 return ips->chipset_power;
 213
 214         /* FIXME: handle per-counter overflow */
 215         total = intel_uncore_read(uncore, DMIEC);
 216         total += intel_uncore_read(uncore, DDREC);
 217         total += intel_uncore_read(uncore, CSIEC);
 218
 219         delta = total - ips->last_count1;
 220
 221         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 222
 223         ips->last_count1 = total;
 224         ips->last_time1 = now;
 225
 226         ips->chipset_power = result;
 227
 228         return result;
 229 }
 230
 231 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 232 {
 233         unsigned int m, x, b;
 234         u32 tsfs;
 235
 236         tsfs = intel_uncore_read(uncore, TSFS);
 237         x = intel_uncore_read8(uncore, TR1);
 238
 239         b = tsfs & TSFS_INTR_MASK;
 240         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 241
 242         return m * x / 127 - b;
 243 }
 244
 245 static int _pxvid_to_vd(u8 pxvid)
 246 {
 247         if (pxvid == 0)
 248                 return 0;
 249
 250         if (pxvid >= 8 && pxvid < 31)
 251                 pxvid = 31;
 252
 253         return (pxvid + 2) * 125;
 254 }
 255
 256 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 257 {
 258         const int vd = _pxvid_to_vd(pxvid);
 259
 260         if (INTEL_INFO(i915)->is_mobile)
 261                 return max(vd - 1125, 0);
 262
 263         return vd;
 264 }
 265
 266 static void __gen5_ips_update(struct intel_ips *ips)
 267 {
 268         struct intel_uncore *uncore =
 269                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 270         u64 now, delta, dt;
 271         u32 count;
 272
 273         lockdep_assert_held(&mchdev_lock);
 274
 275         now = ktime_get_raw_ns();
 276         dt = now - ips->last_time2;
 277         do_div(dt, NSEC_PER_MSEC);
 278
 279         /* Don't divide by 0 */
 280         if (dt <= 10)
 281                 return;
 282
 283         count = intel_uncore_read(uncore, GFXEC);
 284         delta = count - ips->last_count2;
 285
 286         ips->last_count2 = count;
 287         ips->last_time2 = now;
 288
 289         /* More magic constants... */
 290         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 291 }
 292
 293 static void gen5_rps_update(struct intel_rps *rps)
 294 {
 295         spin_lock_irq(&mchdev_lock);
 296         __gen5_ips_update(&rps->ips);
 297         spin_unlock_irq(&mchdev_lock);
 298 }
 299
 300 static bool gen5_rps_set(struct intel_rps *rps, u8 val)
 301 {
 302         struct intel_uncore *uncore = rps_to_uncore(rps);
 303         u16 rgvswctl;
 304
 305         lockdep_assert_held(&mchdev_lock);
 306
 307         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 308         if (rgvswctl & MEMCTL_CMD_STS) {
 309                 DRM_DEBUG("gpu busy, RCS change rejected\n");
 310                 return false; /* still busy with another command */
 311         }
 312
 313         /* Invert the frequency bin into an ips delay */
 314         val = rps->max_freq - val;
 315         val = rps->min_freq + val;
 316
 317         rgvswctl =
 318                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 319                 (val << MEMCTL_FREQ_SHIFT) |
 320                 MEMCTL_SFCAVM;
 321         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 322         intel_uncore_posting_read16(uncore, MEMSWCTL);
 323
 324         rgvswctl |= MEMCTL_CMD_STS;
 325         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 326
 327         return true;
 328 }
 329
 330 static unsigned long intel_pxfreq(u32 vidfreq)
 331 {
 332         int div = (vidfreq & 0x3f0000) >> 16;
 333         int post = (vidfreq & 0x3000) >> 12;
 334         int pre = (vidfreq & 0x7);
 335
 336         if (!pre)
 337                 return 0;
 338
 339         return div * 133333 / (pre << post);
 340 }
 341
 342 static unsigned int init_emon(struct intel_uncore *uncore)
 343 {
 344         u8 pxw[16];
 345         int i;
 346
 347         /* Disable to program */
 348         intel_uncore_write(uncore, ECR, 0);
 349         intel_uncore_posting_read(uncore, ECR);
 350
 351         /* Program energy weights for various events */
 352         intel_uncore_write(uncore, SDEW, 0x15040d00);
 353         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 354         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 355         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 356
 357         for (i = 0; i < 5; i++)
 358                 intel_uncore_write(uncore, PEW(i), 0);
 359         for (i = 0; i < 3; i++)
 360                 intel_uncore_write(uncore, DEW(i), 0);
 361
 362         /* Program P-state weights to account for frequency power adjustment */
 363         for (i = 0; i < 16; i++) {
 364                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 365                 unsigned int freq = intel_pxfreq(pxvidfreq);
 366                 unsigned int vid =
 367                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 368                 unsigned int val;
 369
 370                 val = vid * vid * freq / 1000 * 255;
 371                 val /= 127 * 127 * 900;
 372
 373                 pxw[i] = val;
 374         }
 375         /* Render standby states get 0 weight */
 376         pxw[14] = 0;
 377         pxw[15] = 0;
 378
 379         for (i = 0; i < 4; i++) {
 380                 intel_uncore_write(uncore, PXW(i),
 381                                    pxw[i * 4 + 0] << 24 |
 382                                    pxw[i * 4 + 1] << 16 |
 383                                    pxw[i * 4 + 2] <<  8 |
 384                                    pxw[i * 4 + 3] <<  0);
 385         }
 386
 387         /* Adjust magic regs to magic values (more experimental results) */
 388         intel_uncore_write(uncore, OGW0, 0);
 389         intel_uncore_write(uncore, OGW1, 0);
 390         intel_uncore_write(uncore, EG0, 0x00007f00);
 391         intel_uncore_write(uncore, EG1, 0x0000000e);
 392         intel_uncore_write(uncore, EG2, 0x000e0000);
 393         intel_uncore_write(uncore, EG3, 0x68000300);
 394         intel_uncore_write(uncore, EG4, 0x42000000);
 395         intel_uncore_write(uncore, EG5, 0x00140031);
 396         intel_uncore_write(uncore, EG6, 0);
 397         intel_uncore_write(uncore, EG7, 0);
 398
 399         for (i = 0; i < 8; i++)
 400                 intel_uncore_write(uncore, PXWL(i), 0);
 401
 402         /* Enable PMON + select events */
 403         intel_uncore_write(uncore, ECR, 0x80000019);
 404
 405         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 406 }
 407
 408 static bool gen5_rps_enable(struct intel_rps *rps)
 409 {
 410         struct intel_uncore *uncore = rps_to_uncore(rps);
 411         u8 fstart, vstart;
 412         u32 rgvmodectl;
 413
 414         spin_lock_irq(&mchdev_lock);
 415
 416         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 417
 418         /* Enable temp reporting */
 419         intel_uncore_write16(uncore, PMMISC,
 420                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 421         intel_uncore_write16(uncore, TSC1,
 422                              intel_uncore_read16(uncore, TSC1) | TSE);
 423
 424         /* 100ms RC evaluation intervals */
 425         intel_uncore_write(uncore, RCUPEI, 100000);
 426         intel_uncore_write(uncore, RCDNEI, 100000);
 427
 428         /* Set max/min thresholds to 90ms and 80ms respectively */
 429         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 430         intel_uncore_write(uncore, RCBMINAVG, 80000);
 431
 432         intel_uncore_write(uncore, MEMIHYST, 1);
 433
 434         /* Set up min, max, and cur for interrupt handling */
 435         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 436                 MEMMODE_FSTART_SHIFT;
 437
 438         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 439                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 440
 441         intel_uncore_write(uncore,
 442                            MEMINTREN,
 443                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 444
 445         intel_uncore_write(uncore, VIDSTART, vstart);
 446         intel_uncore_posting_read(uncore, VIDSTART);
 447
 448         rgvmodectl |= MEMMODE_SWMODE_EN;
 449         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 450
 451         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 452                              MEMCTL_CMD_STS) == 0, 10))
 453                 DRM_ERROR("stuck trying to change perf mode\n");
 454         mdelay(1);
 455
 456         gen5_rps_set(rps, rps->cur_freq);
 457
 458         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 459         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 460         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 461         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 462
 463         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 464         rps->ips.last_time2 = ktime_get_raw_ns();
 465
 466         spin_unlock_irq(&mchdev_lock);
 467
 468         rps->ips.corr = init_emon(uncore);
 469
 470         return true;
 471 }
 472
 473 static void gen5_rps_disable(struct intel_rps *rps)
 474 {
 475         struct intel_uncore *uncore = rps_to_uncore(rps);
 476         u16 rgvswctl;
 477
 478         spin_lock_irq(&mchdev_lock);
 479
 480         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 481
 482         /* Ack interrupts, disable EFC interrupt */
 483         intel_uncore_write(uncore, MEMINTREN,
 484                            intel_uncore_read(uncore, MEMINTREN) &
 485                            ~MEMINT_EVAL_CHG_EN);
 486         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 487         intel_uncore_write(uncore, DEIER,
 488                            intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
 489         intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
 490         intel_uncore_write(uncore, DEIMR,
 491                            intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
 492
 493         /* Go back to the starting frequency */
 494         gen5_rps_set(rps, rps->idle_freq);
 495         mdelay(1);
 496         rgvswctl |= MEMCTL_CMD_STS;
 497         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 498         mdelay(1);
 499
 500         spin_unlock_irq(&mchdev_lock);
 501 }
 502
 503 static u32 rps_limits(struct intel_rps *rps, u8 val)
 504 {
 505         u32 limits;
 506
 507         /*
 508          * Only set the down limit when we've reached the lowest level to avoid
 509          * getting more interrupts, otherwise leave this clear. This prevents a
 510          * race in the hw when coming out of rc6: There's a tiny window where
 511          * the hw runs at the minimal clock before selecting the desired
 512          * frequency, if the down threshold expires in that window we will not
 513          * receive a down interrupt.
 514          */
 515         if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
 516                 limits = rps->max_freq_softlimit << 23;
 517                 if (val <= rps->min_freq_softlimit)
 518                         limits |= rps->min_freq_softlimit << 14;
 519         } else {
 520                 limits = rps->max_freq_softlimit << 24;
 521                 if (val <= rps->min_freq_softlimit)
 522                         limits |= rps->min_freq_softlimit << 16;
 523         }
 524
 525         return limits;
 526 }
 527
 528 static void rps_set_power(struct intel_rps *rps, int new_power)
 529 {
 530         struct intel_uncore *uncore = rps_to_uncore(rps);
 531         struct drm_i915_private *i915 = rps_to_i915(rps);
 532         u32 threshold_up = 0, threshold_down = 0; /* in % */
 533         u32 ei_up = 0, ei_down = 0;
 534
 535         lockdep_assert_held(&rps->power.mutex);
 536
 537         if (new_power == rps->power.mode)
 538                 return;
 539
 540         /* Note the units here are not exactly 1us, but 1280ns. */
 541         switch (new_power) {
 542         case LOW_POWER:
 543                 /* Upclock if more than 95% busy over 16ms */
 544                 ei_up = 16000;
 545                 threshold_up = 95;
 546
 547                 /* Downclock if less than 85% busy over 32ms */
 548                 ei_down = 32000;
 549                 threshold_down = 85;
 550                 break;
 551
 552         case BETWEEN:
 553                 /* Upclock if more than 90% busy over 13ms */
 554                 ei_up = 13000;
 555                 threshold_up = 90;
 556
 557                 /* Downclock if less than 75% busy over 32ms */
 558                 ei_down = 32000;
 559                 threshold_down = 75;
 560                 break;
 561
 562         case HIGH_POWER:
 563                 /* Upclock if more than 85% busy over 10ms */
 564                 ei_up = 10000;
 565                 threshold_up = 85;
 566
 567                 /* Downclock if less than 60% busy over 32ms */
 568                 ei_down = 32000;
 569                 threshold_down = 60;
 570                 break;
 571         }
 572
 573         /* When byt can survive without system hang with dynamic
 574          * sw freq adjustments, this restriction can be lifted.
 575          */
 576         if (IS_VALLEYVIEW(i915))
 577                 goto skip_hw_write;
 578
 579         set(uncore, GEN6_RP_UP_EI, GT_INTERVAL_FROM_US(i915, ei_up));
 580         set(uncore, GEN6_RP_UP_THRESHOLD,
 581             GT_INTERVAL_FROM_US(i915, ei_up * threshold_up / 100));
 582
 583         set(uncore, GEN6_RP_DOWN_EI, GT_INTERVAL_FROM_US(i915, ei_down));
 584         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 585             GT_INTERVAL_FROM_US(i915, ei_down * threshold_down / 100));
 586
 587         set(uncore, GEN6_RP_CONTROL,
 588             (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 589             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 590             GEN6_RP_MEDIA_IS_GFX |
 591             GEN6_RP_ENABLE |
 592             GEN6_RP_UP_BUSY_AVG |
 593             GEN6_RP_DOWN_IDLE_AVG);
 594
 595 skip_hw_write:
 596         rps->power.mode = new_power;
 597         rps->power.up_threshold = threshold_up;
 598         rps->power.down_threshold = threshold_down;
 599 }
 600
 601 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 602 {
 603         int new_power;
 604
 605         new_power = rps->power.mode;
 606         switch (rps->power.mode) {
 607         case LOW_POWER:
 608                 if (val > rps->efficient_freq + 1 &&
 609                     val > rps->cur_freq)
 610                         new_power = BETWEEN;
 611                 break;
 612
 613         case BETWEEN:
 614                 if (val <= rps->efficient_freq &&
 615                     val < rps->cur_freq)
 616                         new_power = LOW_POWER;
 617                 else if (val >= rps->rp0_freq &&
 618                          val > rps->cur_freq)
 619                         new_power = HIGH_POWER;
 620                 break;
 621
 622         case HIGH_POWER:
 623                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 624                     val < rps->cur_freq)
 625                         new_power = BETWEEN;
 626                 break;
 627         }
 628         /* Max/min bins are special */
 629         if (val <= rps->min_freq_softlimit)
 630                 new_power = LOW_POWER;
 631         if (val >= rps->max_freq_softlimit)
 632                 new_power = HIGH_POWER;
 633
 634         mutex_lock(&rps->power.mutex);
 635         if (rps->power.interactive)
 636                 new_power = HIGH_POWER;
 637         rps_set_power(rps, new_power);
 638         mutex_unlock(&rps->power.mutex);
 639 }
 640
 641 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 642 {
 643         mutex_lock(&rps->power.mutex);
 644         if (interactive) {
 645                 if (!rps->power.interactive++ && rps->active)
 646                         rps_set_power(rps, HIGH_POWER);
 647         } else {
 648                 GEM_BUG_ON(!rps->power.interactive);
 649                 rps->power.interactive--;
 650         }
 651         mutex_unlock(&rps->power.mutex);
 652 }
 653
 654 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 655 {
 656         struct intel_uncore *uncore = rps_to_uncore(rps);
 657         struct drm_i915_private *i915 = rps_to_i915(rps);
 658         u32 swreq;
 659
 660         if (INTEL_GEN(i915) >= 9)
 661                 swreq = GEN9_FREQUENCY(val);
 662         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 663                 swreq = HSW_FREQUENCY(val);
 664         else
 665                 swreq = (GEN6_FREQUENCY(val) |
 666                          GEN6_OFFSET(0) |
 667                          GEN6_AGGRESSIVE_TURBO);
 668         set(uncore, GEN6_RPNSWREQ, swreq);
 669
 670         return 0;
 671 }
 672
 673 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 674 {
 675         struct drm_i915_private *i915 = rps_to_i915(rps);
 676         int err;
 677
 678         vlv_punit_get(i915);
 679         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 680         vlv_punit_put(i915);
 681
 682         return err;
 683 }
 684
 685 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 686 {
 687         struct drm_i915_private *i915 = rps_to_i915(rps);
 688         int err;
 689
 690         if (INTEL_GEN(i915) < 6)
 691                 return 0;
 692
 693         if (val == rps->last_freq)
 694                 return 0;
 695
 696         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 697                 err = vlv_rps_set(rps, val);
 698         else
 699                 err = gen6_rps_set(rps, val);
 700         if (err)
 701                 return err;
 702
 703         if (update)
 704                 gen6_rps_set_thresholds(rps, val);
 705         rps->last_freq = val;
 706
 707         return 0;
 708 }
 709
 710 void intel_rps_unpark(struct intel_rps *rps)
 711 {
 712         u8 freq;
 713
 714         if (!rps->enabled)
 715                 return;
 716
 717         /*
 718          * Use the user's desired frequency as a guide, but for better
 719          * performance, jump directly to RPe as our starting frequency.
 720          */
 721         mutex_lock(&rps->lock);
 722         rps->active = true;
 723         freq = max(rps->cur_freq, rps->efficient_freq),
 724         freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit);
 725         intel_rps_set(rps, freq);
 726         rps->last_adj = 0;
 727         mutex_unlock(&rps->lock);
 728
 729         if (INTEL_GEN(rps_to_i915(rps)) >= 6)
 730                 rps_enable_interrupts(rps);
 731
 732         if (IS_GEN(rps_to_i915(rps), 5))
 733                 gen5_rps_update(rps);
 734 }
 735
 736 void intel_rps_park(struct intel_rps *rps)
 737 {
 738         struct drm_i915_private *i915 = rps_to_i915(rps);
 739
 740         if (!rps->enabled)
 741                 return;
 742
 743         if (INTEL_GEN(i915) >= 6)
 744                 rps_disable_interrupts(rps);
 745
 746         rps->active = false;
 747         if (rps->last_freq <= rps->idle_freq)
 748                 return;
 749
 750         /*
 751          * The punit delays the write of the frequency and voltage until it
 752          * determines the GPU is awake. During normal usage we don't want to
 753          * waste power changing the frequency if the GPU is sleeping (rc6).
 754          * However, the GPU and driver is now idle and we do not want to delay
 755          * switching to minimum voltage (reducing power whilst idle) as we do
 756          * not expect to be woken in the near future and so must flush the
 757          * change by waking the device.
 758          *
 759          * We choose to take the media powerwell (either would do to trick the
 760          * punit into committing the voltage change) as that takes a lot less
 761          * power than the render powerwell.
 762          */
 763         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 764         rps_set(rps, rps->idle_freq, false);
 765         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 766 }
 767
 768 void intel_rps_boost(struct i915_request *rq)
 769 {
 770         struct intel_rps *rps = &rq->engine->gt->rps;
 771         unsigned long flags;
 772
 773         if (i915_request_signaled(rq) || !rps->active)
 774                 return;
 775
 776         /* Serializes with i915_request_retire() */
 777         spin_lock_irqsave(&rq->lock, flags);
 778         if (!i915_request_has_waitboost(rq) &&
 779             !dma_fence_is_signaled_locked(&rq->fence)) {
 780                 set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);
 781
 782                 if (!atomic_fetch_inc(&rps->num_waiters) &&
 783                     READ_ONCE(rps->cur_freq) < rps->boost_freq)
 784                         schedule_work(&rps->work);
 785
 786                 atomic_inc(&rps->boosts);
 787         }
 788         spin_unlock_irqrestore(&rq->lock, flags);
 789 }
 790
 791 int intel_rps_set(struct intel_rps *rps, u8 val)
 792 {
 793         int err;
 794
 795         lockdep_assert_held(&rps->lock);
 796         GEM_BUG_ON(val > rps->max_freq);
 797         GEM_BUG_ON(val < rps->min_freq);
 798
 799         if (rps->active) {
 800                 err = rps_set(rps, val, true);
 801                 if (err)
 802                         return err;
 803
 804                 /*
 805                  * Make sure we continue to get interrupts
 806                  * until we hit the minimum or maximum frequencies.
 807                  */
 808                 if (INTEL_GEN(rps_to_i915(rps)) >= 6) {
 809                         struct intel_uncore *uncore = rps_to_uncore(rps);
 810
 811                         set(uncore,
 812                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
 813
 814                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
 815                 }
 816         }
 817
 818         rps->cur_freq = val;
 819         return 0;
 820 }
 821
 822 static void gen6_rps_init(struct intel_rps *rps)
 823 {
 824         struct drm_i915_private *i915 = rps_to_i915(rps);
 825         struct intel_uncore *uncore = rps_to_uncore(rps);
 826
 827         /* All of these values are in units of 50MHz */
 828
 829         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
 830         if (IS_GEN9_LP(i915)) {
 831                 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
 832
 833                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
 834                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 835                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
 836         } else {
 837                 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
 838
 839                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
 840                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 841                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
 842         }
 843
 844         /* hw_max = RP0 until we check for overclocking */
 845         rps->max_freq = rps->rp0_freq;
 846
 847         rps->efficient_freq = rps->rp1_freq;
 848         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
 849             IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
 850                 u32 ddcc_status = 0;
 851
 852                 if (sandybridge_pcode_read(i915,
 853                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
 854                                            &ddcc_status, NULL) == 0)
 855                         rps->efficient_freq =
 856                                 clamp_t(u8,
 857                                         (ddcc_status >> 8) & 0xff,
 858                                         rps->min_freq,
 859                                         rps->max_freq);
 860         }
 861
 862         if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
 863                 /* Store the frequency values in 16.66 MHZ units, which is
 864                  * the natural hardware unit for SKL
 865                  */
 866                 rps->rp0_freq *= GEN9_FREQ_SCALER;
 867                 rps->rp1_freq *= GEN9_FREQ_SCALER;
 868                 rps->min_freq *= GEN9_FREQ_SCALER;
 869                 rps->max_freq *= GEN9_FREQ_SCALER;
 870                 rps->efficient_freq *= GEN9_FREQ_SCALER;
 871         }
 872 }
 873
 874 static bool rps_reset(struct intel_rps *rps)
 875 {
 876         /* force a reset */
 877         rps->power.mode = -1;
 878         rps->last_freq = -1;
 879
 880         if (rps_set(rps, rps->min_freq, true)) {
 881                 DRM_ERROR("Failed to reset RPS to initial values\n");
 882                 return false;
 883         }
 884
 885         rps->cur_freq = rps->min_freq;
 886         return true;
 887 }
 888
 889 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
 890 static bool gen9_rps_enable(struct intel_rps *rps)
 891 {
 892         struct drm_i915_private *i915 = rps_to_i915(rps);
 893         struct intel_uncore *uncore = rps_to_uncore(rps);
 894
 895         /* Program defaults and thresholds for RPS */
 896         if (IS_GEN(i915, 9))
 897                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
 898                                       GEN9_FREQUENCY(rps->rp1_freq));
 899
 900         /* 1 second timeout */
 901         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
 902                               GT_INTERVAL_FROM_US(i915, 1000000));
 903
 904         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
 905
 906         return rps_reset(rps);
 907 }
 908
 909 static bool gen8_rps_enable(struct intel_rps *rps)
 910 {
 911         struct intel_uncore *uncore = rps_to_uncore(rps);
 912
 913         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
 914                               HSW_FREQUENCY(rps->rp1_freq));
 915
 916         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
 917         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
 918                               100000000 / 128); /* 1 second timeout */
 919
 920         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
 921
 922         return rps_reset(rps);
 923 }
 924
 925 static bool gen6_rps_enable(struct intel_rps *rps)
 926 {
 927         struct intel_uncore *uncore = rps_to_uncore(rps);
 928
 929         /* Power down if completely idle for over 50ms */
 930         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
 931         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
 932
 933         return rps_reset(rps);
 934 }
 935
 936 static int chv_rps_max_freq(struct intel_rps *rps)
 937 {
 938         struct drm_i915_private *i915 = rps_to_i915(rps);
 939         u32 val;
 940
 941         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
 942
 943         switch (RUNTIME_INFO(i915)->sseu.eu_total) {
 944         case 8:
 945                 /* (2 * 4) config */
 946                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
 947                 break;
 948         case 12:
 949                 /* (2 * 6) config */
 950                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
 951                 break;
 952         case 16:
 953                 /* (2 * 8) config */
 954         default:
 955                 /* Setting (2 * 8) Min RP0 for any other combination */
 956                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
 957                 break;
 958         }
 959
 960         return val & FB_GFX_FREQ_FUSE_MASK;
 961 }
 962
 963 static int chv_rps_rpe_freq(struct intel_rps *rps)
 964 {
 965         struct drm_i915_private *i915 = rps_to_i915(rps);
 966         u32 val;
 967
 968         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
 969         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
 970
 971         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
 972 }
 973
 974 static int chv_rps_guar_freq(struct intel_rps *rps)
 975 {
 976         struct drm_i915_private *i915 = rps_to_i915(rps);
 977         u32 val;
 978
 979         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
 980
 981         return val & FB_GFX_FREQ_FUSE_MASK;
 982 }
 983
 984 static u32 chv_rps_min_freq(struct intel_rps *rps)
 985 {
 986         struct drm_i915_private *i915 = rps_to_i915(rps);
 987         u32 val;
 988
 989         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
 990         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
 991
 992         return val & FB_GFX_FREQ_FUSE_MASK;
 993 }
 994
 995 static bool chv_rps_enable(struct intel_rps *rps)
 996 {
 997         struct intel_uncore *uncore = rps_to_uncore(rps);
 998         struct drm_i915_private *i915 = rps_to_i915(rps);
 999         u32 val;
1000
1001         /* 1: Program defaults and thresholds for RPS*/
1002         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1003         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1004         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1005         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1006         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1007
1008         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1009
1010         /* 2: Enable RPS */
1011         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1012                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1013                               GEN6_RP_MEDIA_IS_GFX |
1014                               GEN6_RP_ENABLE |
1015                               GEN6_RP_UP_BUSY_AVG |
1016                               GEN6_RP_DOWN_IDLE_AVG);
1017
1018         /* Setting Fixed Bias */
1019         vlv_punit_get(i915);
1020
1021         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1022         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1023
1024         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1025
1026         vlv_punit_put(i915);
1027
1028         /* RPS code assumes GPLL is used */
1029         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1030                       "GPLL not enabled\n");
1031
1032         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1033         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
1034
1035         return rps_reset(rps);
1036 }
1037
1038 static int vlv_rps_guar_freq(struct intel_rps *rps)
1039 {
1040         struct drm_i915_private *i915 = rps_to_i915(rps);
1041         u32 val, rp1;
1042
1043         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1044
1045         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1046         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1047
1048         return rp1;
1049 }
1050
1051 static int vlv_rps_max_freq(struct intel_rps *rps)
1052 {
1053         struct drm_i915_private *i915 = rps_to_i915(rps);
1054         u32 val, rp0;
1055
1056         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1057
1058         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1059         /* Clamp to max */
1060         rp0 = min_t(u32, rp0, 0xea);
1061
1062         return rp0;
1063 }
1064
1065 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1066 {
1067         struct drm_i915_private *i915 = rps_to_i915(rps);
1068         u32 val, rpe;
1069
1070         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1071         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1072         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1073         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1074
1075         return rpe;
1076 }
1077
1078 static int vlv_rps_min_freq(struct intel_rps *rps)
1079 {
1080         struct drm_i915_private *i915 = rps_to_i915(rps);
1081         u32 val;
1082
1083         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1084         /*
1085          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1086          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1087          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1088          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1089          * to make sure it matches what Punit accepts.
1090          */
1091         return max_t(u32, val, 0xc0);
1092 }
1093
1094 static bool vlv_rps_enable(struct intel_rps *rps)
1095 {
1096         struct intel_uncore *uncore = rps_to_uncore(rps);
1097         struct drm_i915_private *i915 = rps_to_i915(rps);
1098         u32 val;
1099
1100         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1101         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1102         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1103         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1104         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1105
1106         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1107
1108         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1109                               GEN6_RP_MEDIA_TURBO |
1110                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1111                               GEN6_RP_MEDIA_IS_GFX |
1112                               GEN6_RP_ENABLE |
1113                               GEN6_RP_UP_BUSY_AVG |
1114                               GEN6_RP_DOWN_IDLE_CONT);
1115
1116         vlv_punit_get(i915);
1117
1118         /* Setting Fixed Bias */
1119         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1120         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1121
1122         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1123
1124         vlv_punit_put(i915);
1125
1126         /* RPS code assumes GPLL is used */
1127         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1128                       "GPLL not enabled\n");
1129
1130         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1131         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
1132
1133         return rps_reset(rps);
1134 }
1135
1136 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1137 {
1138         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1139         struct intel_uncore *uncore = rps_to_uncore(rps);
1140         unsigned long t, corr, state1, corr2, state2;
1141         u32 pxvid, ext_v;
1142
1143         lockdep_assert_held(&mchdev_lock);
1144
1145         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1146         pxvid = (pxvid >> 24) & 0x7f;
1147         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1148
1149         state1 = ext_v;
1150
1151         /* Revel in the empirically derived constants */
1152
1153         /* Correction factor in 1/100000 units */
1154         t = ips_mch_val(uncore);
1155         if (t > 80)
1156                 corr = t * 2349 + 135940;
1157         else if (t >= 50)
1158                 corr = t * 964 + 29317;
1159         else /* < 50 */
1160                 corr = t * 301 + 1004;
1161
1162         corr = corr * 150142 * state1 / 10000 - 78642;
1163         corr /= 100000;
1164         corr2 = corr * ips->corr;
1165
1166         state2 = corr2 * state1 / 10000;
1167         state2 /= 100; /* convert to mW */
1168
1169         __gen5_ips_update(ips);
1170
1171         return ips->gfx_power + state2;
1172 }
1173
1174 void intel_rps_enable(struct intel_rps *rps)
1175 {
1176         struct drm_i915_private *i915 = rps_to_i915(rps);
1177         struct intel_uncore *uncore = rps_to_uncore(rps);
1178
1179         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1180         if (IS_CHERRYVIEW(i915))
1181                 rps->enabled = chv_rps_enable(rps);
1182         else if (IS_VALLEYVIEW(i915))
1183                 rps->enabled = vlv_rps_enable(rps);
1184         else if (INTEL_GEN(i915) >= 9)
1185                 rps->enabled = gen9_rps_enable(rps);
1186         else if (INTEL_GEN(i915) >= 8)
1187                 rps->enabled = gen8_rps_enable(rps);
1188         else if (INTEL_GEN(i915) >= 6)
1189                 rps->enabled = gen6_rps_enable(rps);
1190         else if (IS_IRONLAKE_M(i915))
1191                 rps->enabled = gen5_rps_enable(rps);
1192         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1193         if (!rps->enabled)
1194                 return;
1195
1196         drm_WARN_ON(&i915->drm, rps->max_freq < rps->min_freq);
1197         drm_WARN_ON(&i915->drm, rps->idle_freq > rps->max_freq);
1198
1199         drm_WARN_ON(&i915->drm, rps->efficient_freq < rps->min_freq);
1200         drm_WARN_ON(&i915->drm, rps->efficient_freq > rps->max_freq);
1201 }
1202
1203 static void gen6_rps_disable(struct intel_rps *rps)
1204 {
1205         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1206 }
1207
1208 void intel_rps_disable(struct intel_rps *rps)
1209 {
1210         struct drm_i915_private *i915 = rps_to_i915(rps);
1211
1212         rps->enabled = false;
1213
1214         if (INTEL_GEN(i915) >= 6)
1215                 gen6_rps_disable(rps);
1216         else if (IS_IRONLAKE_M(i915))
1217                 gen5_rps_disable(rps);
1218 }
1219
1220 static int byt_gpu_freq(struct intel_rps *rps, int val)
1221 {
1222         /*
1223          * N = val - 0xb7
1224          * Slow = Fast = GPLL ref * N
1225          */
1226         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1227 }
1228
1229 static int byt_freq_opcode(struct intel_rps *rps, int val)
1230 {
1231         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1232 }
1233
1234 static int chv_gpu_freq(struct intel_rps *rps, int val)
1235 {
1236         /*
1237          * N = val / 2
1238          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1239          */
1240         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1241 }
1242
1243 static int chv_freq_opcode(struct intel_rps *rps, int val)
1244 {
1245         /* CHV needs even values */
1246         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1247 }
1248
1249 int intel_gpu_freq(struct intel_rps *rps, int val)
1250 {
1251         struct drm_i915_private *i915 = rps_to_i915(rps);
1252
1253         if (INTEL_GEN(i915) >= 9)
1254                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1255                                          GEN9_FREQ_SCALER);
1256         else if (IS_CHERRYVIEW(i915))
1257                 return chv_gpu_freq(rps, val);
1258         else if (IS_VALLEYVIEW(i915))
1259                 return byt_gpu_freq(rps, val);
1260         else
1261                 return val * GT_FREQUENCY_MULTIPLIER;
1262 }
1263
1264 int intel_freq_opcode(struct intel_rps *rps, int val)
1265 {
1266         struct drm_i915_private *i915 = rps_to_i915(rps);
1267
1268         if (INTEL_GEN(i915) >= 9)
1269                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1270                                          GT_FREQUENCY_MULTIPLIER);
1271         else if (IS_CHERRYVIEW(i915))
1272                 return chv_freq_opcode(rps, val);
1273         else if (IS_VALLEYVIEW(i915))
1274                 return byt_freq_opcode(rps, val);
1275         else
1276                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1277 }
1278
1279 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1280 {
1281         struct drm_i915_private *i915 = rps_to_i915(rps);
1282
1283         rps->gpll_ref_freq =
1284                 vlv_get_cck_clock(i915, "GPLL ref",
1285                                   CCK_GPLL_CLOCK_CONTROL,
1286                                   i915->czclk_freq);
1287
1288         DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq);
1289 }
1290
1291 static void vlv_rps_init(struct intel_rps *rps)
1292 {
1293         struct drm_i915_private *i915 = rps_to_i915(rps);
1294         u32 val;
1295
1296         vlv_iosf_sb_get(i915,
1297                         BIT(VLV_IOSF_SB_PUNIT) |
1298                         BIT(VLV_IOSF_SB_NC) |
1299                         BIT(VLV_IOSF_SB_CCK));
1300
1301         vlv_init_gpll_ref_freq(rps);
1302
1303         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1304         switch ((val >> 6) & 3) {
1305         case 0:
1306         case 1:
1307                 i915->mem_freq = 800;
1308                 break;
1309         case 2:
1310                 i915->mem_freq = 1066;
1311                 break;
1312         case 3:
1313                 i915->mem_freq = 1333;
1314                 break;
1315         }
1316         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
1317
1318         rps->max_freq = vlv_rps_max_freq(rps);
1319         rps->rp0_freq = rps->max_freq;
1320         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
1321                          intel_gpu_freq(rps, rps->max_freq),
1322                          rps->max_freq);
1323
1324         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1325         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
1326                          intel_gpu_freq(rps, rps->efficient_freq),
1327                          rps->efficient_freq);
1328
1329         rps->rp1_freq = vlv_rps_guar_freq(rps);
1330         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1331                          intel_gpu_freq(rps, rps->rp1_freq),
1332                          rps->rp1_freq);
1333
1334         rps->min_freq = vlv_rps_min_freq(rps);
1335         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
1336                          intel_gpu_freq(rps, rps->min_freq),
1337                          rps->min_freq);
1338
1339         vlv_iosf_sb_put(i915,
1340                         BIT(VLV_IOSF_SB_PUNIT) |
1341                         BIT(VLV_IOSF_SB_NC) |
1342                         BIT(VLV_IOSF_SB_CCK));
1343 }
1344
1345 static void chv_rps_init(struct intel_rps *rps)
1346 {
1347         struct drm_i915_private *i915 = rps_to_i915(rps);
1348         u32 val;
1349
1350         vlv_iosf_sb_get(i915,
1351                         BIT(VLV_IOSF_SB_PUNIT) |
1352                         BIT(VLV_IOSF_SB_NC) |
1353                         BIT(VLV_IOSF_SB_CCK));
1354
1355         vlv_init_gpll_ref_freq(rps);
1356
1357         val = vlv_cck_read(i915, CCK_FUSE_REG);
1358
1359         switch ((val >> 2) & 0x7) {
1360         case 3:
1361                 i915->mem_freq = 2000;
1362                 break;
1363         default:
1364                 i915->mem_freq = 1600;
1365                 break;
1366         }
1367         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
1368
1369         rps->max_freq = chv_rps_max_freq(rps);
1370         rps->rp0_freq = rps->max_freq;
1371         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
1372                          intel_gpu_freq(rps, rps->max_freq),
1373                          rps->max_freq);
1374
1375         rps->efficient_freq = chv_rps_rpe_freq(rps);
1376         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
1377                          intel_gpu_freq(rps, rps->efficient_freq),
1378                          rps->efficient_freq);
1379
1380         rps->rp1_freq = chv_rps_guar_freq(rps);
1381         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
1382                          intel_gpu_freq(rps, rps->rp1_freq),
1383                          rps->rp1_freq);
1384
1385         rps->min_freq = chv_rps_min_freq(rps);
1386         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
1387                          intel_gpu_freq(rps, rps->min_freq),
1388                          rps->min_freq);
1389
1390         vlv_iosf_sb_put(i915,
1391                         BIT(VLV_IOSF_SB_PUNIT) |
1392                         BIT(VLV_IOSF_SB_NC) |
1393                         BIT(VLV_IOSF_SB_CCK));
1394
1395         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1396                                    rps->rp1_freq | rps->min_freq) & 1,
1397                       "Odd GPU freq values\n");
1398 }
1399
1400 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1401 {
1402         ei->ktime = ktime_get_raw();
1403         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1404         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1405 }
1406
1407 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1408 {
1409         struct intel_uncore *uncore = rps_to_uncore(rps);
1410         const struct intel_rps_ei *prev = &rps->ei;
1411         struct intel_rps_ei now;
1412         u32 events = 0;
1413
1414         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1415                 return 0;
1416
1417         vlv_c0_read(uncore, &now);
1418
1419         if (prev->ktime) {
1420                 u64 time, c0;
1421                 u32 render, media;
1422
1423                 time = ktime_us_delta(now.ktime, prev->ktime);
1424
1425                 time *= rps_to_i915(rps)->czclk_freq;
1426
1427                 /* Workload can be split between render + media,
1428                  * e.g. SwapBuffers being blitted in X after being rendered in
1429                  * mesa. To account for this we need to combine both engines
1430                  * into our activity counter.
1431                  */
1432                 render = now.render_c0 - prev->render_c0;
1433                 media = now.media_c0 - prev->media_c0;
1434                 c0 = max(render, media);
1435                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1436
1437                 if (c0 > time * rps->power.up_threshold)
1438                         events = GEN6_PM_RP_UP_THRESHOLD;
1439                 else if (c0 < time * rps->power.down_threshold)
1440                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1441         }
1442
1443         rps->ei = now;
1444         return events;
1445 }
1446
1447 static void rps_work(struct work_struct *work)
1448 {
1449         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1450         struct intel_gt *gt = rps_to_gt(rps);
1451         bool client_boost = false;
1452         int new_freq, adj, min, max;
1453         u32 pm_iir = 0;
1454
1455         spin_lock_irq(&gt->irq_lock);
1456         pm_iir = fetch_and_zero(&rps->pm_iir);
1457         client_boost = atomic_read(&rps->num_waiters);
1458         spin_unlock_irq(&gt->irq_lock);
1459
1460         /* Make sure we didn't queue anything we're not going to process. */
1461         if ((pm_iir & rps->pm_events) == 0 && !client_boost)
1462                 goto out;
1463
1464         mutex_lock(&rps->lock);
1465
1466         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1467
1468         adj = rps->last_adj;
1469         new_freq = rps->cur_freq;
1470         min = rps->min_freq_softlimit;
1471         max = rps->max_freq_softlimit;
1472         if (client_boost)
1473                 max = rps->max_freq;
1474         if (client_boost && new_freq < rps->boost_freq) {
1475                 new_freq = rps->boost_freq;
1476                 adj = 0;
1477         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1478                 if (adj > 0)
1479                         adj *= 2;
1480                 else /* CHV needs even encode values */
1481                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1482
1483                 if (new_freq >= rps->max_freq_softlimit)
1484                         adj = 0;
1485         } else if (client_boost) {
1486                 adj = 0;
1487         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1488                 if (rps->cur_freq > rps->efficient_freq)
1489                         new_freq = rps->efficient_freq;
1490                 else if (rps->cur_freq > rps->min_freq_softlimit)
1491                         new_freq = rps->min_freq_softlimit;
1492                 adj = 0;
1493         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1494                 if (adj < 0)
1495                         adj *= 2;
1496                 else /* CHV needs even encode values */
1497                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1498
1499                 if (new_freq <= rps->min_freq_softlimit)
1500                         adj = 0;
1501         } else { /* unknown event */
1502                 adj = 0;
1503         }
1504
1505         rps->last_adj = adj;
1506
1507         /*
1508          * Limit deboosting and boosting to keep ourselves at the extremes
1509          * when in the respective power modes (i.e. slowly decrease frequencies
1510          * while in the HIGH_POWER zone and slowly increase frequencies while
1511          * in the LOW_POWER zone). On idle, we will hit the timeout and drop
1512          * to the next level quickly, and conversely if busy we expect to
1513          * hit a waitboost and rapidly switch into max power.
1514          */
1515         if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
1516             (adj > 0 && rps->power.mode == LOW_POWER))
1517                 rps->last_adj = 0;
1518
1519         /* sysfs frequency interfaces may have snuck in while servicing the
1520          * interrupt
1521          */
1522         new_freq += adj;
1523         new_freq = clamp_t(int, new_freq, min, max);
1524
1525         if (intel_rps_set(rps, new_freq)) {
1526                 DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
1527                 rps->last_adj = 0;
1528         }
1529
1530         mutex_unlock(&rps->lock);
1531
1532 out:
1533         spin_lock_irq(&gt->irq_lock);
1534         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1535         spin_unlock_irq(&gt->irq_lock);
1536 }
1537
1538 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1539 {
1540         struct intel_gt *gt = rps_to_gt(rps);
1541         const u32 events = rps->pm_events & pm_iir;
1542
1543         lockdep_assert_held(&gt->irq_lock);
1544
1545         if (unlikely(!events))
1546                 return;
1547
1548         gen6_gt_pm_mask_irq(gt, events);
1549
1550         rps->pm_iir |= events;
1551         schedule_work(&rps->work);
1552 }
1553
1554 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1555 {
1556         struct intel_gt *gt = rps_to_gt(rps);
1557
1558         if (pm_iir & rps->pm_events) {
1559                 spin_lock(&gt->irq_lock);
1560                 gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events);
1561                 rps->pm_iir |= pm_iir & rps->pm_events;
1562                 schedule_work(&rps->work);
1563                 spin_unlock(&gt->irq_lock);
1564         }
1565
1566         if (INTEL_GEN(gt->i915) >= 8)
1567                 return;
1568
1569         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1570                 intel_engine_signal_breadcrumbs(gt->engine[VECS0]);
1571
1572         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1573                 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1574 }
1575
1576 void gen5_rps_irq_handler(struct intel_rps *rps)
1577 {
1578         struct intel_uncore *uncore = rps_to_uncore(rps);
1579         u32 busy_up, busy_down, max_avg, min_avg;
1580         u8 new_freq;
1581
1582         spin_lock(&mchdev_lock);
1583
1584         intel_uncore_write16(uncore,
1585                              MEMINTRSTS,
1586                              intel_uncore_read(uncore, MEMINTRSTS));
1587
1588         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1589         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1590         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1591         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1592         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1593
1594         /* Handle RCS change request from hw */
1595         new_freq = rps->cur_freq;
1596         if (busy_up > max_avg)
1597                 new_freq++;
1598         else if (busy_down < min_avg)
1599                 new_freq--;
1600         new_freq = clamp(new_freq,
1601                          rps->min_freq_softlimit,
1602                          rps->max_freq_softlimit);
1603
1604         if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
1605                 rps->cur_freq = new_freq;
1606
1607         spin_unlock(&mchdev_lock);
1608 }
1609
1610 void intel_rps_init_early(struct intel_rps *rps)
1611 {
1612         mutex_init(&rps->lock);
1613         mutex_init(&rps->power.mutex);
1614
1615         INIT_WORK(&rps->work, rps_work);
1616
1617         atomic_set(&rps->num_waiters, 0);
1618 }
1619
1620 void intel_rps_init(struct intel_rps *rps)
1621 {
1622         struct drm_i915_private *i915 = rps_to_i915(rps);
1623
1624         if (IS_CHERRYVIEW(i915))
1625                 chv_rps_init(rps);
1626         else if (IS_VALLEYVIEW(i915))
1627                 vlv_rps_init(rps);
1628         else if (INTEL_GEN(i915) >= 6)
1629                 gen6_rps_init(rps);
1630         else if (IS_IRONLAKE_M(i915))
1631                 gen5_rps_init(rps);
1632
1633         /* Derive initial user preferences/limits from the hardware limits */
1634         rps->max_freq_softlimit = rps->max_freq;
1635         rps->min_freq_softlimit = rps->min_freq;
1636
1637         /* After setting max-softlimit, find the overclock max freq */
1638         if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1639                 u32 params = 0;
1640
1641                 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
1642                                        &params, NULL);
1643                 if (params & BIT(31)) { /* OC supported */
1644                         DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1645                                          (rps->max_freq & 0xff) * 50,
1646                                          (params & 0xff) * 50);
1647                         rps->max_freq = params & 0xff;
1648                 }
1649         }
1650
1651         /* Finally allow us to boost to max by default */
1652         rps->boost_freq = rps->max_freq;
1653         rps->idle_freq = rps->min_freq;
1654         rps->cur_freq = rps->idle_freq;
1655
1656         rps->pm_intrmsk_mbz = 0;
1657
1658         /*
1659          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
1660          * if GEN6_PM_UP_EI_EXPIRED is masked.
1661          *
1662          * TODO: verify if this can be reproduced on VLV,CHV.
1663          */
1664         if (INTEL_GEN(i915) <= 7)
1665                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
1666
1667         if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11)
1668                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
1669 }
1670
1671 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
1672 {
1673         struct drm_i915_private *i915 = rps_to_i915(rps);
1674         u32 cagf;
1675
1676         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1677                 cagf = (rpstat >> 8) & 0xff;
1678         else if (INTEL_GEN(i915) >= 9)
1679                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
1680         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
1681                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
1682         else
1683                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
1684
1685         return cagf;
1686 }
1687
1688 static u32 read_cagf(struct intel_rps *rps)
1689 {
1690         struct drm_i915_private *i915 = rps_to_i915(rps);
1691         u32 freq;
1692
1693         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
1694                 vlv_punit_get(i915);
1695                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1696                 vlv_punit_put(i915);
1697         } else {
1698                 freq = intel_uncore_read(rps_to_gt(rps)->uncore, GEN6_RPSTAT1);
1699         }
1700
1701         return intel_rps_get_cagf(rps, freq);
1702 }
1703
1704 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
1705 {
1706         struct intel_runtime_pm *rpm = rps_to_gt(rps)->uncore->rpm;
1707         intel_wakeref_t wakeref;
1708         u32 freq = 0;
1709
1710         with_intel_runtime_pm_if_in_use(rpm, wakeref)
1711                 freq = intel_gpu_freq(rps, read_cagf(rps));
1712
1713         return freq;
1714 }
1715
1716 /* External interface for intel_ips.ko */
1717
1718 static struct drm_i915_private __rcu *ips_mchdev;
1719
1720 /**
1721  * Tells the intel_ips driver that the i915 driver is now loaded, if
1722  * IPS got loaded first.
1723  *
1724  * This awkward dance is so that neither module has to depend on the
1725  * other in order for IPS to do the appropriate communication of
1726  * GPU turbo limits to i915.
1727  */
1728 static void
1729 ips_ping_for_i915_load(void)
1730 {
1731         void (*link)(void);
1732
1733         link = symbol_get(ips_link_to_i915_driver);
1734         if (link) {
1735                 link();
1736                 symbol_put(ips_link_to_i915_driver);
1737         }
1738 }
1739
1740 void intel_rps_driver_register(struct intel_rps *rps)
1741 {
1742         struct intel_gt *gt = rps_to_gt(rps);
1743
1744         /*
1745          * We only register the i915 ips part with intel-ips once everything is
1746          * set up, to avoid intel-ips sneaking in and reading bogus values.
1747          */
1748         if (IS_GEN(gt->i915, 5)) {
1749                 GEM_BUG_ON(ips_mchdev);
1750                 rcu_assign_pointer(ips_mchdev, gt->i915);
1751                 ips_ping_for_i915_load();
1752         }
1753 }
1754
1755 void intel_rps_driver_unregister(struct intel_rps *rps)
1756 {
1757         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
1758                 rcu_assign_pointer(ips_mchdev, NULL);
1759 }
1760
1761 static struct drm_i915_private *mchdev_get(void)
1762 {
1763         struct drm_i915_private *i915;
1764
1765         rcu_read_lock();
1766         i915 = rcu_dereference(ips_mchdev);
1767         if (!kref_get_unless_zero(&i915->drm.ref))
1768                 i915 = NULL;
1769         rcu_read_unlock();
1770
1771         return i915;
1772 }
1773
1774 /**
1775  * i915_read_mch_val - return value for IPS use
1776  *
1777  * Calculate and return a value for the IPS driver to use when deciding whether
1778  * we have thermal and power headroom to increase CPU or GPU power budget.
1779  */
1780 unsigned long i915_read_mch_val(void)
1781 {
1782         struct drm_i915_private *i915;
1783         unsigned long chipset_val = 0;
1784         unsigned long graphics_val = 0;
1785         intel_wakeref_t wakeref;
1786
1787         i915 = mchdev_get();
1788         if (!i915)
1789                 return 0;
1790
1791         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
1792                 struct intel_ips *ips = &i915->gt.rps.ips;
1793
1794                 spin_lock_irq(&mchdev_lock);
1795                 chipset_val = __ips_chipset_val(ips);
1796                 graphics_val = __ips_gfx_val(ips);
1797                 spin_unlock_irq(&mchdev_lock);
1798         }
1799
1800         drm_dev_put(&i915->drm);
1801         return chipset_val + graphics_val;
1802 }
1803 EXPORT_SYMBOL_GPL(i915_read_mch_val);
1804
1805 /**
1806  * i915_gpu_raise - raise GPU frequency limit
1807  *
1808  * Raise the limit; IPS indicates we have thermal headroom.
1809  */
1810 bool i915_gpu_raise(void)
1811 {
1812         struct drm_i915_private *i915;
1813         struct intel_rps *rps;
1814
1815         i915 = mchdev_get();
1816         if (!i915)
1817                 return false;
1818
1819         rps = &i915->gt.rps;
1820
1821         spin_lock_irq(&mchdev_lock);
1822         if (rps->max_freq_softlimit < rps->max_freq)
1823                 rps->max_freq_softlimit++;
1824         spin_unlock_irq(&mchdev_lock);
1825
1826         drm_dev_put(&i915->drm);
1827         return true;
1828 }
1829 EXPORT_SYMBOL_GPL(i915_gpu_raise);
1830
1831 /**
1832  * i915_gpu_lower - lower GPU frequency limit
1833  *
1834  * IPS indicates we're close to a thermal limit, so throttle back the GPU
1835  * frequency maximum.
1836  */
1837 bool i915_gpu_lower(void)
1838 {
1839         struct drm_i915_private *i915;
1840         struct intel_rps *rps;
1841
1842         i915 = mchdev_get();
1843         if (!i915)
1844                 return false;
1845
1846         rps = &i915->gt.rps;
1847
1848         spin_lock_irq(&mchdev_lock);
1849         if (rps->max_freq_softlimit > rps->min_freq)
1850                 rps->max_freq_softlimit--;
1851         spin_unlock_irq(&mchdev_lock);
1852
1853         drm_dev_put(&i915->drm);
1854         return true;
1855 }
1856 EXPORT_SYMBOL_GPL(i915_gpu_lower);
1857
1858 /**
1859  * i915_gpu_busy - indicate GPU business to IPS
1860  *
1861  * Tell the IPS driver whether or not the GPU is busy.
1862  */
1863 bool i915_gpu_busy(void)
1864 {
1865         struct drm_i915_private *i915;
1866         bool ret;
1867
1868         i915 = mchdev_get();
1869         if (!i915)
1870                 return false;
1871
1872         ret = i915->gt.awake;
1873
1874         drm_dev_put(&i915->drm);
1875         return ret;
1876 }
1877 EXPORT_SYMBOL_GPL(i915_gpu_busy);
1878
1879 /**
1880  * i915_gpu_turbo_disable - disable graphics turbo
1881  *
1882  * Disable graphics turbo by resetting the max frequency and setting the
1883  * current frequency to the default.
1884  */
1885 bool i915_gpu_turbo_disable(void)
1886 {
1887         struct drm_i915_private *i915;
1888         struct intel_rps *rps;
1889         bool ret;
1890
1891         i915 = mchdev_get();
1892         if (!i915)
1893                 return false;
1894
1895         rps = &i915->gt.rps;
1896
1897         spin_lock_irq(&mchdev_lock);
1898         rps->max_freq_softlimit = rps->min_freq;
1899         ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
1900         spin_unlock_irq(&mchdev_lock);
1901
1902         drm_dev_put(&i915->drm);
1903         return ret;
1904 }
1905 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);