drivers/gpu/drm/i915/gt/intel_rps.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <drm/i915_drm.h>
   8
   9 #include "i915_drv.h"
  10 #include "intel_gt.h"
  11 #include "intel_gt_clock_utils.h"
  12 #include "intel_gt_irq.h"
  13 #include "intel_gt_pm_irq.h"
  14 #include "intel_rps.h"
  15 #include "intel_sideband.h"
  16 #include "../../../platform/x86/intel_ips.h"
  17
  18 #define BUSY_MAX_EI     20u /* ms */
  19
  20 /*
  21  * Lock protecting IPS related data structures
  22  */
  23 static DEFINE_SPINLOCK(mchdev_lock);
  24
  25 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  26 {
  27         return container_of(rps, struct intel_gt, rps);
  28 }
  29
  30 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  31 {
  32         return rps_to_gt(rps)->i915;
  33 }
  34
  35 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  36 {
  37         return rps_to_gt(rps)->uncore;
  38 }
  39
  40 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  41 {
  42         return mask & ~rps->pm_intrmsk_mbz;
  43 }
  44
  45 static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  46 {
  47         intel_uncore_write_fw(uncore, reg, val);
  48 }
  49
  50 static void rps_timer(struct timer_list *t)
  51 {
  52         struct intel_rps *rps = from_timer(rps, t, timer);
  53         struct intel_engine_cs *engine;
  54         enum intel_engine_id id;
  55         s64 max_busy[3] = {};
  56         ktime_t dt, last;
  57
  58         for_each_engine(engine, rps_to_gt(rps), id) {
  59                 s64 busy;
  60                 int i;
  61
  62                 dt = intel_engine_get_busy_time(engine);
  63                 last = engine->stats.rps;
  64                 engine->stats.rps = dt;
  65
  66                 busy = ktime_to_ns(ktime_sub(dt, last));
  67                 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
  68                         if (busy > max_busy[i])
  69                                 swap(busy, max_busy[i]);
  70                 }
  71         }
  72
  73         dt = ktime_get();
  74         last = rps->pm_timestamp;
  75         rps->pm_timestamp = dt;
  76
  77         if (intel_rps_is_active(rps)) {
  78                 s64 busy;
  79                 int i;
  80
  81                 dt = ktime_sub(dt, last);
  82
  83                 /*
  84                  * Our goal is to evaluate each engine independently, so we run
  85                  * at the lowest clocks required to sustain the heaviest
  86                  * workload. However, a task may be split into sequential
  87                  * dependent operations across a set of engines, such that
  88                  * the independent contributions do not account for high load,
  89                  * but overall the task is GPU bound. For example, consider
  90                  * video decode on vcs followed by colour post-processing
  91                  * on vecs, followed by general post-processing on rcs.
  92                  * Since multi-engines being active does imply a single
  93                  * continuous workload across all engines, we hedge our
  94                  * bets by only contributing a factor of the distributed
  95                  * load into our busyness calculation.
  96                  */
  97                 busy = max_busy[0];
  98                 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
  99                         if (!max_busy[i])
 100                                 break;
 101
 102                         busy += div_u64(max_busy[i], 1 << i);
 103                 }
 104                 GT_TRACE(rps_to_gt(rps),
 105                          "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
 106                          busy, (int)div64_u64(100 * busy, dt),
 107                          max_busy[0], max_busy[1], max_busy[2],
 108                          rps->pm_interval);
 109
 110                 if (100 * busy > rps->power.up_threshold * dt &&
 111                     rps->cur_freq < rps->max_freq_softlimit) {
 112                         rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
 113                         rps->pm_interval = 1;
 114                         schedule_work(&rps->work);
 115                 } else if (100 * busy < rps->power.down_threshold * dt &&
 116                            rps->cur_freq > rps->min_freq_softlimit) {
 117                         rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
 118                         rps->pm_interval = 1;
 119                         schedule_work(&rps->work);
 120                 } else {
 121                         rps->last_adj = 0;
 122                 }
 123
 124                 mod_timer(&rps->timer,
 125                           jiffies + msecs_to_jiffies(rps->pm_interval));
 126                 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
 127         }
 128 }
 129
 130 static void rps_start_timer(struct intel_rps *rps)
 131 {
 132         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 133         rps->pm_interval = 1;
 134         mod_timer(&rps->timer, jiffies + 1);
 135 }
 136
 137 static void rps_stop_timer(struct intel_rps *rps)
 138 {
 139         del_timer_sync(&rps->timer);
 140         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 141         cancel_work_sync(&rps->work);
 142 }
 143
 144 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
 145 {
 146         u32 mask = 0;
 147
 148         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
 149         if (val > rps->min_freq_softlimit)
 150                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
 151                          GEN6_PM_RP_DOWN_THRESHOLD |
 152                          GEN6_PM_RP_DOWN_TIMEOUT);
 153
 154         if (val < rps->max_freq_softlimit)
 155                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
 156
 157         mask &= rps->pm_events;
 158
 159         return rps_pm_sanitize_mask(rps, ~mask);
 160 }
 161
 162 static void rps_reset_ei(struct intel_rps *rps)
 163 {
 164         memset(&rps->ei, 0, sizeof(rps->ei));
 165 }
 166
 167 static void rps_enable_interrupts(struct intel_rps *rps)
 168 {
 169         struct intel_gt *gt = rps_to_gt(rps);
 170
 171         GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
 172                  rps->pm_events, rps_pm_mask(rps, rps->last_freq));
 173
 174         rps_reset_ei(rps);
 175
 176         spin_lock_irq(&gt->irq_lock);
 177         gen6_gt_pm_enable_irq(gt, rps->pm_events);
 178         spin_unlock_irq(&gt->irq_lock);
 179
 180         intel_uncore_write(gt->uncore,
 181                            GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
 182 }
 183
 184 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
 185 {
 186         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
 187 }
 188
 189 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
 190 {
 191         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
 192                 ;
 193 }
 194
 195 static void rps_reset_interrupts(struct intel_rps *rps)
 196 {
 197         struct intel_gt *gt = rps_to_gt(rps);
 198
 199         spin_lock_irq(&gt->irq_lock);
 200         if (INTEL_GEN(gt->i915) >= 11)
 201                 gen11_rps_reset_interrupts(rps);
 202         else
 203                 gen6_rps_reset_interrupts(rps);
 204
 205         rps->pm_iir = 0;
 206         spin_unlock_irq(&gt->irq_lock);
 207 }
 208
 209 static void rps_disable_interrupts(struct intel_rps *rps)
 210 {
 211         struct intel_gt *gt = rps_to_gt(rps);
 212
 213         intel_uncore_write(gt->uncore,
 214                            GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 215
 216         spin_lock_irq(&gt->irq_lock);
 217         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 218         spin_unlock_irq(&gt->irq_lock);
 219
 220         intel_synchronize_irq(gt->i915);
 221
 222         /*
 223          * Now that we will not be generating any more work, flush any
 224          * outstanding tasks. As we are called on the RPS idle path,
 225          * we will reset the GPU to minimum frequencies, so the current
 226          * state of the worker can be discarded.
 227          */
 228         cancel_work_sync(&rps->work);
 229
 230         rps_reset_interrupts(rps);
 231         GT_TRACE(gt, "interrupts:off\n");
 232 }
 233
 234 static const struct cparams {
 235         u16 i;
 236         u16 t;
 237         u16 m;
 238         u16 c;
 239 } cparams[] = {
 240         { 1, 1333, 301, 28664 },
 241         { 1, 1066, 294, 24460 },
 242         { 1, 800, 294, 25192 },
 243         { 0, 1333, 276, 27605 },
 244         { 0, 1066, 276, 27605 },
 245         { 0, 800, 231, 23784 },
 246 };
 247
 248 static void gen5_rps_init(struct intel_rps *rps)
 249 {
 250         struct drm_i915_private *i915 = rps_to_i915(rps);
 251         struct intel_uncore *uncore = rps_to_uncore(rps);
 252         u8 fmax, fmin, fstart;
 253         u32 rgvmodectl;
 254         int c_m, i;
 255
 256         if (i915->fsb_freq <= 3200)
 257                 c_m = 0;
 258         else if (i915->fsb_freq <= 4800)
 259                 c_m = 1;
 260         else
 261                 c_m = 2;
 262
 263         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 264                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 265                         rps->ips.m = cparams[i].m;
 266                         rps->ips.c = cparams[i].c;
 267                         break;
 268                 }
 269         }
 270
 271         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 272
 273         /* Set up min, max, and cur for interrupt handling */
 274         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 275         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 276         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 277                 MEMMODE_FSTART_SHIFT;
 278         drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
 279                 fmax, fmin, fstart);
 280
 281         rps->min_freq = fmax;
 282         rps->efficient_freq = fstart;
 283         rps->max_freq = fmin;
 284 }
 285
 286 static unsigned long
 287 __ips_chipset_val(struct intel_ips *ips)
 288 {
 289         struct intel_uncore *uncore =
 290                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 291         unsigned long now = jiffies_to_msecs(jiffies), dt;
 292         unsigned long result;
 293         u64 total, delta;
 294
 295         lockdep_assert_held(&mchdev_lock);
 296
 297         /*
 298          * Prevent division-by-zero if we are asking too fast.
 299          * Also, we don't get interesting results if we are polling
 300          * faster than once in 10ms, so just return the saved value
 301          * in such cases.
 302          */
 303         dt = now - ips->last_time1;
 304         if (dt <= 10)
 305                 return ips->chipset_power;
 306
 307         /* FIXME: handle per-counter overflow */
 308         total = intel_uncore_read(uncore, DMIEC);
 309         total += intel_uncore_read(uncore, DDREC);
 310         total += intel_uncore_read(uncore, CSIEC);
 311
 312         delta = total - ips->last_count1;
 313
 314         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 315
 316         ips->last_count1 = total;
 317         ips->last_time1 = now;
 318
 319         ips->chipset_power = result;
 320
 321         return result;
 322 }
 323
 324 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 325 {
 326         unsigned int m, x, b;
 327         u32 tsfs;
 328
 329         tsfs = intel_uncore_read(uncore, TSFS);
 330         x = intel_uncore_read8(uncore, TR1);
 331
 332         b = tsfs & TSFS_INTR_MASK;
 333         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 334
 335         return m * x / 127 - b;
 336 }
 337
 338 static int _pxvid_to_vd(u8 pxvid)
 339 {
 340         if (pxvid == 0)
 341                 return 0;
 342
 343         if (pxvid >= 8 && pxvid < 31)
 344                 pxvid = 31;
 345
 346         return (pxvid + 2) * 125;
 347 }
 348
 349 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 350 {
 351         const int vd = _pxvid_to_vd(pxvid);
 352
 353         if (INTEL_INFO(i915)->is_mobile)
 354                 return max(vd - 1125, 0);
 355
 356         return vd;
 357 }
 358
 359 static void __gen5_ips_update(struct intel_ips *ips)
 360 {
 361         struct intel_uncore *uncore =
 362                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 363         u64 now, delta, dt;
 364         u32 count;
 365
 366         lockdep_assert_held(&mchdev_lock);
 367
 368         now = ktime_get_raw_ns();
 369         dt = now - ips->last_time2;
 370         do_div(dt, NSEC_PER_MSEC);
 371
 372         /* Don't divide by 0 */
 373         if (dt <= 10)
 374                 return;
 375
 376         count = intel_uncore_read(uncore, GFXEC);
 377         delta = count - ips->last_count2;
 378
 379         ips->last_count2 = count;
 380         ips->last_time2 = now;
 381
 382         /* More magic constants... */
 383         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 384 }
 385
 386 static void gen5_rps_update(struct intel_rps *rps)
 387 {
 388         spin_lock_irq(&mchdev_lock);
 389         __gen5_ips_update(&rps->ips);
 390         spin_unlock_irq(&mchdev_lock);
 391 }
 392
 393 static bool gen5_rps_set(struct intel_rps *rps, u8 val)
 394 {
 395         struct intel_uncore *uncore = rps_to_uncore(rps);
 396         u16 rgvswctl;
 397
 398         lockdep_assert_held(&mchdev_lock);
 399
 400         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 401         if (rgvswctl & MEMCTL_CMD_STS) {
 402                 DRM_DEBUG("gpu busy, RCS change rejected\n");
 403                 return false; /* still busy with another command */
 404         }
 405
 406         /* Invert the frequency bin into an ips delay */
 407         val = rps->max_freq - val;
 408         val = rps->min_freq + val;
 409
 410         rgvswctl =
 411                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 412                 (val << MEMCTL_FREQ_SHIFT) |
 413                 MEMCTL_SFCAVM;
 414         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 415         intel_uncore_posting_read16(uncore, MEMSWCTL);
 416
 417         rgvswctl |= MEMCTL_CMD_STS;
 418         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 419
 420         return true;
 421 }
 422
 423 static unsigned long intel_pxfreq(u32 vidfreq)
 424 {
 425         int div = (vidfreq & 0x3f0000) >> 16;
 426         int post = (vidfreq & 0x3000) >> 12;
 427         int pre = (vidfreq & 0x7);
 428
 429         if (!pre)
 430                 return 0;
 431
 432         return div * 133333 / (pre << post);
 433 }
 434
 435 static unsigned int init_emon(struct intel_uncore *uncore)
 436 {
 437         u8 pxw[16];
 438         int i;
 439
 440         /* Disable to program */
 441         intel_uncore_write(uncore, ECR, 0);
 442         intel_uncore_posting_read(uncore, ECR);
 443
 444         /* Program energy weights for various events */
 445         intel_uncore_write(uncore, SDEW, 0x15040d00);
 446         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 447         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 448         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 449
 450         for (i = 0; i < 5; i++)
 451                 intel_uncore_write(uncore, PEW(i), 0);
 452         for (i = 0; i < 3; i++)
 453                 intel_uncore_write(uncore, DEW(i), 0);
 454
 455         /* Program P-state weights to account for frequency power adjustment */
 456         for (i = 0; i < 16; i++) {
 457                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 458                 unsigned int freq = intel_pxfreq(pxvidfreq);
 459                 unsigned int vid =
 460                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 461                 unsigned int val;
 462
 463                 val = vid * vid * freq / 1000 * 255;
 464                 val /= 127 * 127 * 900;
 465
 466                 pxw[i] = val;
 467         }
 468         /* Render standby states get 0 weight */
 469         pxw[14] = 0;
 470         pxw[15] = 0;
 471
 472         for (i = 0; i < 4; i++) {
 473                 intel_uncore_write(uncore, PXW(i),
 474                                    pxw[i * 4 + 0] << 24 |
 475                                    pxw[i * 4 + 1] << 16 |
 476                                    pxw[i * 4 + 2] <<  8 |
 477                                    pxw[i * 4 + 3] <<  0);
 478         }
 479
 480         /* Adjust magic regs to magic values (more experimental results) */
 481         intel_uncore_write(uncore, OGW0, 0);
 482         intel_uncore_write(uncore, OGW1, 0);
 483         intel_uncore_write(uncore, EG0, 0x00007f00);
 484         intel_uncore_write(uncore, EG1, 0x0000000e);
 485         intel_uncore_write(uncore, EG2, 0x000e0000);
 486         intel_uncore_write(uncore, EG3, 0x68000300);
 487         intel_uncore_write(uncore, EG4, 0x42000000);
 488         intel_uncore_write(uncore, EG5, 0x00140031);
 489         intel_uncore_write(uncore, EG6, 0);
 490         intel_uncore_write(uncore, EG7, 0);
 491
 492         for (i = 0; i < 8; i++)
 493                 intel_uncore_write(uncore, PXWL(i), 0);
 494
 495         /* Enable PMON + select events */
 496         intel_uncore_write(uncore, ECR, 0x80000019);
 497
 498         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 499 }
 500
 501 static bool gen5_rps_enable(struct intel_rps *rps)
 502 {
 503         struct intel_uncore *uncore = rps_to_uncore(rps);
 504         u8 fstart, vstart;
 505         u32 rgvmodectl;
 506
 507         spin_lock_irq(&mchdev_lock);
 508
 509         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 510
 511         /* Enable temp reporting */
 512         intel_uncore_write16(uncore, PMMISC,
 513                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 514         intel_uncore_write16(uncore, TSC1,
 515                              intel_uncore_read16(uncore, TSC1) | TSE);
 516
 517         /* 100ms RC evaluation intervals */
 518         intel_uncore_write(uncore, RCUPEI, 100000);
 519         intel_uncore_write(uncore, RCDNEI, 100000);
 520
 521         /* Set max/min thresholds to 90ms and 80ms respectively */
 522         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 523         intel_uncore_write(uncore, RCBMINAVG, 80000);
 524
 525         intel_uncore_write(uncore, MEMIHYST, 1);
 526
 527         /* Set up min, max, and cur for interrupt handling */
 528         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 529                 MEMMODE_FSTART_SHIFT;
 530
 531         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 532                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 533
 534         intel_uncore_write(uncore,
 535                            MEMINTREN,
 536                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 537
 538         intel_uncore_write(uncore, VIDSTART, vstart);
 539         intel_uncore_posting_read(uncore, VIDSTART);
 540
 541         rgvmodectl |= MEMMODE_SWMODE_EN;
 542         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 543
 544         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 545                              MEMCTL_CMD_STS) == 0, 10))
 546                 drm_err(&uncore->i915->drm,
 547                         "stuck trying to change perf mode\n");
 548         mdelay(1);
 549
 550         gen5_rps_set(rps, rps->cur_freq);
 551
 552         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 553         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 554         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 555         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 556
 557         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 558         rps->ips.last_time2 = ktime_get_raw_ns();
 559
 560         spin_unlock_irq(&mchdev_lock);
 561
 562         rps->ips.corr = init_emon(uncore);
 563
 564         return true;
 565 }
 566
 567 static void gen5_rps_disable(struct intel_rps *rps)
 568 {
 569         struct intel_uncore *uncore = rps_to_uncore(rps);
 570         u16 rgvswctl;
 571
 572         spin_lock_irq(&mchdev_lock);
 573
 574         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 575
 576         /* Ack interrupts, disable EFC interrupt */
 577         intel_uncore_write(uncore, MEMINTREN,
 578                            intel_uncore_read(uncore, MEMINTREN) &
 579                            ~MEMINT_EVAL_CHG_EN);
 580         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 581         intel_uncore_write(uncore, DEIER,
 582                            intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
 583         intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
 584         intel_uncore_write(uncore, DEIMR,
 585                            intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
 586
 587         /* Go back to the starting frequency */
 588         gen5_rps_set(rps, rps->idle_freq);
 589         mdelay(1);
 590         rgvswctl |= MEMCTL_CMD_STS;
 591         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 592         mdelay(1);
 593
 594         spin_unlock_irq(&mchdev_lock);
 595 }
 596
 597 static u32 rps_limits(struct intel_rps *rps, u8 val)
 598 {
 599         u32 limits;
 600
 601         /*
 602          * Only set the down limit when we've reached the lowest level to avoid
 603          * getting more interrupts, otherwise leave this clear. This prevents a
 604          * race in the hw when coming out of rc6: There's a tiny window where
 605          * the hw runs at the minimal clock before selecting the desired
 606          * frequency, if the down threshold expires in that window we will not
 607          * receive a down interrupt.
 608          */
 609         if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
 610                 limits = rps->max_freq_softlimit << 23;
 611                 if (val <= rps->min_freq_softlimit)
 612                         limits |= rps->min_freq_softlimit << 14;
 613         } else {
 614                 limits = rps->max_freq_softlimit << 24;
 615                 if (val <= rps->min_freq_softlimit)
 616                         limits |= rps->min_freq_softlimit << 16;
 617         }
 618
 619         return limits;
 620 }
 621
 622 static void rps_set_power(struct intel_rps *rps, int new_power)
 623 {
 624         struct intel_gt *gt = rps_to_gt(rps);
 625         struct intel_uncore *uncore = gt->uncore;
 626         u32 threshold_up = 0, threshold_down = 0; /* in % */
 627         u32 ei_up = 0, ei_down = 0;
 628
 629         lockdep_assert_held(&rps->power.mutex);
 630
 631         if (new_power == rps->power.mode)
 632                 return;
 633
 634         threshold_up = 95;
 635         threshold_down = 85;
 636
 637         /* Note the units here are not exactly 1us, but 1280ns. */
 638         switch (new_power) {
 639         case LOW_POWER:
 640                 ei_up = 16000;
 641                 ei_down = 32000;
 642                 break;
 643
 644         case BETWEEN:
 645                 ei_up = 13000;
 646                 ei_down = 32000;
 647                 break;
 648
 649         case HIGH_POWER:
 650                 ei_up = 10000;
 651                 ei_down = 32000;
 652                 break;
 653         }
 654
 655         /* When byt can survive without system hang with dynamic
 656          * sw freq adjustments, this restriction can be lifted.
 657          */
 658         if (IS_VALLEYVIEW(gt->i915))
 659                 goto skip_hw_write;
 660
 661         GT_TRACE(gt,
 662                  "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
 663                  new_power, threshold_up, ei_up, threshold_down, ei_down);
 664
 665         set(uncore, GEN6_RP_UP_EI,
 666             intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
 667         set(uncore, GEN6_RP_UP_THRESHOLD,
 668             intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
 669
 670         set(uncore, GEN6_RP_DOWN_EI,
 671             intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
 672         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 673             intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
 674
 675         set(uncore, GEN6_RP_CONTROL,
 676             (INTEL_GEN(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 677             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 678             GEN6_RP_MEDIA_IS_GFX |
 679             GEN6_RP_ENABLE |
 680             GEN6_RP_UP_BUSY_AVG |
 681             GEN6_RP_DOWN_IDLE_AVG);
 682
 683 skip_hw_write:
 684         rps->power.mode = new_power;
 685         rps->power.up_threshold = threshold_up;
 686         rps->power.down_threshold = threshold_down;
 687 }
 688
 689 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 690 {
 691         int new_power;
 692
 693         new_power = rps->power.mode;
 694         switch (rps->power.mode) {
 695         case LOW_POWER:
 696                 if (val > rps->efficient_freq + 1 &&
 697                     val > rps->cur_freq)
 698                         new_power = BETWEEN;
 699                 break;
 700
 701         case BETWEEN:
 702                 if (val <= rps->efficient_freq &&
 703                     val < rps->cur_freq)
 704                         new_power = LOW_POWER;
 705                 else if (val >= rps->rp0_freq &&
 706                          val > rps->cur_freq)
 707                         new_power = HIGH_POWER;
 708                 break;
 709
 710         case HIGH_POWER:
 711                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 712                     val < rps->cur_freq)
 713                         new_power = BETWEEN;
 714                 break;
 715         }
 716         /* Max/min bins are special */
 717         if (val <= rps->min_freq_softlimit)
 718                 new_power = LOW_POWER;
 719         if (val >= rps->max_freq_softlimit)
 720                 new_power = HIGH_POWER;
 721
 722         mutex_lock(&rps->power.mutex);
 723         if (rps->power.interactive)
 724                 new_power = HIGH_POWER;
 725         rps_set_power(rps, new_power);
 726         mutex_unlock(&rps->power.mutex);
 727 }
 728
 729 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 730 {
 731         GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive));
 732
 733         mutex_lock(&rps->power.mutex);
 734         if (interactive) {
 735                 if (!rps->power.interactive++ && intel_rps_is_active(rps))
 736                         rps_set_power(rps, HIGH_POWER);
 737         } else {
 738                 GEM_BUG_ON(!rps->power.interactive);
 739                 rps->power.interactive--;
 740         }
 741         mutex_unlock(&rps->power.mutex);
 742 }
 743
 744 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 745 {
 746         struct intel_uncore *uncore = rps_to_uncore(rps);
 747         struct drm_i915_private *i915 = rps_to_i915(rps);
 748         u32 swreq;
 749
 750         if (INTEL_GEN(i915) >= 9)
 751                 swreq = GEN9_FREQUENCY(val);
 752         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 753                 swreq = HSW_FREQUENCY(val);
 754         else
 755                 swreq = (GEN6_FREQUENCY(val) |
 756                          GEN6_OFFSET(0) |
 757                          GEN6_AGGRESSIVE_TURBO);
 758         set(uncore, GEN6_RPNSWREQ, swreq);
 759
 760         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
 761                  val, intel_gpu_freq(rps, val), swreq);
 762
 763         return 0;
 764 }
 765
 766 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 767 {
 768         struct drm_i915_private *i915 = rps_to_i915(rps);
 769         int err;
 770
 771         vlv_punit_get(i915);
 772         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 773         vlv_punit_put(i915);
 774
 775         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
 776                  val, intel_gpu_freq(rps, val));
 777
 778         return err;
 779 }
 780
 781 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 782 {
 783         struct drm_i915_private *i915 = rps_to_i915(rps);
 784         int err;
 785
 786         if (INTEL_GEN(i915) < 6)
 787                 return 0;
 788
 789         if (val == rps->last_freq)
 790                 return 0;
 791
 792         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 793                 err = vlv_rps_set(rps, val);
 794         else
 795                 err = gen6_rps_set(rps, val);
 796         if (err)
 797                 return err;
 798
 799         if (update)
 800                 gen6_rps_set_thresholds(rps, val);
 801         rps->last_freq = val;
 802
 803         return 0;
 804 }
 805
 806 void intel_rps_unpark(struct intel_rps *rps)
 807 {
 808         if (!intel_rps_is_enabled(rps))
 809                 return;
 810
 811         GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
 812
 813         /*
 814          * Use the user's desired frequency as a guide, but for better
 815          * performance, jump directly to RPe as our starting frequency.
 816          */
 817         mutex_lock(&rps->lock);
 818
 819         intel_rps_set_active(rps);
 820         intel_rps_set(rps,
 821                       clamp(rps->cur_freq,
 822                             rps->min_freq_softlimit,
 823                             rps->max_freq_softlimit));
 824
 825         mutex_unlock(&rps->lock);
 826
 827         rps->pm_iir = 0;
 828         if (intel_rps_has_interrupts(rps))
 829                 rps_enable_interrupts(rps);
 830         if (intel_rps_uses_timer(rps))
 831                 rps_start_timer(rps);
 832
 833         if (IS_GEN(rps_to_i915(rps), 5))
 834                 gen5_rps_update(rps);
 835 }
 836
 837 void intel_rps_park(struct intel_rps *rps)
 838 {
 839         int adj;
 840
 841         if (!intel_rps_clear_active(rps))
 842                 return;
 843
 844         if (intel_rps_uses_timer(rps))
 845                 rps_stop_timer(rps);
 846         if (intel_rps_has_interrupts(rps))
 847                 rps_disable_interrupts(rps);
 848
 849         if (rps->last_freq <= rps->idle_freq)
 850                 return;
 851
 852         /*
 853          * The punit delays the write of the frequency and voltage until it
 854          * determines the GPU is awake. During normal usage we don't want to
 855          * waste power changing the frequency if the GPU is sleeping (rc6).
 856          * However, the GPU and driver is now idle and we do not want to delay
 857          * switching to minimum voltage (reducing power whilst idle) as we do
 858          * not expect to be woken in the near future and so must flush the
 859          * change by waking the device.
 860          *
 861          * We choose to take the media powerwell (either would do to trick the
 862          * punit into committing the voltage change) as that takes a lot less
 863          * power than the render powerwell.
 864          */
 865         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 866         rps_set(rps, rps->idle_freq, false);
 867         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 868
 869         /*
 870          * Since we will try and restart from the previously requested
 871          * frequency on unparking, treat this idle point as a downclock
 872          * interrupt and reduce the frequency for resume. If we park/unpark
 873          * more frequently than the rps worker can run, we will not respond
 874          * to any EI and never see a change in frequency.
 875          *
 876          * (Note we accommodate Cherryview's limitation of only using an
 877          * even bin by applying it to all.)
 878          */
 879         adj = rps->last_adj;
 880         if (adj < 0)
 881                 adj *= 2;
 882         else /* CHV needs even encode values */
 883                 adj = -2;
 884         rps->last_adj = adj;
 885         rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
 886
 887         GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
 888 }
 889
 890 void intel_rps_boost(struct i915_request *rq)
 891 {
 892         struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
 893         unsigned long flags;
 894
 895         if (i915_request_signaled(rq) || !intel_rps_is_active(rps))
 896                 return;
 897
 898         /* Serializes with i915_request_retire() */
 899         spin_lock_irqsave(&rq->lock, flags);
 900         if (!i915_request_has_waitboost(rq) &&
 901             !dma_fence_is_signaled_locked(&rq->fence)) {
 902                 set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);
 903
 904                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
 905                          rq->fence.context, rq->fence.seqno);
 906
 907                 if (!atomic_fetch_inc(&rps->num_waiters) &&
 908                     READ_ONCE(rps->cur_freq) < rps->boost_freq)
 909                         schedule_work(&rps->work);
 910
 911                 atomic_inc(&rps->boosts);
 912         }
 913         spin_unlock_irqrestore(&rq->lock, flags);
 914 }
 915
 916 int intel_rps_set(struct intel_rps *rps, u8 val)
 917 {
 918         int err;
 919
 920         lockdep_assert_held(&rps->lock);
 921         GEM_BUG_ON(val > rps->max_freq);
 922         GEM_BUG_ON(val < rps->min_freq);
 923
 924         if (intel_rps_is_active(rps)) {
 925                 err = rps_set(rps, val, true);
 926                 if (err)
 927                         return err;
 928
 929                 /*
 930                  * Make sure we continue to get interrupts
 931                  * until we hit the minimum or maximum frequencies.
 932                  */
 933                 if (intel_rps_has_interrupts(rps)) {
 934                         struct intel_uncore *uncore = rps_to_uncore(rps);
 935
 936                         set(uncore,
 937                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
 938
 939                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
 940                 }
 941         }
 942
 943         rps->cur_freq = val;
 944         return 0;
 945 }
 946
 947 static void gen6_rps_init(struct intel_rps *rps)
 948 {
 949         struct drm_i915_private *i915 = rps_to_i915(rps);
 950         struct intel_uncore *uncore = rps_to_uncore(rps);
 951
 952         /* All of these values are in units of 50MHz */
 953
 954         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
 955         if (IS_GEN9_LP(i915)) {
 956                 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
 957
 958                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
 959                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 960                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
 961         } else {
 962                 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
 963
 964                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
 965                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 966                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
 967         }
 968
 969         /* hw_max = RP0 until we check for overclocking */
 970         rps->max_freq = rps->rp0_freq;
 971
 972         rps->efficient_freq = rps->rp1_freq;
 973         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
 974             IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
 975                 u32 ddcc_status = 0;
 976
 977                 if (sandybridge_pcode_read(i915,
 978                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
 979                                            &ddcc_status, NULL) == 0)
 980                         rps->efficient_freq =
 981                                 clamp_t(u8,
 982                                         (ddcc_status >> 8) & 0xff,
 983                                         rps->min_freq,
 984                                         rps->max_freq);
 985         }
 986
 987         if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
 988                 /* Store the frequency values in 16.66 MHZ units, which is
 989                  * the natural hardware unit for SKL
 990                  */
 991                 rps->rp0_freq *= GEN9_FREQ_SCALER;
 992                 rps->rp1_freq *= GEN9_FREQ_SCALER;
 993                 rps->min_freq *= GEN9_FREQ_SCALER;
 994                 rps->max_freq *= GEN9_FREQ_SCALER;
 995                 rps->efficient_freq *= GEN9_FREQ_SCALER;
 996         }
 997 }
 998
 999 static bool rps_reset(struct intel_rps *rps)
1000 {
1001         struct drm_i915_private *i915 = rps_to_i915(rps);
1002
1003         /* force a reset */
1004         rps->power.mode = -1;
1005         rps->last_freq = -1;
1006
1007         if (rps_set(rps, rps->min_freq, true)) {
1008                 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1009                 return false;
1010         }
1011
1012         rps->cur_freq = rps->min_freq;
1013         return true;
1014 }
1015
1016 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1017 static bool gen9_rps_enable(struct intel_rps *rps)
1018 {
1019         struct intel_gt *gt = rps_to_gt(rps);
1020         struct intel_uncore *uncore = gt->uncore;
1021
1022         /* Program defaults and thresholds for RPS */
1023         if (IS_GEN(gt->i915, 9))
1024                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1025                                       GEN9_FREQUENCY(rps->rp1_freq));
1026
1027         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1028
1029         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1030
1031         return rps_reset(rps);
1032 }
1033
1034 static bool gen8_rps_enable(struct intel_rps *rps)
1035 {
1036         struct intel_uncore *uncore = rps_to_uncore(rps);
1037
1038         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1039                               HSW_FREQUENCY(rps->rp1_freq));
1040
1041         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1042
1043         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1044
1045         return rps_reset(rps);
1046 }
1047
1048 static bool gen6_rps_enable(struct intel_rps *rps)
1049 {
1050         struct intel_uncore *uncore = rps_to_uncore(rps);
1051
1052         /* Power down if completely idle for over 50ms */
1053         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1054         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1055
1056         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1057                           GEN6_PM_RP_DOWN_THRESHOLD |
1058                           GEN6_PM_RP_DOWN_TIMEOUT);
1059
1060         return rps_reset(rps);
1061 }
1062
1063 static int chv_rps_max_freq(struct intel_rps *rps)
1064 {
1065         struct drm_i915_private *i915 = rps_to_i915(rps);
1066         u32 val;
1067
1068         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1069
1070         switch (RUNTIME_INFO(i915)->sseu.eu_total) {
1071         case 8:
1072                 /* (2 * 4) config */
1073                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1074                 break;
1075         case 12:
1076                 /* (2 * 6) config */
1077                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1078                 break;
1079         case 16:
1080                 /* (2 * 8) config */
1081         default:
1082                 /* Setting (2 * 8) Min RP0 for any other combination */
1083                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1084                 break;
1085         }
1086
1087         return val & FB_GFX_FREQ_FUSE_MASK;
1088 }
1089
1090 static int chv_rps_rpe_freq(struct intel_rps *rps)
1091 {
1092         struct drm_i915_private *i915 = rps_to_i915(rps);
1093         u32 val;
1094
1095         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1096         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1097
1098         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1099 }
1100
1101 static int chv_rps_guar_freq(struct intel_rps *rps)
1102 {
1103         struct drm_i915_private *i915 = rps_to_i915(rps);
1104         u32 val;
1105
1106         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1107
1108         return val & FB_GFX_FREQ_FUSE_MASK;
1109 }
1110
1111 static u32 chv_rps_min_freq(struct intel_rps *rps)
1112 {
1113         struct drm_i915_private *i915 = rps_to_i915(rps);
1114         u32 val;
1115
1116         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1117         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1118
1119         return val & FB_GFX_FREQ_FUSE_MASK;
1120 }
1121
1122 static bool chv_rps_enable(struct intel_rps *rps)
1123 {
1124         struct intel_uncore *uncore = rps_to_uncore(rps);
1125         struct drm_i915_private *i915 = rps_to_i915(rps);
1126         u32 val;
1127
1128         /* 1: Program defaults and thresholds for RPS*/
1129         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1130         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1131         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1132         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1133         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1134
1135         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1136
1137         /* 2: Enable RPS */
1138         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1139                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1140                               GEN6_RP_MEDIA_IS_GFX |
1141                               GEN6_RP_ENABLE |
1142                               GEN6_RP_UP_BUSY_AVG |
1143                               GEN6_RP_DOWN_IDLE_AVG);
1144
1145         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1146                           GEN6_PM_RP_DOWN_THRESHOLD |
1147                           GEN6_PM_RP_DOWN_TIMEOUT);
1148
1149         /* Setting Fixed Bias */
1150         vlv_punit_get(i915);
1151
1152         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1153         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1154
1155         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1156
1157         vlv_punit_put(i915);
1158
1159         /* RPS code assumes GPLL is used */
1160         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1161                       "GPLL not enabled\n");
1162
1163         drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1164         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1165
1166         return rps_reset(rps);
1167 }
1168
1169 static int vlv_rps_guar_freq(struct intel_rps *rps)
1170 {
1171         struct drm_i915_private *i915 = rps_to_i915(rps);
1172         u32 val, rp1;
1173
1174         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1175
1176         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1177         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1178
1179         return rp1;
1180 }
1181
1182 static int vlv_rps_max_freq(struct intel_rps *rps)
1183 {
1184         struct drm_i915_private *i915 = rps_to_i915(rps);
1185         u32 val, rp0;
1186
1187         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1188
1189         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1190         /* Clamp to max */
1191         rp0 = min_t(u32, rp0, 0xea);
1192
1193         return rp0;
1194 }
1195
1196 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1197 {
1198         struct drm_i915_private *i915 = rps_to_i915(rps);
1199         u32 val, rpe;
1200
1201         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1202         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1203         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1204         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1205
1206         return rpe;
1207 }
1208
1209 static int vlv_rps_min_freq(struct intel_rps *rps)
1210 {
1211         struct drm_i915_private *i915 = rps_to_i915(rps);
1212         u32 val;
1213
1214         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1215         /*
1216          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1217          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1218          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1219          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1220          * to make sure it matches what Punit accepts.
1221          */
1222         return max_t(u32, val, 0xc0);
1223 }
1224
1225 static bool vlv_rps_enable(struct intel_rps *rps)
1226 {
1227         struct intel_uncore *uncore = rps_to_uncore(rps);
1228         struct drm_i915_private *i915 = rps_to_i915(rps);
1229         u32 val;
1230
1231         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1232         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1233         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1234         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1235         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1236
1237         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1238
1239         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1240                               GEN6_RP_MEDIA_TURBO |
1241                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1242                               GEN6_RP_MEDIA_IS_GFX |
1243                               GEN6_RP_ENABLE |
1244                               GEN6_RP_UP_BUSY_AVG |
1245                               GEN6_RP_DOWN_IDLE_CONT);
1246
1247         /* WaGsvRC0ResidencyMethod:vlv */
1248         rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1249
1250         vlv_punit_get(i915);
1251
1252         /* Setting Fixed Bias */
1253         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1254         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1255
1256         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1257
1258         vlv_punit_put(i915);
1259
1260         /* RPS code assumes GPLL is used */
1261         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1262                       "GPLL not enabled\n");
1263
1264         drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1265         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1266
1267         return rps_reset(rps);
1268 }
1269
1270 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1271 {
1272         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1273         struct intel_uncore *uncore = rps_to_uncore(rps);
1274         unsigned long t, corr, state1, corr2, state2;
1275         u32 pxvid, ext_v;
1276
1277         lockdep_assert_held(&mchdev_lock);
1278
1279         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1280         pxvid = (pxvid >> 24) & 0x7f;
1281         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1282
1283         state1 = ext_v;
1284
1285         /* Revel in the empirically derived constants */
1286
1287         /* Correction factor in 1/100000 units */
1288         t = ips_mch_val(uncore);
1289         if (t > 80)
1290                 corr = t * 2349 + 135940;
1291         else if (t >= 50)
1292                 corr = t * 964 + 29317;
1293         else /* < 50 */
1294                 corr = t * 301 + 1004;
1295
1296         corr = corr * 150142 * state1 / 10000 - 78642;
1297         corr /= 100000;
1298         corr2 = corr * ips->corr;
1299
1300         state2 = corr2 * state1 / 10000;
1301         state2 /= 100; /* convert to mW */
1302
1303         __gen5_ips_update(ips);
1304
1305         return ips->gfx_power + state2;
1306 }
1307
1308 static bool has_busy_stats(struct intel_rps *rps)
1309 {
1310         struct intel_engine_cs *engine;
1311         enum intel_engine_id id;
1312
1313         for_each_engine(engine, rps_to_gt(rps), id) {
1314                 if (!intel_engine_supports_stats(engine))
1315                         return false;
1316         }
1317
1318         return true;
1319 }
1320
1321 void intel_rps_enable(struct intel_rps *rps)
1322 {
1323         struct drm_i915_private *i915 = rps_to_i915(rps);
1324         struct intel_uncore *uncore = rps_to_uncore(rps);
1325         bool enabled = false;
1326
1327         if (!HAS_RPS(i915))
1328                 return;
1329
1330         intel_gt_check_clock_frequency(rps_to_gt(rps));
1331
1332         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1333         if (rps->max_freq <= rps->min_freq)
1334                 /* leave disabled, no room for dynamic reclocking */;
1335         else if (IS_CHERRYVIEW(i915))
1336                 enabled = chv_rps_enable(rps);
1337         else if (IS_VALLEYVIEW(i915))
1338                 enabled = vlv_rps_enable(rps);
1339         else if (INTEL_GEN(i915) >= 9)
1340                 enabled = gen9_rps_enable(rps);
1341         else if (INTEL_GEN(i915) >= 8)
1342                 enabled = gen8_rps_enable(rps);
1343         else if (INTEL_GEN(i915) >= 6)
1344                 enabled = gen6_rps_enable(rps);
1345         else if (IS_IRONLAKE_M(i915))
1346                 enabled = gen5_rps_enable(rps);
1347         else
1348                 MISSING_CASE(INTEL_GEN(i915));
1349         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1350         if (!enabled)
1351                 return;
1352
1353         GT_TRACE(rps_to_gt(rps),
1354                  "min:%x, max:%x, freq:[%d, %d]\n",
1355                  rps->min_freq, rps->max_freq,
1356                  intel_gpu_freq(rps, rps->min_freq),
1357                  intel_gpu_freq(rps, rps->max_freq));
1358
1359         GEM_BUG_ON(rps->max_freq < rps->min_freq);
1360         GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1361
1362         GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1363         GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1364
1365         if (has_busy_stats(rps))
1366                 intel_rps_set_timer(rps);
1367         else if (INTEL_GEN(i915) >= 6)
1368                 intel_rps_set_interrupts(rps);
1369         else
1370                 /* Ironlake currently uses intel_ips.ko */ {}
1371
1372         intel_rps_set_enabled(rps);
1373 }
1374
1375 static void gen6_rps_disable(struct intel_rps *rps)
1376 {
1377         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1378 }
1379
1380 void intel_rps_disable(struct intel_rps *rps)
1381 {
1382         struct drm_i915_private *i915 = rps_to_i915(rps);
1383
1384         intel_rps_clear_enabled(rps);
1385         intel_rps_clear_interrupts(rps);
1386         intel_rps_clear_timer(rps);
1387
1388         if (INTEL_GEN(i915) >= 6)
1389                 gen6_rps_disable(rps);
1390         else if (IS_IRONLAKE_M(i915))
1391                 gen5_rps_disable(rps);
1392 }
1393
1394 static int byt_gpu_freq(struct intel_rps *rps, int val)
1395 {
1396         /*
1397          * N = val - 0xb7
1398          * Slow = Fast = GPLL ref * N
1399          */
1400         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1401 }
1402
1403 static int byt_freq_opcode(struct intel_rps *rps, int val)
1404 {
1405         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1406 }
1407
1408 static int chv_gpu_freq(struct intel_rps *rps, int val)
1409 {
1410         /*
1411          * N = val / 2
1412          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1413          */
1414         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1415 }
1416
1417 static int chv_freq_opcode(struct intel_rps *rps, int val)
1418 {
1419         /* CHV needs even values */
1420         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1421 }
1422
1423 int intel_gpu_freq(struct intel_rps *rps, int val)
1424 {
1425         struct drm_i915_private *i915 = rps_to_i915(rps);
1426
1427         if (INTEL_GEN(i915) >= 9)
1428                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1429                                          GEN9_FREQ_SCALER);
1430         else if (IS_CHERRYVIEW(i915))
1431                 return chv_gpu_freq(rps, val);
1432         else if (IS_VALLEYVIEW(i915))
1433                 return byt_gpu_freq(rps, val);
1434         else
1435                 return val * GT_FREQUENCY_MULTIPLIER;
1436 }
1437
1438 int intel_freq_opcode(struct intel_rps *rps, int val)
1439 {
1440         struct drm_i915_private *i915 = rps_to_i915(rps);
1441
1442         if (INTEL_GEN(i915) >= 9)
1443                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1444                                          GT_FREQUENCY_MULTIPLIER);
1445         else if (IS_CHERRYVIEW(i915))
1446                 return chv_freq_opcode(rps, val);
1447         else if (IS_VALLEYVIEW(i915))
1448                 return byt_freq_opcode(rps, val);
1449         else
1450                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1451 }
1452
1453 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1454 {
1455         struct drm_i915_private *i915 = rps_to_i915(rps);
1456
1457         rps->gpll_ref_freq =
1458                 vlv_get_cck_clock(i915, "GPLL ref",
1459                                   CCK_GPLL_CLOCK_CONTROL,
1460                                   i915->czclk_freq);
1461
1462         drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1463                 rps->gpll_ref_freq);
1464 }
1465
1466 static void vlv_rps_init(struct intel_rps *rps)
1467 {
1468         struct drm_i915_private *i915 = rps_to_i915(rps);
1469         u32 val;
1470
1471         vlv_iosf_sb_get(i915,
1472                         BIT(VLV_IOSF_SB_PUNIT) |
1473                         BIT(VLV_IOSF_SB_NC) |
1474                         BIT(VLV_IOSF_SB_CCK));
1475
1476         vlv_init_gpll_ref_freq(rps);
1477
1478         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1479         switch ((val >> 6) & 3) {
1480         case 0:
1481         case 1:
1482                 i915->mem_freq = 800;
1483                 break;
1484         case 2:
1485                 i915->mem_freq = 1066;
1486                 break;
1487         case 3:
1488                 i915->mem_freq = 1333;
1489                 break;
1490         }
1491         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1492
1493         rps->max_freq = vlv_rps_max_freq(rps);
1494         rps->rp0_freq = rps->max_freq;
1495         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1496                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1497
1498         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1499         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1500                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1501
1502         rps->rp1_freq = vlv_rps_guar_freq(rps);
1503         drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1504                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1505
1506         rps->min_freq = vlv_rps_min_freq(rps);
1507         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1508                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1509
1510         vlv_iosf_sb_put(i915,
1511                         BIT(VLV_IOSF_SB_PUNIT) |
1512                         BIT(VLV_IOSF_SB_NC) |
1513                         BIT(VLV_IOSF_SB_CCK));
1514 }
1515
1516 static void chv_rps_init(struct intel_rps *rps)
1517 {
1518         struct drm_i915_private *i915 = rps_to_i915(rps);
1519         u32 val;
1520
1521         vlv_iosf_sb_get(i915,
1522                         BIT(VLV_IOSF_SB_PUNIT) |
1523                         BIT(VLV_IOSF_SB_NC) |
1524                         BIT(VLV_IOSF_SB_CCK));
1525
1526         vlv_init_gpll_ref_freq(rps);
1527
1528         val = vlv_cck_read(i915, CCK_FUSE_REG);
1529
1530         switch ((val >> 2) & 0x7) {
1531         case 3:
1532                 i915->mem_freq = 2000;
1533                 break;
1534         default:
1535                 i915->mem_freq = 1600;
1536                 break;
1537         }
1538         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1539
1540         rps->max_freq = chv_rps_max_freq(rps);
1541         rps->rp0_freq = rps->max_freq;
1542         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1543                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1544
1545         rps->efficient_freq = chv_rps_rpe_freq(rps);
1546         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1547                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1548
1549         rps->rp1_freq = chv_rps_guar_freq(rps);
1550         drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1551                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1552
1553         rps->min_freq = chv_rps_min_freq(rps);
1554         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1555                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1556
1557         vlv_iosf_sb_put(i915,
1558                         BIT(VLV_IOSF_SB_PUNIT) |
1559                         BIT(VLV_IOSF_SB_NC) |
1560                         BIT(VLV_IOSF_SB_CCK));
1561
1562         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1563                                    rps->rp1_freq | rps->min_freq) & 1,
1564                       "Odd GPU freq values\n");
1565 }
1566
1567 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1568 {
1569         ei->ktime = ktime_get_raw();
1570         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1571         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1572 }
1573
1574 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1575 {
1576         struct intel_uncore *uncore = rps_to_uncore(rps);
1577         const struct intel_rps_ei *prev = &rps->ei;
1578         struct intel_rps_ei now;
1579         u32 events = 0;
1580
1581         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1582                 return 0;
1583
1584         vlv_c0_read(uncore, &now);
1585
1586         if (prev->ktime) {
1587                 u64 time, c0;
1588                 u32 render, media;
1589
1590                 time = ktime_us_delta(now.ktime, prev->ktime);
1591
1592                 time *= rps_to_i915(rps)->czclk_freq;
1593
1594                 /* Workload can be split between render + media,
1595                  * e.g. SwapBuffers being blitted in X after being rendered in
1596                  * mesa. To account for this we need to combine both engines
1597                  * into our activity counter.
1598                  */
1599                 render = now.render_c0 - prev->render_c0;
1600                 media = now.media_c0 - prev->media_c0;
1601                 c0 = max(render, media);
1602                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1603
1604                 if (c0 > time * rps->power.up_threshold)
1605                         events = GEN6_PM_RP_UP_THRESHOLD;
1606                 else if (c0 < time * rps->power.down_threshold)
1607                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1608         }
1609
1610         rps->ei = now;
1611         return events;
1612 }
1613
1614 static void rps_work(struct work_struct *work)
1615 {
1616         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1617         struct intel_gt *gt = rps_to_gt(rps);
1618         struct drm_i915_private *i915 = rps_to_i915(rps);
1619         bool client_boost = false;
1620         int new_freq, adj, min, max;
1621         u32 pm_iir = 0;
1622
1623         spin_lock_irq(&gt->irq_lock);
1624         pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1625         client_boost = atomic_read(&rps->num_waiters);
1626         spin_unlock_irq(&gt->irq_lock);
1627
1628         /* Make sure we didn't queue anything we're not going to process. */
1629         if (!pm_iir && !client_boost)
1630                 goto out;
1631
1632         mutex_lock(&rps->lock);
1633         if (!intel_rps_is_active(rps)) {
1634                 mutex_unlock(&rps->lock);
1635                 return;
1636         }
1637
1638         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1639
1640         adj = rps->last_adj;
1641         new_freq = rps->cur_freq;
1642         min = rps->min_freq_softlimit;
1643         max = rps->max_freq_softlimit;
1644         if (client_boost)
1645                 max = rps->max_freq;
1646
1647         GT_TRACE(gt,
1648                  "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1649                  pm_iir, yesno(client_boost),
1650                  adj, new_freq, min, max);
1651
1652         if (client_boost && new_freq < rps->boost_freq) {
1653                 new_freq = rps->boost_freq;
1654                 adj = 0;
1655         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1656                 if (adj > 0)
1657                         adj *= 2;
1658                 else /* CHV needs even encode values */
1659                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1660
1661                 if (new_freq >= rps->max_freq_softlimit)
1662                         adj = 0;
1663         } else if (client_boost) {
1664                 adj = 0;
1665         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1666                 if (rps->cur_freq > rps->efficient_freq)
1667                         new_freq = rps->efficient_freq;
1668                 else if (rps->cur_freq > rps->min_freq_softlimit)
1669                         new_freq = rps->min_freq_softlimit;
1670                 adj = 0;
1671         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1672                 if (adj < 0)
1673                         adj *= 2;
1674                 else /* CHV needs even encode values */
1675                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1676
1677                 if (new_freq <= rps->min_freq_softlimit)
1678                         adj = 0;
1679         } else { /* unknown event */
1680                 adj = 0;
1681         }
1682
1683         /*
1684          * sysfs frequency limits may have snuck in while
1685          * servicing the interrupt
1686          */
1687         new_freq += adj;
1688         new_freq = clamp_t(int, new_freq, min, max);
1689
1690         if (intel_rps_set(rps, new_freq)) {
1691                 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1692                 adj = 0;
1693         }
1694         rps->last_adj = adj;
1695
1696         mutex_unlock(&rps->lock);
1697
1698 out:
1699         spin_lock_irq(&gt->irq_lock);
1700         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1701         spin_unlock_irq(&gt->irq_lock);
1702 }
1703
1704 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1705 {
1706         struct intel_gt *gt = rps_to_gt(rps);
1707         const u32 events = rps->pm_events & pm_iir;
1708
1709         lockdep_assert_held(&gt->irq_lock);
1710
1711         if (unlikely(!events))
1712                 return;
1713
1714         GT_TRACE(gt, "irq events:%x\n", events);
1715
1716         gen6_gt_pm_mask_irq(gt, events);
1717
1718         rps->pm_iir |= events;
1719         schedule_work(&rps->work);
1720 }
1721
1722 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1723 {
1724         struct intel_gt *gt = rps_to_gt(rps);
1725         u32 events;
1726
1727         events = pm_iir & rps->pm_events;
1728         if (events) {
1729                 spin_lock(&gt->irq_lock);
1730
1731                 GT_TRACE(gt, "irq events:%x\n", events);
1732
1733                 gen6_gt_pm_mask_irq(gt, events);
1734                 rps->pm_iir |= events;
1735
1736                 schedule_work(&rps->work);
1737                 spin_unlock(&gt->irq_lock);
1738         }
1739
1740         if (INTEL_GEN(gt->i915) >= 8)
1741                 return;
1742
1743         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1744                 intel_engine_signal_breadcrumbs(gt->engine[VECS0]);
1745
1746         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1747                 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1748 }
1749
1750 void gen5_rps_irq_handler(struct intel_rps *rps)
1751 {
1752         struct intel_uncore *uncore = rps_to_uncore(rps);
1753         u32 busy_up, busy_down, max_avg, min_avg;
1754         u8 new_freq;
1755
1756         spin_lock(&mchdev_lock);
1757
1758         intel_uncore_write16(uncore,
1759                              MEMINTRSTS,
1760                              intel_uncore_read(uncore, MEMINTRSTS));
1761
1762         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1763         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1764         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1765         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1766         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1767
1768         /* Handle RCS change request from hw */
1769         new_freq = rps->cur_freq;
1770         if (busy_up > max_avg)
1771                 new_freq++;
1772         else if (busy_down < min_avg)
1773                 new_freq--;
1774         new_freq = clamp(new_freq,
1775                          rps->min_freq_softlimit,
1776                          rps->max_freq_softlimit);
1777
1778         if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
1779                 rps->cur_freq = new_freq;
1780
1781         spin_unlock(&mchdev_lock);
1782 }
1783
1784 void intel_rps_init_early(struct intel_rps *rps)
1785 {
1786         mutex_init(&rps->lock);
1787         mutex_init(&rps->power.mutex);
1788
1789         INIT_WORK(&rps->work, rps_work);
1790         timer_setup(&rps->timer, rps_timer, 0);
1791
1792         atomic_set(&rps->num_waiters, 0);
1793 }
1794
1795 void intel_rps_init(struct intel_rps *rps)
1796 {
1797         struct drm_i915_private *i915 = rps_to_i915(rps);
1798
1799         if (IS_CHERRYVIEW(i915))
1800                 chv_rps_init(rps);
1801         else if (IS_VALLEYVIEW(i915))
1802                 vlv_rps_init(rps);
1803         else if (INTEL_GEN(i915) >= 6)
1804                 gen6_rps_init(rps);
1805         else if (IS_IRONLAKE_M(i915))
1806                 gen5_rps_init(rps);
1807
1808         /* Derive initial user preferences/limits from the hardware limits */
1809         rps->max_freq_softlimit = rps->max_freq;
1810         rps->min_freq_softlimit = rps->min_freq;
1811
1812         /* After setting max-softlimit, find the overclock max freq */
1813         if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1814                 u32 params = 0;
1815
1816                 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
1817                                        &params, NULL);
1818                 if (params & BIT(31)) { /* OC supported */
1819                         drm_dbg(&i915->drm,
1820                                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1821                                 (rps->max_freq & 0xff) * 50,
1822                                 (params & 0xff) * 50);
1823                         rps->max_freq = params & 0xff;
1824                 }
1825         }
1826
1827         /* Finally allow us to boost to max by default */
1828         rps->boost_freq = rps->max_freq;
1829         rps->idle_freq = rps->min_freq;
1830
1831         /* Start in the middle, from here we will autotune based on workload */
1832         rps->cur_freq = rps->efficient_freq;
1833
1834         rps->pm_intrmsk_mbz = 0;
1835
1836         /*
1837          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
1838          * if GEN6_PM_UP_EI_EXPIRED is masked.
1839          *
1840          * TODO: verify if this can be reproduced on VLV,CHV.
1841          */
1842         if (INTEL_GEN(i915) <= 7)
1843                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
1844
1845         if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11)
1846                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
1847
1848         if (INTEL_GEN(i915) >= 6)
1849                 rps_disable_interrupts(rps);
1850 }
1851
1852 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
1853 {
1854         struct drm_i915_private *i915 = rps_to_i915(rps);
1855         u32 cagf;
1856
1857         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1858                 cagf = (rpstat >> 8) & 0xff;
1859         else if (INTEL_GEN(i915) >= 9)
1860                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
1861         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
1862                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
1863         else
1864                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
1865
1866         return cagf;
1867 }
1868
1869 static u32 read_cagf(struct intel_rps *rps)
1870 {
1871         struct drm_i915_private *i915 = rps_to_i915(rps);
1872         u32 freq;
1873
1874         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
1875                 vlv_punit_get(i915);
1876                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1877                 vlv_punit_put(i915);
1878         } else {
1879                 freq = intel_uncore_read(rps_to_uncore(rps), GEN6_RPSTAT1);
1880         }
1881
1882         return intel_rps_get_cagf(rps, freq);
1883 }
1884
1885 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
1886 {
1887         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
1888         intel_wakeref_t wakeref;
1889         u32 freq = 0;
1890
1891         with_intel_runtime_pm_if_in_use(rpm, wakeref)
1892                 freq = intel_gpu_freq(rps, read_cagf(rps));
1893
1894         return freq;
1895 }
1896
1897 /* External interface for intel_ips.ko */
1898
1899 static struct drm_i915_private __rcu *ips_mchdev;
1900
1901 /**
1902  * Tells the intel_ips driver that the i915 driver is now loaded, if
1903  * IPS got loaded first.
1904  *
1905  * This awkward dance is so that neither module has to depend on the
1906  * other in order for IPS to do the appropriate communication of
1907  * GPU turbo limits to i915.
1908  */
1909 static void
1910 ips_ping_for_i915_load(void)
1911 {
1912         void (*link)(void);
1913
1914         link = symbol_get(ips_link_to_i915_driver);
1915         if (link) {
1916                 link();
1917                 symbol_put(ips_link_to_i915_driver);
1918         }
1919 }
1920
1921 void intel_rps_driver_register(struct intel_rps *rps)
1922 {
1923         struct intel_gt *gt = rps_to_gt(rps);
1924
1925         /*
1926          * We only register the i915 ips part with intel-ips once everything is
1927          * set up, to avoid intel-ips sneaking in and reading bogus values.
1928          */
1929         if (IS_GEN(gt->i915, 5)) {
1930                 GEM_BUG_ON(ips_mchdev);
1931                 rcu_assign_pointer(ips_mchdev, gt->i915);
1932                 ips_ping_for_i915_load();
1933         }
1934 }
1935
1936 void intel_rps_driver_unregister(struct intel_rps *rps)
1937 {
1938         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
1939                 rcu_assign_pointer(ips_mchdev, NULL);
1940 }
1941
1942 static struct drm_i915_private *mchdev_get(void)
1943 {
1944         struct drm_i915_private *i915;
1945
1946         rcu_read_lock();
1947         i915 = rcu_dereference(ips_mchdev);
1948         if (!kref_get_unless_zero(&i915->drm.ref))
1949                 i915 = NULL;
1950         rcu_read_unlock();
1951
1952         return i915;
1953 }
1954
1955 /**
1956  * i915_read_mch_val - return value for IPS use
1957  *
1958  * Calculate and return a value for the IPS driver to use when deciding whether
1959  * we have thermal and power headroom to increase CPU or GPU power budget.
1960  */
1961 unsigned long i915_read_mch_val(void)
1962 {
1963         struct drm_i915_private *i915;
1964         unsigned long chipset_val = 0;
1965         unsigned long graphics_val = 0;
1966         intel_wakeref_t wakeref;
1967
1968         i915 = mchdev_get();
1969         if (!i915)
1970                 return 0;
1971
1972         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
1973                 struct intel_ips *ips = &i915->gt.rps.ips;
1974
1975                 spin_lock_irq(&mchdev_lock);
1976                 chipset_val = __ips_chipset_val(ips);
1977                 graphics_val = __ips_gfx_val(ips);
1978                 spin_unlock_irq(&mchdev_lock);
1979         }
1980
1981         drm_dev_put(&i915->drm);
1982         return chipset_val + graphics_val;
1983 }
1984 EXPORT_SYMBOL_GPL(i915_read_mch_val);
1985
1986 /**
1987  * i915_gpu_raise - raise GPU frequency limit
1988  *
1989  * Raise the limit; IPS indicates we have thermal headroom.
1990  */
1991 bool i915_gpu_raise(void)
1992 {
1993         struct drm_i915_private *i915;
1994         struct intel_rps *rps;
1995
1996         i915 = mchdev_get();
1997         if (!i915)
1998                 return false;
1999
2000         rps = &i915->gt.rps;
2001
2002         spin_lock_irq(&mchdev_lock);
2003         if (rps->max_freq_softlimit < rps->max_freq)
2004                 rps->max_freq_softlimit++;
2005         spin_unlock_irq(&mchdev_lock);
2006
2007         drm_dev_put(&i915->drm);
2008         return true;
2009 }
2010 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2011
2012 /**
2013  * i915_gpu_lower - lower GPU frequency limit
2014  *
2015  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2016  * frequency maximum.
2017  */
2018 bool i915_gpu_lower(void)
2019 {
2020         struct drm_i915_private *i915;
2021         struct intel_rps *rps;
2022
2023         i915 = mchdev_get();
2024         if (!i915)
2025                 return false;
2026
2027         rps = &i915->gt.rps;
2028
2029         spin_lock_irq(&mchdev_lock);
2030         if (rps->max_freq_softlimit > rps->min_freq)
2031                 rps->max_freq_softlimit--;
2032         spin_unlock_irq(&mchdev_lock);
2033
2034         drm_dev_put(&i915->drm);
2035         return true;
2036 }
2037 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2038
2039 /**
2040  * i915_gpu_busy - indicate GPU business to IPS
2041  *
2042  * Tell the IPS driver whether or not the GPU is busy.
2043  */
2044 bool i915_gpu_busy(void)
2045 {
2046         struct drm_i915_private *i915;
2047         bool ret;
2048
2049         i915 = mchdev_get();
2050         if (!i915)
2051                 return false;
2052
2053         ret = i915->gt.awake;
2054
2055         drm_dev_put(&i915->drm);
2056         return ret;
2057 }
2058 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2059
2060 /**
2061  * i915_gpu_turbo_disable - disable graphics turbo
2062  *
2063  * Disable graphics turbo by resetting the max frequency and setting the
2064  * current frequency to the default.
2065  */
2066 bool i915_gpu_turbo_disable(void)
2067 {
2068         struct drm_i915_private *i915;
2069         struct intel_rps *rps;
2070         bool ret;
2071
2072         i915 = mchdev_get();
2073         if (!i915)
2074                 return false;
2075
2076         rps = &i915->gt.rps;
2077
2078         spin_lock_irq(&mchdev_lock);
2079         rps->max_freq_softlimit = rps->min_freq;
2080         ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
2081         spin_unlock_irq(&mchdev_lock);
2082
2083         drm_dev_put(&i915->drm);
2084         return ret;
2085 }
2086 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2087
2088 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2089 #include "selftest_rps.c"
2090 #endif