arch/riscv/kvm/aia.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
   4  * Copyright (C) 2022 Ventana Micro Systems Inc.
   5  *
   6  * Authors:
   7  *      Anup Patel <apatel@ventanamicro.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/bitops.h>
  12 #include <linux/irq.h>
  13 #include <linux/irqdomain.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/percpu.h>
  16 #include <linux/spinlock.h>
  17 #include <asm/hwcap.h>
  18 #include <asm/kvm_aia_imsic.h>
  19
  20 struct aia_hgei_control {
  21         raw_spinlock_t lock;
  22         unsigned long free_bitmap;
  23         struct kvm_vcpu *owners[BITS_PER_LONG];
  24 };
  25 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
  26 static int hgei_parent_irq;
  27
  28 unsigned int kvm_riscv_aia_nr_hgei;
  29 unsigned int kvm_riscv_aia_max_ids;
  30 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
  31
  32 static int aia_find_hgei(struct kvm_vcpu *owner)
  33 {
  34         int i, hgei;
  35         unsigned long flags;
  36         struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
  37
  38         raw_spin_lock_irqsave(&hgctrl->lock, flags);
  39
  40         hgei = -1;
  41         for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) {
  42                 if (hgctrl->owners[i] == owner) {
  43                         hgei = i;
  44                         break;
  45                 }
  46         }
  47
  48         raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
  49
  50         put_cpu_ptr(&aia_hgei);
  51         return hgei;
  52 }
  53
  54 static void aia_set_hvictl(bool ext_irq_pending)
  55 {
  56         unsigned long hvictl;
  57
  58         /*
  59          * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents
  60          * no interrupt in HVICTL.
  61          */
  62
  63         hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID;
  64         hvictl |= ext_irq_pending;
  65         csr_write(CSR_HVICTL, hvictl);
  66 }
  67
  68 #ifdef CONFIG_32BIT
  69 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
  70 {
  71         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
  72         unsigned long mask, val;
  73
  74         if (!kvm_riscv_aia_available())
  75                 return;
  76
  77         if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
  78                 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
  79                 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
  80
  81                 csr->hviph &= ~mask;
  82                 csr->hviph |= val;
  83         }
  84 }
  85
  86 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
  87 {
  88         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
  89
  90         if (kvm_riscv_aia_available())
  91                 csr->vsieh = csr_read(CSR_VSIEH);
  92 }
  93 #endif
  94
  95 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
  96 {
  97         int hgei;
  98         unsigned long seip;
  99
 100         if (!kvm_riscv_aia_available())
 101                 return false;
 102
 103 #ifdef CONFIG_32BIT
 104         if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
 105             (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
 106                 return true;
 107 #endif
 108
 109         seip = vcpu->arch.guest_csr.vsie;
 110         seip &= (unsigned long)mask;
 111         seip &= BIT(IRQ_S_EXT);
 112
 113         if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
 114                 return false;
 115
 116         hgei = aia_find_hgei(vcpu);
 117         if (hgei > 0)
 118                 return !!(csr_read(CSR_HGEIP) & BIT(hgei));
 119
 120         return false;
 121 }
 122
 123 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu)
 124 {
 125         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 126
 127         if (!kvm_riscv_aia_available())
 128                 return;
 129
 130 #ifdef CONFIG_32BIT
 131         csr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph);
 132 #endif
 133         aia_set_hvictl(!!(csr->hvip & BIT(IRQ_VS_EXT)));
 134 }
 135
 136 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu)
 137 {
 138         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 139
 140         if (!kvm_riscv_aia_available())
 141                 return;
 142
 143         csr_write(CSR_VSISELECT, csr->vsiselect);
 144         csr_write(CSR_HVIPRIO1, csr->hviprio1);
 145         csr_write(CSR_HVIPRIO2, csr->hviprio2);
 146 #ifdef CONFIG_32BIT
 147         csr_write(CSR_VSIEH, csr->vsieh);
 148         csr_write(CSR_HVIPH, csr->hviph);
 149         csr_write(CSR_HVIPRIO1H, csr->hviprio1h);
 150         csr_write(CSR_HVIPRIO2H, csr->hviprio2h);
 151 #endif
 152 }
 153
 154 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu)
 155 {
 156         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 157
 158         if (!kvm_riscv_aia_available())
 159                 return;
 160
 161         csr->vsiselect = csr_read(CSR_VSISELECT);
 162         csr->hviprio1 = csr_read(CSR_HVIPRIO1);
 163         csr->hviprio2 = csr_read(CSR_HVIPRIO2);
 164 #ifdef CONFIG_32BIT
 165         csr->vsieh = csr_read(CSR_VSIEH);
 166         csr->hviph = csr_read(CSR_HVIPH);
 167         csr->hviprio1h = csr_read(CSR_HVIPRIO1H);
 168         csr->hviprio2h = csr_read(CSR_HVIPRIO2H);
 169 #endif
 170 }
 171
 172 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu,
 173                                unsigned long reg_num,
 174                                unsigned long *out_val)
 175 {
 176         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 177
 178         if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
 179                 return -EINVAL;
 180
 181         *out_val = 0;
 182         if (kvm_riscv_aia_available())
 183                 *out_val = ((unsigned long *)csr)[reg_num];
 184
 185         return 0;
 186 }
 187
 188 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
 189                                unsigned long reg_num,
 190                                unsigned long val)
 191 {
 192         struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 193
 194         if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
 195                 return -EINVAL;
 196
 197         if (kvm_riscv_aia_available()) {
 198                 ((unsigned long *)csr)[reg_num] = val;
 199
 200 #ifdef CONFIG_32BIT
 201                 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
 202                         WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
 203 #endif
 204         }
 205
 206         return 0;
 207 }
 208
 209 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu,
 210                                  unsigned int csr_num,
 211                                  unsigned long *val,
 212                                  unsigned long new_val,
 213                                  unsigned long wr_mask)
 214 {
 215         /* If AIA not available then redirect trap */
 216         if (!kvm_riscv_aia_available())
 217                 return KVM_INSN_ILLEGAL_TRAP;
 218
 219         /* If AIA not initialized then forward to user space */
 220         if (!kvm_riscv_aia_initialized(vcpu->kvm))
 221                 return KVM_INSN_EXIT_TO_USER_SPACE;
 222
 223         return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI,
 224                                             val, new_val, wr_mask);
 225 }
 226
 227 /*
 228  * External IRQ priority always read-only zero. This means default
 229  * priority order  is always preferred for external IRQs unless
 230  * HVICTL.IID == 9 and HVICTL.IPRIO != 0
 231  */
 232 static int aia_irq2bitpos[] = {
 233 0,     8,   -1,   -1,   16,   24,   -1,   -1, /* 0 - 7 */
 234 32,   -1,   -1,   -1,   -1,   40,   48,   56, /* 8 - 15 */
 235 64,   72,   80,   88,   96,  104,  112,  120, /* 16 - 23 */
 236 -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 24 - 31 */
 237 -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 32 - 39 */
 238 -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 40 - 47 */
 239 -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 48 - 55 */
 240 -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 56 - 63 */
 241 };
 242
 243 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq)
 244 {
 245         unsigned long hviprio;
 246         int bitpos = aia_irq2bitpos[irq];
 247
 248         if (bitpos < 0)
 249                 return 0;
 250
 251         switch (bitpos / BITS_PER_LONG) {
 252         case 0:
 253                 hviprio = csr_read(CSR_HVIPRIO1);
 254                 break;
 255         case 1:
 256 #ifndef CONFIG_32BIT
 257                 hviprio = csr_read(CSR_HVIPRIO2);
 258                 break;
 259 #else
 260                 hviprio = csr_read(CSR_HVIPRIO1H);
 261                 break;
 262         case 2:
 263                 hviprio = csr_read(CSR_HVIPRIO2);
 264                 break;
 265         case 3:
 266                 hviprio = csr_read(CSR_HVIPRIO2H);
 267                 break;
 268 #endif
 269         default:
 270                 return 0;
 271         }
 272
 273         return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK;
 274 }
 275
 276 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio)
 277 {
 278         unsigned long hviprio;
 279         int bitpos = aia_irq2bitpos[irq];
 280
 281         if (bitpos < 0)
 282                 return;
 283
 284         switch (bitpos / BITS_PER_LONG) {
 285         case 0:
 286                 hviprio = csr_read(CSR_HVIPRIO1);
 287                 break;
 288         case 1:
 289 #ifndef CONFIG_32BIT
 290                 hviprio = csr_read(CSR_HVIPRIO2);
 291                 break;
 292 #else
 293                 hviprio = csr_read(CSR_HVIPRIO1H);
 294                 break;
 295         case 2:
 296                 hviprio = csr_read(CSR_HVIPRIO2);
 297                 break;
 298         case 3:
 299                 hviprio = csr_read(CSR_HVIPRIO2H);
 300                 break;
 301 #endif
 302         default:
 303                 return;
 304         }
 305
 306         hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG));
 307         hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG);
 308
 309         switch (bitpos / BITS_PER_LONG) {
 310         case 0:
 311                 csr_write(CSR_HVIPRIO1, hviprio);
 312                 break;
 313         case 1:
 314 #ifndef CONFIG_32BIT
 315                 csr_write(CSR_HVIPRIO2, hviprio);
 316                 break;
 317 #else
 318                 csr_write(CSR_HVIPRIO1H, hviprio);
 319                 break;
 320         case 2:
 321                 csr_write(CSR_HVIPRIO2, hviprio);
 322                 break;
 323         case 3:
 324                 csr_write(CSR_HVIPRIO2H, hviprio);
 325                 break;
 326 #endif
 327         default:
 328                 return;
 329         }
 330 }
 331
 332 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel,
 333                          unsigned long *val, unsigned long new_val,
 334                          unsigned long wr_mask)
 335 {
 336         int i, first_irq, nirqs;
 337         unsigned long old_val;
 338         u8 prio;
 339
 340 #ifndef CONFIG_32BIT
 341         if (isel & 0x1)
 342                 return KVM_INSN_ILLEGAL_TRAP;
 343 #endif
 344
 345         nirqs = 4 * (BITS_PER_LONG / 32);
 346         first_irq = (isel - ISELECT_IPRIO0) * 4;
 347
 348         old_val = 0;
 349         for (i = 0; i < nirqs; i++) {
 350                 prio = aia_get_iprio8(vcpu, first_irq + i);
 351                 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i);
 352         }
 353
 354         if (val)
 355                 *val = old_val;
 356
 357         if (wr_mask) {
 358                 new_val = (old_val & ~wr_mask) | (new_val & wr_mask);
 359                 for (i = 0; i < nirqs; i++) {
 360                         prio = (new_val >> (TOPI_IPRIO_BITS * i)) &
 361                                 TOPI_IPRIO_MASK;
 362                         aia_set_iprio8(vcpu, first_irq + i, prio);
 363                 }
 364         }
 365
 366         return KVM_INSN_CONTINUE_NEXT_SEPC;
 367 }
 368
 369 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
 370                                 unsigned long *val, unsigned long new_val,
 371                                 unsigned long wr_mask)
 372 {
 373         unsigned int isel;
 374
 375         /* If AIA not available then redirect trap */
 376         if (!kvm_riscv_aia_available())
 377                 return KVM_INSN_ILLEGAL_TRAP;
 378
 379         /* First try to emulate in kernel space */
 380         isel = csr_read(CSR_VSISELECT) & ISELECT_MASK;
 381         if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15)
 382                 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask);
 383         else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST &&
 384                  kvm_riscv_aia_initialized(vcpu->kvm))
 385                 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val,
 386                                                     wr_mask);
 387
 388         /* We can't handle it here so redirect to user space */
 389         return KVM_INSN_EXIT_TO_USER_SPACE;
 390 }
 391
 392 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
 393                              void __iomem **hgei_va, phys_addr_t *hgei_pa)
 394 {
 395         int ret = -ENOENT;
 396         unsigned long flags;
 397         struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
 398
 399         if (!kvm_riscv_aia_available() || !hgctrl)
 400                 return -ENODEV;
 401
 402         raw_spin_lock_irqsave(&hgctrl->lock, flags);
 403
 404         if (hgctrl->free_bitmap) {
 405                 ret = __ffs(hgctrl->free_bitmap);
 406                 hgctrl->free_bitmap &= ~BIT(ret);
 407                 hgctrl->owners[ret] = owner;
 408         }
 409
 410         raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
 411
 412         /* TODO: To be updated later by AIA IMSIC HW guest file support */
 413         if (hgei_va)
 414                 *hgei_va = NULL;
 415         if (hgei_pa)
 416                 *hgei_pa = 0;
 417
 418         return ret;
 419 }
 420
 421 void kvm_riscv_aia_free_hgei(int cpu, int hgei)
 422 {
 423         unsigned long flags;
 424         struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
 425
 426         if (!kvm_riscv_aia_available() || !hgctrl)
 427                 return;
 428
 429         raw_spin_lock_irqsave(&hgctrl->lock, flags);
 430
 431         if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
 432                 if (!(hgctrl->free_bitmap & BIT(hgei))) {
 433                         hgctrl->free_bitmap |= BIT(hgei);
 434                         hgctrl->owners[hgei] = NULL;
 435                 }
 436         }
 437
 438         raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
 439 }
 440
 441 void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable)
 442 {
 443         int hgei;
 444
 445         if (!kvm_riscv_aia_available())
 446                 return;
 447
 448         hgei = aia_find_hgei(owner);
 449         if (hgei > 0) {
 450                 if (enable)
 451                         csr_set(CSR_HGEIE, BIT(hgei));
 452                 else
 453                         csr_clear(CSR_HGEIE, BIT(hgei));
 454         }
 455 }
 456
 457 static irqreturn_t hgei_interrupt(int irq, void *dev_id)
 458 {
 459         int i;
 460         unsigned long hgei_mask, flags;
 461         struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
 462
 463         hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
 464         csr_clear(CSR_HGEIE, hgei_mask);
 465
 466         raw_spin_lock_irqsave(&hgctrl->lock, flags);
 467
 468         for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
 469                 if (hgctrl->owners[i])
 470                         kvm_vcpu_kick(hgctrl->owners[i]);
 471         }
 472
 473         raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
 474
 475         put_cpu_ptr(&aia_hgei);
 476         return IRQ_HANDLED;
 477 }
 478
 479 static int aia_hgei_init(void)
 480 {
 481         int cpu, rc;
 482         struct irq_domain *domain;
 483         struct aia_hgei_control *hgctrl;
 484
 485         /* Initialize per-CPU guest external interrupt line management */
 486         for_each_possible_cpu(cpu) {
 487                 hgctrl = per_cpu_ptr(&aia_hgei, cpu);
 488                 raw_spin_lock_init(&hgctrl->lock);
 489                 if (kvm_riscv_aia_nr_hgei) {
 490                         hgctrl->free_bitmap =
 491                                 BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
 492                         hgctrl->free_bitmap &= ~BIT(0);
 493                 } else
 494                         hgctrl->free_bitmap = 0;
 495         }
 496
 497         /* Find INTC irq domain */
 498         domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
 499                                           DOMAIN_BUS_ANY);
 500         if (!domain) {
 501                 kvm_err("unable to find INTC domain\n");
 502                 return -ENOENT;
 503         }
 504
 505         /* Map per-CPU SGEI interrupt from INTC domain */
 506         hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
 507         if (!hgei_parent_irq) {
 508                 kvm_err("unable to map SGEI IRQ\n");
 509                 return -ENOMEM;
 510         }
 511
 512         /* Request per-CPU SGEI interrupt */
 513         rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
 514                                 "riscv-kvm", &aia_hgei);
 515         if (rc) {
 516                 kvm_err("failed to request SGEI IRQ\n");
 517                 return rc;
 518         }
 519
 520         return 0;
 521 }
 522
 523 static void aia_hgei_exit(void)
 524 {
 525         /* Free per-CPU SGEI interrupt */
 526         free_percpu_irq(hgei_parent_irq, &aia_hgei);
 527 }
 528
 529 void kvm_riscv_aia_enable(void)
 530 {
 531         if (!kvm_riscv_aia_available())
 532                 return;
 533
 534         aia_set_hvictl(false);
 535         csr_write(CSR_HVIPRIO1, 0x0);
 536         csr_write(CSR_HVIPRIO2, 0x0);
 537 #ifdef CONFIG_32BIT
 538         csr_write(CSR_HVIPH, 0x0);
 539         csr_write(CSR_HIDELEGH, 0x0);
 540         csr_write(CSR_HVIPRIO1H, 0x0);
 541         csr_write(CSR_HVIPRIO2H, 0x0);
 542 #endif
 543
 544         /* Enable per-CPU SGEI interrupt */
 545         enable_percpu_irq(hgei_parent_irq,
 546                           irq_get_trigger_type(hgei_parent_irq));
 547         csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
 548 }
 549
 550 void kvm_riscv_aia_disable(void)
 551 {
 552         int i;
 553         unsigned long flags;
 554         struct kvm_vcpu *vcpu;
 555         struct aia_hgei_control *hgctrl;
 556
 557         if (!kvm_riscv_aia_available())
 558                 return;
 559         hgctrl = get_cpu_ptr(&aia_hgei);
 560
 561         /* Disable per-CPU SGEI interrupt */
 562         csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
 563         disable_percpu_irq(hgei_parent_irq);
 564
 565         aia_set_hvictl(false);
 566
 567         raw_spin_lock_irqsave(&hgctrl->lock, flags);
 568
 569         for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
 570                 vcpu = hgctrl->owners[i];
 571                 if (!vcpu)
 572                         continue;
 573
 574                 /*
 575                  * We release hgctrl->lock before notifying IMSIC
 576                  * so that we don't have lock ordering issues.
 577                  */
 578                 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
 579
 580                 /* Notify IMSIC */
 581                 kvm_riscv_vcpu_aia_imsic_release(vcpu);
 582
 583                 /*
 584                  * Wakeup VCPU if it was blocked so that it can
 585                  * run on other HARTs
 586                  */
 587                 if (csr_read(CSR_HGEIE) & BIT(i)) {
 588                         csr_clear(CSR_HGEIE, BIT(i));
 589                         kvm_vcpu_kick(vcpu);
 590                 }
 591
 592                 raw_spin_lock_irqsave(&hgctrl->lock, flags);
 593         }
 594
 595         raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
 596
 597         put_cpu_ptr(&aia_hgei);
 598 }
 599
 600 int kvm_riscv_aia_init(void)
 601 {
 602         int rc;
 603
 604         if (!riscv_isa_extension_available(NULL, SxAIA))
 605                 return -ENODEV;
 606
 607         /* Figure-out number of bits in HGEIE */
 608         csr_write(CSR_HGEIE, -1UL);
 609         kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
 610         csr_write(CSR_HGEIE, 0);
 611         if (kvm_riscv_aia_nr_hgei)
 612                 kvm_riscv_aia_nr_hgei--;
 613
 614         /*
 615          * Number of usable HGEI lines should be minimum of per-HART
 616          * IMSIC guest files and number of bits in HGEIE
 617          *
 618          * TODO: To be updated later by AIA IMSIC HW guest file support
 619          */
 620         kvm_riscv_aia_nr_hgei = 0;
 621
 622         /*
 623          * Find number of guest MSI IDs
 624          *
 625          * TODO: To be updated later by AIA IMSIC HW guest file support
 626          */
 627         kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
 628
 629         /* Initialize guest external interrupt line management */
 630         rc = aia_hgei_init();
 631         if (rc)
 632                 return rc;
 633
 634         /* Register device operations */
 635         rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
 636                                      KVM_DEV_TYPE_RISCV_AIA);
 637         if (rc) {
 638                 aia_hgei_exit();
 639                 return rc;
 640         }
 641
 642         /* Enable KVM AIA support */
 643         static_branch_enable(&kvm_riscv_aia_available);
 644
 645         return 0;
 646 }
 647
 648 void kvm_riscv_aia_exit(void)
 649 {
 650         if (!kvm_riscv_aia_available())
 651                 return;
 652
 653         /* Unregister device operations */
 654         kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
 655
 656         /* Cleanup the HGEI state */
 657         aia_hgei_exit();
 658 }