arch/x86/kvm/mmu/page_track.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Support KVM gust page tracking
   4  *
   5  * This feature allows us to track page access in guest. Currently, only
   6  * write access is tracked.
   7  *
   8  * Copyright(C) 2015 Intel Corporation.
   9  *
  10  * Author:
  11  *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
  12  */
  13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14
  15 #include <linux/lockdep.h>
  16 #include <linux/kvm_host.h>
  17 #include <linux/rculist.h>
  18
  19 #include "mmu.h"
  20 #include "mmu_internal.h"
  21 #include "page_track.h"
  22
  23 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
  24 {
  25         return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
  26                !tdp_enabled || kvm_shadow_root_allocated(kvm);
  27 }
  28
  29 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot)
  30 {
  31         kvfree(slot->arch.gfn_write_track);
  32         slot->arch.gfn_write_track = NULL;
  33 }
  34
  35 static int __kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot,
  36                                                  unsigned long npages)
  37 {
  38         const size_t size = sizeof(*slot->arch.gfn_write_track);
  39
  40         if (!slot->arch.gfn_write_track)
  41                 slot->arch.gfn_write_track = __vcalloc(npages, size,
  42                                                        GFP_KERNEL_ACCOUNT);
  43
  44         return slot->arch.gfn_write_track ? 0 : -ENOMEM;
  45 }
  46
  47 int kvm_page_track_create_memslot(struct kvm *kvm,
  48                                   struct kvm_memory_slot *slot,
  49                                   unsigned long npages)
  50 {
  51         if (!kvm_page_track_write_tracking_enabled(kvm))
  52                 return 0;
  53
  54         return __kvm_page_track_write_tracking_alloc(slot, npages);
  55 }
  56
  57 int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot)
  58 {
  59         return __kvm_page_track_write_tracking_alloc(slot, slot->npages);
  60 }
  61
  62 static void update_gfn_write_track(struct kvm_memory_slot *slot, gfn_t gfn,
  63                                    short count)
  64 {
  65         int index, val;
  66
  67         index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
  68
  69         val = slot->arch.gfn_write_track[index];
  70
  71         if (WARN_ON_ONCE(val + count < 0 || val + count > USHRT_MAX))
  72                 return;
  73
  74         slot->arch.gfn_write_track[index] += count;
  75 }
  76
  77 void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
  78                                gfn_t gfn)
  79 {
  80         lockdep_assert_held_write(&kvm->mmu_lock);
  81
  82         lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
  83                             srcu_read_lock_held(&kvm->srcu));
  84
  85         if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
  86                 return;
  87
  88         update_gfn_write_track(slot, gfn, 1);
  89
  90         /*
  91          * new track stops large page mapping for the
  92          * tracked page.
  93          */
  94         kvm_mmu_gfn_disallow_lpage(slot, gfn);
  95
  96         if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
  97                 kvm_flush_remote_tlbs(kvm);
  98 }
  99
 100 void __kvm_write_track_remove_gfn(struct kvm *kvm,
 101                                   struct kvm_memory_slot *slot, gfn_t gfn)
 102 {
 103         lockdep_assert_held_write(&kvm->mmu_lock);
 104
 105         lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||
 106                             srcu_read_lock_held(&kvm->srcu));
 107
 108         if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))
 109                 return;
 110
 111         update_gfn_write_track(slot, gfn, -1);
 112
 113         /*
 114          * allow large page mapping for the tracked page
 115          * after the tracker is gone.
 116          */
 117         kvm_mmu_gfn_allow_lpage(slot, gfn);
 118 }
 119
 120 /*
 121  * check if the corresponding access on the specified guest page is tracked.
 122  */
 123 bool kvm_gfn_is_write_tracked(struct kvm *kvm,
 124                               const struct kvm_memory_slot *slot, gfn_t gfn)
 125 {
 126         int index;
 127
 128         if (!slot)
 129                 return false;
 130
 131         if (!kvm_page_track_write_tracking_enabled(kvm))
 132                 return false;
 133
 134         index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
 135         return !!READ_ONCE(slot->arch.gfn_write_track[index]);
 136 }
 137
 138 #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING
 139 void kvm_page_track_cleanup(struct kvm *kvm)
 140 {
 141         struct kvm_page_track_notifier_head *head;
 142
 143         head = &kvm->arch.track_notifier_head;
 144         cleanup_srcu_struct(&head->track_srcu);
 145 }
 146
 147 int kvm_page_track_init(struct kvm *kvm)
 148 {
 149         struct kvm_page_track_notifier_head *head;
 150
 151         head = &kvm->arch.track_notifier_head;
 152         INIT_HLIST_HEAD(&head->track_notifier_list);
 153         return init_srcu_struct(&head->track_srcu);
 154 }
 155
 156 /*
 157  * register the notifier so that event interception for the tracked guest
 158  * pages can be received.
 159  */
 160 int kvm_page_track_register_notifier(struct kvm *kvm,
 161                                      struct kvm_page_track_notifier_node *n)
 162 {
 163         struct kvm_page_track_notifier_head *head;
 164
 165         if (!kvm || kvm->mm != current->mm)
 166                 return -ESRCH;
 167
 168         kvm_get_kvm(kvm);
 169
 170         head = &kvm->arch.track_notifier_head;
 171
 172         write_lock(&kvm->mmu_lock);
 173         hlist_add_head_rcu(&n->node, &head->track_notifier_list);
 174         write_unlock(&kvm->mmu_lock);
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
 178
 179 /*
 180  * stop receiving the event interception. It is the opposed operation of
 181  * kvm_page_track_register_notifier().
 182  */
 183 void kvm_page_track_unregister_notifier(struct kvm *kvm,
 184                                         struct kvm_page_track_notifier_node *n)
 185 {
 186         struct kvm_page_track_notifier_head *head;
 187
 188         head = &kvm->arch.track_notifier_head;
 189
 190         write_lock(&kvm->mmu_lock);
 191         hlist_del_rcu(&n->node);
 192         write_unlock(&kvm->mmu_lock);
 193         synchronize_srcu(&head->track_srcu);
 194
 195         kvm_put_kvm(kvm);
 196 }
 197 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
 198
 199 /*
 200  * Notify the node that write access is intercepted and write emulation is
 201  * finished at this time.
 202  *
 203  * The node should figure out if the written page is the one that node is
 204  * interested in by itself.
 205  */
 206 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)
 207 {
 208         struct kvm_page_track_notifier_head *head;
 209         struct kvm_page_track_notifier_node *n;
 210         int idx;
 211
 212         head = &kvm->arch.track_notifier_head;
 213
 214         if (hlist_empty(&head->track_notifier_list))
 215                 return;
 216
 217         idx = srcu_read_lock(&head->track_srcu);
 218         hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
 219                                   srcu_read_lock_held(&head->track_srcu))
 220                 if (n->track_write)
 221                         n->track_write(gpa, new, bytes, n);
 222         srcu_read_unlock(&head->track_srcu, idx);
 223 }
 224
 225 /*
 226  * Notify external page track nodes that a memory region is being removed from
 227  * the VM, e.g. so that users can free any associated metadata.
 228  */
 229 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
 230 {
 231         struct kvm_page_track_notifier_head *head;
 232         struct kvm_page_track_notifier_node *n;
 233         int idx;
 234
 235         head = &kvm->arch.track_notifier_head;
 236
 237         if (hlist_empty(&head->track_notifier_list))
 238                 return;
 239
 240         idx = srcu_read_lock(&head->track_srcu);
 241         hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
 242                                   srcu_read_lock_held(&head->track_srcu))
 243                 if (n->track_remove_region)
 244                         n->track_remove_region(slot->base_gfn, slot->npages, n);
 245         srcu_read_unlock(&head->track_srcu, idx);
 246 }
 247
 248 /*
 249  * add guest page to the tracking pool so that corresponding access on that
 250  * page will be intercepted.
 251  *
 252  * @kvm: the guest instance we are interested in.
 253  * @gfn: the guest page.
 254  */
 255 int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)
 256 {
 257         struct kvm_memory_slot *slot;
 258         int idx;
 259
 260         idx = srcu_read_lock(&kvm->srcu);
 261
 262         slot = gfn_to_memslot(kvm, gfn);
 263         if (!slot) {
 264                 srcu_read_unlock(&kvm->srcu, idx);
 265                 return -EINVAL;
 266         }
 267
 268         write_lock(&kvm->mmu_lock);
 269         __kvm_write_track_add_gfn(kvm, slot, gfn);
 270         write_unlock(&kvm->mmu_lock);
 271
 272         srcu_read_unlock(&kvm->srcu, idx);
 273
 274         return 0;
 275 }
 276 EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn);
 277
 278 /*
 279  * remove the guest page from the tracking pool which stops the interception
 280  * of corresponding access on that page.
 281  *
 282  * @kvm: the guest instance we are interested in.
 283  * @gfn: the guest page.
 284  */
 285 int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)
 286 {
 287         struct kvm_memory_slot *slot;
 288         int idx;
 289
 290         idx = srcu_read_lock(&kvm->srcu);
 291
 292         slot = gfn_to_memslot(kvm, gfn);
 293         if (!slot) {
 294                 srcu_read_unlock(&kvm->srcu, idx);
 295                 return -EINVAL;
 296         }
 297
 298         write_lock(&kvm->mmu_lock);
 299         __kvm_write_track_remove_gfn(kvm, slot, gfn);
 300         write_unlock(&kvm->mmu_lock);
 301
 302         srcu_read_unlock(&kvm->srcu, idx);
 303
 304         return 0;
 305 }
 306 EXPORT_SYMBOL_GPL(kvm_write_track_remove_gfn);
 307 #endif