Re: [Qemu-devel] [PATCH] memory: do not do out of bound notification

[Yan Zhao]

On Thu, Jun 20, 2019 at 09:04:43PM +0800, Peter Xu wrote:
> On Thu, Jun 20, 2019 at 08:59:55PM +0800, Peter Xu wrote:
> > On Thu, Jun 20, 2019 at 10:35:29AM +0200, Paolo Bonzini wrote:
> > > On 20/06/19 06:02, Peter Xu wrote:
> > > > Seems workable, to be explicit - we can even cut it into chunks with
> > > > different size to be efficient.
> > > 
> > > Yes, this is not hard (completely untested):
> > > 
> > > diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> > > index 44b1231157..541538bc6c 100644
> > > --- a/hw/i386/intel_iommu.c
> > > +++ b/hw/i386/intel_iommu.c
> > > @@ -3388,39 +3388,34 @@ static void 
> > > vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
> > >      }
> > >  
> > >      assert(start <= end);
> > > -    size = end - start;
> > > +    while (end > start) {
> > > +        size = end - start;
> > > +        /* Only keep the lowest bit of either size or start.  */
> > > +        size = MIN(size & -size, start & -start);
> > 
> > I feel like this can be problematic.  I'm imaging:
> > 
> > start=0x1000_0000, size=0x1000_1000
> > 
> > This will get size=0x1000 but actually we can do size=0x1000_0000 as
> > the first.
> > 
> > > +        /* Should not happen, but limit to address width too just in 
> > > case */
> > > +        size = MIN(size, 1ULL << s->aw_bits);
> > >  
> > > -    if (ctpop64(size) != 1) {
> > > -        /*
> > > -         * This size cannot format a correct mask. Let's enlarge it to
> > > -         * suite the minimum available mask.
> > > -         */
> > > -        int n = 64 - clz64(size);
> > > -        if (n > s->aw_bits) {
> > > -            /* should not happen, but in case it happens, limit it */
> > > -            n = s->aw_bits;
> > > -        }
> > > -        size = 1ULL << n;
> > > -    }
> > > +        assert((start & (size - 1)) == 0);
> > >  
> > > -    entry.target_as = &address_space_memory;
> > > -    /* Adjust iova for the size */
> > > -    entry.iova = n->start & ~(size - 1);
> > > -    /* This field is meaningless for unmap */
> > > -    entry.translated_addr = 0;
> > > -    entry.perm = IOMMU_NONE;
> > > -    entry.addr_mask = size - 1;
> > > +        entry.target_as = &address_space_memory;
> > > +        entry.iova = start;
> > > +        /* This field is meaningless for unmap */
> > > +        entry.translated_addr = 0;
> > > +        entry.perm = IOMMU_NONE;
> > > +        entry.addr_mask = size - 1;
> > 
> > (some of the fields can be moved out of loop because they are
> >  constants)
> > 
> > >  
> > > -    trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> > > -                             VTD_PCI_SLOT(as->devfn),
> > > -                             VTD_PCI_FUNC(as->devfn),
> > > -                             entry.iova, size);
> > > +        trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> > > +                                 VTD_PCI_SLOT(as->devfn),
> > > +                                 VTD_PCI_FUNC(as->devfn),
> > > +                                 entry.iova, size);
> > 
> > Can move this out because this is a trace only so we don't have
> > restriction on mask?
> > 
> > >  
> > > -    map.iova = entry.iova;
> > > -    map.size = entry.addr_mask;
> > > -    iova_tree_remove(as->iova_tree, &map);
> > > +        map.iova = entry.iova;
> > > +        map.size = entry.addr_mask;
> > > +        iova_tree_remove(as->iova_tree, &map);
> > 
> > Same here?
> > 
> > >  
> > > -    memory_region_notify_one(n, &entry);
> > > +        memory_region_notify_one(n, &entry);
> > > +        start += size;
> > > +    }
> > >  }
> > >  
> > >  static void vtd_address_space_unmap_all(IntelIOMMUState *s)
> > > 
> > > 
> > > Yan,
> > > 
> > > if something like this works for you, let me know and I will submit it
> > > as a proper patch.
> > > 
> > > Paolo
> > 
> > Since during review I'm thinking how to generate a correct sequence of
> > these masks... here's my try below with above issues fixed... :)
> > 
> > I've tried compile but not tested.  Yan can test it, or I can do it
> > too tomorrow after I find some machines.
> > 
> > Thanks,
> > 
> > ------------------------------------------------------------
> > diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> > index 44b1231157..cfbd225f0a 100644
> > --- a/hw/i386/intel_iommu.c
> > +++ b/hw/i386/intel_iommu.c
> > @@ -3363,11 +3363,32 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState 
> > *s, PCIBus *bus, int devfn)
> >      return vtd_dev_as;
> >  }
> > 
> > +static uint64_t vtd_get_next_mask(uint64_t start, uint64_t size, int gaw)
> > +{
> > +    /* Tries to find smallest mask from start first */
> > +    uint64_t rmask = start & -start, max_mask = 1ULL << gaw;
> > +
> > +    assert(size && gaw > 0 && gaw < 64);
> > +
> > +    /* Zero start, or too big */
> > +    if (!rmask || rmask > max_mask) {
> > +        rmask = max_mask;
> > +    }
> > +
> > +    /* If the start mask worked, then use it */
> > +    if (rmask <= size) {
> > +        return rmask;
> > +    }
> > +
> > +    /* Find the largest page mask from size */
> > +    return 1ULL << (63 - clz64(size));
> > +}
> > +
> >  /* Unmap the whole range in the notifier's scope. */
> >  static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
> >  {
> >      IOMMUTLBEntry entry;
> > -    hwaddr size;
> > +    hwaddr size, remain;
> >      hwaddr start = n->start;
> >      hwaddr end = n->end;
> >      IntelIOMMUState *s = as->iommu_state;
> > @@ -3388,39 +3409,28 @@ static void vtd_address_space_unmap(VTDAddressSpace 
> > *as, IOMMUNotifier *n)
> >      }
> > 
> >      assert(start <= end);
> > -    size = end - start;
> > -
> > -    if (ctpop64(size) != 1) {
> > -        /*
> > -         * This size cannot format a correct mask. Let's enlarge it to
> > -         * suite the minimum available mask.
> > -         */
> > -        int n = 64 - clz64(size);
> > -        if (n > s->aw_bits) {
> > -            /* should not happen, but in case it happens, limit it */
> > -            n = s->aw_bits;
> > -        }
> > -        size = 1ULL << n;
> > -    }
> > -
> > +    size = remain = end - start;
> >      entry.target_as = &address_space_memory;
> > -    /* Adjust iova for the size */
> > -    entry.iova = n->start & ~(size - 1);
> > +    entry.perm = IOMMU_NONE;
> >      /* This field is meaningless for unmap */
> >      entry.translated_addr = 0;
> > -    entry.perm = IOMMU_NONE;
> > -    entry.addr_mask = size - 1;
> > +
> > +    while (remain) {
> > +        uint64_t mask = vtd_get_next_mask(start, remain, s->aw_bits);
> > +
> > +        entry.iova = start;
> > +        entry.addr_mask = mask - 1;
> > +        memory_region_notify_one(n, &entry);
> 
> Sorry, I at least missed these lines:
> 
>            start += mask;
>            remain -= mask;
> 
> > +    }
> > 
> >      trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> >                               VTD_PCI_SLOT(as->devfn),
> >                               VTD_PCI_FUNC(as->devfn),
> > -                             entry.iova, size);
> > +                             n->start, size);
> > 
> > -    map.iova = entry.iova;
> > -    map.size = entry.addr_mask;
> > +    map.iova = n->start;
> > +    map.size = size;
> >      iova_tree_remove(as->iova_tree, &map);
> > -
> > -    memory_region_notify_one(n, &entry);
> >  }
> > 
> >  static void vtd_address_space_unmap_all(IntelIOMMUState *s)
> > ------------------------------------------------------------
> > 
> > Regards,
> > 
> > -- 
> > Peter Xu
> 
> Regards,
> 
> -- 
> Peter Xu

hi Peter and Paolo,
I tested with code and it's fine in my side.
It's base on your version with some minor modifications, such as size is
now  (end - start + 1) now.
Thanks
Yan

+static uint64_t vtd_get_next_mask(uint64_t start, uint64_t size, int gaw)
+{
+    /* Tries to find smallest mask from start first */
+    uint64_t rmask = start & -start, max_mask = 1ULL << gaw;
+    assert(size && gaw > 0 && gaw < 64);
+    /* Zero start, or too big */
+    if (!rmask || rmask > max_mask) {
+        rmask = max_mask;
+    }
+    /* If the start mask worked, then use it */
+    if (rmask <= size) {
+        return rmask;
+    }
+
+    /* Find the largest page mask from size */
+    return 1ULL << (63 - clz64(size));
+}
+
 /* Unmap the whole range in the notifier's scope. */
 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
 {
     IOMMUTLBEntry entry;
-    hwaddr size;
+    hwaddr size, remain;
     hwaddr start = n->start;
     hwaddr end = n->end;
     IntelIOMMUState *s = as->iommu_state;
@@ -3380,48 +3398,46 @@ static void vtd_address_space_unmap(VTDAddressSpace 
*as, IOMMUNotifier *n)
      * VT-d spec), otherwise we need to consider overflow of 64 bits.
      */

-    if (end > VTD_ADDRESS_SIZE(s->aw_bits)) {
+    if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) {
         /*
          * Don't need to unmap regions that is bigger than the whole
          * VT-d supported address space size
          */
-        end = VTD_ADDRESS_SIZE(s->aw_bits);
+        end = VTD_ADDRESS_SIZE(s->aw_bits) - 1;
     }

     assert(start <= end);
-    size = end - start;

-    if (ctpop64(size) != 1) {
-        /*
-         * This size cannot format a correct mask. Let's enlarge it to
-         * suite the minimum available mask.
-         */
-        int n = 64 - clz64(size);
-        if (n > s->aw_bits) {
-            /* should not happen, but in case it happens, limit it */
-            n = s->aw_bits;
-        }
-        size = 1ULL << n;
-    }
+    size = remain = end - start + 1;

     entry.target_as = &address_space_memory;
-    /* Adjust iova for the size */
-    entry.iova = n->start & ~(size - 1);
+
+    entry.perm = IOMMU_NONE;
     /* This field is meaningless for unmap */
     entry.translated_addr = 0;
-    entry.perm = IOMMU_NONE;
-    entry.addr_mask = size - 1;
+
+    while (remain >= VTD_PAGE_SIZE) {
+        uint64_t mask = vtd_get_next_mask(start, remain, s->aw_bits);
+
+        entry.iova = start;
+        entry.addr_mask = mask - 1;
+        memory_region_notify_one(n, &entry);
+        start += mask;
+        remain -= mask;
+    }
+
+    if (remain) {
+        warn_report("Unmapping unaligned range %lx-%lx", start, end);
+    }

     trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
                              VTD_PCI_SLOT(as->devfn),
                              VTD_PCI_FUNC(as->devfn),
-                             entry.iova, size);
-
-    map.iova = entry.iova;
-    map.size = entry.addr_mask;
+                             n->start, size);
+    map.iova = n->start;
+    map.size = size;
     iova_tree_remove(as->iova_tree, &map);

-    memory_region_notify_one(n, &entry);
 }

 static void vtd_address_space_unmap_all(IntelIOMMUState *s)
--
2.7.4