[Qemu-devel] [PATCH 4/5] memory: dispatch directly via MemoryRegion

[Avi Kivity]

Instead of indirecting via io_mem_region, dispatch directly
through the MemoryRegion obtained from the iotlb or phys_page_find().

Signed-off-by: Avi Kivity <address@hidden>
---
 exec-all.h         |    9 +++--
 exec.c             |  102 ++++++++++++++++++---------------------------------
 memory.c           |    8 ++--
 softmmu_template.h |   48 ++++++++++++------------
 4 files changed, 69 insertions(+), 98 deletions(-)

diff --git a/exec-all.h b/exec-all.h
index 3ffe9dd..4e8c7f5 100644
--- a/exec-all.h
+++ b/exec-all.h
@@ -299,10 +299,11 @@ extern void *tci_tb_ptr;
 
 #if !defined(CONFIG_USER_ONLY)
 
-target_phys_addr_t section_to_ioaddr(target_phys_addr_t section_io_addr);
-uint64_t io_mem_read(int index, target_phys_addr_t addr, unsigned size);
-void io_mem_write(int index, target_phys_addr_t addr, uint64_t value,
-                  unsigned size);
+struct MemoryRegion *iotlb_to_region(target_phys_addr_t index);
+uint64_t io_mem_read(struct MemoryRegion *mr, target_phys_addr_t addr,
+                     unsigned size);
+void io_mem_write(struct MemoryRegion *mr, target_phys_addr_t addr,
+                  uint64_t value, unsigned size);
 extern struct MemoryRegion *io_mem_region[IO_MEM_NB_ENTRIES];
 
 void tlb_fill(CPUState *env1, target_ulong addr, int is_write, int mmu_idx,
diff --git a/exec.c b/exec.c
index f26d1b0..6e14048 100644
--- a/exec.c
+++ b/exec.c
@@ -3399,7 +3399,7 @@ static uint64_t subpage_read(void *opaque, 
target_phys_addr_t addr,
     addr += mmio->base;
     addr -= section->offset_within_address_space;
     addr += section->offset_within_region;
-    return io_mem_read(section->mr->ram_addr, addr, len);
+    return io_mem_read(section->mr, addr, len);
 }
 
 static void subpage_write(void *opaque, target_phys_addr_t addr,
@@ -3418,7 +3418,7 @@ static void subpage_write(void *opaque, 
target_phys_addr_t addr,
     addr += mmio->base;
     addr -= section->offset_within_address_space;
     addr += section->offset_within_region;
-    io_mem_write(section->mr->ram_addr, addr, value, len);
+    io_mem_write(section->mr, addr, value, len);
 }
 
 static const MemoryRegionOps subpage_ops = {
@@ -3562,13 +3562,9 @@ static uint16_t dummy_section(MemoryRegion *mr)
     return phys_section_add(&section);
 }
 
-target_phys_addr_t section_to_ioaddr(target_phys_addr_t section_io_addr)
+MemoryRegion *iotlb_to_region(target_phys_addr_t index)
 {
-    MemoryRegionSection *section;
-
-    section = &phys_sections[section_io_addr & ~TARGET_PAGE_MASK];
-    return (section_io_addr & TARGET_PAGE_MASK)
-        | (section->mr->ram_addr & ~TARGET_PAGE_MASK);
+    return phys_sections[index & ~TARGET_PAGE_MASK].mr;
 }
 
 static void io_mem_init(void)
@@ -3838,7 +3834,7 @@ int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                             int len, int is_write)
 {
-    int l, io_index;
+    int l;
     uint8_t *ptr;
     uint32_t val;
     target_phys_addr_t page;
@@ -3854,25 +3850,23 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, 
uint8_t *buf,
         if (is_write) {
             if (!memory_region_is_ram(section->mr)) {
                 target_phys_addr_t addr1;
-                io_index = memory_region_get_ram_addr(section->mr)
-                    & (IO_MEM_NB_ENTRIES - 1);
                 addr1 = section_addr(section, addr);
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
                 if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit write access */
                     val = ldl_p(buf);
-                    io_mem_write(io_index, addr1, val, 4);
+                    io_mem_write(section->mr, addr1, val, 4);
                     l = 4;
                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
                     /* 16 bit write access */
                     val = lduw_p(buf);
-                    io_mem_write(io_index, addr1, val, 2);
+                    io_mem_write(section->mr, addr1, val, 2);
                     l = 2;
                 } else {
                     /* 8 bit write access */
                     val = ldub_p(buf);
-                    io_mem_write(io_index, addr1, val, 1);
+                    io_mem_write(section->mr, addr1, val, 1);
                     l = 1;
                 }
             } else if (!section->readonly) {
@@ -3895,22 +3889,20 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, 
uint8_t *buf,
             if (!is_ram_rom_romd(section)) {
                 target_phys_addr_t addr1;
                 /* I/O case */
-                io_index = memory_region_get_ram_addr(section->mr)
-                    & (IO_MEM_NB_ENTRIES - 1);
                 addr1 = section_addr(section, addr);
                 if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit read access */
-                    val = io_mem_read(io_index, addr1, 4);
+                    val = io_mem_read(section->mr, addr1, 4);
                     stl_p(buf, val);
                     l = 4;
                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
                     /* 16 bit read access */
-                    val = io_mem_read(io_index, addr1, 2);
+                    val = io_mem_read(section->mr, addr1, 2);
                     stw_p(buf, val);
                     l = 2;
                 } else {
                     /* 8 bit read access */
-                    val = io_mem_read(io_index, addr1, 1);
+                    val = io_mem_read(section->mr, addr1, 1);
                     stb_p(buf, val);
                     l = 1;
                 }
@@ -4106,7 +4098,6 @@ void cpu_physical_memory_unmap(void *buffer, 
target_phys_addr_t len,
 static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
                                          enum device_endian endian)
 {
-    int io_index;
     uint8_t *ptr;
     uint32_t val;
     MemoryRegionSection *section;
@@ -4115,10 +4106,8 @@ static inline uint32_t 
ldl_phys_internal(target_phys_addr_t addr,
 
     if (!is_ram_rom_romd(section)) {
         /* I/O case */
-        io_index = memory_region_get_ram_addr(section->mr)
-            & (IO_MEM_NB_ENTRIES - 1);
         addr = section_addr(section, addr);
-        val = io_mem_read(io_index, addr, 4);
+        val = io_mem_read(section->mr, addr, 4);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap32(val);
@@ -4167,7 +4156,6 @@ uint32_t ldl_be_phys(target_phys_addr_t addr)
 static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
                                          enum device_endian endian)
 {
-    int io_index;
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
@@ -4176,18 +4164,16 @@ static inline uint64_t 
ldq_phys_internal(target_phys_addr_t addr,
 
     if (!is_ram_rom_romd(section)) {
         /* I/O case */
-        io_index = memory_region_get_ram_addr(section->mr)
-            & (IO_MEM_NB_ENTRIES - 1);
         addr = section_addr(section, addr);
 
         /* XXX This is broken when device endian != cpu endian.
                Fix and add "endian" variable check */
 #ifdef TARGET_WORDS_BIGENDIAN
-        val = io_mem_read(io_index, addr, 4) << 32;
-        val |= io_mem_read(io_index, addr + 4, 4);
+        val = io_mem_read(section->mr, addr, 4) << 32;
+        val |= io_mem_read(section->mr, addr + 4, 4);
 #else
-        val = io_mem_read(io_index, addr, 4);
-        val |= io_mem_read(io_index, addr + 4, 4) << 32;
+        val = io_mem_read(section->mr, addr, 4);
+        val |= io_mem_read(section->mr, addr + 4, 4) << 32;
 #endif
     } else {
         /* RAM case */
@@ -4236,7 +4222,6 @@ uint32_t ldub_phys(target_phys_addr_t addr)
 static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
                                           enum device_endian endian)
 {
-    int io_index;
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
@@ -4245,10 +4230,8 @@ static inline uint32_t 
lduw_phys_internal(target_phys_addr_t addr,
 
     if (!is_ram_rom_romd(section)) {
         /* I/O case */
-        io_index = memory_region_get_ram_addr(section->mr)
-            & (IO_MEM_NB_ENTRIES - 1);
         addr = section_addr(section, addr);
-        val = io_mem_read(io_index, addr, 2);
+        val = io_mem_read(section->mr, addr, 2);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap16(val);
@@ -4298,20 +4281,17 @@ uint32_t lduw_be_phys(target_phys_addr_t addr)
    bits are used to track modified PTEs */
 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
 {
-    int io_index;
     uint8_t *ptr;
     MemoryRegionSection *section;
 
     section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
+        addr = section_addr(section, addr);
         if (memory_region_is_ram(section->mr)) {
-            io_index = io_mem_rom.ram_addr;
-        } else {
-            io_index = memory_region_get_ram_addr(section->mr);
+            section = &phys_sections[phys_section_rom];
         }
-        addr = section_addr(section, addr);
-        io_mem_write(io_index, addr, val, 4);
+        io_mem_write(section->mr, addr, val, 4);
     } else {
         unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
                                & TARGET_PAGE_MASK)
@@ -4333,26 +4313,22 @@ void stl_phys_notdirty(target_phys_addr_t addr, 
uint32_t val)
 
 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
 {
-    int io_index;
     uint8_t *ptr;
     MemoryRegionSection *section;
 
     section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
+        addr = section_addr(section, addr);
         if (memory_region_is_ram(section->mr)) {
-                io_index = io_mem_rom.ram_addr;
-        } else {
-            io_index = memory_region_get_ram_addr(section->mr)
-                & (IO_MEM_NB_ENTRIES - 1);
+            section = &phys_sections[phys_section_rom];
         }
-        addr = section_addr(section, addr);
 #ifdef TARGET_WORDS_BIGENDIAN
-        io_mem_write(io_index, addr, val >> 32, 4);
-        io_mem_write(io_index, addr + 4, (uint32_t)val, 4);
+        io_mem_write(section->mr, addr, val >> 32, 4);
+        io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
 #else
-        io_mem_write(io_index, addr, (uint32_t)val, 4);
-        io_mem_write(io_index, addr + 4, val >> 32, 4);
+        io_mem_write(section->mr, addr, (uint32_t)val, 4);
+        io_mem_write(section->mr, addr + 4, val >> 32, 4);
 #endif
     } else {
         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
@@ -4366,20 +4342,16 @@ void stq_phys_notdirty(target_phys_addr_t addr, 
uint64_t val)
 static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
                                      enum device_endian endian)
 {
-    int io_index;
     uint8_t *ptr;
     MemoryRegionSection *section;
 
     section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
+        addr = section_addr(section, addr);
         if (memory_region_is_ram(section->mr)) {
-            io_index = io_mem_rom.ram_addr;
-        } else {
-            io_index = memory_region_get_ram_addr(section->mr)
-                & (IO_MEM_NB_ENTRIES - 1);
+            section = &phys_sections[phys_section_rom];
         }
-        addr = section_addr(section, addr);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap32(val);
@@ -4389,7 +4361,7 @@ static inline void stl_phys_internal(target_phys_addr_t 
addr, uint32_t val,
             val = bswap32(val);
         }
 #endif
-        io_mem_write(io_index, addr, val, 4);
+        io_mem_write(section->mr, addr, val, 4);
     } else {
         unsigned long addr1;
         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
@@ -4443,20 +4415,16 @@ void stb_phys(target_phys_addr_t addr, uint32_t val)
 static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
                                      enum device_endian endian)
 {
-    int io_index;
     uint8_t *ptr;
     MemoryRegionSection *section;
 
     section = phys_page_find(addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
+        addr = section_addr(section, addr);
         if (memory_region_is_ram(section->mr)) {
-            io_index = io_mem_rom.ram_addr;
-        } else {
-            io_index = memory_region_get_ram_addr(section->mr)
-                & (IO_MEM_NB_ENTRIES - 1);
+            section = &phys_sections[phys_section_rom];
         }
-        addr = section_addr(section, addr);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap16(val);
@@ -4466,7 +4434,7 @@ static inline void stw_phys_internal(target_phys_addr_t 
addr, uint32_t val,
             val = bswap16(val);
         }
 #endif
-        io_mem_write(io_index, addr, val, 2);
+        io_mem_write(section->mr, addr, val, 2);
     } else {
         unsigned long addr1;
         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
@@ -4678,6 +4646,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, 
target_ulong addr)
 {
     int mmu_idx, page_index, pd;
     void *p;
+    MemoryRegion *mr;
 
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = cpu_mmu_index(env1);
@@ -4686,8 +4655,9 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, 
target_ulong addr)
         ldub_code(addr);
     }
     pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK;
-    if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr
-        && !io_mem_region[pd]->rom_device) {
+    mr = iotlb_to_region(pd);
+    if (mr != &io_mem_ram && mr != &io_mem_rom
+        && mr != &io_mem_notdirty && !mr->rom_device) {
 #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)
         cpu_unassigned_access(env1, addr, 0, 1, 0, 4);
 #else
diff --git a/memory.c b/memory.c
index 4c3dc49..bc76f55 100644
--- a/memory.c
+++ b/memory.c
@@ -1501,15 +1501,15 @@ void set_system_io_map(MemoryRegion *mr)
     memory_region_update_topology(NULL);
 }
 
-uint64_t io_mem_read(int io_index, target_phys_addr_t addr, unsigned size)
+uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)
 {
-    return memory_region_dispatch_read(io_mem_region[io_index], addr, size);
+    return memory_region_dispatch_read(mr, addr, size);
 }
 
-void io_mem_write(int io_index, target_phys_addr_t addr,
+void io_mem_write(MemoryRegion *mr, target_phys_addr_t addr,
                   uint64_t val, unsigned size)
 {
-    memory_region_dispatch_write(io_mem_region[io_index], addr, val, size);
+    memory_region_dispatch_write(mr, addr, val, size);
 }
 
 typedef struct MemoryRegionList MemoryRegionList;
diff --git a/softmmu_template.h b/softmmu_template.h
index 7c7e15b..e395020 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -62,27 +62,27 @@ static inline DATA_TYPE glue(io_read, 
SUFFIX)(target_phys_addr_t physaddr,
                                               void *retaddr)
 {
     DATA_TYPE res;
-    int index;
-    index = physaddr & (IO_MEM_NB_ENTRIES - 1);
+    MemoryRegion *mr = iotlb_to_region(physaddr);
+
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
     env->mem_io_pc = (unsigned long)retaddr;
-    if (index != io_mem_ram.ram_addr && index != io_mem_rom.ram_addr
-        && index != io_mem_unassigned.ram_addr
-        && index != io_mem_notdirty.ram_addr
+    if (mr != &io_mem_ram && mr != &io_mem_rom
+        && mr != &io_mem_unassigned
+        && mr != &io_mem_notdirty
             && !can_do_io(env)) {
         cpu_io_recompile(env, retaddr);
     }
 
     env->mem_io_vaddr = addr;
 #if SHIFT <= 2
-    res = io_mem_read(index, physaddr, 1 << SHIFT);
+    res = io_mem_read(mr, physaddr, 1 << SHIFT);
 #else
 #ifdef TARGET_WORDS_BIGENDIAN
-    res = io_mem_read(index, physaddr, 4) << 32;
-    res |= io_mem_read(index, physaddr + 4, 4);
+    res = io_mem_read(mr, physaddr, 4) << 32;
+    res |= io_mem_read(mr, physaddr + 4, 4);
 #else
-    res = io_mem_read(index, physaddr, 4);
-    res |= io_mem_read(index, physaddr + 4, 4) << 32;
+    res = io_mem_read(mr, physaddr, 4);
+    res |= io_mem_read(mr, physaddr + 4, 4) << 32;
 #endif
 #endif /* SHIFT > 2 */
     return res;
@@ -110,7 +110,7 @@ DATA_TYPE REGPARM glue(glue(__ld, SUFFIX), 
MMUSUFFIX)(target_ulong addr,
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             retaddr = GETPC();
-            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);
+            ioaddr = env->iotlb[mmu_idx][index];
             res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= 
TARGET_PAGE_SIZE) {
             /* slow unaligned access (it spans two pages or IO) */
@@ -164,7 +164,7 @@ static DATA_TYPE glue(glue(slow_ld, SUFFIX), 
MMUSUFFIX)(target_ulong addr,
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);
+            ioaddr = env->iotlb[mmu_idx][index];
             res = glue(io_read, SUFFIX)(ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= 
TARGET_PAGE_SIZE) {
         do_unaligned_access:
@@ -207,12 +207,12 @@ static inline void glue(io_write, 
SUFFIX)(target_phys_addr_t physaddr,
                                           target_ulong addr,
                                           void *retaddr)
 {
-    int index;
-    index = physaddr & (IO_MEM_NB_ENTRIES - 1);
+    MemoryRegion *mr = iotlb_to_region(physaddr);
+
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
-    if (index != io_mem_ram.ram_addr && index != io_mem_rom.ram_addr
-        && index != io_mem_unassigned.ram_addr
-        && index != io_mem_notdirty.ram_addr
+    if (mr != &io_mem_ram && mr != &io_mem_rom
+        && mr != &io_mem_unassigned
+        && mr != &io_mem_notdirty
             && !can_do_io(env)) {
         cpu_io_recompile(env, retaddr);
     }
@@ -220,14 +220,14 @@ static inline void glue(io_write, 
SUFFIX)(target_phys_addr_t physaddr,
     env->mem_io_vaddr = addr;
     env->mem_io_pc = (unsigned long)retaddr;
 #if SHIFT <= 2
-    io_mem_write(index, physaddr, val, 1 << SHIFT);
+    io_mem_write(mr, physaddr, val, 1 << SHIFT);
 #else
 #ifdef TARGET_WORDS_BIGENDIAN
-    io_mem_write(index, physaddr, (val >> 32), 4);
-    io_mem_write(index, physaddr + 4, (uint32_t)val, 4);
+    io_mem_write(mr, physaddr, (val >> 32), 4);
+    io_mem_write(mr, physaddr + 4, (uint32_t)val, 4);
 #else
-    io_mem_write(index, physaddr, (uint32_t)val, 4);
-    io_mem_write(index, physaddr + 4, val >> 32, 4);
+    io_mem_write(mr, physaddr, (uint32_t)val, 4);
+    io_mem_write(mr, physaddr + 4, val >> 32, 4);
 #endif
 #endif /* SHIFT > 2 */
 }
@@ -251,7 +251,7 @@ void REGPARM glue(glue(__st, SUFFIX), 
MMUSUFFIX)(target_ulong addr,
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             retaddr = GETPC();
-            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);
+            ioaddr = env->iotlb[mmu_idx][index];
             glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= 
TARGET_PAGE_SIZE) {
         do_unaligned_access:
@@ -303,7 +303,7 @@ static void glue(glue(slow_st, SUFFIX), 
MMUSUFFIX)(target_ulong addr,
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            ioaddr = section_to_ioaddr(env->iotlb[mmu_idx][index]);
+            ioaddr = env->iotlb[mmu_idx][index];
             glue(io_write, SUFFIX)(ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= 
TARGET_PAGE_SIZE) {
         do_unaligned_access:
-- 
1.7.9