Changes in kernel/arch/mips32/src/mm/tlb.c [8fe2c9bd:1dbc43f] in mainline

File:

• kernel/arch/mips32/src/mm/tlb.c (modified) (19 diffs)

kernel/arch/mips32/src/mm/tlb.c

@@ -48 +48 @@
 #include <symtab.h>
 
-#define PFN_SHIFT       12
-#define VPN_SHIFT       12
-#define ADDR2VPN(a)     ((a) >> VPN_SHIFT)
-#define ADDR2VPN2(a)    (ADDR2VPN((a)) >> 1)
-#define VPN2ADDR(vpn)   ((vpn) << VPN_SHIFT)
-#define VPN22ADDR(vpn2) (VPN2ADDR(vpn2) << 1)
-#define PFN2ADDR(pfn)   ((pfn) << PFN_SHIFT)
-
-#define BANK_SELECT_BIT(a)      (((a) >> PAGE_WIDTH) & 1) 
-        
+static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *);
 
 /** Initialize TLB.
@@ -93 +84 @@
 {
         entry_lo_t lo;
+        entry_hi_t hi;
+        asid_t asid;
         uintptr_t badvaddr;
-        uintptr_t page;
         pte_t *pte;
         
         badvaddr = cp0_badvaddr_read();
-        page = ALIGN_DOWN(badvaddr, PAGE_SIZE);
-
-        pte = page_mapping_find(AS, page, true);
-        if (pte && pte->p) {
+        asid = AS->asid;
+        
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate);
+        if (pte) {
                 /*
                  * Record access to PTE.
@@ -107 +99 @@
                 pte->a = 1;
 
+                tlb_prepare_entry_hi(&hi, asid, badvaddr);
                 tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d,
                     pte->cacheable, pte->pfn);
@@ -113 +106 @@
                  * New entry is to be inserted into TLB
                  */
-                if (BANK_SELECT_BIT(badvaddr) == 0) {
+                cp0_entry_hi_write(hi.value);
+                if ((badvaddr / PAGE_SIZE) % 2 == 0) {
                         cp0_entry_lo0_write(lo.value);
                         cp0_entry_lo1_write(0);
@@ -122 +116 @@
                 cp0_pagemask_write(TLB_PAGE_MASK_16K);
                 tlbwr();
-                return;
-        }
-
-        (void) as_page_fault(page, PF_ACCESS_READ, istate);
+        }
 }
 
@@ -134 +125 @@
 void tlb_invalid(istate_t *istate)
 {
-        entry_lo_t lo;
         tlb_index_t index;
         uintptr_t badvaddr;
-        uintptr_t page;
+        entry_lo_t lo;
+        entry_hi_t hi;
         pte_t *pte;
+
+        badvaddr = cp0_badvaddr_read();
 
         /*
          * Locate the faulting entry in TLB.
          */
+        hi.value = cp0_entry_hi_read();
+        tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
+        cp0_entry_hi_write(hi.value);
         tlbp();
         index.value = cp0_index_read();
 
-#if defined(PROCESSOR_4Kc)
-        /*
-         * This can happen on a 4Kc when Status.EXL is 1 and there is a TLB miss.
-         * EXL is 1 when interrupts are disabled. The combination of a TLB miss
-         * and disabled interrupts is possible in copy_to/from_uspace().
-         */
-        if (index.p) {
-                tlb_refill(istate);
-                return;
-        }
-#endif
-
         ASSERT(!index.p);
 
-        badvaddr = cp0_badvaddr_read();
-        page = ALIGN_DOWN(badvaddr, PAGE_SIZE);
-
-        pte = page_mapping_find(AS, page, true);
-        if (pte && pte->p) {
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate);
+        if (pte) {
                 /*
                  * Read the faulting TLB entry.
@@ -181 +162 @@
                  * The entry is to be updated in TLB.
                  */
-                if (BANK_SELECT_BIT(badvaddr) == 0)
+                if ((badvaddr / PAGE_SIZE) % 2 == 0)
                         cp0_entry_lo0_write(lo.value);
                 else
                         cp0_entry_lo1_write(lo.value);
+                cp0_pagemask_write(TLB_PAGE_MASK_16K);
                 tlbwi();
-                return;
-        }
-
-        (void) as_page_fault(page, PF_ACCESS_READ, istate);
+        }
 }
 
@@ -198 +177 @@
 void tlb_modified(istate_t *istate)
 {
-        entry_lo_t lo;
         tlb_index_t index;
         uintptr_t badvaddr;
-        uintptr_t page;
+        entry_lo_t lo;
+        entry_hi_t hi;
         pte_t *pte;
 
         badvaddr = cp0_badvaddr_read();
-        page = ALIGN_DOWN(badvaddr, PAGE_SIZE);
 
         /*
          * Locate the faulting entry in TLB.
          */
+        hi.value = cp0_entry_hi_read();
+        tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
+        cp0_entry_hi_write(hi.value);
         tlbp();
         index.value = cp0_index_read();
 
         /*
-         * Emit warning if the entry is not in TLB.
-         *
-         * We do not assert on this because this could be a manifestation of
-         * an emulator bug, such as QEMU Bug #1128935:
-         * https://bugs.launchpad.net/qemu/+bug/1128935  
+         * Fail if the entry is not in TLB.
          */
-        if (index.p) {
-                printf("%s: TLBP failed in exception handler (badvaddr=%#"
-                    PRIxn ", ASID=%d).\n", __func__, badvaddr,
-                    AS ? AS->asid : -1);
-                return;
-        }
-
-        pte = page_mapping_find(AS, page, true);
-        if (pte && pte->p && pte->w) {
+        ASSERT(!index.p);
+
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate);
+        if (pte) {
                 /*
                  * Read the faulting TLB entry.
@@ -246 +218 @@
                  * The entry is to be updated in TLB.
                  */
-                if (BANK_SELECT_BIT(badvaddr) == 0)
+                if ((badvaddr / PAGE_SIZE) % 2 == 0)
                         cp0_entry_lo0_write(lo.value);
                 else
                         cp0_entry_lo1_write(lo.value);
+                cp0_pagemask_write(TLB_PAGE_MASK_16K);
                 tlbwi();
-                return;
-        }
-
-        (void) as_page_fault(page, PF_ACCESS_WRITE, istate);
+        }
+}
+
+/** Try to find PTE for faulting address.
+ *
+ * @param badvaddr      Faulting virtual address.
+ * @param access        Access mode that caused the fault.
+ * @param istate        Pointer to interrupted state.
+ *
+ * @return              PTE on success, NULL otherwise.
+ */
+pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate)
+{
+        entry_hi_t hi;
+        pte_t *pte;
+
+        hi.value = cp0_entry_hi_read();
+
+        ASSERT(hi.asid == AS->asid);
+
+        /*
+         * Check if the mapping exists in page tables.
+         */     
+        pte = page_mapping_find(AS, badvaddr, true);
+        if (pte && pte->p && (pte->w || access != PF_ACCESS_WRITE)) {
+                /*
+                 * Mapping found in page tables.
+                 * Immediately succeed.
+                 */
+                return pte;
+        }
+
+        /*
+         * Mapping not found in page tables.
+         * Resort to higher-level page fault handler.
+         */
+        if (as_page_fault(badvaddr, access, istate) == AS_PF_OK) {
+                pte = page_mapping_find(AS, badvaddr, true);
+                ASSERT(pte && pte->p);
+                ASSERT(pte->w || access != PF_ACCESS_WRITE);
+                return pte;
+        }
+
+        return NULL;
 }
 
@@ -271 +284 @@
 void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
 {
-        hi->value = 0;
-        hi->vpn2 = ADDR2VPN2(ALIGN_DOWN(addr, PAGE_SIZE));
+        hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
         hi->asid = asid;
 }
@@ -279 +291 @@
 void tlb_print(void)
 {
-        page_mask_t mask, mask_save;
-        entry_lo_t lo0, lo0_save, lo1, lo1_save;
+        page_mask_t mask;
+        entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         unsigned int i;
 
         hi_save.value = cp0_entry_hi_read();
-        lo0_save.value = cp0_entry_lo0_read();
-        lo1_save.value = cp0_entry_lo1_read();
-        mask_save.value = cp0_pagemask_read();
-        
-        printf("[nr] [asid] [vpn2    ] [mask] [gvdc] [pfn     ]\n");
+        
+        printf("[nr] [asid] [vpn2] [mask] [gvdc] [pfn ]\n");
         
         for (i = 0; i < TLB_ENTRY_COUNT; i++) {
@@ -300 +309 @@
                 lo1.value = cp0_entry_lo1_read();
                 
-                printf("%-4u %-6u %0#10x %-#6x  %1u%1u%1u%1u  %0#10x\n",
-                    i, hi.asid, VPN22ADDR(hi.vpn2), mask.mask,
-                    lo0.g, lo0.v, lo0.d, lo0.c, PFN2ADDR(lo0.pfn));
-                printf("                               %1u%1u%1u%1u  %0#10x\n",
-                    lo1.g, lo1.v, lo1.d, lo1.c, PFN2ADDR(lo1.pfn));
+                printf("%-4u %-6u %#6x %#6x  %1u%1u%1u%1u  %#6x\n",
+                    i, hi.asid, hi.vpn2, mask.mask,
+                    lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
+                printf("                           %1u%1u%1u%1u  %#6x\n",
+                    lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
         }
         
         cp0_entry_hi_write(hi_save.value);
-        cp0_entry_lo0_write(lo0_save.value);
-        cp0_entry_lo1_write(lo1_save.value);
-        cp0_pagemask_write(mask_save.value);
 }
 
@@ -316 +322 @@
 void tlb_invalidate_all(void)
 {
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi_save;
         int i;
 
-        ASSERT(interrupts_disabled());
-
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
 
         for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
@@ -340 +346 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }
@@ -349 +356 @@
 void tlb_invalidate_asid(asid_t asid)
 {
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         int i;
 
-        ASSERT(interrupts_disabled());
         ASSERT(asid != ASID_INVALID);
 
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
         
         for (i = 0; i < TLB_ENTRY_COUNT; i++) {
@@ -378 +386 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }
@@ -391 +400 @@
 {
         unsigned int i;
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         tlb_index_t index;
-
-        ASSERT(interrupts_disabled());
         
         if (asid == ASID_INVALID)
@@ -401 +409 @@
 
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
 
         for (i = 0; i < cnt + 1; i += 2) {
+                hi.value = 0;
                 tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
                 cp0_entry_hi_write(hi.value);
@@ -429 +439 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }