Changes in kernel/arch/mips32/src/mm/tlb.c [59fb782:aaa6af2] in mainline

File:

• kernel/arch/mips32/src/mm/tlb.c (modified) (15 diffs)

kernel/arch/mips32/src/mm/tlb.c

@@ -48 +48 @@
 #include <symtab.h>
 
-#define PFN_SHIFT       12
-#define VPN_SHIFT       12
-#define ADDR2VPN(a)     ((a) >> VPN_SHIFT)
-#define ADDR2VPN2(a)    (ADDR2VPN((a)) >> 1)
-#define VPN2ADDR(vpn)   ((vpn) << VPN_SHIFT)
-#define VPN22ADDR(vpn2) (VPN2ADDR(vpn2) << 1)
-#define PFN2ADDR(pfn)   ((pfn) << PFN_SHIFT)
-
-#define BANK_SELECT_BIT(a)      (((a) >> PAGE_WIDTH) & 1) 
-        
+static void tlb_refill_fail(istate_t *);
+static void tlb_invalid_fail(istate_t *);
+static void tlb_modified_fail(istate_t *);
+
+static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *, int *);
 
 /** Initialize TLB.
@@ -93 +88 @@
 {
         entry_lo_t lo;
+        entry_hi_t hi;
+        asid_t asid;
         uintptr_t badvaddr;
         pte_t *pte;
+        int pfrc;
         
         badvaddr = cp0_badvaddr_read();
-
-        pte = page_mapping_find(AS, badvaddr, true);
-        if (pte && pte->p) {
-                /*
-                 * Record access to PTE.
-                 */
-                pte->a = 1;
-
-                tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d,
-                    pte->cacheable, pte->pfn);
-
-                /*
-                 * New entry is to be inserted into TLB
-                 */
-                if (BANK_SELECT_BIT(badvaddr) == 0) {
-                        cp0_entry_lo0_write(lo.value);
-                        cp0_entry_lo1_write(0);
-                } else {
-                        cp0_entry_lo0_write(0);
-                        cp0_entry_lo1_write(lo.value);
+        asid = AS->asid;
+        
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
+        if (!pte) {
+                switch (pfrc) {
+                case AS_PF_FAULT:
+                        goto fail;
+                        break;
+                case AS_PF_DEFER:
+                        /*
+                         * The page fault came during copy_from_uspace()
+                         * or copy_to_uspace().
+                         */
+                        return;
+                default:
+                        panic("Unexpected pfrc (%d).", pfrc);
                 }
-                cp0_pagemask_write(TLB_PAGE_MASK_16K);
-                tlbwr();
-                return;
-        }
-
-        (void) as_page_fault(badvaddr, PF_ACCESS_READ, istate);
+        }
+
+        /*
+         * Record access to PTE.
+         */
+        pte->a = 1;
+
+        tlb_prepare_entry_hi(&hi, asid, badvaddr);
+        tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
+            pte->pfn);
+
+        /*
+         * New entry is to be inserted into TLB
+         */
+        cp0_entry_hi_write(hi.value);
+        if ((badvaddr / PAGE_SIZE) % 2 == 0) {
+                cp0_entry_lo0_write(lo.value);
+                cp0_entry_lo1_write(0);
+        }
+        else {
+                cp0_entry_lo0_write(0);
+                cp0_entry_lo1_write(lo.value);
+        }
+        cp0_pagemask_write(TLB_PAGE_MASK_16K);
+        tlbwr();
+
+        return;
+        
+fail:
+        tlb_refill_fail(istate);
 }
 
@@ -132 +150 @@
 void tlb_invalid(istate_t *istate)
 {
-        entry_lo_t lo;
         tlb_index_t index;
         uintptr_t badvaddr;
+        entry_lo_t lo;
+        entry_hi_t hi;
         pte_t *pte;
+        int pfrc;
+
+        badvaddr = cp0_badvaddr_read();
 
         /*
          * Locate the faulting entry in TLB.
          */
+        hi.value = cp0_entry_hi_read();
+        tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
+        cp0_entry_hi_write(hi.value);
         tlbp();
         index.value = cp0_index_read();
 
-#if defined(PROCESSOR_4Kc)
-        /*
-         * This can happen on a 4Kc when Status.EXL is 1 and there is a TLB miss.
-         * EXL is 1 when interrupts are disabled. The combination of a TLB miss
-         * and disabled interrupts is possible in copy_to/from_uspace().
+        /*
+         * Fail if the entry is not in TLB.
          */
         if (index.p) {
-                tlb_refill(istate);
-                return;
-        }
-#endif
-
-        ASSERT(!index.p);
-
-        badvaddr = cp0_badvaddr_read();
-
-        pte = page_mapping_find(AS, badvaddr, true);
-        if (pte && pte->p) {
-                /*
-                 * Read the faulting TLB entry.
-                 */
-                tlbr();
-
-                /*
-                 * Record access to PTE.
-                 */
-                pte->a = 1;
-
-                tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d,
-                    pte->cacheable, pte->pfn);
-
-                /*
-                 * The entry is to be updated in TLB.
-                 */
-                if (BANK_SELECT_BIT(badvaddr) == 0)
-                        cp0_entry_lo0_write(lo.value);
-                else
-                        cp0_entry_lo1_write(lo.value);
-                tlbwi();
-                return;
-        }
-
-        (void) as_page_fault(badvaddr, PF_ACCESS_READ, istate);
+                printf("TLB entry not found.\n");
+                goto fail;
+        }
+
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
+        if (!pte) {
+                switch (pfrc) {
+                case AS_PF_FAULT:
+                        goto fail;
+                        break;
+                case AS_PF_DEFER:
+                        /*
+                         * The page fault came during copy_from_uspace()
+                         * or copy_to_uspace().
+                         */
+                        return;
+                default:
+                        panic("Unexpected pfrc (%d).", pfrc);
+                }
+        }
+
+        /*
+         * Read the faulting TLB entry.
+         */
+        tlbr();
+
+        /*
+         * Record access to PTE.
+         */
+        pte->a = 1;
+
+        tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
+            pte->pfn);
+
+        /*
+         * The entry is to be updated in TLB.
+         */
+        if ((badvaddr / PAGE_SIZE) % 2 == 0)
+                cp0_entry_lo0_write(lo.value);
+        else
+                cp0_entry_lo1_write(lo.value);
+        cp0_pagemask_write(TLB_PAGE_MASK_16K);
+        tlbwi();
+
+        return;
+        
+fail:
+        tlb_invalid_fail(istate);
 }
 
@@ -194 +228 @@
 void tlb_modified(istate_t *istate)
 {
-        entry_lo_t lo;
         tlb_index_t index;
         uintptr_t badvaddr;
+        entry_lo_t lo;
+        entry_hi_t hi;
         pte_t *pte;
+        int pfrc;
 
         badvaddr = cp0_badvaddr_read();
@@ -204 +240 @@
          * Locate the faulting entry in TLB.
          */
+        hi.value = cp0_entry_hi_read();
+        tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
+        cp0_entry_hi_write(hi.value);
         tlbp();
         index.value = cp0_index_read();
 
         /*
-         * Emit warning if the entry is not in TLB.
-         *
-         * We do not assert on this because this could be a manifestation of
-         * an emulator bug, such as QEMU Bug #1128935:
-         * https://bugs.launchpad.net/qemu/+bug/1128935  
+         * Fail if the entry is not in TLB.
          */
         if (index.p) {
-                printf("%s: TLBP failed in exception handler (badvaddr=%#"
-                    PRIxn ", ASID=%d).\n", __func__, badvaddr,
-                    AS ? AS->asid : -1);
-                return;
-        }
-
+                printf("TLB entry not found.\n");
+                goto fail;
+        }
+
+        pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
+        if (!pte) {
+                switch (pfrc) {
+                case AS_PF_FAULT:
+                        goto fail;
+                        break;
+                case AS_PF_DEFER:
+                        /*
+                         * The page fault came during copy_from_uspace()
+                         * or copy_to_uspace().
+                         */
+                        return;
+                default:
+                        panic("Unexpected pfrc (%d).", pfrc);
+                }
+        }
+
+        /*
+         * Read the faulting TLB entry.
+         */
+        tlbr();
+
+        /*
+         * Record access and write to PTE.
+         */
+        pte->a = 1;
+        pte->d = 1;
+
+        tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
+            pte->pfn);
+
+        /*
+         * The entry is to be updated in TLB.
+         */
+        if ((badvaddr / PAGE_SIZE) % 2 == 0)
+                cp0_entry_lo0_write(lo.value);
+        else
+                cp0_entry_lo1_write(lo.value);
+        cp0_pagemask_write(TLB_PAGE_MASK_16K);
+        tlbwi();
+
+        return;
+        
+fail:
+        tlb_modified_fail(istate);
+}
+
+void tlb_refill_fail(istate_t *istate)
+{
+        uintptr_t va = cp0_badvaddr_read();
+        
+        fault_if_from_uspace(istate, "TLB Refill Exception on %p.",
+            (void *) va);
+        panic_memtrap(istate, PF_ACCESS_UNKNOWN, va, "TLB Refill Exception.");
+}
+
+
+void tlb_invalid_fail(istate_t *istate)
+{
+        uintptr_t va = cp0_badvaddr_read();
+        
+        fault_if_from_uspace(istate, "TLB Invalid Exception on %p.",
+            (void *) va);
+        panic_memtrap(istate, PF_ACCESS_UNKNOWN, va, "TLB Invalid Exception.");
+}
+
+void tlb_modified_fail(istate_t *istate)
+{
+        uintptr_t va = cp0_badvaddr_read();
+        
+        fault_if_from_uspace(istate, "TLB Modified Exception on %p.",
+            (void *) va);
+        panic_memtrap(istate, PF_ACCESS_WRITE, va, "TLB Modified Exception.");
+}
+
+/** Try to find PTE for faulting address.
+ *
+ * @param badvaddr      Faulting virtual address.
+ * @param access        Access mode that caused the fault.
+ * @param istate        Pointer to interrupted state.
+ * @param pfrc          Pointer to variable where as_page_fault() return code
+ *                      will be stored.
+ *
+ * @return              PTE on success, NULL otherwise.
+ */
+pte_t *
+find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
+    int *pfrc)
+{
+        entry_hi_t hi;
+        pte_t *pte;
+
+        hi.value = cp0_entry_hi_read();
+
+        /*
+         * Handler cannot succeed if the ASIDs don't match.
+         */
+        if (hi.asid != AS->asid) {
+                printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
+                return NULL;
+        }
+
+        /*
+         * Check if the mapping exists in page tables.
+         */     
         pte = page_mapping_find(AS, badvaddr, true);
-        if (pte && pte->p && pte->w) {
+        if (pte && pte->p && (pte->w || access != PF_ACCESS_WRITE)) {
                 /*
-                 * Read the faulting TLB entry.
+                 * Mapping found in page tables.
+                 * Immediately succeed.
                  */
-                tlbr();
-
+                return pte;
+        } else {
+                int rc;
+                
                 /*
-                 * Record access and write to PTE.
+                 * Mapping not found in page tables.
+                 * Resort to higher-level page fault handler.
                  */
-                pte->a = 1;
-                pte->d = 1;
-
-                tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w,
-                    pte->cacheable, pte->pfn);
-
-                /*
-                 * The entry is to be updated in TLB.
-                 */
-                if (BANK_SELECT_BIT(badvaddr) == 0)
-                        cp0_entry_lo0_write(lo.value);
-                else
-                        cp0_entry_lo1_write(lo.value);
-                tlbwi();
-                return;
-        }
-
-        (void) as_page_fault(badvaddr, PF_ACCESS_WRITE, istate);
+                switch (rc = as_page_fault(badvaddr, access, istate)) {
+                case AS_PF_OK:
+                        /*
+                         * The higher-level page fault handler succeeded,
+                         * The mapping ought to be in place.
+                         */
+                        pte = page_mapping_find(AS, badvaddr, true);
+                        ASSERT(pte && pte->p);
+                        ASSERT(pte->w || access != PF_ACCESS_WRITE);
+                        return pte;
+                case AS_PF_DEFER:
+                        *pfrc = AS_PF_DEFER;
+                        return NULL;
+                case AS_PF_FAULT:
+                        *pfrc = AS_PF_FAULT;
+                        return NULL;
+                default:
+                        panic("Unexpected rc (%d).", rc);
+                }
+                
+        }
 }
 
@@ -265 +410 @@
 void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
 {
-        hi->value = 0;
-        hi->vpn2 = ADDR2VPN2(ALIGN_DOWN(addr, PAGE_SIZE));
+        hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
         hi->asid = asid;
 }
@@ -273 +417 @@
 void tlb_print(void)
 {
-        page_mask_t mask, mask_save;
-        entry_lo_t lo0, lo0_save, lo1, lo1_save;
+        page_mask_t mask;
+        entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         unsigned int i;
 
         hi_save.value = cp0_entry_hi_read();
-        lo0_save.value = cp0_entry_lo0_read();
-        lo1_save.value = cp0_entry_lo1_read();
-        mask_save.value = cp0_pagemask_read();
-        
-        printf("[nr] [asid] [vpn2    ] [mask] [gvdc] [pfn     ]\n");
+        
+        printf("[nr] [asid] [vpn2] [mask] [gvdc] [pfn ]\n");
         
         for (i = 0; i < TLB_ENTRY_COUNT; i++) {
@@ -294 +435 @@
                 lo1.value = cp0_entry_lo1_read();
                 
-                printf("%-4u %-6u %0#10x %-#6x  %1u%1u%1u%1u  %0#10x\n",
-                    i, hi.asid, VPN22ADDR(hi.vpn2), mask.mask,
-                    lo0.g, lo0.v, lo0.d, lo0.c, PFN2ADDR(lo0.pfn));
-                printf("                               %1u%1u%1u%1u  %0#10x\n",
-                    lo1.g, lo1.v, lo1.d, lo1.c, PFN2ADDR(lo1.pfn));
+                printf("%-4u %-6u %#6x %#6x  %1u%1u%1u%1u  %#6x\n",
+                    i, hi.asid, hi.vpn2, mask.mask,
+                    lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
+                printf("                           %1u%1u%1u%1u  %#6x\n",
+                    lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
         }
         
         cp0_entry_hi_write(hi_save.value);
-        cp0_entry_lo0_write(lo0_save.value);
-        cp0_entry_lo1_write(lo1_save.value);
-        cp0_pagemask_write(mask_save.value);
 }
 
@@ -310 +448 @@
 void tlb_invalidate_all(void)
 {
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi_save;
         int i;
 
-        ASSERT(interrupts_disabled());
-
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
 
         for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
@@ -334 +472 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }
@@ -343 +482 @@
 void tlb_invalidate_asid(asid_t asid)
 {
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         int i;
 
-        ASSERT(interrupts_disabled());
         ASSERT(asid != ASID_INVALID);
 
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
         
         for (i = 0; i < TLB_ENTRY_COUNT; i++) {
@@ -372 +512 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }
@@ -385 +526 @@
 {
         unsigned int i;
+        ipl_t ipl;
         entry_lo_t lo0, lo1;
         entry_hi_t hi, hi_save;
         tlb_index_t index;
-
-        ASSERT(interrupts_disabled());
         
         if (asid == ASID_INVALID)
@@ -395 +535 @@
 
         hi_save.value = cp0_entry_hi_read();
+        ipl = interrupts_disable();
 
         for (i = 0; i < cnt + 1; i += 2) {
+                hi.value = 0;
                 tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
                 cp0_entry_hi_write(hi.value);
@@ -423 +565 @@
         }
         
+        interrupts_restore(ipl);
         cp0_entry_hi_write(hi_save.value);
 }