Changes in kernel/genarch/src/mm/page_pt.c [a2789d2:de73242] in mainline

File:

• kernel/genarch/src/mm/page_pt.c (modified) (10 diffs)

kernel/genarch/src/mm/page_pt.c

@@ -43 +43 @@
 #include <arch/mm/page.h>
 #include <arch/mm/as.h>
+#include <arch/barrier.h>
 #include <typedefs.h>
 #include <arch/asm.h>
@@ -48 +49 @@
 #include <align.h>
 #include <macros.h>
+#include <bitops.h>
 
 static void pt_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int);
@@ -85 +87 @@
                 SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page), KA2PA(newpt));
                 SET_PTL1_FLAGS(ptl0, PTL0_INDEX(page),
-                    PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
+                    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
                     PAGE_WRITE);
+                /*
+                 * Make sure that a concurrent hardware page table walk or
+                 * pt_mapping_find() will see the new PTL1 only after it is
+                 * fully initialized.
+                 */
+                write_barrier();
+                SET_PTL1_PRESENT(ptl0, PTL0_INDEX(page));
         }
         
@@ -97 +106 @@
                 SET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page), KA2PA(newpt));
                 SET_PTL2_FLAGS(ptl1, PTL1_INDEX(page),
-                    PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
+                    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
                     PAGE_WRITE);
+                /*
+                 * Make the new PTL2 visible only after it is fully initialized.
+                 */
+                write_barrier();
+                SET_PTL2_PRESENT(ptl1, PTL1_INDEX(page));       
         }
         
@@ -109 +123 @@
                 SET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page), KA2PA(newpt));
                 SET_PTL3_FLAGS(ptl2, PTL2_INDEX(page),
-                    PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
+                    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
                     PAGE_WRITE);
+                /*
+                 * Make the new PTL3 visible only after it is fully initialized.
+                 */
+                write_barrier();
+                SET_PTL3_PRESENT(ptl2, PTL2_INDEX(page));
         }
         
@@ -116 +135 @@
         
         SET_FRAME_ADDRESS(ptl3, PTL3_INDEX(page), frame);
-        SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags);
+        SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags | PAGE_NOT_PRESENT);
+        /*
+         * Make the new mapping visible only after it is fully initialized.
+         */
+        write_barrier();
+        SET_FRAME_PRESENT(ptl3, PTL3_INDEX(page));
 }
 
@@ -278 +302 @@
         if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT)
                 return NULL;
+
+        read_barrier();
         
         pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
         if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT)
                 return NULL;
+
+#if (PTL1_ENTRIES != 0)
+        /*
+         * Always read ptl2 only after we are sure it is present.
+         */
+        read_barrier();
+#endif
         
         pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
         if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT)
                 return NULL;
+
+#if (PTL2_ENTRIES != 0)
+        /*
+         * Always read ptl3 only after we are sure it is present.
+         */
+        read_barrier();
+#endif
         
         pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
         
         return &ptl3[PTL3_INDEX(page)];
+}
+
+/** Return the size of the region mapped by a single PTL0 entry.
+ *
+ * @return Size of the region mapped by a single PTL0 entry.
+ */
+static uintptr_t ptl0_step_get(void)
+{
+        size_t va_bits;
+
+        va_bits = fnzb(PTL0_ENTRIES) + fnzb(PTL1_ENTRIES) + fnzb(PTL2_ENTRIES) +
+            fnzb(PTL3_ENTRIES) + PAGE_WIDTH;
+
+        return 1UL << (va_bits - fnzb(PTL0_ENTRIES));
 }
 
@@ -309 +363 @@
 {
         uintptr_t ptl0 = PA2KA((uintptr_t) AS_KERNEL->genarch.page_table);
-        uintptr_t ptl0step = (((uintptr_t) -1) / PTL0_ENTRIES) + 1;
+        uintptr_t ptl0_step = ptl0_step_get();
         size_t order;
         uintptr_t addr;
@@ -321 +375 @@
 #endif
 
-        ASSERT(ispwr2(ptl0step));
         ASSERT(size > 0);
 
-        for (addr = ALIGN_DOWN(base, ptl0step); addr - 1 < base + size - 1;
-            addr += ptl0step) {
+        for (addr = ALIGN_DOWN(base, ptl0_step); addr - 1 < base + size - 1;
+            addr += ptl0_step) {
                 uintptr_t l1;
 
@@ -332 +385 @@
                 SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(addr), KA2PA(l1));
                 SET_PTL1_FLAGS(ptl0, PTL0_INDEX(addr),
-                    PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
-                    PAGE_WRITE);
+                    PAGE_PRESENT | PAGE_USER | PAGE_CACHEABLE |
+                    PAGE_EXEC | PAGE_WRITE | PAGE_READ);
         }
 }