Changes in / [321052f7:3fe58d3c] in mainline

Location:

kernel

Files:

• arch/arm32/include/mm/km.h (modified) (1 diff)
• arch/arm32/src/mm/page.c (modified) (1 diff)
• arch/ia32/src/mm/frame.c (modified) (1 diff)
• genarch/src/mm/page_pt.c (modified) (1 diff)
• generic/include/mm/frame.h (modified) (1 diff)
• generic/include/mm/km.h (modified) (2 diffs)
• generic/src/mm/backend_anon.c (modified) (7 diffs)
• generic/src/mm/backend_elf.c (modified) (12 diffs)
• generic/src/mm/km.c (modified) (2 diffs)
• generic/src/mm/page.c (modified) (1 diff)

kernel/arch/arm32/include/mm/km.h

@@ -39 +39 @@
 
 #define KM_ARM32_IDENTITY_START         UINT32_C(0x80000000)
-#define KM_ARM32_IDENTITY_SIZE          UINT32_C(0x40000000)
+#define KM_ARM32_IDENTITY_SIZE          UINT32_C(0x70000000)
 
-#define KM_ARM32_NON_IDENTITY_START     UINT32_C(0xc0000000)
-#define KM_ARM32_NON_IDENTITY_SIZE      UINT32_C(0x40000000)
+#define KM_ARM32_NON_IDENTITY_START     UINT32_C(0xf0000000)
+/*
+ * The last virtual megabyte contains the high exception vectors (0xFFFF0000).
+ * Do not include this range into kernel non-identity.
+ */
+#define KM_ARM32_NON_IDENTITY_SIZE      UINT32_C(0x0ff00000)
 
 extern void km_identity_arch_init(void);

kernel/arch/arm32/src/mm/page.c

@@ -65 +65 @@
                 page_mapping_insert(AS_KERNEL, PA2KA(cur), cur, flags);
         
+#ifdef HIGH_EXCEPTION_VECTORS
         /* Create mapping for exception table at high offset */
-#ifdef HIGH_EXCEPTION_VECTORS
-        // XXX: fixme to use proper non-identity page
-        void *virtaddr = frame_alloc(ONE_FRAME, FRAME_KA);
-        page_mapping_insert(AS_KERNEL, EXC_BASE_ADDRESS, KA2PA(virtaddr),
-            flags);
+        uintptr_t ev_frame = (uintptr_t) frame_alloc(ONE_FRAME, FRAME_NONE);
+        page_mapping_insert(AS_KERNEL, EXC_BASE_ADDRESS, ev_frame, flags);
 #else
 #error "Only high exception vector supported now"
 #endif
-        cur = ALIGN_DOWN(0x50008010, FRAME_SIZE);
-        page_mapping_insert(AS_KERNEL, PA2KA(cur), cur, flags);
 
         page_table_unlock(AS_KERNEL, true);

kernel/arch/ia32/src/mm/frame.c

@@ -54 +54 @@
         
         for (i = 0; i < e820counter; i++) {
-                uintptr_t base = (uintptr_t) e820table[i].base_address;
-                size_t size = (size_t) e820table[i].size;
+                uint64_t base64 = e820table[i].base_address;
+                uint64_t size64 = e820table[i].size;
+
+#ifdef KARCH_ia32
+                /*
+                 * Restrict the e820 table entries to 32-bits.
+                 */
+                if (base64 >= 0x100000000ULL)
+                        continue;
+                if (base64 + size64 > 0x100000000ULL)
+                        size64 -= base64 + size64 - 0x100000000ULL;
+#endif
+
+                uintptr_t base = (uintptr_t) base64;
+                size_t size = (size_t) size64;
                 
                 if (!frame_adjust_zone_bounds(low, &base, &size))

kernel/genarch/src/mm/page_pt.c

@@ -322 +322 @@
 
         ASSERT(ispwr2(ptl0step));
-
-        for (addr = ALIGN_DOWN(base, ptl0step); addr < base + size;
+        ASSERT(size > 0);
+
+        for (addr = ALIGN_DOWN(base, ptl0step); addr - 1 < base + size - 1;
             addr += ptl0step) {
                 uintptr_t l1;

kernel/generic/include/mm/frame.h

@@ -83 +83 @@
 #define FRAME_TO_ZONE_FLAGS(ff) \
         ((((ff) & FRAME_LOWMEM) ? ZONE_LOWMEM : \
-            (((ff) & FRAME_HIGHMEM) ? ZONE_HIGHMEM : ZONE_NONE)) | \
-            (ZONE_AVAILABLE | ZONE_LOWMEM /* | ZONE_HIGHMEM */)) 
+            (((ff) & FRAME_HIGHMEM) ? ZONE_HIGHMEM : \
+            ZONE_LOWMEM /* | ZONE_HIGHMEM */)) | \
+            ZONE_AVAILABLE) 
 
 #define ZONE_FLAGS_MATCH(zf, f) \

kernel/generic/include/mm/km.h

@@ -37 +37 @@
 
 #include <typedefs.h>
+#include <mm/frame.h>
 
 extern void km_identity_init(void);
@@ -48 +49 @@
 extern bool km_is_non_identity(uintptr_t);
 
+extern uintptr_t km_temporary_page_get(uintptr_t *, frame_flags_t);
+extern void km_temporary_page_put(uintptr_t);
+
 #endif
 

kernel/generic/src/mm/backend_anon.c

@@ -44 +44 @@
 #include <mm/frame.h>
 #include <mm/slab.h>
+#include <mm/km.h>
 #include <synch/mutex.h>
 #include <adt/list.h>
@@ -155 +156 @@
 int anon_page_fault(as_area_t *area, uintptr_t addr, pf_access_t access)
 {
+        uintptr_t upage = ALIGN_DOWN(addr, PAGE_SIZE);
+        uintptr_t kpage;
         uintptr_t frame;
 
@@ -175 +178 @@
                 mutex_lock(&area->sh_info->lock);
                 frame = (uintptr_t) btree_search(&area->sh_info->pagemap,
-                    ALIGN_DOWN(addr, PAGE_SIZE) - area->base, &leaf);
+                    upage - area->base, &leaf);
                 if (!frame) {
                         bool allocate = true;
@@ -185 +188 @@
                          */
                         for (i = 0; i < leaf->keys; i++) {
-                                if (leaf->key[i] ==
-                                    ALIGN_DOWN(addr, PAGE_SIZE) - area->base) {
+                                if (leaf->key[i] == upage - area->base) {
                                         allocate = false;
                                         break;
@@ -192 +194 @@
                         }
                         if (allocate) {
-                                frame = (uintptr_t) frame_alloc_noreserve(
-                                    ONE_FRAME, 0);
-                                memsetb((void *) PA2KA(frame), FRAME_SIZE, 0);
+                                kpage = km_temporary_page_get(&frame,
+                                    FRAME_NO_RESERVE);
+                                memsetb((void *) kpage, PAGE_SIZE, 0);
+                                km_temporary_page_put(kpage);
                                 
                                 /*
@@ -201 +204 @@
                                  */
                                 btree_insert(&area->sh_info->pagemap,
-                                    ALIGN_DOWN(addr, PAGE_SIZE) - area->base,
-                                    (void *) frame, leaf);
+                                    upage - area->base, (void *) frame, leaf);
                         }
                 }
@@ -223 +225 @@
                  *   the different causes
                  */
-                frame = (uintptr_t) frame_alloc_noreserve(ONE_FRAME, 0);
-                memsetb((void *) PA2KA(frame), FRAME_SIZE, 0);
+                kpage = km_temporary_page_get(&frame, FRAME_NO_RESERVE);
+                memsetb((void *) kpage, PAGE_SIZE, 0);
+                km_temporary_page_put(kpage);
         }
         
         /*
-         * Map 'page' to 'frame'.
+         * Map 'upage' to 'frame'.
          * Note that TLB shootdown is not attempted as only new information is
          * being inserted into page tables.
          */
-        page_mapping_insert(AS, addr, frame, as_area_get_flags(area));
-        if (!used_space_insert(area, ALIGN_DOWN(addr, PAGE_SIZE), 1))
+        page_mapping_insert(AS, upage, frame, as_area_get_flags(area));
+        if (!used_space_insert(area, upage, 1))
                 panic("Cannot insert used space.");
                 

kernel/generic/src/mm/backend_elf.c

@@ -44 +44 @@
 #include <mm/page.h>
 #include <mm/reserve.h>
+#include <mm/km.h>
 #include <genarch/mm/page_pt.h>
 #include <genarch/mm/page_ht.h>
@@ -229 +230 @@
         elf_segment_header_t *entry = area->backend_data.segment;
         btree_node_t *leaf;
-        uintptr_t base, frame, page, start_anon;
+        uintptr_t base;
+        uintptr_t frame;
+        uintptr_t kpage;
+        uintptr_t upage;
+        uintptr_t start_anon;
         size_t i;
         bool dirty = false;
@@ -249 +254 @@
             (((void *) elf) + ALIGN_DOWN(entry->p_offset, PAGE_SIZE));
 
-        /* Virtual address of faulting page*/
-        page = ALIGN_DOWN(addr, PAGE_SIZE);
+        /* Virtual address of faulting page */
+        upage = ALIGN_DOWN(addr, PAGE_SIZE);
 
         /* Virtual address of the end of initialized part of segment */
@@ -264 +269 @@
                 mutex_lock(&area->sh_info->lock);
                 frame = (uintptr_t) btree_search(&area->sh_info->pagemap,
-                    page - area->base, &leaf);
+                    upage - area->base, &leaf);
                 if (!frame) {
                         unsigned int i;
@@ -273 +278 @@
 
                         for (i = 0; i < leaf->keys; i++) {
-                                if (leaf->key[i] == page - area->base) {
+                                if (leaf->key[i] == upage - area->base) {
                                         found = true;
                                         break;
@@ -281 +286 @@
                 if (frame || found) {
                         frame_reference_add(ADDR2PFN(frame));
-                        page_mapping_insert(AS, addr, frame,
+                        page_mapping_insert(AS, upage, frame,
                             as_area_get_flags(area));
-                        if (!used_space_insert(area, page, 1))
+                        if (!used_space_insert(area, upage, 1))
                                 panic("Cannot insert used space.");
                         mutex_unlock(&area->sh_info->lock);
@@ -294 +299 @@
          * mapping.
          */
-        if (page >= entry->p_vaddr && page + PAGE_SIZE <= start_anon) {
+        if (upage >= entry->p_vaddr && upage + PAGE_SIZE <= start_anon) {
                 /*
                  * Initialized portion of the segment. The memory is backed
@@ -304 +309 @@
                  */
                 if (entry->p_flags & PF_W) {
-                        frame = (uintptr_t)frame_alloc_noreserve(ONE_FRAME, 0);
-                        memcpy((void *) PA2KA(frame),
-                            (void *) (base + i * FRAME_SIZE), FRAME_SIZE);
+                        kpage = km_temporary_page_get(&frame, FRAME_NO_RESERVE);
+                        memcpy((void *) kpage, (void *) (base + i * PAGE_SIZE),
+                            PAGE_SIZE);
                         if (entry->p_flags & PF_X) {
-                                smc_coherence_block((void *) PA2KA(frame),
-                                    FRAME_SIZE);
+                                smc_coherence_block((void *) kpage, PAGE_SIZE);
                         }
+                        km_temporary_page_put(kpage);
                         dirty = true;
                 } else {
                         frame = KA2PA(base + i * FRAME_SIZE);
                 }       
-        } else if (page >= start_anon) {
+        } else if (upage >= start_anon) {
                 /*
                  * This is the uninitialized portion of the segment.
@@ -322 +327 @@
                  * and cleared.
                  */
-                frame = (uintptr_t) frame_alloc_noreserve(ONE_FRAME, 0);
-                memsetb((void *) PA2KA(frame), FRAME_SIZE, 0);
+                kpage = km_temporary_page_get(&frame, FRAME_NO_RESERVE);
+                memsetb((void *) kpage, PAGE_SIZE, 0);
+                km_temporary_page_put(kpage);
                 dirty = true;
         } else {
@@ -334 +340 @@
                  * (The segment can be and often is shorter than 1 page).
                  */
-                if (page < entry->p_vaddr)
-                        pad_lo = entry->p_vaddr - page;
+                if (upage < entry->p_vaddr)
+                        pad_lo = entry->p_vaddr - upage;
                 else
                         pad_lo = 0;
 
-                if (start_anon < page + PAGE_SIZE)
-                        pad_hi = page + PAGE_SIZE - start_anon;
+                if (start_anon < upage + PAGE_SIZE)
+                        pad_hi = upage + PAGE_SIZE - start_anon;
                 else
                         pad_hi = 0;
 
-                frame = (uintptr_t) frame_alloc_noreserve(ONE_FRAME, 0);
-                memcpy((void *) (PA2KA(frame) + pad_lo),
-                    (void *) (base + i * FRAME_SIZE + pad_lo),
-                    FRAME_SIZE - pad_lo - pad_hi);
+                kpage = km_temporary_page_get(&frame, FRAME_NO_RESERVE);
+                memcpy((void *) (kpage + pad_lo),
+                    (void *) (base + i * PAGE_SIZE + pad_lo),
+                    PAGE_SIZE - pad_lo - pad_hi);
                 if (entry->p_flags & PF_X) {
-                        smc_coherence_block((void *) (PA2KA(frame) + pad_lo), 
-                            FRAME_SIZE - pad_lo - pad_hi);
+                        smc_coherence_block((void *) (kpage + pad_lo), 
+                            PAGE_SIZE - pad_lo - pad_hi);
                 }
-                memsetb((void *) PA2KA(frame), pad_lo, 0);
-                memsetb((void *) (PA2KA(frame) + FRAME_SIZE - pad_hi), pad_hi,
-                    0);
+                memsetb((void *) kpage, pad_lo, 0);
+                memsetb((void *) (kpage + PAGE_SIZE - pad_hi), pad_hi, 0);
+                km_temporary_page_put(kpage);
                 dirty = true;
         }
@@ -360 +366 @@
         if (dirty && area->sh_info) {
                 frame_reference_add(ADDR2PFN(frame));
-                btree_insert(&area->sh_info->pagemap, page - area->base,
+                btree_insert(&area->sh_info->pagemap, upage - area->base,
                     (void *) frame, leaf);
         }
@@ -367 +373 @@
                 mutex_unlock(&area->sh_info->lock);
 
-        page_mapping_insert(AS, addr, frame, as_area_get_flags(area));
-        if (!used_space_insert(area, page, 1))
+        page_mapping_insert(AS, upage, frame, as_area_get_flags(area));
+        if (!used_space_insert(area, upage, 1))
                 panic("Cannot insert used space.");
 

kernel/generic/src/mm/km.c

@@ -39 +39 @@
 #include <arch/mm/km.h>
 #include <mm/page.h>
+#include <mm/frame.h>
+#include <mm/asid.h>
 #include <config.h>
 #include <typedefs.h>
 #include <lib/ra.h>
 #include <debug.h>
+#include <arch.h>
 
 static ra_arena_t *km_ni_arena;
+
+#define DEFERRED_PAGES_MAX      (PAGE_SIZE / sizeof(uintptr_t)) 
+
+/** Number of freed pages in the deferred buffer. */
+static volatile unsigned deferred_pages;
+/** Buffer of deferred freed pages. */
+static uintptr_t deferred_page[DEFERRED_PAGES_MAX];
+
+/** Flush the buffer of deferred freed pages.
+ *
+ * @return              Number of freed pages.
+ */
+static unsigned km_flush_deferred(void)
+{
+        unsigned i = 0;
+        ipl_t ipl;
+
+        ipl = tlb_shootdown_start(TLB_INVL_ASID, ASID_KERNEL, 0, 0);
+
+        for (i = 0; i < deferred_pages; i++) {
+                page_mapping_remove(AS_KERNEL, deferred_page[i]);
+                km_page_free(deferred_page[i], PAGE_SIZE);
+        }
+
+        tlb_invalidate_asid(ASID_KERNEL);
+
+        as_invalidate_translation_cache(AS_KERNEL, 0, -1);
+        tlb_shootdown_finalize(ipl);
+
+        return i;
+}
 
 /** Architecture dependent setup of identity-mapped kernel memory. */
@@ -87 +121 @@
 }
 
+/** Unmap kernen non-identity page.
+ *
+ * @param[in] page      Non-identity page to be unmapped.
+ */
+static void km_unmap_deferred(uintptr_t page)
+{
+        page_table_lock(AS_KERNEL, true);
+
+        if (deferred_pages == DEFERRED_PAGES_MAX) {
+                (void) km_flush_deferred();
+                deferred_pages = 0;
+        }
+
+        deferred_page[deferred_pages++] = page;
+
+        page_table_unlock(AS_KERNEL, true);
+}
+
+/** Create a temporary page.
+ *
+ * The page is mapped read/write to a newly allocated frame of physical memory.
+ * The page must be returned back to the system by a call to
+ * km_temporary_page_put().
+ *
+ * @param[inout] framep Pointer to a variable which will receive the physical
+ *                      address of the allocated frame.
+ * @param[in] flags     Frame allocation flags. FRAME_NONE or FRAME_NO_RESERVE.
+ * @return              Virtual address of the allocated frame.
+ */
+uintptr_t km_temporary_page_get(uintptr_t *framep, frame_flags_t flags)
+{
+        uintptr_t frame;
+        uintptr_t page;
+
+        ASSERT(THREAD);
+        ASSERT(framep);
+        ASSERT(!(flags & ~FRAME_NO_RESERVE));
+
+        /*
+         * Allocate a frame, preferably from high memory.
+         */
+        frame = (uintptr_t) frame_alloc(ONE_FRAME,
+            FRAME_HIGHMEM | FRAME_ATOMIC | flags); 
+        if (frame) {
+                page = km_page_alloc(PAGE_SIZE, PAGE_SIZE);
+                ASSERT(page);   // FIXME
+                page_table_lock(AS_KERNEL, true);
+                page_mapping_insert(AS_KERNEL, page, frame,
+                    PAGE_CACHEABLE | PAGE_READ | PAGE_WRITE);
+                page_table_unlock(AS_KERNEL, true);
+        } else {
+                frame = (uintptr_t) frame_alloc_noreserve(ONE_FRAME,
+                    FRAME_LOWMEM);
+                page = PA2KA(frame);
+        }
+
+        *framep = frame;
+        return page;    
+}
+
+/** Destroy a temporary page.
+ *
+ * This function destroys a temporary page previously created by
+ * km_temporary_page_get(). The page destruction may be immediate or deferred.
+ * The frame mapped by the destroyed page is not freed.
+ *
+ * @param[in] page      Temporary page to be destroyed.
+ */
+void km_temporary_page_put(uintptr_t page)
+{
+        ASSERT(THREAD);
+
+        if (km_is_non_identity(page))
+                km_unmap_deferred(page);
+}
 
 /** @}

kernel/generic/src/mm/page.c

@@ -202 +202 @@
         asize = ALIGN_UP(size, PAGE_SIZE);
         align = ispwr2(size) ? size : (1U << (fnzb(size) + 1));
-        virtaddr = km_page_alloc(asize, align);
+        virtaddr = km_page_alloc(asize, max(PAGE_SIZE, align));
 
         page_table_lock(AS_KERNEL, true);