Changes in kernel/generic/src/mm/as.c [6b9e85b:fc47885] in mainline
- File:
-
- 1 edited
-
kernel/generic/src/mm/as.c (modified) (90 diffs)
Legend:
- Unmodified
- Added
- Removed
-
kernel/generic/src/mm/as.c
r6b9e85b rfc47885 79 79 #include <arch/interrupt.h> 80 80 81 #ifdef CONFIG_VIRT_IDX_DCACHE 82 #include <arch/mm/cache.h> 83 #endif /* CONFIG_VIRT_IDX_DCACHE */ 84 81 85 /** 82 86 * Each architecture decides what functions will be used to carry out 83 87 * address space operations such as creating or locking page tables. 84 *85 88 */ 86 89 as_operations_t *as_operations = NULL; 87 90 88 /** 89 * Slab for as_t objects. 91 /** Slab for as_t objects. 90 92 * 91 93 */ 92 94 static slab_cache_t *as_slab; 93 95 94 /** 95 * This lock serializes access to the ASID subsystem.96 * Itprotects:96 /** ASID subsystem lock. 97 * 98 * This lock protects: 97 99 * - inactive_as_with_asid_head list 98 100 * - as->asid for each as of the as_t type … … 103 105 104 106 /** 105 * This list contains address spaces that are not active on any 106 * processor and that have valid ASID. 107 * 107 * Inactive address spaces (on all processors) 108 * that have valid ASID. 108 109 */ 109 110 LIST_INITIALIZE(inactive_as_with_asid_head); … … 119 120 mutex_initialize(&as->lock, MUTEX_PASSIVE); 120 121 121 int rc = as_constructor_arch(as, flags); 122 123 return rc; 122 return as_constructor_arch(as, flags); 124 123 } 125 124 126 125 NO_TRACE static size_t as_destructor(void *obj) 127 126 { 128 as_t *as = (as_t *) obj; 129 return as_destructor_arch(as); 127 return as_destructor_arch((as_t *) obj); 130 128 } 131 129 … … 142 140 panic("Cannot create kernel address space."); 143 141 144 /* Make sure the kernel address space 142 /* 143 * Make sure the kernel address space 145 144 * reference count never drops to zero. 146 145 */ … … 191 190 { 192 191 DEADLOCK_PROBE_INIT(p_asidlock); 193 192 194 193 ASSERT(as != AS); 195 194 ASSERT(atomic_get(&as->refcount) == 0); … … 199 198 * lock its mutex. 200 199 */ 201 200 202 201 /* 203 202 * We need to avoid deadlock between TLB shootdown and asidlock. … … 206 205 * disabled to prevent nested context switches. We also depend on the 207 206 * fact that so far no spinlocks are held. 208 *209 207 */ 210 208 preemption_disable(); … … 231 229 spinlock_unlock(&asidlock); 232 230 interrupts_restore(ipl); 233 231 234 232 235 233 /* … … 237 235 * The B+tree must be walked carefully because it is 238 236 * also being destroyed. 239 *240 237 */ 241 238 bool cond = true; … … 264 261 /** Hold a reference to an address space. 265 262 * 266 * Holding a reference to an address space prevents destruction of that address267 * space.263 * Holding a reference to an address space prevents destruction 264 * of that address space. 268 265 * 269 266 * @param as Address space to be held. … … 277 274 /** Release a reference to an address space. 278 275 * 279 * The last one to release a reference to an address space destroys the address280 * space.276 * The last one to release a reference to an address space 277 * destroys the address space. 281 278 * 282 279 * @param asAddress space to be released. … … 306 303 /* 307 304 * We don't want any area to have conflicts with NULL page. 308 *309 305 */ 310 306 if (overlaps(va, size, (uintptr_t) NULL, PAGE_SIZE)) … … 317 313 * record in the left neighbour, the leftmost record in the right 318 314 * neighbour and all records in the leaf node itself. 319 *320 315 */ 321 316 btree_node_t *leaf; … … 378 373 * So far, the area does not conflict with other areas. 379 374 * Check if it doesn't conflict with kernel address space. 380 *381 375 */ 382 376 if (!KERNEL_ADDRESS_SPACE_SHADOWED) { … … 433 427 area->attributes = attrs; 434 428 area->pages = SIZE2FRAMES(size); 429 area->resident = 0; 435 430 area->base = base; 436 431 area->sh_info = NULL; … … 475 470 * to find out whether this is a miss or va belongs to an address 476 471 * space area found there. 477 *478 472 */ 479 473 … … 495 489 * Second, locate the left neighbour and test its last record. 496 490 * Because of its position in the B+tree, it must have base < va. 497 *498 491 */ 499 492 btree_node_t *lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); … … 530 523 /* 531 524 * Locate the area. 532 *533 525 */ 534 526 as_area_t *area = find_area_and_lock(as, address); … … 542 534 * Remapping of address space areas associated 543 535 * with memory mapped devices is not supported. 544 *545 536 */ 546 537 mutex_unlock(&area->lock); … … 553 544 * Remapping of shared address space areas 554 545 * is not supported. 555 *556 546 */ 557 547 mutex_unlock(&area->lock); … … 564 554 /* 565 555 * Zero size address space areas are not allowed. 566 *567 556 */ 568 557 mutex_unlock(&area->lock); … … 577 566 * Shrinking the area. 578 567 * No need to check for overlaps. 579 *580 568 */ 581 569 … … 584 572 /* 585 573 * Start TLB shootdown sequence. 586 *587 574 */ 588 575 ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, as->asid, … … 595 582 * is also the right way to remove part of the used_space 596 583 * B+tree leaf list. 597 *598 584 */ 599 585 bool cond = true; … … 619 605 * completely in the resized 620 606 * address space area. 621 *622 607 */ 623 608 break; … … 628 613 * to b and c overlaps with the resized 629 614 * address space area. 630 *631 615 */ 632 616 … … 669 653 /* 670 654 * Finish TLB shootdown sequence. 671 *672 655 */ 673 656 … … 677 660 /* 678 661 * Invalidate software translation caches (e.g. TSB on sparc64). 679 *680 662 */ 681 663 as_invalidate_translation_cache(as, area->base + … … 688 670 * Growing the area. 689 671 * Check for overlaps with other address space areas. 690 *691 672 */ 692 673 if (!check_area_conflicts(as, address, pages * PAGE_SIZE, … … 809 790 /* 810 791 * Finish TLB shootdown sequence. 811 *812 792 */ 813 793 … … 817 797 * Invalidate potential software translation caches (e.g. TSB on 818 798 * sparc64). 819 *820 799 */ 821 800 as_invalidate_translation_cache(as, area->base, area->pages); … … 835 814 /* 836 815 * Remove the empty area from address space. 837 *838 816 */ 839 817 btree_remove(&as->as_area_btree, base, NULL); … … 877 855 /* 878 856 * Could not find the source address space area. 879 *880 857 */ 881 858 mutex_unlock(&src_as->lock); … … 887 864 * There is no backend or the backend does not 888 865 * know how to share the area. 889 *890 866 */ 891 867 mutex_unlock(&src_area->lock); … … 914 890 * First, prepare the area for sharing. 915 891 * Then it will be safe to unlock it. 916 *917 892 */ 918 893 share_info_t *sh_info = src_area->sh_info; … … 926 901 /* 927 902 * Call the backend to setup sharing. 928 *929 903 */ 930 904 src_area->backend->share(src_area); … … 945 919 * The flags of the source area are masked against dst_flags_mask 946 920 * to support sharing in less privileged mode. 947 *948 921 */ 949 922 as_area_t *dst_area = as_area_create(dst_as, dst_flags_mask, src_size, … … 962 935 * fully initialized. Clear the AS_AREA_ATTR_PARTIAL 963 936 * attribute and set the sh_info. 964 *965 937 */ 966 938 mutex_lock(&dst_as->lock); … … 985 957 NO_TRACE bool as_area_check_access(as_area_t *area, pf_access_t access) 986 958 { 959 ASSERT(mutex_locked(&area->lock)); 960 987 961 int flagmap[] = { 988 962 [PF_ACCESS_READ] = AS_AREA_READ, … … 990 964 [PF_ACCESS_EXEC] = AS_AREA_EXEC 991 965 }; 992 993 ASSERT(mutex_locked(&area->lock));994 966 995 967 if (!(area->flags & flagmap[access])) … … 1062 1034 /* 1063 1035 * Compute total number of used pages in the used_space B+tree 1064 *1065 1036 */ 1066 1037 size_t used_pages = 0; … … 1084 1055 /* 1085 1056 * Start TLB shootdown sequence. 1086 *1087 1057 */ 1088 1058 ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, … … 1092 1062 * Remove used pages from page tables and remember their frame 1093 1063 * numbers. 1094 *1095 1064 */ 1096 1065 size_t frame_idx = 0; … … 1123 1092 /* 1124 1093 * Finish TLB shootdown sequence. 1125 *1126 1094 */ 1127 1095 … … 1131 1099 * Invalidate potential software translation caches (e.g. TSB on 1132 1100 * sparc64). 1133 *1134 1101 */ 1135 1102 as_invalidate_translation_cache(as, area->base, area->pages); … … 1213 1180 * No area contained mapping for 'page'. 1214 1181 * Signal page fault to low-level handler. 1215 *1216 1182 */ 1217 1183 mutex_unlock(&AS->lock); … … 1233 1199 * The address space area is not backed by any backend 1234 1200 * or the backend cannot handle page faults. 1235 *1236 1201 */ 1237 1202 mutex_unlock(&area->lock); … … 1245 1210 * To avoid race condition between two page faults on the same address, 1246 1211 * we need to make sure the mapping has not been already inserted. 1247 *1248 1212 */ 1249 1213 pte_t *pte; … … 1263 1227 /* 1264 1228 * Resort to the backend page fault handler. 1265 *1266 1229 */ 1267 1230 if (area->backend->page_fault(area, page, access) != AS_PF_OK) { … … 1318 1281 * preemption is disabled. We should not be 1319 1282 * holding any other lock. 1320 *1321 1283 */ 1322 1284 (void) interrupts_enable(); … … 1338 1300 * list of inactive address spaces with assigned 1339 1301 * ASID. 1340 *1341 1302 */ 1342 1303 ASSERT(old_as->asid != ASID_INVALID); … … 1349 1310 * Perform architecture-specific tasks when the address space 1350 1311 * is being removed from the CPU. 1351 *1352 1312 */ 1353 1313 as_deinstall_arch(old_as); … … 1356 1316 /* 1357 1317 * Second, prepare the new address space. 1358 *1359 1318 */ 1360 1319 if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { … … 1372 1331 * Perform architecture-specific steps. 1373 1332 * (e.g. write ASID to hardware register etc.) 1374 *1375 1333 */ 1376 1334 as_install_arch(new_as); … … 1391 1349 { 1392 1350 ASSERT(mutex_locked(&area->lock)); 1393 1351 1394 1352 return area_flags_to_page_flags(area->flags); 1395 1353 } … … 1512 1470 * @param count Number of page to be marked. 1513 1471 * 1514 * @return Zero on failure and non-zeroon success.1515 * 1516 */ 1517 intused_space_insert(as_area_t *area, uintptr_t page, size_t count)1472 * @return False on failure or true on success. 1473 * 1474 */ 1475 bool used_space_insert(as_area_t *area, uintptr_t page, size_t count) 1518 1476 { 1519 1477 ASSERT(mutex_locked(&area->lock)); … … 1526 1484 /* 1527 1485 * We hit the beginning of some used space. 1528 * 1529 */ 1530 return 0; 1486 */ 1487 return false; 1531 1488 } 1532 1489 1533 1490 if (!leaf->keys) { 1534 1491 btree_insert(&area->used_space, page, (void *) count, leaf); 1535 return 1;1492 goto success; 1536 1493 } 1537 1494 … … 1547 1504 * somewhere between the rightmost interval of 1548 1505 * the left neigbour and the first interval of the leaf. 1549 *1550 1506 */ 1551 1507 … … 1555 1511 left_cnt * PAGE_SIZE)) { 1556 1512 /* The interval intersects with the left interval. */ 1557 return 0;1513 return false; 1558 1514 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1559 1515 right_cnt * PAGE_SIZE)) { 1560 1516 /* The interval intersects with the right interval. */ 1561 return 0;1517 return false; 1562 1518 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1563 1519 (page + count * PAGE_SIZE == right_pg)) { … … 1565 1521 * The interval can be added by merging the two already 1566 1522 * present intervals. 1567 *1568 1523 */ 1569 1524 node->value[node->keys - 1] += count + right_cnt; 1570 1525 btree_remove(&area->used_space, right_pg, leaf); 1571 return 1;1526 goto success; 1572 1527 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1573 1528 /* 1574 1529 * The interval can be added by simply growing the left 1575 1530 * interval. 1576 *1577 1531 */ 1578 1532 node->value[node->keys - 1] += count; 1579 return 1;1533 goto success; 1580 1534 } else if (page + count * PAGE_SIZE == right_pg) { 1581 1535 /* … … 1583 1537 * the right interval down and increasing its size 1584 1538 * accordingly. 1585 *1586 1539 */ 1587 1540 leaf->value[0] += count; 1588 1541 leaf->key[0] = page; 1589 return 1;1542 goto success; 1590 1543 } else { 1591 1544 /* 1592 1545 * The interval is between both neigbouring intervals, 1593 1546 * but cannot be merged with any of them. 1594 *1595 1547 */ 1596 1548 btree_insert(&area->used_space, page, (void *) count, 1597 1549 leaf); 1598 return 1;1550 goto success; 1599 1551 } 1600 1552 } else if (page < leaf->key[0]) { … … 1605 1557 * Investigate the border case in which the left neighbour does 1606 1558 * not exist but the interval fits from the left. 1607 *1608 1559 */ 1609 1560 … … 1611 1562 right_cnt * PAGE_SIZE)) { 1612 1563 /* The interval intersects with the right interval. */ 1613 return 0;1564 return false; 1614 1565 } else if (page + count * PAGE_SIZE == right_pg) { 1615 1566 /* … … 1617 1568 * right interval down and increasing its size 1618 1569 * accordingly. 1619 *1620 1570 */ 1621 1571 leaf->key[0] = page; 1622 1572 leaf->value[0] += count; 1623 return 1;1573 goto success; 1624 1574 } else { 1625 1575 /* 1626 1576 * The interval doesn't adjoin with the right interval. 1627 1577 * It must be added individually. 1628 *1629 1578 */ 1630 1579 btree_insert(&area->used_space, page, (void *) count, 1631 1580 leaf); 1632 return 1;1581 goto success; 1633 1582 } 1634 1583 } … … 1645 1594 * somewhere between the leftmost interval of 1646 1595 * the right neigbour and the last interval of the leaf. 1647 *1648 1596 */ 1649 1597 … … 1653 1601 left_cnt * PAGE_SIZE)) { 1654 1602 /* The interval intersects with the left interval. */ 1655 return 0;1603 return false; 1656 1604 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1657 1605 right_cnt * PAGE_SIZE)) { 1658 1606 /* The interval intersects with the right interval. */ 1659 return 0;1607 return false; 1660 1608 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1661 1609 (page + count * PAGE_SIZE == right_pg)) { … … 1663 1611 * The interval can be added by merging the two already 1664 1612 * present intervals. 1665 *1666 1613 */ 1667 1614 leaf->value[leaf->keys - 1] += count + right_cnt; 1668 1615 btree_remove(&area->used_space, right_pg, node); 1669 return 1;1616 goto success; 1670 1617 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1671 1618 /* 1672 1619 * The interval can be added by simply growing the left 1673 1620 * interval. 1674 *1675 1621 */ 1676 leaf->value[leaf->keys - 1] += 1677 return 1;1622 leaf->value[leaf->keys - 1] += count; 1623 goto success; 1678 1624 } else if (page + count * PAGE_SIZE == right_pg) { 1679 1625 /* … … 1681 1627 * the right interval down and increasing its size 1682 1628 * accordingly. 1683 *1684 1629 */ 1685 1630 node->value[0] += count; 1686 1631 node->key[0] = page; 1687 return 1;1632 goto success; 1688 1633 } else { 1689 1634 /* 1690 1635 * The interval is between both neigbouring intervals, 1691 1636 * but cannot be merged with any of them. 1692 *1693 1637 */ 1694 1638 btree_insert(&area->used_space, page, (void *) count, 1695 1639 leaf); 1696 return 1;1640 goto success; 1697 1641 } 1698 1642 } else if (page >= leaf->key[leaf->keys - 1]) { … … 1703 1647 * Investigate the border case in which the right neighbour 1704 1648 * does not exist but the interval fits from the right. 1705 *1706 1649 */ 1707 1650 … … 1709 1652 left_cnt * PAGE_SIZE)) { 1710 1653 /* The interval intersects with the left interval. */ 1711 return 0;1654 return false; 1712 1655 } else if (left_pg + left_cnt * PAGE_SIZE == page) { 1713 1656 /* 1714 1657 * The interval can be added by growing the left 1715 1658 * interval. 1716 *1717 1659 */ 1718 1660 leaf->value[leaf->keys - 1] += count; 1719 return 1;1661 goto success; 1720 1662 } else { 1721 1663 /* 1722 1664 * The interval doesn't adjoin with the left interval. 1723 1665 * It must be added individually. 1724 *1725 1666 */ 1726 1667 btree_insert(&area->used_space, page, (void *) count, 1727 1668 leaf); 1728 return 1;1669 goto success; 1729 1670 } 1730 1671 } … … 1734 1675 * only between two other intervals of the leaf. The two border cases 1735 1676 * were already resolved. 1736 *1737 1677 */ 1738 1678 btree_key_t i; … … 1746 1686 /* 1747 1687 * The interval fits between left_pg and right_pg. 1748 *1749 1688 */ 1750 1689 … … 1754 1693 * The interval intersects with the left 1755 1694 * interval. 1756 *1757 1695 */ 1758 return 0;1696 return false; 1759 1697 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1760 1698 right_cnt * PAGE_SIZE)) { … … 1762 1700 * The interval intersects with the right 1763 1701 * interval. 1764 *1765 1702 */ 1766 return 0;1703 return false; 1767 1704 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1768 1705 (page + count * PAGE_SIZE == right_pg)) { … … 1770 1707 * The interval can be added by merging the two 1771 1708 * already present intervals. 1772 *1773 1709 */ 1774 1710 leaf->value[i - 1] += count + right_cnt; 1775 1711 btree_remove(&area->used_space, right_pg, leaf); 1776 return 1;1712 goto success; 1777 1713 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1778 1714 /* 1779 1715 * The interval can be added by simply growing 1780 1716 * the left interval. 1781 *1782 1717 */ 1783 1718 leaf->value[i - 1] += count; 1784 return 1;1719 goto success; 1785 1720 } else if (page + count * PAGE_SIZE == right_pg) { 1786 1721 /* … … 1788 1723 * base of the right interval down and 1789 1724 * increasing its size accordingly. 1790 *1791 1725 */ 1792 1726 leaf->value[i] += count; 1793 1727 leaf->key[i] = page; 1794 return 1;1728 goto success; 1795 1729 } else { 1796 1730 /* … … 1798 1732 * intervals, but cannot be merged with any of 1799 1733 * them. 1800 *1801 1734 */ 1802 1735 btree_insert(&area->used_space, page, 1803 1736 (void *) count, leaf); 1804 return 1;1737 goto success; 1805 1738 } 1806 1739 } … … 1809 1742 panic("Inconsistency detected while adding %zu pages of used " 1810 1743 "space at %p.", count, (void *) page); 1744 1745 success: 1746 area->resident += count; 1747 return true; 1811 1748 } 1812 1749 … … 1819 1756 * @param count Number of page to be marked. 1820 1757 * 1821 * @return Zero on failure and non-zeroon success.1822 * 1823 */ 1824 intused_space_remove(as_area_t *area, uintptr_t page, size_t count)1758 * @return False on failure or true on success. 1759 * 1760 */ 1761 bool used_space_remove(as_area_t *area, uintptr_t page, size_t count) 1825 1762 { 1826 1763 ASSERT(mutex_locked(&area->lock)); … … 1833 1770 /* 1834 1771 * We are lucky, page is the beginning of some interval. 1835 *1836 1772 */ 1837 1773 if (count > pages) { 1838 return 0;1774 return false; 1839 1775 } else if (count == pages) { 1840 1776 btree_remove(&area->used_space, page, leaf); 1841 return 1;1777 goto success; 1842 1778 } else { 1843 1779 /* 1844 1780 * Find the respective interval. 1845 1781 * Decrease its size and relocate its start address. 1846 *1847 1782 */ 1848 1783 btree_key_t i; … … 1851 1786 leaf->key[i] += count * PAGE_SIZE; 1852 1787 leaf->value[i] -= count; 1853 return 1;1788 goto success; 1854 1789 } 1855 1790 } 1791 1856 1792 goto error; 1857 1793 } … … 1872 1808 * removed by updating the size of the bigger 1873 1809 * interval. 1874 *1875 1810 */ 1876 1811 node->value[node->keys - 1] -= count; 1877 return 1;1812 goto success; 1878 1813 } else if (page + count * PAGE_SIZE < 1879 1814 left_pg + left_cnt*PAGE_SIZE) { … … 1884 1819 * the original interval and also inserting a 1885 1820 * new interval. 1886 *1887 1821 */ 1888 1822 size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - … … 1891 1825 btree_insert(&area->used_space, page + 1892 1826 count * PAGE_SIZE, (void *) new_cnt, leaf); 1893 return 1;1827 goto success; 1894 1828 } 1895 1829 } 1896 return 0; 1830 1831 return false; 1897 1832 } else if (page < leaf->key[0]) 1898 return 0;1833 return false; 1899 1834 1900 1835 if (page > leaf->key[leaf->keys - 1]) { … … 1910 1845 * interval of the leaf and can be removed by 1911 1846 * updating the size of the bigger interval. 1912 *1913 1847 */ 1914 1848 leaf->value[leaf->keys - 1] -= count; 1915 return 1;1849 goto success; 1916 1850 } else if (page + count * PAGE_SIZE < left_pg + 1917 1851 left_cnt * PAGE_SIZE) { … … 1922 1856 * original interval and also inserting a new 1923 1857 * interval. 1924 *1925 1858 */ 1926 1859 size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - … … 1929 1862 btree_insert(&area->used_space, page + 1930 1863 count * PAGE_SIZE, (void *) new_cnt, leaf); 1931 return 1;1864 goto success; 1932 1865 } 1933 1866 } 1934 return 0; 1867 1868 return false; 1935 1869 } 1936 1870 1937 1871 /* 1938 1872 * The border cases have been already resolved. 1939 * Now the interval can be only between intervals of the leaf. 1873 * Now the interval can be only between intervals of the leaf. 1940 1874 */ 1941 1875 btree_key_t i; … … 1958 1892 * be removed by updating the size of 1959 1893 * the bigger interval. 1960 *1961 1894 */ 1962 1895 leaf->value[i - 1] -= count; 1963 return 1;1896 goto success; 1964 1897 } else if (page + count * PAGE_SIZE < 1965 1898 left_pg + left_cnt * PAGE_SIZE) { … … 1979 1912 count * PAGE_SIZE, (void *) new_cnt, 1980 1913 leaf); 1981 return 1;1914 goto success; 1982 1915 } 1983 1916 } 1984 return 0; 1917 1918 return false; 1985 1919 } 1986 1920 } … … 1989 1923 panic("Inconsistency detected while removing %zu pages of used " 1990 1924 "space from %p.", count, (void *) page); 1925 1926 success: 1927 area->resident -= count; 1928 return true; 1991 1929 } 1992 1930
Note:
See TracChangeset
for help on using the changeset viewer.
