Changes in kernel/generic/src/mm/as.c [fc47885:6b9e85b] in mainline
- File:
-
- 1 edited
-
kernel/generic/src/mm/as.c (modified) (90 diffs)
Legend:
- Unmodified
- Added
- Removed
-
kernel/generic/src/mm/as.c
rfc47885 r6b9e85b 79 79 #include <arch/interrupt.h> 80 80 81 #ifdef CONFIG_VIRT_IDX_DCACHE82 #include <arch/mm/cache.h>83 #endif /* CONFIG_VIRT_IDX_DCACHE */84 85 81 /** 86 82 * Each architecture decides what functions will be used to carry out 87 83 * address space operations such as creating or locking page tables. 84 * 88 85 */ 89 86 as_operations_t *as_operations = NULL; 90 87 91 /** Slab for as_t objects. 88 /** 89 * Slab for as_t objects. 92 90 * 93 91 */ 94 92 static slab_cache_t *as_slab; 95 93 96 /** ASID subsystem lock.97 * 98 * This lockprotects:94 /** 95 * This lock serializes access to the ASID subsystem. 96 * It protects: 99 97 * - inactive_as_with_asid_head list 100 98 * - as->asid for each as of the as_t type … … 105 103 106 104 /** 107 * Inactive address spaces (on all processors) 108 * that have valid ASID. 105 * This list contains address spaces that are not active on any 106 * processor and that have valid ASID. 107 * 109 108 */ 110 109 LIST_INITIALIZE(inactive_as_with_asid_head); … … 120 119 mutex_initialize(&as->lock, MUTEX_PASSIVE); 121 120 122 return as_constructor_arch(as, flags); 121 int rc = as_constructor_arch(as, flags); 122 123 return rc; 123 124 } 124 125 125 126 NO_TRACE static size_t as_destructor(void *obj) 126 127 { 127 return as_destructor_arch((as_t *) obj); 128 as_t *as = (as_t *) obj; 129 return as_destructor_arch(as); 128 130 } 129 131 … … 140 142 panic("Cannot create kernel address space."); 141 143 142 /* 143 * Make sure the kernel address space 144 /* Make sure the kernel address space 144 145 * reference count never drops to zero. 145 146 */ … … 190 191 { 191 192 DEADLOCK_PROBE_INIT(p_asidlock); 192 193 193 194 ASSERT(as != AS); 194 195 ASSERT(atomic_get(&as->refcount) == 0); … … 198 199 * lock its mutex. 199 200 */ 200 201 201 202 /* 202 203 * We need to avoid deadlock between TLB shootdown and asidlock. … … 205 206 * disabled to prevent nested context switches. We also depend on the 206 207 * fact that so far no spinlocks are held. 208 * 207 209 */ 208 210 preemption_disable(); … … 229 231 spinlock_unlock(&asidlock); 230 232 interrupts_restore(ipl); 231 233 232 234 233 235 /* … … 235 237 * The B+tree must be walked carefully because it is 236 238 * also being destroyed. 239 * 237 240 */ 238 241 bool cond = true; … … 261 264 /** Hold a reference to an address space. 262 265 * 263 * Holding a reference to an address space prevents destruction 264 * of that addressspace.266 * Holding a reference to an address space prevents destruction of that address 267 * space. 265 268 * 266 269 * @param as Address space to be held. … … 274 277 /** Release a reference to an address space. 275 278 * 276 * The last one to release a reference to an address space 277 * destroys the addressspace.279 * The last one to release a reference to an address space destroys the address 280 * space. 278 281 * 279 282 * @param asAddress space to be released. … … 303 306 /* 304 307 * We don't want any area to have conflicts with NULL page. 308 * 305 309 */ 306 310 if (overlaps(va, size, (uintptr_t) NULL, PAGE_SIZE)) … … 313 317 * record in the left neighbour, the leftmost record in the right 314 318 * neighbour and all records in the leaf node itself. 319 * 315 320 */ 316 321 btree_node_t *leaf; … … 373 378 * So far, the area does not conflict with other areas. 374 379 * Check if it doesn't conflict with kernel address space. 380 * 375 381 */ 376 382 if (!KERNEL_ADDRESS_SPACE_SHADOWED) { … … 427 433 area->attributes = attrs; 428 434 area->pages = SIZE2FRAMES(size); 429 area->resident = 0;430 435 area->base = base; 431 436 area->sh_info = NULL; … … 470 475 * to find out whether this is a miss or va belongs to an address 471 476 * space area found there. 477 * 472 478 */ 473 479 … … 489 495 * Second, locate the left neighbour and test its last record. 490 496 * Because of its position in the B+tree, it must have base < va. 497 * 491 498 */ 492 499 btree_node_t *lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); … … 523 530 /* 524 531 * Locate the area. 532 * 525 533 */ 526 534 as_area_t *area = find_area_and_lock(as, address); … … 534 542 * Remapping of address space areas associated 535 543 * with memory mapped devices is not supported. 544 * 536 545 */ 537 546 mutex_unlock(&area->lock); … … 544 553 * Remapping of shared address space areas 545 554 * is not supported. 555 * 546 556 */ 547 557 mutex_unlock(&area->lock); … … 554 564 /* 555 565 * Zero size address space areas are not allowed. 566 * 556 567 */ 557 568 mutex_unlock(&area->lock); … … 566 577 * Shrinking the area. 567 578 * No need to check for overlaps. 579 * 568 580 */ 569 581 … … 572 584 /* 573 585 * Start TLB shootdown sequence. 586 * 574 587 */ 575 588 ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, as->asid, … … 582 595 * is also the right way to remove part of the used_space 583 596 * B+tree leaf list. 597 * 584 598 */ 585 599 bool cond = true; … … 605 619 * completely in the resized 606 620 * address space area. 621 * 607 622 */ 608 623 break; … … 613 628 * to b and c overlaps with the resized 614 629 * address space area. 630 * 615 631 */ 616 632 … … 653 669 /* 654 670 * Finish TLB shootdown sequence. 671 * 655 672 */ 656 673 … … 660 677 /* 661 678 * Invalidate software translation caches (e.g. TSB on sparc64). 679 * 662 680 */ 663 681 as_invalidate_translation_cache(as, area->base + … … 670 688 * Growing the area. 671 689 * Check for overlaps with other address space areas. 690 * 672 691 */ 673 692 if (!check_area_conflicts(as, address, pages * PAGE_SIZE, … … 790 809 /* 791 810 * Finish TLB shootdown sequence. 811 * 792 812 */ 793 813 … … 797 817 * Invalidate potential software translation caches (e.g. TSB on 798 818 * sparc64). 819 * 799 820 */ 800 821 as_invalidate_translation_cache(as, area->base, area->pages); … … 814 835 /* 815 836 * Remove the empty area from address space. 837 * 816 838 */ 817 839 btree_remove(&as->as_area_btree, base, NULL); … … 855 877 /* 856 878 * Could not find the source address space area. 879 * 857 880 */ 858 881 mutex_unlock(&src_as->lock); … … 864 887 * There is no backend or the backend does not 865 888 * know how to share the area. 889 * 866 890 */ 867 891 mutex_unlock(&src_area->lock); … … 890 914 * First, prepare the area for sharing. 891 915 * Then it will be safe to unlock it. 916 * 892 917 */ 893 918 share_info_t *sh_info = src_area->sh_info; … … 901 926 /* 902 927 * Call the backend to setup sharing. 928 * 903 929 */ 904 930 src_area->backend->share(src_area); … … 919 945 * The flags of the source area are masked against dst_flags_mask 920 946 * to support sharing in less privileged mode. 947 * 921 948 */ 922 949 as_area_t *dst_area = as_area_create(dst_as, dst_flags_mask, src_size, … … 935 962 * fully initialized. Clear the AS_AREA_ATTR_PARTIAL 936 963 * attribute and set the sh_info. 964 * 937 965 */ 938 966 mutex_lock(&dst_as->lock); … … 957 985 NO_TRACE bool as_area_check_access(as_area_t *area, pf_access_t access) 958 986 { 959 ASSERT(mutex_locked(&area->lock));960 961 987 int flagmap[] = { 962 988 [PF_ACCESS_READ] = AS_AREA_READ, … … 964 990 [PF_ACCESS_EXEC] = AS_AREA_EXEC 965 991 }; 992 993 ASSERT(mutex_locked(&area->lock)); 966 994 967 995 if (!(area->flags & flagmap[access])) … … 1034 1062 /* 1035 1063 * Compute total number of used pages in the used_space B+tree 1064 * 1036 1065 */ 1037 1066 size_t used_pages = 0; … … 1055 1084 /* 1056 1085 * Start TLB shootdown sequence. 1086 * 1057 1087 */ 1058 1088 ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, … … 1062 1092 * Remove used pages from page tables and remember their frame 1063 1093 * numbers. 1094 * 1064 1095 */ 1065 1096 size_t frame_idx = 0; … … 1092 1123 /* 1093 1124 * Finish TLB shootdown sequence. 1125 * 1094 1126 */ 1095 1127 … … 1099 1131 * Invalidate potential software translation caches (e.g. TSB on 1100 1132 * sparc64). 1133 * 1101 1134 */ 1102 1135 as_invalidate_translation_cache(as, area->base, area->pages); … … 1180 1213 * No area contained mapping for 'page'. 1181 1214 * Signal page fault to low-level handler. 1215 * 1182 1216 */ 1183 1217 mutex_unlock(&AS->lock); … … 1199 1233 * The address space area is not backed by any backend 1200 1234 * or the backend cannot handle page faults. 1235 * 1201 1236 */ 1202 1237 mutex_unlock(&area->lock); … … 1210 1245 * To avoid race condition between two page faults on the same address, 1211 1246 * we need to make sure the mapping has not been already inserted. 1247 * 1212 1248 */ 1213 1249 pte_t *pte; … … 1227 1263 /* 1228 1264 * Resort to the backend page fault handler. 1265 * 1229 1266 */ 1230 1267 if (area->backend->page_fault(area, page, access) != AS_PF_OK) { … … 1281 1318 * preemption is disabled. We should not be 1282 1319 * holding any other lock. 1320 * 1283 1321 */ 1284 1322 (void) interrupts_enable(); … … 1300 1338 * list of inactive address spaces with assigned 1301 1339 * ASID. 1340 * 1302 1341 */ 1303 1342 ASSERT(old_as->asid != ASID_INVALID); … … 1310 1349 * Perform architecture-specific tasks when the address space 1311 1350 * is being removed from the CPU. 1351 * 1312 1352 */ 1313 1353 as_deinstall_arch(old_as); … … 1316 1356 /* 1317 1357 * Second, prepare the new address space. 1358 * 1318 1359 */ 1319 1360 if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { … … 1331 1372 * Perform architecture-specific steps. 1332 1373 * (e.g. write ASID to hardware register etc.) 1374 * 1333 1375 */ 1334 1376 as_install_arch(new_as); … … 1349 1391 { 1350 1392 ASSERT(mutex_locked(&area->lock)); 1351 1393 1352 1394 return area_flags_to_page_flags(area->flags); 1353 1395 } … … 1470 1512 * @param count Number of page to be marked. 1471 1513 * 1472 * @return False on failure or trueon success.1473 * 1474 */ 1475 boolused_space_insert(as_area_t *area, uintptr_t page, size_t count)1514 * @return Zero on failure and non-zero on success. 1515 * 1516 */ 1517 int used_space_insert(as_area_t *area, uintptr_t page, size_t count) 1476 1518 { 1477 1519 ASSERT(mutex_locked(&area->lock)); … … 1484 1526 /* 1485 1527 * We hit the beginning of some used space. 1486 */ 1487 return false; 1528 * 1529 */ 1530 return 0; 1488 1531 } 1489 1532 1490 1533 if (!leaf->keys) { 1491 1534 btree_insert(&area->used_space, page, (void *) count, leaf); 1492 goto success;1535 return 1; 1493 1536 } 1494 1537 … … 1504 1547 * somewhere between the rightmost interval of 1505 1548 * the left neigbour and the first interval of the leaf. 1549 * 1506 1550 */ 1507 1551 … … 1511 1555 left_cnt * PAGE_SIZE)) { 1512 1556 /* The interval intersects with the left interval. */ 1513 return false;1557 return 0; 1514 1558 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1515 1559 right_cnt * PAGE_SIZE)) { 1516 1560 /* The interval intersects with the right interval. */ 1517 return false;1561 return 0; 1518 1562 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1519 1563 (page + count * PAGE_SIZE == right_pg)) { … … 1521 1565 * The interval can be added by merging the two already 1522 1566 * present intervals. 1567 * 1523 1568 */ 1524 1569 node->value[node->keys - 1] += count + right_cnt; 1525 1570 btree_remove(&area->used_space, right_pg, leaf); 1526 goto success;1571 return 1; 1527 1572 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1528 1573 /* 1529 1574 * The interval can be added by simply growing the left 1530 1575 * interval. 1576 * 1531 1577 */ 1532 1578 node->value[node->keys - 1] += count; 1533 goto success;1579 return 1; 1534 1580 } else if (page + count * PAGE_SIZE == right_pg) { 1535 1581 /* … … 1537 1583 * the right interval down and increasing its size 1538 1584 * accordingly. 1585 * 1539 1586 */ 1540 1587 leaf->value[0] += count; 1541 1588 leaf->key[0] = page; 1542 goto success;1589 return 1; 1543 1590 } else { 1544 1591 /* 1545 1592 * The interval is between both neigbouring intervals, 1546 1593 * but cannot be merged with any of them. 1594 * 1547 1595 */ 1548 1596 btree_insert(&area->used_space, page, (void *) count, 1549 1597 leaf); 1550 goto success;1598 return 1; 1551 1599 } 1552 1600 } else if (page < leaf->key[0]) { … … 1557 1605 * Investigate the border case in which the left neighbour does 1558 1606 * not exist but the interval fits from the left. 1607 * 1559 1608 */ 1560 1609 … … 1562 1611 right_cnt * PAGE_SIZE)) { 1563 1612 /* The interval intersects with the right interval. */ 1564 return false;1613 return 0; 1565 1614 } else if (page + count * PAGE_SIZE == right_pg) { 1566 1615 /* … … 1568 1617 * right interval down and increasing its size 1569 1618 * accordingly. 1619 * 1570 1620 */ 1571 1621 leaf->key[0] = page; 1572 1622 leaf->value[0] += count; 1573 goto success;1623 return 1; 1574 1624 } else { 1575 1625 /* 1576 1626 * The interval doesn't adjoin with the right interval. 1577 1627 * It must be added individually. 1628 * 1578 1629 */ 1579 1630 btree_insert(&area->used_space, page, (void *) count, 1580 1631 leaf); 1581 goto success;1632 return 1; 1582 1633 } 1583 1634 } … … 1594 1645 * somewhere between the leftmost interval of 1595 1646 * the right neigbour and the last interval of the leaf. 1647 * 1596 1648 */ 1597 1649 … … 1601 1653 left_cnt * PAGE_SIZE)) { 1602 1654 /* The interval intersects with the left interval. */ 1603 return false;1655 return 0; 1604 1656 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1605 1657 right_cnt * PAGE_SIZE)) { 1606 1658 /* The interval intersects with the right interval. */ 1607 return false;1659 return 0; 1608 1660 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1609 1661 (page + count * PAGE_SIZE == right_pg)) { … … 1611 1663 * The interval can be added by merging the two already 1612 1664 * present intervals. 1665 * 1613 1666 */ 1614 1667 leaf->value[leaf->keys - 1] += count + right_cnt; 1615 1668 btree_remove(&area->used_space, right_pg, node); 1616 goto success;1669 return 1; 1617 1670 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1618 1671 /* 1619 1672 * The interval can be added by simply growing the left 1620 1673 * interval. 1674 * 1621 1675 */ 1622 leaf->value[leaf->keys - 1] += count;1623 goto success;1676 leaf->value[leaf->keys - 1] += count; 1677 return 1; 1624 1678 } else if (page + count * PAGE_SIZE == right_pg) { 1625 1679 /* … … 1627 1681 * the right interval down and increasing its size 1628 1682 * accordingly. 1683 * 1629 1684 */ 1630 1685 node->value[0] += count; 1631 1686 node->key[0] = page; 1632 goto success;1687 return 1; 1633 1688 } else { 1634 1689 /* 1635 1690 * The interval is between both neigbouring intervals, 1636 1691 * but cannot be merged with any of them. 1692 * 1637 1693 */ 1638 1694 btree_insert(&area->used_space, page, (void *) count, 1639 1695 leaf); 1640 goto success;1696 return 1; 1641 1697 } 1642 1698 } else if (page >= leaf->key[leaf->keys - 1]) { … … 1647 1703 * Investigate the border case in which the right neighbour 1648 1704 * does not exist but the interval fits from the right. 1705 * 1649 1706 */ 1650 1707 … … 1652 1709 left_cnt * PAGE_SIZE)) { 1653 1710 /* The interval intersects with the left interval. */ 1654 return false;1711 return 0; 1655 1712 } else if (left_pg + left_cnt * PAGE_SIZE == page) { 1656 1713 /* 1657 1714 * The interval can be added by growing the left 1658 1715 * interval. 1716 * 1659 1717 */ 1660 1718 leaf->value[leaf->keys - 1] += count; 1661 goto success;1719 return 1; 1662 1720 } else { 1663 1721 /* 1664 1722 * The interval doesn't adjoin with the left interval. 1665 1723 * It must be added individually. 1724 * 1666 1725 */ 1667 1726 btree_insert(&area->used_space, page, (void *) count, 1668 1727 leaf); 1669 goto success;1728 return 1; 1670 1729 } 1671 1730 } … … 1675 1734 * only between two other intervals of the leaf. The two border cases 1676 1735 * were already resolved. 1736 * 1677 1737 */ 1678 1738 btree_key_t i; … … 1686 1746 /* 1687 1747 * The interval fits between left_pg and right_pg. 1748 * 1688 1749 */ 1689 1750 … … 1693 1754 * The interval intersects with the left 1694 1755 * interval. 1756 * 1695 1757 */ 1696 return false;1758 return 0; 1697 1759 } else if (overlaps(page, count * PAGE_SIZE, right_pg, 1698 1760 right_cnt * PAGE_SIZE)) { … … 1700 1762 * The interval intersects with the right 1701 1763 * interval. 1764 * 1702 1765 */ 1703 return false;1766 return 0; 1704 1767 } else if ((page == left_pg + left_cnt * PAGE_SIZE) && 1705 1768 (page + count * PAGE_SIZE == right_pg)) { … … 1707 1770 * The interval can be added by merging the two 1708 1771 * already present intervals. 1772 * 1709 1773 */ 1710 1774 leaf->value[i - 1] += count + right_cnt; 1711 1775 btree_remove(&area->used_space, right_pg, leaf); 1712 goto success;1776 return 1; 1713 1777 } else if (page == left_pg + left_cnt * PAGE_SIZE) { 1714 1778 /* 1715 1779 * The interval can be added by simply growing 1716 1780 * the left interval. 1781 * 1717 1782 */ 1718 1783 leaf->value[i - 1] += count; 1719 goto success;1784 return 1; 1720 1785 } else if (page + count * PAGE_SIZE == right_pg) { 1721 1786 /* … … 1723 1788 * base of the right interval down and 1724 1789 * increasing its size accordingly. 1790 * 1725 1791 */ 1726 1792 leaf->value[i] += count; 1727 1793 leaf->key[i] = page; 1728 goto success;1794 return 1; 1729 1795 } else { 1730 1796 /* … … 1732 1798 * intervals, but cannot be merged with any of 1733 1799 * them. 1800 * 1734 1801 */ 1735 1802 btree_insert(&area->used_space, page, 1736 1803 (void *) count, leaf); 1737 goto success;1804 return 1; 1738 1805 } 1739 1806 } … … 1742 1809 panic("Inconsistency detected while adding %zu pages of used " 1743 1810 "space at %p.", count, (void *) page); 1744 1745 success:1746 area->resident += count;1747 return true;1748 1811 } 1749 1812 … … 1756 1819 * @param count Number of page to be marked. 1757 1820 * 1758 * @return False on failure or trueon success.1759 * 1760 */ 1761 boolused_space_remove(as_area_t *area, uintptr_t page, size_t count)1821 * @return Zero on failure and non-zero on success. 1822 * 1823 */ 1824 int used_space_remove(as_area_t *area, uintptr_t page, size_t count) 1762 1825 { 1763 1826 ASSERT(mutex_locked(&area->lock)); … … 1770 1833 /* 1771 1834 * We are lucky, page is the beginning of some interval. 1835 * 1772 1836 */ 1773 1837 if (count > pages) { 1774 return false;1838 return 0; 1775 1839 } else if (count == pages) { 1776 1840 btree_remove(&area->used_space, page, leaf); 1777 goto success;1841 return 1; 1778 1842 } else { 1779 1843 /* 1780 1844 * Find the respective interval. 1781 1845 * Decrease its size and relocate its start address. 1846 * 1782 1847 */ 1783 1848 btree_key_t i; … … 1786 1851 leaf->key[i] += count * PAGE_SIZE; 1787 1852 leaf->value[i] -= count; 1788 goto success;1853 return 1; 1789 1854 } 1790 1855 } 1791 1792 1856 goto error; 1793 1857 } … … 1808 1872 * removed by updating the size of the bigger 1809 1873 * interval. 1874 * 1810 1875 */ 1811 1876 node->value[node->keys - 1] -= count; 1812 goto success;1877 return 1; 1813 1878 } else if (page + count * PAGE_SIZE < 1814 1879 left_pg + left_cnt*PAGE_SIZE) { … … 1819 1884 * the original interval and also inserting a 1820 1885 * new interval. 1886 * 1821 1887 */ 1822 1888 size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - … … 1825 1891 btree_insert(&area->used_space, page + 1826 1892 count * PAGE_SIZE, (void *) new_cnt, leaf); 1827 goto success;1893 return 1; 1828 1894 } 1829 1895 } 1830 1831 return false; 1896 return 0; 1832 1897 } else if (page < leaf->key[0]) 1833 return false;1898 return 0; 1834 1899 1835 1900 if (page > leaf->key[leaf->keys - 1]) { … … 1845 1910 * interval of the leaf and can be removed by 1846 1911 * updating the size of the bigger interval. 1912 * 1847 1913 */ 1848 1914 leaf->value[leaf->keys - 1] -= count; 1849 goto success;1915 return 1; 1850 1916 } else if (page + count * PAGE_SIZE < left_pg + 1851 1917 left_cnt * PAGE_SIZE) { … … 1856 1922 * original interval and also inserting a new 1857 1923 * interval. 1924 * 1858 1925 */ 1859 1926 size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - … … 1862 1929 btree_insert(&area->used_space, page + 1863 1930 count * PAGE_SIZE, (void *) new_cnt, leaf); 1864 goto success;1931 return 1; 1865 1932 } 1866 1933 } 1867 1868 return false; 1934 return 0; 1869 1935 } 1870 1936 1871 1937 /* 1872 1938 * The border cases have been already resolved. 1873 * Now the interval can be only between intervals of the leaf. 1939 * Now the interval can be only between intervals of the leaf. 1874 1940 */ 1875 1941 btree_key_t i; … … 1892 1958 * be removed by updating the size of 1893 1959 * the bigger interval. 1960 * 1894 1961 */ 1895 1962 leaf->value[i - 1] -= count; 1896 goto success;1963 return 1; 1897 1964 } else if (page + count * PAGE_SIZE < 1898 1965 left_pg + left_cnt * PAGE_SIZE) { … … 1912 1979 count * PAGE_SIZE, (void *) new_cnt, 1913 1980 leaf); 1914 goto success;1981 return 1; 1915 1982 } 1916 1983 } 1917 1918 return false; 1984 return 0; 1919 1985 } 1920 1986 } … … 1923 1989 panic("Inconsistency detected while removing %zu pages of used " 1924 1990 "space from %p.", count, (void *) page); 1925 1926 success:1927 area->resident -= count;1928 return true;1929 1991 } 1930 1992
Note:
See TracChangeset
for help on using the changeset viewer.
