diff --git a/sys/arch/aarch64/aarch64/aarch64_tlb.c b/sys/arch/aarch64/aarch64/aarch64_tlb.c index 76b6dfceb3f3..6b378109e77f 100644 --- a/sys/arch/aarch64/aarch64/aarch64_tlb.c +++ b/sys/arch/aarch64/aarch64/aarch64_tlb.c @@ -46,9 +46,11 @@ tlb_asid_t tlb_get_asid(void) { - return __SHIFTOUT(reg_ttbr0_el1_read(), TTBR_ASID); + return (reg_tcr_el1_read() & TCR_EPD0) != 0 ? + KERNEL_PID :__SHIFTOUT(reg_ttbr0_el1_read(), TTBR_ASID); } +#if 0 void tlb_set_asid(tlb_asid_t asid, pmap_t pm) { @@ -58,6 +60,7 @@ tlb_set_asid(tlb_asid_t asid, pmap_t pm) cpu_set_ttbr0(ttbr); } +#endif void tlb_invalidate_all(void) diff --git a/sys/arch/aarch64/aarch64/genassym.cf b/sys/arch/aarch64/aarch64/genassym.cf index 85c47e21d368..7da36242483f 100644 --- a/sys/arch/aarch64/aarch64/genassym.cf +++ b/sys/arch/aarch64/aarch64/genassym.cf @@ -360,6 +360,7 @@ define SCTLR_nAA SCTLR_nAA define SCTLR_ITD SCTLR_ITD define SCTLR_SED SCTLR_SED define SCTLR_UMA SCTLR_UMA +define SCTLR_EOS SCTLR_EOS define SCTLR_I SCTLR_I define SCTLR_DZE SCTLR_DZE define SCTLR_UCT SCTLR_UCT diff --git a/sys/arch/aarch64/aarch64/locore.S b/sys/arch/aarch64/aarch64/locore.S index 6d7ae1bd2d2e..a4ede19ef55d 100644 --- a/sys/arch/aarch64/aarch64/locore.S +++ b/sys/arch/aarch64/aarch64/locore.S @@ -1015,6 +1015,7 @@ sctlr_set: SCTLR_SED | /* SETEND instruction disable */ \ SCTLR_C | /* Cache enable */ \ SCTLR_M | /* MMU Enable */ \ + SCTLR_EOS | /* RES1 if no FEAT_ExS */ \ SCTLR_SA0_CONFIG | \ SCTLR_SA_CONFIG | \ SCTLR_A_CONFIG | \ diff --git a/sys/arch/aarch64/aarch64/pmap.c b/sys/arch/aarch64/aarch64/pmap.c index 53063d9dc92f..5e498b5d1af0 100644 --- a/sys/arch/aarch64/aarch64/pmap.c +++ b/sys/arch/aarch64/aarch64/pmap.c @@ -1387,6 +1387,57 @@ pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) pm_unlock(pm); } +void +pmap_md_xtab_activate(pmap_t pm, struct lwp *l) +{ + + KASSERT(kpreempt_disabled()); + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); + + /* + * Assume that TTBR1 has only global mappings and TTBR0 only + * has non-global mappings. To prevent speculation from doing + * evil things we disable translation table walks using TTBR0 + * before setting the CONTEXTIDR (ASID) or new TTBR0 value. + * Once both are set, table walks are reenabled. + */ + + if (pm == pmap_kernel()) + return; + +// struct cpu_info * const ci = curcpu(); + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); + + const uint64_t ttbr = + __SHIFTIN(pai->pai_asid, TTBR_ASID) | + __SHIFTIN(pmap_l0pa(pm), TTBR_BADDR); + + cpu_set_ttbr0(ttbr); + + uint64_t tcrel1 = reg_tcr_el1_read(); + reg_tcr_el1_write(tcrel1 & ~TCR_EPD0); + // XXXNH don't need this when EOS=1 for non-efirt. + isb(); +} + + +void +pmap_md_xtab_deactivate(pmap_t pm) +{ + KASSERT(kpreempt_disabled()); + +// struct cpu_info * const ci = curcpu(); + + /* + * Disable translation table walks from TTBR0 while no pmap has been + * activated. + */ + const uint64_t old_tcrel1 = reg_tcr_el1_read(); + reg_tcr_el1_write(old_tcrel1 | TCR_EPD0); + isb(); + +} + #if defined(EFI_RUNTIME) void pmap_activate_efirt(void) @@ -1401,14 +1452,8 @@ pmap_activate_efirt(void) KASSERT(kpreempt_disabled()); ci->ci_pmap_asid_cur = pai->pai_asid; - UVMHIST_LOG(pmaphist, "setting asid to %#jx", pai->pai_asid, - 0, 0, 0); - tlb_set_asid(pai->pai_asid, pm); - - /* Re-enable translation table walks using TTBR0 */ - uint64_t tcr = reg_tcr_el1_read(); - reg_tcr_el1_write(tcr & ~TCR_EPD0); - isb(); + UVMHIST_LOG(pmaphist, "setting asid to %#jx", pai->pai_asid, 0, 0, 0); + pmap_md_xtab_activate(pm, NULL); pm->pm_activated = true; PMAP_COUNT(activate); @@ -1419,7 +1464,6 @@ void pmap_activate(struct lwp *l) { struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap; - uint64_t tcr; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, kernel=%u)", l, @@ -1435,17 +1479,9 @@ pmap_activate(struct lwp *l) KASSERT(pm->pm_l0table != NULL); - /* this calls tlb_set_asid which calls cpu_set_ttbr0 */ + /* this calls pmap_md_xtab_activate */ pmap_tlb_asid_acquire(pm, l); - UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, - PMAP_PAI(pm, cpu_tlb_info(ci))->pai_asid, 0, 0); - - /* Re-enable translation table walks using TTBR0 */ - tcr = reg_tcr_el1_read(); - reg_tcr_el1_write(tcr & ~TCR_EPD0); - isb(); - pm->pm_activated = true; PMAP_COUNT(activate); @@ -1462,19 +1498,12 @@ pmap_deactivate_efirt(void) KASSERT(kpreempt_disabled()); - /* Disable translation table walks using TTBR0 */ - uint64_t tcr = reg_tcr_el1_read(); - reg_tcr_el1_write(tcr | TCR_EPD0); - isb(); - - UVMHIST_LOG(pmaphist, "setting asid to %#jx", KERNEL_PID, - 0, 0, 0); - ci->ci_pmap_asid_cur = KERNEL_PID; - tlb_set_asid(KERNEL_PID, pmap_kernel()); - + pmap_md_xtab_deactivate(pm); pm->pm_activated = false; + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); + PMAP_COUNT(deactivate); } #endif @@ -1483,26 +1512,21 @@ void pmap_deactivate(struct lwp *l) { struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap; - uint64_t tcr; UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, (kernel=%u))", l, l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0); KASSERT(kpreempt_disabled()); - - /* Disable translation table walks using TTBR0 */ - tcr = reg_tcr_el1_read(); - reg_tcr_el1_write(tcr | TCR_EPD0); - isb(); - UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, PMAP_PAI(pm, cpu_tlb_info(ci))->pai_asid, 0, 0); +// ci->ci_pmap_asid_cur = KERNEL_PID; pmap_tlb_asid_deactivate(pm); - pm->pm_activated = false; + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); + PMAP_COUNT(deactivate); } diff --git a/sys/arch/aarch64/aarch64/pmap_machdep.c b/sys/arch/aarch64/aarch64/pmap_machdep.c index ad7088ee2cc7..5005d9a1948a 100644 --- a/sys/arch/aarch64/aarch64/pmap_machdep.c +++ b/sys/arch/aarch64/aarch64/pmap_machdep.c @@ -525,19 +525,26 @@ pmap_md_xtab_activate(pmap_t pm, struct lwp *l) KASSERT(kpreempt_disabled()); + struct cpu_info * const ci = curcpu(); + /* * Assume that TTBR1 has only global mappings and TTBR0 only * has non-global mappings. To prevent speculation from doing * evil things we disable translation table walks using TTBR0 - * before setting the CONTEXTIDR (ASID) or new TTBR0 value. + * before setting the ASID or new translation table base. + * These are set in the same system register (TTBR0_EL1). * Once both are set, table walks are reenabled. */ + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); - const uint64_t old_tcrel1 = reg_tcr_el1_read(); - reg_tcr_el1_write(old_tcrel1 | TCR_EPD0); - isb(); + if (pm == pmap_kernel()) { + KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", + ci->ci_pmap_asid_cur); + KASSERTMSG(ci->ci_pmap_cur == pmap_kernel(), + "ci_pmap_cur %p", ci->ci_pmap_cur); + return; + } - struct cpu_info * const ci = curcpu(); struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); const uint64_t ttbr = @@ -546,10 +553,10 @@ pmap_md_xtab_activate(pmap_t pm, struct lwp *l) cpu_set_ttbr0(ttbr); - if (pm != pmap_kernel()) { - reg_tcr_el1_write(old_tcrel1 & ~TCR_EPD0); - isb(); - } + const uint64_t old_tcrel1 = reg_tcr_el1_read(); + reg_tcr_el1_write(old_tcrel1 & ~TCR_EPD0); + // XXXNH don't need this when EOS=1 for non-efirt. + isb(); UVMHIST_LOG(maphist, " pm %#jx pm->pm_l0 %016jx pm->pm_l0_pa %016jx asid %ju... done", (uintptr_t)pm, (uintptr_t)pm->pm_pdetab, (uintptr_t)pm->pm_l0_pa, @@ -564,7 +571,8 @@ pmap_md_xtab_activate(pmap_t pm, struct lwp *l) void pmap_md_xtab_deactivate(pmap_t pm) { - UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); + UVMHIST_FUNC(__func__); + UVMHIST_CALLARGS(pmaphist, " (pm=%#jx)", (uintptr_t)pm, 0, 0, 0); KASSERT(kpreempt_disabled()); @@ -577,11 +585,10 @@ pmap_md_xtab_deactivate(pmap_t pm) reg_tcr_el1_write(old_tcrel1 | TCR_EPD0); isb(); - cpu_set_ttbr0(0); - ci->ci_pmap_cur = pmap_kernel(); KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", ci->ci_pmap_asid_cur); + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); } @@ -593,11 +600,14 @@ pmap_md_activate_efirt(void) pmap_md_xtab_activate(pmap_efirt(), NULL); } + void pmap_md_deactivate_efirt(void) { pmap_md_xtab_deactivate(pmap_efirt()); + KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); + kpreempt_enable(); } #endif @@ -625,7 +635,16 @@ pmap_md_page_syncicache(struct vm_page_md *mdpg, const kcpuset_t *onproc) { UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); - //XXXNH + /* + * This handles the PIPT case where we can use the direct map + * to sync the icache without worrying about aliasing. + */ + + struct vm_page *pg = VM_MD_TO_PAGE(mdpg); + const paddr_t pa = VM_PAGE_TO_PHYS(pg); + const vaddr_t kva = pmap_md_direct_map_paddr(pa); + cpu_icache_sync_range(kva, PAGE_SIZE); + } diff --git a/sys/arch/aarch64/include/pmap.h b/sys/arch/aarch64/include/pmap.h index 52f909696cf8..8fd88787efb6 100644 --- a/sys/arch/aarch64/include/pmap.h +++ b/sys/arch/aarch64/include/pmap.h @@ -246,6 +246,9 @@ void pmap_db_pteinfo(vaddr_t, void (*)(const char *, ...) __printflike(1, 2)); void pmap_db_ttbrdump(bool, vaddr_t, void (*)(const char *, ...) __printflike(1, 2)); #endif +void pmap_md_xtab_activate(pmap_t, struct lwp *); +void pmap_md_xtab_deactivate(pmap_t); + #define LX_BLKPAG_OS_WIRED LX_BLKPAG_OS_2 #define LX_BLKPAG_OS_BOOT LX_BLKPAG_OS_3 diff --git a/sys/arch/aarch64/include/pmap_machdep.h b/sys/arch/aarch64/include/pmap_machdep.h index 2f6d58bb3252..48bd3fcca154 100644 --- a/sys/arch/aarch64/include/pmap_machdep.h +++ b/sys/arch/aarch64/include/pmap_machdep.h @@ -156,8 +156,6 @@ void pmap_md_vca_clean(struct vm_page_md *, int); void pmap_md_vca_remove(struct vm_page_md *, vaddr_t, bool, bool); bool pmap_md_ok_to_steal_p(const uvm_physseg_t, size_t); -void pmap_md_xtab_activate(pmap_t, struct lwp *); -void pmap_md_xtab_deactivate(pmap_t); vaddr_t pmap_md_direct_map_paddr(paddr_t); @@ -289,7 +287,8 @@ pte_nv_entry(bool kernel_p) static inline pt_entry_t pte_prot_downgrade(pt_entry_t pte, vm_prot_t prot) { - +// XXXNH +// XXXNH exec? return (pte & ~LX_BLKPAG_AP) | (((prot) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ ? LX_BLKPAG_AP_RO : LX_BLKPAG_AP_RW); } diff --git a/sys/arch/arm/arm32/arm32_tlb.c b/sys/arch/arm/arm32/arm32_tlb.c index a0bfc390ce14..d10aa212bf66 100644 --- a/sys/arch/arm/arm32/arm32_tlb.c +++ b/sys/arch/arm/arm32/arm32_tlb.c @@ -46,9 +46,11 @@ bool arm_has_mpext_p; // CPU supports MP extensions tlb_asid_t tlb_get_asid(void) { - return armreg_contextidr_read() & 0xff; + return (armreg_ttbcr_read() & TTBCR_S_PD0) != 0 ? + KERNEL_PID : (armreg_contextidr_read() & 0xff); } +#if 0 void tlb_set_asid(tlb_asid_t asid, pmap_t pm) { @@ -60,6 +62,7 @@ tlb_set_asid(tlb_asid_t asid, pmap_t pm) armreg_contextidr_write(asid); isb(); } +#endif void tlb_invalidate_all(void) diff --git a/sys/arch/arm/arm32/pmap.c b/sys/arch/arm/arm32/pmap.c index 5711952ed915..7ecb9c8884aa 100644 --- a/sys/arch/arm/arm32/pmap.c +++ b/sys/arch/arm/arm32/pmap.c @@ -470,11 +470,6 @@ EVCNT_ATTACH_STATIC(pmap_ev_activations); #define PMAPCOUNT(x) ((void)0) #endif -#ifdef ARM_MMU_EXTENDED -void pmap_md_pdetab_activate(pmap_t, struct lwp *); -void pmap_md_pdetab_deactivate(pmap_t pm); -#endif - /* * pmap copy/zero page, and mem(5) hook point */ @@ -4915,38 +4910,47 @@ pmap_unwire(pmap_t pm, vaddr_t va) #ifdef ARM_MMU_EXTENDED void -pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) +pmap_md_xtab_activate(pmap_t pm, struct lwp *l) { UVMHIST_FUNC(__func__); + UVMHIST_CALLARGS(maphist, " (pm=%#jx l=%#jx)", (uintptr_t)pm, (uintptr_t)l, 0, 0); + + KASSERT(kpreempt_disabled()); + KASSERT((armreg_ttbcr_read() & TTBCR_S_PD0) != 0); + struct cpu_info * const ci = curcpu(); struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); - UVMHIST_CALLARGS(maphist, "pm %#jx (pm->pm_l1_pa %08jx asid %ju)", - (uintptr_t)pm, pm->pm_l1_pa, pai->pai_asid, 0); - /* * Assume that TTBR1 has only global mappings and TTBR0 only * has non-global mappings. To prevent speculation from doing - * evil things we disable translation table walks using TTBR0 + * evil thingstranslation table walks using TTBR0 are disabled * before setting the CONTEXTIDR (ASID) or new TTBR0 value. * Once both are set, table walks are reenabled. */ - const uint32_t old_ttbcr = armreg_ttbcr_read(); - armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); - isb(); - pmap_tlb_asid_acquire(pm, l); + if (pm == pmap_kernel()) { + KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", + ci->ci_pmap_asid_cur); + KASSERTMSG(ci->ci_pmap_cur == pmap_kernel(), + "ci_pmap_cur %p", ci->ci_pmap_cur); + UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0); - cpu_setttb(pm->pm_l1_pa, pai->pai_asid); + return; + } + armreg_contextidr_write(pai->pai_asid); + armreg_ttbr_write(pm->pm_l1_pa | + (ci->ci_mpidr ? TTBR_MPATTR : TTBR_UPATTR)); + + // cpu_setttb(pm->pm_l1_pa, pai->pai_asid); /* * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 * have been updated. */ isb(); - if (pm != pmap_kernel()) { - armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); - } + const uint32_t old_ttbcr = armreg_ttbcr_read(); + armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); cpu_cpwait(); KASSERTMSG(ci->ci_pmap_asid_cur == pai->pai_asid, "%u vs %u", @@ -4954,14 +4958,16 @@ pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) ci->ci_pmap_cur = pm; } + void -pmap_md_pdetab_deactivate(pmap_t pm) +pmap_md_xtab_deactivate(pmap_t pm) { UVMHIST_FUNC(__func__); UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); - kpreempt_disable(); + KASSERT(kpreempt_disabled()); + struct cpu_info * const ci = curcpu(); /* * Disable translation table walks from TTBR0 while no pmap has been @@ -4970,14 +4976,15 @@ pmap_md_pdetab_deactivate(pmap_t pm) const uint32_t old_ttbcr = armreg_ttbcr_read(); armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); isb(); - pmap_tlb_asid_deactivate(pm); + + /* not really needed */ cpu_setttb(pmap_kernel()->pm_l1_pa, KERNEL_PID); isb(); ci->ci_pmap_cur = pmap_kernel(); KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", ci->ci_pmap_asid_cur); - kpreempt_enable(); + } #endif @@ -5143,7 +5150,9 @@ pmap_activate(struct lwp *l) #endif #ifdef ARM_MMU_EXTENDED - pmap_md_pdetab_activate(npm, l); + /* this calls pmap_md_xtab_activate */ + pmap_tlb_asid_acquire(npm, l); + #else cpu_domains(ndacr); if (npm == pmap_kernel() || npm == rpm) { @@ -5203,7 +5212,7 @@ pmap_deactivate(struct lwp *l) #ifdef ARM_MMU_EXTENDED KASSERT(kpreempt_disabled()); - pmap_md_pdetab_deactivate(pm); + pmap_tlb_asid_deactivate(pm); #else /* * If the process is exiting, make sure pmap_activate() does @@ -5298,7 +5307,8 @@ pmap_update(pmap_t pm) pm->pm_remove_all = false; KASSERT(pm != pmap_kernel()); - pmap_md_pdetab_activate(pm, curlwp); + // XXXNH + pmap_md_xtab_activate(pm, curlwp); } if (arm_has_mpext_p) diff --git a/sys/arch/arm/include/arm32/pmap.h b/sys/arch/arm/include/arm32/pmap.h index 0daa1dec78a4..b309fef5614e 100644 --- a/sys/arch/arm/include/arm32/pmap.h +++ b/sys/arch/arm/include/arm32/pmap.h @@ -370,6 +370,9 @@ void pmap_prefer(vaddr_t, vaddr_t *, int); int pmap_maxproc_set(int); struct pmap * pmap_efirt(void); + +void pmap_md_xtab_activate(pmap_t, struct lwp *); +void pmap_md_xtab_deactivate(pmap_t pm); #endif void pmap_icache_sync_range(pmap_t, vaddr_t, vaddr_t); diff --git a/sys/arch/evbarm/conf/GENERIC64_PMAPMI b/sys/arch/evbarm/conf/GENERIC64_PMAPMI index 9872befe8cd0..62c030dcf2d8 100644 --- a/sys/arch/evbarm/conf/GENERIC64_PMAPMI +++ b/sys/arch/evbarm/conf/GENERIC64_PMAPMI @@ -3,6 +3,19 @@ include "arch/evbarm/conf/GENERIC64" options PMAP_MI +# Hardware management of the Access flag and dirty state (HAFDBS). +no options ARMV81_HAFDBS + +# Privileged Access Never (PAN). +no options ARMV81_PAN + +# Pointer Authentication (PAC). +#makeoptions ARMV83_PAC=1 +#options ARMV83_PAC + +# Branch Target Identification (BTI). +#makeoptions ARMV85_BTI=1 +#options ARMV85_BTI #options VERBOSE_INIT_ARM # verbose bootstrapping messages diff --git a/sys/arch/m68k/include/pmap_coldfire.h b/sys/arch/m68k/include/pmap_coldfire.h index 1fd56723f451..f7c98c7f253f 100644 --- a/sys/arch/m68k/include/pmap_coldfire.h +++ b/sys/arch/m68k/include/pmap_coldfire.h @@ -139,15 +139,16 @@ pmap_md_tlb_asid_max(void) static inline void pmap_md_xtab_activate(struct pmap *pm, struct lwp *l) { + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); - /* nothing */ + tlb_set_asid(pai->pai_asid, pm); } static inline void pmap_md_xtab_deactivate(struct pmap *pm) { - /* nothing */ + tlb_set_asid(KERNEL_PID, pmap_kernel()); } #endif diff --git a/sys/arch/mips/include/pmap.h b/sys/arch/mips/include/pmap.h index cf9acfa6675e..f5f65a143865 100644 --- a/sys/arch/mips/include/pmap.h +++ b/sys/arch/mips/include/pmap.h @@ -145,15 +145,18 @@ pmap_md_cache_prefer_mask(void) static inline void pmap_md_xtab_activate(struct pmap *pm, struct lwp *l) { +#if notyet + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); - /* nothing */ + tlb_set_asid(pai->pai_asid, pm); +#endif } static inline void pmap_md_xtab_deactivate(struct pmap *pm) { - /* nothing */ + tlb_set_asid(KERNEL_PID, pmap_kernel()); } #endif /* __PMAP_PRIVATE */ diff --git a/sys/arch/powerpc/include/booke/pmap.h b/sys/arch/powerpc/include/booke/pmap.h index f89028e7512a..a66a6f8d5aae 100644 --- a/sys/arch/powerpc/include/booke/pmap.h +++ b/sys/arch/powerpc/include/booke/pmap.h @@ -158,15 +158,18 @@ pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs) static __inline void pmap_md_xtab_activate(struct pmap *pm, struct lwp *l) { +#if 0 + struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); - /* nothing */ + tlb_set_asid(pai->pai_asid, pm); +#endif } static __inline void pmap_md_xtab_deactivate(struct pmap *pm) { - /* nothing */ + tlb_set_asid(KERNEL_PID, pmap_kernel()); } #endif diff --git a/sys/arch/riscv/include/pte.h b/sys/arch/riscv/include/pte.h index a3ec99597a99..f1383217a7c4 100644 --- a/sys/arch/riscv/include/pte.h +++ b/sys/arch/riscv/include/pte.h @@ -142,7 +142,6 @@ pte_modified_p(pt_entry_t pte) static inline bool pte_cached_p(pt_entry_t pte) { - /* TODO: This seems wrong... */ return true; } diff --git a/sys/uvm/pmap/pmap.c b/sys/uvm/pmap/pmap.c index 0824c694a716..fd42735e97a3 100644 --- a/sys/uvm/pmap/pmap.c +++ b/sys/uvm/pmap/pmap.c @@ -173,6 +173,7 @@ PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); +PMAP_COUNTER(exec_synced_protect_1, "exec pages synced 1 (P)"); PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); @@ -184,6 +185,8 @@ PMAP_COUNTER(reference, "references"); PMAP_COUNTER(dereference, "dereferences"); PMAP_COUNTER(destroy, "destroyed"); PMAP_COUNTER(activate, "activations"); +PMAP_COUNTER(activate_kernel, "activations (kernel)"); +PMAP_COUNTER(activate_notcurlwp, "activations (not curlwp)"); PMAP_COUNTER(deactivate, "deactivations"); PMAP_COUNTER(update, "updates"); #ifdef MULTIPROCESSOR @@ -877,6 +880,18 @@ pmap_activate(struct lwp *l) (uintptr_t)pmap, 0, 0); PMAP_COUNT(activate); + if (pmap == pmap_kernel()) { + UVMHIST_LOG(pmaphist, " <-- done (kernel)", 0, 0, 0, 0); + PMAP_COUNT(activate_kernel); + return; + } + if (l != curlwp) { + UVMHIST_LOG(pmaphist, " <-- done (not curlwp %p vs %p)", + l, curlwp, 0, 0); + PMAP_COUNT(activate_notcurlwp); + return; + } + kpreempt_disable(); pmap_tlb_miss_lock_enter(); pmap_tlb_asid_acquire(pmap, l); @@ -902,8 +917,10 @@ pmap_deactivate(struct lwp *l) PMAP_COUNT(deactivate); kpreempt_disable(); + KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); pmap_tlb_miss_lock_enter(); + // calls pmap_md_tlb_asid_deactivate pmap_tlb_asid_deactivate(pmap); pmap_segtab_deactivate(pmap); pmap_tlb_miss_lock_exit(); @@ -1270,9 +1287,26 @@ pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, pt_entry_t pte = *ptep; if (!pte_valid_p(pte)) continue; + + bool syncicache = false; + // XXXNH __HAVE_PMAP_PV_TRACK pp vs pg? struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); if (pg != NULL && pte_modified_p(pte)) { struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); + + // XXXNH check all mappings not just the page version + if (!VM_PAGEMD_EXECPAGE_P(mdpg) && (prot & VM_PROT_EXECUTE)) { + /* non-exec -> exec */ + UVMHIST_LOG(pmapexechist, + "pg %#jx (pa %#jx): " + "syncicached performed", + (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), + 0, 0); + syncicache = true; + pmap_page_syncicache(pg); + PMAP_COUNT(exec_synced_protect_1); + } + if (VM_PAGEMD_EXECPAGE_P(mdpg)) { KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); #ifdef PMAP_VIRTUAL_CACHE_ALIASES @@ -1290,6 +1324,8 @@ pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, #endif } } + + //XXX non-exec -> exec pte = pte_prot_downgrade(pte, prot); if (*ptep != pte) { pmap_tlb_miss_lock_enter(); @@ -1298,6 +1334,11 @@ pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, * Update the TLB if needed. */ pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI); + + if (syncicache) { + pmap_page_syncicache(pg); + } + pmap_tlb_miss_lock_exit(); } } @@ -1567,7 +1608,8 @@ pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) KASSERT(prot & VM_PROT_WRITE); PMAP_COUNT(exec_mappings); pmap_page_syncicache(pg); - pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); + pmap_page_set_attributes(mdpg, VM_PAGEMD_EXECPAGE); +// pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); UVMHIST_LOG(*histp, "va=%#jx pg %#jx: immediate syncicache (writeable)", va, (uintptr_t)pg, 0, 0); @@ -1697,6 +1739,16 @@ pmap_kremove(vaddr_t va, vsize_t len) UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); } +#if 1 +bool +pmap_remove_all(struct pmap *pmap) +{ + UVMHIST_FUNC(__func__); + UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); + + return false; +} +#else bool pmap_remove_all(struct pmap *pmap) { @@ -1734,6 +1786,7 @@ pmap_remove_all(struct pmap *pmap) UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); return false; } +#endif /* * Routine: pmap_unwire diff --git a/sys/uvm/pmap/pmap_segtab.c b/sys/uvm/pmap/pmap_segtab.c index f2458ceb63bd..93bb6fb7e44c 100644 --- a/sys/uvm/pmap/pmap_segtab.c +++ b/sys/uvm/pmap/pmap_segtab.c @@ -946,7 +946,6 @@ pmap_segtab_activate(struct pmap *pm, struct lwp *l) { if (l == curlwp) { KASSERT(pm == l->l_proc->p_vmspace->vm_map.pmap); - pmap_md_xtab_activate(pm, l); #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) struct cpu_info * const ci = l->l_cpu; if (pm == pmap_kernel()) { @@ -967,8 +966,6 @@ pmap_segtab_activate(struct pmap *pm, struct lwp *l) void pmap_segtab_deactivate(pmap_t pm) { - pmap_md_xtab_deactivate(pm); - #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS; #ifdef _LP64 diff --git a/sys/uvm/pmap/pmap_tlb.c b/sys/uvm/pmap/pmap_tlb.c index 5b7a8bb1f47d..82ef8031c04e 100644 --- a/sys/uvm/pmap/pmap_tlb.c +++ b/sys/uvm/pmap/pmap_tlb.c @@ -800,6 +800,8 @@ pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid); TLBINFO_LOCK(ti); + // XXXNH maybe this should be using a tlb_invalidate_globalva + // XXXNH for pmap_kernel(). if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { pmap_tlb_asid_check(); UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx", @@ -985,7 +987,7 @@ pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) ci->ci_pmap_asid_cur = pai->pai_asid; UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid, 0, 0, 0); - tlb_set_asid(pai->pai_asid, pm); + pmap_md_xtab_activate(pm, l); pmap_tlb_asid_check(); } TLBINFO_UNLOCK(ti); @@ -1021,8 +1023,7 @@ pmap_tlb_asid_deactivate(pmap_t pm) } #endif curcpu()->ci_pmap_asid_cur = KERNEL_PID; - tlb_set_asid(KERNEL_PID, pmap_kernel()); - + pmap_md_xtab_deactivate(pm); pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false); #if defined(DEBUG) pmap_tlb_asid_check();