diff -upr dev/misc/hwpmc.freebsd/hwpmc_mod.c dev/misc/hwpmc/hwpmc_mod.c --- dev/misc/hwpmc.freebsd/hwpmc_mod.c +++ dev/misc/hwpmc/hwpmc_mod.c @@ -26,29 +26,26 @@ */ #include -__FBSDID("$FreeBSD: src/sys/dev/hwpmc/hwpmc_mod.c,v 1.29 2007/06/05 00:00:50 jeff Exp $"); +/* __FBSDID("$FreeBSD: src/sys/dev/hwpmc/hwpmc_mod.c,v 1.29 2007/06/05 00:00:50 jeff Exp $"); */ #include #include #include #include #include -#include #include #include #include -#include #include #include #include -#include #include #include #include #include #include -#include -#include +#include +#include #include #include #include @@ -57,7 +54,52 @@ __FBSDID("$FreeBSD: src/sys/dev/hwpmc/hw #include /* needs to be after */ #include +#include #include +#include +#include /* XXX */ + +KTR_INFO_MASTER(hwpmctr); +KTR_INFO(!0, hwpmctr, regval, 0, "r=%lld", sizeof(int64_t)); + +#define PROC_LOCK(p) do { (void)p; } while(0) +#define PROC_UNLOCK(p) do { (void)p; } while(0) + +static __inline void +critical_enter(void) +{ + /* no preemption in the dragonfly kernel */ +} +static __inline void +critical_exit(void) +{ +} + +/* I think we need giant for process manipulation etc -- agg */ +#define DROP_GIANT() do {} while(0) +#define PICKUP_GIANT() do {} while(0) + +#define PRIV_PMC_MANAGE 0 +#define PRIV_PMC_SYSTEM 0 +static __inline int +priv_check(struct thread *td, int priv) +{ + /* + * FIXME: don't have privs in dfly, just use root for now -- agg + * XXX: jail? + */ + return suser(td); +} + +static __inline int +p_candebug(struct thread *td, struct proc *p) +{ + /* + * FIXME: need to bring in some stuff from freebsd which will + * require some code reorganization. TBD -- agg + */ + return 0; +} /* * Types @@ -79,7 +121,6 @@ pmc_value_t *pmc_pcpu_saved; /* saved P #define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)] -struct mtx_pool *pmc_mtxpool; static int *pmc_pmcdisp; /* PMC row dispositions */ #define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0) @@ -94,7 +135,7 @@ static int *pmc_pmcdisp; /* PMC row di KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ atomic_add_int(&pmc_pmcdisp[(R)], -1); \ - KASSERT(pmc_pmcdisp[(R)] >= (-mp_ncpus), ("[pmc,%d] row " \ + KASSERT(pmc_pmcdisp[(R)] >= (-ncpus), ("[pmc,%d] row " \ "disposition error", __LINE__)); \ } while (0) @@ -130,7 +171,7 @@ struct pmc_mdep *md; * Hash tables mapping owner processes and target threads to PMCs. */ -struct mtx pmc_processhash_mtx; /* spin mutex */ +struct spinlock pmc_processhash_lock; /* spin lock */ static u_long pmc_processhashmask; static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash; @@ -183,8 +224,8 @@ static void pmc_link_target_process(stru static void pmc_maybe_remove_owner(struct pmc_owner *po); static void pmc_process_csw_in(struct thread *td); static void pmc_process_csw_out(struct thread *td); -static void pmc_process_exit(void *arg, struct proc *p); -static void pmc_process_fork(void *arg, struct proc *p1, +static void pmc_process_exit(struct thread *td); +static void pmc_process_fork(struct proc *p1, struct proc *p2, int n); static void pmc_process_samples(int cpu); static void pmc_release_pmc_descriptor(struct pmc *pmc); @@ -195,7 +236,7 @@ static void pmc_save_cpu_binding(struct static void pmc_select_cpu(int cpu); static int pmc_start(struct pmc *pm); static int pmc_stop(struct pmc *pm); -static int pmc_syscall_handler(struct thread *td, void *syscall_args); +static int pmc_syscall_handler(void *syscall_args); static void pmc_unlink_target_process(struct pmc *pmc, struct pmc_process *pp); @@ -206,12 +247,25 @@ static void pmc_unlink_target_process(st SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); #ifdef DEBUG +#ifndef agg +#undef PMC_DEBUG_DEFAULT_FLAGS +#define PMC_DEBUG_DEFAULT_FLAGS { \ + .pdb_CPU = 0x00000000, \ + .pdb_CSW = 0x00000000, \ + .pdb_LOG = 0xffffffff, \ + .pdb_MDP = 0x00000000, \ + .pdb_MOD = (1 << PMC_DEBUG_MIN_PMS), \ + .pdb_OWN = 0xffffffff, \ + .pdb_PMC = 0xffffffff, \ + .pdb_PRC = 0xffffffff, \ + .pdb_SAM = 0xffffffff,} +#endif struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; char pmc_debugstr[PMC_DEBUG_STRSIZE]; TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, sizeof(pmc_debugstr)); SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, - CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN, + CTLTYPE_STRING|CTLFLAG_RW, 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); #endif @@ -222,7 +276,7 @@ SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debug static int pmc_hashsize = PMC_HASH_SIZE; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize); -SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD, +SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RD, &pmc_hashsize, 0, "rows in hash tables"); /* @@ -231,20 +285,10 @@ SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsi static int pmc_nsamples = PMC_NSAMPLES; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples); -SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD, +SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RD, &pmc_nsamples, 0, "number of PC samples per CPU"); /* - * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. - */ - -static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; -TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size); -SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD, - &pmc_mtxpool_size, 0, "size of spin mutex pool"); - - -/* * security.bsd.unprivileged_syspmcs -- allow non-root processes to * allocate system-wide PMCs. * @@ -254,8 +298,8 @@ SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoo */ static int pmc_unprivileged_syspmcs = 0; -TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs); -SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW, +TUNABLE_INT("kern.unprivileged_syspmcs", &pmc_unprivileged_syspmcs); +SYSCTL_INT(_kern, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW, &pmc_unprivileged_syspmcs, 0, "allow unprivileged process to allocate system PMCs"); @@ -282,7 +326,8 @@ SYSCTL_INT(_security_bsd, OID_AUTO, unpr /* The `sysent' for the new syscall */ static struct sysent pmc_sysent = { 2, /* sy_narg */ - pmc_syscall_handler /* sy_call */ + pmc_syscall_handler, /* sy_call */ + NULL }; static struct syscall_module_data pmc_syscall_mod = { @@ -299,7 +344,7 @@ static moduledata_t pmc_mod = { &pmc_syscall_mod }; -DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY); +DECLARE_MODULE(pmc, pmc_mod, SI_SUB_CONFIGURE, SI_ORDER_ANY); MODULE_VERSION(pmc, PMC_VERSION); #ifdef DEBUG @@ -495,7 +540,7 @@ pmc_debugflags_sysctl_handler(SYSCTL_HAN * * The driver uses four locking strategies for its operation: * - * - The global SX lock "pmc_sx" is used to protect internal + * - The global lockmgr lock "pmc_lock" is used to protect internal * data structures. * * Calls into the module by syscall() start with this lock being @@ -511,15 +556,15 @@ pmc_debugflags_sysctl_handler(SYSCTL_HAN * It is only examined with the sx lock held in exclusive mode. The * kernel module is allowed to be unloaded only with the sx lock held * in exclusive mode. In normal syscall handling, after acquiring the - * pmc_sx lock we first check that 'pmc_hook' is non-null before + * pmc_lock lock we first check that 'pmc_hook' is non-null before * proceeding. This prevents races between the thread unloading the module * and other threads seeking to use the module. * * - Lookups of target process structures and owner process structures - * cannot use the global "pmc_sx" SX lock because these lookups need + * cannot use the global "pmc_lock" lock because these lookups need * to happen during context switches and in other critical sections * where sleeping is not allowed. We protect these lookup tables - * with their own private spin-mutexes, "pmc_processhash_mtx" and + * with their own private spin-mutexes, "pmc_processhash_lock" and * "pmc_ownerhash_mtx". * * - Interrupt handlers work in a lock free manner. At interrupt @@ -587,14 +632,41 @@ pmc_debugflags_sysctl_handler(SYSCTL_HAN * save the cpu binding of the current kthread */ +/* + * count set bits + */ +static int +csb(cpumask_t mask) +{ + unsigned i, cnt; + + for (i = 0, cnt = 0; i < (sizeof(cpumask_t) * 8); ++i, mask >>= 1) { + if (mask & 1) { + ++cnt; + } + } + return cnt; +} + +static int +sched_is_bound(struct thread *td) +{ + struct lwp *lwp; + + lwp = td->td_lwp; + if (!lwp) { + panic("can't happen\n"); + } + /* FIXME: could be better -- agg */ + return csb(lwp->lwp_cpumask) == ((sizeof(cpumask_t) * 8) - 1); +} + static void pmc_save_cpu_binding(struct pmc_binding *pb) { PMCDBG(CPU,BND,2, "%s", "save-cpu"); - thread_lock(curthread); pb->pb_bound = sched_is_bound(curthread); - pb->pb_cpu = curthread->td_oncpu; - thread_unlock(curthread); + pb->pb_cpu = mycpu->gd_cpuid; PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu); } @@ -606,13 +678,11 @@ static void pmc_restore_cpu_binding(struct pmc_binding *pb) { PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d", - curthread->td_oncpu, pb->pb_cpu); - thread_lock(curthread); - if (pb->pb_bound) - sched_bind(curthread, pb->pb_cpu); - else - sched_unbind(curthread); - thread_unlock(curthread); + mycpu->gd_cpuid, pb->pb_cpu); + + /* unconditionally return to original cpu */ + lwkt_migratecpu(pb->pb_cpu); + PMCDBG(CPU,BND,2, "%s", "restore-cpu done"); } @@ -623,7 +693,7 @@ pmc_restore_cpu_binding(struct pmc_bindi static void pmc_select_cpu(int cpu) { - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && ((1<td_oncpu == cpu, + lwkt_migratecpu(cpu); + + KASSERT(mycpu->gd_cpuid == cpu, ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, - cpu, curthread->td_oncpu)); + cpu, mycpu->gd_cpuid)); PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); } @@ -652,8 +721,8 @@ pmc_select_cpu(int cpu) static void pmc_force_context_switch(void) { - - pause("pmcctx", 1); + static int pillow; + tsleep(&pillow, 0, "pmcctx", 1); } /* @@ -669,9 +738,9 @@ pmc_getfilename(struct vnode *v, char ** td = curthread; *fullpath = "unknown"; *freepath = NULL; - vn_lock(v, LK_CANRECURSE | LK_EXCLUSIVE | LK_RETRY, td); - vn_fullpath(td, v, fullpath, freepath); - VOP_UNLOCK(v, 0, td); + vn_lock(v, LK_CANRECURSE | LK_EXCLUSIVE | LK_RETRY); + vn_fullpath(td->td_proc, v, fullpath, freepath); + vn_unlock(v); } /* @@ -683,7 +752,7 @@ pmc_remove_owner(struct pmc_owner *po) { struct pmc *pm, *tmp; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); PMCDBG(OWN,ORM,1, "remove-owner po=%p", po); @@ -691,7 +760,7 @@ pmc_remove_owner(struct pmc_owner *po) LIST_REMOVE(po, po_next); /* release all owned PMC descriptors */ - LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { + LIST_FOREACH_MUTABLE(pm, &po->po_pmcs, pm_next, tmp) { PMCDBG(OWN,ORM,2, "pmc=%p", pm); KASSERT(pm->pm_owner == po, ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); @@ -742,7 +811,7 @@ pmc_link_target_process(struct pmc *pm, int ri; struct pmc_target *pt; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); @@ -799,7 +868,7 @@ pmc_unlink_target_process(struct pmc *pm struct proc *p; struct pmc_target *ptgt; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); @@ -843,7 +912,7 @@ pmc_unlink_target_process(struct pmc *pm if (LIST_EMPTY(&pm->pm_targets)) { p = pm->pm_owner->po_owner; PROC_LOCK(p); - psignal(p, SIGIO); + lwpsignal(p, NULL, SIGIO); PROC_UNLOCK(p); PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p, @@ -917,8 +986,9 @@ pmc_attach_one_process(struct proc *p, s char *fullpath, *freepath; struct pmc_process *pp; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); + kprintf("FTLOD\n"); PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); @@ -976,7 +1046,7 @@ pmc_attach_process(struct proc *p, struc int error; struct proc *top; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); @@ -998,11 +1068,10 @@ pmc_attach_process(struct proc *p, struc * this PMC. */ - sx_slock(&proctree_lock); - top = p; for (;;) { + kprintf("attaching: p %lu\n", p->p_pid); if ((error = pmc_attach_one_process(p, pm)) != 0) break; if (!LIST_EMPTY(&p->p_children)) @@ -1022,7 +1091,6 @@ pmc_attach_process(struct proc *p, struc (void) pmc_detach_process(top, pm); done: - sx_sunlock(&proctree_lock); return error; } @@ -1038,7 +1106,7 @@ pmc_detach_one_process(struct proc *p, s int ri; struct pmc_process *pp; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); KASSERT(pm != NULL, ("[pmc,%d] null pm pointer", __LINE__)); @@ -1093,7 +1161,7 @@ pmc_detach_process(struct proc *p, struc { struct proc *top; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); @@ -1107,8 +1175,6 @@ pmc_detach_process(struct proc *p, struc * partially attached proc tree. */ - sx_slock(&proctree_lock); - top = p; for (;;) { @@ -1128,8 +1194,6 @@ pmc_detach_process(struct proc *p, struc } done: - sx_sunlock(&proctree_lock); - if (LIST_EMPTY(&pm->pm_targets)) pm->pm_flags &= ~PMC_F_ATTACH_DONE; @@ -1163,12 +1227,12 @@ pmc_process_csw_in(struct thread *td) critical_enter(); /* no preemption from this point */ - cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ + cpu = mycpu->gd_cpuid; PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu < ncpus, ("[pmc,%d] wierd CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; @@ -1217,18 +1281,18 @@ pmc_process_csw_in(struct thread *td) * inherited across descendants. */ if (PMC_TO_MODE(pm) == PMC_MODE_TS) { - mtx_pool_lock_spin(pmc_mtxpool, pm); + spin_lock_wr(&pm->lock); newvalue = PMC_PCPU_SAVED(cpu,ri) = pp->pp_pmcs[ri].pp_pmcval; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, PMC_TO_MODE(pm))); - mtx_pool_lock_spin(pmc_mtxpool, pm); + spin_lock_wr(&pm->lock); newvalue = PMC_PCPU_SAVED(cpu, ri) = pm->pm_gv.pm_savedvalue; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); } PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); @@ -1265,6 +1329,7 @@ pmc_process_csw_out(struct thread *td) int64_t tmp; pmc_value_t newvalue; + /* * Locate our process descriptor; this may be NULL if * this process is exiting and we have already removed @@ -1288,12 +1353,12 @@ pmc_process_csw_out(struct thread *td) critical_enter(); - cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ + cpu = mycpu->gd_cpuid; PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu < ncpus, ("[pmc,%d wierd CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; @@ -1366,12 +1431,12 @@ pmc_process_csw_out(struct thread *td) */ if (tmp < 0) tmp += pm->pm_sc.pm_reloadcount; - mtx_pool_lock_spin(pmc_mtxpool, pm); + spin_lock_wr(&pm->lock); pp->pp_pmcs[ri].pp_pmcval -= tmp; if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0) pp->pp_pmcs[ri].pp_pmcval += pm->pm_sc.pm_reloadcount; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); } else { @@ -1381,16 +1446,18 @@ pmc_process_csw_out(struct thread *td) * increasing monotonically, modulo a 64 * bit wraparound. */ + + KTR_LOG(hwpmctr_regval, PMC_PCPU_SAVED(cpu, ri)); KASSERT((int64_t) tmp >= 0, ("[pmc,%d] negative increment cpu=%d " "ri=%d newvalue=%jx saved=%jx " "incr=%jx", __LINE__, cpu, ri, newvalue, PMC_PCPU_SAVED(cpu,ri), tmp)); - mtx_pool_lock_spin(pmc_mtxpool, pm); + spin_lock_wr(&pm->lock); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); if (pm->pm_flags & PMC_F_LOG_PROCCSW) pmclog_process_proccsw(pm, pp, tmp); @@ -1420,7 +1487,7 @@ pmc_process_kld_load(struct pmckern_map_ { struct pmc_owner *po; - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); /* * Notify owners of system sampling PMCs about KLD operations. @@ -1443,7 +1510,7 @@ pmc_process_kld_unload(struct pmckern_ma { struct pmc_owner *po; - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) @@ -1572,7 +1639,7 @@ pmc_hook_handler(struct thread *td, int struct pmc_process *pp; struct pmckern_procexec *pk; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); p = td->td_proc; pmc_getfilename(p->p_textvp, &fullpath, &freepath); @@ -1701,28 +1768,28 @@ pmc_hook_handler(struct thread *td, int * had already processed the interrupt). We don't * lose the interrupt sample. */ - atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid))); - pmc_process_samples(PCPU_GET(cpuid)); + atomic_clear_int(&pmc_cpumask, (1 << mycpu->gd_cpuid)); + pmc_process_samples(mycpu->gd_cpuid); break; case PMC_FN_KLD_LOAD: - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); pmc_process_kld_load((struct pmckern_map_in *) arg); break; case PMC_FN_KLD_UNLOAD: - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); pmc_process_kld_unload((struct pmckern_map_out *) arg); break; case PMC_FN_MMAP: - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); pmc_process_mmap(td, (struct pmckern_map_in *) arg); break; case PMC_FN_MUNMAP: - sx_assert(&pmc_sx, SX_LOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_SHARED); pmc_process_munmap(td, (struct pmckern_map_out *) arg); break; @@ -1763,7 +1830,7 @@ pmc_allocate_owner_descriptor(struct pro LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ TAILQ_INIT(&po->po_logbuffers); - mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN); + spin_init(&po->po_lock); PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p", p, p->p_pid, p->p_comm, po); @@ -1778,7 +1845,7 @@ pmc_destroy_owner_descriptor(struct pmc_ PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)", po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); - mtx_destroy(&po->po_mtx); + spin_uninit(&po->po_lock); FREE(po, M_PMC); } @@ -1811,7 +1878,7 @@ pmc_find_process_descriptor(struct proc sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK); } - mtx_lock_spin(&pmc_processhash_mtx); + spin_lock_wr(&pmc_processhash_lock); LIST_FOREACH(pp, pph, pp_next) if (pp->pp_proc == p) break; @@ -1826,7 +1893,7 @@ pmc_find_process_descriptor(struct proc pp = ppnew; ppnew = NULL; } - mtx_unlock_spin(&pmc_processhash_mtx); + spin_unlock_wr(&pmc_processhash_lock); if (pp != NULL && ppnew != NULL) FREE(ppnew, M_PMC); @@ -1845,9 +1912,9 @@ pmc_remove_process_descriptor(struct pmc ("[pmc,%d] Removing process descriptor %p with count %d", __LINE__, pp, pp->pp_refcnt)); - mtx_lock_spin(&pmc_processhash_mtx); + spin_lock_wr(&pmc_processhash_lock); LIST_REMOVE(pp, pp_next); - mtx_unlock_spin(&pmc_processhash_mtx); + spin_unlock_wr(&pmc_processhash_lock); } @@ -1929,7 +1996,7 @@ pmc_wait_for_pmc_idle(struct pmc *pm) #ifdef DEBUG volatile int maxloop; - maxloop = 100 * mp_ncpus; + maxloop = 100 * ncpus; #endif /* @@ -1971,7 +2038,7 @@ pmc_release_pmc_descriptor(struct pmc *p struct pmc_target *ptgt, *tmp; struct pmc_binding pb; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); @@ -2065,7 +2132,7 @@ pmc_release_pmc_descriptor(struct pmc *p * it from the hash table. The module-wide SX lock * protects us from races. */ - LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { + LIST_FOREACH_MUTABLE(ptgt, &pm->pm_targets, pt_next, tmp) { pp = ptgt->pt_process; pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ @@ -2082,7 +2149,7 @@ pmc_release_pmc_descriptor(struct pmc *p } } - cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ + cpu = curthread->td_gd->gd_cpuid; /* setup cpu for pmd_release() */ } @@ -2119,7 +2186,7 @@ pmc_register_owner(struct proc *p, struc { struct pmc_owner *po; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); if ((po = pmc_find_owner_descriptor(p)) == NULL) if ((po = pmc_allocate_owner_descriptor(p)) == NULL) @@ -2222,7 +2289,7 @@ pmc_can_allocate_row(int ri, enum pmc_mo { enum pmc_disp disp; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode); @@ -2456,7 +2523,7 @@ pmc_stop(struct pmc *pm) cpu = PMC_TO_CPU(pm); - KASSERT(cpu >= 0 && cpu < mp_ncpus, + KASSERT(cpu >= 0 && cpu < ncpus, ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); if (pmc_cpu_is_disabled(cpu)) @@ -2503,25 +2570,27 @@ static const char *pmc_op_to_name[] = { */ #define PMC_GET_SX_XLOCK(...) do { \ - sx_xlock(&pmc_sx); \ + lockmgr(&pmc_lock, LK_EXCLUSIVE); \ if (pmc_hook == NULL) { \ - sx_xunlock(&pmc_sx); \ + lockmgr(&pmc_lock, LK_RELEASE); \ return __VA_ARGS__; \ } \ } while (0) #define PMC_DOWNGRADE_SX() do { \ - sx_downgrade(&pmc_sx); \ + lockmgr(&pmc_lock, LK_DOWNGRADE); \ is_sx_downgraded = 1; \ } while (0) static int -pmc_syscall_handler(struct thread *td, void *syscall_args) +pmc_syscall_handler(void *syscall_args) { int error, is_sx_downgraded, op; struct pmc_syscall_args *c; void *arg; + struct thread *td; + td = curthread; PMC_GET_SX_XLOCK(ENOSYS); DROP_GIANT(); @@ -2557,7 +2626,7 @@ pmc_syscall_handler(struct thread *td, v struct pmckern_map_in *km, *kmbase; struct pmc_op_configurelog cl; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); if ((error = copyin(arg, &cl, sizeof(cl))) != 0) break; @@ -2617,7 +2686,7 @@ pmc_syscall_handler(struct thread *td, v { struct pmc_owner *po; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; @@ -2637,7 +2706,7 @@ pmc_syscall_handler(struct thread *td, v struct pmc_op_getcpuinfo gci; gci.pm_cputype = md->pmd_cputype; - gci.pm_ncpu = mp_ncpus; + gci.pm_ncpu = ncpus; gci.pm_npmc = md->pmd_npmc; gci.pm_nclass = md->pmd_nclass; bcopy(md->pmd_classes, &gci.pm_classes, @@ -2705,7 +2774,7 @@ pmc_syscall_handler(struct thread *td, v if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) break; - if (cpu >= (unsigned int) mp_ncpus) { + if (cpu >= (unsigned int) ncpus) { error = EINVAL; break; } @@ -2785,7 +2854,7 @@ pmc_syscall_handler(struct thread *td, v struct pmc_op_pmcadmin pma; struct pmc_binding pb; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); @@ -2799,7 +2868,7 @@ pmc_syscall_handler(struct thread *td, v cpu = pma.pm_cpu; - if (cpu < 0 || cpu >= mp_ncpus) { + if (cpu < 0 || cpu >= ncpus) { error = EINVAL; break; } @@ -2892,7 +2961,7 @@ pmc_syscall_handler(struct thread *td, v if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && mode != PMC_MODE_TS && mode != PMC_MODE_TC) || - (cpu != (u_int) PMC_CPU_ANY && cpu >= (u_int) mp_ncpus)) { + (cpu != (u_int) PMC_CPU_ANY && cpu >= (u_int) ncpus)) { error = EINVAL; break; } @@ -2925,7 +2994,7 @@ pmc_syscall_handler(struct thread *td, v */ if (PMC_IS_SYSTEM_MODE(mode)) { - if (jailed(curthread->td_ucred)) { + if (jailed(curthread->td_proc->p_ucred)) { error = EPERM; break; } @@ -3017,7 +3086,7 @@ pmc_syscall_handler(struct thread *td, v pmc_can_allocate_rowindex( curthread->td_proc, n, PMC_CPU_ANY) == 0 && - md->pmd_allocate_pmc(curthread->td_oncpu, + md->pmd_allocate_pmc(curthread->td_gd->gd_cpuid, n, pmc, &pa) == 0) break; } @@ -3119,7 +3188,7 @@ pmc_syscall_handler(struct thread *td, v struct proc *p; struct pmc_op_pmcattach a; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); if ((error = copyin(arg, &a, sizeof(a))) != 0) break; @@ -3405,8 +3474,8 @@ pmc_syscall_handler(struct thread *td, v ri = PMC_TO_ROWINDEX(pm); - mtx_pool_lock_spin(pmc_mtxpool, pm); - cpu = curthread->td_oncpu; + spin_lock_wr(&pm->lock); + cpu = curthread->td_gd->gd_cpuid; if (prw.pm_flags & PMC_F_OLDVALUE) { if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && @@ -3419,7 +3488,7 @@ pmc_syscall_handler(struct thread *td, v if (prw.pm_flags & PMC_F_NEWVALUE) pm->pm_gv.pm_savedvalue = prw.pm_value; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); } else { /* System mode PMCs */ cpu = PMC_TO_CPU(pm); @@ -3512,7 +3581,7 @@ pmc_syscall_handler(struct thread *td, v struct pmc *pm; struct pmc_op_simple sp; - sx_assert(&pmc_sx, SX_XLOCKED); + KKASSERT(lockstatus(&pmc_lock, curthread) == LK_EXCLUSIVE); if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; @@ -3614,10 +3683,7 @@ pmc_syscall_handler(struct thread *td, v break; } - if (is_sx_downgraded) - sx_sunlock(&pmc_sx); - else - sx_xunlock(&pmc_sx); + lockmgr(&pmc_lock, LK_RELEASE); if (error) atomic_add_int(&pmc_stats.pm_syscall_errors, 1); @@ -3708,14 +3774,13 @@ pmc_process_samples(int cpu) { int n, ri; struct pmc *pm; - struct thread *td; struct pmc_owner *po; struct pmc_sample *ps; struct pmc_samplebuffer *psb; - KASSERT(PCPU_GET(cpuid) == cpu, + KASSERT(mycpu->gd_cpuid == cpu, ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, - PCPU_GET(cpuid), cpu)); + mycpu->gd_cpuid, cpu)); psb = pmc_pcpu[cpu]->pc_sb; @@ -3749,8 +3814,7 @@ pmc_process_samples(int cpu) */ if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { if (ps->ps_usermode) { - td = FIRST_THREAD_IN_PROC(po->po_owner); - addupc_intr(td, ps->ps_pc, 1); + addupc_intr(po->po_owner, ps->ps_pc, 1); } goto entrydone; } @@ -3824,7 +3888,7 @@ pmc_process_samples(int cpu) */ static void -pmc_process_exit(void *arg __unused, struct proc *p) +pmc_process_exit(struct thread *td) { int is_using_hwpmcs; int cpu; @@ -3833,6 +3897,10 @@ pmc_process_exit(void *arg __unused, str struct pmc_process *pp; struct pmc_owner *po; pmc_value_t newvalue, tmp; + struct proc *p; + + p = td->td_proc; + KKASSERT(p); PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; @@ -3869,7 +3937,7 @@ pmc_process_exit(void *arg __unused, str critical_enter(); /* no preemption */ - cpu = curthread->td_oncpu; + cpu = curthread->td_gd->gd_cpuid; if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_REMOVE)) != NULL) { @@ -3925,10 +3993,10 @@ pmc_process_exit(void *arg __unused, str tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); - mtx_pool_lock_spin(pmc_mtxpool, pm); + spin_lock_wr(&pm->lock); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; - mtx_pool_unlock_spin(pmc_mtxpool, pm); + spin_unlock_wr(&pm->lock); } atomic_subtract_rel_32(&pm->pm_runcount,1); @@ -3976,7 +4044,7 @@ pmc_process_exit(void *arg __unused, str pmc_destroy_owner_descriptor(po); } - sx_xunlock(&pmc_sx); + lockmgr(&pmc_lock, LK_RELEASE); } /* @@ -3987,7 +4055,7 @@ pmc_process_exit(void *arg __unused, str */ static void -pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, +pmc_process_fork(struct proc *p1, struct proc *newproc, int flags) { int is_using_hwpmcs; @@ -4068,7 +4136,7 @@ pmc_process_fork(void *arg __unused, str PROC_UNLOCK(newproc); done: - sx_xunlock(&pmc_sx); + lockmgr(&pmc_lock, LK_RELEASE); } @@ -4105,10 +4173,10 @@ pmc_initialize(void) /* check kernel version */ if (pmc_kernel_version != PMC_VERSION) { if (pmc_kernel_version == 0) - printf("hwpmc: this kernel has not been compiled with " + kprintf("hwpmc: this kernel has not been compiled with " "'options HWPMC_HOOKS'.\n"); else - printf("hwpmc: kernel version (0x%x) does not match " + kprintf("hwpmc: kernel version (0x%x) does not match " "module version (0x%x).\n", pmc_kernel_version, PMC_VERSION); return EPROGMISMATCH; @@ -4119,13 +4187,13 @@ pmc_initialize(void) */ if (pmc_hashsize <= 0) { - (void) printf("hwpmc: tunable hashsize=%d must be greater " + (void) kprintf("hwpmc: tunable hashsize=%d must be greater " "than zero.\n", pmc_hashsize); pmc_hashsize = PMC_HASH_SIZE; } if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { - (void) printf("hwpmc: tunable nsamples=%d out of range.\n", + (void) kprintf("hwpmc: tunable nsamples=%d out of range.\n", pmc_nsamples); pmc_nsamples = PMC_NSAMPLES; } @@ -4136,16 +4204,16 @@ pmc_initialize(void) return ENOSYS; /* allocate space for the per-cpu array */ - MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *), + MALLOC(pmc_pcpu, struct pmc_cpu **, ncpus * sizeof(struct pmc_cpu *), M_PMC, M_WAITOK|M_ZERO); /* per-cpu 'saved values' for managing process-mode PMCs */ MALLOC(pmc_pcpu_saved, pmc_value_t *, - sizeof(pmc_value_t) * mp_ncpus * md->pmd_npmc, M_PMC, M_WAITOK); + sizeof(pmc_value_t) * ncpus * md->pmd_npmc, M_PMC, M_WAITOK); /* perform cpu dependent initialization */ pmc_save_cpu_binding(&pb); - for (cpu = 0; cpu < mp_ncpus; cpu++) { + for (cpu = 0; cpu < ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; pmc_select_cpu(cpu); @@ -4158,7 +4226,7 @@ pmc_initialize(void) return error; /* allocate space for the sample array */ - for (cpu = 0; cpu < mp_ncpus; cpu++) { + for (cpu = 0; cpu < ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; MALLOC(sb, struct pmc_samplebuffer *, @@ -4175,7 +4243,7 @@ pmc_initialize(void) } /* allocate space for the row disposition array */ - pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, + pmc_pmcdisp = kmalloc(sizeof(enum pmc_mode) * md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); KASSERT(pmc_pmcdisp != NULL, @@ -4191,25 +4259,22 @@ pmc_initialize(void) pmc_processhash = hashinit(pmc_hashsize, M_PMC, &pmc_processhashmask); - mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf", - MTX_SPIN); + spin_init(&pmc_processhash_lock); LIST_INIT(&pmc_ss_owners); pmc_ss_count = 0; - /* allocate a pool of spin mutexes */ - pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size, - MTX_SPIN); - PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx " "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, pmc_processhash, pmc_processhashmask); /* register process {exit,fork,exec} handlers */ - pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit, - pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY); - pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork, - pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY); + if (at_exit(pmc_process_exit)) { + panic("cannot register exit() handler\n"); + } + if (at_fork(pmc_process_fork)) { + panic("cannot register fork() handler\n"); + } /* initialize logging */ pmclog_initialize(); @@ -4219,9 +4284,9 @@ pmc_initialize(void) pmc_hook = pmc_hook_handler; if (error == 0) { - printf(PMC_MODULE_NAME ":"); + kprintf(PMC_MODULE_NAME ":"); for (n = 0; n < (int) md->pmd_nclass; n++) { - printf(" %s/%d/0x%b", + kprintf(" %s/%d/0x%b", pmc_name_of_pmcclass[md->pmd_classes[n].pm_class], md->pmd_nclasspmcs[n], md->pmd_classes[n].pm_caps, @@ -4230,7 +4295,7 @@ pmc_initialize(void) "\6REA\7WRI\10INV\11QUA\12PRC" "\13TAG\14CSC"); } - printf("\n"); + kprintf("\n"); } return error; @@ -4254,9 +4319,9 @@ pmc_cleanup(void) atomic_store_rel_int(&pmc_cpumask, 0); pmc_intr = NULL; - sx_xlock(&pmc_sx); + lockmgr(&pmc_lock, LK_EXCLUSIVE); if (pmc_hook == NULL) { /* being unloaded already */ - sx_xunlock(&pmc_sx); + lockmgr(&pmc_lock, LK_RELEASE); return; } @@ -4271,7 +4336,7 @@ pmc_cleanup(void) for (ph = pmc_ownerhash; ph <= &pmc_ownerhash[pmc_ownerhashmask]; ph++) { - LIST_FOREACH_SAFE(po, ph, po_next, tmp) { + LIST_FOREACH_MUTABLE(po, ph, po_next, tmp) { pmc_remove_owner(po); /* send SIGBUS to owner processes */ @@ -4281,18 +4346,14 @@ pmc_cleanup(void) po->po_owner->p_comm); PROC_LOCK(po->po_owner); - psignal(po->po_owner, SIGBUS); + lwpsignal(po->po_owner, NULL, SIGBUS); PROC_UNLOCK(po->po_owner); pmc_destroy_owner_descriptor(po); } } - /* reclaim allocated data structures */ - if (pmc_mtxpool) - mtx_pool_destroy(&pmc_mtxpool); - - mtx_destroy(&pmc_processhash_mtx); + spin_uninit(&pmc_processhash_lock); if (pmc_processhash) { #ifdef DEBUG struct pmc_process *pp; @@ -4321,7 +4382,7 @@ pmc_cleanup(void) ("[pmc,%d] Global SS count not empty", __LINE__)); /* free the per-cpu sample buffers */ - for (cpu = 0; cpu < mp_ncpus; cpu++) { + for (cpu = 0; cpu < ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, @@ -4335,7 +4396,7 @@ pmc_cleanup(void) PMCDBG(MOD,INI,3, "%s", "md cleanup"); if (md) { pmc_save_cpu_binding(&pb); - for (cpu = 0; cpu < mp_ncpus; cpu++) { + for (cpu = 0; cpu < ncpus; cpu++) { PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", cpu, pmc_pcpu[cpu]); if (pmc_cpu_is_disabled(cpu)) @@ -4363,7 +4424,7 @@ pmc_cleanup(void) pmclog_shutdown(); - sx_xunlock(&pmc_sx); /* we are done */ + lockmgr(&pmc_lock, LK_RELEASE); /* we are done */ } /* @@ -4384,7 +4445,7 @@ load (struct module *module __unused, in if (error != 0) break; PMCDBG(MOD,INI,1, "syscall=%d ncpus=%d", - pmc_syscall_num, mp_ncpus); + pmc_syscall_num, ncpus); break;