Sprinkle hooks and support code all over the kernel. Needed for proper operation. Obtained-from: FreeBSD Index: platform/pc32/apic/apic_abi.c =================================================================== retrieving revision 1.12 diff -u -p -r1.12 apic_abi.c --- platform/pc32/apic/apic_abi.c +++ platform/pc32/apic/apic_abi.c @@ -40,6 +40,8 @@ * $DragonFly: src/sys/platform/pc32/apic/apic_abi.c,v 1.12 2007/04/30 16:45:55 dillon Exp $ */ +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -240,6 +242,11 @@ apic_finalize(void) temp = lapic.lvt_lint1; temp &= ~APIC_LVT_MASKED; lapic.lvt_lint1 = temp; +#ifdef HWPMC_HOOKS + /* Program the PMC LVT entry if present. */ + if (((lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT) >= LVT_PMC) + lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); +#endif if (bootverbose) apic_dump("bsp_apic_configure()"); Index: platform/pc32/apic/mpapic.c =================================================================== retrieving revision 1.21 diff -u -p -r1.21 mpapic.c --- platform/pc32/apic/mpapic.c +++ platform/pc32/apic/mpapic.c @@ -26,6 +26,8 @@ * $DragonFly: src/sys/platform/pc32/apic/mpapic.c,v 1.21 2007/04/30 16:45:55 dillon Exp $ */ +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -82,6 +84,11 @@ apic_initialize(void) APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK); temp |= APIC_LVT_MASKED | APIC_LVT_DM_NMI; lapic.lvt_lint1 = temp; +#ifdef HWPMC_HOOKS + /* Program the PMC LVT entry if present. */ + if (((lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT) >= LVT_PMC) + lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); +#endif /* * Mask the apic error interrupt, apic performance counter @@ -151,7 +158,6 @@ apic_dump(char* str) lapic.lvt_lint0, lapic.lvt_lint1, lapic.tpr, lapic.svr); } - #if defined(APIC_IO) /* Index: platform/pc32/i386/trap.c =================================================================== retrieving revision 1.107 diff -u -p -r1.107 trap.c --- platform/pc32/i386/trap.c +++ platform/pc32/i386/trap.c @@ -48,6 +48,7 @@ #include "opt_cpu.h" #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include "opt_ktrace.h" #include "opt_clock.h" #include "opt_trap.h" @@ -69,6 +70,9 @@ #ifdef KTRACE #include #endif +#ifdef HWPMC_HOOKS +#include +#endif #include #include #include @@ -408,6 +412,20 @@ trap(struct trapframe *frame) } #endif +#ifdef HWPMC_HOOKS + /* + * CPU PMCs interrupt using an NMI so we check for that first. + * If the HWPMC module is active, 'pmc_hook' will point to + * the function to be called. A return value of '1' from the + * hook means that the NMI was handled by it and that we can + * return immediately. + */ + if (type == T_NMI && pmc_intr && + (*pmc_intr)(mycpu->gd_cpuid, (uintptr_t) frame->tf_eip, + TRAPF_USERMODE(frame))) + goto out; +#endif + eva = 0; ++gd->gd_trap_nesting_level; if (frame->tf_trapno == T_PAGEFLT) { Index: kern/kern_exec.c =================================================================== retrieving revision 1.61 diff -u -p -r1.61 kern_exec.c --- kern/kern_exec.c +++ kern/kern_exec.c @@ -27,6 +27,8 @@ * $DragonFly: src/sys/kern/kern_exec.c,v 1.61 2007/07/30 17:41:23 pavalos Exp $ */ +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -66,6 +68,10 @@ #include #include +#ifdef HWPMC_HOOKS +#include +#endif + #include #include @@ -177,6 +183,10 @@ kern_execve(struct nlookupdata *nd, stru struct image_params image_params, *imgp; struct vattr attr; int (*img_first) (struct image_params *); + int credential_changing; +#ifdef HWPMC_HOOKS + struct pmckern_procexec pe; +#endif if (debug_execve_args) { kprintf("%s()\n", __func__); @@ -378,6 +388,7 @@ interpret: wakeup((caddr_t)p->p_pptr); } + credential_changing = 0; /* * Implement image setuid/setgid. * @@ -393,6 +404,7 @@ interpret: * root. Record any set-id flags first to make sure that * we do not regain any tracing during a possible block. */ + credential_changing = !0; setsugid(); if (p->p_tracenode && suser(td) != 0) { ktrdestroy(&p->p_tracenode); @@ -460,6 +472,23 @@ interpret: /* clear "fork but no exec" flag, as we _are_ execing */ p->p_acflag &= ~AFORK; +#ifdef HWPMC_HOOKS + /* + * Check if system-wide sampling is in effect or if the + * current process is using PMCs. If so, do exec() time + * processing. This processing needs to happen AFTER the + * P_INEXEC flag is cleared. + * + * The proc lock needs to be released before taking the PMC + * SX. + */ + if (PMC_SYSTEM_SAMPLING_ACTIVE() || PMC_PROC_IS_USING_PMCS(p)) { + pe.pm_credentialschanged = credential_changing; + pe.pm_entryaddr = imgp->entry_addr; + + PMC_CALL_HOOK_X(td, PMC_FN_PROCESS_EXEC, (void *) &pe); + } +#endif /* Set values passed into the program in registers. */ exec_setregs(imgp->entry_addr, (u_long)(uintptr_t)stack_base, imgp->ps_strings); Index: kern/kern_linker.c =================================================================== retrieving revision 1.38 diff -u -p -r1.38 kern_linker.c --- kern/kern_linker.c +++ kern/kern_linker.c @@ -28,6 +28,7 @@ */ #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include #include @@ -51,6 +52,10 @@ #include #endif +#ifdef HWPMC_HOOKS +#include +#endif + #ifdef KLD_DEBUG int kld_debug = 0; #endif @@ -359,6 +364,20 @@ linker_find_file_by_id(int fileid) return lf; } +static int +linker_file_foreach(int (*predicate)(linker_file_t, void*), void *context) +{ + linker_file_t lf; + int retval = 0; + + TAILQ_FOREACH(lf, &linker_files, link) { + retval = predicate(lf, context); + if (retval != 0) + break; + } + return (retval); +} + linker_file_t linker_make_file(const char* pathname, void* priv, struct linker_file_ops* ops) { @@ -713,6 +732,9 @@ linker_ddb_symbol_values(c_linker_sym_t int sys_kldload(struct kldload_args *uap) { +#ifdef HWPMC_HOOKS + struct pmckern_map_in pkm; +#endif struct thread *td = curthread; char* filename = NULL, *modulename; linker_file_t lf; @@ -743,6 +765,11 @@ sys_kldload(struct kldload_args *uap) if ((error = linker_load_file(filename, &lf)) != 0) goto out; +#ifdef HWPMC_HOOKS + pkm.pm_file = lf->filename; + pkm.pm_address = (uintptr_t) lf->address; + PMC_CALL_HOOK(td, PMC_FN_KLD_LOAD, (void *) &pkm); +#endif lf->userrefs++; uap->sysmsg_result = lf->id; @@ -756,6 +783,9 @@ out: int sys_kldunload(struct kldunload_args *uap) { +#ifdef HWPMC_HOOKS + struct pmckern_map_out pkm; +#endif struct thread *td = curthread; linker_file_t lf; int error = 0; @@ -774,6 +804,11 @@ sys_kldunload(struct kldunload_args *uap error = EBUSY; goto out; } +#ifdef HWPMC_HOOKS + /* Save data needed by hwpmc(4) before unloading. */ + pkm.pm_address = (uintptr_t) lf->address; + pkm.pm_size = lf->size; +#endif lf->userrefs--; error = linker_file_unload(lf); if (error) @@ -782,6 +817,10 @@ sys_kldunload(struct kldunload_args *uap error = ENOENT; out: +#ifdef HWPMC_HOOKS + if (error == 0) + PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm); +#endif return error; } @@ -1169,3 +1208,63 @@ linker_search_path(const char *name) } return(NULL); } + +#ifdef HWPMC_HOOKS + +struct hwpmc_context { + int nobjects; + int nmappings; + struct pmckern_map_in *kobase; +}; + +static int +linker_hwpmc_list_object(linker_file_t lf, void *arg) +{ + struct hwpmc_context *hc; + + hc = arg; + + /* If we run out of mappings, fail. */ + if (hc->nobjects >= hc->nmappings) + return (1); + + /* Save the info for this linker file. */ + hc->kobase[hc->nobjects].pm_file = lf->filename; + hc->kobase[hc->nobjects].pm_address = (uintptr_t)lf->address; + hc->nobjects++; + return (0); +} + +/* + * Inform hwpmc about the set of kernel modules currently loaded. + */ +void * +linker_hwpmc_list_objects(void) +{ + struct hwpmc_context hc; + + hc.nmappings = 15; /* a reasonable default */ + + retry: + /* allocate nmappings+1 entries */ + MALLOC(hc.kobase, struct pmckern_map_in *, + (hc.nmappings + 1) * sizeof(struct pmckern_map_in), M_LINKER, + M_WAITOK | M_ZERO); + + hc.nobjects = 0; + if (linker_file_foreach(linker_hwpmc_list_object, &hc) != 0) { + hc.nmappings = hc.nobjects; + FREE(hc.kobase, M_LINKER); + goto retry; + } + + KASSERT(hc.nobjects > 0, ("linker_hpwmc_list_objects: no kernel " + "objects?")); + + /* The last entry of the malloced area comprises of all zeros. */ + KASSERT(hc.kobase[hc.nobjects].pm_file == NULL, + ("linker_hwpmc_list_objects: last object not NULL")); + + return ((void *)hc.kobase); +} +#endif Index: kern/kern_timeout.c =================================================================== retrieving revision 1.26 diff -u -p -r1.26 kern_timeout.c --- kern/kern_timeout.c +++ kern/kern_timeout.c @@ -99,6 +99,7 @@ * The per-cpu augmentation was done by Matthew Dillon. */ +#include "opt_hwpmc_hooks.h" #include "opt_ddb.h" #include @@ -110,6 +111,10 @@ #include #include +#ifdef HWPMC_HOOKS +#include +#endif + #ifndef MAX_SOFTCLOCK_STEPS #define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */ #endif @@ -217,6 +222,13 @@ hardclock_softtick(globaldata_t gd) ++sc->curticks; if (sc->isrunning) return; +#ifdef HWPMC_HOOKS + /* + * XXX: does hook sleep? is that ok here? -- agg + */ + if (PMC_CPU_HAS_SAMPLES(mycpu->gd_cpuid)) + PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL); +#endif if (sc->softticks == sc->curticks) { /* * in sync, only wakeup the thread if there is something to Index: kern/lwkt_thread.c =================================================================== retrieving revision 1.109 diff -u -p -r1.109 lwkt_thread.c --- kern/lwkt_thread.c +++ kern/lwkt_thread.c @@ -43,6 +43,8 @@ #ifdef _KERNEL +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -93,6 +95,10 @@ #endif +#ifdef HWPMC_HOOKS +#include +#endif + static int untimely_switch = 0; #ifdef INVARIANTS static int panic_on_cscount = 0; @@ -749,7 +755,15 @@ using_idle_thread: #endif if (td != ntd) { ++switch_count; +#ifdef HWPMC_HOOKS + if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc)) + PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); +#endif td->td_switch(ntd); +#ifdef HWPMC_HOOKS + if (td->td_proc && PMC_PROC_IS_USING_PMCS(td->td_proc)) + PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN); +#endif } /* NOTE: current cpu may have changed after switch */ crit_exit_quick(td); Index: vm/vm_mmap.c =================================================================== retrieving revision 1.39 diff -u -p -r1.39 vm_mmap.c --- vm/vm_mmap.c +++ vm/vm_mmap.c @@ -46,6 +46,8 @@ * Mapped file (mmap) interface to VM */ +#include "opt_hwpmc_hooks.h" + #include #include #include @@ -77,6 +79,10 @@ #include #include +#ifdef HWPMC_HOOKS +#include +#endif + #include #include @@ -148,6 +154,10 @@ int kern_mmap(struct vmspace *vms, caddr_t uaddr, size_t ulen, int uprot, int uflags, int fd, off_t upos, void **res) { +#ifdef HWPMC_HOOKS + struct pmckern_map_in pkm; + objtype_t handle_type; +#endif struct thread *td = curthread; struct proc *p = td->td_proc; struct file *fp = NULL; @@ -248,6 +258,7 @@ kern_mmap(struct vmspace *vms, caddr_t u * Mapping blank space is trivial. */ handle = NULL; + handle_type = OBJT_DEFAULT; maxprot = VM_PROT_ALL; pos = 0; } else { @@ -312,6 +323,7 @@ kern_mmap(struct vmspace *vms, caddr_t u */ if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) { handle = NULL; + handle_type = OBJT_DEFAULT; maxprot = VM_PROT_ALL; flags |= MAP_ANON; pos = 0; @@ -385,6 +397,7 @@ kern_mmap(struct vmspace *vms, caddr_t u maxprot |= VM_PROT_WRITE; } handle = (void *)vp; + handle_type = OBJT_VNODE; } } @@ -401,6 +414,15 @@ kern_mmap(struct vmspace *vms, caddr_t u error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, flags, handle, pos); +#ifdef HWPMC_HOOKS + /* inform hwpmc(4) if an executable is being mapped */ + if (error == 0 && handle_type == OBJT_VNODE && + (prot & PROT_EXEC)) { + pkm.pm_file = handle; + pkm.pm_address = (uintptr_t) addr; + PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); + } +#endif if (error == 0) *res = (void *)(addr + pageoff); done: @@ -495,6 +517,10 @@ sys_msync(struct msync_args *uap) int sys_munmap(struct munmap_args *uap) { +#ifdef HWPMC_HOOKS + struct pmckern_map_out pkm; + vm_map_entry_t entry; +#endif struct proc *p = curproc; vm_offset_t addr; vm_size_t size, pageoff; @@ -527,6 +553,26 @@ sys_munmap(struct munmap_args *uap) */ if (!vm_map_check_protection(map, addr, addr + size, VM_PROT_NONE)) return (EINVAL); +#ifdef HWPMC_HOOKS + /* + * Inform hwpmc if the address range being unmapped contains + * an executable region. + */ + if (vm_map_lookup_entry(map, addr, &entry)) { + for (; + entry != &map->header && entry->start < addr + size; + entry = entry->next) { + if (vm_map_check_protection(map, entry->start, + entry->end, VM_PROT_EXECUTE) == TRUE) { + pkm.pm_address = (uintptr_t) addr; + pkm.pm_size = (size_t) size; + PMC_CALL_HOOK(curthread, PMC_FN_MUNMAP, + (void *) &pkm); + break; + } + } + } +#endif /* returns nothing but KERN_SUCCESS anyway */ vm_map_remove(map, addr, addr + size); return (0); Index: sys/linker.h =================================================================== retrieving revision 1.10 diff -u -p -r1.10 linker.h --- sys/linker.h +++ sys/linker.h @@ -225,6 +225,9 @@ int linker_ddb_search_symbol(caddr_t _va int linker_ddb_symbol_values(c_linker_sym_t _sym, linker_symval_t *_symval); +/* HWPMC helper */ +void *linker_hwpmc_list_objects(void); + #endif /* _KERNEL */ /* Index: cpu/i386/include/cpu.h =================================================================== RCS file: /home/aggelos/imports/vcs/dcvs/src/sys/cpu/i386/include/cpu.h,v retrieving revision 1.25 diff -u -p -r1.25 cpu.h --- cpu/i386/include/cpu.h 1 Mar 2007 01:46:52 -0000 1.25 +++ cpu/i386/include/cpu.h 29 Aug 2007 23:14:39 -0000 @@ -67,6 +67,10 @@ #define CLKF_INTR(framep) (mycpu->gd_intr_nesting_level > 1 || (curthread->td_flags & TDF_INTTHREAD)) #define CLKF_PC(framep) ((framep)->if_eip) +#define TRAPF_USERMODE(framep) \ + ((ISPL((framep)->tf_cs) == SEL_UPL) || ((framep)->tf_eflags & PSL_VM)) +#define TRAPF_PC(framep) ((framep)->tf_eip) + /* * Preempt the current process if in interrupt from user mode, * or after the current trap/syscall if in system mode.