gernot@tintenfass:~# dtrace -n 'profile-1001 { @[stack()] = count() }' dtrace: description 'profile-1001 ' matched 1 probe ^C unix`i86_monitor+0x10 unix`cpu_idle_mwait+0xbe unix`idle+0x114 unix`thread_start+0x8 1 unix`mutex_enter+0x10 genunix`taskq_thread_wait+0x84 genunix`taskq_thread+0x308 unix`thread_start+0x8 1 unix`mutex_enter+0x10 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`mutex_enter+0x10 genunix`cdev_ioctl+0x45 specfs`spec_ioctl+0x5a genunix`fop_ioctl+0x7b genunix`ioctl+0x18e unix`sys_syscall+0x17a 1 unix`mutex_enter+0x10 unix`page_free+0x10d unix`page_destroy+0x104 unix`segkmem_free_vn+0xe6 unix`segkmem_free+0x27 genunix`vmem_xfree+0x104 genunix`vmem_free+0x29 unix`kfreea+0x54 unix`i_ddi_mem_free+0x5d rootnex`rootnex_teardown_copybuf+0x24 rootnex`rootnex_coredma_unbindhdl+0x90 rootnex`rootnex_dma_unbindhdl+0x35 genunix`ddi_dma_unbind_handle+0x41 ata`ghd_dmafree_attr+0x2b ata`ata_disk_memfree+0x20 gda`gda_free+0x39 dadk`dadk_iodone+0xbf dadk`dadk_pktcb+0xc6 ata`ata_disk_complete+0x119 ata`ata_hba_complete+0x38 1 i915`i915_gem_retire_requests+0xc i915`i915_gem_retire_work_handler+0x3d genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 1 SDC`sysdc_update+0x1f5 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 1 genunix`fsflush_do_pages+0xb3 genunix`fsflush+0x39a unix`thread_start+0x8 1 unix`cpu_idle_exit+0x1fc unix`cpu_idle_mwait+0xfb unix`idle+0x114 unix`thread_start+0x8 1 unix`tsc_read+0x5 genunix`gethrtime_unscaled+0xd genunix`syscall_mstate+0x4a unix`sys_syscall+0x10e 1 unix`tsc_read+0x5 genunix`gethrtime_unscaled+0xd genunix`new_mstate+0x4b unix`trap+0x1fc unix`0xfffffffffb8001d6 1 unix`page_nextn+0xfe genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 unix`tsc_read+0x9 genunix`gethrtime_unscaled+0xd genunix`syscall_mstate+0x4a unix`0xfffffffffb800c86 1 unix`sys_syscall32+0x102 1 genunix`ioctl+0x4a unix`sys_syscall+0x17a 1 unix`do_splx+0x8d unix`xc_common+0x231 unix`xc_call+0x46 unix`hat_tlb_inval+0x283 unix`x86pte_inval+0xaa unix`hat_pte_unmap+0xed unix`hat_unload_callback+0x193 unix`hat_unload+0x41 unix`segkmem_free_vn+0x6f unix`segkmem_free+0x27 genunix`vmem_xfree+0x104 genunix`vmem_free+0x29 unix`kfreea+0x54 unix`i_ddi_mem_free+0x5d rootnex`rootnex_teardown_copybuf+0x24 rootnex`rootnex_coredma_unbindhdl+0x90 rootnex`rootnex_dma_unbindhdl+0x35 genunix`ddi_dma_unbind_handle+0x41 ata`ghd_dmafree_attr+0x2b ata`ata_disk_memfree+0x20 1 unix`page_nextn+0x12 genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 unix`page_nextn+0x14 genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 genunix`callout_downheap+0x7e genunix`callout_heap_delete+0x18b genunix`callout_normal+0x27 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 genunix`syscall_mstate+0x3e unix`sys_syscall+0x1a1 1 unix`tsc_gethrtime+0x65 genunix`gethrtime+0xd genunix`timeout_generic+0x46 genunix`timeout+0x5b SDC`sysdc_update+0x311 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 1 unix`page_nextn+0x46 genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 sha1`sha1_block_data_order+0xdff 1 unix`ddi_get16+0x10 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 1 unix`cpu_idle_enter+0x109 unix`cpu_idle_mwait+0xdc unix`idle+0x114 unix`thread_start+0x8 1 unix`page_lookup_create+0xd0 unix`page_lookup+0x26 genunix`swap_getapage+0xaf genunix`swap_getpage+0x85 genunix`fop_getpage+0x9a genunix`anon_zero+0xa3 genunix`segvn_faultpage+0x2a4 genunix`segvn_fault+0xc13 genunix`as_fault+0x5ee unix`pagefault+0x99 unix`trap+0xe63 unix`0xfffffffffb8001d6 1 genunix`fsflush_do_pages+0x31d genunix`fsflush+0x39a unix`thread_start+0x8 1 genunix`cv_unsleep+0x64 genunix`setrun_locked+0x96 genunix`setrun+0x1e genunix`cv_wakeup+0x36 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_realtime+0x26 genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`tsc_gethrtime+0x8d genunix`gethrtime+0xd genunix`lbolt_event_driven+0x18 genunix`ddi_get_lbolt+0xd unix`setbackdq+0x122 genunix`sleepq_wakeone_chan+0x89 genunix`cv_signal+0x8e genunix`taskq_dispatch+0x37c genunix`callout_normal+0x121 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 dtrace`dtrace_dynvar_clean+0x31 dtrace`dtrace_state_clean+0x23 genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 genunix`pcache_poll+0x480 genunix`poll_common+0x427 genunix`pollsys+0xea unix`sys_syscall+0x17a 1 genunix`syscall_mstate+0x81 unix`sys_syscall+0x1a1 1 TS`ts_setrun+0x1d genunix`cv_unsleep+0x8c genunix`setrun_locked+0x96 genunix`setrun+0x1e genunix`cv_wakeup+0x36 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_realtime+0x26 genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 rootnex`rootnex_get_sgl+0x1 rootnex`rootnex_dma_bindhdl+0x4c genunix`ddi_dma_buf_bind_handle+0x117 ata`ghd_dma_buf_bind_attr+0x98 ata`ata_disk_memsetup+0xc3 gda`gda_pktprep+0xab dadk`dadk_pktprep+0x3a dadk`dadk_pkt+0x4d strategy`dmult_enque+0x7d dadk`dadk_strategy+0x7d cmdk`cmdkstrategy+0x16c genunix`bdev_strategy+0x75 genunix`ldi_strategy+0x59 zfs`vdev_disk_io_start+0xd0 zfs`zio_vdev_io_start+0x1ea zfs`zio_execute+0x8d genunix`taskq_thread+0x285 unix`thread_start+0x8 1 genunix`fsflush_do_pages+0x358 genunix`fsflush+0x39a unix`thread_start+0x8 1 genunix`read 1 unix`kcopy+0x1b genunix`copyin_nowatch+0x48 genunix`copyin_args32+0x3a genunix`syscall_entry+0xb3 unix`_sys_sysenter_post_swapgs+0x12a 1 unix`cmt_ev_thread_swtch+0x2c unix`pg_ev_thread_swtch+0x10d unix`swtch+0xdb genunix`cv_wait_sig_swap_core+0x174 genunix`cv_wait_sig_swap+0x18 genunix`cv_waituntil_sig+0x13c genunix`poll_common+0x47f genunix`pollsys+0xea unix`sys_syscall+0x17a 1 unix`page_nextn+0x9a genunix`fsflush_do_pages+0x104 genunix`fsflush+0x39a unix`thread_start+0x8 1 genunix`cpu_decay+0x27 genunix`cpu_update_pct+0x89 unix`setbackdq+0x2b3 genunix`sleepq_wakeone_chan+0x89 genunix`cv_signal+0x8e genunix`taskq_dispatch+0x37c genunix`callout_normal+0x121 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`ddi_io_put8+0xf ata`ata_disk_start_dma_out+0x88 ata`ata_ctlr_fsm+0x1fb ata`ata_hba_start+0x84 ata`ghd_waitq_process_and_mutex_hold+0xdf ata`ghd_intr+0x8d ata`ata_intr+0x27 unix`av_dispatch_autovect+0x7c unix`dispatch_hardint+0x33 unix`switch_sp_and_call+0x13 1 unix`ddi_io_put8+0xf ata`ata_disk_start_common+0xda ata`ata_disk_start_dma_out+0x32 ata`ata_ctlr_fsm+0x1fb ata`ata_hba_start+0x84 ata`ghd_waitq_process_and_mutex_hold+0xdf ata`ghd_intr+0x8d ata`ata_intr+0x27 unix`av_dispatch_autovect+0x7c unix`dispatch_hardint+0x33 unix`switch_sp_and_call+0x13 1 unix`rw_exit+0xf genunix`callout_heap_delete+0x1e3 genunix`callout_realtime+0x1e genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`setbackdq+0x328 genunix`cv_unsleep+0x8c genunix`setrun_locked+0x96 genunix`setrun+0x1e genunix`cv_wakeup+0x36 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_realtime+0x26 genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`av_check_softint_pending+0x8 unix`av_dispatch_softvect+0x48 unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 genunix`thread_lock+0x41 genunix`setrun+0x16 genunix`cv_wakeup+0x36 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_realtime+0x26 genunix`cyclic_softint+0xdc unix`cbe_low_level+0x17 unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 unix`splr+0x92 genunix`thread_lock+0x1d genunix`post_syscall+0x669 unix`0xfffffffffb800c91 1 genunix`fsflush_do_pages+0x395 genunix`fsflush+0x39a unix`thread_start+0x8 1 unix`plcnt_inc_dec+0xa3 unix`page_ctr_sub_internal+0x5d unix`page_ctr_sub+0x78 unix`page_get_mnode_freelist+0x346 unix`page_get_anylist+0x200 unix`page_create_io+0x1e8 unix`page_create_io_wrapper+0x57 unix`segkmem_xalloc+0xc0 unix`segkmem_alloc_io_4G+0x3b genunix`vmem_xalloc+0x546 genunix`vmem_alloc+0x161 unix`kalloca+0x203 unix`i_ddi_mem_alloc+0x173 rootnex`rootnex_setup_copybuf+0x11f rootnex`rootnex_bind_slowpath+0x70 rootnex`rootnex_coredma_bindhdl+0x334 rootnex`rootnex_dma_bindhdl+0x4c genunix`ddi_dma_buf_bind_handle+0x117 ata`ghd_dma_buf_bind_attr+0x98 ata`ata_disk_memsetup+0xc3 1 unix`mul32+0xd genunix`scalehrtime+0x19 genunix`cpu_update_pct+0x76 unix`setbackdq+0x2b3 genunix`sleepq_wakeone_chan+0x89 genunix`cv_signal+0x8e genunix`taskq_dispatch+0x37c genunix`callout_normal+0x121 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 1 genunix`cyclic_timer+0x89 genunix`cyclic_softint+0xdc unix`cbe_softclock+0x1a unix`av_dispatch_softvect+0x5f unix`dispatch_softint+0x34 unix`switch_sp_and_call+0x13 2 unix`atomic_swap_64+0x7 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 2 unix`page_nextn genunix`fsflush+0x39a unix`thread_start+0x8 2 unix`ddi_io_put16+0x10 genunix`callout_list_expire+0x77 genunix`callout_expire+0x31 genunix`callout_execute+0x1e genunix`taskq_thread+0x285 unix`thread_start+0x8 2 genunix`fsflush_do_pages+0x2fa genunix`fsflush+0x39a unix`thread_start+0x8 3 genunix`fsflush_do_pages+0x13e genunix`fsflush+0x39a unix`thread_start+0x8 4 unix`dispatch_softint+0x27 unix`switch_sp_and_call+0x13 5 genunix`fsflush_do_pages+0x124 genunix`fsflush+0x39a unix`thread_start+0x8 7 18 unix`i86_mwait+0xd unix`cpu_idle_mwait+0xf1 unix`idle+0x114 unix`thread_start+0x8 16862