Something appears to have disabled IRQs for 657 usecs.
Lee
preemption latency trace v1.1.5 on 2.6.13-rt12
--------------------------------------------------------------------
latency: 657 us, #950/950, CPU#0 | (M:rt VP:0, KP:1, SP:1 HP:1)
-----------------
| task: epiphany-14475 (uid:1000 nice:0 policy:0 rt_prio:0)
-----------------
_------=> CPU#
/ _-----=> irqs-off
| / _----=> need-resched
|| / _---=> hardirq/softirq
||| / _--=> preempt-depth
|||| /
||||| delay
cmd pid ||||| time | caller
\ / ||||| \ | /
resolvco-17100 0Dnh2 0us : __trace_start_sched_wakeup (try_to_wake_up)
resolvco-17100 0Dnh2 1us : __trace_start_sched_wakeup <epiphany-14475> (73 0)
resolvco-17100 0Dnh1 1us : try_to_wake_up <epiphany-14475> (73 7d)
resolvco-17100 0Dnh. 2us : check_raw_flags (try_to_wake_up)
resolvco-17100 0Dnh. 2us : preempt_schedule (try_to_wake_up)
resolvco-17100 0Dnh. 3us : wake_up_process_mutex (__up_mutex_waiter_savestate)
resolvco-17100 0Dnh. 4us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 5us : __mod_page_state (free_pte_range)
resolvco-17100 0Dnh. 5us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 6us : check_raw_flags (__mod_page_state)
resolvco-17100 0Dn.. 7us : _spin_unlock (exit_mmap)
resolvco-17100 0Dn.. 7us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 8us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 8us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 9us : _spin_unlock (exit_mmap)
resolvco-17100 0Dn.. 10us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 10us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 11us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 12us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 12us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 13us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 14us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 15us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 16us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 17us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 17us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 18us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 19us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 19us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 20us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 20us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 21us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 22us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 22us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 23us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 24us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 24us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 25us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 26us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 26us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 27us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 28us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 28us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 29us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 30us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 30us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 31us : __fput (fput)
resolvco-17100 0Dn.. 32us : __might_sleep (__fput)
resolvco-17100 0Dn.. 32us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 33us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 34us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 35us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 35us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 36us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 37us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 38us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 39us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 40us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 41us : file_kill (__fput)
resolvco-17100 0Dn.. 42us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 43us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 43us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 44us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 44us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 45us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 46us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 46us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 47us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 48us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 48us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 49us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 50us : dput (__fput)
resolvco-17100 0Dn.. 51us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 51us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 52us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 52us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 53us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 54us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 55us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 56us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 56us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 57us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 58us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 58us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 59us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 60us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 61us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 61us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 62us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 63us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 63us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 64us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 64us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 65us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 65us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 66us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 67us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 67us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 68us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 69us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 70us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 70us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 71us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 71us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 72us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 73us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 73us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 74us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 75us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 75us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 76us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 77us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 77us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 78us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 79us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 79us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 80us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 80us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 81us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 82us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 83us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 84us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 84us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 85us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 86us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 87us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 87us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 88us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 89us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 89us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 90us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 91us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 91us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 92us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 92us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 93us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 94us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 95us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 95us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 96us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 97us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 98us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 98us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 99us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 99us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 100us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 101us : __fput (fput)
resolvco-17100 0Dn.. 102us : __might_sleep (__fput)
resolvco-17100 0Dn.. 102us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 103us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 104us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 105us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 105us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 106us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 107us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 107us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 108us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 109us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 109us : file_kill (__fput)
resolvco-17100 0Dn.. 110us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 111us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 111us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 112us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 113us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 113us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 114us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 114us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 115us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 116us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 116us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 117us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 118us : dput (__fput)
resolvco-17100 0Dn.. 118us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 119us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 120us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 120us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 121us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 122us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 122us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 123us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 124us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 124us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 125us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 126us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 126us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 127us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 128us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 133us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 134us : memcpy (memmove)
resolvco-17100 0Dn.. 135us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 135us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 136us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 136us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 137us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 138us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 139us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 139us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 140us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 140us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 141us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 142us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 142us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 143us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 144us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 145us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 146us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 146us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 147us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 148us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 148us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 149us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 150us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 150us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 151us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 151us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 152us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 153us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 153us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 154us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 155us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 155us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 156us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 157us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 157us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 158us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 159us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 159us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 160us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 161us : __fput (fput)
resolvco-17100 0Dn.. 161us : __might_sleep (__fput)
resolvco-17100 0Dn.. 162us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 163us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 164us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 164us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 165us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 165us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 166us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 167us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 168us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 168us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 169us : file_kill (__fput)
resolvco-17100 0Dn.. 170us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 170us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 171us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 171us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 172us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 173us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 173us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 174us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 175us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 175us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 176us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 176us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 177us : dput (__fput)
resolvco-17100 0Dn.. 178us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 179us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 179us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 180us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 181us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 181us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 182us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 183us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 183us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 184us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 185us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 185us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 186us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 187us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 187us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 188us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 188us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 189us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 190us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 190us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 191us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 192us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 195us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 195us : memcpy (memmove)
resolvco-17100 0Dn.. 196us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 197us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 197us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 198us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 199us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 199us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 200us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 201us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 201us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 202us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 203us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 203us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 204us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 205us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 205us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 206us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 207us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 207us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 208us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 208us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 209us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 210us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 211us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 211us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 212us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 213us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 213us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 214us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 215us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 215us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 216us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 217us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 217us : __fput (fput)
resolvco-17100 0Dn.. 218us : __might_sleep (__fput)
resolvco-17100 0Dn.. 219us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 219us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 220us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 221us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 221us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 222us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 223us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 223us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 224us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 225us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 226us : file_kill (__fput)
resolvco-17100 0Dn.. 226us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 227us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 228us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 228us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 229us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 230us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 230us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 231us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 231us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 232us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 233us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 233us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 234us : dput (__fput)
resolvco-17100 0Dn.. 235us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 235us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 236us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 236us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 237us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 238us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 239us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 239us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 240us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 241us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 241us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 242us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 243us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 243us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 244us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 244us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 245us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 246us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 246us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 247us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 247us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 248us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 249us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 249us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 250us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 251us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 251us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 252us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 253us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 253us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 254us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 254us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 255us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 256us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 257us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 257us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 258us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 259us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 259us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 260us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 261us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 261us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 262us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 263us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 263us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 264us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 265us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 265us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 266us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 266us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 267us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 268us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 268us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 269us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 270us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 270us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 271us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 272us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 272us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 273us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 274us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 274us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 275us : __fput (fput)
resolvco-17100 0Dn.. 276us : __might_sleep (__fput)
resolvco-17100 0Dn.. 277us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 277us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 278us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 279us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 279us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 280us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 281us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 281us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 282us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 283us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 283us : file_kill (__fput)
resolvco-17100 0Dn.. 284us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 285us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 285us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 286us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 286us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 287us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 288us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 288us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 289us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 290us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 290us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 291us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 292us : dput (__fput)
resolvco-17100 0Dn.. 292us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 293us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 294us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 294us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 295us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 296us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 296us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 297us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 298us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 298us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 299us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 300us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 300us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 301us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 301us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 302us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 303us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 303us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 304us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 305us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 305us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 306us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 306us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 307us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 308us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 308us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 309us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 310us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 311us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 311us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 312us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 312us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 313us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 314us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 314us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 315us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 315us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 316us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 317us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 317us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 318us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 319us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 319us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 320us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 320us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 321us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 322us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 322us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 323us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 324us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 324us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 325us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 326us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 326us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 327us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 328us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 328us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 329us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 330us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 330us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 331us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 331us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 332us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 333us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 333us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 334us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 335us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 335us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 336us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 337us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 337us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 338us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 338us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 339us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 340us : __fput (fput)
resolvco-17100 0Dn.. 341us : __might_sleep (__fput)
resolvco-17100 0Dn.. 341us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 342us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 343us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 343us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 344us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 344us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 345us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 346us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 347us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 347us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 348us : file_kill (__fput)
resolvco-17100 0Dn.. 349us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 349us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 350us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 351us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 351us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 352us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 353us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 353us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 354us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 355us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 358us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 359us : memcpy (memmove)
resolvco-17100 0Dn.. 360us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 360us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 361us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 362us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 362us : dput (__fput)
resolvco-17100 0Dn.. 363us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 364us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 364us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 365us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 366us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 366us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 367us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 367us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 368us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 369us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 370us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 370us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 371us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 371us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 372us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 372us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 373us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 374us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 374us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 375us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 376us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 376us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 380us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 381us : memcpy (memmove)
resolvco-17100 0Dn.. 382us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 382us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 383us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 383us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 384us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 385us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 385us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 386us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 387us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 388us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 388us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 389us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 390us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 390us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 391us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 391us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 392us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 393us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 393us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 394us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 395us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 395us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 396us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 396us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 397us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 398us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 398us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 399us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 400us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 401us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 401us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 402us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 403us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 404us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 404us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 405us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 406us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 406us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 407us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 407us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 408us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 409us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 409us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 410us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 411us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 411us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 412us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 413us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 413us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 414us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 415us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 415us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 416us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 417us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 417us : __fput (fput)
resolvco-17100 0Dn.. 418us : __might_sleep (__fput)
resolvco-17100 0Dn.. 419us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 419us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 420us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 421us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 421us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 422us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 423us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 423us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 424us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 425us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 426us : file_kill (__fput)
resolvco-17100 0Dn.. 426us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 427us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 427us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 428us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 429us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 430us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 430us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 431us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 431us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 432us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 432us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 433us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 434us : dput (__fput)
resolvco-17100 0Dn.. 435us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 435us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 436us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 436us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 437us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 438us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 439us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 439us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 440us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 441us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 441us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 442us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 443us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 443us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 444us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 444us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 445us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 446us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 446us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 447us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 448us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 448us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 449us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 449us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 450us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 451us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 452us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 453us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 453us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 454us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 455us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 455us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 456us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 457us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 458us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 458us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 459us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 459us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 460us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 460us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 461us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 462us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 463us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 463us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 464us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 465us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 465us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 466us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 467us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 467us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 468us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 469us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 469us : __fput (fput)
resolvco-17100 0Dn.. 470us : __might_sleep (__fput)
resolvco-17100 0Dn.. 471us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 471us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 472us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 473us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 473us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 474us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 475us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 476us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 476us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 477us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 478us : file_kill (__fput)
resolvco-17100 0Dn.. 478us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 479us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 479us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 480us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 481us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 482us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 482us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 483us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 483us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 484us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 485us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 485us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 486us : dput (__fput)
resolvco-17100 0Dn.. 487us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 487us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 488us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 488us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 489us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 490us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 491us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 491us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 492us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 493us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 493us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 494us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 495us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 495us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 496us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 499us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 500us : memcpy (memmove)
resolvco-17100 0Dn.. 501us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 501us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 502us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 503us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 503us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 504us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 505us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 505us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 506us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 506us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 507us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 508us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 509us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 510us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 510us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 511us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 512us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 512us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 513us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 514us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 514us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 515us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 515us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 516us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 517us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 517us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 518us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 519us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 519us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 520us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 520us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 521us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 522us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 522us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 523us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 524us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 524us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 525us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 526us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 526us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 527us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 528us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 528us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 529us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 529us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 530us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 531us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 531us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 532us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 533us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 533us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 534us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 534us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 535us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 536us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 536us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 537us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 538us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 539us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 539us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 540us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 541us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 542us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 542us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 543us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 543us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 544us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 545us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 545us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 546us : cache_flusharray (kmem_cache_free)
resolvco-17100 0Dn.. 546us+: free_block (cache_flusharray)
resolvco-17100 0Dn.. 549us : memmove (cache_flusharray)
resolvco-17100 0Dn.. 550us : memcpy (memmove)
resolvco-17100 0Dn.. 550us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 551us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 552us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 552us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 553us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 554us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 554us : _spin_lock (remove_vm_struct)
resolvco-17100 0Dn.. 555us : __remove_shared_vm_struct (remove_vm_struct)
resolvco-17100 0Dn.. 556us : vma_prio_tree_remove (__remove_shared_vm_struct)
resolvco-17100 0Dn.. 556us : _spin_unlock (remove_vm_struct)
resolvco-17100 0Dn.. 557us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 558us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 558us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 559us : fput (remove_vm_struct)
resolvco-17100 0Dn.. 560us : __fput (fput)
resolvco-17100 0Dn.. 560us : __might_sleep (__fput)
resolvco-17100 0Dn.. 561us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 562us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 562us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 563us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 563us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 564us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 565us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 566us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 567us : ext3_release_file (__fput)
resolvco-17100 0Dn.. 567us : dummy_file_free_security (__fput)
resolvco-17100 0Dn.. 568us : file_kill (__fput)
resolvco-17100 0Dn.. 568us : _spin_lock (file_kill)
resolvco-17100 0Dn.. 569us : _spin_unlock (file_kill)
resolvco-17100 0Dn.. 570us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 570us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 571us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 572us : kmem_cache_free (__fput)
resolvco-17100 0Dn.. 572us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 573us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 574us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 574us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 575us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 576us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 576us : dput (__fput)
resolvco-17100 0Dn.. 577us : atomic_dec_and_spin_lock (dput)
resolvco-17100 0Dn.. 578us : up_mutex (atomic_dec_and_spin_lock)
resolvco-17100 0Dnh. 578us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 579us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 580us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 580us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 581us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 582us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 582us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 583us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 584us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 584us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 585us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 585us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 586us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 587us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 587us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 588us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 589us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 589us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 590us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 590us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 591us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 592us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 592us : remove_vm_struct (exit_mmap)
resolvco-17100 0Dn.. 593us : __might_sleep (remove_vm_struct)
resolvco-17100 0Dn.. 594us : anon_vma_unlink (remove_vm_struct)
resolvco-17100 0Dn.. 594us : _spin_lock (anon_vma_unlink)
resolvco-17100 0Dn.. 595us : _spin_unlock (anon_vma_unlink)
resolvco-17100 0Dn.. 596us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 596us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 597us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 598us : kmem_cache_free (anon_vma_unlink)
resolvco-17100 0Dn.. 598us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 599us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 599us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 600us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 601us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 601us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 602us : kmem_cache_free (remove_vm_struct)
resolvco-17100 0Dn.. 603us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 603us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 604us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 604us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 605us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 606us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 607us : __mmdrop (mmput)
resolvco-17100 0Dn.. 608us : pgd_free (__mmdrop)
resolvco-17100 0Dn.. 608us : kmem_cache_free (pgd_free)
resolvco-17100 0Dn.. 609us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 610us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 610us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 611us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 612us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 612us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 613us : destroy_context (__mmdrop)
resolvco-17100 0Dn.. 614us : kmem_cache_free (__mmdrop)
resolvco-17100 0Dn.. 615us : __local_save_flags (kmem_cache_free)
resolvco-17100 0Dn.. 615us : _spin_lock (kmem_cache_free)
resolvco-17100 0Dn.. 616us : _spin_unlock (kmem_cache_free)
resolvco-17100 0Dn.. 617us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 617us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 618us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 619us : exit_sem (do_exit)
resolvco-17100 0Dn.. 620us : _spin_lock (do_exit)
resolvco-17100 0Dn.. 621us : _spin_unlock (do_exit)
resolvco-17100 0Dn.. 621us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 622us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 622us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 623us : put_files_struct (do_exit)
resolvco-17100 0Dn.. 625us : filp_close (put_files_struct)
resolvco-17100 0Dn.. 626us : dnotify_flush (filp_close)
resolvco-17100 0Dn.. 627us : locks_remove_posix (filp_close)
resolvco-17100 0Dn.. 627us : fput (filp_close)
resolvco-17100 0Dn.. 628us : __fput (fput)
resolvco-17100 0Dn.. 628us : __might_sleep (__fput)
resolvco-17100 0Dn.. 629us : inotify_dentry_parent_queue_event (__fput)
resolvco-17100 0Dn.. 630us : _spin_lock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 631us : _spin_unlock (inotify_dentry_parent_queue_event)
resolvco-17100 0Dn.. 631us : up_mutex (_spin_unlock)
resolvco-17100 0Dnh. 632us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 633us : check_raw_flags (up_mutex)
resolvco-17100 0Dn.. 633us : inotify_inode_queue_event (__fput)
resolvco-17100 0Dn.. 634us : locks_remove_flock (__fput)
resolvco-17100 0Dn.. 635us : pipe_read_release (__fput)
resolvco-17100 0Dn.. 636us : pipe_read_fasync (pipe_read_release)
resolvco-17100 0Dn.. 636us : rt_down (pipe_read_fasync)
resolvco-17100 0Dn.. 637us : fasync_helper (pipe_read_fasync)
resolvco-17100 0Dn.. 638us : _write_lock_irq (fasync_helper)
resolvco-17100 0Dn.. 639us : _write_unlock_irq (fasync_helper)
resolvco-17100 0Dn.. 639us : up_write_mutex (_write_unlock_irq)
resolvco-17100 0Dnh. 640us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 641us : check_raw_flags (up_write_mutex)
resolvco-17100 0Dn.. 641us : rt_up (pipe_read_fasync)
resolvco-17100 0Dnh1 642us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh1 643us : check_raw_flags (rt_up)
resolvco-17100 0Dn.. 644us : preempt_schedule (rt_up)
resolvco-17100 0Dnh. 645us : __local_save_flags (trace_irqs_off)
resolvco-17100 0Dnh. 646us : __schedule (preempt_schedule)
resolvco-17100 0Dnh. 646us : profile_hit (__schedule)
resolvco-17100 0Dnh1 647us : sched_clock (__schedule)
resolvco-17100 0Dnh1 648us : __local_save_flags (trace_irqs_off)
epiphany-14475 0Dnh2 652us : __switch_to (__schedule)
epiphany-14475 0Dnh2 653us : __schedule <resolvco-17100> (7d 73)
epiphany-14475 0...1 654us : trace_stop_sched_switched (__schedule)
epiphany-14475 0Dnh1 654us : __local_save_flags (trace_irqs_off)
epiphany-14475 0Dnh2 655us : trace_stop_sched_switched <epiphany-14475> (73 0)
epiphany-14475 0Dnh2 657us : trace_stop_sched_switched (__schedule)
vim:ft=help
* Lee Revell <[email protected]> wrote:
> Something appears to have disabled IRQs for 657 usecs.
hm ... i fixed one such bug in rt.c recently, but more could be lurking.
Could you send me your .config?
Ingo
Ingo Molnar <[email protected]> wrote:
>> Something appears to have disabled IRQs for 657 usecs.
>hm ... i fixed one such bug in rt.c recently, but more could be lurking.
>Could you send me your .config?
Ingo,
could this be the same bug that's been hitting me since 2.6.13* ?
usenetgateway (heavy used server) with scsi/gig-E ethernet which
crashes within couple of days.
Last kernel i tried was 2.6.14-rc3-git5 and that was the first kernel
which also reported:
------------
warning: many lost ticks.
Your time source seems to be instable or some driver is hogging interupts
rip release_console_sem+0x151/0x210
Falling back to HPET
printk: 4 messages suppressed.
------------
The crash itself:
scsi0:0:0:0: Attempting to queue an ABORT message:CDB: 0x2a 0x0 0x1 0x79 0x80 0x72 0x0 0x0 0x28 0x0
scsi0: At time of recovery, card was not paused
>>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<
scsi0: Dumping Card State at program address 0x26 Mode 0x33
Card was paused
HS_MAILBOX[0x0] INTCTL[0xc0]:(SWTMINTEN|SWTMINTMASK)
SEQINTSTAT[0x10]:(SEQ_SWTMRTO) SAVED_MODE[0x11]
DFFSTAT[0x33]:(CURRFIFO_NONE|FIFO0FREE|FIFO1FREE)
SCSISIGI[0x0]:(P_DATAOUT) SCSIPHASE[0x0] SCSIBUS[0x0]
LASTPHASE[0x1]:(P_DATAOUT|P_BUSFREE) SCSISEQ0[0x0]
SCSISEQ1[0x12]:(ENAUTOATNP|ENRSELI) SEQCTL0[0x0]
SEQINTCTL[0x0] SEQ_FLAGS[0x0] SEQ_FLAGS2[0x0] SSTAT0[0x0]
SSTAT1[0x0] SSTAT2[0x0] SSTAT3[0x0] PERRDIAG[0x0]
SIMODE1[0xa4]:(ENSCSIPERR|ENSCSIRST|ENSELTIMO)
LQISTAT0[0x0] LQISTAT1[0x0] LQISTAT2[0x0] LQOSTAT0[0x0]
LQOSTAT1[0x0] LQOSTAT2[0xe1]:(LQOSTOP0|LQOPKT)
SCB Count = 128 CMDS_PENDING = 32 LASTSCB 0x6 CURRSCB 0x1c NEXTSCB 0xff80
qinstart = 44079 qinfifonext = 44079
QINFIFO:
WAITING_TID_QUEUES:
Pending list:
14 FIFO_USE[0x0] SCB_CONTROL[0x60]:(TAG_ENB|DISCENB) SCB_SCSIID[0x7]
[SNIP]
------------
On all previous kernel crashes either scsi or ethernet suddenly quit working.
Since scsi driver is exact same version as 2.6.12-mm1 which for me keeps working
perfectly, it sounds logic that if the IRQ system gets fedup some driver is likely
to grind the kernel to a painfull halt.
Just my feedback
Danny
On Saturday 08 October 2005 13:33, Danny ter Haar wrote:
> Ingo,
> could this be the same bug that's been hitting me since 2.6.13* ?
> usenetgateway (heavy used server) with scsi/gig-E ethernet which
> crashes within couple of days.
Danny,
I wasn't sure from your post whether you are running Ingo's RT patchset or
not?
Also, maybe you should not run unstable kernels on a production machine? :-)
Felix Oxley <[email protected]> wrote:
>I wasn't sure from your post whether you are running Ingo's RT patchset or
>not?
<blush> am _not_ </blush>
>Also, maybe you should not run unstable kernels on a production machine? :-)
How else could i help the linux community ?
A machine which is 99.9% idle isn't going to get caught by bugs that easy.
Danny
* Lee Revell <[email protected]> wrote:
> Something appears to have disabled IRQs for 657 usecs.
how hard is it to reproduce this latency?
Ingo
On Tue, 2005-10-11 at 13:20 +0200, Ingo Molnar wrote:
> * Lee Revell <[email protected]> wrote:
>
> > Something appears to have disabled IRQs for 657 usecs.
>
> how hard is it to reproduce this latency?
Very easy. Here's the current max latency.
The max latency is close to the period of the timer IRQ. So either it's
an instrumentation bug or something really disables IRQs for up to 1/HZ
seconds.
preemption latency trace v1.1.5 on 2.6.13-rt12
--------------------------------------------------------------------
latency: 928 us, #1359/1359, CPU#0 | (M:rt VP:0, KP:1, SP:1 HP:1)
-----------------
| task: softirq-tasklet-7 (uid:0 nice:0 policy:1 rt_prio:1)
-----------------
_------=> CPU#
/ _-----=> irqs-off
| / _----=> need-resched
|| / _---=> hardirq/softirq
||| / _--=> preempt-depth
|||| /
||||| delay
cmd pid ||||| time | caller
\ / ||||| \ | /
kswapd0-100 0Dnh2 0us : __trace_start_sched_wakeup (try_to_wake_up)
kswapd0-100 0Dnh2 1us : __trace_start_sched_wakeup <softirq--7> (62 0)
kswapd0-100 0Dnh1 1us : preempt_schedule (__trace_start_sched_wakeup)
kswapd0-100 0Dnh1 2us : try_to_wake_up <softirq--7> (62 73)
kswapd0-100 0Dnh. 2us : check_raw_flags (try_to_wake_up)
kswapd0-100 0Dnh. 3us : preempt_schedule (try_to_wake_up)
kswapd0-100 0Dnh. 4us : wake_up_process_mutex (__up_mutex_waiter_savestate)
kswapd0-100 0Dnh. 4us+: check_raw_flags (up_mutex)
kswapd0-100 0Dn.. 8us : memmove (cache_flusharray)
kswapd0-100 0Dn.. 8us : memcpy (memmove)
kswapd0-100 0Dn.. 9us : _spin_unlock (kmem_cache_free)
kswapd0-100 0Dn.. 9us : up_mutex (_spin_unlock)
kswapd0-100 0Dnh. 10us : check_raw_flags (up_mutex)
kswapd0-100 0Dn.. 10us : clear_inode (dispose_list)
kswapd0-100 0Dn.. 11us : __might_sleep (clear_inode)
kswapd0-100 0Dn.. 12us : invalidate_inode_buffers (clear_inode)
kswapd0-100 0Dn.. 12us : inode_has_buffers (invalidate_inode_buffers)
kswapd0-100 0Dn.. 13us : __might_sleep (clear_inode)
kswapd0-100 0Dn.. 13us : ext3_clear_inode (clear_inode)
kswapd0-100 0Dn.. 14us : ext3_discard_reservation (ext3_clear_inode)
kswapd0-100 0Dn.. 15us : kfree (ext3_clear_inode)
kswapd0-100 0Dn.. 15us : _spin_lock (dispose_list)
kswapd0-100 0Dn.. 16us : _spin_unlock (dispose_list)
kswapd0-100 0Dn.. 17us : up_mutex (_spin_unlock)
kswapd0-100 0Dnh. 17us : check_raw_flags (up_mutex)
kswapd0-100 0Dn.. 18us : wake_up_inode (dispose_list)
kswapd0-100 0Dn.. 18us : wake_up_bit (wake_up_inode)
kswapd0-100 0Dn.. 19us : bit_waitqueue (wake_up_bit)
kswapd0-100 0Dn.. 20us : __wake_up_bit (wake_up_bit)
This part repeats:
kswapd0-100 0Dn.. 20us : destroy_inode (dispose_list)
kswapd0-100 0Dn.. 21us : inode_has_buffers (destroy_inode)
kswapd0-100 0Dn.. 22us : dummy_inode_free_security (destroy_inode)
kswapd0-100 0Dn.. 22us : ext3_destroy_inode (destroy_inode)
kswapd0-100 0Dn.. 23us : kmem_cache_free (ext3_destroy_inode)
kswapd0-100 0Dn.. 23us : __local_save_flags (kmem_cache_free)
kswapd0-100 0Dn.. 24us : _spin_lock (kmem_cache_free)
kswapd0-100 0Dn.. 24us : _spin_unlock (kmem_cache_free)
kswapd0-100 0Dn.. 25us : up_mutex (_spin_unlock)
kswapd0-100 0Dnh. 25us : check_raw_flags (up_mutex)
kswapd0-100 0Dn.. 26us : clear_inode (dispose_list)
kswapd0-100 0Dn.. 27us : __might_sleep (clear_inode)
kswapd0-100 0Dn.. 27us : invalidate_inode_buffers (clear_inode)
kswapd0-100 0Dn.. 28us : inode_has_buffers (invalidate_inode_buffers)
kswapd0-100 0Dn.. 29us : __might_sleep (clear_inode)
kswapd0-100 0Dn.. 29us : ext3_clear_inode (clear_inode)
kswapd0-100 0Dn.. 30us : ext3_discard_reservation (ext3_clear_inode)
kswapd0-100 0Dn.. 31us : kfree (ext3_clear_inode)
kswapd0-100 0Dn.. 31us : _spin_lock (dispose_list)
kswapd0-100 0Dn.. 33us : _spin_unlock (dispose_list)
kswapd0-100 0Dn.. 33us : up_mutex (_spin_unlock)
kswapd0-100 0Dnh. 34us : check_raw_flags (up_mutex)
kswapd0-100 0Dn.. 34us : wake_up_inode (dispose_list)
kswapd0-100 0Dn.. 35us : wake_up_bit (wake_up_inode)
kswapd0-100 0Dn.. 36us : bit_waitqueue (wake_up_bit)
kswapd0-100 0Dn.. 36us : __wake_up_bit (wake_up_bit)
kswapd0-100 0Dn.. 37us : destroy_inode (dispose_list)
kswapd0-100 0Dn.. 38us : inode_has_buffers (destroy_inode)
etc
Finally the timer interrupt saves the day:
kswapd0-100 0Dnh. 859us : do_IRQ (c0150297 0 0)
kswapd0-100 0Dnh1 861us+: mask_and_ack_8259A (__do_IRQ)
kswapd0-100 0Dnh1 865us : check_raw_flags (mask_and_ack_8259A)
kswapd0-100 0Dnh1 866us : preempt_schedule (mask_and_ack_8259A)
kswapd0-100 0Dnh1 866us : redirect_hardirq (__do_IRQ)
kswapd0-100 0Dnh. 867us : preempt_schedule (__do_IRQ)
kswapd0-100 0Dnh. 868us : handle_IRQ_event (__do_IRQ)
kswapd0-100 0Dnh. 869us : timer_interrupt (handle_IRQ_event)
kswapd0-100 0Dnh1 869us+: mark_offset_pmtmr (timer_interrupt)
kswapd0-100 0Dnh1 872us : preempt_schedule (mark_offset_pmtmr)
kswapd0-100 0Dnh1 873us : do_timer (timer_interrupt)
kswapd0-100 0Dnh1 874us : update_process_times (timer_interrupt)
kswapd0-100 0Dnh1 874us : account_system_time (update_process_times)
kswapd0-100 0Dnh1 876us : acct_update_integrals (account_system_time)
kswapd0-100 0Dnh1 876us : update_mem_hiwater (account_system_time)
kswapd0-100 0Dnh1 877us : run_local_timers (update_process_times)
kswapd0-100 0Dnh1 878us : raise_softirq (run_local_timers)
kswapd0-100 0Dnh1 878us : check_raw_flags (raise_softirq)
kswapd0-100 0Dnh1 879us : rcu_pending (update_process_times)
kswapd0-100 0Dnh1 880us : rcu_check_callbacks (update_process_times)
kswapd0-100 0Dnh1 880us : rcu_try_flip (rcu_check_callbacks)
kswapd0-100 0Dnh1 881us : check_raw_flags (rcu_try_flip)
kswapd0-100 0Dnh1 882us : preempt_schedule (rcu_try_flip)
kswapd0-100 0Dnh2 882us : __rcu_advance_callbacks (rcu_check_callbacks)
kswapd0-100 0Dnh1 883us : check_raw_flags (rcu_check_callbacks)
kswapd0-100 0Dnh1 884us : preempt_schedule (rcu_check_callbacks)
kswapd0-100 0Dnh1 885us : __tasklet_schedule (rcu_check_callbacks)
kswapd0-100 0Dnh1 885us : check_raw_flags (__tasklet_schedule)
kswapd0-100 0Dnh1 886us : scheduler_tick (update_process_times)
kswapd0-100 0Dnh1 887us : sched_clock (scheduler_tick)
kswapd0-100 0Dnh2 888us : task_timeslice (scheduler_tick)
kswapd0-100 0Dnh1 889us : preempt_schedule (scheduler_tick)
kswapd0-100 0Dnh1 890us : smp_local_timer_interrupt (timer_interrupt)
kswapd0-100 0Dnh. 891us : preempt_schedule (timer_interrupt)
kswapd0-100 0Dnh1 892us : note_interrupt (__do_IRQ)
kswapd0-100 0Dnh1 892us : enable_8259A_irq (__do_IRQ)
kswapd0-100 0Dnh1 894us : check_raw_flags (enable_8259A_irq)
kswapd0-100 0Dnh1 895us : preempt_schedule (enable_8259A_irq)
kswapd0-100 0Dnh. 896us : preempt_schedule (__do_IRQ)
kswapd0-100 0Dnh. 896us : irq_exit (do_IRQ)
kswapd0-100 0Dnh1 897us : do_softirq (irq_exit)
kswapd0-100 0Dnh1 898us : __do_softirq (do_softirq)
kswapd0-100 0Dnh1 899us : trigger_softirqs (__do_softirq)
kswapd0-100 0Dnh1 899us : wakeup_softirqd (trigger_softirqs)
kswapd0-100 0Dnh1 900us : wake_up_process (wakeup_softirqd)
kswapd0-100 0Dnh1 901us : check_preempt_wakeup (wake_up_process)
kswapd0-100 0Dnh1 901us : try_to_wake_up (wake_up_process)
kswapd0-100 0Dnh2 902us : activate_task (try_to_wake_up)
kswapd0-100 0Dnh2 903us : sched_clock (activate_task)
kswapd0-100 0Dnh2 904us : recalc_task_prio (activate_task)
kswapd0-100 0Dnh2 905us : __recalc_task_prio (recalc_task_prio)
kswapd0-100 0Dnh2 905us : __recalc_task_prio <<...>-3> (62 62)
kswapd0-100 0Dnh2 906us : activate_task <<...>-3> (62 3)
kswapd0-100 0Dnh2 907us : enqueue_task (activate_task)
kswapd0-100 0Dnh2 908us : __trace_start_sched_wakeup (try_to_wake_up)
kswapd0-100 0Dnh2 909us : preempt_schedule (__trace_start_sched_wakeup)
kswapd0-100 0Dnh2 909us : try_to_wake_up <<...>-3> (62 73)
kswapd0-100 0Dnh1 910us : check_raw_flags (try_to_wake_up)
kswapd0-100 0Dnh1 910us : preempt_schedule (try_to_wake_up)
kswapd0-100 0Dnh1 911us : wake_up_process (wakeup_softirqd)
kswapd0-100 0Dnh1 912us : wakeup_softirqd (trigger_softirqs)
kswapd0-100 0Dnh1 912us : wake_up_process (wakeup_softirqd)
kswapd0-100 0Dnh1 913us : check_preempt_wakeup (wake_up_process)
kswapd0-100 0Dnh1 913us : try_to_wake_up (wake_up_process)
kswapd0-100 0Dnh1 915us : check_raw_flags (try_to_wake_up)
kswapd0-100 0Dnh1 915us : preempt_schedule (try_to_wake_up)
kswapd0-100 0Dnh1 916us : wake_up_process (wakeup_softirqd)
kswapd0-100 0Dnh1 917us : check_raw_flags (do_softirq)
kswapd0-100 0Dnh. 917us : preempt_schedule_irq (need_resched)
kswapd0-100 0Dnh. 918us : __schedule (preempt_schedule_irq)
kswapd0-100 0Dnh. 919us : profile_hit (__schedule)
kswapd0-100 0Dnh1 920us : sched_clock (__schedule)
softirq--7 0Dnh2 923us : __switch_to (__schedule)
softirq--7 0Dnh2 924us : __schedule <kswapd0-100> (73 62)
softirq--7 0...1 925us : trace_stop_sched_switched (__schedule)
softirq--7 0Dnh2 926us : trace_stop_sched_switched <softirq--7> (62 0)
softirq--7 0Dnh2 927us : trace_stop_sched_switched (__schedule)
On Tue, 2005-10-11 at 12:28 -0400, Lee Revell wrote:
> On Tue, 2005-10-11 at 13:20 +0200, Ingo Molnar wrote:
> > * Lee Revell <[email protected]> wrote:
> >
> > > Something appears to have disabled IRQs for 657 usecs.
> >
> > how hard is it to reproduce this latency?
>
> Very easy. Here's the current max latency.
>
> The max latency is close to the period of the timer IRQ. So either it's
> an instrumentation bug or something really disables IRQs for up to 1/HZ
> seconds.
I can no longer reproduce this with 2.6.14-rc4-rt1. It seems to have
been a real IRQs disabled bug (not an instrumentation bug) as I was also
getting xruns with 2.6.13-rt12.
Lee
* Lee Revell <[email protected]> wrote:
> > The max latency is close to the period of the timer IRQ. So either it's
> > an instrumentation bug or something really disables IRQs for up to 1/HZ
> > seconds.
>
> I can no longer reproduce this with 2.6.14-rc4-rt1. It seems to have
> been a real IRQs disabled bug (not an instrumentation bug) as I was
> also getting xruns with 2.6.13-rt12.
ok, please re-report if it occurs again. I havent fixed any preemption
bug since -rt12, but maybe as a side-effect of other things? Some false
positive latencies were fixed in the latency tracer, but since you also
got xruns, they must have been real latencies. (albeit saving a long
false-positive latency trace can cause a latency by itself!)
Ingo