Kernel Stack Trace of Kernel 2.6.43-r2

2.6.34-r2-kernel_stack_trace.txt (text/plain), 12.37 KB, created by Phillip Merensky on 2010-07-19 20:36:24 UTC

INFO: task mysqld:14337 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld        D 00000001001ac631     0 14337      1 0x00000008
 ffff8801fdc6e820 0000000000000086 ffff8801e1d17a88 0000000000004000
 ffff880009c51fd8 ffff880009c51fd8 ffff8801f1f85040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801f1f85040 00000000ae406820
Call Trace:
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd7b7>] ? __mutex_lock_slowpath+0x57/0x190
 [<ffffffff8111e5d8>] ? reiserfs_update_sd_size+0x308/0x330
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d86d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff81137c8b>] ? do_journal_begin_r+0x8b/0x350
 [<ffffffff814fd7b7>] ? __mutex_lock_slowpath+0x57/0x190
 [<ffffffff811380e1>] ? journal_begin+0x81/0x150
 [<ffffffff8111e7ec>] ? reiserfs_write_end+0x1ec/0x310
 [<ffffffff81074ca4>] ? generic_file_buffered_write+0x164/0x240
 [<ffffffff81076c18>] ? __generic_file_aio_write+0x278/0x450
 [<ffffffff81076e3f>] ? generic_file_aio_write+0x4f/0xb0
 [<ffffffff810b0bc0>] ? do_sync_write+0xc0/0x100
 [<ffffffff81002c4e>] ? apic_timer_interrupt+0xe/0x20
 [<ffffffff810b169b>] ? vfs_write+0xcb/0x180
 [<ffffffff810b1853>] ? sys_write+0x53/0xa0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task if_eth1:14429 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
if_eth1       D ffff8801ae406820     0 14429  14333 0x00000004
 ffff8801bb9ea040 0000000000000082 0000000000000000 0000000000004000
 ffff880009dbdfd8 ffff880009dbdfd8 ffff8801ae406820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801ae406820 ffff8801f1f85040
Call Trace:
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff81137c83>] ? do_journal_begin_r+0x83/0x350
 [<ffffffff814fd6a2>] ? mutex_unlock+0x12/0x20
 [<ffffffff811380e1>] ? journal_begin+0x81/0x150
 [<ffffffff8104f490>] ? autoremove_wake_function+0x0/0x30
 [<ffffffff81126f8e>] ? reiserfs_write_dquot+0x4e/0xc0
 [<ffffffff811278ab>] ? remove_save_link+0xdb/0x100
 [<ffffffff810fab6f>] ? dqput+0xff/0x1c0
 [<ffffffff810fac94>] ? __dquot_drop+0x64/0x80
 [<ffffffff810c5aff>] ? clear_inode+0x4f/0xf0
 [<ffffffff811215ba>] ? reiserfs_delete_inode+0x8a/0x160
 [<ffffffff810e07f6>] ? fsnotify_clear_marks_by_inode+0x26/0xe0
 [<ffffffff810c62ef>] ? generic_delete_inode+0x7f/0x130
 [<ffffffff810c21b7>] ? d_kill+0x47/0x80
 [<ffffffff810c29c7>] ? dput+0x97/0x160
 [<ffffffff810b235d>] ? __fput+0x13d/0x1d0
 [<ffffffff810aef46>] ? filp_close+0x56/0x90
 [<ffffffff810bef6a>] ? sys_dup3+0x13a/0x1a0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task flush-8:0:1591 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
flush-8:0     D 00000001001ae3b4     0  1591      2 0x00000000
 ffffffff8181f020 0000000000000046 0000000000000007 0000000000004000
 ffff8801fa367fd8 ffff8801fa367fd8 ffff8801fcea5040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801fcea5040 0000000004553d80
Call Trace:
 [<ffffffff810759e0>] ? sync_page+0x0/0x60
 [<ffffffff814fd07b>] ? io_schedule+0x4b/0x70
 [<ffffffff81075a15>] ? sync_page+0x35/0x60
 [<ffffffff814fd44a>] ? __wait_on_bit_lock+0x4a/0x90
 [<ffffffff810759be>] ? __lock_page+0x5e/0x70
 [<ffffffff8104f4c0>] ? wake_bit_function+0x0/0x30
 [<ffffffff8107ebfa>] ? pagevec_lookup_tag+0x1a/0x30
 [<ffffffff8107d363>] ? write_cache_pages+0x193/0x370
 [<ffffffff8107cbf0>] ? __writepage+0x0/0x40
 [<ffffffff810cf182>] ? writeback_single_inode+0xf2/0x370
 [<ffffffff810cff98>] ? writeback_inodes_wb+0x3a8/0x5e0
 [<ffffffff810d02f6>] ? wb_writeback+0x126/0x1f0
 [<ffffffff810d0624>] ? wb_do_writeback+0x184/0x190
 [<ffffffff810d066b>] ? bdi_writeback_task+0x3b/0xc0
 [<ffffffff81089f20>] ? bdi_start_fn+0x0/0xf0
 [<ffffffff81089f9e>] ? bdi_start_fn+0x7e/0xf0
 [<ffffffff81089f20>] ? bdi_start_fn+0x0/0xf0
 [<ffffffff8104efe6>] ? kthread+0x96/0xa0
 [<ffffffff81003094>] ? kernel_thread_helper+0x4/0x10
 [<ffffffff8104ef50>] ? kthread+0x0/0xa0
 [<ffffffff81003090>] ? kernel_thread_helper+0x0/0x10
INFO: task mysqld:14337 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld        D 00000001001ac631     0 14337      1 0x00000008
 ffff8801fdc6e820 0000000000000086 ffff8801e1d17a88 0000000000004000
 ffff880009c51fd8 ffff880009c51fd8 ffff8801f1f85040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801f1f85040 00000000ae406820
Call Trace:
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd7b7>] ? __mutex_lock_slowpath+0x57/0x190
 [<ffffffff8111e5d8>] ? reiserfs_update_sd_size+0x308/0x330
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d86d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff81137c8b>] ? do_journal_begin_r+0x8b/0x350
 [<ffffffff814fd7b7>] ? __mutex_lock_slowpath+0x57/0x190
 [<ffffffff811380e1>] ? journal_begin+0x81/0x150
 [<ffffffff8111e7ec>] ? reiserfs_write_end+0x1ec/0x310
 [<ffffffff81074ca4>] ? generic_file_buffered_write+0x164/0x240
 [<ffffffff81076c18>] ? __generic_file_aio_write+0x278/0x450
 [<ffffffff81076e3f>] ? generic_file_aio_write+0x4f/0xb0
 [<ffffffff810b0bc0>] ? do_sync_write+0xc0/0x100
 [<ffffffff81002c4e>] ? apic_timer_interrupt+0xe/0x20
 [<ffffffff810b169b>] ? vfs_write+0xcb/0x180
 [<ffffffff810b1853>] ? sys_write+0x53/0xa0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task mysqld:14437 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld        D ffff880009d9f040     0 14437      1 0x00000000
 ffff8801fe73f040 0000000000000086 ffff8801fe73f040 0000000000004000
 ffff8801f1c65fd8 ffff8801f1c65fd8 ffff880009d9f040 0000000000004000
 0000000000012f80 0000000000012f80 ffff880009d9f040 ffff8800f1802600
Call Trace:
 [<ffffffff810fb5ef>] ? dqget+0x19f/0x340
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d86d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff8111b6a1>] ? reiserfs_create+0xd1/0x250
 [<ffffffff810c29e0>] ? dput+0xb0/0x160
 [<ffffffff810b9942>] ? __follow_mount+0x22/0x90
 [<ffffffff810c41d2>] ? __d_lookup+0xa2/0x130
 [<ffffffff810b9528>] ? acl_permission_check+0x68/0xa0
 [<ffffffff810ba8a6>] ? vfs_create+0xa6/0xf0
 [<ffffffff810bb9b1>] ? do_last+0x561/0x6e0
 [<ffffffff810bdceb>] ? do_filp_open+0x1fb/0x630
 [<ffffffff810b49e7>] ? vfs_fstatat+0x37/0x80
 [<ffffffff810af10f>] ? do_sys_open+0x8f/0x140
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task java:7264 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
java          D 00000001001b1010     0  7264   7077 0x00000000
 ffffffff8181f020 0000000000000082 00000000000010fe 0000000000004000
 ffff8801d909dfd8 ffff8801d909dfd8 ffff8801e3508820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801e3508820 00000000d90494c0
Call Trace:
 [<ffffffff814fe8ed>] ? rwsem_down_failed_common+0xcd/0x200
 [<ffffffff814fea3f>] ? rwsem_down_write_failed+0x1f/0x30
 [<ffffffff81223d23>] ? call_rwsem_down_write_failed+0x13/0x20
 [<ffffffff814fdfbc>] ? down_write+0x1c/0x20
 [<ffffffff810959cc>] ? sys_mprotect+0xec/0x250
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task java:7281 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
java          D ffff8801f7d88040     0  7281   7077 0x00000000
 ffff8801fdc6e820 0000000000000082 ffff880006292f80 0000000000004000
 ffff8801d9141fd8 ffff8801d9141fd8 ffff8801f7d88040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801f7d88040 ffff8801f7d88040
Call Trace:
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff81052c29>] ? lock_hrtimer_base+0x29/0x60
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d80d>] ? reiserfs_write_lock_once+0x2d/0x60
 [<ffffffff81127418>] ? reiserfs_dirty_inode+0x28/0xc0
 [<ffffffff8121de6f>] ? __prop_inc_single+0x2f/0x60
 [<ffffffff8103e02e>] ? current_fs_time+0x1e/0x30
 [<ffffffff810cf893>] ? __mark_inode_dirty+0x33/0x170
 [<ffffffff810c4fc6>] ? file_update_time+0x116/0x1a0
 [<ffffffff8108c1ca>] ? do_wp_page+0x16a/0x810
 [<ffffffff8108e696>] ? handle_mm_fault+0x846/0x970
 [<ffffffff81021257>] ? do_page_fault+0x197/0x390
 [<ffffffff810328a0>] ? finish_task_switch+0x80/0xc0
 [<ffffffff814fcc2c>] ? schedule+0x30c/0x710
 [<ffffffff8105eb52>] ? sys_futex+0x82/0x160
 [<ffffffff814ff09f>] ? page_fault+0x1f/0x30
INFO: task java:8309 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
java          D 00000001001ac850     0  8309   7077 0x00000000
 ffff8801fe608820 0000000000000082 ffffffff8102cf40 0000000000004000
 ffff8801c95cffd8 ffff8801c95cffd8 ffff8801bb9ea040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801bb9ea040 00000000fdc6e820
Call Trace:
 [<ffffffff8102cf40>] ? update_curr+0xe0/0x120
 [<ffffffff812260a4>] ? __bitmap_weight+0x84/0xb0
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d80d>] ? reiserfs_write_lock_once+0x2d/0x60
 [<ffffffff81127418>] ? reiserfs_dirty_inode+0x28/0xc0
 [<ffffffff8121de6f>] ? __prop_inc_single+0x2f/0x60
 [<ffffffff8103e02e>] ? current_fs_time+0x1e/0x30
 [<ffffffff810cf893>] ? __mark_inode_dirty+0x33/0x170
 [<ffffffff810c4fc6>] ? file_update_time+0x116/0x1a0
 [<ffffffff8108c1ca>] ? do_wp_page+0x16a/0x810
 [<ffffffff814fcc2c>] ? schedule+0x30c/0x710
 [<ffffffff8108e696>] ? handle_mm_fault+0x846/0x970
 [<ffffffff81021257>] ? do_page_fault+0x197/0x390
 [<ffffffff810e32b6>] ? ep_remove+0xa6/0xd0
 [<ffffffff810e3c81>] ? sys_epoll_ctl+0xb1/0x500
 [<ffffffff814ff09f>] ? page_fault+0x1f/0x30
INFO: task java:8312 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
java          D 00000001001af8c6     0  8312   7077 0x00000000
 ffff8801fe608820 0000000000000082 ffffffff8112fe08 0000000000004000
 ffff8801bbbb9fd8 ffff8801bbbb9fd8 ffff8801bbac2040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801bbac2040 00000000bbac2040
Call Trace:
 [<ffffffff8112fe08>] ? pathrelse+0x28/0x40
 [<ffffffff8113d86d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff81136c57>] ? do_journal_end+0x297/0xf20
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d80d>] ? reiserfs_write_lock_once+0x2d/0x60
 [<ffffffff81127418>] ? reiserfs_dirty_inode+0x28/0xc0
 [<ffffffff8121de6f>] ? __prop_inc_single+0x2f/0x60
 [<ffffffff8103e02e>] ? current_fs_time+0x1e/0x30
 [<ffffffff810cf893>] ? __mark_inode_dirty+0x33/0x170
 [<ffffffff810c4fc6>] ? file_update_time+0x116/0x1a0
 [<ffffffff8108c1ca>] ? do_wp_page+0x16a/0x810
 [<ffffffff814fd345>] ? schedule_timeout+0x1a5/0x200
 [<ffffffff8105c08f>] ? wake_futex+0x3f/0x80
 [<ffffffff8108e696>] ? handle_mm_fault+0x846/0x970
 [<ffffffff813d5c23>] ? release_sock+0x13/0xb0
 [<ffffffff8104f490>] ? autoremove_wake_function+0x0/0x30
 [<ffffffff81021257>] ? do_page_fault+0x197/0x390
 [<ffffffff810aee3e>] ? fd_install+0x2e/0x60
 [<ffffffff813d45c5>] ? sys_accept4+0x1a5/0x1e0
 [<ffffffff814fcc2c>] ? schedule+0x30c/0x710
 [<ffffffff8105eb52>] ? sys_futex+0x82/0x160
 [<ffffffff814ff09f>] ? page_fault+0x1f/0x30
INFO: task clamd:15019 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
clamd         D 00000001001af586     0 15019      1 0x00000000
 ffffffff8181f020 0000000000000082 ffffffff810b9528 0000000000004000
 ffff8800e0a07fd8 ffff8800e0a07fd8 ffff8800e2ed1820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8800e2ed1820 00000000dff4f844
Call Trace:
 [<ffffffff810b9528>] ? acl_permission_check+0x68/0xa0
 [<ffffffff814fd897>] ? __mutex_lock_slowpath+0x137/0x190
 [<ffffffff810c9803>] ? mntput_no_expire+0x23/0x100
 [<ffffffff814fd66a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810ba017>] ? lookup_create+0x27/0xc0
 [<ffffffff810bd3dd>] ? sys_mkdirat+0x7d/0x130
 [<ffffffff810b1d64>] ? generic_file_llseek+0x54/0x70
 [<ffffffff810b0844>] ? sys_lseek+0x54/0x90
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b