Stacktrace from dmesg about the kernel crash

dmesg_kernel_crash.txt (text/plain), 10.45 KB, created by Phillip Merensky on 2010-07-15 15:33:02 UTC

INFO: task flush-8:0:1571 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
flush-8:0     D 0000000000000002     0  1571      2 0x00000000
 ffff8801fc87d040 0000000000000046 ffff8801fa371a78 0000000000004000
 ffff8801fa371fd8 ffff8801fa371fd8 ffff8801fcea2820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801fcea2820 0000000000000000
Call Trace:
 [<ffffffff8121d95e>] ? prio_tree_next+0xae/0x240
 [<ffffffff810759e0>] ? sync_page+0x0/0x60
 [<ffffffff814fcbab>] ? io_schedule+0x4b/0x70
 [<ffffffff81075a15>] ? sync_page+0x35/0x60
 [<ffffffff814fcf7a>] ? __wait_on_bit_lock+0x4a/0x90
 [<ffffffff810759be>] ? __lock_page+0x5e/0x70
 [<ffffffff8104f480>] ? wake_bit_function+0x0/0x30
 [<ffffffff8107cbfa>] ? __writepage+0xa/0x40
 [<ffffffff8107d363>] ? write_cache_pages+0x193/0x370
 [<ffffffff8107cbf0>] ? __writepage+0x0/0x40
 [<ffffffff810cf152>] ? writeback_single_inode+0xf2/0x370
 [<ffffffff810cff68>] ? writeback_inodes_wb+0x3a8/0x5e0
 [<ffffffff810d02c6>] ? wb_writeback+0x126/0x1f0
 [<ffffffff810d05eb>] ? wb_do_writeback+0x17b/0x180
 [<ffffffff810d0652>] ? bdi_writeback_task+0x62/0xa0
 [<ffffffff81089f20>] ? bdi_start_fn+0x0/0xf0
 [<ffffffff81089f9e>] ? bdi_start_fn+0x7e/0xf0
 [<ffffffff81089f20>] ? bdi_start_fn+0x0/0xf0
 [<ffffffff8104efa6>] ? kthread+0x96/0xa0
 [<ffffffff81003094>] ? kernel_thread_helper+0x4/0x10
 [<ffffffff8104ef10>] ? kthread+0x0/0xa0
 [<ffffffff81003090>] ? kernel_thread_helper+0x0/0x10
INFO: task mysqld:3703 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld        D 0000000101516f5e     0  3703      1 0x00000000
 ffff8801fe608820 0000000000000082 ffffffff810b4790 0000000000004000
 ffff8800d5733fd8 ffff8800d5733fd8 ffff8801e422c040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801e422c040 00000000fe73d040
Call Trace:
 [<ffffffff810b4790>] ? inode_get_bytes+0x20/0x50
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd2ce>] ? __mutex_lock_slowpath+0x3e/0x170
 [<ffffffff8111e468>] ? reiserfs_update_sd_size+0x308/0x330
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d70d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff81137b2b>] ? do_journal_begin_r+0x8b/0x350
 [<ffffffff814fd2ce>] ? __mutex_lock_slowpath+0x3e/0x170
 [<ffffffff81137f81>] ? journal_begin+0x81/0x150
 [<ffffffff8111e67c>] ? reiserfs_write_end+0x1ec/0x310
 [<ffffffff81074ca4>] ? generic_file_buffered_write+0x164/0x240
 [<ffffffff81076c18>] ? __generic_file_aio_write+0x278/0x450
 [<ffffffff81076e3f>] ? generic_file_aio_write+0x4f/0xb0
 [<ffffffff810b0bc0>] ? do_sync_write+0xc0/0x100
 [<ffffffff81032841>] ? finish_task_switch+0x41/0xc0
 [<ffffffff810b169b>] ? vfs_write+0xcb/0x180
 [<ffffffff810b1853>] ? sys_write+0x53/0xa0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task mysqld:3764 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld        D ffff8801559e5fd8     0  3764      1 0x00000000
 ffff8801fcf18040 0000000000000082 ffff880006292f80 0000000000004000
 ffff8801559e5fd8 ffff8801559e5fd8 ffff88007846f820 0000000000004000
 0000000000012f80 0000000000012f80 ffff88007846f820 ffff8801ba57a898
Call Trace:
 [<ffffffff810fb47f>] ? dqget+0x19f/0x340
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d70d>] ? reiserfs_write_lock+0x2d/0x50
 [<ffffffff8111b531>] ? reiserfs_create+0xd1/0x250
 [<ffffffff810c29a0>] ? dput+0xb0/0x160
 [<ffffffff810b9942>] ? __follow_mount+0x22/0x90
 [<ffffffff810c4192>] ? __d_lookup+0xa2/0x130
 [<ffffffff810b9528>] ? acl_permission_check+0x68/0xa0
 [<ffffffff810ba8a6>] ? vfs_create+0xa6/0xf0
 [<ffffffff810bb9da>] ? do_last+0x58a/0x6a0
 [<ffffffff810bdcab>] ? do_filp_open+0x1fb/0x630
 [<ffffffff810b49e7>] ? vfs_fstatat+0x37/0x80
 [<ffffffff810af10f>] ? do_sys_open+0x8f/0x140
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task java:6387 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
java          D ffff8801cd9f9fd8     0  6387   6338 0x00000000
 ffff8801fc87d040 0000000000000082 ffffffff81032841 0000000000004000
 ffff8801cd9f9fd8 ffff8801cd9f9fd8 ffff8801cdb38820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801cdb38820 ffff8801cd9f9fd8
Call Trace:
 [<ffffffff81032841>] ? finish_task_switch+0x41/0xc0
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff8113d6ad>] ? reiserfs_write_lock_once+0x2d/0x60
 [<ffffffff811272b8>] ? reiserfs_dirty_inode+0x28/0xc0
 [<ffffffff8121dcbf>] ? __prop_inc_single+0x2f/0x60
 [<ffffffff8103e00e>] ? current_fs_time+0x1e/0x30
 [<ffffffff810cf863>] ? __mark_inode_dirty+0x33/0x170
 [<ffffffff810c4f86>] ? file_update_time+0x116/0x1a0
 [<ffffffff8108c1ca>] ? do_wp_page+0x16a/0x810
 [<ffffffff8108e696>] ? handle_mm_fault+0x846/0x970
 [<ffffffff81021237>] ? do_page_fault+0x197/0x390
 [<ffffffff814fe94e>] ? common_interrupt+0xe/0x13
 [<ffffffff8105eb12>] ? sys_futex+0x82/0x160
 [<ffffffff814feb5f>] ? page_fault+0x1f/0x30
INFO: task clamd:3984 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
clamd         D 0000000101517a68     0  3984      1 0x00000000
 ffff8801fe608820 0000000000000082 ffff8801fe34bd70 0000000000004000
 ffff880078541fd8 ffff880078541fd8 ffff880060d83820 0000000000004000
 0000000000012f80 0000000000012f80 ffff880060d83820 0000000060d83820
Call Trace:
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810ba017>] ? lookup_create+0x27/0xc0
 [<ffffffff810bd39d>] ? sys_mkdirat+0x7d/0x130
 [<ffffffff810b1d64>] ? generic_file_llseek+0x54/0x70
 [<ffffffff810b0844>] ? sys_lseek+0x54/0x90
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task clamd:4094 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
clamd         D 0000000101518e61     0  4094      1 0x00000000
 ffff8801fe608820 0000000000000082 ffff8801fe34bd70 0000000000004000
 ffff880009f93fd8 ffff880009f93fd8 ffff8801fcf12820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801fcf12820 00000000fcf12820
Call Trace:
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810ba017>] ? lookup_create+0x27/0xc0
 [<ffffffff810bd39d>] ? sys_mkdirat+0x7d/0x130
 [<ffffffff810b1d64>] ? generic_file_llseek+0x54/0x70
 [<ffffffff810b0844>] ? sys_lseek+0x54/0x90
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task proc_pri:3761 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
proc_pri      D ffff8800d56affd8     0  3761      1 0x00000004
 ffff8801fc87d040 0000000000000082 0000000000000000 0000000000004000
 ffff8800d56affd8 ffff8800d56affd8 ffff8801fe73d040 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801fe73d040 ffff8801e422c040
Call Trace:
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff81137b23>] ? do_journal_begin_r+0x83/0x350
 [<ffffffff814fd1d2>] ? mutex_unlock+0x12/0x20
 [<ffffffff81137f81>] ? journal_begin+0x81/0x150
 [<ffffffff8104f450>] ? autoremove_wake_function+0x0/0x30
 [<ffffffff81126e2e>] ? reiserfs_write_dquot+0x4e/0xc0
 [<ffffffff8112774b>] ? remove_save_link+0xdb/0x100
 [<ffffffff810fa9ff>] ? dqput+0xff/0x1c0
 [<ffffffff810fab24>] ? __dquot_drop+0x64/0x80
 [<ffffffff810c5abf>] ? clear_inode+0x4f/0xf0
 [<ffffffff8112144a>] ? reiserfs_delete_inode+0x8a/0x160
 [<ffffffff810e0796>] ? fsnotify_clear_marks_by_inode+0x26/0xe0
 [<ffffffff810c62af>] ? generic_delete_inode+0x7f/0x130
 [<ffffffff810c2177>] ? d_kill+0x47/0x80
 [<ffffffff810c2987>] ? dput+0x97/0x160
 [<ffffffff810b235d>] ? __fput+0x13d/0x1d0
 [<ffffffff810aef46>] ? filp_close+0x56/0x90
 [<ffffffff810bef2a>] ? sys_dup3+0x13a/0x1a0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task apache2:3947 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2       D 0000000101517e77     0  3947   9622 0x00000000
 ffff8801fe608820 0000000000000082 0000000800000002 0000000000004000
 ffff880166415fd8 ffff880166415fd8 ffff8801f25ec820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801f25ec820 0000000094800001
Call Trace:
 [<ffffffff810b9942>] ? __follow_mount+0x22/0x90
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff810bbd58>] ? link_path_walk+0x78/0xc70
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810bb4fd>] ? do_last+0xad/0x6a0
 [<ffffffff810bdcab>] ? do_filp_open+0x1fb/0x630
 [<ffffffff810c7bea>] ? alloc_fd+0x4a/0x140
 [<ffffffff810af10f>] ? do_sys_open+0x8f/0x140
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task apache2:3987 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2       D 0000000101518a9e     0  3987   9622 0x00000000
 ffffffff8181f020 0000000000000082 ffff88008ca91005 0000000000004000
 ffff8800875cffd8 ffff8800875cffd8 ffff8801fe660820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8801fe660820 000000000ff3e818
Call Trace:
 [<ffffffff810c97c3>] ? mntput_no_expire+0x23/0x100
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff810bcc33>] ? do_path_lookup+0x33/0x60
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810bd092>] ? do_unlinkat+0xb2/0x1d0
 [<ffffffff810939a8>] ? do_munmap+0x2d8/0x3b0
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b
INFO: task processes:3989 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
processes     D 0000000101517b6b     0  3989      1 0x00000004
 ffff8801fe608820 0000000000000086 ffff88008cb5dd78 0000000000004000
 ffff88008cb5dfd8 ffff88008cb5dfd8 ffff8800c3e76820 0000000000004000
 0000000000012f80 0000000000012f80 ffff8800c3e76820 0000000037b16001
Call Trace:
 [<ffffffff810b9942>] ? __follow_mount+0x22/0x90
 [<ffffffff814fd3a8>] ? __mutex_lock_slowpath+0x118/0x170
 [<ffffffff810bbd58>] ? link_path_walk+0x78/0xc70
 [<ffffffff814fd19a>] ? mutex_lock+0x1a/0x40
 [<ffffffff810bb4fd>] ? do_last+0xad/0x6a0
 [<ffffffff810bdcab>] ? do_filp_open+0x1fb/0x630
 [<ffffffff810b060f>] ? sys_statfs+0x9f/0xe0
 [<ffffffff810c7bea>] ? alloc_fd+0x4a/0x140
 [<ffffffff810af10f>] ? do_sys_open+0x8f/0x140
 [<ffffffff8100236b>] ? system_call_fastpath+0x16/0x1b