[OpenIndiana-discuss] why zpool import cost a long time or hung?
Dang Zhiqiang
c_dddd at 163.com
Tue Jun 10 03:27:26 UTC 2014
Hi,
I want to know why zpool import cost a long time or hung? and how to fix it?
thanks.
OS: openindiana 151a9
root at host1:~# time zpool import
pool: data
id: 11364155362532848638
state: ONLINE
status: The pool was last accessed by another system.
action: The pool can be imported using its name or numeric identifier and
the '-f' flag.
see: http://illumos.org/msg/ZFS-8000-EY
config:
data ONLINE
c3t5000C500470098D7d0 ONLINE
real 16m59.173s
user 0m0.008s
sys 0m0.099s
When do zpool import:
mdb -k
> ::ps -f !grep import
R 25285 20254 20254 20179 0 0x4a004000 ffffff02f50a9090 bash -c grep import
R 25017 7778 25017 7778 0 0x4a004000 ffffff02fce5c040 zpool import
> ffffff02fce5c040::ps -t
S PID PPID PGID SID UID FLAGS ADDR NAME
R 25017 7778 25017 7778 0 0x4a004000 ffffff02fce5c040 zpool
T 0xffffff02fbda4100 <TS_SLEEP>
T 0xffffff02fa1df400 <TS_SLEEP>
T 0xffffff02d8a96000 <TS_SLEEP>
T 0xffffff02fbc90ba0 <TS_SLEEP>
T 0xffffff02fa3fe460 <TS_SLEEP>
T 0xffffff02fa0c6460 <TS_SLEEP>
T 0xffffff02fa49a3c0 <TS_SLEEP>
T 0xffffff02d8561840 <TS_SLEEP>
T 0xffffff02f58de460 <TS_SLEEP>
> 0xffffff02fbda4100::findstack -v
stack pointer for thread ffffff02fbda4100: ffffff000fec2cc0
[ ffffff000fec2cc0 _resume_from_idle+0xf4() ]
ffffff000fec2cf0 swtch+0x145()
ffffff000fec2d50 cv_wait_sig_swap_core+0x174(ffffff02fbda42ee, ffffff02fbda42f0, 0)
ffffff000fec2d70 cv_wait_sig_swap+0x18(ffffff02fbda42ee, ffffff02fbda42f0)
ffffff000fec2de0 cv_waituntil_sig+0x13c(ffffff02fbda42ee, ffffff02fbda42f0, 0, 0)
ffffff000fec2e80 lwp_park+0x157(0, 0)
ffffff000fec2ec0 syslwp_park+0x31(0, 0, 0)
ffffff000fec2f10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02fa1df400::findstack -v
stack pointer for thread ffffff02fa1df400: ffffff0011c0d920
[ ffffff0011c0d920 _resume_from_idle+0xf4() ]
ffffff0011c0d950 swtch+0x145()
ffffff0011c0d990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0011c0da20 sdopen+0xe0(ffffff0011c0daa8, 2001, 2, ffffff02f7219760)
ffffff0011c0da50 dev_open+0x3c(ffffff0011c0daa8, 2001, 2, ffffff02f7219760)
ffffff0011c0db00 spec_open+0x5dc(ffffff0011c0dbf0, 2001, ffffff02f7219760, 0)
ffffff0011c0db70 fop_open+0xbf(ffffff0011c0dbf0, 2001, ffffff02f7219760, 0)
ffffff0011c0dd20 vn_openat+0x6ce(80def08, 0, 2001, c59, ffffff0011c0dd78, 0, 12, ffffff02f047be00, e)
ffffff0011c0de90 copen+0x49e(7, 80def08, 2001, fed8ac59)
ffffff0011c0dec0 openat64+0x2d(7, 80def08, 0, fed8ac59)
ffffff0011c0df10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02d8a96000::findstack -v
stack pointer for thread ffffff02d8a96000: ffffff0011192920
[ ffffff0011192920 _resume_from_idle+0xf4() ]
ffffff0011192950 swtch+0x145()
ffffff0011192990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0011192a20 sdopen+0xe0(ffffff0011192aa8, 2001, 2, ffffff02f7219760)
ffffff0011192a50 dev_open+0x3c(ffffff0011192aa8, 2001, 2, ffffff02f7219760)
ffffff0011192b00 spec_open+0x5dc(ffffff0011192bf0, 2001, ffffff02f7219760, 0)
ffffff0011192b70 fop_open+0xbf(ffffff0011192bf0, 2001, ffffff02f7219760, 0)
ffffff0011192d20 vn_openat+0x6ce(80deee8, 0, 2001, c59, ffffff0011192d78, 0, 12, ffffff02f047be00, 9)
ffffff0011192e90 copen+0x49e(7, 80deee8, 2001, fed8ac59)
ffffff0011192ec0 openat64+0x2d(7, 80deee8, 0, fed8ac59)
ffffff0011192f10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02fbc90ba0::findstack -v
stack pointer for thread ffffff02fbc90ba0: ffffff0010820920
[ ffffff0010820920 _resume_from_idle+0xf4() ]
ffffff0010820950 swtch+0x145()
ffffff0010820990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0010820a20 sdopen+0xe0(ffffff0010820aa8, 2001, 2, ffffff02f7219760)
ffffff0010820a50 dev_open+0x3c(ffffff0010820aa8, 2001, 2, ffffff02f7219760)
ffffff0010820b00 spec_open+0x5dc(ffffff0010820bf0, 2001, ffffff02f7219760, 0)
ffffff0010820b70 fop_open+0xbf(ffffff0010820bf0, 2001, ffffff02f7219760, 0)
ffffff0010820d20 vn_openat+0x6ce(80dee68, 0, 2001, c59, ffffff0010820d78, 0, 12, ffffff02f047be00, a)
ffffff0010820e90 copen+0x49e(7, 80dee68, 2001, fed8ac59)
ffffff0010820ec0 openat64+0x2d(7, 80dee68, 0, fed8ac59)
ffffff0010820f10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02fa3fe460::findstack -v
stack pointer for thread ffffff02fa3fe460: ffffff0012128690
[ ffffff0012128690 _resume_from_idle+0xf4() ]
ffffff00121286c0 swtch+0x145()
ffffff0012128700 sema_p+0x1d9(ffffff03425d1d40)
ffffff0012128730 biowait+0x76(ffffff03425d1c80)
ffffff00121287f0 scsi_uscsi_handle_cmd+0x190(f700000100, 1, ffffff032ed5b9b0, fffffffff816f870, 0, ffffff03308569b8)
ffffff0012128860 sd_ssc_send+0x1fd(ffffff02f5105480, ffffff0012128880, 80000000, 1, 0)
ffffff0012128910 sd_send_scsi_TEST_UNIT_READY+0x10b(ffffff02f5105480, 0)
ffffff0012128990 sd_ready_and_valid+0x211(ffffff02f5105480, c)
ffffff0012128a20 sdopen+0x2d5(ffffff0012128aa8, 2001, 2, ffffff02f7219760)
ffffff0012128a50 dev_open+0x3c(ffffff0012128aa8, 2001, 2, ffffff02f7219760)
ffffff0012128b00 spec_open+0x5dc(ffffff0012128bf0, 2001, ffffff02f7219760, 0)
ffffff0012128b70 fop_open+0xbf(ffffff0012128bf0, 2001, ffffff02f7219760, 0)
ffffff0012128d20 vn_openat+0x6ce(808ff60, 0, 2001, c59, ffffff0012128d78, 0, 12, ffffff02f047be00, c)
ffffff0012128e90 copen+0x49e(7, 808ff60, 2001, fed8ac59)
ffffff0012128ec0 openat64+0x2d(7, 808ff60, 0, fed8ac59)
ffffff0012128f10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02fa0c6460::findstack -v
stack pointer for thread ffffff02fa0c6460: ffffff0011d4f920
[ ffffff0011d4f920 _resume_from_idle+0xf4() ]
ffffff0011d4f950 swtch+0x145()
ffffff0011d4f990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0011d4fa20 sdopen+0xe0(ffffff0011d4faa8, 2001, 2, ffffff02f7219760)
ffffff0011d4fa50 dev_open+0x3c(ffffff0011d4faa8, 2001, 2, ffffff02f7219760)
ffffff0011d4fb00 spec_open+0x5dc(ffffff0011d4fbf0, 2001, ffffff02f7219760, 0)
ffffff0011d4fb70 fop_open+0xbf(ffffff0011d4fbf0, 2001, ffffff02f7219760, 0)
ffffff0011d4fd20 vn_openat+0x6ce(808ffb0, 0, 2001, c59, ffffff0011d4fd78, 0, 12, ffffff02f047be00, b)
ffffff0011d4fe90 copen+0x49e(7, 808ffb0, 2001, fed8ac59)
ffffff0011d4fec0 openat64+0x2d(7, 808ffb0, 0, fed8ac59)
ffffff0011d4ff10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02fa49a3c0::findstack -v
stack pointer for thread ffffff02fa49a3c0: ffffff0012104920
[ ffffff0012104920 _resume_from_idle+0xf4() ]
ffffff0012104950 swtch+0x145()
ffffff0012104990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0012104a20 sdopen+0xe0(ffffff0012104aa8, 2001, 2, ffffff02f7219760)
ffffff0012104a50 dev_open+0x3c(ffffff0012104aa8, 2001, 2, ffffff02f7219760)
ffffff0012104b00 spec_open+0x5dc(ffffff0012104bf0, 2001, ffffff02f7219760, 0)
ffffff0012104b70 fop_open+0xbf(ffffff0012104bf0, 2001, ffffff02f7219760, 0)
ffffff0012104d20 vn_openat+0x6ce(80def28, 0, 2001, c59, ffffff0012104d78, 0, 12, ffffff02f047be00, 8)
ffffff0012104e90 copen+0x49e(7, 80def28, 2001, fed8ac59)
ffffff0012104ec0 openat64+0x2d(7, 80def28, 0, fed8ac59)
ffffff0012104f10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02d8561840::findstack -v
stack pointer for thread ffffff02d8561840: ffffff0010c4a920
[ ffffff0010c4a920 _resume_from_idle+0xf4() ]
ffffff0010c4a950 swtch+0x145()
ffffff0010c4a990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff0010c4aa20 sdopen+0xe0(ffffff0010c4aaa8, 2001, 2, ffffff02f7219760)
ffffff0010c4aa50 dev_open+0x3c(ffffff0010c4aaa8, 2001, 2, ffffff02f7219760)
ffffff0010c4ab00 spec_open+0x5dc(ffffff0010c4abf0, 2001, ffffff02f7219760, 0)
ffffff0010c4ab70 fop_open+0xbf(ffffff0010c4abf0, 2001, ffffff02f7219760, 0)
ffffff0010c4ad20 vn_openat+0x6ce(80deec8, 0, 2001, c59, ffffff0010c4ad78, 0, 12, ffffff02f047be00, d)
ffffff0010c4ae90 copen+0x49e(7, 80deec8, 2001, fed8ac59)
ffffff0010c4aec0 openat64+0x2d(7, 80deec8, 0, fed8ac59)
ffffff0010c4af10 _sys_sysenter_post_swapgs+0x149()
> 0xffffff02f58de460::findstack -v
stack pointer for thread ffffff02f58de460: ffffff001119e920
[ ffffff001119e920 _resume_from_idle+0xf4() ]
ffffff001119e950 swtch+0x145()
ffffff001119e990 sema_p+0x1d9(ffffff02ee02e0e8)
ffffff001119ea20 sdopen+0xe0(ffffff001119eaa8, 2001, 2, ffffff02f7219760)
ffffff001119ea50 dev_open+0x3c(ffffff001119eaa8, 2001, 2, ffffff02f7219760)
ffffff001119eb00 spec_open+0x5dc(ffffff001119ebf0, 2001, ffffff02f7219760, 0)
ffffff001119eb70 fop_open+0xbf(ffffff001119ebf0, 2001, ffffff02f7219760, 0)
ffffff001119ed20 vn_openat+0x6ce(80deea8, 0, 2001, c59, ffffff001119ed78, 0, 12, ffffff02f047be00, f)
ffffff001119ee90 copen+0x49e(7, 80deea8, 2001, fed8ac59)
ffffff001119eec0 openat64+0x2d(7, 80deea8, 0, fed8ac59)
ffffff001119ef10 _sys_sysenter_post_swapgs+0x149()
root at host1:~# ps -ef |grep import
root 25017 7778 0 11:07:32 pts/1 0:00 zpool import
root 26348 25996 0 11:14:18 pts/4 0:00 grep import
root at host1:~# truss -f -p 25017
25017/1: lwp_park(0x00000000, 0) (sleeping...)
25017/6: openat64(7, "c3t5000C5000B74F2DBd0s10", O_RDONLY) Err#5 EIO
25017/6: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/7: openat64(7, "c3t5000C5000B74F2DBd0p0", O_RDONLY) Err#5 EIO
25017/7: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/2: openat64(7, "c3t5000C5000B74F2DBd0p1", O_RDONLY) Err#5 EIO
25017/2: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/5: openat64(7, "c3t5000C5000B74F2DBd0s13", O_RDONLY) Err#5 EIO
25017/5: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/3: openat64(7, "c3t5000C5000B74F2DBd0s14", O_RDONLY) Err#5 EIO
25017/3: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/3: openat64(7, "c3t5000C50010B03872d0", O_RDONLY) = 9
25017/3: fstat64(9, 0xFE6BFEB0) = 0
25017/3: fstat64(9, 0xFE6BFDE0) = 0
25017/3: pread64(9, "EB04 M 3 . 0FAFCBE\0 |BF".., 262144, 0) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 262144) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 0x0000007470B80000) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 0x0000007470BC0000) = 262144
25017/3: close(9) = 0
25017/3: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
25017/3: openat64(7, "c3t5000C50010B03872d0p0", O_RDONLY) = 9
25017/3: fstat64(9, 0xFE6BFEB0) = 0
25017/3: fstat64(9, 0xFE6BFDE0) = 0
25017/3: pread64(9, "EB04 M 3 . 0FAFCBE\0 |BF".., 262144, 0) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 262144) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 0x0000007470B80000) = 262144
25017/3: pread64(9, "\0\0\0\0\0\0\0\0\0\0\0\0".., 262144, 0x0000007470BC0000) = 262144
25017/3: close(9) = 0
25017/3: lwp_sigmask(SIG_SETMASK, 0xFFBFFEFF, 0xFFFFFFF7, 0x000000FF, 0x00000000) = 0xFFBFFEFF [0xFFFFFFFF]
……
root at host1:~# zpool status
pool: data1
state: ONLINE
scan: scrub repaired 0 in 0h3m with 0 errors on Sat Jun 7 18:03:48 2014
config:
NAME STATE READ WRITE CKSUM
data1 ONLINE 0 0 0
c3t5000C5004700B33Fd0 ONLINE 0 0 0
errors: No known data errors
pool: new
state: ONLINE
scan: scrub repaired 0 in 0h0m with 0 errors on Sat Jun 7 18:00:35 2014
config:
NAME STATE READ WRITE CKSUM
new ONLINE 0 0 0
c3t5000C50010B03872d0 ONLINE 0 0 0
errors: No known data errors
pool: rpool1
state: ONLINE
status: Some supported features are not enabled on the pool. The pool can
still be used, but some features are unavailable.
action: Enable all features using 'zpool upgrade'. Once this is done,
the pool may no longer be accessible by software that does not support
the features. See zpool-features(5) for details.
scan: none requested
config:
NAME STATE READ WRITE CKSUM
rpool1 ONLINE 0 0 0
c0d0s0 ONLINE 0 0 0
errors: No known data errors
pool: tank
state: DEGRADED
status: One or more devices has experienced an error resulting in data
corruption. Applications may be affected.
action: Restore the file in question if possible. Otherwise restore the
entire pool from backup.
see: http://illumos.org/msg/ZFS-8000-8A
scan: scrub repaired 0 in 0h28m with 110 errors on Sat Jun 7 18:28:54 2014
config:
NAME STATE READ WRITE CKSUM
tank DEGRADED 0 0 110
mirror-0 DEGRADED 0 0 220
c3t5000C5000B74F2DBd0 FAULTED 13 64 52 too many errors
c3t5000C50043FFA61Fd0 DEGRADED 0 0 220 too many errors
errors: 110 data errors, use '-v' for a list
More information about the openindiana-discuss
mailing list