[Swan] klips + ipsec whack --shutdown causes lockup

Brian T btuch at usa.net
Mon Sep 9 13:11:07 UTC 2019


> 
> Found the dependency modules in the kernel config that were causing xfrm 
> to build and removed them.  The lockup issue is still present.

Here is the trace when shutting down.  ipsec.ko is unloaded, but I do not see
anything related to it in the trace.



[  570.847388] skbuff: skb_over_panic: text:c04d4e3c len:70 put:70 head: 
(null) data:  (null) tail:0x46 end:0x0 dev:eth0
[  570.858268] ------------[ cut here ]------------
[  570.862909] Kernel BUG at c0596118 [verbose debug info unavailable]
[  570.869201] Internal error: Oops - BUG: 0 [#1] PREEMPT ARM
[  570.874708] Modules linked in: ipsec(O) cmac gcm ccm cdc_acm nf_log_ipv6
nf_conntrack_ipv6 nf_defrag_ipv6 nf_log_ipv4 nf_log_common xt_multiport
xt_TCPMSS xt_conntrack xt_LOG xt_limit iptable_mangle ipt_MASQUERADE
nf_nat_masquerade_ipv4 xt_REDIRECT nf_nat_redirect iptable_nat
nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_raw iptable_filter
ip_tables ath9k_htc mac80211 ath9k_common ath9k_hw ath cfg80211 smsc95xx
cdc_ncm cdc_ether GobiNet(O) usbnet mii GobiSerial(O) ti_am335x_adc kfifo_buf
industrialio i2c_hid usbhid hid_generic hid xr_usb_serial_common(O) option
usb_wwan usbserial ppp_async ppp_generic slhc gpio_keys evdev sd_mod sg
mtd_spi_sram spi_omap2_mcspi lm75 ads1015 hwmon at25 at24 nvmem_core usb_f_ecm
dwc3_omap sxni_iodb(O) g_ether usb_f_rndis libcomposite u_ether configfs
jbm_feature [last unloaded: ipsec]
[  570.948854] CPU: 0 PID: 0 Comm: swapper Tainted: G           O    4.9.119
#1
[  570.955930] Hardware name: Generic AM43 (Flattened Device Tree)
[  570.961873] task: c0a08c38 task.stack: c0a00000
[  570.966436] PC is at skb_panic+0x64/0x68
[  570.970381] LR is at irq_work_queue+0xdc/0x104
[  570.974844] pc : [<c0596118>]    lr : [<c0190e0c>]    psr: 20000113
[  570.974844] sp : c0a01cc0  ip : c0a01b88  fp : c0a01ce4
[  570.986369] r10: c0112390  r9 : 00000046  r8 : 000a0000
[  570.991613] r7 : 00000000  r6 : c075c714  r5 : 00000000  r4 : 00000046
[  570.998166] r3 : c0a00000  r2 : 00000101  r1 : c0a257fc  r0 : 0000006a
[  571.004721] Flags: nzCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment
none
[  571.011884] Control: 10c53c7d  Table: 9df80059  DAC: 00000051
[  571.017652] Process swapper (pid: 0, stack limit = 0xc0a00210)
[  571.023507] Stack: (0xc0a01cc0 to 0xc0a02000)
[  571.027885] 1cc0: 00000046 00000000 00000000 00000046 00000000 df46f800
c0a01d04 c0a01ce8
[  571.036100] 1ce0: c0596f80 c05960c0 dcccda00 df46f800 df467c10 dc8e3540
c0a01d44 c0a01d08
[  571.044314] 1d00: c04d4e3c c0596f34 df46b640 e0a74ec0 c0a01d44 c0a01d20
c036bbb4 e0a74ec0
[  571.052528] 1d20: df2a7d90 df46b7d0 dcccda00 00000046 000a0000 c0112390
c0a01d7c c0a01d48
[  571.060743] 1d40: c04cd678 c04d4d9c 00000000 c01642d4 00000001 df2a7d90
000a0000 e0a74ec0
[  571.068957] 1d60: df46b7d0 4a102ec0 20000113 00000046 c0a01db4 c0a01d80
c04cd844 c04cd5dc
[  571.077170] 1d80: c0164fd4 e0a74000 c0a2efd0 00000000 df2a7d90 00000040
00000040 df467c38
[  571.085383] 1da0: c0a28280 c0a01e18 c0a01dd4 c0a01db8 c04ce8b0 c04cd6a0
00000001 df467c38
[  571.093597] 1dc0: 00000000 00000040 c0a01dfc c0a01dd8 c04d675c c04ce878
df467c38 00000001
[  571.101812] 1de0: 0000012c 000069cd 00000040 c0a28280 c0a01e54 c0a01e00
c05a76d4 c04d671c
[  571.110025] 1e00: c082d65c c0834060 c082d318 c0a0373c c0a55128 c0a54e40
c0a01e18 c0a01e18
[  571.118238] 1e20: c0a01e20 c0a01e20 df004000 00000000 c0a586cc 40000003
ffffe000 c0a586c0
[  571.126452] 1e40: 00000101 00000003 c0a01eb4 c0a01e58 c0132f10 c05a75d0
c0161cec c01642d4
[  571.134667] 1e60: 00000084 00200100 c0a28280 000069cc c07d700c c0703778
0000000a c0a58680
[  571.142880] 1e80: c0a25588 c0a586c0 c0a01eb4 c0a2efd0 00000000 00000000
00000001 df004000
[  571.151094] 1ea0: c0a00000 00000001 c0a01ec4 c0a01eb8 c01333e4 c0132e38
c0a01eec c0a01ec8
[  571.159307] 1ec0: c0161490 c01332e0 c0a02b34 fa24010c c0a01f18 fa240100
fa241100 c0a00000
[  571.167522] 1ee0: c0a01f14 c0a01ef0 c01014fc c016143c c0183d08 20000013
ffffffff c0a01f4c
[  571.175736] 1f00: c0a4b108 c0a00000 c0a01f74 c0a01f18 c010c20c c01014c8
00000000 00000084
[  571.183949] 1f20: e88e1d61 000069cb ffffe000 c0a028b4 00000000 00000000
c0a4b108 c0a54fa7
[  571.192163] 1f40: 00000001 c0a01f74 c0a01e80 c0a01f68 c0357168 c0183d08
20000013 ffffffff
[  571.200377] 1f60: 00000051 00000000 c0a01f94 c0a01f78 c015a004 c0183ce0
c06d1b98 c07d8864
[  571.208591] 1f80: 00000002 c0a02840 c0a01fac c0a01f98 c06cf144 c0159fcc
00000000 c0a57000
[  571.216805] 1fa0: c0a01ff4 c0a01fb0 c0900c8c c06cf0c4 ffffffff ffffffff
00000000 c09006f0
[  571.225019] 1fc0: 00000000 c09398c4 00000000 c0a571d4 c0a0285c c09398c0
c0a09db4 80004059
[  571.233232] 1fe0: 412fc09a 00000000 00000000 c0a01ff8 80008078 c0900984
00000000 00000000
[  571.241439] Backtrace: 
[  571.243909] [<c05960b4>] (skb_panic) from [<c0596f80>] (skb_put+0x58/0x5c)
[  571.250827] [<c0596f28>] (skb_put) from [<c04d4e3c>]
(cpsw_rx_handler+0xac/0x250)
[  571.258343]  r7:dc8e3540 r6:df467c10 r5:df46f800 r4:dcccda00
[  571.264044] [<c04d4d90>] (cpsw_rx_handler) from [<c04cd678>]
(__cpdma_chan_free+0xa8/0xc4)
[  571.272345]  r10:c0112390 r9:000a0000 r8:00000046 r7:dcccda00 r6:df46b7d0
r5:df2a7d90
[  571.280204]  r4:e0a74ec0
[  571.282752] [<c04cd5d0>] (__cpdma_chan_free) from [<c04cd844>]
(__cpdma_chan_process+0x1b0/0x1d0)
[  571.291663]  r10:00000046 r9:20000113 r8:4a102ec0 r7:df46b7d0 r6:e0a74ec0
r5:000a0000
[  571.299521]  r4:df2a7d90
[  571.302068] [<c04cd694>] (__cpdma_chan_process) from [<c04ce8b0>]
(cpdma_chan_process+0x44/0x5c)
[  571.310892]  r10:c0a01e18 r9:c0a28280 r8:df467c38 r7:00000040 r6:00000040
r5:df2a7d90
[  571.318751]  r4:00000000
[  571.321298] [<c04ce86c>] (cpdma_chan_process) from [<c04d675c>]
(cpsw_rx_poll+0x4c/0xc4)
[  571.329422]  r7:00000040 r6:00000000 r5:df467c38 r4:00000001
[  571.335119] [<c04d6710>] (cpsw_rx_poll) from [<c05a76d4>]
(net_rx_action+0x110/0x2c4)
[  571.342985]  r9:c0a28280 r8:00000040 r7:000069cd r6:0000012c r5:00000001
r4:df467c38
[  571.350776] [<c05a75c4>] (net_rx_action) from [<c0132f10>]
(__do_softirq+0xe4/0x268)
[  571.358553]  r10:00000003 r9:00000101 r8:c0a586c0 r7:ffffe000 r6:40000003
r5:c0a586cc
[  571.366412]  r4:00000000
[  571.368959] [<c0132e2c>] (__do_softirq) from [<c01333e4>]
(irq_exit+0x110/0x144)
[  571.376389]  r10:00000001 r9:c0a00000 r8:df004000 r7:00000001 r6:00000000
r5:00000000
[  571.384247]  r4:c0a2efd0
[  571.386798] [<c01332d4>] (irq_exit) from [<c0161490>]
(__handle_domain_irq+0x60/0xb0)
[  571.394666] [<c0161430>] (__handle_domain_irq) from [<c01014fc>]
(gic_handle_irq+0x40/0x6c)
[  571.403055]  r9:c0a00000 r8:fa241100 r7:fa240100 r6:c0a01f18 r5:fa24010c
r4:c0a02b34
[  571.410834] [<c01014bc>] (gic_handle_irq) from [<c010c20c>]
(__irq_svc+0x6c/0xa8)
[  571.418345] Exception stack(0xc0a01f18 to 0xc0a01f60)
[  571.423415] 1f00:                                                      
00000000 00000084
[  571.431630] 1f20: e88e1d61 000069cb ffffe000 c0a028b4 00000000 00000000
c0a4b108 c0a54fa7
[  571.439843] 1f40: 00000001 c0a01f74 c0a01e80 c0a01f68 c0357168 c0183d08
20000013 ffffffff
[  571.448057]  r9:c0a00000 r8:c0a4b108 r7:c0a01f4c r6:ffffffff r5:20000013
r4:c0183d08
[  571.455849] [<c0183cd4>] (tick_nohz_idle_enter) from [<c015a004>]
(cpu_startup_entry+0x44/0x174)
[  571.464681] [<c0159fc0>] (cpu_startup_entry) from [<c06cf144>]
(rest_init+0x8c/0x90)
[  571.472454]  r7:c0a02840
[  571.475001] [<c06cf0b8>] (rest_init) from [<c0900c8c>]
(start_kernel+0x314/0x320)
[  571.482513]  r5:c0a57000 r4:00000000
[  571.486104] [<c0900978>] (start_kernel) from [<80008078>] (0x80008078)
[  571.492661] Code: e34c0082 e58d400c e58de010 ebf03935 (e7f001f2) 
[  571.498782] ---[ end trace adcee6f4e49b6bb7 ]---
[  571.503418] Kernel panic - not syncing: Fatal exception in interrupt
[  571.509799] Rebooting in 30 seconds..


If I comment out "rmmod ipsec" from _stackmanager and shutdown I get this
(which still in the bottom part says ipsec was unloaded?)

Sep  9 08:07:19 kernel: [  373.641887] IPSEC EVENT: KLIPS device ipsec0 shut
down.
Sep  9 08:07:20 kernel: [  374.376539] 
Sep  9 08:07:20 kernel: [  374.529318] 
Sep  9 08:07:20 kernel: [  374.529347] klips_info:pfkey_cleanup: shutting down
PF_KEY domain sockets.
Sep  9 08:07:20 kernel: [  374.559223] NET: Unregistered protocol family 15
[  374.816756] Internal error: Oops: 5 [#1] PREEMPT ARM
[  374.821751] Modules linked in: cmac gcm ccm cdc_acm nf_log_ipv6
nf_conntrack_ipv6 nf_defrag_ipv6 nf_log_ipv4 nf_log_common xt_multiport
xt_TCPMSS xt_conntrack xt_LOG xt_limit iptable_mangle ipt_MASQUERADE
nf_nat_masquerade_ipv4 xt_REDIRECT nf_nat_redirect iptable_nat
nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_raw iptable_filter
ip_tables ath9k_htc mac80211 ath9k_common ath9k_hw ath cfg80211 smsc95xx
cdc_ncm cdc_ether GobiNet(O) usbnet mii GobiSerial(O) ti_am335x_adc kfifo_buf
industrialio i2c_hid usbhid hid_generic hid xr_usb_serial_common(O) option
usb_wwan usbserial ppp_async ppp_generic slhc gpio_keys evdev sd_mod sg
mtd_spi_sram spi_omap2_mcspi lm75 ads1015 hwmon at25 at24 nvmem_core usb_f_ecm
dwc3_omap sxni_iodb(O) g_ether usb_f_rndis libcomposite u_ether configfs
jbm_feature [last unloaded: ipsec]
[  374.895105] CPU: 0 PID: 1678 Comm: sled_watcher Tainted: G           O   
4.9.119 #1
[  374.902880] Hardware name: Generic AM43 (Flattened Device Tree)
[  374.908823] task: df1ed100 task.stack: dddb4000
[  374.913386] PC is at copy_process.part.3+0xab4/0x1590
[  374.918457] LR is at 0x0
[  374.921000] pc : [<c012e048>]    lr : [<00000000>]    psr: 00010013
[  374.921000] sp : dddb5eb8  ip : ffff0002  fp : dddb5f3c
[  374.932525] r10: 00000000  r9 : 00000000  r8 : 00000000
[  374.937769] r7 : c0a58200  r6 : ddf8d540  r5 : ddc7e1b8  r4 : dd30daa8
[  374.944322] r3 : 000000b8  r2 : de0863c4  r1 : 00000000  r0 : ddc46798
[  374.950877] Flags: nzcv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment
none
[  374.958041] Control: 10c53c7d  Table: 9d9e8059  DAC: 00000051
[  374.963809] Process sled_watcher (pid: 1678, stack limit = 0xdddb4210)
[  374.970361] Stack: (0xdddb5eb8 to 0xdddb6000)
[  374.974735] 5ea0:                                                      
dddb5ee4 fff7cdff
[  374.982949] 5ec0: 024000c0 ddf8d574 ddd131f4 ffffe000 ddf8d544 ddf8d540
00000000 00000000
[  374.991164] 5ee0: 01200011 ddd03448 ddd03280 ddd131c0 00000000 b6fd7c78
00000000 00000000
[  374.999377] 5f00: 44431ce7 00000057 dddb5f34 fffffff4 c013a9d0 01200011
00000000 b6fd80d0
[  375.007590] 5f20: 00000000 00000000 dddb4000 00000000 dddb5f8c dddb5f40
c012ec84 c012d5a0
[  375.015804] 5f40: 00000000 00000000 ffffffff be909fc4 00000051 be909f44
00000051 be909fc4
[  375.024017] 5f60: 00000008 b6fd7c78 00000000 b6fd80d0 00000078 c0107d04
dddb4000 00000000
[  375.032230] 5f80: dddb5fa4 dddb5f90 c012eff8 c012ebe8 b6fd7c78 00000000
00000000 dddb5fa8
[  375.040443] 5fa0: c0107b40 c012efdc b6fd7c78 00000000 01200011 00000000
00000000 00000000
[  375.048656] 5fc0: b6fd7c78 00000000 b6fd80d0 00000078 b6f9a000 00000000
00000000 b6fd7c10
[  375.056869] 5fe0: 00000078 be909ef0 b6f1995f b6ebd206 20010030 01200011
00000000 00000000
[  375.065075] Backtrace: 
[  375.067544] [<c012d594>] (copy_process.part.3) from [<c012ec84>]
(_do_fork+0xa8/0x328)
[  375.075498]  r10:00000000 r9:dddb4000 r8:00000000 r7:00000000 r6:b6fd80d0
r5:00000000
[  375.083358]  r4:01200011
[  375.085906] [<c012ebdc>] (_do_fork) from [<c012eff8>]
(SyS_clone+0x28/0x30)
[  375.092900]  r10:00000000 r9:dddb4000 r8:c0107d04 r7:00000078 r6:b6fd80d0
r5:00000000
[  375.100759]  r4:b6fd7c78
[  375.103315] [<c012efd0>] (SyS_clone) from [<c0107b40>]
(ret_fast_syscall+0x0/0x48)
[  375.110919] Code: e3130b02 0a000006 e28130b8 f5d3f000 (e1930f9f) 
[  375.123423] ---[ end trace adcee6f4e49b6bb7 ]---
Sep  9 08:07:21 kernel: [  374.816726] Unable to handle kernel NULL pointer
dereference at virtual address 000000b8
Sep  9 08:07:21 kernel: [  374.816738] pgd = dd9e8000
Sep  9 08:07:21 kernel: [  374.816743] [000000b8] *pgd=00000000
Sep  9 08:07:21 kernel: [  374.816756] Internal error: Oops: 5 [#1] PREEMPT
ARM
Sep  9 08:07:21 kernel: [  374.821751] Modules linked in: cmac gcm ccm cdc_acm
nf_log_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 nf_log_ipv4 nf_log_common
xt_multiport xt_TCPMSS xt_conntrack xt_LOG xt_limit iptable_mangle
ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_REDIRECT nf_nat_redirect iptable_nat
nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_raw iptable_filter
ip_tables ath9k_htc mac80211 ath9k_common ath9k_hw ath cfg80211 smsc95xx
cdc_ncm cdc_ether GobiNet(O) usbnet mii GobiSerial(O) ti_am335x_adc kfifo_buf
industrialio i2c_hid usbhid hid_generic hid xr_usb_serial_common(O) option
usb_wwan usbserial ppp_async ppp_generic slhc gpio_keys evdev sd_mod sg
mtd_spi_sram spi_omap2_mcspi lm75 ads1015 hwmon at25 at24 nvmem_core usb_f_ecm
dwc3_omap sxni_iodb(O) g_ether usb_f_rndis libcomposite u_ether configfs
jbm_feature
Sep  9 08:07:21 kernel: [  374.892904]  [last unloaded: ipsec]
Sep  9 08:07:21 kernel: [  374.895105] CPU: 0 PID: 1678 Comm: sled_watcher
Tainted: G           O    4.9.119 #1
Sep  9 08:07:21 kernel: [  374.902880] Hardware name: Generic AM43 (Flattened
Device Tree)
Sep  9 08:07:21 kernel: [  374.908823] task: df1ed100 task.stack: dddb4000
Sep  9 08:07:21 kernel: [  374.913386] PC is at
copy_process.part.3+0xab4/0x1590
Sep  9 08:07:21 kernel: [  374.918457] LR is at 0x0
Sep  9 08:07:21 kernel: [  374.921000] pc : [<c012e048>]    lr : [<00000000>] 
  psr: 00010013
Sep  9 08:07:21 kernel: [  374.921000] sp : dddb5eb8  ip : ffff0002  fp :
dddb5f3c
Sep  9 08:07:21 kernel: [  374.932525] r10: 00000000  r9 : 00000000  r8 :
00000000
Sep  9 08:07:21 kernel: [  374.937769] r7 : c0a58200  r6 : ddf8d540  r5 :
ddc7e1b8  r4 : dd30daa8
Sep  9 08:07:21 kernel: [  374.944322] r3 : 000000b8  r2 : de0863c4  r1 :
00000000  r0 : ddc46798
Sep  9 08:07:21 kernel: [  374.950877] Flags: nzcv  IRQs on  FIQs on  Mode
SVC_32  ISA ARM  Segment none
Sep  9 08:07:21 kernel: [  374.958041] Control: 10c53c7d  Table: 9d9e8059 
DAC: 00000051
Sep  9 08:07:21 kernel: [  374.963809] Process sled_watcher (pid: 1678, stack
limit = 0xdddb4210)
Sep  9 08:07:21 kernel: [  374.970361] Stack: (0xdddb5eb8 to 0xdddb6000)
Sep  9 08:07:21 kernel: [  374.974735] 5ea0:                                  
                    dddb5ee4 fff7cdff
Sep  9 08:07:21 kernel: [  374.982949] 5ec0: 024000c0 ddf8d574 ddd131f4
ffffe000 ddf8d544 ddf8d540 00000000 00000000
Sep  9 08:07:21 kernel: [  374.991164] 5ee0: 01200011 ddd03448 ddd03280
ddd131c0 00000000 b6fd7c78 00000000 00000000
Sep  9 08:07:21 kernel: [  374.999377] 5f00: 44431ce7 00000057 dddb5f34
fffffff4 c013a9d0 01200011 00000000 b6fd80d0
Sep  9 08:07:21 kernel: [  375.007590] 5f20: 00000000 00000000 dddb4000
00000000 dddb5f8c dddb5f40 c012ec84 c012d5a0
Sep  9 08:07:21 kernel: [  375.015804] 5f40: 00000000 00000000 ffffffff
be909fc4 00000051 be909f44 00000051 be909fc4
Sep  9 08:07:21 kernel: [  375.024017] 5f60: 00000008 b6fd7c78 00000000
b6fd80d0 00000078 c0107d04 dddb4000 00000000
Sep  9 08:07:21 kernel: [  375.032230] 5f80: dddb5fa4 dddb5f90 c012eff8
c012ebe8 b6fd7c78 00000000 00000000 dddb5fa8
Sep  9 08:07:21 kernel: [  375.040443] 5fa0: c0107b40 c012efdc b6fd7c78
00000000 01200011 00000000 00000000 00000000
Sep  9 08:07:21 kernel: [  375.048656] 5fc0: b6fd7c78 00000000 b6fd80d0
00000078 b6f9a000 00000000 00000000 b6fd7c10
Sep  9 08:07:21 kernel: [  375.056869] 5fe0: 00000078 be909ef0 b6f1995f
b6ebd206 20010030 01200011 00000000 00000000
Sep  9 08:07:21 kernel: [  375.065075] Backtrace: 
Sep  9 08:07:21 kernel: [  375.067544] [<c012d594>] (copy_process.part.3) from
[<c012ec84>] (_do_fork+0xa8/0x328)
Sep  9 08:07:21 kernel: [  375.075498]  r10:00000000 r9:dddb4000 r8:00000000
r7:00000000 r6:b6fd80d0 r5:00000000
Sep  9 08:07:21 kernel: [  375.083358]  r4:01200011
Sep  9 08:07:21 kernel: [  375.085906] [<c012ebdc>] (_do_fork) from
[<c012eff8>] (SyS_clone+0x28/0x30)
Sep  9 08:07:21 kernel: [  375.092900]  r10:00000000 r9:dddb4000 r8:c0107d04
r7:00000078 r6:b6fd80d0 r5:00000000
Sep  9 08:07:21 kernel: [  375.100759]  r4:b6fd7c78
Sep  9 08:07:21 kernel: [  375.103315] [<c012efd0>] (SyS_clone) from
[<c0107b40>] (ret_fast_syscall+0x0/0x48)
Sep  9 08:07:21 kernel: [  375.110919] Code: e3130b02 0a000006 e28130b8
f5d3f000 (e1930f9f) 
Sep  9 08:07:21 kernel: [  375.123423] ---[ end trace adcee6f4e49b6bb7 ]---






More information about the Swan mailing list