Code to reproduce the panic and system info

forcedethPanic_testcode_2007-08-13.txt (text/plain), 51.70 KB, created by slowfood on 2007-08-13 23:38:42 UTC

Some additional info as requested by:
  http://kernel.org/pub/linux/docs/lkml/reporting-bugs.html
[4.] Kernel version (from /proc/version):
# cat /proc/version
Linux version 2.6.22-gentoo-r1 (root@cl34) (gcc version 4.1.2 (Gentoo 4.1.2)) #1 SMP PREEMPT Tue Jul 24 14:22:37 PDT 2007

[6.] A small shell script or example program which triggers the
     problem (if possible)
See below for two test programs (server and client)

[7.] Environment
Two SuperMicro H8DME-2 (Dual AMD64 Opteron 2212HE dual core) motherboards talkign to each other over Gig-E network.
[7.1.] Software (add the output of the ver_linux script here):
Don'thave that script, but:
# uname -a
Linux cl34 2.6.22-gentoo-r1 #1 SMP PREEMPT Tue Jul 24 14:22:37 PDT 2007 x86_64 Dual-Core AMD Opteron(tm) Processor 2212 AuthenticAMD GNU/Linux

[7.2.] Processor information (from /proc/cpuinfo):
# cat /proc/cpuinfo
processor       : 0
vendor_id       : AuthenticAMD
cpu family      : 15
model           : 65
model name      : Dual-Core AMD Opteron(tm) Processor 2212
stepping        : 2
cpu MHz         : 2010.318
cache size      : 1024 KB
physical id     : 0
siblings        : 2
core id         : 0
cpu cores       : 2
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16
lahf_lm cmp_legacy svm extapic cr8_legacy
bogomips        : 4023.95
TLB size        : 1024 4K pages
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp tm stc

processor       : 1
vendor_id       : AuthenticAMD
cpu family      : 15
model           : 65
model name      : Dual-Core AMD Opteron(tm) Processor 2212
stepping        : 2
cpu MHz         : 2010.318
cache size      : 1024 KB
physical id     : 0
siblings        : 2
core id         : 1
cpu cores       : 2
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16
lahf_lm cmp_legacy svm extapic cr8_legacy
bogomips        : 4021.04
TLB size        : 1024 4K pages
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp tm stc

processor       : 2
vendor_id       : AuthenticAMD
cpu family      : 15
model           : 65
model name      : Dual-Core AMD Opteron(tm) Processor 2212
stepping        : 2
cpu MHz         : 2010.318
cache size      : 1024 KB
physical id     : 1
siblings        : 2
core id         : 0
cpu cores       : 2
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16
lahf_lm cmp_legacy svm extapic cr8_legacy
bogomips        : 4020.96
TLB size        : 1024 4K pages
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp tm stc

processor       : 3
vendor_id       : AuthenticAMD
cpu family      : 15
model           : 65
model name      : Dual-Core AMD Opteron(tm) Processor 2212
stepping        : 2
cpu MHz         : 2010.318
cache size      : 1024 KB
physical id     : 1
siblings        : 2
core id         : 1
cpu cores       : 2
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16
lahf_lm cmp_legacy svm extapic cr8_legacy
bogomips        : 4020.96
TLB size        : 1024 4K pages
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management: ts fid vid ttp tm stc

[7.3.] Module information (from /proc/modules):
all compiled in, no modules used.
[7.4.] Loaded driver and hardware information (/proc/ioports, /proc/iomem)
# cat /proc/ioports
0000-001f : dma1
0020-0021 : pic1
0040-0043 : timer0
0050-0053 : timer1
0060-006f : keyboard
0070-0077 : rtc
0080-008f : dma page reg
00a0-00a1 : pic2
00c0-00df : dma2
00f0-00ff : fpu
0170-0177 : 0000:00:04.0
01f0-01f7 : 0000:00:04.0
0290-029f : pnp 00:0a
02f8-02ff : serial
0376-0376 : 0000:00:04.0
03c0-03df : vga+
03f6-03f6 : 0000:00:04.0
03f8-03ff : serial
0ca2-0ca2 : ipmi_si
0ca3-0ca3 : ipmi_si
0cf8-0cff : PCI conf1
2000-2003 : ACPI PM1a_EVT_BLK
2004-2005 : ACPI PM1a_CNT_BLK
2008-200b : ACPI PM_TMR
2010-2015 : ACPI CPU throttle
2020-2027 : ACPI GPE0_BLK
24a0-24af : ACPI GPE1_BLK
2d00-2d3f : 0000:00:01.1
  2d00-2d3f : nForce2_smbus
2e00-2e3f : 0000:00:01.1
  2e00-2e3f : nForce2_smbus
b080-b087 : 0000:00:09.0
  b080-b087 : forcedeth
b400-b407 : 0000:00:08.0
  b400-b407 : forcedeth
b480-b48f : 0000:00:05.2
  b480-b48f : libata
b800-b803 : 0000:00:05.2
  b800-b803 : libata
b880-b887 : 0000:00:05.2
  b880-b887 : libata
bc00-bc03 : 0000:00:05.2
  bc00-bc03 : libata
c000-c007 : 0000:00:05.2
  c000-c007 : libata
c080-c08f : 0000:00:05.1
  c080-c08f : libata
c400-c403 : 0000:00:05.1
  c400-c403 : libata
c480-c487 : 0000:00:05.1
  c480-c487 : libata
c800-c803 : 0000:00:05.1
  c800-c803 : libata
c880-c887 : 0000:00:05.1
  c880-c887 : libata
cc00-cc0f : 0000:00:05.0
  cc00-cc0f : libata
d000-d003 : 0000:00:05.0
  d000-d003 : libata
d080-d087 : 0000:00:05.0
  d080-d087 : libata
d400-d403 : 0000:00:05.0
  d400-d403 : libata
d480-d487 : 0000:00:05.0
  d480-d487 : libata
dc00-dc3f : 0000:00:01.1
e000-efff : PCI Bus #01
  e000-e0ff : 0000:01:05.0
ffa0-ffaf : 0000:00:04.0
  ffa0-ffa7 : ide0
#  cat /proc/iomem
00000000-0009fbff : System RAM
  00000000-00000000 : Crash kernel
0009fc00-0009ffff : reserved
000c0000-000cffff : pnp 00:0c
000e0000-000fffff : reserved
00100000-dffeffff : System RAM
  00200000-0065dc8e : Kernel code
  0065dc8f-0084e50f : Kernel data
  04000000-07ffffff : GART
dfff0000-dfffdfff : ACPI Tables
dfffe000-dfffffff : ACPI Non-volatile Storage
e0000000-efffffff : pnp 00:0b
f0000000-f7ffffff : PCI Bus #01
  f0000000-f7ffffff : 0000:01:05.0
    f0000000-f7ffffff : radeonfb framebuffer
fea80000-feabffff : pnp 00:07
feaf4c00-feaf4c0f : 0000:00:09.0
  feaf4c00-feaf4c0f : forcedeth
feaf5000-feaf5fff : 0000:00:09.0
  feaf5000-feaf5fff : forcedeth
feaf6000-feaf6fff : 0000:00:08.0
  feaf6000-feaf6fff : forcedeth
feaf7000-feaf7fff : 0000:00:05.2
  feaf7000-feaf7fff : sata_nv
feaf8000-feaf8fff : 0000:00:05.1
  feaf8000-feaf8fff : sata_nv
feaf9000-feaf9fff : 0000:00:05.0
  feaf9000-feaf9fff : sata_nv
feafa000-feafa0ff : 0000:00:09.0
  feafa000-feafa0ff : forcedeth
feafa400-feafa40f : 0000:00:08.0
  feafa400-feafa40f : forcedeth
feafa800-feafa8ff : 0000:00:08.0
  feafa800-feafa8ff : forcedeth
feafac00-feafacff : 0000:00:02.1
  feafac00-feafacff : ehci_hcd
feafb000-feafbfff : 0000:00:02.0
  feafb000-feafbfff : ohci_hcd
feb00000-febfffff : PCI Bus #01
  feb00000-feb1ffff : 0000:01:05.0
  febf0000-febfffff : 0000:01:05.0
    febf0000-febfffff : radeonfb mmio
fec00000-fec00fff : IOAPIC 0
  fec00000-fec00fff : reserved
fee00000-fee00fff : Local APIC
fee01000-feefffff : pnp 00:07
ff700000-ffffffff : reserved
100000000-41fffffff : System RAM


[7.5.] PCI information ('lspci -vvv' as root)
# # lspci -vvv
00:00.0 RAM memory: nVidia Corporation MCP55 Memory Controller (rev a2)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0
        Capabilities: [44] HyperTransport: Slave or Primary Interface
                Command: BaseUnitID=0 UnitCnt=15 MastHost- DefDir- DUL-
                Link Control 0: CFlE+ CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0 IsocEn- LSEn+ ExtCTL- 64b-
                Link Config 0: MLWI=16bit DwFcIn- MLWO=16bit DwFcOut- LWI=16bit DwFcInEn- LWO=16bit DwFcOutEn-
                Link Control 1: CFlE- CST- CFE- <LkFail+ Init- EOC+ TXO+ <CRCErr=0 IsocEn- LSEn- ExtCTL- 64b-
                Link Config 1: MLWI=8bit DwFcIn- MLWO=8bit DwFcOut- LWI=8bit DwFcInEn- LWO=8bit DwFcOutEn-
                Revision ID: 1.03
                Link Frequency 0: 1.0GHz
                Link Error 0: <Prot- <Ovfl- <EOC- CTLTm-
                Link Frequency Capability 0: 200MHz+ 300MHz+ 400MHz+ 500MHz+ 600MHz+ 800MHz+ 1.0GHz+ 1.2GHz- 1.4GHz- 1.6GHz- Vend-
                Feature Capability: IsocFC+ LDTSTOP+ CRCTM- ECTLT- 64bA- UIDRD-
                Link Frequency 1: 200MHz
                Link Error 1: <Prot- <Ovfl- <EOC- CTLTm-
                Link Frequency Capability 1: 200MHz- 300MHz- 400MHz- 500MHz- 600MHz- 800MHz- 1.0GHz- 1.2GHz- 1.4GHz- 1.6GHz- Vend-
                Error Handling: PFlE+ OFlE+ PFE- OFE- EOCFE- RFE- CRCFE- SERRFE- CF- RE- PNFE- ONFE- EOCNFE- RNFE- CRCNFE- SERRNFE-
                Prefetchable memory behind bridge Upper: 00-00
                Bus Number: 00
        Capabilities: [dc] HyperTransport: MSI Mapping

00:01.0 ISA bridge: nVidia Corporation MCP55 LPC Bridge (rev a3)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle+ MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0

00:01.1 SMBus: nVidia Corporation MCP55 SMBus (rev a3)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Interrupt: pin A routed to IRQ 11
        Region 0: I/O ports at dc00 [size=64]
        Region 4: I/O ports at 2d00 [size=64]
        Region 5: I/O ports at 2e00 [size=64]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:01.2 RAM memory: nVidia Corporation MCP55 Memory Controller (rev a3)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:02.0 USB Controller: nVidia Corporation MCP55 USB Controller (rev a1) (prog-if 10 [OHCI])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Interrupt: pin A routed to IRQ 21
        Region 0: Memory at feafb000 (32-bit, non-prefetchable) [size=4K]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:02.1 USB Controller: nVidia Corporation MCP55 USB Controller (rev a2) (prog-if 20 [EHCI])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Interrupt: pin B routed to IRQ 22
        Region 0: Memory at feafac00 (32-bit, non-prefetchable) [size=256]
        Capabilities: [44] Debug port
        Capabilities: [80] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:04.0 IDE interface: nVidia Corporation MCP55 IDE (rev a1) (prog-if 8a [Master SecP PriP])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Region 0: [virtual] Memory at 000001f0 (32-bit, non-prefetchable) [disabled] [size=8]
        Region 1: [virtual] Memory at 000003f0 (type 3, non-prefetchable) [disabled] [size=1]
        Region 2: [virtual] Memory at 00000170 (32-bit, non-prefetchable) [disabled] [size=8]
        Region 3: [virtual] Memory at 00000370 (type 3, non-prefetchable) [disabled] [size=1]
        Region 4: I/O ports at ffa0 [size=16]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-

00:05.0 IDE interface: nVidia Corporation MCP55 SATA Controller (rev a3) (prog-if 85 [Master SecO PriO])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Interrupt: pin A routed to IRQ 21
        Region 0: I/O ports at d480 [size=8]
        Region 1: I/O ports at d400 [size=4]
        Region 2: I/O ports at d080 [size=8]
        Region 3: I/O ports at d000 [size=4]
        Region 4: I/O ports at cc00 [size=16]
        Region 5: Memory at feaf9000 (32-bit, non-prefetchable) [size=4K]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [b0] Message Signalled Interrupts: Mask- 64bit+ Queue=0/2 Enable-
                Address: 0000000000000000  Data: 0000
        Capabilities: [cc] HyperTransport: MSI Mapping

00:05.1 IDE interface: nVidia Corporation MCP55 SATA Controller (rev a3) (prog-if 85 [Master SecO PriO])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Interrupt: pin B routed to IRQ 20
        Region 0: I/O ports at c880 [size=8]
        Region 1: I/O ports at c800 [size=4]
        Region 2: I/O ports at c480 [size=8]
        Region 3: I/O ports at c400 [size=4]
        Region 4: I/O ports at c080 [size=16]
        Region 5: Memory at feaf8000 (32-bit, non-prefetchable) [size=4K]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [b0] Message Signalled Interrupts: Mask- 64bit+ Queue=0/2 Enable-
                Address: 0000000000000000  Data: 0000
        Capabilities: [cc] HyperTransport: MSI Mapping

00:05.2 IDE interface: nVidia Corporation MCP55 SATA Controller (rev a3) (prog-if 85 [Master SecO PriO])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (750ns min, 250ns max)
        Interrupt: pin C routed to IRQ 23
        Region 0: I/O ports at c000 [size=8]
        Region 1: I/O ports at bc00 [size=4]
        Region 2: I/O ports at b880 [size=8]
        Region 3: I/O ports at b800 [size=4]
        Region 4: I/O ports at b480 [size=16]
        Region 5: Memory at feaf7000 (32-bit, non-prefetchable) [size=4K]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [b0] Message Signalled Interrupts: Mask- 64bit+ Queue=0/2 Enable-
                Address: 0000000000000000  Data: 0000
        Capabilities: [cc] HyperTransport: MSI Mapping

00:06.0 PCI bridge: nVidia Corporation Unknown device 0370 (rev a2) (prog-if 01 [Subtractive decode])
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0
        Bus: primary=00, secondary=01, subordinate=01, sec-latency=64
        I/O behind bridge: 0000e000-0000efff
        Memory behind bridge: feb00000-febfffff
        Prefetchable memory behind bridge: f0000000-f7ffffff
        Secondary status: 66MHz- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA+ MAbort- >Reset- FastB2B-
        Capabilities: [b8] Subsystem: nVidia Corporation Unknown device cb84
        Capabilities: [8c] HyperTransport: MSI Mapping

00:08.0 Bridge: nVidia Corporation MCP55 Ethernet (rev a3)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (250ns min, 5000ns max)
        Interrupt: pin A routed to IRQ 377
        Region 0: Memory at feaf6000 (32-bit, non-prefetchable) [size=4K]
        Region 1: I/O ports at b400 [size=8]
        Region 2: Memory at feafa800 (32-bit, non-prefetchable) [size=256]
        Region 3: Memory at feafa400 (32-bit, non-prefetchable) [size=16]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable+ DSel=0 DScale=0 PME-
        Capabilities: [70] MSI-X: Enable- Mask- TabSize=8
                Vector table: BAR=2 offset=00000000
                PBA: BAR=3 offset=00000000
        Capabilities: [50] Message Signalled Interrupts: Mask+ 64bit+ Queue=0/3 Enable+
                Address: 00000000fee0f00c  Data: 41a1
                Masking: 000000fe  Pending: 00000000
        Capabilities: [6c] HyperTransport: MSI Mapping

00:09.0 Bridge: nVidia Corporation MCP55 Ethernet (rev a3)
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz+ UDF- FastB2B+ ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0 (250ns min, 5000ns max)
        Interrupt: pin A routed to IRQ 376
        Region 0: Memory at feaf5000 (32-bit, non-prefetchable) [size=4K]
        Region 1: I/O ports at b080 [size=8]
        Region 2: Memory at feafa000 (32-bit, non-prefetchable) [size=256]
        Region 3: Memory at feaf4c00 (32-bit, non-prefetchable) [size=16]
        Capabilities: [44] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable+ DSel=0 DScale=0 PME-
        Capabilities: [70] MSI-X: Enable- Mask- TabSize=8
                Vector table: BAR=2 offset=00000000
                PBA: BAR=3 offset=00000000
        Capabilities: [50] Message Signalled Interrupts: Mask+ 64bit+ Queue=0/3 Enable+
                Address: 00000000fee0f00c  Data: 41a9
                Masking: 000000fe  Pending: 00000000
        Capabilities: [6c] HyperTransport: MSI Mapping

00:0a.0 PCI bridge: nVidia Corporation Unknown device 0376 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=02, subordinate=04, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4149
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x8, ASPM L0s L1, Port 5
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x8
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:0b.0 PCI bridge: nVidia Corporation Unknown device 0374 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=05, subordinate=05, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4151
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x0, ASPM L0s L1, Port 4
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x4
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:0c.0 PCI bridge: nVidia Corporation Unknown device 0374 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=06, subordinate=06, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4159
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x0, ASPM L0s L1, Port 3
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x4
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:0d.0 PCI bridge: nVidia Corporation Unknown device 0378 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=07, subordinate=07, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4161
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x4, ASPM L0s L1, Port 2
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x4
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:0e.0 PCI bridge: nVidia Corporation Unknown device 0375 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=08, subordinate=08, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4169
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x8, ASPM L0s L1, Port 1
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x8
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:0f.0 PCI bridge: nVidia Corporation Unknown device 0377 (rev a3) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=00, secondary=09, subordinate=09, sec-latency=0
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- <SERR- <PERR-
        BridgeCtl: Parity- SERR+ NoISA- VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Subsystem: nVidia Corporation Unknown device 0000
        Capabilities: [48] Power Management version 2
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0+,D1+,D2+,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] Message Signalled Interrupts: Mask- 64bit+ Queue=0/1 Enable+
                Address: 00000000fee0f00c  Data: 4171
        Capabilities: [60] HyperTransport: MSI Mapping
        Capabilities: [80] Express Root Port (Slot+) IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: Errors: Correctable+ Non-Fatal+ Fatal+ Unsupported+
                Device: RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x8, ASPM L0s L1, Port 0
                Link: Latency L0s <512ns, L1 <4us
                Link: ASPM Disabled RCB 64 bytes CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x16
                Slot: AtnBtn- PwrCtrl- MRL- AtnInd- PwrInd- HotPlug- Surpise-
                Slot: Number 0, PowerLimit 0.000000
                Slot: Enabled AtnBtn- PwrFlt- MRL- PresDet- CmdCplt- HPIrq-
                Slot: AttnInd Off, PwrInd On, Power-
                Root: Correctable- Non-Fatal- Fatal- PME-

00:18.0 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] HyperTransport Technology Configuration
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Capabilities: [80] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=16bit LWO=16bit
                Revision ID: 1.02
        Capabilities: [a0] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=16bit LWO=16bit
                Revision ID: 1.02
        Capabilities: [c0] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=16bit LWO=16bit
                Revision ID: 1.02

00:18.1 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] Address Map
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:18.2 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] DRAM Controller
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:18.3 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] Miscellaneous Control
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Capabilities: [f0] #0f [0010]

00:19.0 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] HyperTransport Technology Configuration
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Capabilities: [80] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=16bit LWO=16bit
                Revision ID: 1.02
        Capabilities: [a0] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail- Init+ EOC- TXO- <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=16bit LWO=16bit
                Revision ID: 1.02
        Capabilities: [c0] HyperTransport: Host or Secondary Interface
                !!! Possibly incomplete decoding
                Command: WarmRst+ DblEnd-
                Link Control: CFlE- CST- CFE- <LkFail+ Init- EOC+ TXO+ <CRCErr=0
                Link Config: MLWI=16bit MLWO=16bit LWI=N/C LWO=N/C
                Revision ID: 1.02

00:19.1 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] Address Map
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:19.2 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] DRAM Controller
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-

00:19.3 Host bridge: Advanced Micro Devices [AMD] K8 [Athlon64/Opteron] Miscellaneous Control
        Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Capabilities: [f0] #0f [0010]

01:05.0 VGA compatible controller: ATI Technologies Inc ES1000 (rev 02) (prog-if 00 [VGA])
        Subsystem: Super Micro Computer Inc Unknown device 1611
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping+ SERR- FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B+ ParErr- DEVSEL=medium >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 64 (2000ns min), Cache Line Size: 64 bytes
        Interrupt: pin A routed to IRQ 19
        Region 0: Memory at f0000000 (32-bit, prefetchable) [size=128M]
        Region 1: I/O ports at e000 [size=256]
        Region 2: Memory at febf0000 (32-bit, non-prefetchable) [size=64K]
        Expansion ROM at feb00000 [disabled] [size=128K]
        Capabilities: [50] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2+ AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-

02:00.0 PCI bridge: NEC Corporation uPD720400 PCI Express - PCI/PCI-X Bridge (rev 07) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=02, secondary=03, subordinate=03, sec-latency=64
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz+ FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
        BridgeCtl: Parity+ SERR+ NoISA+ VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Express PCI/PCI-X Bridge IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: AtnBtn- AtnInd- PwrInd-
                Device: Errors: Correctable- Non-Fatal- Fatal- Unsupported-
                Device: RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x8, ASPM L0s L1, Port 0
                Link: Latency L0s <4us, L1 <16us
                Link: ASPM Disabled CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x8
        Capabilities: [54] PCI-X bridge device
                Secondary Status: 64bit+ 133MHz+ SCD- USC- SCO- SRD- Freq=133MHz
                Status: Dev=02:00.0 64bit- 133MHz- SCD- USC- SCO- SRD-
                Upstream: Capacity=32 CommitmentLimit=32
                Downstream: Capacity=16 CommitmentLimit=16
        Capabilities: [64] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2- AuxCurrent=55mA PME(D0+,D1+,D2-,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
                Bridge: PM- B3+

02:00.1 PCI bridge: NEC Corporation uPD720400 PCI Express - PCI/PCI-X Bridge (rev 07) (prog-if 00 [Normal decode])
        Control: I/O- Mem- BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR+ FastB2B-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR-
        Latency: 0, Cache Line Size: 64 bytes
        Bus: primary=02, secondary=04, subordinate=04, sec-latency=64
        I/O behind bridge: 0000f000-00000fff
        Memory behind bridge: fff00000-000fffff
        Prefetchable memory behind bridge: 00000000fff00000-00000000000fffff
        Secondary status: 66MHz+ FastB2B- ParErr- DEVSEL=slow >TAbort- <TAbort- <MAbort+ <SERR- <PERR-
        BridgeCtl: Parity+ SERR+ NoISA+ VGA- MAbort- >Reset- FastB2B-
        Capabilities: [40] Express PCI/PCI-X Bridge IRQ 0
                Device: Supported: MaxPayload 256 bytes, PhantFunc 0, ExtTag-
                Device: Latency L0s <64ns, L1 <1us
                Device: AtnBtn- AtnInd- PwrInd-
                Device: Errors: Correctable- Non-Fatal- Fatal- Unsupported-
                Device: RlxdOrd- ExtTag- PhantFunc- AuxPwr- NoSnoop-
                Device: MaxPayload 128 bytes, MaxReadReq 512 bytes
                Link: Supported Speed 2.5Gb/s, Width x8, ASPM L0s L1, Port 0
                Link: Latency L0s <4us, L1 <16us
                Link: ASPM Disabled CommClk- ExtSynch-
                Link: Speed 2.5Gb/s, Width x8
        Capabilities: [54] PCI-X bridge device
                Secondary Status: 64bit+ 133MHz+ SCD- USC- SCO- SRD- Freq=133MHz
                Status: Dev=02:00.1 64bit- 133MHz- SCD- USC- SCO- SRD-
                Upstream: Capacity=32 CommitmentLimit=32
                Downstream: Capacity=16 CommitmentLimit=16
        Capabilities: [64] Power Management version 2
                Flags: PMEClk- DSI- D1+ D2- AuxCurrent=55mA PME(D0+,D1+,D2-,D3hot+,D3cold+)
                Status: D0 PME-Enable- DSel=0 DScale=0 PME-
                Bridge: PM- B3+


[7.6.] SCSI information (from /proc/scsi/scsi)
no scsi devices, only SATA

[7.7.] Other information that might be relevant to the problem
       (please look in /proc and include all information that you
       think to be relevant):
Happy to provide anything else folks think might help, nothing jumps out at me.)

[X.] Other notes, patches, fixes, workarounds:
This kernel does seem to have the patch from Ayaz described in:
   http://bugzilla.kernel.org/show_bug.cgi?id=8058
that talks about what seemed to be a similar problem, but Opps-ed in a
differnt place.

The folowing two programs reproduce the error easily. They use the boost option handling code, but are otherwise very straght forward. (emerge -av boost first,
then compile these on the respective server and client machiens:

# cat crashSvr.cc
#include <iostream>
#include <boost/thread.hpp>
#include <arpa/inet.h>

using namespace std;

const int PortNum = 3311;

class Listener
{
  int sock_;
public:
  Listener(int sock) : sock_(sock) { ; }
  Listener(const Listener& that) : sock_(that.sock_) { ; }
  bool getSize(int& size) {
    char* buf = reinterpret_cast<char*>(&size);
    int numBytes = sizeof(size);
    int offset = 0;
    int n;
    while ((n = read(sock_, (buf+offset), numBytes - offset)) > 0) {
      offset += n;
      if (offset >= numBytes) {
        return true;
      }
    }
    return false;
  }
  bool getMsg(const int size, char* buf) {
    int numBytes = size;
    int offset = 0;
    int n;
    while ((n = read(sock_, (buf+offset), numBytes - offset)) > 0) {
      offset += n;
      if (offset >= numBytes) {
        return true;
      }
    }
    return false;
  }
  bool sendMsg(const int size, char* buf) {
    if (write(sock_, (void*)(&size), sizeof(size)) != sizeof(size)) {
      cout << "Error writing back message size" << endl;
      return false;
    }
    if (write(sock_, (void*)(buf), size) != size) {
      cout << "Error writing back message" << endl;
      return false;
    }
    return true;
  }
  void operator()()
  {
    cout << "Listening on socket " << sock_ << endl;
    bool closed = false;
    int msgNum = 0;
    int totalSize = 0;
    while (!closed) {
      int size;
      if (!getSize(size)) {
        closed = true;
      } else {
        char buf[size+1];
        if (getMsg(size, buf) && sendMsg(size, buf)) {
          totalSize += size;
          ++msgNum;
        } else {
          closed = true;
        }
      }
    }
    cout << "Received & sent back " << msgNum << " messages on socket " << sock_ << ". Avg size: " << totalSize/msgNum << endl;
    close(sock_);
  }
};

void accept_(int sock)
{
  struct sockaddr_in remoteAddr;
  socklen_t remoteAddrSize = sizeof(remoteAddr);
  int remoteSock = accept(sock, (struct sockaddr*)&remoteAddr, &remoteAddrSize);
  if (remoteSock == -1) {
    cout << "Error in accepting connection on socket " << sock << endl;
    perror("accept: ");
  } else {
    // Connection received. Spawning thread to listen on it...
    new boost::thread(Listener(remoteSock));
  }
}

int main(int argc, char* argv[])
{
  struct sockaddr_in sin;
  memset(&sin, 0, sizeof(sin));
  int sock = socket(PF_INET, SOCK_STREAM, 0);
  if (sock == -1) {
    perror ("Unable to create socket");
    return -1;
  }
  sin.sin_family = AF_INET;
  sin.sin_port = htons(PortNum);
  sin.sin_addr.s_addr = INADDR_ANY;
  if (bind(sock, (struct sockaddr*)&sin, sizeof(sin)) == -1) {
    perror("Could not bind to socket. Quitting!");
    return -1;
  }
  if (listen(sock, 10) == -1) {
    perror("Could not start listening on socket. Quitting!");
    return -1;
  }
  cout << "Server running..." << endl;
  fd_set masterSet;
  FD_ZERO(&masterSet);
  FD_SET(sock, &masterSet);
  int loopNum = 0;
  while (true) {
    fd_set fds = masterSet;
    if (select(sock+1, &fds, 0, 0, 0) == -1) {
      perror("Select failure. Why?");
      return -1;
    }
    if (FD_ISSET(sock, &fds)) {
      cout << "Got connection on the socket. Woo-hoo. Now on to spawning a thread..." << endl;
      accept_(sock);
    }
  }
  cout << "Hello!" << endl;
  return 0;
}

# cat crashClnt.cc
#include <iostream>
#include <boost/program_options.hpp>
#include <boost/thread.hpp>
#include <arpa/inet.h>
#include <netdb.h>

namespace progOpts = boost::program_options;
using namespace std;

int DefaultNumThreads = 2;
int DefaultNumMsgs = 100;
int DefaultAvgMsgSize = 1000;  // (uniform distribution: 1 to 2*DefaultAvgMsgSize, inclusive)

bool parseProgOptions(progOpts::variables_map& vm, int argc, char* argv[])
{
  progOpts::options_description desc;
  desc.add_options()
    ("help", "produce help message")
    ("hostName,h", progOpts::value<string>(), "host name")
    ("portNum,p", progOpts::value<int>(), "port number on which the host will be listening")
    ("numThreads,t", progOpts::value<int>()->default_value(DefaultNumThreads), "Number of threads in the test")
    ("numMsgs,n", progOpts::value<int>()->default_value(DefaultNumMsgs), "Number of messages sent in each thread")
    ("avgSize,a", progOpts::value<int>()->default_value(DefaultAvgMsgSize), "Average message size")
    ("verbose,v", "Verbose output")
    ;

  progOpts::positional_options_description pd;
  pd.add("hostName", 1);
  pd.add("portNum", 1);

  progOpts::store(progOpts::command_line_parser(argc, argv).
                  options(desc).positional(pd).run(), vm);
  progOpts::notify(vm);

  if (vm.count("help")) {
    cout << desc << endl;
    return false;
  }
  if (!vm.count("hostName") || !vm.count("portNum")) {
    cout << "Must specify hostName & portNum" << endl;
    cout << desc << endl;
    return false;
  }
  return true;
}

class Sender {
private: // data
  int id_;
  string hostName_;
  int portNum_;
  int numMsgs_, avgSize_;
public: // members
  Sender(int id, const string& hostName, int portNum, int numMsgs, int avgSize)
    : id_(id), hostName_(hostName), portNum_(portNum), numMsgs_(numMsgs), avgSize_(avgSize)
  { ; }
  Sender(const Sender& that) :
    id_(that.id_), hostName_(that.hostName_), portNum_(that.portNum_), numMsgs_(that.numMsgs_), avgSize_(that.avgSize_)
  { ; }
  // connect to hostName_:portNum_
  int connect_()
  {
    struct hostent* he = gethostbyname(hostName_.c_str());
    if (he == NULL) {
      cout << "Unable to get hostname in thread " << id_ << ". Quitting thread!" << endl;
      perror("gethostbyname: ");
      return -1;
    }
    sockaddr_in hostAddr;
    hostAddr.sin_family = AF_INET;
    hostAddr.sin_port = htons(portNum_);
    memcpy(&hostAddr.sin_addr, he->h_addr_list[0], sizeof(hostAddr.sin_addr));

    int sock = socket(PF_INET, SOCK_STREAM, 0);
    if (sock == -1) {
      cout << "Unable to create socket in thread " << id_ << ". Quitting thread!" << endl;
      perror("socket: ");
      return -1;
    }

    if (connect(sock, (struct sockaddr *)(&hostAddr), sizeof(hostAddr)) == -1) {
      cout << "Unable to connectsocket in thread " << id_ << ". Quitting thread!" << endl;
      perror("connect:");
      return -1;
    }
    return sock;
  }
  // Get a size from a socket
  bool getSize(int sock, int& size) {
    char* buf = reinterpret_cast<char*>(&size);
    int numBytes = sizeof(size);
    int offset = 0;
    int n;
    while ((n = read(sock, (buf+offset), numBytes - offset)) > 0) {
      offset += n;
      if (offset >= numBytes) {
        return true;
      }
    }
    return false;
  }
  // get a message of given size back from a socket
  bool getMsg(int sock, const int size, char* buf) {
    int numBytes = size;
    int offset = 0;
    int n;
    while ((n = read(sock, (buf+offset), numBytes - offset)) > 0) {
      offset += n;
      if (offset >= numBytes) {
        return true;
      }
    }
    return false;
  }
  // Send messages on a socket
  void sendMsgs_(int sock)
  {
    unsigned char buf[avgSize_*2];
    for (int j=0; j<avgSize_*2; ++j) {
      unsigned char c = random() & 0x0ff;
      buf[j] = c;
    }
    for (int i=0; i<numMsgs_; ++i) {
      int size = (random() % (avgSize_*2)) + 1; // between 1 and avgSize_*2, inclusive
      if (write(sock, (void*)(&size), sizeof(size)) != sizeof(size)) {
        cout << "Error writing message #" << i << "(size). Quitting thread " << id_ << endl;
        return;
      }
      if (write(sock, (void*)(buf), size) != size) {
        cout << "Error writing message #" << i << ". Quitting thread " << id_ << endl;
        return;
      }
      int inSize;
      char inBuf[avgSize_*2];
      if (getSize(sock, inSize) && getMsg(sock, inSize, inBuf)) {
        if (size != inSize) {
          cout << "Insize(" << inSize << ") != outsize(" << size << "). Quitting thread " << id_ << endl;
          return;
        }
        if (memcmp(buf, inBuf, inSize) != 0) {
          cout << "Received message differs from sent message. Quitting thread " << id_ << endl;
          return;
        }
      } else {
        cout << "Error receiving back message #" << i << ". Quitting thread " << id_ << endl;
        return;
      }
    }
  }
  // Main entry point for the thread
  void operator()() {
    cout << "Executing thread " << id_ << endl;
    int sock = connect_();
    if (sock != -1) {
      sendMsgs_(sock);
    }
  }
};

int main(int argc, char* argv[])
{
  progOpts::variables_map vm;
  if (!parseProgOptions(vm, argc, argv)) {
    return 1;
  }
  string hostName = vm["hostName"].as<string>();
  int portNum = vm["portNum"].as<int>();
  int numThreads = vm["numThreads"].as<int>();
  int numMsgs = vm["numMsgs"].as<int>();
  int avgSize = vm["avgSize"].as<int>();

  std::vector<boost::thread*> threads;
  for (int i=0; i<numThreads; ++i) {
    threads.push_back(new boost::thread(Sender(i, hostName, portNum, numMsgs, avgSize)));
  }
  for (int i=0; i<numThreads; ++i) {
    threads[i]->join();
  }
  cout << "Server is: " << hostName << ":" << portNum << endl;
  cout << "Sent & received " << numMsgs << " msgs of avg. size " << avgSize << " with " << numThreads << " threads" << endl;
  cout << "Grand total: " << 2*numMsgs*(avgSize+sizeof(int))*numThreads << " bytes, or " << 2*numMsgs*(avgSize+sizeof(int))*numThreads*8 << " bits" << endl;
  return 0;
}