100% reproducible panic on wg-quick up wg0 unless pf is disabled

classic Classic list List threaded Threaded
5 messages Options
Reply | Threaded
Open this post in threaded view
|

100% reproducible panic on wg-quick up wg0 unless pf is disabled

Abel Abraham Camarillo Ojeda-2
>Synopsis: 100% reproducible panic on wg-quick up wg0 unless pf is disabled
>Category: amd64 kernel
>Environment:
System      : OpenBSD 6.7
Details     : OpenBSD 6.7-current (GENERIC) #330: Sun Jul 12 15:04:16 MDT
2020
[hidden email]:/usr/src/sys/arch/amd64/compile/GENERIC

Architecture: OpenBSD.amd64
Machine     : amd64
>Description:
machine panics as soon as I try to set a wireguard vpn against a ubuntu
18.04 LTS host
>How-To-Repeat:

(I can provide serial access to both machines in case its needed, but
both hosts are fresh installs + wireguard)

# cat > /etc/wireguard/wg0.conf
[Interface]
Address = 172.17.255.65/24
PrivateKey = xxxx

[Peer]
Publickey = xxx
AllowedIPs = 172.17.255.0/24
Endpoint = lkf-mongo.srv.verlet.org:45000
PersistentKeepalive = 25
^D
# wg-quick up wg0

(panic)
login: panic: kernel diagnostic assertion "m->m_pkthdr.pf.statekey == NULL"
failed: file "/usr/src/sys/net/pf.c", line 7455
Stopped at      db_enter+0x10:  popq    %rbp
   TID    PID    UID     PRFLAGS     PFLAGS  CPU  COMMAND
*143795  57046      0     0x14000      0x200    0  softnet
db_enter() at db_enter+0x10
panic(ffffffff81de2358) at panic+0x128
__assert(ffffffff81e4b90a,ffffffff81dbf1ea,1d1f,ffffffff81dda0a3) at
__assert+0
x2b
pf_test(2,1,ffff800000b06000,ffff80000fd67ee8) at pf_test+0x10d3
ip_input_if(ffff80000fd67ee8,ffff80000fd67ef4,4,0,ffff800000b06000) at
ip_input
_if+0x21c
ipv4_input(ffff800000b06000,fffffd8022e88f00) at ipv4_input+0x39
wg_deliver_in(ffff800000b4bcc0) at wg_deliver_in+0x122
taskq_thread(ffff80000002b080) at taskq_thread+0x6d
end trace frame: 0x0, count: 7
https://www.openbsd.org/ddb.html describes the minimum info required in bug
reports.  Insufficient info makes it difficult to find and fix bugs.
ddb> ps
PID     TID   PPID    UID  S       FLAGS  WAIT          COMMAND
64313  298349  29939   1000  3    0x1000b3  poll          ping
29939  207491  78041   1000  3    0x10008b  pause         ksh
40065  294858  39965      0  3    0x100083  netio         route
39965  383625  36980      0  3        0x81  wait          bash
36980  383852      1      0  3        0x81  piperd        bash
20112  494715      0      0  3     0x14200  bored         wg_crypt
64021  520594      0      0  3     0x14200  bored         wg_handshake
97182  322104      0      0  3     0x14200  bored         wg_handshake
15342   43565  59958      0  3    0x100083  ttyin         ksh
59958  366149  78041   1000  3    0x10008b  pause         ksh
78041  301999      1   1000  3    0x100080  kqread        tmux
65305   40237  21583   1000  3    0x100083  kqread        tmux
21583   80353  83375   1000  3    0x10008b  pause         ksh
83375   46912  88185   1000  3        0x90  select        sshd
88185  175068  21979      0  3        0x92  poll          sshd
77291  254480  92025      0  2         0x2                ld
92025  463410  87587      0  3    0x10008a  pause         sh
87587  153773  79407      0  3    0x10008a  pause         make
71758  245697      1      0  3    0x100083  ttyin         getty
79407  146691      1      0  3    0x10008b  pause         ksh
12242  231832      1      0  3    0x100098  poll          cron
74639  439451      1    799  3    0x100083  piperd        logger
71370  252357      1    799  3        0x83  thrsleep      prometheus
71370  251649      1    799  3   0x4000083  thrsleep      prometheus
71370  248926      1    799  3   0x4000083  thrsleep      prometheus
71370  149870      1    799  3   0x4000083  thrsleep      prometheus
71370   31938      1    799  3   0x4000083  thrsleep      prometheus
71370   72648      1    799  3   0x4000083  thrsleep      prometheus
71370  399570      1    799  3   0x4000083  thrsleep      prometheus
71370   80023      1    799  3   0x4000083  kqread        prometheus
 8600   96556  59675     83  3    0x100092  poll          ntpd
59675  478796  51352     83  3    0x100092  poll          ntpd
51352  136872      1      0  3    0x100080  poll          ntpd
11860  346331      1      0  3    0x200083  poll          python2.7
29334  404391      1      0  3    0x200083  poll          python2.7
57618  293489      1     99  3    0x100090  poll          sndiod
24972  419748      1    110  3    0x100090  poll          sndiod
84757   96029  94673     95  3    0x100092  kqread        smtpd
91831  306120  94673    103  3    0x100092  kqread        smtpd
62568  398870  94673     95  3    0x100092  kqread        smtpd
14093  363011  94673     95  3    0x100092  kqread        smtpd
43743  109460  94673     95  3    0x100092  kqread        smtpd
60150   90504  94673     95  3    0x100092  kqread        smtpd
94673  424303      1      0  3    0x100080  kqread        smtpd
21979  426534      1      0  3        0x80  select        sshd
 1833  235419  31696     74  3    0x100092  bpf           pflogd
31696   71235      1      0  3        0x80  netio         pflogd
65515  344509  69516     73  3    0x100090  kqread        syslogd
69516   92912      1      0  3    0x100082  netio         syslogd
 2542  450870      1     77  3    0x100090  poll          dhclient
30280  266106      1      0  3        0x80  poll          dhclient
31789  399932   5631    115  3    0x100092  kqread        slaacd
71049  223703   5631    115  3    0x100092  kqread        slaacd
 5631  133940      1      0  3    0x100080  kqread        slaacd
13348   37048      0      0  3     0x14200  bored         smr
 2173  179292      0      0  3     0x14200  pgzero        zerothread
72653  284344      0      0  3     0x14200  aiodoned      aiodoned
45404  332929      0      0  3     0x14200  syncer        update
49719   27730      0      0  3     0x14200  cleaner       cleaner
 3358  116913      0      0  3     0x14200  reaper        reaper
89672  315981      0      0  3     0x14200  pgdaemon      pagedaemon
86368   88441      0      0  3     0x14200  bored         crynlk
60092   77370      0      0  3     0x14200  bored         crypto
 1039  108860      0      0  3  0x40014200  acpi0         acpi0
*57046  143795      0      0  7     0x14200                softnet
41705  350350      0      0  3     0x14200  bored         systqmp
80965  321876      0      0  3     0x14200  bored         systq
62269  286795      0      0  3  0x40014200  bored         softclock
 4738  322079      0      0  3  0x40014200                idle0
    1  510750      0      0  3        0x82  wait          init
    0       0     -1      0  3     0x10200  scheduler     swapper

>Fix:
   pfctl -d;
   wg-quick up wg0;
   # ping via wg; works



/etc/pf.conf:
#       $OpenBSD: pf.conf,v 1.55 2017/12/03 20:40:04 sthen Exp $
#
# See pf.conf(5) and /etc/examples/pf.conf

set skip on lo

protectssh="keep state (max-src-conn-rate 2/30 overload <badhosts> flush
global)"

block return    # block stateless traffic
pass            # establish keep-state

block log from <badhosts>
pass in log on egress proto tcp to port { ssh } $protectssh
# By default, do not permit remote connections to X11
block return in on ! lo0 proto tcp to port 6000:6010

# Port build user does not need network
block return out log proto {tcp udp} user _pbuild


SENDBUG: dmesg, pcidump, acpidump and usbdevs are attached.
SENDBUG: Feel free to delete or use the -D flag if they contain sensitive
information.

dmesg:
OpenBSD 6.7-current (GENERIC) #330: Sun Jul 12 15:04:16 MDT 2020
    [hidden email]:/usr/src/sys/arch/amd64/compile/GENERIC
real mem = 627036160 (597MB)
avail mem = 593141760 (565MB)
random: good seed from bootblocks
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0: SMBIOS rev. 2.4 @ 0xf2ac0 (12 entries)
bios0: vendor Google version "Google" date 01/01/2011
bios0: Google Google Compute Engine
acpi0 at bios0: ACPI 2.0
acpi0: sleep states S3 S4 S5
acpi0: tables DSDT FACP SSDT APIC WAET SRAT
acpi0: wakeup devices
acpitimer0 at acpi0: 3579545 Hz, 24 bits
acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
cpu0 at mainbus0: apid 0 (boot processor)
cpu0: Intel(R) Xeon(R) CPU @ 2.30GHz, 2300.43 MHz, 06-3f-00
cpu0:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,SS,HTT,SSE3,PCLMUL,SSSE3,FMA3,CX16,PCID,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,HV,NXE,PAGE1GB,RDTSCP,LONG,LAHF,ABM,ITSC,FSGSBASE,TSC_ADJUST,BMI1,AVX2,SMEP,BMI2,ERMS,INVPCID,IBRS,IBPB,STIBP,SSBD,ARAT,XSAVEOPT,MELTDOWN
cpu0: 256KB 64b/line 8-way L2 cache
cpu0: smt 0, core 0, package 0
mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
cpu0: apic clock running at 2141MHz
ioapic0 at mainbus0: apid 0 pa 0xfec00000, version 11, 24 pins
acpiprt0 at acpi0: bus 0 (PCI0)
acpicpu0 at acpi0: C1(@1 halt!)
"ACPI0006" at acpi0 not configured
acpipci0 at acpi0 PCI0
extent `acpipci0 pcibus' (0x0 - 0xff), flags=0
extent `pciio' (0x0 - 0xffffffff), flags=0
     0x10000 - 0xffffffff
extent `pcimem' (0x0 - 0xffffffffffffffff), flags=0
     0x0 - 0x265fffff
     0xfffbc000 - 0xffffffff
     0x40000000000 - 0xffffffffffffffff
acpicmos0 at acpi0
"QEMU0001" at acpi0 not configured
"ACPI0007" at acpi0 not configured
cpu0: using Broadwell MDS workaround
pvbus0 at mainbus0: KVM
pvclock0 at pvbus0
pci0 at mainbus0 bus 0
pchb0 at pci0 dev 0 function 0 "Intel 82441FX" rev 0x02
pcib0 at pci0 dev 1 function 0 "Intel 82371AB PIIX4 ISA" rev 0x03
piixpm0 at pci0 dev 1 function 3 "Intel 82371AB Power" rev 0x03: SMBus
disabled
virtio0 at pci0 dev 3 function 0 "Qumranet Virtio SCSI" rev 0x00
vioscsi0 at virtio0: qsize 8192
scsibus1 at vioscsi0: 253 targets
sd0 at scsibus1 targ 1 lun 0: <Google, PersistentDisk, 1>
serial.Google_PersistentDisk_
sd0: 10240MB, 512 bytes/sector, 20971520 sectors, thin
virtio0: msix shared
virtio1 at pci0 dev 4 function 0 "Qumranet Virtio Network" rev 0x00
vio0 at virtio1: address 42:01:0a:80:00:02
virtio1: msix shared
isa0 at pcib0
isadma0 at isa0
com0 at isa0 port 0x3f8/8 irq 4: ns16550a, 16 byte fifo
com0: console
com1 at isa0 port 0x2f8/8 irq 3: ns16550a, 16 byte fifo
com2 at isa0 port 0x3e8/8 irq 5: ns16550a, 16 byte fifo
pckbc0 at isa0 port 0x60/5 irq 1 irq 12
pckbd0 at pckbc0 (kbd slot)
wskbd0 at pckbd0 mux 1
pms0 at pckbc0 (aux slot)
wsmouse0 at pms0 mux 0
pcppi0 at isa0 port 0x61
spkr0 at pcppi0
vscsi0 at root
scsibus2 at vscsi0: 256 targets
softraid0 at root
scsibus3 at softraid0: 256 targets
root on sd0a (ac423de5edd3cadd.a) swap on sd0b dump on sd0b
WARNING: / was not properly unmounted

usbdevs:
usbdevs: no USB controllers found

pcidump:
Domain /dev/pci0:
 0:0:0: Intel 82441FX
0x0000: Vendor ID: 8086, Product ID: 1237
0x0004: Command: 0103, Status: 0280
0x0008: Class: 06 Bridge, Subclass: 00 Host,
Interface: 00, Revision: 02
0x000c: BIST: 00, Header Type: 00, Latency Timer: 00,
Cache Line Size: 00
0x0010: BAR empty (00000000)
0x0014: BAR empty (00000000)
0x0018: BAR empty (00000000)
0x001c: BAR empty (00000000)
0x0020: BAR empty (00000000)
0x0024: BAR empty (00000000)
0x0028: Cardbus CIS: 00000000
0x002c: Subsystem Vendor ID: 1af4 Product ID: 1100
0x0030: Expansion ROM Base Address: 00000000
0x0038: 00000000
0x003c: Interrupt Pin: 00 Line: 00 Min Gnt: 00 Max Lat: 00
0x0000: 12378086 02800103 06000002 00000000
0x0010: 00000000 00000000 00000000 00000000
0x0020: 00000000 00000000 00000000 11001af4
0x0030: 00000000 00000000 00000000 00000000
0x0040: 00000000 00000000 00000000 00000000
0x0050: 00000000 00000000 33333000 33333333
0x0060: 00000000 00000000 00000000 00000000
0x0070: 00000000 00000000 00000000 00000000
0x0080: 00000000 00000000 00000000 00000000
0x0090: 00000000 00000000 00000000 00000000
0x00a0: 00000000 00000000 00000000 00000000
0x00b0: 00000000 00000000 00000000 00000000
0x00c0: 00000000 00000000 00000000 00000000
0x00d0: 00000000 00000000 00000000 00000000
0x00e0: 00000000 00000000 00000000 00000000
0x00f0: 00000000 00000000 00000000 00000000
 0:1:0: Intel 82371AB PIIX4 ISA
0x0000: Vendor ID: 8086, Product ID: 7110
0x0004: Command: 0103, Status: 0000
0x0008: Class: 06 Bridge, Subclass: 01 ISA,
Interface: 00, Revision: 03
0x000c: BIST: 00, Header Type: 80, Latency Timer: 00,
Cache Line Size: 00
0x0010: BAR empty (00000000)
0x0014: BAR empty (00000000)
0x0018: BAR empty (00000000)
0x001c: BAR empty (00000000)
0x0020: BAR empty (00000000)
0x0024: BAR empty (00000000)
0x0028: Cardbus CIS: 00000000
0x002c: Subsystem Vendor ID: 0000 Product ID: 0000
0x0030: Expansion ROM Base Address: 00000000
0x0038: 00000000
0x003c: Interrupt Pin: 00 Line: 00 Min Gnt: 00 Max Lat: 00
0x0000: 71108086 00000103 06010003 00800000
0x0010: 00000000 00000000 00000000 00000000
0x0020: 00000000 00000000 00000000 00000000
0x0030: 00000000 00000000 00000000 00000000
0x0040: 00000000 00000000 00000000 00030000
0x0050: 00000000 00000000 00000000 00000000
0x0060: 0b0b0a0a 00000000 00000000 00000000
0x0070: 00000000 00000000 00000000 00000000
0x0080: 00000000 00000000 00000000 00000000
0x0090: 00000000 00000000 00000000 00000000
0x00a0: 00000000 00000000 00000000 00000000
0x00b0: 00000000 00000000 00000000 00000000
0x00c0: 00000000 00000000 00000000 00000000
0x00d0: 00000000 00000000 00000000 00000000
0x00e0: 00000000 00000000 00000000 00000000
0x00f0: 00000000 00000000 00000000 00000000
 0:1:3: Intel 82371AB Power
0x0000: Vendor ID: 8086, Product ID: 7113
0x0004: Command: 0103, Status: 0000
0x0008: Class: 06 Bridge, Subclass: 80 Miscellaneous,
Interface: 00, Revision: 03
0x000c: BIST: 00, Header Type: 00, Latency Timer: 00,
Cache Line Size: 00
0x0010: BAR empty (00000000)
0x0014: BAR empty (00000000)
0x0018: BAR empty (00000000)
0x001c: BAR empty (00000000)
0x0020: BAR empty (00000000)
0x0024: BAR empty (00000000)
0x0028: Cardbus CIS: 00000000
0x002c: Subsystem Vendor ID: 0000 Product ID: 0000
0x0030: Expansion ROM Base Address: 00000000
0x0038: 00000000
0x003c: Interrupt Pin: 01 Line: 09 Min Gnt: 00 Max Lat: 00
0x0000: 71138086 00000103 06800003 00000000
0x0010: 00000000 00000000 00000000 00000000
0x0020: 00000000 00000000 00000000 00000000
0x0030: 00000000 00000000 00000000 00000109
0x0040: 0000b001 00000000 00000000 00000000
0x0050: 00000000 00000000 02000000 00000000
0x0060: 00000000 89000000 00000000 00000000
0x0070: 00000000 00000000 00000000 00000000
0x0080: 00000000 00000000 00000000 00000000
0x0090: 00000000 00000000 00000000 00000000
0x00a0: 00000000 00000000 00000000 00000000
0x00b0: 00000000 00000000 00000000 00000000
0x00c0: 00000000 00000000 00000000 00000000
0x00d0: 00000000 00000000 00000000 00000000
0x00e0: 00000000 00000000 00000000 00000000
0x00f0: 00000000 00000000 00000000 00000000
 0:3:0: Qumranet Virtio SCSI
0x0000: Vendor ID: 1af4, Product ID: 1004
0x0004: Command: 0107, Status: 0018
0x0008: Class: 00 Prehistoric, Subclass: 00 Miscellaneous,
Interface: 00, Revision: 00
0x000c: BIST: 00, Header Type: 00, Latency Timer: 00,
Cache Line Size: 00
0x0010: BAR io addr: 0x0000c000/0x0040
0x0014: BAR mem 32bit addr: 0xfebfe000/0x00000080
0x0018: BAR empty (00000000)
0x001c: BAR empty (00000000)
0x0020: BAR empty (00000000)
0x0024: BAR empty (00000000)
0x0028: Cardbus CIS: 00000000
0x002c: Subsystem Vendor ID: 1af4 Product ID: 0008
0x0030: Expansion ROM Base Address: 00000000
0x0038: 00000000
0x003c: Interrupt Pin: 01 Line: 0b Min Gnt: 00 Max Lat: 00
0x0080: Capability 0x11: Extended Message Signalled Interrupts (MSI-X)
Enabled: yes; table size 4 (BAR 1:8)
0x0000: 10041af4 00180107 00000000 00000000
0x0010: 0000c001 febfe000 00000000 00000000
0x0020: 00000000 00000000 00000000 00081af4
0x0030: 00000000 00000080 00000000 0000010b
0x0040: 00000000 00000000 00000000 00000000
0x0050: 00000000 00000000 00000000 00000000
0x0060: 00000000 00000000 00000000 00000000
0x0070: 00000000 00000000 00000000 00000000
0x0080: 80030011 00000009 00000001 00000000
0x0090: 00000000 00000000 00000000 00000000
0x00a0: 00000000 00000000 00000000 00000000
0x00b0: 00000000 00000000 00000000 00000000
0x00c0: 00000000 00000000 00000000 00000000
0x00d0: 00000000 00000000 00000000 00000000
0x00e0: 00000000 00000000 00000000 00000000
0x00f0: 00000000 00000000 00000000 00000000
 0:4:0: Qumranet Virtio Network
0x0000: Vendor ID: 1af4, Product ID: 1000
0x0004: Command: 0107, Status: 0010
0x0008: Class: 02 Network, Subclass: 00 Ethernet,
Interface: 00, Revision: 00
0x000c: BIST: 00, Header Type: 00, Latency Timer: 00,
Cache Line Size: 00
0x0010: BAR io addr: 0x0000c040/0x0040
0x0014: BAR mem 32bit addr: 0xfebff000/0x00000040
0x0018: BAR empty (00000000)
0x001c: BAR empty (00000000)
0x0020: BAR empty (00000000)
0x0024: BAR empty (00000000)
0x0028: Cardbus CIS: 00000000
0x002c: Subsystem Vendor ID: 1af4 Product ID: 0001
0x0030: Expansion ROM Base Address: 00000000
0x0038: 00000000
0x003c: Interrupt Pin: 01 Line: 0b Min Gnt: 00 Max Lat: 00
0x0080: Capability 0x11: Extended Message Signalled Interrupts (MSI-X)
Enabled: yes; table size 3 (BAR 1:8)
0x0000: 10001af4 00100107 02000000 00000000
0x0010: 0000c041 febff000 00000000 00000000
0x0020: 00000000 00000000 00000000 00011af4
0x0030: 00000000 00000080 00000000 0000010b
0x0040: 00000000 00000000 00000000 00000000
0x0050: 00000000 00000000 00000000 00000000
0x0060: 00000000 00000000 00000000 00000000
0x0070: 00000000 00000000 00000000 00000000
0x0080: 80020011 00000009 00000001 00000000
0x0090: 00000000 00000000 00000000 00000000
0x00a0: 00000000 00000000 00000000 00000000
0x00b0: 00000000 00000000 00000000 00000000
0x00c0: 00000000 00000000 00000000 00000000
0x00d0: 00000000 00000000 00000000 00000000
0x00e0: 00000000 00000000 00000000 00000000
0x00f0: 00000000 00000000 00000000 00000000

acpidump:
begin-base64 644 APIC.4
QVBJQ24AAAABOUdvb2dsZUdPT0dBUElDAQAAAEdPT0cBAAAAAADg/gEAAAAACAAAAQAAAAEMAAAA
AMD+AAAAAAIKAAUFAAAADQACCgAJCQAAAA0AAgoACgoAAAANAAIKAAsLAAAADQAEBv8AAAE=
====
begin-base64 644 DSDT.2
RFNEVLIXAAAB40dvb2dsZUdPT0dEU0RUAQAAAEdPT0cBAAAAoEIMABVQMFNfAQAVUDBFXwEAFVAx
Vl8BABVQMVNfAwAVUDFFXwMAFVAxTF8DABVQQ1MwAQAVUENFMAEAFVBDUzEBABVQQ0UxAQAVUENT
MgEAFVBDRTIBABVQQ1MzAQAVUENFMwEAFVBDUzQBABVQQ0U0AQAVUENTNQEAFVBDRTUBABVQQ1M2
AQAVUENFNgEAFVwvA19TQl9QQ0kwUENOVAgCFVwuX1NCX05URlkIAhVDUE9OBAAVXC5fU0JfTU5P
VAgCEEkEXABbgERCR18BCwIEAVuBC0RCR18BREJHQggULERCVUcBmGhglmBgdIdgAWFwAGKiEJVi
YXCDiGBiAERCR0J1YnAKCkRCR0IQIl9TQl9bghtQQ0kwCF9ISUQMQdAKAwhfQURSAAhfVUlEARBL
MC5fU0JfUENJMAhDUkVTEUIHCm6IDQACDAAAAAAA/wAAAAABRwH4DPgMAQiIDQABDAMAAAAA9wwA
APgMiA0AAQwDAAAADf//AAAA84cXAAAMAwAAAAAAAAoA//8LAAAAAAAAAAIAhxcAAAwBAAAAAAAA
AOD//7/+AAAAAAAAwB55AAhDUldNER8KHIcXAAAMAQAAAAAAAADg//+//gAAAAAAAMAeeQAIQ1I2
NBEzCjCKKwAADAMAAAAAAAAAAAAAAACAAAAA//////8AAAAAAAAAAAAAAAAAAACAAAAAeQAUSSJf
Q1JTAIpDUkVTClxQUzMyikNSRVMKYFBFMzKKQ1JFUwpoUEwzMqAskpNQQ1MwAHBQQ1MwUFMzMnBQ
Q0UwUEUzMnBydFBDRTBQQ1MwAAEAUEwzMqElcFAwU19QUzMycFAwRV9QRTMycHJ0UDBFX1AwU18A
AQBQTDMycENSRVNgikNSV00KClJTMzKKQ1JXTQoOUkUzMopDUldNChZSTDMyoDOSk1BDUzEAcFBD
UzFSUzMycFBDRTFSRTMycHJ0UENFMVBDUzEAAQBSTDMyhGBDUldNYKAzkpNQQ1MyAHBQQ1MyUlMz
MnBQQ0UyUkUzMnBydFBDRTJQQ1MyAAEAUkwzMoRgQ1JXTWCgM5KTUENTMwBwUENTM1JTMzJwUENF
M1JFMzJwcnRQQ0UzUENTMwABAFJMMzKEYENSV01goDOSk1BDUzQAcFBDUzRSUzMycFBDRTRSRTMy
cHJ0UENFNFBDUzQAAQBSTDMyhGBDUldNYKAzkpNQQ1M1AHBQQ1M1UlMzMnBQQ0U1UkUzMnBydFBD
RTVQQ1M1AAEAUkwzMoRgQ1JXTWCgM5KTUENTNgBwUENTNlJTMzJwUENFNlJFMzJwcnRQQ0U2UENT
NgABAFJMMzKEYENSV01goAmTUDFWXwCkYI9DUjY0Cg5QUzY0j0NSNjQKFlBFNjSPQ1I2NAomUEw2
NHBQMVNfUFM2NHBQMUVfUEU2NHBQMUxfUEw2NIRgQ1I2NGCkYBAFX1NCXxBABi5fU0JfUENJMFuC
QwVWR0FfCF9BRFIMAAACAFuAUENJQwIACgRbgQtQQ0lDA1ZFTkQgFAhfUzFEAKQAFAhfUzJEAKQA
FBlfUzNEAKAOk1ZFTkQMNhsAAaQKA6EDpAAQJS5fU0JfUENJMFuCGVBYMTMIX0FEUgwDAAEAW4BQ
MTNDAgAK/xBGBS5fU0JfUENJMFuCSQRJU0FfCF9BRFIMAAABAFuAUDQwQwIKYAoEW4EmXi5QWDEz
UDEzQwAASC8AB0xQRU4BADgAA0NBRU4BAANDQkVOAQhGREVOABBMMC8DX1NCX1BDSTBJU0FfW4BJ
U0FFAQsArQFbgStJU0FFQVNFUkEBU0VSQgFLQkRDAVdQRU4BU0VSQwFTRVJEAQABTEdDWQFbgi1S
VENfCF9ISUQMQdALAAhfQ1JTERgKFUcBcABwABACIgABRwFyAHIAAgZ5AFuCRwVLQkRfCF9ISUQM
QdADAxQoX1NUQQBwTEdDWWCgFpNgAHBLQkRDYaAGk2EApAChBKQKD6EEpAoPCF9DUlMRGAoVRwFg
AGAAAQFHAWQAZAABASICAHkAW4JHBE1PVV8IX0hJRAxB0A8TFChfU1RBAHBMR0NZYKAWk2AAcEtC
RENhoAaTYQCkAKEEpAoPoQSkCg8IX0NSUxEICgUiABB5AFuCGEZEQzAIX0hJRAxB0AcAFAhfU1RB
AKQAW4IYTFBUXwhfSElEDEHQBAAUCF9TVEEApABbgkQGQ09NMQhfSElEDEHQBQEIX1VJRAEUN19T
VEEAcExHQ1lgoBaTYABwU0VSQWGgBpNhAKQAoQSkCg+hE3BDQUVOYaAGk2EApAChBKQKDwhfQ1JT
ERAKDUcB+AP4AwAIIhAAeQBbgkUGQ09NMghfSElEDEHQBQEIX1VJRAoCFDdfU1RBAHBMR0NZYKAW
k2AAcFNFUkJhoAaTYQCkAKEEpAoPoRNwQ0JFTmGgBpNhAKQAoQSkCg8IX0NSUxEQCg1HAfgC+AIA
CCIIAHkAW4JPBENPTTMIX0hJRAxB0AUBCF9VSUQKAxQhX1NUQQBwTEdDWWCgEpNgAHBTRVJDYaAH
k2EBpAoPpAAIX0NSUxEQCg1HAegD6AMACCJAAHkAW4JPBENPTTQIX0hJRAxB0AUBCF9VSUQKBBQh
X1NUQQBwTEdDWWCgEpNgAHBTRVJEYaAHk2EBpAoPpAAIX0NSUxEQCg1HAegC6AIACCKAAHkAW4JO
BFBFTl8IX0hJRAxcI/AEFCdfU1RBAHBMR0NZYKAWk2AAcFdQRU5hoAaTYQCkAKEEpAoPoQOkAAhf
Q1JTERAKDUcBAAIIAgAIIoAAeQAQSQguX1NCX1BDSTBbgFBDU1QBCwCuCghbgRBQQ1NUQ1BDSVUg
UENJRCBbgFNFSl8BCwiuCgRbgQtTRUpfQ0IwRUogFA9QQ0VKAXB5AWgAQjBFShQ2UENORgBwAGCi
LJVgCh91YKARe1BDSVV5AWAAAFBDTlRgAaASe1BDSUR5AWAAAFBDTlRgCgMQSqBfU0JfEEd0UENJ
MAhfUFJUEktzgBILBAv//wBMTktEABILBAv//wFMTktBABIMBAv//woCTE5LQgASDAQL//8KA0xO
S0MAEg0EDP//AQAATE5LUwASDQQM//8BAAFMTktCABIOBAz//wEACgJMTktDABIOBAz//wEACgNM
TktEABINBAz//wIAAExOS0IAEg0EDP//AgABTE5LQwASDgQM//8CAAoCTE5LRAASDgQM//8CAAoD
TE5LQQASDQQM//8DAABMTktDABINBAz//wMAAUxOS0QAEg4EDP//AwAKAkxOS0EAEg4EDP//AwAK
A0xOS0IAEg0EDP//BAAATE5LRAASDQQM//8EAAFMTktBABIOBAz//wQACgJMTktCABIOBAz//wQA
CgNMTktDABINBAz//wUAAExOS0EAEg0EDP//BQABTE5LQgASDgQM//8FAAoCTE5LQwASDgQM//8F
AAoDTE5LRAASDQQM//8GAABMTktCABINBAz//wYAAUxOS0MAEg4EDP//BgAKAkxOS0QAEg4EDP//
BgAKA0xOS0EAEg0EDP//BwAATE5LQwASDQQM//8HAAFMTktEABIOBAz//wcACgJMTktBABIOBAz/
/wcACgNMTktCABINBAz//wgAAExOS0QAEg0EDP//CAABTE5LQQASDgQM//8IAAoCTE5LQgASDgQM
//8IAAoDTE5LQwASDQQM//8JAABMTktBABINBAz//wkAAUxOS0IAEg4EDP//CQAKAkxOS0MAEg4E
DP//CQAKA0xOS0QAEg0EDP//CgAATE5LQgASDQQM//8KAAFMTktDABIOBAz//woACgJMTktEABIO
BAz//woACgNMTktBABINBAz//wsAAExOS0MAEg0EDP//CwABTE5LRAASDgQM//8LAAoCTE5LQQAS
DgQM//8LAAoDTE5LQgASDQQM//8MAABMTktEABINBAz//wwAAUxOS0EAEg4EDP//DAAKAkxOS0IA
Eg4EDP//DAAKA0xOS0MAEg0EDP//DQAATE5LQQASDQQM//8NAAFMTktCABIOBAz//w0ACgJMTktD
ABIOBAz//w0ACgNMTktEABINBAz//w4AAExOS0IAEg0EDP//DgABTE5LQwASDgQM//8OAAoCTE5L
RAASDgQM//8OAAoDTE5LQQASDQQM//8PAABMTktDABINBAz//w8AAUxOS0QAEg4EDP//DwAKAkxO
S0EAEg4EDP//DwAKA0xOS0IAEg0EDP//EAAATE5LRAASDQQM//8QAAFMTktBABIOBAz//xAACgJM
TktCABIOBAz//xAACgNMTktDABINBAz//xEAAExOS0EAEg0EDP//EQABTE5LQgASDgQM//8RAAoC
TE5LQwASDgQM//8RAAoDTE5LRAASDQQM//8SAABMTktCABINBAz//xIAAUxOS0MAEg4EDP//EgAK
AkxOS0QAEg4EDP//EgAKA0xOS0EAEg0EDP//EwAATE5LQwASDQQM//8TAAFMTktEABIOBAz//xMA
CgJMTktBABIOBAz//xMACgNMTktCABINBAz//xQAAExOS0QAEg0EDP//FAABTE5LQQASDgQM//8U
AAoCTE5LQgASDgQM//8UAAoDTE5LQwASDQQM//8VAABMTktBABINBAz//xUAAUxOS0IAEg4EDP//
FQAKAkxOS0MAEg4EDP//FQAKA0xOS0QAEg0EDP//FgAATE5LQgASDQQM//8WAAFMTktDABIOBAz/
/xYACgJMTktEABIOBAz//xYACgNMTktBABINBAz//xcAAExOS0MAEg0EDP//FwABTE5LRAASDgQM
//8XAAoCTE5LQQASDgQM//8XAAoDTE5LQgASDQQM//8YAABMTktEABINBAz//xgAAUxOS0EAEg4E
DP//GAAKAkxOS0IAEg4EDP//GAAKA0xOS0MAEg0EDP//GQAATE5LQQASDQQM//8ZAAFMTktCABIO
BAz//xkACgJMTktDABIOBAz//xkACgNMTktEABINBAz//xoAAExOS0IAEg0EDP//GgABTE5LQwAS
DgQM//8aAAoCTE5LRAASDgQM//8aAAoDTE5LQQASDQQM//8bAABMTktDABINBAz//xsAAUxOS0QA
Eg4EDP//GwAKAkxOS0EAEg4EDP//GwAKA0xOS0IAEg0EDP//HAAATE5LRAASDQQM//8cAAFMTktB
ABIOBAz//xwACgJMTktCABIOBAz//xwACgNMTktDABINBAz//x0AAExOS0EAEg0EDP//HQABTE5L
QgASDgQM//8dAAoCTE5LQwASDgQM//8dAAoDTE5LRAASDQQM//8eAABMTktCABINBAz//x4AAUxO
S0MAEg4EDP//HgAKAkxOS0QAEg4EDP//HgAKA0xOS0EAEg0EDP//HwAATE5LQwASDQQM//8fAAFM
TktEABIOBAz//x8ACgJMTktBABIOBAz//x8ACgNMTktCAFuBJC8DUENJMElTQV9QNDBDAVBSUTAI
UFJRMQhQUlEyCFBSUTMIFBNJUVNUAaAJewqAaACkCgmkCgsUNklRQ1IJCFBSUjARDgoLiQYACQEA
AAAAeQCKUFJSMAoFUFJSSaALlWgKgHBoUFJSSaRQUlIwW4JMB0xOS0EIX0hJRAxB0AwPCF9VSUQA
CF9QUlMRFgoTiQ4ACQMFAAAACgAAAAsAAAB5ABQPX1NUQQCkSVFTVFBSUTAUEV9ESVMAfVBSUTAK
gFBSUTAUD19DUlMApElRQ1JQUlEwFBdfU1JTAYpoCgVQUlJJcFBSUklQUlEwW4JMB0xOS0IIX0hJ
RAxB0AwPCF9VSUQBCF9QUlMRFgoTiQ4ACQMFAAAACgAAAAsAAAB5ABQPX1NUQQCkSVFTVFBSUTEU
EV9ESVMAfVBSUTEKgFBSUTEUD19DUlMApElRQ1JQUlExFBdfU1JTAYpoCgVQUlJJcFBSUklQUlEx
W4JNB0xOS0MIX0hJRAxB0AwPCF9VSUQKAghfUFJTERYKE4kOAAkDBQAAAAoAAAALAAAAeQAUD19T
VEEApElRU1RQUlEyFBFfRElTAH1QUlEyCoBQUlEyFA9fQ1JTAKRJUUNSUFJRMhQXX1NSUwGKaAoF
UFJSSXBQUlJJUFJRMluCTQdMTktECF9ISUQMQdAMDwhfVUlECgMIX1BSUxEWChOJDgAJAwUAAAAK
AAAACwAAAHkAFA9fU1RBAKRJUVNUUFJRMxQRX0RJUwB9UFJRMwqAUFJRMxQPX0NSUwCkSVFDUlBS
UTMUF19TUlMBimgKBVBSUklwUFJSSVBSUTNbgk8ETE5LUwhfSElEDEHQDA8IX1VJRAoECF9QUlMR
DgoLiQYACQEJAAAAeQAUCV9TVEEApAoLFAZfRElTABQLX0NSUwCkX1BSUxQGX1NSUwEQRxJfU0Jf
FDVDUE1BAXCDiENQT05oAGBwEQsKCAAIAAAAAAAAYXBoiGEKAgBwaIhhCgMAcGCIYQoEAKRhFBpD
UFNUAXCDiENQT05oAGCgBWCkCg+hA6QAFEoEQ1BFSgJEQlVHDUNQRUoAREJVR2hEQlVHaaAqk2kB
cBEDCiBgcHpoCgMAYXB7aAoHAGJweQFiAGNwY4hgYQBwYFBSU19bIgrIW4BQUlNUAQsArwqAW4EM
UFJTVAFQUlNfQEAUSgZQUlNDAHBQUlNfZXAAYnAAYKJGBZVgh0NQT05wg4hDUE9OYABhoAp7YAoH
AHpiAWKhDHCDiGV6YAoDAABicHtiAQBjoCKSk2FjcGOIQ1BPTmAAoAqTYwFOVEZZYAGhCE5URllg
CgN1YBBKL19TQl9bAU1NVFgAW4BNT1JfAQuArwqAW4E9TU9SXwNNSFBFIE1DTlQgTVNFTCBNRkxH
IE1CTF8gTUJIXyBNTExfIE1MSF8gTVBESSBNT1NFIE1PU0MgFDpNU1RBAXAAYFsjTU1UWP//cGhN
U0VMoBqQkpN7TUZMRwEAAJN7TUZMRwoEAABwCg9gWydNTVRYpGAUTBNNQ1JTAXARMwowiisAAAwD
AAAAAAAAAAAAAAAAAAAAAP7/////////AAAAAAAAAAD//////////3kAYIpgCg5NSU5MimAKEk1J
TkiKYAoWTUFYTIpgChpNQVhIimAKJkxFTkyKYAoqTEVOSFsjTU1UWP//cGhNU0VMcE1CTF9NSU5M
cE1CSF9NSU5IcE1MTF9MRU5McE1MSF9MRU5Ick1JTkxMRU5MTUFYTHJNSU5ITEVOSE1BWEigH32V
TUFYTE1JTkyVTUFYTExFTkwAck1BWEgBTUFYSHRNQVhMAU1BWEygEZNNQVhM/3RNQVhIAU1BWEhE
QlVHDU1DUlMAREJVR2hEQlVHTUlOTERCVUdNSU5IREJVR01BWExEQlVHTUFYSERCVUdMRU5MREJV
R0xFTkhbJ01NVFikYBQ2TVBYTQFbI01NVFj//3BoTVNFTHBNUERJYERCVUcNTVBYTQBEQlVHaERC
VUdgWydNTVRYpGAURQRNT1NUBFsjTU1UWP//cGhNU0VMcGlNT1NFcGpNT1NDREJVRw1NT1NUAERC
VUdoREJVR2lEQlVHakRCVUdrWydNTVRYFD5NRUowAlsjTU1UWP//REJVRw1NRUowAERCVUdoREJV
R2mgEZNpAXBoTVNFTHAKBE1GTEdbJ01NVFhbIgrIFE0GTVNDQQBbI01NVFj//0RCVUcNTVNDQQBw
TUNOVGBwAGGiRASVYWBwYU1TRUxwTUZMR2KgLpKTe2IKAgAAREJVRw1NU0NBIC0tPiBNTk9UAERC
VUdhTU5PVGEBcAoCTUZMR3JhAWFbJ01NVFgQSApfR1BFCF9ISUQNQUNQSTAwMDYAFAZfTDAwABQV
X0UwMQBcLwNfU0JfUENJMFBDTkYUEF9FMDIAXC5fU0JfUFJTQxQQX0UwMwBcLl9TQl9NU0NBFAZf
TDA0ABQGX0wwNQAUBl9MMDYAFAZfTDA3ABQGX0wwOAAUBl9MMDkAFAZfTDBBABQGX0wwQgAUBl9M
MEMAFAZfTDBEABQGX0wwRQAUBl9MMEYA
====
begin-base64 644 FACP.1
RkFDUPQAAAACOkdvb2dsZUdPT0dGQUNQAQAAAEdPT0cBAAAAwP5fJlDcXyYBAAkAsgAAAPHwAAAA
sAAAAAAAAASwAAAAAAAAAAAAAAiwAADgrwAAAAAAAAQCAAQEAAAA/w//DwAAAAAAAAAAAAAAAIWE
AAABCAAA+QwAAAAAAAAGAAAAwP5fJgAAAAAAAAAAAAAAAAEgAAAAsAAAAAAAAAAAAAAAAAAAAAAA
AAEQAAAEsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEgAAAIsAAAAAAAAAEgAADgrwAA
AAAAAAAAAAAAAAAAAAAAAA==
====
begin-base64 644 RSDT.0
UlNEVDgAAAABXEdvb2dsZUdPT0dSU0RUAQAAAEdPT0cBAAAAAP9fJvD1XyYA9V8m0PRfJhD0XyY=
====
begin-base64 644 SRAT.6
U1JBVLgAAAAB/Udvb2dsZUdPT0dTUkFUAQAAAEdPT0cBAAAAAQAAAAAAAAAAAAAAABAAAAEAAAAA
AAAAAAAAAAEoAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAABAAAAAAAAAAAAAAABKAAAAAAAAAAA
EAAAAAAAAABQJgAAAAAAAAAAAQAAAAAAAAAAAAAAASgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAA==
====
begin-base64 644 SSDT.3
U1NEVM8IAAABGkdvb2dsZUdPT0dTU0RUAQAAAEdPT0cBAAAAoCEAFVwuX1NCX1BDSTAGABVcLwNf
U0JfUENJMElTQV8GABBODVwACFAwU18MAAAAgAhQMEVfDP//v/4IUENTMAwAAAAACFBDRTAMAAAA
AAhQQ1MxDAAAAAAIUENFMQwAAAAACFBDUzIMAAAAAAhQQ0UyDAAAAAAIUENTMwwAAAAACFBDRTMM
AAAAAAhQQ1M0DAAAAAAIUENFNAwAAAAACFBDUzUMAAAAAAhQQ0U1DAAAAAAIUENTNgwAAAAACFBD
RTYMAAAAAAhQMVZfCgAIUDFTXxELCggAAAAAAAAAAAhQMUVfEQsKCAAAAAAAAAAACFAxTF8RCwoI
AAAAAAAAAAAQKVwACF9TM18SBgQBAQAACF9TNF8SCAQKAAoAAAAIX1M1XxIGBAAAAAAQQAxcLwNf
U0JfUENJMElTQV9bgk0KUEVWVAhfSElEDVFFTVUwMDAxAAhQRVNUCwUFW4BQRU9SAVBFU1QBW4EL
UEVPUgFQRVBUCBQYX1NUQQBwUEVTVGCgBpNgAKQAoQSkCg8UDlJEUFQAcFBFUFRgpGAUDFdSUFQB
cGhQRVBUCF9DUlMRDQoKRwEAAAAAAQF5AItfQ1JTCgJJT01Oi19DUlMKBElPTVgUGF9JTkkAcFBF
U1RJT01OcFBFU1RJT01YEI5rAF9TQl9bg0IFQ1AwMAAQsAAABghJRF9fCgAIX0hJRA1BQ1BJMDAw
NwAUD19NQVQApENQTUFJRF9fFA9fU1RBAKRDUFNUSURfXxQPX0VKMAFDUEVKSURfX2gUQwFOVEZZ
AqALk2gKAIZDUDAwaQhDUE9OEkQAAQEURwBNTk9UAhCMYwBQQ0kwW4IlUzAxXwhfU1VOCgEIX0FE
UgwAAAEAFA5FSjBfAVBDRUpfU1VOW4IlUzAyXwhfU1VOCgIIX0FEUgwAAAIAFA5FSjBfAVBDRUpf
U1VOW4IlUzAzXwhfU1VOCgMIX0FEUgwAAAMAFA5FSjBfAVBDRUpfU1VOW4IlUzA0XwhfU1VOCgQI
X0FEUgwAAAQAFA5FSjBfAVBDRUpfU1VOW4IlUzA1XwhfU1VOCgUIX0FEUgwAAAUAFA5FSjBfAVBD
RUpfU1VOW4IlUzA2XwhfU1VOCgYIX0FEUgwAAAYAFA5FSjBfAVBDRUpfU1VOW4IlUzA3XwhfU1VO
CgcIX0FEUgwAAAcAFA5FSjBfAVBDRUpfU1VOW4IlUzA4XwhfU1VOCggIX0FEUgwAAAgAFA5FSjBf
AVBDRUpfU1VOW4IlUzA5XwhfU1VOCgkIX0FEUgwAAAkAFA5FSjBfAVBDRUpfU1VOW4IlUzBBXwhf
U1VOCgoIX0FEUgwAAAoAFA5FSjBfAVBDRUpfU1VOW4IlUzBCXwhfU1VOCgsIX0FEUgwAAAsAFA5F
SjBfAVBDRUpfU1VOW4IlUzBDXwhfU1VOCgwIX0FEUgwAAAwAFA5FSjBfAVBDRUpfU1VOW4IlUzBE
XwhfU1VOCg0IX0FEUgwAAA0AFA5FSjBfAVBDRUpfU1VOW4IlUzBFXwhfU1VOCg4IX0FEUgwAAA4A
FA5FSjBfAVBDRUpfU1VOW4IlUzBGXwhfU1VOCg8IX0FEUgwAAA8AFA5FSjBfAVBDRUpfU1VOW4Il
UzEwXwhfU1VOChAIX0FEUgwAABAAFA5FSjBfAVBDRUpfU1VOW4IlUzExXwhfU1VOChEIX0FEUgwA
ABEAFA5FSjBfAVBDRUpfU1VOW4IlUzEyXwhfU1VOChIIX0FEUgwAABIAFA5FSjBfAVBDRUpfU1VO
W4IlUzEzXwhfU1VOChMIX0FEUgwAABMAFA5FSjBfAVBDRUpfU1VOW4IlUzE0XwhfU1VOChQIX0FE
UgwAABQAFA5FSjBfAVBDRUpfU1VOW4IlUzE1XwhfU1VOChUIX0FEUgwAABUAFA5FSjBfAVBDRUpf
U1VOW4IlUzE2XwhfU1VOChYIX0FEUgwAABYAFA5FSjBfAVBDRUpfU1VOW4IlUzE3XwhfU1VOChcI
X0FEUgwAABcAFA5FSjBfAVBDRUpfU1VOW4IlUzE4XwhfU1VOChgIX0FEUgwAABgAFA5FSjBfAVBD
RUpfU1VOW4IlUzE5XwhfU1VOChkIX0FEUgwAABkAFA5FSjBfAVBDRUpfU1VOW4IlUzFBXwhfU1VO
ChoIX0FEUgwAABoAFA5FSjBfAVBDRUpfU1VOW4IlUzFCXwhfU1VOChsIX0FEUgwAABsAFA5FSjBf
AVBDRUpfU1VOW4IlUzFDXwhfU1VOChwIX0FEUgwAABwAFA5FSjBfAVBDRUpfU1VOW4IlUzFEXwhf
U1VOCh0IX0FEUgwAAB0AFA5FSjBfAVBDRUpfU1VOW4IlUzFFXwhfU1VOCh4IX0FEUgwAAB4AFA5F
SjBfAVBDRUpfU1VOW4IlUzFGXwhfU1VOCh8IX0FEUgwAAB8AFA5FSjBfAVBDRUpfU1VOFEsXUENO
VAKgC5NoCgGGUzAxX2mgC5NoCgKGUzAyX2mgC5NoCgOGUzAzX2mgC5NoCgSGUzA0X2mgC5NoCgWG
UzA1X2mgC5NoCgaGUzA2X2mgC5NoCgeGUzA3X2mgC5NoCgiGUzA4X2mgC5NoCgmGUzA5X2mgC5No
CgqGUzBBX2mgC5NoCguGUzBCX2mgC5NoCgyGUzBDX2mgC5NoCg2GUzBEX2mgC5NoCg6GUzBFX2mg
C5NoCg+GUzBGX2mgC5NoChCGUzEwX2mgC5NoChGGUzExX2mgC5NoChKGUzEyX2mgC5NoChOGUzEz
X2mgC5NoChSGUzE0X2mgC5NoChWGUzE1X2mgC5NoChaGUzE2X2mgC5NoCheGUzE3X2mgC5NoChiG
UzE4X2mgC5NoChmGUzE5X2mgC5NoChqGUzFBX2mgC5NoChuGUzFCX2mgC5NoChyGUzFDX2mgC5No
Ch2GUzFEX2mgC5NoCh6GUzFFX2mgC5NoCh+GUzFGX2k=
====
begin-base64 644 WAET.5
V0FFVCgAAAABvEdvb2dsZUdPT0dXQUVUAQAAAEdPT0cBAAAAAgAAAA==
====
begin-base64 644 headers
ClJTRCBQVFI6IENoZWNrc3VtPTE5LCBPRU1JRD1Hb29nbGUsIFJldmlzaW9uPTAsIFJzZHRBZGRy
ZXNzPTB4MjY1ZmRjMTAKCgpSU0RUOiBMZW5ndGg9NTYsIFJldmlzaW9uPTEsIENoZWNrc3VtPTky
LAoJT0VNSUQ9R29vZ2xlLCBPRU0gVGFibGUgSUQ9R09PR1JTRFQsIE9FTSBSZXZpc2lvbj0weDEs
CglDcmVhdG9yIElEPUdPT0csIENyZWF0b3IgUmV2aXNpb249MHgxCgoKCUVudHJpZXM9eyAweDI2
NWZmZjAwLCAweDI2NWZmNWYwLCAweDI2NWZmNTAwLCAweDI2NWZmNGQwLCAweDI2NWZmNDEwIH0K
CgoJRFNEVD0weDI2NWZkYzUwCglJTlRfTU9ERUw9QVBJQwoJU0NJX0lOVD05CglTTUlfQ01EPTB4
YjIsIEFDUElfRU5BQkxFPTB4ZjEsIEFDUElfRElTQUJMRT0weGYwLCBTNEJJT1NfUkVRPTB4MAoJ
UE0xYV9FVlRfQkxLPTB4YjAwMC0weGIwMDMKCVBNMWFfQ05UX0JMSz0weGIwMDQtMHhiMDA1CglQ
TTJfVE1SX0JMSz0weGIwMDgtMHhiMDBiCglQTTJfR1BFMF9CTEs9MHhhZmUwLTB4YWZlMwoJUF9M
VkwyX0xBVD00MDk1bXMsIFBfTFZMM19MQVQ9NDA5NW1zCglGTFVTSF9TSVpFPTAsIEZMVVNIX1NU
UklERT0wCglEVVRZX09GRlNFVD0wLCBEVVRZX1dJRFRIPTAKCURBWV9BTFJNPTAsIE1PTl9BTFJN
PTAsIENFTlRVUlk9MAoJRmxhZ3M9e1dCSU5WRCxQUk9DX0MxLFJUQ19TNH0KCgpEU0RUOiBMZW5n
dGg9NjA2NiwgUmV2aXNpb249MSwgQ2hlY2tzdW09MjI3LAoJT0VNSUQ9R29vZ2xlLCBPRU0gVGFi
bGUgSUQ9R09PR0RTRFQsIE9FTSBSZXZpc2lvbj0weDEsCglDcmVhdG9yIElEPUdPT0csIENyZWF0
b3IgUmV2aXNpb249MHgxCgoKU1NEVDogTGVuZ3RoPTIyNTUsIFJldmlzaW9uPTEsIENoZWNrc3Vt
PTI2LAoJT0VNSUQ9R29vZ2xlLCBPRU0gVGFibGUgSUQ9R09PR1NTRFQsIE9FTSBSZXZpc2lvbj0w
eDEsCglDcmVhdG9yIElEPUdPT0csIENyZWF0b3IgUmV2aXNpb249MHgxCgoKQVBJQzogTGVuZ3Ro
PTExMCwgUmV2aXNpb249MSwgQ2hlY2tzdW09NTcsCglPRU1JRD1Hb29nbGUsIE9FTSBUYWJsZSBJ
RD1HT09HQVBJQywgT0VNIFJldmlzaW9uPTB4MSwKCUNyZWF0b3IgSUQ9R09PRywgQ3JlYXRvciBS
ZXZpc2lvbj0weDEKCgpXQUVUOiBMZW5ndGg9NDAsIFJldmlzaW9uPTEsIENoZWNrc3VtPTE4OCwK
CU9FTUlEPUdvb2dsZSwgT0VNIFRhYmxlIElEPUdPT0dXQUVULCBPRU0gUmV2aXNpb249MHgxLAoJ
Q3JlYXRvciBJRD1HT09HLCBDcmVhdG9yIFJldmlzaW9uPTB4MQoKClNSQVQ6IExlbmd0aD0xODQs
IFJldmlzaW9uPTEsIENoZWNrc3VtPTI1MywKCU9FTUlEPUdvb2dsZSwgT0VNIFRhYmxlIElEPUdP
T0dTUkFULCBPRU0gUmV2aXNpb249MHgxLAoJQ3JlYXRvciBJRD1HT09HLCBDcmVhdG9yIFJldmlz
aW9uPTB4MQoK
====
Reply | Threaded
Open this post in threaded view
|

Re: 100% reproducible panic on wg-quick up wg0 unless pf is disabled

Theo Buehler-3
> login: panic: kernel diagnostic assertion "m->m_pkthdr.pf.statekey == NULL"
> failed: file "/usr/src/sys/net/pf.c", line 7455

Ran into this as well. Should be fixed in the next snap.
https://marc.info/?l=openbsd-tech&m=159462210226639&w=2

Index: if_wg.c
===================================================================
RCS file: /var/cvs/src/sys/net/if_wg.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -p -r1.10 -r1.11
--- if_wg.c 12 Jul 2020 18:54:23 -0000 1.10
+++ if_wg.c 13 Jul 2020 08:29:34 -0000 1.11
@@ -1,4 +1,4 @@
-/* $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
+/* $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
 
 /*
  * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All Rights Reserved.
@@ -18,6 +18,7 @@
  */
 
 #include "bpfilter.h"
+#include "pf.h"
 
 #include <sys/types.h>
 #include <sys/systm.h>

Reply | Threaded
Open this post in threaded view
|

Re: 100% reproducible panic on wg-quick up wg0 unless pf is disabled

Abel Abraham Camarillo Ojeda-2
On Mon, Jul 13, 2020 at 4:48 AM Theo Buehler <[hidden email]> wrote:

> > login: panic: kernel diagnostic assertion "m->m_pkthdr.pf.statekey ==
> NULL"
> > failed: file "/usr/src/sys/net/pf.c", line 7455
>
> Ran into this as well. Should be fixed in the next snap.
> https://marc.info/?l=openbsd-tech&m=159462210226639&w=2


thank you !


>
> Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>
>
Reply | Threaded
Open this post in threaded view
|

Re: 100% reproducible panic on wg-quick up wg0 unless pf is disabled

Alexandr Nedvedicky
In reply to this post by Abel Abraham Camarillo Ojeda-2
Hello Abel,

On Mon, Jul 13, 2020 at 04:42:23AM -0500, Abel Abraham Camarillo Ojeda wrote:
</snip>

> # wg-quick up wg0
>
> (panic)
> login: panic: kernel diagnostic assertion "m->m_pkthdr.pf.statekey == NULL"
> failed: file "/usr/src/sys/net/pf.c", line 7455
> Stopped at      db_enter+0x10:  popq    %rbp
>    TID    PID    UID     PRFLAGS     PFLAGS  CPU  COMMAND
> *143795  57046      0     0x14000      0x200    0  softnet
> db_enter() at db_enter+0x10
> panic(ffffffff81de2358) at panic+0x128
> __assert(ffffffff81e4b90a,ffffffff81dbf1ea,1d1f,ffffffff81dda0a3) at
> __assert+0
> x2b
> pf_test(2,1,ffff800000b06000,ffff80000fd67ee8) at pf_test+0x10d3
> ip_input_if(ffff80000fd67ee8,ffff80000fd67ef4,4,0,ffff800000b06000) at
> ip_input
> _if+0x21c
> ipv4_input(ffff800000b06000,fffffd8022e88f00) at ipv4_input+0x39
> wg_deliver_in(ffff800000b4bcc0) at wg_deliver_in+0x122
> taskq_thread(ffff80000002b080) at taskq_thread+0x6d
> end trace frame: 0x0, count: 7

</snip>

    thanks for reporting. and sorry for inconveniences.

I wonder if diff below fixes the assertion panic?

I suspect wg forgot to tell the IP address of decrypted packet got changed.  I
have no wireguard by hand ready for try that out so the diff is kind of
shooting in-the-dark.

thanks for testing.

regards
sashan

--------8<---------------8<---------------8<------------------8<--------
diff --git a/sys/net/if_wg.c b/sys/net/if_wg.c
index 3f59681fed8..2745d20ff6c 100644
--- a/sys/net/if_wg.c
+++ b/sys/net/if_wg.c
@@ -18,6 +18,7 @@
  */
 
 #include "bpfilter.h"
+#include "pf.h"
 
 #include <sys/types.h>
 #include <sys/systm.h>
@@ -2050,6 +2051,10 @@ wg_input(void *_sc, struct mbuf *m, struct ip *ip, struct ip6_hdr *ip6,
  t->t_mbuf = NULL;
  t->t_done = 0;
 
+#if NPF > 0
+ pf_pkt_addr_changed(m);
+#endif /* NPF */
+
  if (wg_queue_in(sc, t->t_peer, m) != 0)
  counters_inc(sc->sc_if.if_counters,
     ifc_iqdrops);

Reply | Threaded
Open this post in threaded view
|

Re: 100% reproducible panic on wg-quick up wg0 unless pf is disabled

Abel Abraham Camarillo Ojeda-2
In reply to this post by Theo Buehler-3
I confirm this commit, fixes wg for me.

Thanks to all!

----------------------------
revision 1.11
date: 2020/07/13 08:29:34;  author: tb;  state: Exp;  lines: +2 -1;
 commitid: Kt4hgscFLJKqzC1v;
Unbreak wg(4).

Previous may have fixed the build without pf(4), but broke wireguard in
normal kernels: the condition NPF > 0 is false if pf.h is not in scope.


On Mon, Jul 13, 2020 at 4:48 AM Theo Buehler <[hidden email]> wrote:

> > login: panic: kernel diagnostic assertion "m->m_pkthdr.pf.statekey ==
> NULL"
> > failed: file "/usr/src/sys/net/pf.c", line 7455
>
> Ran into this as well. Should be fixed in the next snap.
> https://marc.info/?l=openbsd-tech&m=159462210226639&w=2
>
> Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
> Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */Index: if_wg.c
> ===================================================================
> RCS file: /var/cvs/src/sys/net/if_wg.c,v
> retrieving revision 1.10
> retrieving revision 1.11
> diff -u -p -r1.10 -r1.11
> --- if_wg.c     12 Jul 2020 18:54:23 -0000      1.10
> +++ if_wg.c     13 Jul 2020 08:29:34 -0000      1.11
> @@ -1,4 +1,4 @@
> -/*     $OpenBSD: if_wg.c,v 1.10 2020/07/12 18:54:23 kn Exp $ */
> +/*     $OpenBSD: if_wg.c,v 1.11 2020/07/13 08:29:34 tb Exp $ */
>
>  /*
>   * Copyright (C) 2015-2020 Jason A. Donenfeld <[hidden email]>. All
> Rights Reserved.
> @@ -18,6 +18,7 @@
>   */
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>
>
>  #include "bpfilter.h"
> +#include "pf.h"
>
>  #include <sys/types.h>
>  #include <sys/systm.h>
>