Skip to content

Commit 0a14842

Browse files
Eric Dumazetdavem330
Eric Dumazet
authored and
davem330
committed
net: filter: Just In Time compiler for x86-64
In order to speedup packet filtering, here is an implementation of a JIT compiler for x86_64 It is disabled by default, and must be enabled by the admin. echo 1 >/proc/sys/net/core/bpf_jit_enable It uses module_alloc() and module_free() to get memory in the 2GB text kernel range since we call helpers functions from the generated code. EAX : BPF A accumulator EBX : BPF X accumulator RDI : pointer to skb (first argument given to JIT function) RBP : frame pointer (even if CONFIG_FRAME_POINTER=n) r9d : skb->len - skb->data_len (headlen) r8 : skb->data To get a trace of generated code, use : echo 2 >/proc/sys/net/core/bpf_jit_enable Example of generated code : # tcpdump -p -n -s 0 -i eth1 host 192.168.20.0/24 flen=18 proglen=147 pass=3 image=ffffffffa00b5000 JIT code: ffffffffa00b5000: 55 48 89 e5 48 83 ec 60 48 89 5d f8 44 8b 4f 60 JIT code: ffffffffa00b5010: 44 2b 4f 64 4c 8b 87 b8 00 00 00 be 0c 00 00 00 JIT code: ffffffffa00b5020: e8 24 7b f7 e0 3d 00 08 00 00 75 28 be 1a 00 00 JIT code: ffffffffa00b5030: 00 e8 fe 7a f7 e0 24 00 3d 00 14 a8 c0 74 49 be JIT code: ffffffffa00b5040: 1e 00 00 00 e8 eb 7a f7 e0 24 00 3d 00 14 a8 c0 JIT code: ffffffffa00b5050: 74 36 eb 3b 3d 06 08 00 00 74 07 3d 35 80 00 00 JIT code: ffffffffa00b5060: 75 2d be 1c 00 00 00 e8 c8 7a f7 e0 24 00 3d 00 JIT code: ffffffffa00b5070: 14 a8 c0 74 13 be 26 00 00 00 e8 b5 7a f7 e0 24 JIT code: ffffffffa00b5080: 00 3d 00 14 a8 c0 75 07 b8 ff ff 00 00 eb 02 31 JIT code: ffffffffa00b5090: c0 c9 c3 BPF program is 144 bytes long, so native program is almost same size ;) (000) ldh [12] (001) jeq #0x800 jt 2 jf 8 (002) ld [26] (003) and #0xffffff00 (004) jeq #0xc0a81400 jt 16 jf 5 (005) ld [30] (006) and #0xffffff00 (007) jeq #0xc0a81400 jt 16 jf 17 (008) jeq #0x806 jt 10 jf 9 (009) jeq #0x8035 jt 10 jf 17 (010) ld [28] (011) and #0xffffff00 (012) jeq #0xc0a81400 jt 16 jf 13 (013) ld [38] (014) and #0xffffff00 (015) jeq #0xc0a81400 jt 16 jf 17 (016) ret #65535 (017) ret #0 Signed-off-by: Eric Dumazet <[email protected]> Cc: Arnaldo Carvalho de Melo <[email protected]> Cc: Ben Hutchings <[email protected]> Cc: Hagen Paul Pfeifer <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent b678027 commit 0a14842

File tree

14 files changed

+918
-62
lines changed

14 files changed

+918
-62
lines changed

‎Documentation/sysctl/net.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,17 @@ Table : Subdirectories in /proc/sys/net
3232
1. /proc/sys/net/core - Network core options
3333
-------------------------------------------------------
3434

35+
bpf_jit_enable
36+
--------------
37+
38+
This enables Berkeley Packet Filter Just in Time compiler.
39+
Currently supported on x86_64 architecture, bpf_jit provides a framework
40+
to speed packet filtering, the one used by tcpdump/libpcap for example.
41+
Values :
42+
0 - disable the JIT (default value)
43+
1 - enable the JIT
44+
2 - enable the JIT and ask the compiler to emit traces on kernel log.
45+
3546
rmem_default
3647
------------
3748

‎MAINTAINERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4372,6 +4372,7 @@ S: Maintained
43724372
F: net/ipv4/
43734373
F: net/ipv6/
43744374
F: include/net/ip*
4375+
F: arch/x86/net/*
43754376

43764377
NETWORKING [LABELED] (NetLabel, CIPSO, Labeled IPsec, SECMARK)
43774378
M: Paul Moore <[email protected]>

‎arch/x86/Kbuild

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ obj-y += vdso/
1515
obj-$(CONFIG_IA32_EMULATION) += ia32/
1616

1717
obj-y += platform/
18+
obj-y += net/

‎arch/x86/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ config X86
7272
select IRQ_FORCED_THREADING
7373
select USE_GENERIC_SMP_HELPERS if SMP
7474
select ARCH_NO_SYSDEV_OPS
75+
select HAVE_BPF_JIT if X86_64
7576

7677
config INSTRUCTION_DECODER
7778
def_bool (KPROBES || PERF_EVENTS)

‎arch/x86/net/Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#
2+
# Arch-specific network modules
3+
#
4+
obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o

‎arch/x86/net/bpf_jit.S

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/* bpf_jit.S : BPF JIT helper functions
2+
*
3+
* Copyright (C) 2011 Eric Dumazet ([email protected])
4+
*
5+
* This program is free software; you can redistribute it and/or
6+
* modify it under the terms of the GNU General Public License
7+
* as published by the Free Software Foundation; version 2
8+
* of the License.
9+
*/
10+
#include <linux/linkage.h>
11+
#include <asm/dwarf2.h>
12+
13+
/*
14+
* Calling convention :
15+
* rdi : skb pointer
16+
* esi : offset of byte(s) to fetch in skb (can be scratched)
17+
* r8 : copy of skb->data
18+
* r9d : hlen = skb->len - skb->data_len
19+
*/
20+
#define SKBDATA %r8
21+
22+
sk_load_word_ind:
23+
.globl sk_load_word_ind
24+
25+
add %ebx,%esi /* offset += X */
26+
# test %esi,%esi /* if (offset < 0) goto bpf_error; */
27+
js bpf_error
28+
29+
sk_load_word:
30+
.globl sk_load_word
31+
32+
mov %r9d,%eax # hlen
33+
sub %esi,%eax # hlen - offset
34+
cmp $3,%eax
35+
jle bpf_slow_path_word
36+
mov (SKBDATA,%rsi),%eax
37+
bswap %eax /* ntohl() */
38+
ret
39+
40+
41+
sk_load_half_ind:
42+
.globl sk_load_half_ind
43+
44+
add %ebx,%esi /* offset += X */
45+
js bpf_error
46+
47+
sk_load_half:
48+
.globl sk_load_half
49+
50+
mov %r9d,%eax
51+
sub %esi,%eax # hlen - offset
52+
cmp $1,%eax
53+
jle bpf_slow_path_half
54+
movzwl (SKBDATA,%rsi),%eax
55+
rol $8,%ax # ntohs()
56+
ret
57+
58+
sk_load_byte_ind:
59+
.globl sk_load_byte_ind
60+
add %ebx,%esi /* offset += X */
61+
js bpf_error
62+
63+
sk_load_byte:
64+
.globl sk_load_byte
65+
66+
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
67+
jle bpf_slow_path_byte
68+
movzbl (SKBDATA,%rsi),%eax
69+
ret
70+
71+
/**
72+
* sk_load_byte_msh - BPF_S_LDX_B_MSH helper
73+
*
74+
* Implements BPF_S_LDX_B_MSH : ldxb 4*([offset]&0xf)
75+
* Must preserve A accumulator (%eax)
76+
* Inputs : %esi is the offset value, already known positive
77+
*/
78+
ENTRY(sk_load_byte_msh)
79+
CFI_STARTPROC
80+
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
81+
jle bpf_slow_path_byte_msh
82+
movzbl (SKBDATA,%rsi),%ebx
83+
and $15,%bl
84+
shl $2,%bl
85+
ret
86+
CFI_ENDPROC
87+
ENDPROC(sk_load_byte_msh)
88+
89+
bpf_error:
90+
# force a return 0 from jit handler
91+
xor %eax,%eax
92+
mov -8(%rbp),%rbx
93+
leaveq
94+
ret
95+
96+
/* rsi contains offset and can be scratched */
97+
#define bpf_slow_path_common(LEN) \
98+
push %rdi; /* save skb */ \
99+
push %r9; \
100+
push SKBDATA; \
101+
/* rsi already has offset */ \
102+
mov $LEN,%ecx; /* len */ \
103+
lea -12(%rbp),%rdx; \
104+
call skb_copy_bits; \
105+
test %eax,%eax; \
106+
pop SKBDATA; \
107+
pop %r9; \
108+
pop %rdi
109+
110+
111+
bpf_slow_path_word:
112+
bpf_slow_path_common(4)
113+
js bpf_error
114+
mov -12(%rbp),%eax
115+
bswap %eax
116+
ret
117+
118+
bpf_slow_path_half:
119+
bpf_slow_path_common(2)
120+
js bpf_error
121+
mov -12(%rbp),%ax
122+
rol $8,%ax
123+
movzwl %ax,%eax
124+
ret
125+
126+
bpf_slow_path_byte:
127+
bpf_slow_path_common(1)
128+
js bpf_error
129+
movzbl -12(%rbp),%eax
130+
ret
131+
132+
bpf_slow_path_byte_msh:
133+
xchg %eax,%ebx /* dont lose A , X is about to be scratched */
134+
bpf_slow_path_common(1)
135+
js bpf_error
136+
movzbl -12(%rbp),%eax
137+
and $15,%al
138+
shl $2,%al
139+
xchg %eax,%ebx
140+
ret

0 commit comments

Comments
 (0)