diff options
author | Michael Holzheu <holzheu@linux.vnet.ibm.com> | 2015-04-01 16:08:32 +0200 |
---|---|---|
committer | Martin Schwidefsky <schwidefsky@de.ibm.com> | 2015-04-15 12:23:49 +0200 |
commit | 054623105728b06852f077299e2bf1bf3d5f2b0b (patch) | |
tree | e05f39e160591d88255da22719bc40813698a89f /arch | |
parent | 1c6e4b1811bffc6b0520310f6d8a6b4ec760fdbc (diff) |
s390/bpf: Add s390x eBPF JIT compiler backend
Replace 32 bit BPF JIT backend with new 64 bit eBPF backend.
Signed-off-by: Michael Holzheu <holzheu@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Diffstat (limited to 'arch')
-rw-r--r-- | arch/s390/Kconfig | 2 | ||||
-rw-r--r-- | arch/s390/net/bpf_jit.S | 197 | ||||
-rw-r--r-- | arch/s390/net/bpf_jit.h | 58 | ||||
-rw-r--r-- | arch/s390/net/bpf_jit_comp.c | 1780 |
4 files changed, 1234 insertions, 803 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a5ced5c3c1e0..53a91425d92d 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -115,7 +115,7 @@ config S390 select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE - select HAVE_BPF_JIT if PACK_STACK + select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z9_109_FEATURES select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_KMEMLEAK diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S index ba44c9f55346..a1c917d881ec 100644 --- a/arch/s390/net/bpf_jit.S +++ b/arch/s390/net/bpf_jit.S @@ -1,134 +1,115 @@ /* * BPF Jit compiler for s390, help functions. * - * Copyright IBM Corp. 2012 + * Copyright IBM Corp. 2012,2015 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> */ + #include <linux/linkage.h> +#include "bpf_jit.h" /* * Calling convention: - * registers %r2, %r6-%r8, %r10-%r11, %r13, %r15 are call saved - * %r2: skb pointer - * %r3: offset parameter - * %r5: BPF A accumulator - * %r8: return address - * %r9: save register for skb pointer - * %r10: skb->data - * %r11: skb->len - skb->data_len (headlen) - * %r12: BPF X accumulator + * registers %r7-%r10, %r11,%r13, and %r15 are call saved + * + * Input (64 bit): + * %r3 (%b2) = offset into skb data + * %r6 (%b5) = return address + * %r7 (%b6) = skb pointer + * %r12 = skb data pointer + * + * Output: + * %r14= %b0 = return value (read skb value) + * + * Work registers: %r2,%r4,%r5,%r14 * * skb_copy_bits takes 4 parameters: * %r2 = skb pointer * %r3 = offset into skb data * %r4 = pointer to temp buffer * %r5 = length to copy + * Return value in %r2: 0 = ok + * + * bpf_internal_load_pointer_neg_helper takes 3 parameters: + * %r2 = skb pointer + * %r3 = offset into data + * %r4 = length to copy + * Return value in %r2: Pointer to data */ -#define SKBDATA %r8 - - /* A = *(u32 *) (skb->data+K+X) */ -ENTRY(sk_load_word_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc - - /* A = *(u32 *) (skb->data+K) */ -ENTRY(sk_load_word) - llgfr %r1,%r3 # extend offset - ahi %r3,4 # offset + 4 - clr %r11,%r3 # hlen <= offset + 4 ? - jl sk_load_word_slow - l %r5,0(%r1,%r10) # get word from skb - xr %r1,%r1 # set cc to zero - br %r8 -sk_load_word_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,160(%r15) # pointer to temp buffer - lghi %r5,4 # 4 bytes - brasl %r14,skb_copy_bits # get data from skb - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +#define SKF_MAX_NEG_OFF -0x200000 /* SKF_LL_OFF from filter.h */ - /* A = *(u16 *) (skb->data+K+X) */ -ENTRY(sk_load_half_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc - - /* A = *(u16 *) (skb->data+K) */ -ENTRY(sk_load_half) - llgfr %r1,%r3 # extend offset - ahi %r3,2 # offset + 2 - clr %r11,%r3 # hlen <= offset + 2 ? - jl sk_load_half_slow - llgh %r5,0(%r1,%r10) # get half from skb - xr %r1,%r1 # set cc to zero - br %r8 - -sk_load_half_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,162(%r15) # pointer to temp buffer - lghi %r5,2 # 2 bytes - brasl %r14,skb_copy_bits # get data from skb - xc 160(2,%r15),160(%r15) - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +/* + * Load SIZE bytes from SKB + */ +#define sk_load_common(NAME, SIZE, LOAD) \ +ENTRY(sk_load_##NAME); \ + ltgr %r3,%r3; /* Is offset negative? */ \ + jl sk_load_##NAME##_slow_neg; \ +ENTRY(sk_load_##NAME##_pos); \ + aghi %r3,SIZE; /* Offset + SIZE */ \ + clg %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */ \ + jh sk_load_##NAME##_slow; \ + LOAD %r14,-SIZE(%r3,%r12); /* Get data from skb */ \ + b OFF_OK(%r6); /* Return */ \ + \ +sk_load_##NAME##_slow:; \ + lgr %r2,%r7; /* Arg1 = skb pointer */ \ + aghi %r3,-SIZE; /* Arg2 = offset */ \ + la %r4,STK_OFF_TMP(%r15); /* Arg3 = temp bufffer */ \ + lghi %r5,SIZE; /* Arg4 = size */ \ + brasl %r14,skb_copy_bits; /* Get data from skb */ \ + LOAD %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */ \ + ltgr %r2,%r2; /* Set cc to (%r2 != 0) */ \ + br %r6; /* Return */ - /* A = *(u8 *) (skb->data+K+X) */ -ENTRY(sk_load_byte_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc +sk_load_common(word, 4, llgf) /* r14 = *(u32 *) (skb->data+offset) */ +sk_load_common(half, 2, llgh) /* r14 = *(u16 *) (skb->data+offset) */ - /* A = *(u8 *) (skb->data+K) */ +/* + * Load 1 byte from SKB (optimized version) + */ + /* r14 = *(u8 *) (skb->data+offset) */ ENTRY(sk_load_byte) - llgfr %r1,%r3 # extend offset - clr %r11,%r3 # hlen < offset ? - jle sk_load_byte_slow - lhi %r5,0 - ic %r5,0(%r1,%r10) # get byte from skb - xr %r1,%r1 # set cc to zero - br %r8 + ltgr %r3,%r3 # Is offset negative? + jl sk_load_byte_slow_neg +ENTRY(sk_load_byte_pos) + clg %r3,STK_OFF_HLEN(%r15) # Offset >= hlen? + jnl sk_load_byte_slow + llgc %r14,0(%r3,%r12) # Get byte from skb + b OFF_OK(%r6) # Return OK sk_load_byte_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,163(%r15) # pointer to temp buffer - lghi %r5,1 # 1 byte - brasl %r14,skb_copy_bits # get data from skb - xc 160(3,%r15),160(%r15) - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 + lgr %r2,%r7 # Arg1 = skb pointer + # Arg2 = offset + la %r4,STK_OFF_TMP(%r15) # Arg3 = pointer to temp buffer + lghi %r5,1 # Arg4 = size (1 byte) + brasl %r14,skb_copy_bits # Get data from skb + llgc %r14,STK_OFF_TMP(%r15) # Load result from temp buffer + ltgr %r2,%r2 # Set cc to (%r2 != 0) + br %r6 # Return cc + +#define sk_negative_common(NAME, SIZE, LOAD) \ +sk_load_##NAME##_slow_neg:; \ + cgfi %r3,SKF_MAX_NEG_OFF; \ + jl bpf_error; \ + lgr %r2,%r7; /* Arg1 = skb pointer */ \ + /* Arg2 = offset */ \ + lghi %r4,SIZE; /* Arg3 = size */ \ + brasl %r14,bpf_internal_load_pointer_neg_helper; \ + ltgr %r2,%r2; \ + jz bpf_error; \ + LOAD %r14,0(%r2); /* Get data from pointer */ \ + xr %r3,%r3; /* Set cc to zero */ \ + br %r6; /* Return cc */ - /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ -ENTRY(sk_load_byte_msh) - llgfr %r1,%r3 # extend offset - clr %r11,%r3 # hlen < offset ? - jle sk_load_byte_msh_slow - lhi %r12,0 - ic %r12,0(%r1,%r10) # get byte from skb - nill %r12,0x0f - sll %r12,2 - xr %r1,%r1 # set cc to zero - br %r8 +sk_negative_common(word, 4, llgf) +sk_negative_common(half, 2, llgh) +sk_negative_common(byte, 1, llgc) -sk_load_byte_msh_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,163(%r15) # pointer to temp buffer - lghi %r5,1 # 1 byte - brasl %r14,skb_copy_bits # get data from skb - xc 160(3,%r15),160(%r15) - l %r12,160(%r15) # load result from temp buffer - nill %r12,0x0f - sll %r12,2 - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +bpf_error: +# force a return 0 from jit handler + ltgr %r15,%r15 # Set condition code + br %r6 diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h new file mode 100644 index 000000000000..ba8593a515ba --- /dev/null +++ b/arch/s390/net/bpf_jit.h @@ -0,0 +1,58 @@ +/* + * BPF Jit compiler defines + * + * Copyright IBM Corp. 2012,2015 + * + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> + */ + +#ifndef __ARCH_S390_NET_BPF_JIT_H +#define __ARCH_S390_NET_BPF_JIT_H + +#ifndef __ASSEMBLY__ + +#include <linux/filter.h> +#include <linux/types.h> + +extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[]; +extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; + +#endif /* __ASSEMBLY__ */ + +/* + * Stackframe layout (packed stack): + * + * ^ high + * +---------------+ | + * | old backchain | | + * +---------------+ | + * | r15 - r6 | | + * BFP -> +===============+ | + * | | | + * | BPF stack | | + * | | | + * +---------------+ | + * | 8 byte hlen | | + * R15+168 -> +---------------+ | + * | 4 byte align | | + * +---------------+ | + * | 4 byte temp | | + * | for bpf_jit.S | | + * R15+160 -> +---------------+ | + * | new backchain | | + * R15+152 -> +---------------+ | + * | + 152 byte SA | | + * R15 -> +---------------+ + low + * + * We get 160 bytes stack space from calling function, but only use + * 11 * 8 byte (old backchain + r15 - r6) for storing registers. + */ +#define STK_OFF (MAX_BPF_STACK + 8 + 4 + 4 + (160 - 11 * 8)) +#define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */ +#define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */ + +/* Offset to skip condition code check */ +#define OFF_OK 4 + +#endif /* __ARCH_S390_NET_BPF_JIT_H */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index bbd1981cc150..7690dc8e1ab5 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1,817 +1,1209 @@ /* * BPF Jit compiler for s390. * - * Copyright IBM Corp. 2012 + * Minimum build requirements: + * + * - HAVE_MARCH_Z196_FEATURES: laal, laalg + * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj + * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf + * - PACK_STACK + * - 64BIT + * + * Copyright IBM Corp. 2012,2015 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> */ + +#define KMSG_COMPONENT "bpf_jit" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include <linux/netdevice.h> -#include <linux/if_vlan.h> #include <linux/filter.h> #include <linux/init.h> #include <asm/cacheflush.h> -#include <asm/facility.h> #include <asm/dis.h> +#include "bpf_jit.h" -/* - * Conventions: - * %r2 = skb pointer - * %r3 = offset parameter - * %r4 = scratch register / length parameter - * %r5 = BPF A accumulator - * %r8 = return address - * %r9 = save register for skb pointer - * %r10 = skb->data - * %r11 = skb->len - skb->data_len (headlen) - * %r12 = BPF X accumulator - * %r13 = literal pool pointer - * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS - */ int bpf_jit_enable __read_mostly; +struct bpf_jit { + u32 seen; /* Flags to remember seen eBPF instructions */ + u32 seen_reg[16]; /* Array to remember which registers are used */ + u32 *addrs; /* Array with relative instruction addresses */ + u8 *prg_buf; /* Start of program */ + int size; /* Size of program and literal pool */ + int size_prg; /* Size of program */ + int prg; /* Current position in program */ + int lit_start; /* Start of literal pool */ + int lit; /* Current position in literal pool */ + int base_ip; /* Base address for literal pool */ + int ret0_ip; /* Address of return 0 */ + int exit_ip; /* Address of exit */ +}; + +#define BPF_SIZE_MAX 4096 /* Max size for program */ + +#define SEEN_SKB 1 /* skb access */ +#define SEEN_MEM 2 /* use mem[] for temporary storage */ +#define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */ +#define SEEN_LITERAL 8 /* code uses literals */ +#define SEEN_FUNC 16 /* calls C functions */ +#define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB) + /* - * assembly code in arch/x86/net/bpf_jit.S + * s390 registers */ -extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; -extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; +#define REG_W0 (__MAX_BPF_REG+0) /* Work register 1 (even) */ +#define REG_W1 (__MAX_BPF_REG+1) /* Work register 2 (odd) */ +#define REG_SKB_DATA (__MAX_BPF_REG+2) /* SKB data register */ +#define REG_L (__MAX_BPF_REG+3) /* Literal pool register */ +#define REG_15 (__MAX_BPF_REG+4) /* Register 15 */ +#define REG_0 REG_W0 /* Register 0 */ +#define REG_2 BPF_REG_1 /* Register 2 */ +#define REG_14 BPF_REG_0 /* Register 14 */ -struct bpf_jit { - unsigned int seen; - u8 *start; - u8 *prg; - u8 *mid; - u8 *lit; - u8 *end; - u8 *base_ip; - u8 *ret0_ip; - u8 *exit_ip; - unsigned int off_load_word; - unsigned int off_load_half; - unsigned int off_load_byte; - unsigned int off_load_bmsh; - unsigned int off_load_iword; - unsigned int off_load_ihalf; - unsigned int off_load_ibyte; +/* + * Mapping of BPF registers to s390 registers + */ +static const int reg2hex[] = { + /* Return code */ + [BPF_REG_0] = 14, + /* Function parameters */ + [BPF_REG_1] = 2, + [BPF_REG_2] = 3, + [BPF_REG_3] = 4, + [BPF_REG_4] = 5, + [BPF_REG_5] = 6, + /* Call saved registers */ + [BPF_REG_6] = 7, + [BPF_REG_7] = 8, + [BPF_REG_8] = 9, + [BPF_REG_9] = 10, + /* BPF stack pointer */ + [BPF_REG_FP] = 13, + /* SKB data pointer */ + [REG_SKB_DATA] = 12, + /* Work registers for s390x backend */ + [REG_W0] = 0, + [REG_W1] = 1, + [REG_L] = 11, + [REG_15] = 15, }; -#define BPF_SIZE_MAX 4096 /* Max size for program */ +static inline u32 reg(u32 dst_reg, u32 src_reg) +{ + return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; +} + +static inline u32 reg_high(u32 reg) +{ + return reg2hex[reg] << 4; +} + +static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) +{ + u32 r1 = reg2hex[b1]; + + if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15) + jit->seen_reg[r1] = 1; +} + +#define REG_SET_SEEN(b1) \ +({ \ + reg_set_seen(jit, b1); \ +}) + +#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]] + +/* + * EMIT macros for code generation + */ + +#define _EMIT2(op) \ +({ \ + if (jit->prg_buf) \ + *(u16 *) (jit->prg_buf + jit->prg) = op; \ + jit->prg += 2; \ +}) -#define SEEN_DATAREF 1 /* might call external helpers */ -#define SEEN_XREG 2 /* ebx is used */ -#define SEEN_MEM 4 /* use mem[] for temporary storage */ -#define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ -#define SEEN_LITERAL 16 /* code uses literals */ -#define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ -#define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ -#define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ -#define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ -#define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ -#define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ -#define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ - -#define EMIT2(op) \ -({ \ - if (jit->prg + 2 <= jit->mid) \ - *(u16 *) jit->prg = op; \ - jit->prg += 2; \ +#define EMIT2(op, b1, b2) \ +({ \ + _EMIT2(op | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT4(op) \ -({ \ - if (jit->prg + 4 <= jit->mid) \ - *(u32 *) jit->prg = op; \ - jit->prg += 4; \ +#define _EMIT4(op) \ +({ \ + if (jit->prg_buf) \ + *(u32 *) (jit->prg_buf + jit->prg) = op; \ + jit->prg += 4; \ }) -#define EMIT4_DISP(op, disp) \ -({ \ - unsigned int __disp = (disp) & 0xfff; \ - EMIT4(op | __disp); \ +#define EMIT4(op, b1, b2) \ +({ \ + _EMIT4(op | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT4_IMM(op, imm) \ -({ \ - unsigned int __imm = (imm) & 0xffff; \ - EMIT4(op | __imm); \ +#define EMIT4_RRF(op, b1, b2, b3) \ +({ \ + _EMIT4(op | reg_high(b3) << 8 | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ }) -#define EMIT4_PCREL(op, pcrel) \ -({ \ - long __pcrel = ((pcrel) >> 1) & 0xffff; \ - EMIT4(op | __pcrel); \ +#define _EMIT4_DISP(op, disp) \ +({ \ + unsigned int __disp = (disp) & 0xfff; \ + _EMIT4(op | __disp); \ }) -#define EMIT6(op1, op2) \ -({ \ - if (jit->prg + 6 <= jit->mid) { \ - *(u32 *) jit->prg = op1; \ - *(u16 *) (jit->prg + 4) = op2; \ - } \ - jit->prg += 6; \ +#define EMIT4_DISP(op, b1, b2, disp) \ +({ \ + _EMIT4_DISP(op | reg_high(b1) << 16 | \ + reg_high(b2) << 8, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT6_DISP(op1, op2, disp) \ -({ \ - unsigned int __disp = (disp) & 0xfff; \ - EMIT6(op1 | __disp, op2); \ +#define EMIT4_IMM(op, b1, imm) \ +({ \ + unsigned int __imm = (imm) & 0xffff; \ + _EMIT4(op | reg_high(b1) << 16 | __imm); \ + REG_SET_SEEN(b1); \ }) -#define EMIT6_IMM(op, imm) \ -({ \ - unsigned int __imm = (imm); \ - EMIT6(op | (__imm >> 16), __imm & 0xffff); \ +#define EMIT4_PCREL(op, pcrel) \ +({ \ + long __pcrel = ((pcrel) >> 1) & 0xffff; \ + _EMIT4(op | __pcrel); \ }) -#define EMIT_CONST(val) \ -({ \ - unsigned int ret; \ - ret = (unsigned int) (jit->lit - jit->base_ip); \ - jit->seen |= SEEN_LITERAL; \ - if (jit->lit + 4 <= jit->end) \ - *(u32 *) jit->lit = val; \ - jit->lit += 4; \ - ret; \ +#define _EMIT6(op1, op2) \ +({ \ + if (jit->prg_buf) { \ + *(u32 *) (jit->prg_buf + jit->prg) = op1; \ + *(u16 *) (jit->prg_buf + jit->prg + 4) = op2; \ + } \ + jit->prg += 6; \ }) -#define EMIT_FN_CONST(bit, fn) \ -({ \ - unsigned int ret; \ - ret = (unsigned int) (jit->lit - jit->base_ip); \ - if (jit->seen & bit) { \ - jit->seen |= SEEN_LITERAL; \ - if (jit->lit + 8 <= jit->end) \ - *(void **) jit->lit = fn; \ - jit->lit += 8; \ - } \ - ret; \ +#define _EMIT6_DISP(op1, op2, disp) \ +({ \ + unsigned int __disp = (disp) & 0xfff; \ + _EMIT6(op1 | __disp, op2); \ }) -static void bpf_jit_fill_hole(void *area, unsigned int size) +#define EMIT6_DISP(op1, op2, b1, b2, b3, disp) \ +({ \ + _EMIT6_DISP(op1 | reg(b1, b2) << 16 | \ + reg_high(b3) << 8, op2, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ +}) + +#define _EMIT6_DISP_LH(op1, op2, disp) \ +({ \ + unsigned int __disp_h = ((u32)disp) & 0xff000; \ + unsigned int __disp_l = ((u32)disp) & 0x00fff; \ + _EMIT6(op1 | __disp_l, op2 | __disp_h >> 4); \ +}) + +#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ +({ \ + _EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 | \ + reg_high(b3) << 8, op2, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ +}) + +#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ +({ \ + /* Branch instruction needs 6 bytes */ \ + int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\ + _EMIT6(op1 | reg(b1, b2) << 16 | rel, op2 | mask); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ +}) + +#define _EMIT6_IMM(op, imm) \ +({ \ + unsigned int __imm = (imm); \ + _EMIT6(op | (__imm >> 16), __imm & 0xffff); \ +}) + +#define EMIT6_IMM(op, b1, imm) \ +({ \ + _EMIT6_IMM(op | reg_high(b1) << 16, imm); \ + REG_SET_SEEN(b1); \ +}) + +#define EMIT_CONST_U32(val) \ +({ \ + unsigned int ret; \ + ret = jit->lit - jit->base_ip; \ + jit->seen |= SEEN_LITERAL; \ + if (jit->prg_buf) \ + *(u32 *) (jit->prg_buf + jit->lit) = (u32) val; \ + jit->lit += 4; \ + ret; \ +}) + +#define EMIT_CONST_U64(val) \ +({ \ + unsigned int ret; \ + ret = jit->lit - jit->base_ip; \ + jit->seen |= SEEN_LITERAL; \ + if (jit->prg_buf) \ + *(u64 *) (jit->prg_buf + jit->lit) = (u64) val; \ + jit->lit += 8; \ + ret; \ +}) + +#define EMIT_ZERO(b1) \ +({ \ + /* llgfr %dst,%dst (zero extend to 64 bit) */ \ + EMIT4(0xb9160000, b1, b1); \ + REG_SET_SEEN(b1); \ +}) + +/* + * Fill whole space with illegal instructions + */ +static void jit_fill_hole(void *area, unsigned int size) { - /* Fill whole space with illegal instructions */ memset(area, 0, size); } -static void bpf_jit_prologue(struct bpf_jit *jit) +/* + * Save registers from "rs" (register start) to "re" (register end) on stack + */ +static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) +{ + u32 off = 72 + (rs - 6) * 8; + + if (rs == re) + /* stg %rs,off(%r15) */ + _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); + else + /* stmg %rs,%re,off(%r15) */ + _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); +} + +/* + * Restore registers from "rs" (register start) to "re" (register end) on stack + */ +static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) { - /* Save registers and create stack frame if necessary */ - if (jit->seen & SEEN_DATAREF) { - /* stmg %r8,%r15,88(%r15) */ - EMIT6(0xeb8ff058, 0x0024); - /* lgr %r14,%r15 */ - EMIT4(0xb90400ef); - /* aghi %r15,<offset> */ - EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80); - /* stg %r14,152(%r15) */ - EMIT6(0xe3e0f098, 0x0024); - } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) - /* stmg %r12,%r13,120(%r15) */ - EMIT6(0xebcdf078, 0x0024); - else if (jit->seen & SEEN_XREG) - /* stg %r12,120(%r15) */ - EMIT6(0xe3c0f078, 0x0024); - else if (jit->seen & SEEN_LITERAL) - /* stg %r13,128(%r15) */ - EMIT6(0xe3d0f080, 0x0024); + u32 off = 72 + (rs - 6) * 8; + + if (jit->seen & SEEN_STACK) + off += STK_OFF; + + if (rs == re) + /* lg %rs,off(%r15) */ + _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); + else + /* lmg %rs,%re,off(%r15) */ + _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); +} +/* + * Return first seen register (from start) + */ +static int get_start(struct bpf_jit *jit, int start) +{ + int i; + + for (i = start; i <= 15; i++) { + if (jit->seen_reg[i]) + return i; + } + return 0; +} + +/* + * Return last seen register (from start) (gap >= 2) + */ +static int get_end(struct bpf_jit *jit, int start) +{ + int i; + + for (i = start; i < 15; i++) { + if (!jit->seen_reg[i] && !jit->seen_reg[i + 1]) + return i - 1; + } + return jit->seen_reg[15] ? 15 : 14; +} + +#define REGS_SAVE 1 +#define REGS_RESTORE 0 +/* + * Save and restore clobbered registers (6-15) on stack. + * We save/restore registers in chunks with gap >= 2 registers. + */ +static void save_restore_regs(struct bpf_jit *jit, int op) +{ + + int re = 6, rs; + + do { + rs = get_start(jit, re); + if (!rs) + break; + re = get_end(jit, rs + 1); + if (op == REGS_SAVE) + save_regs(jit, rs, re); + else + restore_regs(jit, rs, re); + re++; + } while (re <= 15); +} + +/* + * Emit function prologue + * + * Save registers and create stack frame if necessary. + * See stack frame layout desription in "bpf_jit.h"! + */ +static void bpf_jit_prologue(struct bpf_jit *jit) +{ + /* Save registers */ + save_restore_regs(jit, REGS_SAVE); /* Setup literal pool */ if (jit->seen & SEEN_LITERAL) { /* basr %r13,0 */ - EMIT2(0x0dd0); + EMIT2(0x0d00, REG_L, REG_0); jit->base_ip = jit->prg; } - jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); - jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); - jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); - jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); - jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); - jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); - jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); - - /* Filter needs to access skb data */ - if (jit->seen & SEEN_DATAREF) { - /* l %r11,<len>(%r2) */ - EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); - /* s %r11,<data_len>(%r2) */ - EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); - /* lg %r10,<data>(%r2) */ - EMIT6_DISP(0xe3a02000, 0x0004, - offsetof(struct sk_buff, data)); + /* Setup stack and backchain */ + if (jit->seen & SEEN_STACK) { + /* lgr %bfp,%r15 (BPF frame pointer) */ + EMIT4(0xb9040000, BPF_REG_FP, REG_15); + /* aghi %r15,-STK_OFF */ + EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF); + if (jit->seen & SEEN_FUNC) + /* stg %bfp,152(%r15) (backchain) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_FP, REG_0, + REG_15, 152); + } + /* + * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" + * we store the SKB header length on the stack and the SKB data + * pointer in REG_SKB_DATA. + */ + if (jit->seen & SEEN_SKB) { + /* Header length: llgf %w1,<len>(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1, + offsetof(struct sk_buff, len)); + /* s %w1,<data_len>(%b1) */ + EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1, + offsetof(struct sk_buff, data_len)); + /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, + STK_OFF_HLEN); + /* lg %skb_data,data_off(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, + BPF_REG_1, offsetof(struct sk_buff, data)); } + /* BPF compatibility: clear A (%b7) and X (%b8) registers */ + if (REG_SEEN(BPF_REG_7)) + /* lghi %b7,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_7, 0); + if (REG_SEEN(BPF_REG_8)) + /* lghi %b8,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_8, 0); } +/* + * Function epilogue + */ static void bpf_jit_epilogue(struct bpf_jit *jit) { /* Return 0 */ if (jit->seen & SEEN_RET0) { jit->ret0_ip = jit->prg; - /* lghi %r2,0 */ - EMIT4(0xa7290000); + /* lghi %b0,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_0, 0); } jit->exit_ip = jit->prg; + /* Load exit code: lgr %r2,%b0 */ + EMIT4(0xb9040000, REG_2, BPF_REG_0); /* Restore registers */ - if (jit->seen & SEEN_DATAREF) - /* lmg %r8,%r15,<offset>(%r15) */ - EMIT6_DISP(0xeb8ff000, 0x0004, - (jit->seen & SEEN_MEM) ? 200 : 168); - else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) - /* lmg %r12,%r13,120(%r15) */ - EMIT6(0xebcdf078, 0x0004); - else if (jit->seen & SEEN_XREG) - /* lg %r12,120(%r15) */ - EMIT6(0xe3c0f078, 0x0004); - else if (jit->seen & SEEN_LITERAL) - /* lg %r13,128(%r15) */ - EMIT6(0xe3d0f080, 0x0004); + save_restore_regs(jit, REGS_RESTORE); /* br %r14 */ - EMIT2(0x07fe); + _EMIT2(0x07fe); } /* - * make sure we dont leak kernel information to user + * Compile one eBPF instruction into s390x code */ -static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) +static int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i) { - /* Clear temporary memory if (seen & SEEN_MEM) */ - if (jit->seen & SEEN_MEM) - /* xc 0(64,%r15),0(%r15) */ - EMIT6(0xd73ff000, 0xf000); - /* Clear X if (seen & SEEN_XREG) */ - if (jit->seen & SEEN_XREG) - /* lhi %r12,0 */ - EMIT4(0xa7c80000); - /* Clear A if the first register does not set it. */ - switch (filter[0].code) { - case BPF_LD | BPF_W | BPF_ABS: - case BPF_LD | BPF_H | BPF_ABS: - case BPF_LD | BPF_B | BPF_ABS: - case BPF_LD | BPF_W | BPF_LEN: - case BPF_LD | BPF_W | BPF_IND: - case BPF_LD | BPF_H | BPF_IND: - case BPF_LD | BPF_B | BPF_IND: - case BPF_LD | BPF_IMM: - case BPF_LD | BPF_MEM: - case BPF_MISC | BPF_TXA: - case BPF_RET | BPF_K: - /* first instruction sets A register */ - break; - default: /* A = 0 */ - /* lhi %r5,0 */ - EMIT4(0xa7580000); - } -} + struct bpf_insn *insn = &fp->insnsi[i]; + int jmp_off, last, insn_count = 1; + unsigned int func_addr, mask; + u32 dst_reg = insn->dst_reg; + u32 src_reg = insn->src_reg; + u32 *addrs = jit->addrs; + s32 imm = insn->imm; + s16 off = insn->off; -static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, - unsigned int *addrs, int i, int last) -{ - unsigned int K; - int offset; - unsigned int mask; - u16 code; - - K = filter->k; - code = bpf_anc_helper(filter); - - switch (code) { - case BPF_ALU | BPF_ADD | BPF_X: /* A += X */ - jit->seen |= SEEN_XREG; - /* ar %r5,%r12 */ - EMIT2(0x1a5c); - break; - case BPF_ALU | BPF_ADD | BPF_K: /* A += K */ - if (!K) + switch (insn->code) { + /* + * BPF_MOV + */ + case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */ + /* llgfr %dst,%src */ + EMIT4(0xb9160000, dst_reg, src_reg); + break; + case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ + /* lgr %dst,%src */ + EMIT4(0xb9040000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ + /* llilf %dst,imm */ + EMIT6_IMM(0xc00f0000, dst_reg, imm); + break; + case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ + /* lgfi %dst,imm */ + EMIT6_IMM(0xc0010000, dst_reg, imm); + break; + /* + * BPF_LD 64 + */ + case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ + { + /* 16 byte instruction that uses two 'struct bpf_insn' */ + u64 imm64; + + imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; + /* lg %dst,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L, + EMIT_CONST_U64(imm64)); + insn_count = 2; + break; + } + /* + * BPF_ADD + */ + case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ + /* ar %dst,%src */ + EMIT2(0x1a00, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ + /* agr %dst,%src */ + EMIT4(0xb9080000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ + if (!imm) break; - if (K <= 16383) - /* ahi %r5,<K> */ - EMIT4_IMM(0xa75a0000, K); - else if (test_facility(21)) - /* alfi %r5,<K> */ - EMIT6_IMM(0xc25b0000, K); - else - /* a %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); + /* alfi %dst,imm */ + EMIT6_IMM(0xc20b0000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ + if (!imm) + break; + /* agfi %dst,imm */ + EMIT6_IMM(0xc2080000, dst_reg, imm); + break; + /* + * BPF_SUB + */ + case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ + /* sr %dst,%src */ + EMIT2(0x1b00, dst_reg, src_reg); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_SUB | BPF_X: /* A -= X */ - jit->seen |= SEEN_XREG; - /* sr %r5,%r12 */ - EMIT2(0x1b5c); + case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ + /* sgr %dst,%src */ + EMIT4(0xb9090000, dst_reg, src_reg); break; - case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ - if (!K) + case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ + if (!imm) break; - if (K <= 16384) - /* ahi %r5,-K */ - EMIT4_IMM(0xa75a0000, -K); - else if (test_facility(21)) - /* alfi %r5,-K */ - EMIT6_IMM(0xc25b0000, -K); - else - /* s %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); - break; - case BPF_ALU | BPF_MUL | BPF_X: /* A *= X */ - jit->seen |= SEEN_XREG; - /* msr %r5,%r12 */ - EMIT4(0xb252005c); - break; - case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ - if (K <= 16383) - /* mhi %r5,K */ - EMIT4_IMM(0xa75c0000, K); - else if (test_facility(34)) - /* msfi %r5,<K> */ - EMIT6_IMM(0xc2510000, K); - else - /* ms %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x7150d000, EMIT_CONST(K)); + /* alfi %dst,-imm */ + EMIT6_IMM(0xc20b0000, dst_reg, -imm); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_DIV | BPF_X: /* A /= X */ - jit->seen |= SEEN_XREG | SEEN_RET0; - /* ltr %r12,%r12 */ - EMIT2(0x12cc); - /* jz <ret0> */ - EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); - /* lhi %r4,0 */ - EMIT4(0xa7480000); - /* dlr %r4,%r12 */ - EMIT4(0xb997004c); - break; - case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ - if (K == 1) + case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ + if (!imm) + break; + /* agfi %dst,-imm */ + EMIT6_IMM(0xc2080000, dst_reg, -imm); + break; + /* + * BPF_MUL + */ + case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ + /* msr %dst,%src */ + EMIT4(0xb2520000, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ + /* msgr %dst,%src */ + EMIT4(0xb90c0000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ + if (imm == 1) + break; + /* msfi %r5,imm */ + EMIT6_IMM(0xc2010000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm * |