/*
* Just-In-Time compiler for BPF filters on 32bit ARM
*
* Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation; version 2 of the License.
*/
#include <linux/bitops.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/filter.h>
#include <linux/netdevice.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/if_vlan.h>
#include <asm/cacheflush.h>
#include <asm/hwcap.h>
#include <asm/opcodes.h>
#include "bpf_jit_32.h"
/*
* ABI:
*
* r0 scratch register
* r4 BPF register A
* r5 BPF register X
* r6 pointer to the skb
* r7 skb->data
* r8 skb_headlen(skb)
*/
#define r_scratch ARM_R0
/* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */
#define r_off ARM_R1
#define r_A ARM_R4
#define r_X ARM_R5
#define r_skb ARM_R6
#define r_skb_data ARM_R7
#define r_skb_hl ARM_R8
#define SCRATCH_SP_OFFSET 0
#define SCRATCH_OFF(k) (SCRATCH_SP_OFFSET + 4 * (k))
#define SEEN_MEM ((1 << BPF_MEMWORDS) - 1)
#define SEEN_MEM_WORD(k) (1 << (k))
#define SEEN_X (1 << BPF_MEMWORDS)
#define SEEN_CALL (1 << (BPF_MEMWORDS + 1))
#define SEEN_SKB (1 << (BPF_MEMWORDS + 2))
#define SEEN_DATA (1 << (BPF_MEMWORDS + 3))
#define FLAG_NEED_X_RESET (1 << 0)
#define FLAG_IMM_OVERFLOW (1 << 1)
struct jit_ctx {
const struct bpf_prog *skf;
unsigned idx;
unsigned prologue_bytes;
int ret0_fp_idx;
u32 seen;
u32 flags;
u32 *offsets;
u32 *target;
#if __LINUX_ARM_ARCH__ < 7
u16 epilogue_bytes;
u16 imm_count;
u32 *imms;
#endif
};
int bpf_jit_enable __read_mostly;
static inline int call_neg_helper(struct sk_buff *skb, int offset, void *ret,
unsigned int size)
{
void *ptr = bpf_internal_load_pointer_neg_helper(skb, offset, size);
if (!ptr)
return -EFAULT;
memcpy(ret, ptr, size);
return 0;
}
static u64 jit_get_skb_b(struct sk_buff *skb, int offset)
{
u8 ret;
int err;
if (offset < 0)
err = call_neg_helper(skb, offset, &ret, 1);
else
err = skb_copy_bits(skb, offset, &ret, 1);
return (u64)err << 32 | ret;
}
static u64 jit_get_skb_h(struct sk_buff *skb, int offset)
{
u16 ret;
int err;
if (offset < 0)
err = call_neg_helper(skb, offset, &ret, 2);
else
err = skb_copy_bits(skb, offset, &ret, 2);
return (u64)err << 32 | ntohs(ret);
}
static u64 jit_get_skb_w(struct sk_buff *skb, int offset)
{
u32 ret;
int err;
if (offset < 0)
err = call_neg_helper(skb, offset, &ret, 4);
else
err = skb_copy_bits(skb, offset, &ret, 4);
return (u64)err << 32 | ntohl(ret);
}
/*
* Wrappers which handle both OABI and EABI and assures Thumb2 interworking
* (where the assembly routines like __aeabi_uidiv could cause problems).
*/
static u32 jit_udiv(u32 dividend, u32 divisor)
{
return dividend / divisor;
}
static u32 jit_mod(u32 dividend, u32 divisor)
{
return dividend % divisor;
}
static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
{
inst |= (cond << 28);
inst = __opcode_to_mem_arm(inst);
if (ctx->target != NULL)
ctx->target[ctx->idx] = inst;
ctx->idx++;
}
/*
* Emit an instruction that will be executed unconditionally.
*/
static inline void emit(u32 inst, struct jit_ctx *ctx)
{
_emit(ARM_COND_AL, inst, ctx);
}
static u16 saved_regs(struct