163 lines
3.3 KiB
ArmAsm
163 lines
3.3 KiB
ArmAsm
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||
|
|
#include <asm/ptrace.h>
|
||
|
|
|
||
|
|
#include "bpf_jit_64.h"
|
||
|
|
|
||
|
|
#define SAVE_SZ 176
|
||
|
|
#define SCRATCH_OFF STACK_BIAS + 128
|
||
|
|
#define BE_PTR(label) be,pn %xcc, label
|
||
|
|
#define SIGN_EXTEND(reg) sra reg, 0, reg
|
||
|
|
|
||
|
|
#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
|
||
|
|
|
||
|
|
.text
|
||
|
|
.globl bpf_jit_load_word
|
||
|
|
bpf_jit_load_word:
|
||
|
|
cmp r_OFF, 0
|
||
|
|
bl bpf_slow_path_word_neg
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_word_positive_offset
|
||
|
|
bpf_jit_load_word_positive_offset:
|
||
|
|
sub r_HEADLEN, r_OFF, r_TMP
|
||
|
|
cmp r_TMP, 3
|
||
|
|
ble bpf_slow_path_word
|
||
|
|
add r_SKB_DATA, r_OFF, r_TMP
|
||
|
|
andcc r_TMP, 3, %g0
|
||
|
|
bne load_word_unaligned
|
||
|
|
nop
|
||
|
|
retl
|
||
|
|
ld [r_TMP], r_RESULT
|
||
|
|
load_word_unaligned:
|
||
|
|
ldub [r_TMP + 0x0], r_OFF
|
||
|
|
ldub [r_TMP + 0x1], r_TMP2
|
||
|
|
sll r_OFF, 8, r_OFF
|
||
|
|
or r_OFF, r_TMP2, r_OFF
|
||
|
|
ldub [r_TMP + 0x2], r_TMP2
|
||
|
|
sll r_OFF, 8, r_OFF
|
||
|
|
or r_OFF, r_TMP2, r_OFF
|
||
|
|
ldub [r_TMP + 0x3], r_TMP2
|
||
|
|
sll r_OFF, 8, r_OFF
|
||
|
|
retl
|
||
|
|
or r_OFF, r_TMP2, r_RESULT
|
||
|
|
|
||
|
|
.globl bpf_jit_load_half
|
||
|
|
bpf_jit_load_half:
|
||
|
|
cmp r_OFF, 0
|
||
|
|
bl bpf_slow_path_half_neg
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_half_positive_offset
|
||
|
|
bpf_jit_load_half_positive_offset:
|
||
|
|
sub r_HEADLEN, r_OFF, r_TMP
|
||
|
|
cmp r_TMP, 1
|
||
|
|
ble bpf_slow_path_half
|
||
|
|
add r_SKB_DATA, r_OFF, r_TMP
|
||
|
|
andcc r_TMP, 1, %g0
|
||
|
|
bne load_half_unaligned
|
||
|
|
nop
|
||
|
|
retl
|
||
|
|
lduh [r_TMP], r_RESULT
|
||
|
|
load_half_unaligned:
|
||
|
|
ldub [r_TMP + 0x0], r_OFF
|
||
|
|
ldub [r_TMP + 0x1], r_TMP2
|
||
|
|
sll r_OFF, 8, r_OFF
|
||
|
|
retl
|
||
|
|
or r_OFF, r_TMP2, r_RESULT
|
||
|
|
|
||
|
|
.globl bpf_jit_load_byte
|
||
|
|
bpf_jit_load_byte:
|
||
|
|
cmp r_OFF, 0
|
||
|
|
bl bpf_slow_path_byte_neg
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_byte_positive_offset
|
||
|
|
bpf_jit_load_byte_positive_offset:
|
||
|
|
cmp r_OFF, r_HEADLEN
|
||
|
|
bge bpf_slow_path_byte
|
||
|
|
nop
|
||
|
|
retl
|
||
|
|
ldub [r_SKB_DATA + r_OFF], r_RESULT
|
||
|
|
|
||
|
|
#define bpf_slow_path_common(LEN) \
|
||
|
|
save %sp, -SAVE_SZ, %sp; \
|
||
|
|
mov %i0, %o0; \
|
||
|
|
mov %i1, %o1; \
|
||
|
|
add %fp, SCRATCH_OFF, %o2; \
|
||
|
|
call skb_copy_bits; \
|
||
|
|
mov (LEN), %o3; \
|
||
|
|
cmp %o0, 0; \
|
||
|
|
restore;
|
||
|
|
|
||
|
|
bpf_slow_path_word:
|
||
|
|
bpf_slow_path_common(4)
|
||
|
|
bl bpf_error
|
||
|
|
ld [%sp + SCRATCH_OFF], r_RESULT
|
||
|
|
retl
|
||
|
|
nop
|
||
|
|
bpf_slow_path_half:
|
||
|
|
bpf_slow_path_common(2)
|
||
|
|
bl bpf_error
|
||
|
|
lduh [%sp + SCRATCH_OFF], r_RESULT
|
||
|
|
retl
|
||
|
|
nop
|
||
|
|
bpf_slow_path_byte:
|
||
|
|
bpf_slow_path_common(1)
|
||
|
|
bl bpf_error
|
||
|
|
ldub [%sp + SCRATCH_OFF], r_RESULT
|
||
|
|
retl
|
||
|
|
nop
|
||
|
|
|
||
|
|
#define bpf_negative_common(LEN) \
|
||
|
|
save %sp, -SAVE_SZ, %sp; \
|
||
|
|
mov %i0, %o0; \
|
||
|
|
mov %i1, %o1; \
|
||
|
|
SIGN_EXTEND(%o1); \
|
||
|
|
call bpf_internal_load_pointer_neg_helper; \
|
||
|
|
mov (LEN), %o2; \
|
||
|
|
mov %o0, r_TMP; \
|
||
|
|
cmp %o0, 0; \
|
||
|
|
BE_PTR(bpf_error); \
|
||
|
|
restore;
|
||
|
|
|
||
|
|
bpf_slow_path_word_neg:
|
||
|
|
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||
|
|
cmp r_OFF, r_TMP
|
||
|
|
bl bpf_error
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_word_negative_offset
|
||
|
|
bpf_jit_load_word_negative_offset:
|
||
|
|
bpf_negative_common(4)
|
||
|
|
andcc r_TMP, 3, %g0
|
||
|
|
bne load_word_unaligned
|
||
|
|
nop
|
||
|
|
retl
|
||
|
|
ld [r_TMP], r_RESULT
|
||
|
|
|
||
|
|
bpf_slow_path_half_neg:
|
||
|
|
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||
|
|
cmp r_OFF, r_TMP
|
||
|
|
bl bpf_error
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_half_negative_offset
|
||
|
|
bpf_jit_load_half_negative_offset:
|
||
|
|
bpf_negative_common(2)
|
||
|
|
andcc r_TMP, 1, %g0
|
||
|
|
bne load_half_unaligned
|
||
|
|
nop
|
||
|
|
retl
|
||
|
|
lduh [r_TMP], r_RESULT
|
||
|
|
|
||
|
|
bpf_slow_path_byte_neg:
|
||
|
|
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
|
||
|
|
cmp r_OFF, r_TMP
|
||
|
|
bl bpf_error
|
||
|
|
nop
|
||
|
|
.globl bpf_jit_load_byte_negative_offset
|
||
|
|
bpf_jit_load_byte_negative_offset:
|
||
|
|
bpf_negative_common(1)
|
||
|
|
retl
|
||
|
|
ldub [r_TMP], r_RESULT
|
||
|
|
|
||
|
|
bpf_error:
|
||
|
|
/* Make the JIT program itself return zero. */
|
||
|
|
ret
|
||
|
|
restore %g0, %g0, %o0
|