1#include <asm/ptrace.h> 2 3#include "bpf_jit.h" 4 5#ifdef CONFIG_SPARC64 6#define SAVE_SZ 176 7#define SCRATCH_OFF STACK_BIAS + 128 8#define BE_PTR(label) be,pn %xcc, label 9#else 10#define SAVE_SZ 96 11#define SCRATCH_OFF 72 12#define BE_PTR(label) be label 13#endif 14 15#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ 16 17 .text 18 .globl bpf_jit_load_word 19bpf_jit_load_word: 20 cmp r_OFF, 0 21 bl bpf_slow_path_word_neg 22 nop 23 .globl bpf_jit_load_word_positive_offset 24bpf_jit_load_word_positive_offset: 25 sub r_HEADLEN, r_OFF, r_TMP 26 cmp r_TMP, 3 27 ble bpf_slow_path_word 28 add r_SKB_DATA, r_OFF, r_TMP 29 andcc r_TMP, 3, %g0 30 bne load_word_unaligned 31 nop 32 retl 33 ld [r_TMP], r_A 34load_word_unaligned: 35 ldub [r_TMP + 0x0], r_OFF 36 ldub [r_TMP + 0x1], r_TMP2 37 sll r_OFF, 8, r_OFF 38 or r_OFF, r_TMP2, r_OFF 39 ldub [r_TMP + 0x2], r_TMP2 40 sll r_OFF, 8, r_OFF 41 or r_OFF, r_TMP2, r_OFF 42 ldub [r_TMP + 0x3], r_TMP2 43 sll r_OFF, 8, r_OFF 44 retl 45 or r_OFF, r_TMP2, r_A 46 47 .globl bpf_jit_load_half 48bpf_jit_load_half: 49 cmp r_OFF, 0 50 bl bpf_slow_path_half_neg 51 nop 52 .globl bpf_jit_load_half_positive_offset 53bpf_jit_load_half_positive_offset: 54 sub r_HEADLEN, r_OFF, r_TMP 55 cmp r_TMP, 1 56 ble bpf_slow_path_half 57 add r_SKB_DATA, r_OFF, r_TMP 58 andcc r_TMP, 1, %g0 59 bne load_half_unaligned 60 nop 61 retl 62 lduh [r_TMP], r_A 63load_half_unaligned: 64 ldub [r_TMP + 0x0], r_OFF 65 ldub [r_TMP + 0x1], r_TMP2 66 sll r_OFF, 8, r_OFF 67 retl 68 or r_OFF, r_TMP2, r_A 69 70 .globl bpf_jit_load_byte 71bpf_jit_load_byte: 72 cmp r_OFF, 0 73 bl bpf_slow_path_byte_neg 74 nop 75 .globl bpf_jit_load_byte_positive_offset 76bpf_jit_load_byte_positive_offset: 77 cmp r_OFF, r_HEADLEN 78 bge bpf_slow_path_byte 79 nop 80 retl 81 ldub [r_SKB_DATA + r_OFF], r_A 82 83 .globl bpf_jit_load_byte_msh 84bpf_jit_load_byte_msh: 85 cmp r_OFF, 0 86 bl bpf_slow_path_byte_msh_neg 87 nop 88 .globl bpf_jit_load_byte_msh_positive_offset 89bpf_jit_load_byte_msh_positive_offset: 90 cmp r_OFF, r_HEADLEN 91 bge bpf_slow_path_byte_msh 92 nop 93 ldub [r_SKB_DATA + r_OFF], r_OFF 94 and r_OFF, 0xf, r_OFF 95 retl 96 sll r_OFF, 2, r_X 97 98#define bpf_slow_path_common(LEN) \ 99 save %sp, -SAVE_SZ, %sp; \ 100 mov %i0, %o0; \ 101 mov r_OFF, %o1; \ 102 add %fp, SCRATCH_OFF, %o2; \ 103 call skb_copy_bits; \ 104 mov (LEN), %o3; \ 105 cmp %o0, 0; \ 106 restore; 107 108bpf_slow_path_word: 109 bpf_slow_path_common(4) 110 bl bpf_error 111 ld [%sp + SCRATCH_OFF], r_A 112 retl 113 nop 114bpf_slow_path_half: 115 bpf_slow_path_common(2) 116 bl bpf_error 117 lduh [%sp + SCRATCH_OFF], r_A 118 retl 119 nop 120bpf_slow_path_byte: 121 bpf_slow_path_common(1) 122 bl bpf_error 123 ldub [%sp + SCRATCH_OFF], r_A 124 retl 125 nop 126bpf_slow_path_byte_msh: 127 bpf_slow_path_common(1) 128 bl bpf_error 129 ldub [%sp + SCRATCH_OFF], r_A 130 and r_OFF, 0xf, r_OFF 131 retl 132 sll r_OFF, 2, r_X 133 134#define bpf_negative_common(LEN) \ 135 save %sp, -SAVE_SZ, %sp; \ 136 mov %i0, %o0; \ 137 mov r_OFF, %o1; \ 138 call bpf_internal_load_pointer_neg_helper; \ 139 mov (LEN), %o2; \ 140 mov %o0, r_TMP; \ 141 cmp %o0, 0; \ 142 BE_PTR(bpf_error); \ 143 restore; 144 145bpf_slow_path_word_neg: 146 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 147 cmp r_OFF, r_TMP 148 bl bpf_error 149 nop 150 .globl bpf_jit_load_word_negative_offset 151bpf_jit_load_word_negative_offset: 152 bpf_negative_common(4) 153 andcc r_TMP, 3, %g0 154 bne load_word_unaligned 155 nop 156 retl 157 ld [r_TMP], r_A 158 159bpf_slow_path_half_neg: 160 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 161 cmp r_OFF, r_TMP 162 bl bpf_error 163 nop 164 .globl bpf_jit_load_half_negative_offset 165bpf_jit_load_half_negative_offset: 166 bpf_negative_common(2) 167 andcc r_TMP, 1, %g0 168 bne load_half_unaligned 169 nop 170 retl 171 lduh [r_TMP], r_A 172 173bpf_slow_path_byte_neg: 174 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 175 cmp r_OFF, r_TMP 176 bl bpf_error 177 nop 178 .globl bpf_jit_load_byte_negative_offset 179bpf_jit_load_byte_negative_offset: 180 bpf_negative_common(1) 181 retl 182 ldub [r_TMP], r_A 183 184bpf_slow_path_byte_msh_neg: 185 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 186 cmp r_OFF, r_TMP 187 bl bpf_error 188 nop 189 .globl bpf_jit_load_byte_msh_negative_offset 190bpf_jit_load_byte_msh_negative_offset: 191 bpf_negative_common(1) 192 ldub [r_TMP], r_OFF 193 and r_OFF, 0xf, r_OFF 194 retl 195 sll r_OFF, 2, r_X 196 197bpf_error: 198 /* Make the JIT program return zero. The JIT epilogue 199 * stores away the original %o7 into r_saved_O7. The 200 * normal leaf function return is to use "retl" which 201 * would evalute to "jmpl %o7 + 8, %g0" but we want to 202 * use the saved value thus the sequence you see here. 203 */ 204 jmpl r_saved_O7 + 8, %g0 205 clr %o0 206