1/* 2 * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF 3 * compiler. 4 * 5 * Copyright (C) 2015 Imagination Technologies Ltd. 6 * Author: Markos Chandras <markos.chandras@imgtec.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the 10 * Free Software Foundation; version 2 of the License. 11 */ 12 13#include <asm/asm.h> 14#include <asm/isa-rev.h> 15#include <asm/regdef.h> 16#include "bpf_jit.h" 17 18/* ABI 19 * 20 * r_skb_hl skb header length 21 * r_skb_data skb data 22 * r_off(a1) offset register 23 * r_A BPF register A 24 * r_X PF register X 25 * r_skb(a0) *skb 26 * r_M *scratch memory 27 * r_skb_le skb length 28 * r_s0 Scratch register 0 29 * r_s1 Scratch register 1 30 * 31 * On entry: 32 * a0: *skb 33 * a1: offset (imm or imm + X) 34 * 35 * All non-BPF-ABI registers are free for use. On return, we only 36 * care about r_ret. The BPF-ABI registers are assumed to remain 37 * unmodified during the entire filter operation. 38 */ 39 40#define skb a0 41#define offset a1 42#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */ 43 44 /* We know better :) so prevent assembler reordering etc */ 45 .set noreorder 46 47#define is_offset_negative(TYPE) \ 48 /* If offset is negative we have more work to do */ \ 49 slti t0, offset, 0; \ 50 bgtz t0, bpf_slow_path_##TYPE##_neg; \ 51 /* Be careful what follows in DS. */ 52 53#define is_offset_in_header(SIZE, TYPE) \ 54 /* Reading from header? */ \ 55 addiu $r_s0, $r_skb_hl, -SIZE; \ 56 slt t0, $r_s0, offset; \ 57 bgtz t0, bpf_slow_path_##TYPE; \ 58 59LEAF(sk_load_word) 60 is_offset_negative(word) 61FEXPORT(sk_load_word_positive) 62 is_offset_in_header(4, word) 63 /* Offset within header boundaries */ 64 PTR_ADDU t1, $r_skb_data, offset 65 .set reorder 66 lw $r_A, 0(t1) 67 .set noreorder 68#ifdef CONFIG_CPU_LITTLE_ENDIAN 69# if MIPS_ISA_REV >= 2 70 wsbh t0, $r_A 71 rotr $r_A, t0, 16 72# else 73 sll t0, $r_A, 24 74 srl t1, $r_A, 24 75 srl t2, $r_A, 8 76 or t0, t0, t1 77 andi t2, t2, 0xff00 78 andi t1, $r_A, 0xff00 79 or t0, t0, t2 80 sll t1, t1, 8 81 or $r_A, t0, t1 82# endif 83#endif 84 jr $r_ra 85 move $r_ret, zero 86 END(sk_load_word) 87 88LEAF(sk_load_half) 89 is_offset_negative(half) 90FEXPORT(sk_load_half_positive) 91 is_offset_in_header(2, half) 92 /* Offset within header boundaries */ 93 PTR_ADDU t1, $r_skb_data, offset 94 lhu $r_A, 0(t1) 95#ifdef CONFIG_CPU_LITTLE_ENDIAN 96# if MIPS_ISA_REV >= 2 97 wsbh $r_A, $r_A 98# else 99 sll t0, $r_A, 8 100 srl t1, $r_A, 8 101 andi t0, t0, 0xff00 102 or $r_A, t0, t1 103# endif 104#endif 105 jr $r_ra 106 move $r_ret, zero 107 END(sk_load_half) 108 109LEAF(sk_load_byte) 110 is_offset_negative(byte) 111FEXPORT(sk_load_byte_positive) 112 is_offset_in_header(1, byte) 113 /* Offset within header boundaries */ 114 PTR_ADDU t1, $r_skb_data, offset 115 lbu $r_A, 0(t1) 116 jr $r_ra 117 move $r_ret, zero 118 END(sk_load_byte) 119 120/* 121 * call skb_copy_bits: 122 * (prototype in linux/skbuff.h) 123 * 124 * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len) 125 * 126 * o32 mandates we leave 4 spaces for argument registers in case 127 * the callee needs to use them. Even though we don't care about 128 * the argument registers ourselves, we need to allocate that space 129 * to remain ABI compliant since the callee may want to use that space. 130 * We also allocate 2 more spaces for $r_ra and our return register (*to). 131 * 132 * n64 is a bit different. The *caller* will allocate the space to preserve 133 * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no 134 * good reason but it does not matter that much really. 135 * 136 * (void *to) is returned in r_s0 137 * 138 */ 139#ifdef CONFIG_CPU_LITTLE_ENDIAN 140#define DS_OFFSET(SIZE) (4 * SZREG) 141#else 142#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE)) 143#endif 144#define bpf_slow_path_common(SIZE) \ 145 /* Quick check. Are we within reasonable boundaries? */ \ 146 LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \ 147 sltu $r_s0, offset, $r_s1; \ 148 beqz $r_s0, fault; \ 149 /* Load 4th argument in DS */ \ 150 LONG_ADDIU a3, zero, SIZE; \ 151 PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ 152 PTR_LA t0, skb_copy_bits; \ 153 PTR_S $r_ra, (5 * SZREG)($r_sp); \ 154 /* Assign low slot to a2 */ \ 155 PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \ 156 jalr t0; \ 157 /* Reset our destination slot (DS but it's ok) */ \ 158 INT_S zero, (4 * SZREG)($r_sp); \ 159 /* \ 160 * skb_copy_bits returns 0 on success and -EFAULT \ 161 * on error. Our data live in a2. Do not bother with \ 162 * our data if an error has been returned. \ 163 */ \ 164 /* Restore our frame */ \ 165 PTR_L $r_ra, (5 * SZREG)($r_sp); \ 166 INT_L $r_s0, (4 * SZREG)($r_sp); \ 167 bltz v0, fault; \ 168 PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ 169 move $r_ret, zero; \ 170 171NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp) 172 bpf_slow_path_common(4) 173#ifdef CONFIG_CPU_LITTLE_ENDIAN 174# if MIPS_ISA_REV >= 2 175 wsbh t0, $r_s0 176 jr $r_ra 177 rotr $r_A, t0, 16 178# else 179 sll t0, $r_s0, 24 180 srl t1, $r_s0, 24 181 srl t2, $r_s0, 8 182 or t0, t0, t1 183 andi t2, t2, 0xff00 184 andi t1, $r_s0, 0xff00 185 or t0, t0, t2 186 sll t1, t1, 8 187 jr $r_ra 188 or $r_A, t0, t1 189# endif 190#else 191 jr $r_ra 192 move $r_A, $r_s0 193#endif 194 195 END(bpf_slow_path_word) 196 197NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp) 198 bpf_slow_path_common(2) 199#ifdef CONFIG_CPU_LITTLE_ENDIAN 200# if MIPS_ISA_REV >= 2 201 jr $r_ra 202 wsbh $r_A, $r_s0 203# else 204 sll t0, $r_s0, 8 205 andi t1, $r_s0, 0xff00 206 andi t0, t0, 0xff00 207 srl t1, t1, 8 208 jr $r_ra 209 or $r_A, t0, t1 210# endif 211#else 212 jr $r_ra 213 move $r_A, $r_s0 214#endif 215 216 END(bpf_slow_path_half) 217 218NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp) 219 bpf_slow_path_common(1) 220 jr $r_ra 221 move $r_A, $r_s0 222 223 END(bpf_slow_path_byte) 224 225/* 226 * Negative entry points 227 */ 228 .macro bpf_is_end_of_data 229 li t0, SKF_LL_OFF 230 /* Reading link layer data? */ 231 slt t1, offset, t0 232 bgtz t1, fault 233 /* Be careful what follows in DS. */ 234 .endm 235/* 236 * call skb_copy_bits: 237 * (prototype in linux/filter.h) 238 * 239 * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, 240 * int k, unsigned int size) 241 * 242 * see above (bpf_slow_path_common) for ABI restrictions 243 */ 244#define bpf_negative_common(SIZE) \ 245 PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ 246 PTR_LA t0, bpf_internal_load_pointer_neg_helper; \ 247 PTR_S $r_ra, (5 * SZREG)($r_sp); \ 248 jalr t0; \ 249 li a2, SIZE; \ 250 PTR_L $r_ra, (5 * SZREG)($r_sp); \ 251 /* Check return pointer */ \ 252 beqz v0, fault; \ 253 PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ 254 /* Preserve our pointer */ \ 255 move $r_s0, v0; \ 256 /* Set return value */ \ 257 move $r_ret, zero; \ 258 259bpf_slow_path_word_neg: 260 bpf_is_end_of_data 261NESTED(sk_load_word_negative, (6 * SZREG), $r_sp) 262 bpf_negative_common(4) 263 jr $r_ra 264 lw $r_A, 0($r_s0) 265 END(sk_load_word_negative) 266 267bpf_slow_path_half_neg: 268 bpf_is_end_of_data 269NESTED(sk_load_half_negative, (6 * SZREG), $r_sp) 270 bpf_negative_common(2) 271 jr $r_ra 272 lhu $r_A, 0($r_s0) 273 END(sk_load_half_negative) 274 275bpf_slow_path_byte_neg: 276 bpf_is_end_of_data 277NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp) 278 bpf_negative_common(1) 279 jr $r_ra 280 lbu $r_A, 0($r_s0) 281 END(sk_load_byte_negative) 282 283fault: 284 jr $r_ra 285 addiu $r_ret, zero, 1 286