1/* 2 * bpf_jit_asm64.S: Packet/header access helper functions 3 * for PPC64 BPF compiler. 4 * 5 * Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on bpf_jit_asm.S by Matt Evans 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; version 2 13 * of the License. 14 */ 15 16#include <asm/ppc_asm.h> 17#include <asm/ptrace.h> 18#include "bpf_jit64.h" 19 20/* 21 * All of these routines are called directly from generated code, 22 * with the below register usage: 23 * r27 skb pointer (ctx) 24 * r25 skb header length 25 * r26 skb->data pointer 26 * r4 offset 27 * 28 * Result is passed back in: 29 * r8 data read in host endian format (accumulator) 30 * 31 * r9 is used as a temporary register 32 */ 33 34#define r_skb r27 35#define r_hlen r25 36#define r_data r26 37#define r_off r4 38#define r_val r8 39#define r_tmp r9 40 41_GLOBAL_TOC(sk_load_word) 42 cmpdi r_off, 0 43 blt bpf_slow_path_word_neg 44 b sk_load_word_positive_offset 45 46_GLOBAL_TOC(sk_load_word_positive_offset) 47 /* Are we accessing past headlen? */ 48 subi r_tmp, r_hlen, 4 49 cmpd r_tmp, r_off 50 blt bpf_slow_path_word 51 /* Nope, just hitting the header. cr0 here is eq or gt! */ 52 LWZX_BE r_val, r_data, r_off 53 blr /* Return success, cr0 != LT */ 54 55_GLOBAL_TOC(sk_load_half) 56 cmpdi r_off, 0 57 blt bpf_slow_path_half_neg 58 b sk_load_half_positive_offset 59 60_GLOBAL_TOC(sk_load_half_positive_offset) 61 subi r_tmp, r_hlen, 2 62 cmpd r_tmp, r_off 63 blt bpf_slow_path_half 64 LHZX_BE r_val, r_data, r_off 65 blr 66 67_GLOBAL_TOC(sk_load_byte) 68 cmpdi r_off, 0 69 blt bpf_slow_path_byte_neg 70 b sk_load_byte_positive_offset 71 72_GLOBAL_TOC(sk_load_byte_positive_offset) 73 cmpd r_hlen, r_off 74 ble bpf_slow_path_byte 75 lbzx r_val, r_data, r_off 76 blr 77 78/* 79 * Call out to skb_copy_bits: 80 * Allocate a new stack frame here to remain ABI-compliant in 81 * stashing LR. 82 */ 83#define bpf_slow_path_common(SIZE) \ 84 mflr r0; \ 85 std r0, PPC_LR_STKOFF(r1); \ 86 stdu r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1); \ 87 mr r3, r_skb; \ 88 /* r4 = r_off as passed */ \ 89 addi r5, r1, STACK_FRAME_MIN_SIZE; \ 90 li r6, SIZE; \ 91 bl skb_copy_bits; \ 92 nop; \ 93 /* save r5 */ \ 94 addi r5, r1, STACK_FRAME_MIN_SIZE; \ 95 /* r3 = 0 on success */ \ 96 addi r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS; \ 97 ld r0, PPC_LR_STKOFF(r1); \ 98 mtlr r0; \ 99 cmpdi r3, 0; \ 100 blt bpf_error; /* cr0 = LT */ 101 102bpf_slow_path_word: 103 bpf_slow_path_common(4) 104 /* Data value is on stack, and cr0 != LT */ 105 LWZX_BE r_val, 0, r5 106 blr 107 108bpf_slow_path_half: 109 bpf_slow_path_common(2) 110 LHZX_BE r_val, 0, r5 111 blr 112 113bpf_slow_path_byte: 114 bpf_slow_path_common(1) 115 lbzx r_val, 0, r5 116 blr 117 118/* 119 * Call out to bpf_internal_load_pointer_neg_helper 120 */ 121#define sk_negative_common(SIZE) \ 122 mflr r0; \ 123 std r0, PPC_LR_STKOFF(r1); \ 124 stdu r1, -STACK_FRAME_MIN_SIZE(r1); \ 125 mr r3, r_skb; \ 126 /* r4 = r_off, as passed */ \ 127 li r5, SIZE; \ 128 bl bpf_internal_load_pointer_neg_helper; \ 129 nop; \ 130 addi r1, r1, STACK_FRAME_MIN_SIZE; \ 131 ld r0, PPC_LR_STKOFF(r1); \ 132 mtlr r0; \ 133 /* R3 != 0 on success */ \ 134 cmpldi r3, 0; \ 135 beq bpf_error_slow; /* cr0 = EQ */ 136 137bpf_slow_path_word_neg: 138 lis r_tmp, -32 /* SKF_LL_OFF */ 139 cmpd r_off, r_tmp /* addr < SKF_* */ 140 blt bpf_error /* cr0 = LT */ 141 b sk_load_word_negative_offset 142 143_GLOBAL_TOC(sk_load_word_negative_offset) 144 sk_negative_common(4) 145 LWZX_BE r_val, 0, r3 146 blr 147 148bpf_slow_path_half_neg: 149 lis r_tmp, -32 /* SKF_LL_OFF */ 150 cmpd r_off, r_tmp /* addr < SKF_* */ 151 blt bpf_error /* cr0 = LT */ 152 b sk_load_half_negative_offset 153 154_GLOBAL_TOC(sk_load_half_negative_offset) 155 sk_negative_common(2) 156 LHZX_BE r_val, 0, r3 157 blr 158 159bpf_slow_path_byte_neg: 160 lis r_tmp, -32 /* SKF_LL_OFF */ 161 cmpd r_off, r_tmp /* addr < SKF_* */ 162 blt bpf_error /* cr0 = LT */ 163 b sk_load_byte_negative_offset 164 165_GLOBAL_TOC(sk_load_byte_negative_offset) 166 sk_negative_common(1) 167 lbzx r_val, 0, r3 168 blr 169 170bpf_error_slow: 171 /* fabricate a cr0 = lt */ 172 li r_tmp, -1 173 cmpdi r_tmp, 0 174bpf_error: 175 /* 176 * Entered with cr0 = lt 177 * Generated code will 'blt epilogue', returning 0. 178 */ 179 li r_val, 0 180 blr 181