1 /*
2 * Copyright (c) 2017 Imagination Technologies.
3 *
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in the documentation and/or other materials provided with
15 * the distribution.
16 * * Neither the name of Imagination Technologies nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32 #include <string.h>
33
34 #define ENABLE_PREFETCH 1
35 #define op_t unsigned long int
36 #define op_size sizeof (op_t)
37
38 #if ENABLE_PREFETCH
39 #define PREFETCH(addr) __builtin_prefetch (addr, 0, 1);
40 #else
41 #define PREFETCH(addr)
42 #endif
43
44 #if __mips64 || __mips_isa_rev >= 2
45 static inline void * __attribute__ ((always_inline))
do_bytes(const op_t * w,op_t inval)46 do_bytes (const op_t* w, op_t inval)
47 {
48 const unsigned char *p = (const unsigned char *) w;
49 op_t outval = 0;
50 #if __mips64
51 __asm__ volatile (
52 "dsbh %1, %0 \n\t"
53 "dshd %0, %1 \n\t"
54 "dclz %1, %0 \n\t"
55 : "+r" (inval), "+r" (outval)
56 );
57 #else
58 __asm__ volatile (
59 "wsbh %1, %0 \n\t"
60 "rotr %0, %1, 16 \n\t"
61 "clz %1, %0 \n\t"
62 : "+r" (inval), "+r" (outval)
63 );
64 #endif
65 p += (outval >> 3);
66 return (void *) p;
67 }
68
69 #define DO_WORD(in, val) { \
70 op_t tmp = ((val - mask_1) & ~val) & mask_128; \
71 if (tmp != 0) \
72 return do_bytes(in, tmp); \
73 }
74 #else
75 static inline void * __attribute__ ((always_inline))
do_bytes(const op_t * w,unsigned char ch)76 do_bytes (const op_t* w, unsigned char ch)
77 {
78 const unsigned char *p = (const unsigned char *) w;
79 for (; *p != ch; ++p);
80 return (void *) p;
81 }
82
83 #define DO_WORD(in, val) { \
84 op_t tmp = ((val - mask_1) & ~val) & mask_128; \
85 if (tmp != 0) \
86 return do_bytes(in, ch); \
87 }
88 #endif
89
90 #define DO_WORDS(w) { \
91 op_t* w1 = (op_t*) w; \
92 op_t val0 = w1[0] ^ mask_c; \
93 op_t val1 = w1[1] ^ mask_c; \
94 op_t val2 = w1[2] ^ mask_c; \
95 op_t val3 = w1[3] ^ mask_c; \
96 DO_WORD(w1, val0) \
97 DO_WORD(w1 + 1, val1) \
98 DO_WORD(w1 + 2, val2) \
99 DO_WORD(w1 + 3, val3) \
100 }
101
102 void *
memchr(void const * s,int c_in,size_t n)103 memchr (void const *s, int c_in, size_t n) __overloadable
104 {
105 if (n != 0) {
106 const unsigned char *p = (const unsigned char *) s;
107 const op_t *w;
108 op_t mask_1, mask_128, mask_c;
109 unsigned char ch = (unsigned char) c_in;
110
111 /*
112 * Check bytewize till initial alignment
113 */
114 for (; n > 0 && ((size_t) p % op_size) != 0; --n, ++p) {
115 if (*p == ch)
116 return (void *) p;
117 }
118
119 w = (const op_t *) p;
120
121 mask_c = ch | (ch << 8);
122 mask_c |= mask_c << 16;
123 __asm__ volatile (
124 "li %0, 0x01010101 \n\t"
125 : "=r" (mask_1)
126 );
127 #if __mips64
128 mask_1 |= mask_1 << 32;
129 mask_c |= mask_c << 32;
130 #endif
131 mask_128 = mask_1 << 7;
132
133 /*
134 * Check op_size byteswize after initial alignment
135 */
136 #if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
137 PREFETCH (w);
138 PREFETCH (w + 8);
139 while (n >= 24 * op_size) {
140 PREFETCH(w + 16);
141 DO_WORDS(w);
142 DO_WORDS(w + 4);
143 w += 8;
144 n -= 8 * op_size;
145 }
146 while (n >= 8 * op_size) {
147 DO_WORDS(w);
148 DO_WORDS(w + 4);
149 w += 8;
150 n -= 8 * op_size;
151 }
152 #else
153 PREFETCH (w);
154 PREFETCH (w + 4);
155 while (n >= 12 * op_size) {
156 PREFETCH(w + 8);
157 DO_WORDS(w);
158 w += 4;
159 n -= 4 * op_size;
160 }
161 while (n >= 4 * op_size) {
162 DO_WORDS(w);
163 w += 4;
164 n -= 4 * op_size;
165 }
166 #endif
167
168 while (n >= op_size) {
169 op_t val = *w ^ mask_c;
170 DO_WORD(w, val);
171 w++;
172 n -= op_size;
173 }
174
175 /*
176 * Check bytewize for remaining bytes
177 */
178 p = (const unsigned char *) w;
179 for (; n > 0; --n, ++p) {
180 if (*p == ch)
181 return (void *) p;
182 }
183 }
184 return NULL;
185 }
186