1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/lib/string.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8 /*
9 * This file should be used only for "library" routines that may have
10 * alternative implementations on specific architectures (generally
11 * found in <asm-xx/string.h>), or get overloaded by FORTIFY_SOURCE.
12 * (Specifically, this file is built with __NO_FORTIFY.)
13 *
14 * Other helper functions should live in string_helpers.c.
15 */
16
17 #define __NO_FORTIFY
18 #include <linux/bits.h>
19 #include <linux/bug.h>
20 #include <linux/ctype.h>
21 #include <linux/errno.h>
22 #include <linux/limits.h>
23 #include <linux/linkage.h>
24 #include <linux/stddef.h>
25 #include <linux/string.h>
26 #include <linux/types.h>
27
28 #include <asm/page.h>
29 #include <asm/rwonce.h>
30 #include <linux/unaligned.h>
31 #include <asm/word-at-a-time.h>
32
33 #ifndef __HAVE_ARCH_STRNCASECMP
34 /**
35 * strncasecmp - Case insensitive, length-limited string comparison
36 * @s1: One string
37 * @s2: The other string
38 * @len: the maximum number of characters to compare
39 */
strncasecmp(const char * s1,const char * s2,size_t len)40 int strncasecmp(const char *s1, const char *s2, size_t len)
41 {
42 /* Yes, Virginia, it had better be unsigned */
43 unsigned char c1, c2;
44
45 if (!len)
46 return 0;
47
48 do {
49 c1 = *s1++;
50 c2 = *s2++;
51 if (!c1 || !c2)
52 break;
53 if (c1 == c2)
54 continue;
55 c1 = tolower(c1);
56 c2 = tolower(c2);
57 if (c1 != c2)
58 break;
59 } while (--len);
60 return (int)c1 - (int)c2;
61 }
62 EXPORT_SYMBOL(strncasecmp);
63 #endif
64
65 #ifndef __HAVE_ARCH_STRCASECMP
strcasecmp(const char * s1,const char * s2)66 int strcasecmp(const char *s1, const char *s2)
67 {
68 int c1, c2;
69
70 do {
71 c1 = tolower(*s1++);
72 c2 = tolower(*s2++);
73 } while (c1 == c2 && c1 != 0);
74 return c1 - c2;
75 }
76 EXPORT_SYMBOL(strcasecmp);
77 #endif
78
79 #ifndef __HAVE_ARCH_STRCPY
strcpy(char * dest,const char * src)80 char *strcpy(char *dest, const char *src)
81 {
82 char *tmp = dest;
83
84 while ((*dest++ = *src++) != '\0')
85 /* nothing */;
86 return tmp;
87 }
88 EXPORT_SYMBOL(strcpy);
89 #endif
90
91 #ifndef __HAVE_ARCH_STRNCPY
strncpy(char * dest,const char * src,size_t count)92 char *strncpy(char *dest, const char *src, size_t count)
93 {
94 char *tmp = dest;
95
96 while (count) {
97 if ((*tmp = *src) != 0)
98 src++;
99 tmp++;
100 count--;
101 }
102 return dest;
103 }
104 EXPORT_SYMBOL(strncpy);
105 #endif
106
sized_strscpy(char * dest,const char * src,size_t count)107 ssize_t sized_strscpy(char *dest, const char *src, size_t count)
108 {
109 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
110 size_t max = count;
111 long res = 0;
112
113 if (count == 0 || WARN_ON_ONCE(count > INT_MAX))
114 return -E2BIG;
115
116 #ifndef CONFIG_DCACHE_WORD_ACCESS
117 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
118 /*
119 * If src is unaligned, don't cross a page boundary,
120 * since we don't know if the next page is mapped.
121 */
122 if ((long)src & (sizeof(long) - 1)) {
123 size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1));
124 if (limit < max)
125 max = limit;
126 }
127 #else
128 /* If src or dest is unaligned, don't do word-at-a-time. */
129 if (((long) dest | (long) src) & (sizeof(long) - 1))
130 max = 0;
131 #endif
132 #endif
133
134 /*
135 * load_unaligned_zeropad() or read_word_at_a_time() below may read
136 * uninitialized bytes after the trailing zero and use them in
137 * comparisons. Disable this optimization under KMSAN to prevent
138 * false positive reports.
139 */
140 if (IS_ENABLED(CONFIG_KMSAN))
141 max = 0;
142
143 while (max >= sizeof(unsigned long)) {
144 unsigned long c, data;
145
146 #ifdef CONFIG_DCACHE_WORD_ACCESS
147 c = load_unaligned_zeropad(src+res);
148 #else
149 c = read_word_at_a_time(src+res);
150 #endif
151 if (has_zero(c, &data, &constants)) {
152 data = prep_zero_mask(c, data, &constants);
153 data = create_zero_mask(data);
154 *(unsigned long *)(dest+res) = c & zero_bytemask(data);
155 return res + find_zero(data);
156 }
157 *(unsigned long *)(dest+res) = c;
158 res += sizeof(unsigned long);
159 count -= sizeof(unsigned long);
160 max -= sizeof(unsigned long);
161 }
162
163 while (count) {
164 char c;
165
166 c = src[res];
167 dest[res] = c;
168 if (!c)
169 return res;
170 res++;
171 count--;
172 }
173
174 /* Hit buffer length without finding a NUL; force NUL-termination. */
175 if (res)
176 dest[res-1] = '\0';
177
178 return -E2BIG;
179 }
180 EXPORT_SYMBOL(sized_strscpy);
181
182 /**
183 * stpcpy - copy a string from src to dest returning a pointer to the new end
184 * of dest, including src's %NUL-terminator. May overrun dest.
185 * @dest: pointer to end of string being copied into. Must be large enough
186 * to receive copy.
187 * @src: pointer to the beginning of string being copied from. Must not overlap
188 * dest.
189 *
190 * stpcpy differs from strcpy in a key way: the return value is a pointer
191 * to the new %NUL-terminating character in @dest. (For strcpy, the return
192 * value is a pointer to the start of @dest). This interface is considered
193 * unsafe as it doesn't perform bounds checking of the inputs. As such it's
194 * not recommended for usage. Instead, its definition is provided in case
195 * the compiler lowers other libcalls to stpcpy.
196 */
197 char *stpcpy(char *__restrict__ dest, const char *__restrict__ src);
stpcpy(char * __restrict__ dest,const char * __restrict__ src)198 char *stpcpy(char *__restrict__ dest, const char *__restrict__ src)
199 {
200 while ((*dest++ = *src++) != '\0')
201 /* nothing */;
202 return --dest;
203 }
204 EXPORT_SYMBOL(stpcpy);
205
206 #ifndef __HAVE_ARCH_STRCAT
strcat(char * dest,const char * src)207 char *strcat(char *dest, const char *src)
208 {
209 char *tmp = dest;
210
211 while (*dest)
212 dest++;
213 while ((*dest++ = *src++) != '\0')
214 ;
215 return tmp;
216 }
217 EXPORT_SYMBOL(strcat);
218 #endif
219
220 #ifndef __HAVE_ARCH_STRNCAT
strncat(char * dest,const char * src,size_t count)221 char *strncat(char *dest, const char *src, size_t count)
222 {
223 char *tmp = dest;
224
225 if (count) {
226 while (*dest)
227 dest++;
228 while ((*dest++ = *src++) != 0) {
229 if (--count == 0) {
230 *dest = '\0';
231 break;
232 }
233 }
234 }
235 return tmp;
236 }
237 EXPORT_SYMBOL(strncat);
238 #endif
239
240 #ifndef __HAVE_ARCH_STRLCAT
strlcat(char * dest,const char * src,size_t count)241 size_t strlcat(char *dest, const char *src, size_t count)
242 {
243 size_t dsize = strlen(dest);
244 size_t len = strlen(src);
245 size_t res = dsize + len;
246
247 /* This would be a bug */
248 BUG_ON(dsize >= count);
249
250 dest += dsize;
251 count -= dsize;
252 if (len >= count)
253 len = count-1;
254 __builtin_memcpy(dest, src, len);
255 dest[len] = 0;
256 return res;
257 }
258 EXPORT_SYMBOL(strlcat);
259 #endif
260
261 #ifndef __HAVE_ARCH_STRCMP
262 /**
263 * strcmp - Compare two strings
264 * @cs: One string
265 * @ct: Another string
266 */
strcmp(const char * cs,const char * ct)267 int strcmp(const char *cs, const char *ct)
268 {
269 unsigned char c1, c2;
270
271 while (1) {
272 c1 = *cs++;
273 c2 = *ct++;
274 if (c1 != c2)
275 return c1 < c2 ? -1 : 1;
276 if (!c1)
277 break;
278 }
279 return 0;
280 }
281 EXPORT_SYMBOL(strcmp);
282 #endif
283
284 #ifndef __HAVE_ARCH_STRNCMP
285 /**
286 * strncmp - Compare two length-limited strings
287 * @cs: One string
288 * @ct: Another string
289 * @count: The maximum number of bytes to compare
290 */
strncmp(const char * cs,const char * ct,size_t count)291 int strncmp(const char *cs, const char *ct, size_t count)
292 {
293 unsigned char c1, c2;
294
295 while (count) {
296 c1 = *cs++;
297 c2 = *ct++;
298 if (c1 != c2)
299 return c1 < c2 ? -1 : 1;
300 if (!c1)
301 break;
302 count--;
303 }
304 return 0;
305 }
306 EXPORT_SYMBOL(strncmp);
307 #endif
308
309 #ifndef __HAVE_ARCH_STRCHR
310 /**
311 * strchr - Find the first occurrence of a character in a string
312 * @s: The string to be searched
313 * @c: The character to search for
314 *
315 * Note that the %NUL-terminator is considered part of the string, and can
316 * be searched for.
317 */
strchr(const char * s,int c)318 char *strchr(const char *s, int c)
319 {
320 for (; *s != (char)c; ++s)
321 if (*s == '\0')
322 return NULL;
323 return (char *)s;
324 }
325 EXPORT_SYMBOL(strchr);
326 #endif
327
328 #ifndef __HAVE_ARCH_STRCHRNUL
329 /**
330 * strchrnul - Find and return a character in a string, or end of string
331 * @s: The string to be searched
332 * @c: The character to search for
333 *
334 * Returns pointer to first occurrence of 'c' in s. If c is not found, then
335 * return a pointer to the null byte at the end of s.
336 */
strchrnul(const char * s,int c)337 char *strchrnul(const char *s, int c)
338 {
339 while (*s && *s != (char)c)
340 s++;
341 return (char *)s;
342 }
343 EXPORT_SYMBOL(strchrnul);
344 #endif
345
346 /**
347 * strnchrnul - Find and return a character in a length limited string,
348 * or end of string
349 * @s: The string to be searched
350 * @count: The number of characters to be searched
351 * @c: The character to search for
352 *
353 * Returns pointer to the first occurrence of 'c' in s. If c is not found,
354 * then return a pointer to the last character of the string.
355 */
strnchrnul(const char * s,size_t count,int c)356 char *strnchrnul(const char *s, size_t count, int c)
357 {
358 while (count-- && *s && *s != (char)c)
359 s++;
360 return (char *)s;
361 }
362
363 #ifndef __HAVE_ARCH_STRRCHR
364 /**
365 * strrchr - Find the last occurrence of a character in a string
366 * @s: The string to be searched
367 * @c: The character to search for
368 */
strrchr(const char * s,int c)369 char *strrchr(const char *s, int c)
370 {
371 const char *last = NULL;
372 do {
373 if (*s == (char)c)
374 last = s;
375 } while (*s++);
376 return (char *)last;
377 }
378 EXPORT_SYMBOL(strrchr);
379 #endif
380
381 #ifndef __HAVE_ARCH_STRNCHR
382 /**
383 * strnchr - Find a character in a length limited string
384 * @s: The string to be searched
385 * @count: The number of characters to be searched
386 * @c: The character to search for
387 *
388 * Note that the %NUL-terminator is considered part of the string, and can
389 * be searched for.
390 */
strnchr(const char * s,size_t count,int c)391 char *strnchr(const char *s, size_t count, int c)
392 {
393 while (count--) {
394 if (*s == (char)c)
395 return (char *)s;
396 if (*s++ == '\0')
397 break;
398 }
399 return NULL;
400 }
401 EXPORT_SYMBOL(strnchr);
402 #endif
403
404 #ifndef __HAVE_ARCH_STRLEN
strlen(const char * s)405 size_t strlen(const char *s)
406 {
407 const char *sc;
408
409 for (sc = s; *sc != '\0'; ++sc)
410 /* nothing */;
411 return sc - s;
412 }
413 EXPORT_SYMBOL(strlen);
414 #endif
415
416 #ifndef __HAVE_ARCH_STRNLEN
strnlen(const char * s,size_t count)417 size_t strnlen(const char *s, size_t count)
418 {
419 const char *sc;
420
421 for (sc = s; count-- && *sc != '\0'; ++sc)
422 /* nothing */;
423 return sc - s;
424 }
425 EXPORT_SYMBOL(strnlen);
426 #endif
427
428 #ifndef __HAVE_ARCH_STRSPN
429 /**
430 * strspn - Calculate the length of the initial substring of @s which only contain letters in @accept
431 * @s: The string to be searched
432 * @accept: The string to search for
433 */
strspn(const char * s,const char * accept)434 size_t strspn(const char *s, const char *accept)
435 {
436 const char *p;
437
438 for (p = s; *p != '\0'; ++p) {
439 if (!strchr(accept, *p))
440 break;
441 }
442 return p - s;
443 }
444 EXPORT_SYMBOL(strspn);
445 #endif
446
447 #ifndef __HAVE_ARCH_STRCSPN
448 /**
449 * strcspn - Calculate the length of the initial substring of @s which does not contain letters in @reject
450 * @s: The string to be searched
451 * @reject: The string to avoid
452 */
strcspn(const char * s,const char * reject)453 size_t strcspn(const char *s, const char *reject)
454 {
455 const char *p;
456
457 for (p = s; *p != '\0'; ++p) {
458 if (strchr(reject, *p))
459 break;
460 }
461 return p - s;
462 }
463 EXPORT_SYMBOL(strcspn);
464 #endif
465
466 #ifndef __HAVE_ARCH_STRPBRK
467 /**
468 * strpbrk - Find the first occurrence of a set of characters
469 * @cs: The string to be searched
470 * @ct: The characters to search for
471 */
strpbrk(const char * cs,const char * ct)472 char *strpbrk(const char *cs, const char *ct)
473 {
474 const char *sc;
475
476 for (sc = cs; *sc != '\0'; ++sc) {
477 if (strchr(ct, *sc))
478 return (char *)sc;
479 }
480 return NULL;
481 }
482 EXPORT_SYMBOL(strpbrk);
483 #endif
484
485 #ifndef __HAVE_ARCH_STRSEP
486 /**
487 * strsep - Split a string into tokens
488 * @s: The string to be searched
489 * @ct: The characters to search for
490 *
491 * strsep() updates @s to point after the token, ready for the next call.
492 *
493 * It returns empty tokens, too, behaving exactly like the libc function
494 * of that name. In fact, it was stolen from glibc2 and de-fancy-fied.
495 * Same semantics, slimmer shape. ;)
496 */
strsep(char ** s,const char * ct)497 char *strsep(char **s, const char *ct)
498 {
499 char *sbegin = *s;
500 char *end;
501
502 if (sbegin == NULL)
503 return NULL;
504
505 end = strpbrk(sbegin, ct);
506 if (end)
507 *end++ = '\0';
508 *s = end;
509 return sbegin;
510 }
511 EXPORT_SYMBOL(strsep);
512 #endif
513
514 #ifndef __HAVE_ARCH_MEMSET
515 /**
516 * memset - Fill a region of memory with the given value
517 * @s: Pointer to the start of the area.
518 * @c: The byte to fill the area with
519 * @count: The size of the area.
520 *
521 * Do not use memset() to access IO space, use memset_io() instead.
522 */
memset(void * s,int c,size_t count)523 void *memset(void *s, int c, size_t count)
524 {
525 char *xs = s;
526
527 while (count--)
528 *xs++ = c;
529 return s;
530 }
531 EXPORT_SYMBOL(memset);
532 #endif
533
534 #ifndef __HAVE_ARCH_MEMSET16
535 /**
536 * memset16() - Fill a memory area with a uint16_t
537 * @s: Pointer to the start of the area.
538 * @v: The value to fill the area with
539 * @count: The number of values to store
540 *
541 * Differs from memset() in that it fills with a uint16_t instead
542 * of a byte. Remember that @count is the number of uint16_ts to
543 * store, not the number of bytes.
544 */
memset16(uint16_t * s,uint16_t v,size_t count)545 void *memset16(uint16_t *s, uint16_t v, size_t count)
546 {
547 uint16_t *xs = s;
548
549 while (count--)
550 *xs++ = v;
551 return s;
552 }
553 EXPORT_SYMBOL(memset16);
554 #endif
555
556 #ifndef __HAVE_ARCH_MEMSET32
557 /**
558 * memset32() - Fill a memory area with a uint32_t
559 * @s: Pointer to the start of the area.
560 * @v: The value to fill the area with
561 * @count: The number of values to store
562 *
563 * Differs from memset() in that it fills with a uint32_t instead
564 * of a byte. Remember that @count is the number of uint32_ts to
565 * store, not the number of bytes.
566 */
memset32(uint32_t * s,uint32_t v,size_t count)567 void *memset32(uint32_t *s, uint32_t v, size_t count)
568 {
569 uint32_t *xs = s;
570
571 while (count--)
572 *xs++ = v;
573 return s;
574 }
575 EXPORT_SYMBOL(memset32);
576 #endif
577
578 #ifndef __HAVE_ARCH_MEMSET64
579 /**
580 * memset64() - Fill a memory area with a uint64_t
581 * @s: Pointer to the start of the area.
582 * @v: The value to fill the area with
583 * @count: The number of values to store
584 *
585 * Differs from memset() in that it fills with a uint64_t instead
586 * of a byte. Remember that @count is the number of uint64_ts to
587 * store, not the number of bytes.
588 */
memset64(uint64_t * s,uint64_t v,size_t count)589 void *memset64(uint64_t *s, uint64_t v, size_t count)
590 {
591 uint64_t *xs = s;
592
593 while (count--)
594 *xs++ = v;
595 return s;
596 }
597 EXPORT_SYMBOL(memset64);
598 #endif
599
600 #ifndef __HAVE_ARCH_MEMCPY
601 /**
602 * memcpy - Copy one area of memory to another
603 * @dest: Where to copy to
604 * @src: Where to copy from
605 * @count: The size of the area.
606 *
607 * You should not use this function to access IO space, use memcpy_toio()
608 * or memcpy_fromio() instead.
609 */
memcpy(void * dest,const void * src,size_t count)610 void *memcpy(void *dest, const void *src, size_t count)
611 {
612 char *tmp = dest;
613 const char *s = src;
614
615 while (count--)
616 *tmp++ = *s++;
617 return dest;
618 }
619 EXPORT_SYMBOL(memcpy);
620 #endif
621
622 #ifndef __HAVE_ARCH_MEMMOVE
623 /**
624 * memmove - Copy one area of memory to another
625 * @dest: Where to copy to
626 * @src: Where to copy from
627 * @count: The size of the area.
628 *
629 * Unlike memcpy(), memmove() copes with overlapping areas.
630 */
memmove(void * dest,const void * src,size_t count)631 void *memmove(void *dest, const void *src, size_t count)
632 {
633 char *tmp;
634 const char *s;
635
636 if (dest <= src) {
637 tmp = dest;
638 s = src;
639 while (count--)
640 *tmp++ = *s++;
641 } else {
642 tmp = dest;
643 tmp += count;
644 s = src;
645 s += count;
646 while (count--)
647 *--tmp = *--s;
648 }
649 return dest;
650 }
651 EXPORT_SYMBOL(memmove);
652 #endif
653
654 #ifndef __HAVE_ARCH_MEMCMP
655 /**
656 * memcmp - Compare two areas of memory
657 * @cs: One area of memory
658 * @ct: Another area of memory
659 * @count: The size of the area.
660 */
661 #undef memcmp
memcmp(const void * cs,const void * ct,size_t count)662 __visible int memcmp(const void *cs, const void *ct, size_t count)
663 {
664 const unsigned char *su1, *su2;
665 int res = 0;
666
667 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
668 if (count >= sizeof(unsigned long)) {
669 const unsigned long *u1 = cs;
670 const unsigned long *u2 = ct;
671 do {
672 if (get_unaligned(u1) != get_unaligned(u2))
673 break;
674 u1++;
675 u2++;
676 count -= sizeof(unsigned long);
677 } while (count >= sizeof(unsigned long));
678 cs = u1;
679 ct = u2;
680 }
681 #endif
682 for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--)
683 if ((res = *su1 - *su2) != 0)
684 break;
685 return res;
686 }
687 EXPORT_SYMBOL(memcmp);
688 #endif
689
690 #ifndef __HAVE_ARCH_BCMP
691 /**
692 * bcmp - returns 0 if and only if the buffers have identical contents.
693 * @a: pointer to first buffer.
694 * @b: pointer to second buffer.
695 * @len: size of buffers.
696 *
697 * The sign or magnitude of a non-zero return value has no particular
698 * meaning, and architectures may implement their own more efficient bcmp(). So
699 * while this particular implementation is a simple (tail) call to memcmp, do
700 * not rely on anything but whether the return value is zero or non-zero.
701 */
bcmp(const void * a,const void * b,size_t len)702 int bcmp(const void *a, const void *b, size_t len)
703 {
704 return memcmp(a, b, len);
705 }
706 EXPORT_SYMBOL(bcmp);
707 #endif
708
709 #ifndef __HAVE_ARCH_MEMSCAN
710 /**
711 * memscan - Find a character in an area of memory.
712 * @addr: The memory area
713 * @c: The byte to search for
714 * @size: The size of the area.
715 *
716 * returns the address of the first occurrence of @c, or 1 byte past
717 * the area if @c is not found
718 */
memscan(void * addr,int c,size_t size)719 void *memscan(void *addr, int c, size_t size)
720 {
721 unsigned char *p = addr;
722
723 while (size) {
724 if (*p == (unsigned char)c)
725 return (void *)p;
726 p++;
727 size--;
728 }
729 return (void *)p;
730 }
731 EXPORT_SYMBOL(memscan);
732 #endif
733
734 #ifndef __HAVE_ARCH_STRSTR
735 /**
736 * strstr - Find the first substring in a %NUL terminated string
737 * @s1: The string to be searched
738 * @s2: The string to search for
739 */
strstr(const char * s1,const char * s2)740 char *strstr(const char *s1, const char *s2)
741 {
742 size_t l1, l2;
743
744 l2 = strlen(s2);
745 if (!l2)
746 return (char *)s1;
747 l1 = strlen(s1);
748 while (l1 >= l2) {
749 l1--;
750 if (!memcmp(s1, s2, l2))
751 return (char *)s1;
752 s1++;
753 }
754 return NULL;
755 }
756 EXPORT_SYMBOL(strstr);
757 #endif
758
759 #ifndef __HAVE_ARCH_STRNSTR
760 /**
761 * strnstr - Find the first substring in a length-limited string
762 * @s1: The string to be searched
763 * @s2: The string to search for
764 * @len: the maximum number of characters to search
765 */
strnstr(const char * s1,const char * s2,size_t len)766 char *strnstr(const char *s1, const char *s2, size_t len)
767 {
768 size_t l2;
769
770 l2 = strlen(s2);
771 if (!l2)
772 return (char *)s1;
773 while (len >= l2) {
774 len--;
775 if (!memcmp(s1, s2, l2))
776 return (char *)s1;
777 s1++;
778 }
779 return NULL;
780 }
781 EXPORT_SYMBOL(strnstr);
782 #endif
783
784 #ifndef __HAVE_ARCH_MEMCHR
785 /**
786 * memchr - Find a character in an area of memory.
787 * @s: The memory area
788 * @c: The byte to search for
789 * @n: The size of the area.
790 *
791 * returns the address of the first occurrence of @c, or %NULL
792 * if @c is not found
793 */
memchr(const void * s,int c,size_t n)794 void *memchr(const void *s, int c, size_t n)
795 {
796 const unsigned char *p = s;
797 while (n-- != 0) {
798 if ((unsigned char)c == *p++) {
799 return (void *)(p - 1);
800 }
801 }
802 return NULL;
803 }
804 EXPORT_SYMBOL(memchr);
805 #endif
806
check_bytes8(const u8 * start,u8 value,unsigned int bytes)807 static void *check_bytes8(const u8 *start, u8 value, unsigned int bytes)
808 {
809 while (bytes) {
810 if (*start != value)
811 return (void *)start;
812 start++;
813 bytes--;
814 }
815 return NULL;
816 }
817
818 /**
819 * memchr_inv - Find an unmatching character in an area of memory.
820 * @start: The memory area
821 * @c: Find a character other than c
822 * @bytes: The size of the area.
823 *
824 * returns the address of the first character other than @c, or %NULL
825 * if the whole buffer contains just @c.
826 */
memchr_inv(const void * start,int c,size_t bytes)827 void *memchr_inv(const void *start, int c, size_t bytes)
828 {
829 u8 value = c;
830 u64 value64;
831 unsigned int words, prefix;
832
833 if (bytes <= 16)
834 return check_bytes8(start, value, bytes);
835
836 value64 = value;
837 #if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64
838 value64 *= 0x0101010101010101ULL;
839 #elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER)
840 value64 *= 0x01010101;
841 value64 |= value64 << 32;
842 #else
843 value64 |= value64 << 8;
844 value64 |= value64 << 16;
845 value64 |= value64 << 32;
846 #endif
847
848 prefix = (unsigned long)start % 8;
849 if (prefix) {
850 u8 *r;
851
852 prefix = 8 - prefix;
853 r = check_bytes8(start, value, prefix);
854 if (r)
855 return r;
856 start += prefix;
857 bytes -= prefix;
858 }
859
860 words = bytes / 8;
861
862 while (words) {
863 if (*(u64 *)start != value64)
864 return check_bytes8(start, value, 8);
865 start += 8;
866 words--;
867 }
868
869 return check_bytes8(start, value, bytes % 8);
870 }
871 EXPORT_SYMBOL(memchr_inv);
872