686 bool this_cpu_buffer_push(struct percpu_buffer *buffer,  in this_cpu_buffer_push()  argument
700 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_push()
701 		if (offset == buffer->c[cpu].buflen)  in this_cpu_buffer_push()
704 		targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];  in this_cpu_buffer_push()
706 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_buffer_push()
726 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,  in this_cpu_buffer_pop()  argument
739 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_pop()
744 		head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);  in this_cpu_buffer_pop()
746 		targetptr = (intptr_t *)&buffer->c[cpu].offset;  in this_cpu_buffer_pop()
748 			(intptr_t *)&buffer->c[cpu].array[offset - 1],  in this_cpu_buffer_pop()
763 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,  in __percpu_buffer_pop()  argument
769 	offset = buffer->c[cpu].offset;  in __percpu_buffer_pop()
772 	head = buffer->c[cpu].array[offset - 1];  in __percpu_buffer_pop()
773 	buffer->c[cpu].offset = offset - 1;  in __percpu_buffer_pop()
780 	struct percpu_buffer *buffer = (struct percpu_buffer *)arg;  in test_percpu_buffer_thread()  local
789 		node = this_cpu_buffer_pop(buffer, NULL);  in test_percpu_buffer_thread()
793 			if (!this_cpu_buffer_push(buffer, node, NULL)) {  in test_percpu_buffer_thread()
814 	struct percpu_buffer buffer;  in test_percpu_buffer()  local
818 	memset(&buffer, 0, sizeof(buffer));  in test_percpu_buffer()
826 		buffer.c[i].array =  in test_percpu_buffer()
827 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *  in test_percpu_buffer()
829 		assert(buffer.c[i].array);  in test_percpu_buffer()
830 		buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;  in test_percpu_buffer()
846 			buffer.c[i].array[j - 1] = node;  in test_percpu_buffer()
847 			buffer.c[i].offset++;  in test_percpu_buffer()
853 				     test_percpu_buffer_thread, &buffer);  in test_percpu_buffer()
876 		while ((node = __percpu_buffer_pop(&buffer, i))) {  in test_percpu_buffer()
880 		free(buffer.c[i].array);  in test_percpu_buffer()
891 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,  in this_cpu_memcpy_buffer_push()  argument
906 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_push()
907 		if (offset == buffer->c[cpu].buflen)  in this_cpu_memcpy_buffer_push()
909 		destptr = (char *)&buffer->c[cpu].array[offset];  in this_cpu_memcpy_buffer_push()
914 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_push()
935 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,  in this_cpu_memcpy_buffer_pop()  argument
950 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_pop()
954 		srcptr = (char *)&buffer->c[cpu].array[offset - 1];  in this_cpu_memcpy_buffer_pop()
958 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_pop()
977 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,  in __percpu_memcpy_buffer_pop()  argument
983 	offset = buffer->c[cpu].offset;  in __percpu_memcpy_buffer_pop()
986 	memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));  in __percpu_memcpy_buffer_pop()
987 	buffer->c[cpu].offset = offset - 1;  in __percpu_memcpy_buffer_pop()
994 	struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;  in test_percpu_memcpy_buffer_thread()  local
1004 		result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);  in test_percpu_memcpy_buffer_thread()
1008 			if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {  in test_percpu_memcpy_buffer_thread()
1029 	struct percpu_memcpy_buffer buffer;  in test_percpu_memcpy_buffer()  local
1033 	memset(&buffer, 0, sizeof(buffer));  in test_percpu_memcpy_buffer()
1041 		buffer.c[i].array =  in test_percpu_memcpy_buffer()
1042 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *  in test_percpu_memcpy_buffer()
1044 		assert(buffer.c[i].array);  in test_percpu_memcpy_buffer()
1045 		buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;  in test_percpu_memcpy_buffer()
1056 			buffer.c[i].array[j - 1].data1 = j;  in test_percpu_memcpy_buffer()
1057 			buffer.c[i].array[j - 1].data2 = j + 1;  in test_percpu_memcpy_buffer()
1058 			buffer.c[i].offset++;  in test_percpu_memcpy_buffer()
1065 				     &buffer);  in test_percpu_memcpy_buffer()
1088 		while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {  in test_percpu_memcpy_buffer()
1092 		free(buffer.c[i].array);  in test_percpu_memcpy_buffer()