782 bool this_cpu_buffer_push(struct percpu_buffer *buffer,  in this_cpu_buffer_push()  argument
796 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_push()
797 		if (offset == buffer->c[cpu].buflen)  in this_cpu_buffer_push()
800 		targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];  in this_cpu_buffer_push()
802 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_buffer_push()
817 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,  in this_cpu_buffer_pop()  argument
830 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_pop()
835 		head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);  in this_cpu_buffer_pop()
837 		targetptr = (intptr_t *)&buffer->c[cpu].offset;  in this_cpu_buffer_pop()
840 			(intptr_t *)&buffer->c[cpu].array[offset - 1],  in this_cpu_buffer_pop()
855 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,  in __percpu_buffer_pop()  argument
861 	offset = buffer->c[cpu].offset;  in __percpu_buffer_pop()
864 	head = buffer->c[cpu].array[offset - 1];  in __percpu_buffer_pop()
865 	buffer->c[cpu].offset = offset - 1;  in __percpu_buffer_pop()
872 	struct percpu_buffer *buffer = (struct percpu_buffer *)arg;  in test_percpu_buffer_thread()  local
881 		node = this_cpu_buffer_pop(buffer, NULL);  in test_percpu_buffer_thread()
885 			if (!this_cpu_buffer_push(buffer, node, NULL)) {  in test_percpu_buffer_thread()
906 	struct percpu_buffer buffer;  in test_percpu_buffer()  local
910 	memset(&buffer, 0, sizeof(buffer));  in test_percpu_buffer()
918 		buffer.c[i].array =  in test_percpu_buffer()
919 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *  in test_percpu_buffer()
921 		assert(buffer.c[i].array);  in test_percpu_buffer()
922 		buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;  in test_percpu_buffer()
938 			buffer.c[i].array[j - 1] = node;  in test_percpu_buffer()
939 			buffer.c[i].offset++;  in test_percpu_buffer()
945 				     test_percpu_buffer_thread, &buffer);  in test_percpu_buffer()
968 		while ((node = __percpu_buffer_pop(&buffer, i))) {  in test_percpu_buffer()
972 		free(buffer.c[i].array);  in test_percpu_buffer()
983 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,  in this_cpu_memcpy_buffer_push()  argument
998 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_push()
999 		if (offset == buffer->c[cpu].buflen)  in this_cpu_memcpy_buffer_push()
1001 		destptr = (char *)&buffer->c[cpu].array[offset];  in this_cpu_memcpy_buffer_push()
1006 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_push()
1023 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,  in this_cpu_memcpy_buffer_pop()  argument
1038 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_pop()
1042 		srcptr = (char *)&buffer->c[cpu].array[offset - 1];  in this_cpu_memcpy_buffer_pop()
1046 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_pop()
1065 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,  in __percpu_memcpy_buffer_pop()  argument
1071 	offset = buffer->c[cpu].offset;  in __percpu_memcpy_buffer_pop()
1074 	memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));  in __percpu_memcpy_buffer_pop()
1075 	buffer->c[cpu].offset = offset - 1;  in __percpu_memcpy_buffer_pop()
1082 	struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;  in test_percpu_memcpy_buffer_thread()  local
1092 		result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);  in test_percpu_memcpy_buffer_thread()
1096 			if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {  in test_percpu_memcpy_buffer_thread()
1117 	struct percpu_memcpy_buffer buffer;  in test_percpu_memcpy_buffer()  local
1121 	memset(&buffer, 0, sizeof(buffer));  in test_percpu_memcpy_buffer()
1129 		buffer.c[i].array =  in test_percpu_memcpy_buffer()
1130 			malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *  in test_percpu_memcpy_buffer()
1132 		assert(buffer.c[i].array);  in test_percpu_memcpy_buffer()
1133 		buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;  in test_percpu_memcpy_buffer()
1144 			buffer.c[i].array[j - 1].data1 = j;  in test_percpu_memcpy_buffer()
1145 			buffer.c[i].array[j - 1].data2 = j + 1;  in test_percpu_memcpy_buffer()
1146 			buffer.c[i].offset++;  in test_percpu_memcpy_buffer()
1153 				     &buffer);  in test_percpu_memcpy_buffer()
1176 		while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {  in test_percpu_memcpy_buffer()
1180 		free(buffer.c[i].array);  in test_percpu_memcpy_buffer()