39 void tracing_map_update_sum(struct tracing_map_elt *elt, unsigned int i, u64 n)  in tracing_map_update_sum()  argument
41 	atomic64_add(n, &elt->fields[i].sum);  in tracing_map_update_sum()
56 u64 tracing_map_read_sum(struct tracing_map_elt *elt, unsigned int i)  in tracing_map_read_sum()  argument
58 	return (u64)atomic64_read(&elt->fields[i].sum);  in tracing_map_read_sum()
71 void tracing_map_set_var(struct tracing_map_elt *elt, unsigned int i, u64 n)  in tracing_map_set_var()  argument
73 	atomic64_set(&elt->vars[i], n);  in tracing_map_set_var()
74 	elt->var_set[i] = true;  in tracing_map_set_var()
86 bool tracing_map_var_set(struct tracing_map_elt *elt, unsigned int i)  in tracing_map_var_set()  argument
88 	return elt->var_set[i];  in tracing_map_var_set()
103 u64 tracing_map_read_var(struct tracing_map_elt *elt, unsigned int i)  in tracing_map_read_var()  argument
105 	return (u64)atomic64_read(&elt->vars[i]);  in tracing_map_read_var()
122 u64 tracing_map_read_var_once(struct tracing_map_elt *elt, unsigned int i)  in tracing_map_read_var_once()  argument
124 	elt->var_set[i] = false;  in tracing_map_read_var_once()
125 	return (u64)atomic64_read(&elt->vars[i]);  in tracing_map_read_var_once()
358 static void tracing_map_elt_clear(struct tracing_map_elt *elt)  in tracing_map_elt_clear()  argument
362 	for (i = 0; i < elt->map->n_fields; i++)  in tracing_map_elt_clear()
363 		if (elt->fields[i].cmp_fn == tracing_map_cmp_atomic64)  in tracing_map_elt_clear()
364 			atomic64_set(&elt->fields[i].sum, 0);  in tracing_map_elt_clear()
366 	for (i = 0; i < elt->map->n_vars; i++) {  in tracing_map_elt_clear()
367 		atomic64_set(&elt->vars[i], 0);  in tracing_map_elt_clear()
368 		elt->var_set[i] = false;  in tracing_map_elt_clear()
371 	if (elt->map->ops && elt->map->ops->elt_clear)  in tracing_map_elt_clear()
372 		elt->map->ops->elt_clear(elt);  in tracing_map_elt_clear()
375 static void tracing_map_elt_init_fields(struct tracing_map_elt *elt)  in tracing_map_elt_init_fields()  argument
379 	tracing_map_elt_clear(elt);  in tracing_map_elt_init_fields()
381 	for (i = 0; i < elt->map->n_fields; i++) {  in tracing_map_elt_init_fields()
382 		elt->fields[i].cmp_fn = elt->map->fields[i].cmp_fn;  in tracing_map_elt_init_fields()
384 		if (elt->fields[i].cmp_fn != tracing_map_cmp_atomic64)  in tracing_map_elt_init_fields()
385 			elt->fields[i].offset = elt->map->fields[i].offset;  in tracing_map_elt_init_fields()
389 static void tracing_map_elt_free(struct tracing_map_elt *elt)  in tracing_map_elt_free()  argument
391 	if (!elt)  in tracing_map_elt_free()
394 	if (elt->map->ops && elt->map->ops->elt_free)  in tracing_map_elt_free()
395 		elt->map->ops->elt_free(elt);  in tracing_map_elt_free()
396 	kfree(elt->fields);  in tracing_map_elt_free()
397 	kfree(elt->vars);  in tracing_map_elt_free()
398 	kfree(elt->var_set);  in tracing_map_elt_free()
399 	kfree(elt->key);  in tracing_map_elt_free()
400 	kfree(elt);  in tracing_map_elt_free()
405 	struct tracing_map_elt *elt;  in tracing_map_elt_alloc()  local
408 	elt = kzalloc(sizeof(*elt), GFP_KERNEL);  in tracing_map_elt_alloc()
409 	if (!elt)  in tracing_map_elt_alloc()
412 	elt->map = map;  in tracing_map_elt_alloc()
414 	elt->key = kzalloc(map->key_size, GFP_KERNEL);  in tracing_map_elt_alloc()
415 	if (!elt->key) {  in tracing_map_elt_alloc()
420 	elt->fields = kcalloc(map->n_fields, sizeof(*elt->fields), GFP_KERNEL);  in tracing_map_elt_alloc()
421 	if (!elt->fields) {  in tracing_map_elt_alloc()
426 	elt->vars = kcalloc(map->n_vars, sizeof(*elt->vars), GFP_KERNEL);  in tracing_map_elt_alloc()
427 	if (!elt->vars) {  in tracing_map_elt_alloc()
432 	elt->var_set = kcalloc(map->n_vars, sizeof(*elt->var_set), GFP_KERNEL);  in tracing_map_elt_alloc()
433 	if (!elt->var_set) {  in tracing_map_elt_alloc()
438 	tracing_map_elt_init_fields(elt);  in tracing_map_elt_alloc()
441 		err = map->ops->elt_alloc(elt);  in tracing_map_elt_alloc()
445 	return elt;  in tracing_map_elt_alloc()
447 	tracing_map_elt_free(elt);  in tracing_map_elt_alloc()
454 	struct tracing_map_elt *elt = NULL;  in get_free_elt()  local
459 		elt = *(TRACING_MAP_ELT(map->elts, idx));  in get_free_elt()
461 			map->ops->elt_init(elt);  in get_free_elt()
464 	return elt;  in get_free_elt()
567 				struct tracing_map_elt *elt;  in __tracing_map_insert()  local
569 				elt = get_free_elt(map);  in __tracing_map_insert()
570 				if (!elt) {  in __tracing_map_insert()
576 				memcpy(elt->key, key, map->key_size);  in __tracing_map_insert()
582 				WRITE_ONCE(entry->val, elt);  in __tracing_map_insert()
853 	if (memcmp(a->key, b->key, a->elt->map->key_size))  in cmp_entries_dup()
872 	elt_a = a->elt;  in cmp_entries_sum()
873 	elt_b = b->elt;  in cmp_entries_sum()
903 	elt_a = a->elt;  in cmp_entries_key()
904 	elt_b = b->elt;  in cmp_entries_key()
928 		tracing_map_elt_free(entry->elt);  in destroy_sort_entry()
952 create_sort_entry(void *key, struct tracing_map_elt *elt)  in create_sort_entry()  argument
961 	sort_entry->elt = elt;  in create_sort_entry()