259 int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)  in mlx4_table_get()  argument
261 	u32 i = (obj & (table->num_obj - 1)) /  in mlx4_table_get()
262 			(MLX4_TABLE_CHUNK_SIZE / table->obj_size);  in mlx4_table_get()
265 	mutex_lock(&table->mutex);  in mlx4_table_get()
267 	if (table->icm[i]) {  in mlx4_table_get()
268 		++table->icm[i]->refcount;  in mlx4_table_get()
272 	table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT,  in mlx4_table_get()
273 				       (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |  in mlx4_table_get()
274 				       __GFP_NOWARN, table->coherent);  in mlx4_table_get()
275 	if (!table->icm[i]) {  in mlx4_table_get()
280 	if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +  in mlx4_table_get()
282 		mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_table_get()
283 		table->icm[i] = NULL;  in mlx4_table_get()
288 	++table->icm[i]->refcount;  in mlx4_table_get()
291 	mutex_unlock(&table->mutex);  in mlx4_table_get()
295 void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)  in mlx4_table_put()  argument
300 	i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size);  in mlx4_table_put()
302 	mutex_lock(&table->mutex);  in mlx4_table_put()
304 	if (--table->icm[i]->refcount == 0) {  in mlx4_table_put()
306 		mlx4_UNMAP_ICM(dev, table->virt + offset,  in mlx4_table_put()
308 		mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_table_put()
309 		table->icm[i] = NULL;  in mlx4_table_put()
312 	mutex_unlock(&table->mutex);  in mlx4_table_put()
315 void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj,  in mlx4_table_find()  argument
324 	if (!table->lowmem)  in mlx4_table_find()
327 	mutex_lock(&table->mutex);  in mlx4_table_find()
329 	idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size;  in mlx4_table_find()
330 	icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE];  in mlx4_table_find()
341 			if (table->coherent) {  in mlx4_table_find()
379 	mutex_unlock(&table->mutex);  in mlx4_table_find()
383 int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,  in mlx4_table_get_range()  argument
386 	int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size;  in mlx4_table_get_range()
391 		err = mlx4_table_get(dev, table, i);  in mlx4_table_get_range()
401 		mlx4_table_put(dev, table, i);  in mlx4_table_get_range()
407 void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,  in mlx4_table_put_range()  argument
412 	for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size)  in mlx4_table_put_range()
413 		mlx4_table_put(dev, table, i);  in mlx4_table_put_range()
416 int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,  in mlx4_init_icm_table()  argument
431 	table->icm      = kvcalloc(num_icm, sizeof(*table->icm), GFP_KERNEL);  in mlx4_init_icm_table()
432 	if (!table->icm)  in mlx4_init_icm_table()
434 	table->virt     = virt;  in mlx4_init_icm_table()
435 	table->num_icm  = num_icm;  in mlx4_init_icm_table()
436 	table->num_obj  = nobj;  in mlx4_init_icm_table()
437 	table->obj_size = obj_size;  in mlx4_init_icm_table()
438 	table->lowmem   = use_lowmem;  in mlx4_init_icm_table()
439 	table->coherent = use_coherent;  in mlx4_init_icm_table()
440 	mutex_init(&table->mutex);  in mlx4_init_icm_table()
449 		table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,  in mlx4_init_icm_table()
452 		if (!table->icm[i])  in mlx4_init_icm_table()
454 		if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {  in mlx4_init_icm_table()
455 			mlx4_free_icm(dev, table->icm[i], use_coherent);  in mlx4_init_icm_table()
456 			table->icm[i] = NULL;  in mlx4_init_icm_table()
464 		++table->icm[i]->refcount;  in mlx4_init_icm_table()
471 		if (table->icm[i]) {  in mlx4_init_icm_table()
474 			mlx4_free_icm(dev, table->icm[i], use_coherent);  in mlx4_init_icm_table()
477 	kvfree(table->icm);  in mlx4_init_icm_table()
482 void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table)  in mlx4_cleanup_icm_table()  argument
486 	for (i = 0; i < table->num_icm; ++i)  in mlx4_cleanup_icm_table()
487 		if (table->icm[i]) {  in mlx4_cleanup_icm_table()
488 			mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,  in mlx4_cleanup_icm_table()
490 			mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_cleanup_icm_table()
493 	kvfree(table->icm);  in mlx4_cleanup_icm_table()