Lines Matching refs:sma
281 static void unmerge_queues(struct sem_array *sma) in unmerge_queues() argument
286 if (sma->complex_count) in unmerge_queues()
293 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in unmerge_queues()
295 curr = &sma->sems[q->sops[0].sem_num]; in unmerge_queues()
299 INIT_LIST_HEAD(&sma->pending_alter); in unmerge_queues()
311 static void merge_queues(struct sem_array *sma) in merge_queues() argument
314 for (i = 0; i < sma->sem_nsems; i++) { in merge_queues()
315 struct sem *sem = &sma->sems[i]; in merge_queues()
317 list_splice_init(&sem->pending_alter, &sma->pending_alter); in merge_queues()
324 struct sem_array *sma = container_of(p, struct sem_array, sem_perm); in sem_rcu_free() local
326 security_sem_free(&sma->sem_perm); in sem_rcu_free()
327 kvfree(sma); in sem_rcu_free()
334 static void complexmode_enter(struct sem_array *sma) in complexmode_enter() argument
339 if (sma->use_global_lock > 0) { in complexmode_enter()
345 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
348 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
350 for (i = 0; i < sma->sem_nsems; i++) { in complexmode_enter()
351 sem = &sma->sems[i]; in complexmode_enter()
361 static void complexmode_tryleave(struct sem_array *sma) in complexmode_tryleave() argument
363 if (sma->complex_count) { in complexmode_tryleave()
369 if (sma->use_global_lock == 1) { in complexmode_tryleave()
372 smp_store_release(&sma->use_global_lock, 0); in complexmode_tryleave()
374 sma->use_global_lock--; in complexmode_tryleave()
386 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops, in sem_lock() argument
394 ipc_lock_object(&sma->sem_perm); in sem_lock()
397 complexmode_enter(sma); in sem_lock()
408 idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in sem_lock()
409 sem = &sma->sems[idx]; in sem_lock()
415 if (!sma->use_global_lock) { in sem_lock()
423 if (!smp_load_acquire(&sma->use_global_lock)) { in sem_lock()
431 ipc_lock_object(&sma->sem_perm); in sem_lock()
433 if (sma->use_global_lock == 0) { in sem_lock()
445 ipc_unlock_object(&sma->sem_perm); in sem_lock()
457 static inline void sem_unlock(struct sem_array *sma, int locknum) in sem_unlock() argument
460 unmerge_queues(sma); in sem_unlock()
461 complexmode_tryleave(sma); in sem_unlock()
462 ipc_unlock_object(&sma->sem_perm); in sem_unlock()
464 struct sem *sem = &sma->sems[locknum]; in sem_unlock()
496 static inline void sem_lock_and_putref(struct sem_array *sma) in sem_lock_and_putref() argument
498 sem_lock(sma, NULL, -1); in sem_lock_and_putref()
499 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in sem_lock_and_putref()
509 struct sem_array *sma; in sem_alloc() local
511 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) in sem_alloc()
514 sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL_ACCOUNT); in sem_alloc()
515 if (unlikely(!sma)) in sem_alloc()
518 return sma; in sem_alloc()
531 struct sem_array *sma; in newary() local
542 sma = sem_alloc(nsems); in newary()
543 if (!sma) in newary()
546 sma->sem_perm.mode = (semflg & S_IRWXUGO); in newary()
547 sma->sem_perm.key = key; in newary()
549 sma->sem_perm.security = NULL; in newary()
550 retval = security_sem_alloc(&sma->sem_perm); in newary()
552 kvfree(sma); in newary()
557 INIT_LIST_HEAD(&sma->sems[i].pending_alter); in newary()
558 INIT_LIST_HEAD(&sma->sems[i].pending_const); in newary()
559 spin_lock_init(&sma->sems[i].lock); in newary()
562 sma->complex_count = 0; in newary()
563 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in newary()
564 INIT_LIST_HEAD(&sma->pending_alter); in newary()
565 INIT_LIST_HEAD(&sma->pending_const); in newary()
566 INIT_LIST_HEAD(&sma->list_id); in newary()
567 sma->sem_nsems = nsems; in newary()
568 sma->sem_ctime = ktime_get_real_seconds(); in newary()
571 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); in newary()
573 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in newary()
578 sem_unlock(sma, -1); in newary()
581 return sma->sem_perm.id; in newary()
590 struct sem_array *sma; in sem_more_checks() local
592 sma = container_of(ipcp, struct sem_array, sem_perm); in sem_more_checks()
593 if (params->u.nsems > sma->sem_nsems) in sem_more_checks()
643 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop_slow() argument
657 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop_slow()
658 curr = &sma->sems[idx]; in perform_atomic_semop_slow()
685 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid); in perform_atomic_semop_slow()
707 sma->sems[sop->sem_num].semval -= sem_op; in perform_atomic_semop_slow()
716 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop() argument
729 return perform_atomic_semop_slow(sma, q); in perform_atomic_semop()
738 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop()
740 curr = &sma->sems[idx]; in perform_atomic_semop()
764 curr = &sma->sems[sop->sem_num]; in perform_atomic_semop()
797 static void unlink_queue(struct sem_array *sma, struct sem_queue *q) in unlink_queue() argument
801 sma->complex_count--; in unlink_queue()
814 static inline int check_restart(struct sem_array *sma, struct sem_queue *q) in check_restart() argument
817 if (!list_empty(&sma->pending_alter)) in check_restart()
852 static int wake_const_ops(struct sem_array *sma, int semnum, in wake_const_ops() argument
860 pending_list = &sma->pending_const; in wake_const_ops()
862 pending_list = &sma->sems[semnum].pending_const; in wake_const_ops()
865 int error = perform_atomic_semop(sma, q); in wake_const_ops()
870 unlink_queue(sma, q); in wake_const_ops()
891 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, in do_smart_wakeup_zero() argument
903 if (sma->sems[num].semval == 0) { in do_smart_wakeup_zero()
905 semop_completed |= wake_const_ops(sma, num, wake_q); in do_smart_wakeup_zero()
913 for (i = 0; i < sma->sem_nsems; i++) { in do_smart_wakeup_zero()
914 if (sma->sems[i].semval == 0) { in do_smart_wakeup_zero()
916 semop_completed |= wake_const_ops(sma, i, wake_q); in do_smart_wakeup_zero()
925 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
947 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q) in update_queue() argument
954 pending_list = &sma->pending_alter; in update_queue()
956 pending_list = &sma->sems[semnum].pending_alter; in update_queue()
969 if (semnum != -1 && sma->sems[semnum].semval == 0) in update_queue()
972 error = perform_atomic_semop(sma, q); in update_queue()
978 unlink_queue(sma, q); in update_queue()
984 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
985 restart = check_restart(sma, q); in update_queue()
1003 static void set_semotime(struct sem_array *sma, struct sembuf *sops) in set_semotime() argument
1006 sma->sems[0].sem_otime = ktime_get_real_seconds(); in set_semotime()
1008 sma->sems[sops[0].sem_num].sem_otime = in set_semotime()
1027 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops, in do_smart_update() argument
1032 otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q); in do_smart_update()
1034 if (!list_empty(&sma->pending_alter)) { in do_smart_update()
1036 otime |= update_queue(sma, -1, wake_q); in do_smart_update()
1043 for (i = 0; i < sma->sem_nsems; i++) in do_smart_update()
1044 otime |= update_queue(sma, i, wake_q); in do_smart_update()
1057 otime |= update_queue(sma, in do_smart_update()
1064 set_semotime(sma, sops); in do_smart_update()
1070 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q, in check_qop() argument
1104 static int count_semcnt(struct sem_array *sma, ushort semnum, in count_semcnt() argument
1114 l = &sma->sems[semnum].pending_const; in count_semcnt()
1116 l = &sma->sems[semnum].pending_alter; in count_semcnt()
1126 list_for_each_entry(q, &sma->pending_alter, list) { in count_semcnt()
1127 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1130 list_for_each_entry(q, &sma->pending_const, list) { in count_semcnt()
1131 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1145 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in freeary() local
1150 ipc_assert_locked_object(&sma->sem_perm); in freeary()
1151 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) { in freeary()
1161 list_for_each_entry_safe(q, tq, &sma->pending_const, list) { in freeary()
1162 unlink_queue(sma, q); in freeary()
1166 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in freeary()
1167 unlink_queue(sma, q); in freeary()
1170 for (i = 0; i < sma->sem_nsems; i++) { in freeary()
1171 struct sem *sem = &sma->sems[i]; in freeary()
1173 unlink_queue(sma, q); in freeary()
1177 unlink_queue(sma, q); in freeary()
1184 sem_rmid(ns, sma); in freeary()
1185 sem_unlock(sma, -1); in freeary()
1189 ns->used_sems -= sma->sem_nsems; in freeary()
1190 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in freeary()
1217 static time64_t get_semotime(struct sem_array *sma) in get_semotime() argument
1222 res = sma->sems[0].sem_otime; in get_semotime()
1223 for (i = 1; i < sma->sem_nsems; i++) { in get_semotime()
1224 time64_t to = sma->sems[i].sem_otime; in get_semotime()
1235 struct sem_array *sma; in semctl_stat() local
1243 sma = sem_obtain_object(ns, semid); in semctl_stat()
1244 if (IS_ERR(sma)) { in semctl_stat()
1245 err = PTR_ERR(sma); in semctl_stat()
1249 sma = sem_obtain_object_check(ns, semid); in semctl_stat()
1250 if (IS_ERR(sma)) { in semctl_stat()
1251 err = PTR_ERR(sma); in semctl_stat()
1258 audit_ipc_obj(&sma->sem_perm); in semctl_stat()
1261 if (ipcperms(ns, &sma->sem_perm, S_IRUGO)) in semctl_stat()
1265 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_stat()
1269 ipc_lock_object(&sma->sem_perm); in semctl_stat()
1271 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_stat()
1272 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1277 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm); in semctl_stat()
1278 semotime = get_semotime(sma); in semctl_stat()
1280 semid64->sem_ctime = sma->sem_ctime; in semctl_stat()
1283 semid64->sem_ctime_high = sma->sem_ctime >> 32; in semctl_stat()
1285 semid64->sem_nsems = sma->sem_nsems; in semctl_stat()
1298 err = sma->sem_perm.id; in semctl_stat()
1300 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1345 struct sem_array *sma; in semctl_setval() local
1354 sma = sem_obtain_object_check(ns, semid); in semctl_setval()
1355 if (IS_ERR(sma)) { in semctl_setval()
1357 return PTR_ERR(sma); in semctl_setval()
1360 if (semnum < 0 || semnum >= sma->sem_nsems) { in semctl_setval()
1366 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) { in semctl_setval()
1371 err = security_sem_semctl(&sma->sem_perm, SETVAL); in semctl_setval()
1377 sem_lock(sma, NULL, -1); in semctl_setval()
1379 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_setval()
1380 sem_unlock(sma, -1); in semctl_setval()
1385 semnum = array_index_nospec(semnum, sma->sem_nsems); in semctl_setval()
1386 curr = &sma->sems[semnum]; in semctl_setval()
1388 ipc_assert_locked_object(&sma->sem_perm); in semctl_setval()
1389 list_for_each_entry(un, &sma->list_id, list_id) in semctl_setval()
1394 sma->sem_ctime = ktime_get_real_seconds(); in semctl_setval()
1396 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_setval()
1397 sem_unlock(sma, -1); in semctl_setval()
1406 struct sem_array *sma; in semctl_main() local
1414 sma = sem_obtain_object_check(ns, semid); in semctl_main()
1415 if (IS_ERR(sma)) { in semctl_main()
1417 return PTR_ERR(sma); in semctl_main()
1420 nsems = sma->sem_nsems; in semctl_main()
1423 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO)) in semctl_main()
1426 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_main()
1437 sem_lock(sma, NULL, -1); in semctl_main()
1438 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1443 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1447 sem_unlock(sma, -1); in semctl_main()
1452 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1457 sem_lock_and_putref(sma); in semctl_main()
1458 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1463 for (i = 0; i < sma->sem_nsems; i++) in semctl_main()
1464 sem_io[i] = sma->sems[i].semval; in semctl_main()
1465 sem_unlock(sma, -1); in semctl_main()
1477 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1487 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1493 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1500 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1506 sem_lock_and_putref(sma); in semctl_main()
1507 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1513 sma->sems[i].semval = sem_io[i]; in semctl_main()
1514 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current)); in semctl_main()
1517 ipc_assert_locked_object(&sma->sem_perm); in semctl_main()
1518 list_for_each_entry(un, &sma->list_id, list_id) { in semctl_main()
1522 sma->sem_ctime = ktime_get_real_seconds(); in semctl_main()
1524 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_main()
1534 sem_lock(sma, NULL, -1); in semctl_main()
1535 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1541 curr = &sma->sems[semnum]; in semctl_main()
1551 err = count_semcnt(sma, semnum, 0); in semctl_main()
1554 err = count_semcnt(sma, semnum, 1); in semctl_main()
1559 sem_unlock(sma, -1); in semctl_main()
1603 struct sem_array *sma; in semctl_down() local
1617 sma = container_of(ipcp, struct sem_array, sem_perm); in semctl_down()
1619 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_down()
1625 sem_lock(sma, NULL, -1); in semctl_down()
1630 sem_lock(sma, NULL, -1); in semctl_down()
1634 sma->sem_ctime = ktime_get_real_seconds(); in semctl_down()
1642 sem_unlock(sma, -1); in semctl_down()
1907 struct sem_array *sma; in find_alloc_undo() local
1925 sma = sem_obtain_object_check(ns, semid); in find_alloc_undo()
1926 if (IS_ERR(sma)) { in find_alloc_undo()
1928 return ERR_CAST(sma); in find_alloc_undo()
1931 nsems = sma->sem_nsems; in find_alloc_undo()
1932 if (!ipc_rcu_getref(&sma->sem_perm)) { in find_alloc_undo()
1942 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in find_alloc_undo()
1948 sem_lock_and_putref(sma); in find_alloc_undo()
1949 if (!ipc_valid_object(&sma->sem_perm)) { in find_alloc_undo()
1950 sem_unlock(sma, -1); in find_alloc_undo()
1972 ipc_assert_locked_object(&sma->sem_perm); in find_alloc_undo()
1973 list_add(&new->list_id, &sma->list_id); in find_alloc_undo()
1978 sem_unlock(sma, -1); in find_alloc_undo()
1987 struct sem_array *sma; in do_semtimedop() local
2058 sma = sem_obtain_object_check(ns, semid); in do_semtimedop()
2059 if (IS_ERR(sma)) { in do_semtimedop()
2061 error = PTR_ERR(sma); in do_semtimedop()
2066 if (max >= sma->sem_nsems) { in do_semtimedop()
2072 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) { in do_semtimedop()
2077 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter); in do_semtimedop()
2084 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2093 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2112 error = perform_atomic_semop(sma, &queue); in do_semtimedop()
2121 do_smart_update(sma, sops, nsops, 1, &wake_q); in do_semtimedop()
2123 set_semotime(sma, sops); in do_semtimedop()
2125 sem_unlock(sma, locknum); in do_semtimedop()
2140 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in do_semtimedop()
2141 curr = &sma->sems[idx]; in do_semtimedop()
2144 if (sma->complex_count) { in do_semtimedop()
2146 &sma->pending_alter); in do_semtimedop()
2156 if (!sma->complex_count) in do_semtimedop()
2157 merge_queues(sma); in do_semtimedop()
2160 list_add_tail(&queue.list, &sma->pending_alter); in do_semtimedop()
2162 list_add_tail(&queue.list, &sma->pending_const); in do_semtimedop()
2164 sma->complex_count++; in do_semtimedop()
2174 sem_unlock(sma, locknum); in do_semtimedop()
2202 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2204 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2226 unlink_queue(sma, &queue); in do_semtimedop()
2229 sem_unlock(sma, locknum); in do_semtimedop()
2329 struct sem_array *sma; in exit_sem() local
2361 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid); in exit_sem()
2363 if (IS_ERR(sma)) { in exit_sem()
2368 sem_lock(sma, NULL, -1); in exit_sem()
2370 if (!ipc_valid_object(&sma->sem_perm)) { in exit_sem()
2371 sem_unlock(sma, -1); in exit_sem()
2380 sem_unlock(sma, -1); in exit_sem()
2386 ipc_assert_locked_object(&sma->sem_perm); in exit_sem()
2394 for (i = 0; i < sma->sem_nsems; i++) { in exit_sem()
2395 struct sem *semaphore = &sma->sems[i]; in exit_sem()
2419 do_smart_update(sma, NULL, 0, 1, &wake_q); in exit_sem()
2420 sem_unlock(sma, -1); in exit_sem()
2434 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in sysvipc_sem_proc_show() local
2443 complexmode_enter(sma); in sysvipc_sem_proc_show()
2445 sem_otime = get_semotime(sma); in sysvipc_sem_proc_show()
2449 sma->sem_perm.key, in sysvipc_sem_proc_show()
2450 sma->sem_perm.id, in sysvipc_sem_proc_show()
2451 sma->sem_perm.mode, in sysvipc_sem_proc_show()
2452 sma->sem_nsems, in sysvipc_sem_proc_show()
2453 from_kuid_munged(user_ns, sma->sem_perm.uid), in sysvipc_sem_proc_show()
2454 from_kgid_munged(user_ns, sma->sem_perm.gid), in sysvipc_sem_proc_show()
2455 from_kuid_munged(user_ns, sma->sem_perm.cuid), in sysvipc_sem_proc_show()
2456 from_kgid_munged(user_ns, sma->sem_perm.cgid), in sysvipc_sem_proc_show()
2458 sma->sem_ctime); in sysvipc_sem_proc_show()
2460 complexmode_tryleave(sma); in sysvipc_sem_proc_show()