Lines Matching refs:sma
258 static void unmerge_queues(struct sem_array *sma) in unmerge_queues() argument
263 if (sma->complex_count) in unmerge_queues()
270 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in unmerge_queues()
272 curr = &sma->sems[q->sops[0].sem_num]; in unmerge_queues()
276 INIT_LIST_HEAD(&sma->pending_alter); in unmerge_queues()
288 static void merge_queues(struct sem_array *sma) in merge_queues() argument
291 for (i = 0; i < sma->sem_nsems; i++) { in merge_queues()
292 struct sem *sem = &sma->sems[i]; in merge_queues()
294 list_splice_init(&sem->pending_alter, &sma->pending_alter); in merge_queues()
301 struct sem_array *sma = container_of(p, struct sem_array, sem_perm); in sem_rcu_free() local
303 security_sem_free(&sma->sem_perm); in sem_rcu_free()
304 kvfree(sma); in sem_rcu_free()
311 static void complexmode_enter(struct sem_array *sma) in complexmode_enter() argument
316 if (sma->use_global_lock > 0) { in complexmode_enter()
322 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
325 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in complexmode_enter()
327 for (i = 0; i < sma->sem_nsems; i++) { in complexmode_enter()
328 sem = &sma->sems[i]; in complexmode_enter()
338 static void complexmode_tryleave(struct sem_array *sma) in complexmode_tryleave() argument
340 if (sma->complex_count) { in complexmode_tryleave()
346 if (sma->use_global_lock == 1) { in complexmode_tryleave()
353 smp_store_release(&sma->use_global_lock, 0); in complexmode_tryleave()
355 sma->use_global_lock--; in complexmode_tryleave()
367 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops, in sem_lock() argument
375 ipc_lock_object(&sma->sem_perm); in sem_lock()
378 complexmode_enter(sma); in sem_lock()
389 idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in sem_lock()
390 sem = &sma->sems[idx]; in sem_lock()
396 if (!sma->use_global_lock) { in sem_lock()
404 if (!smp_load_acquire(&sma->use_global_lock)) { in sem_lock()
412 ipc_lock_object(&sma->sem_perm); in sem_lock()
414 if (sma->use_global_lock == 0) { in sem_lock()
426 ipc_unlock_object(&sma->sem_perm); in sem_lock()
438 static inline void sem_unlock(struct sem_array *sma, int locknum) in sem_unlock() argument
441 unmerge_queues(sma); in sem_unlock()
442 complexmode_tryleave(sma); in sem_unlock()
443 ipc_unlock_object(&sma->sem_perm); in sem_unlock()
445 struct sem *sem = &sma->sems[locknum]; in sem_unlock()
477 static inline void sem_lock_and_putref(struct sem_array *sma) in sem_lock_and_putref() argument
479 sem_lock(sma, NULL, -1); in sem_lock_and_putref()
480 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in sem_lock_and_putref()
490 struct sem_array *sma; in sem_alloc() local
492 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) in sem_alloc()
495 sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL); in sem_alloc()
496 if (unlikely(!sma)) in sem_alloc()
499 return sma; in sem_alloc()
512 struct sem_array *sma; in newary() local
523 sma = sem_alloc(nsems); in newary()
524 if (!sma) in newary()
527 sma->sem_perm.mode = (semflg & S_IRWXUGO); in newary()
528 sma->sem_perm.key = key; in newary()
530 sma->sem_perm.security = NULL; in newary()
531 retval = security_sem_alloc(&sma->sem_perm); in newary()
533 kvfree(sma); in newary()
538 INIT_LIST_HEAD(&sma->sems[i].pending_alter); in newary()
539 INIT_LIST_HEAD(&sma->sems[i].pending_const); in newary()
540 spin_lock_init(&sma->sems[i].lock); in newary()
543 sma->complex_count = 0; in newary()
544 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in newary()
545 INIT_LIST_HEAD(&sma->pending_alter); in newary()
546 INIT_LIST_HEAD(&sma->pending_const); in newary()
547 INIT_LIST_HEAD(&sma->list_id); in newary()
548 sma->sem_nsems = nsems; in newary()
549 sma->sem_ctime = ktime_get_real_seconds(); in newary()
552 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); in newary()
554 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in newary()
559 sem_unlock(sma, -1); in newary()
562 return sma->sem_perm.id; in newary()
572 struct sem_array *sma; in sem_more_checks() local
574 sma = container_of(ipcp, struct sem_array, sem_perm); in sem_more_checks()
575 if (params->u.nsems > sma->sem_nsems) in sem_more_checks()
625 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop_slow() argument
639 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop_slow()
640 curr = &sma->sems[idx]; in perform_atomic_semop_slow()
667 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid); in perform_atomic_semop_slow()
689 sma->sems[sop->sem_num].semval -= sem_op; in perform_atomic_semop_slow()
698 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q) in perform_atomic_semop() argument
711 return perform_atomic_semop_slow(sma, q); in perform_atomic_semop()
720 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop()
722 curr = &sma->sems[idx]; in perform_atomic_semop()
746 curr = &sma->sems[sop->sem_num]; in perform_atomic_semop()
780 static void unlink_queue(struct sem_array *sma, struct sem_queue *q) in unlink_queue() argument
784 sma->complex_count--; in unlink_queue()
797 static inline int check_restart(struct sem_array *sma, struct sem_queue *q) in check_restart() argument
800 if (!list_empty(&sma->pending_alter)) in check_restart()
835 static int wake_const_ops(struct sem_array *sma, int semnum, in wake_const_ops() argument
843 pending_list = &sma->pending_const; in wake_const_ops()
845 pending_list = &sma->sems[semnum].pending_const; in wake_const_ops()
848 int error = perform_atomic_semop(sma, q); in wake_const_ops()
853 unlink_queue(sma, q); in wake_const_ops()
874 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops, in do_smart_wakeup_zero() argument
886 if (sma->sems[num].semval == 0) { in do_smart_wakeup_zero()
888 semop_completed |= wake_const_ops(sma, num, wake_q); in do_smart_wakeup_zero()
896 for (i = 0; i < sma->sem_nsems; i++) { in do_smart_wakeup_zero()
897 if (sma->sems[i].semval == 0) { in do_smart_wakeup_zero()
899 semop_completed |= wake_const_ops(sma, i, wake_q); in do_smart_wakeup_zero()
908 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
930 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q) in update_queue() argument
937 pending_list = &sma->pending_alter; in update_queue()
939 pending_list = &sma->sems[semnum].pending_alter; in update_queue()
952 if (semnum != -1 && sma->sems[semnum].semval == 0) in update_queue()
955 error = perform_atomic_semop(sma, q); in update_queue()
961 unlink_queue(sma, q); in update_queue()
967 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
968 restart = check_restart(sma, q); in update_queue()
986 static void set_semotime(struct sem_array *sma, struct sembuf *sops) in set_semotime() argument
989 sma->sems[0].sem_otime = ktime_get_real_seconds(); in set_semotime()
991 sma->sems[sops[0].sem_num].sem_otime = in set_semotime()
1010 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops, in do_smart_update() argument
1015 otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q); in do_smart_update()
1017 if (!list_empty(&sma->pending_alter)) { in do_smart_update()
1019 otime |= update_queue(sma, -1, wake_q); in do_smart_update()
1026 for (i = 0; i < sma->sem_nsems; i++) in do_smart_update()
1027 otime |= update_queue(sma, i, wake_q); in do_smart_update()
1040 otime |= update_queue(sma, in do_smart_update()
1047 set_semotime(sma, sops); in do_smart_update()
1053 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q, in check_qop() argument
1087 static int count_semcnt(struct sem_array *sma, ushort semnum, in count_semcnt() argument
1097 l = &sma->sems[semnum].pending_const; in count_semcnt()
1099 l = &sma->sems[semnum].pending_alter; in count_semcnt()
1109 list_for_each_entry(q, &sma->pending_alter, list) { in count_semcnt()
1110 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1113 list_for_each_entry(q, &sma->pending_const, list) { in count_semcnt()
1114 semcnt += check_qop(sma, semnum, q, count_zero); in count_semcnt()
1128 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in freeary() local
1133 ipc_assert_locked_object(&sma->sem_perm); in freeary()
1134 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) { in freeary()
1144 list_for_each_entry_safe(q, tq, &sma->pending_const, list) { in freeary()
1145 unlink_queue(sma, q); in freeary()
1149 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in freeary()
1150 unlink_queue(sma, q); in freeary()
1153 for (i = 0; i < sma->sem_nsems; i++) { in freeary()
1154 struct sem *sem = &sma->sems[i]; in freeary()
1156 unlink_queue(sma, q); in freeary()
1160 unlink_queue(sma, q); in freeary()
1167 sem_rmid(ns, sma); in freeary()
1168 sem_unlock(sma, -1); in freeary()
1172 ns->used_sems -= sma->sem_nsems; in freeary()
1173 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in freeary()
1200 static time64_t get_semotime(struct sem_array *sma) in get_semotime() argument
1205 res = sma->sems[0].sem_otime; in get_semotime()
1206 for (i = 1; i < sma->sem_nsems; i++) { in get_semotime()
1207 time64_t to = sma->sems[i].sem_otime; in get_semotime()
1218 struct sem_array *sma; in semctl_stat() local
1226 sma = sem_obtain_object(ns, semid); in semctl_stat()
1227 if (IS_ERR(sma)) { in semctl_stat()
1228 err = PTR_ERR(sma); in semctl_stat()
1232 sma = sem_obtain_object_check(ns, semid); in semctl_stat()
1233 if (IS_ERR(sma)) { in semctl_stat()
1234 err = PTR_ERR(sma); in semctl_stat()
1241 audit_ipc_obj(&sma->sem_perm); in semctl_stat()
1244 if (ipcperms(ns, &sma->sem_perm, S_IRUGO)) in semctl_stat()
1248 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_stat()
1252 ipc_lock_object(&sma->sem_perm); in semctl_stat()
1254 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_stat()
1255 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1260 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm); in semctl_stat()
1261 semotime = get_semotime(sma); in semctl_stat()
1263 semid64->sem_ctime = sma->sem_ctime; in semctl_stat()
1266 semid64->sem_ctime_high = sma->sem_ctime >> 32; in semctl_stat()
1268 semid64->sem_nsems = sma->sem_nsems; in semctl_stat()
1281 err = sma->sem_perm.id; in semctl_stat()
1283 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1328 struct sem_array *sma; in semctl_setval() local
1337 sma = sem_obtain_object_check(ns, semid); in semctl_setval()
1338 if (IS_ERR(sma)) { in semctl_setval()
1340 return PTR_ERR(sma); in semctl_setval()
1343 if (semnum < 0 || semnum >= sma->sem_nsems) { in semctl_setval()
1349 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) { in semctl_setval()
1354 err = security_sem_semctl(&sma->sem_perm, SETVAL); in semctl_setval()
1360 sem_lock(sma, NULL, -1); in semctl_setval()
1362 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_setval()
1363 sem_unlock(sma, -1); in semctl_setval()
1368 semnum = array_index_nospec(semnum, sma->sem_nsems); in semctl_setval()
1369 curr = &sma->sems[semnum]; in semctl_setval()
1371 ipc_assert_locked_object(&sma->sem_perm); in semctl_setval()
1372 list_for_each_entry(un, &sma->list_id, list_id) in semctl_setval()
1377 sma->sem_ctime = ktime_get_real_seconds(); in semctl_setval()
1379 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_setval()
1380 sem_unlock(sma, -1); in semctl_setval()
1389 struct sem_array *sma; in semctl_main() local
1397 sma = sem_obtain_object_check(ns, semid); in semctl_main()
1398 if (IS_ERR(sma)) { in semctl_main()
1400 return PTR_ERR(sma); in semctl_main()
1403 nsems = sma->sem_nsems; in semctl_main()
1406 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO)) in semctl_main()
1409 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_main()
1420 sem_lock(sma, NULL, -1); in semctl_main()
1421 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1426 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1430 sem_unlock(sma, -1); in semctl_main()
1435 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1440 sem_lock_and_putref(sma); in semctl_main()
1441 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1446 for (i = 0; i < sma->sem_nsems; i++) in semctl_main()
1447 sem_io[i] = sma->sems[i].semval; in semctl_main()
1448 sem_unlock(sma, -1); in semctl_main()
1460 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1470 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1476 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1483 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1489 sem_lock_and_putref(sma); in semctl_main()
1490 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1496 sma->sems[i].semval = sem_io[i]; in semctl_main()
1497 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current)); in semctl_main()
1500 ipc_assert_locked_object(&sma->sem_perm); in semctl_main()
1501 list_for_each_entry(un, &sma->list_id, list_id) { in semctl_main()
1505 sma->sem_ctime = ktime_get_real_seconds(); in semctl_main()
1507 do_smart_update(sma, NULL, 0, 0, &wake_q); in semctl_main()
1517 sem_lock(sma, NULL, -1); in semctl_main()
1518 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1524 curr = &sma->sems[semnum]; in semctl_main()
1534 err = count_semcnt(sma, semnum, 0); in semctl_main()
1537 err = count_semcnt(sma, semnum, 1); in semctl_main()
1542 sem_unlock(sma, -1); in semctl_main()
1586 struct sem_array *sma; in semctl_down() local
1600 sma = container_of(ipcp, struct sem_array, sem_perm); in semctl_down()
1602 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_down()
1608 sem_lock(sma, NULL, -1); in semctl_down()
1613 sem_lock(sma, NULL, -1); in semctl_down()
1617 sma->sem_ctime = ktime_get_real_seconds(); in semctl_down()
1625 sem_unlock(sma, -1); in semctl_down()
1890 struct sem_array *sma; in find_alloc_undo() local
1908 sma = sem_obtain_object_check(ns, semid); in find_alloc_undo()
1909 if (IS_ERR(sma)) { in find_alloc_undo()
1911 return ERR_CAST(sma); in find_alloc_undo()
1914 nsems = sma->sem_nsems; in find_alloc_undo()
1915 if (!ipc_rcu_getref(&sma->sem_perm)) { in find_alloc_undo()
1925 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in find_alloc_undo()
1931 sem_lock_and_putref(sma); in find_alloc_undo()
1932 if (!ipc_valid_object(&sma->sem_perm)) { in find_alloc_undo()
1933 sem_unlock(sma, -1); in find_alloc_undo()
1955 ipc_assert_locked_object(&sma->sem_perm); in find_alloc_undo()
1956 list_add(&new->list_id, &sma->list_id); in find_alloc_undo()
1961 sem_unlock(sma, -1); in find_alloc_undo()
1970 struct sem_array *sma; in do_semtimedop() local
2041 sma = sem_obtain_object_check(ns, semid); in do_semtimedop()
2042 if (IS_ERR(sma)) { in do_semtimedop()
2044 error = PTR_ERR(sma); in do_semtimedop()
2049 if (max >= sma->sem_nsems) { in do_semtimedop()
2055 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) { in do_semtimedop()
2060 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter); in do_semtimedop()
2067 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2076 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2095 error = perform_atomic_semop(sma, &queue); in do_semtimedop()
2104 do_smart_update(sma, sops, nsops, 1, &wake_q); in do_semtimedop()
2106 set_semotime(sma, sops); in do_semtimedop()
2108 sem_unlock(sma, locknum); in do_semtimedop()
2123 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in do_semtimedop()
2124 curr = &sma->sems[idx]; in do_semtimedop()
2127 if (sma->complex_count) { in do_semtimedop()
2129 &sma->pending_alter); in do_semtimedop()
2139 if (!sma->complex_count) in do_semtimedop()
2140 merge_queues(sma); in do_semtimedop()
2143 list_add_tail(&queue.list, &sma->pending_alter); in do_semtimedop()
2145 list_add_tail(&queue.list, &sma->pending_const); in do_semtimedop()
2147 sma->complex_count++; in do_semtimedop()
2155 sem_unlock(sma, locknum); in do_semtimedop()
2187 locknum = sem_lock(sma, sops, nsops); in do_semtimedop()
2189 if (!ipc_valid_object(&sma->sem_perm)) in do_semtimedop()
2208 unlink_queue(sma, &queue); in do_semtimedop()
2211 sem_unlock(sma, locknum); in do_semtimedop()
2311 struct sem_array *sma; in exit_sem() local
2343 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid); in exit_sem()
2345 if (IS_ERR(sma)) { in exit_sem()
2350 sem_lock(sma, NULL, -1); in exit_sem()
2352 if (!ipc_valid_object(&sma->sem_perm)) { in exit_sem()
2353 sem_unlock(sma, -1); in exit_sem()
2362 sem_unlock(sma, -1); in exit_sem()
2368 ipc_assert_locked_object(&sma->sem_perm); in exit_sem()
2378 for (i = 0; i < sma->sem_nsems; i++) { in exit_sem()
2379 struct sem *semaphore = &sma->sems[i]; in exit_sem()
2403 do_smart_update(sma, NULL, 0, 1, &wake_q); in exit_sem()
2404 sem_unlock(sma, -1); in exit_sem()
2418 struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm); in sysvipc_sem_proc_show() local
2427 complexmode_enter(sma); in sysvipc_sem_proc_show()
2429 sem_otime = get_semotime(sma); in sysvipc_sem_proc_show()
2433 sma->sem_perm.key, in sysvipc_sem_proc_show()
2434 sma->sem_perm.id, in sysvipc_sem_proc_show()
2435 sma->sem_perm.mode, in sysvipc_sem_proc_show()
2436 sma->sem_nsems, in sysvipc_sem_proc_show()
2437 from_kuid_munged(user_ns, sma->sem_perm.uid), in sysvipc_sem_proc_show()
2438 from_kgid_munged(user_ns, sma->sem_perm.gid), in sysvipc_sem_proc_show()
2439 from_kuid_munged(user_ns, sma->sem_perm.cuid), in sysvipc_sem_proc_show()
2440 from_kgid_munged(user_ns, sma->sem_perm.cgid), in sysvipc_sem_proc_show()
2442 sma->sem_ctime); in sysvipc_sem_proc_show()
2444 complexmode_tryleave(sma); in sysvipc_sem_proc_show()