• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2022, The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 use crate::keystore2_client_test_utils::{
16     create_signing_operation, execute_op_run_as_child, perform_sample_sign_operation,
17     BarrierReached, ForcedOp, TestOutcome,
18 };
19 use android_hardware_security_keymint::aidl::android::hardware::security::keymint::{
20     Digest::Digest, ErrorCode::ErrorCode, KeyPurpose::KeyPurpose,
21 };
22 use android_system_keystore2::aidl::android::system::keystore2::{
23     CreateOperationResponse::CreateOperationResponse, Domain::Domain,
24     IKeystoreOperation::IKeystoreOperation, ResponseCode::ResponseCode,
25 };
26 use keystore2_test_utils::{
27     authorizations, key_generations, key_generations::Error, run_as, SecLevel,
28 };
29 use nix::unistd::{getuid, Gid, Uid};
30 use rustutils::users::AID_USER_OFFSET;
31 use std::sync::{
32     atomic::{AtomicBool, Ordering},
33     Arc,
34 };
35 use std::thread;
36 use std::thread::JoinHandle;
37 
38 /// Create `max_ops` number child processes with the given context and perform an operation under each
39 /// child process.
40 ///
41 /// # Safety
42 ///
43 /// Must only be called from a single-threaded process (e.g. as enforced by `AndroidTest.xml`
44 /// setting `--test-threads=1`).
create_operations( target_ctx: &'static str, forced_op: ForcedOp, max_ops: i32, ) -> Vec<run_as::ChildHandle<TestOutcome, BarrierReached>>45 pub unsafe fn create_operations(
46     target_ctx: &'static str,
47     forced_op: ForcedOp,
48     max_ops: i32,
49 ) -> Vec<run_as::ChildHandle<TestOutcome, BarrierReached>> {
50     let alias = format!("ks_op_test_key_{}", getuid());
51     let base_gid = 99 * AID_USER_OFFSET + 10001;
52     let base_uid = 99 * AID_USER_OFFSET + 10001;
53     (0..max_ops)
54         // Safety: The caller guarantees that there are no other threads.
55         .map(|i| unsafe {
56             execute_op_run_as_child(
57                 target_ctx,
58                 Domain::APP,
59                 key_generations::SELINUX_SHELL_NAMESPACE,
60                 Some(alias.to_string()),
61                 Uid::from_raw(base_uid + (i as u32)),
62                 Gid::from_raw(base_gid + (i as u32)),
63                 forced_op,
64             )
65         })
66         .collect()
67 }
68 
69 /// Executes an operation in a thread. Expect an `OPERATION_BUSY` error in case of operation
70 /// failure. Returns True if `OPERATION_BUSY` error is encountered otherwise returns false.
perform_op_busy_in_thread(op: binder::Strong<dyn IKeystoreOperation>) -> JoinHandle<bool>71 fn perform_op_busy_in_thread(op: binder::Strong<dyn IKeystoreOperation>) -> JoinHandle<bool> {
72     thread::spawn(move || {
73         for _n in 1..1000 {
74             match key_generations::map_ks_error(op.update(b"my message")) {
75                 Ok(_) => continue,
76                 Err(e) => {
77                     assert_eq!(Error::Rc(ResponseCode::OPERATION_BUSY), e);
78                     return true;
79                 }
80             }
81         }
82         let sig = op.finish(None, None).unwrap();
83         assert!(sig.is_some());
84         false
85     })
86 }
87 
88 /// This test verifies that backend service throws BACKEND_BUSY error when all
89 /// operations slots are full. This test creates operations in child processes and
90 /// collects the status of operations performed in each child proc and determines
91 /// whether any child proc exited with error status.
92 #[test]
keystore2_backend_busy_test()93 fn keystore2_backend_busy_test() {
94     const MAX_OPS: i32 = 100;
95     static TARGET_CTX: &str = "u:r:untrusted_app:s0:c91,c256,c10,c20";
96 
97     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
98     // `--test-threads=1`), and nothing yet done with binder.
99     let mut child_handles = unsafe { create_operations(TARGET_CTX, ForcedOp(false), MAX_OPS) };
100 
101     // Wait until all child procs notifies us to continue,
102     // so that there are definitely enough operations outstanding to trigger a BACKEND_BUSY.
103     for ch in child_handles.iter_mut() {
104         ch.recv();
105     }
106     // Notify each child to resume and finish.
107     for ch in child_handles.iter_mut() {
108         ch.send(&BarrierReached {});
109     }
110 
111     // Collect the result and validate whether backend busy has occurred.
112     let mut busy_count = 0;
113     for ch in child_handles.into_iter() {
114         if ch.get_result() == TestOutcome::BackendBusy {
115             busy_count += 1;
116         }
117     }
118     assert!(busy_count > 0)
119 }
120 
121 /// This test confirms that forced operation is having high pruning power.
122 /// 1. Initially create regular operations such that there are enough operations outstanding
123 ///    to trigger BACKEND_BUSY.
124 /// 2. Then, create a forced operation. System should be able to prune one of the regular
125 ///    operations and create a slot for forced operation successfully.
126 #[test]
keystore2_forced_op_after_backendbusy_test()127 fn keystore2_forced_op_after_backendbusy_test() {
128     const MAX_OPS: i32 = 100;
129     static TARGET_CTX: &str = "u:r:untrusted_app:s0:c91,c256,c10,c20";
130 
131     // Create regular operations.
132     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
133     // `--test-threads=1`), and nothing yet done with binder.
134     let mut child_handles = unsafe { create_operations(TARGET_CTX, ForcedOp(false), MAX_OPS) };
135 
136     // Wait until all child procs notifies us to continue, so that there are enough
137     // operations outstanding to trigger a BACKEND_BUSY.
138     for ch in child_handles.iter_mut() {
139         ch.recv();
140     }
141 
142     // Create a forced operation.
143     let auid = 99 * AID_USER_OFFSET + 10604;
144     let agid = 99 * AID_USER_OFFSET + 10604;
145     let force_op_fn = move || {
146         let alias = format!("ks_prune_forced_op_key_{}", getuid());
147 
148         // To make room for this forced op, system should be able to prune one of the
149         // above created regular operations and create a slot for this forced operation
150         // successfully.
151         create_signing_operation(
152             ForcedOp(true),
153             KeyPurpose::SIGN,
154             Digest::SHA_2_256,
155             Domain::SELINUX,
156             100,
157             Some(alias),
158         )
159         .expect("Client failed to create forced operation after BACKEND_BUSY state.");
160     };
161 
162     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
163     // `--test-threads=1`), and nothing yet done with binder.
164     unsafe {
165         run_as::run_as(
166             key_generations::TARGET_VOLD_CTX,
167             Uid::from_raw(auid),
168             Gid::from_raw(agid),
169             force_op_fn,
170         );
171     };
172 
173     // Notify each child to resume and finish.
174     for ch in child_handles.iter_mut() {
175         ch.send(&BarrierReached {});
176     }
177 
178     // Collect the results of above created regular operations.
179     let mut pruned_count = 0;
180     let mut busy_count = 0;
181     let mut _other_err = 0;
182     for ch in child_handles.into_iter() {
183         match ch.get_result() {
184             TestOutcome::BackendBusy => {
185                 busy_count += 1;
186             }
187             TestOutcome::InvalidHandle => {
188                 pruned_count += 1;
189             }
190             _ => {
191                 _other_err += 1;
192             }
193         }
194     }
195     // Verify that there should be at least one backend busy has occurred while creating
196     // above regular operations.
197     assert!(busy_count > 0);
198 
199     // Verify that there should be at least one pruned operation which should have failed while
200     // performing operation.
201     assert!(pruned_count > 0);
202 }
203 
204 /// This test confirms that forced operations can't be pruned.
205 ///  1. Creates an initial forced operation and tries to complete the operation after BACKEND_BUSY
206 ///     error is triggered.
207 ///  2. Create MAX_OPS number of forced operations so that definitely enough number of operations
208 ///     outstanding to trigger a BACKEND_BUSY.
209 ///  3. Try to use initially created forced operation (in step #1) and able to perform the
210 ///     operation successfully. This confirms that none of the later forced operations evicted the
211 ///     initial forced operation.
212 #[test]
keystore2_max_forced_ops_test()213 fn keystore2_max_forced_ops_test() {
214     const MAX_OPS: i32 = 100;
215     let auid = 99 * AID_USER_OFFSET + 10205;
216     let agid = 99 * AID_USER_OFFSET + 10205;
217 
218     // Create initial forced operation in a child process
219     // and wait for the parent to notify to perform operation.
220     let alias = format!("ks_forced_op_key_{}", getuid());
221 
222     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
223     // `--test-threads=1`), and nothing yet done with binder.
224     let mut first_op_handle = unsafe {
225         execute_op_run_as_child(
226             key_generations::TARGET_SU_CTX,
227             Domain::SELINUX,
228             key_generations::SELINUX_SHELL_NAMESPACE,
229             Some(alias),
230             Uid::from_raw(auid),
231             Gid::from_raw(agid),
232             ForcedOp(true),
233         )
234     };
235 
236     // Wait until above child proc notifies us to continue, so that there is definitely a forced
237     // operation outstanding to perform a operation.
238     first_op_handle.recv();
239 
240     // Create MAX_OPS number of forced operations.
241     let mut child_handles =
242     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
243     // `--test-threads=1`), and nothing yet done with binder.
244         unsafe { create_operations(key_generations::TARGET_SU_CTX, ForcedOp(true), MAX_OPS) };
245 
246     // Wait until all child procs notifies us to continue, so that  there are enough operations
247     // outstanding to trigger a BACKEND_BUSY.
248     for ch in child_handles.iter_mut() {
249         ch.recv();
250     }
251 
252     // Notify initial created forced operation to continue performing the operations.
253     first_op_handle.send(&BarrierReached {});
254 
255     // Collect initially created forced operation result and is expected to complete operation
256     // successfully.
257     let first_op_result = first_op_handle.get_result();
258     assert_eq!(first_op_result, TestOutcome::Ok);
259 
260     // Notify each child to resume and finish.
261     for ch in child_handles.iter_mut() {
262         ch.send(&BarrierReached {});
263     }
264 
265     // Collect the result and validate whether backend busy has occurred with MAX_OPS number
266     // of forced operations.
267     let busy_count = child_handles
268         .into_iter()
269         .map(|ch| ch.get_result())
270         .filter(|r| *r == TestOutcome::BackendBusy)
271         .count();
272     assert!(busy_count > 0);
273 }
274 
275 /// This test will verify the use case with the same owner(UID) requesting `n` number of operations.
276 /// This test confirms that when all operation slots are full and a new operation is requested,
277 /// an operation which is least recently used and lived longest will be pruned to make a room
278 /// for a new operation. Pruning strategy should prevent the operations of the other owners(UID)
279 /// from being pruned.
280 ///
281 /// 1. Create an operation in a child process with `untrusted_app` context and wait for parent
282 ///    notification to complete the operation.
283 /// 2. Let parent process create `n` number of operations such that there are enough operations
284 ///    outstanding to trigger cannibalizing their own sibling operations.
285 /// 3. Sequentially try to use above created `n` number of operations and also add a new operation,
286 ///    so that it should trigger cannibalizing one of their own sibling operations.
287 ///    3.1 While trying to use these pruned operations an `INVALID_OPERATION_HANDLE` error is
288 ///        expected as they are already pruned.
289 /// 4. Notify the child process to resume and complete the operation. It is expected to complete the
290 ///    operation successfully.
291 /// 5. Try to use the latest operation of parent. It is expected to complete the operation
292 ///    successfully.
293 #[test]
keystore2_ops_prune_test()294 fn keystore2_ops_prune_test() {
295     const MAX_OPS: usize = 40; // This should be at least 32 with sec_level TEE.
296 
297     static TARGET_CTX: &str = "u:r:untrusted_app:s0";
298     const USER_ID: u32 = 99;
299     const APPLICATION_ID: u32 = 10601;
300 
301     let uid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
302     let gid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
303 
304     // Create an operation in an untrusted_app context. Wait until the parent notifies to continue.
305     // Once the parent notifies, this operation is expected to be completed successfully.
306     let alias = format!("ks_reg_op_key_{}", getuid());
307 
308     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
309     // `--test-threads=1`), and nothing yet done with binder.
310     let mut child_handle = unsafe {
311         execute_op_run_as_child(
312             TARGET_CTX,
313             Domain::APP,
314             -1,
315             Some(alias),
316             Uid::from_raw(uid),
317             Gid::from_raw(gid),
318             ForcedOp(false),
319         )
320     };
321 
322     // Wait until child process notifies us to continue, so that an operation from child process is
323     // outstanding to complete the operation.
324     child_handle.recv();
325 
326     // Generate a key to use in below operations.
327     let sl = SecLevel::tee();
328     let alias = format!("ks_prune_op_test_key_{}", getuid());
329     let key_metadata = key_generations::generate_ec_p256_signing_key(
330         &sl,
331         Domain::SELINUX,
332         key_generations::SELINUX_SHELL_NAMESPACE,
333         Some(alias),
334         None,
335     )
336     .unwrap();
337 
338     // Create multiple operations in this process to trigger cannibalizing sibling operations.
339     let mut ops: Vec<binder::Result<CreateOperationResponse>> = (0..MAX_OPS)
340         .map(|_| {
341             sl.binder.createOperation(
342                 &key_metadata.key,
343                 &authorizations::AuthSetBuilder::new()
344                     .purpose(KeyPurpose::SIGN)
345                     .digest(Digest::SHA_2_256),
346                 false,
347             )
348         })
349         .collect();
350 
351     // Sequentially try to use operation handles created above and also add a new operation.
352     for vec_index in 0..MAX_OPS {
353         match &ops[vec_index] {
354             Ok(CreateOperationResponse { iOperation: Some(op), .. }) => {
355                 // Older operation handle is pruned, if we try to use that an error is expected.
356                 assert_eq!(
357                     Err(Error::Km(ErrorCode::INVALID_OPERATION_HANDLE)),
358                     key_generations::map_ks_error(op.update(b"my message"))
359                 );
360             }
361             _ => panic!("Operation should have created successfully."),
362         }
363 
364         // Create a new operation, it should trigger to cannibalize one of their own sibling
365         // operations.
366         ops.push(
367             sl.binder.createOperation(
368                 &key_metadata.key,
369                 &authorizations::AuthSetBuilder::new()
370                     .purpose(KeyPurpose::SIGN)
371                     .digest(Digest::SHA_2_256),
372                 false,
373             ),
374         );
375     }
376 
377     // Notify child process to continue the operation.
378     child_handle.send(&BarrierReached {});
379     assert!((child_handle.get_result() == TestOutcome::Ok), "Failed to perform an operation");
380 
381     // Try to use the latest operation created by parent, should be able to use it successfully.
382     match ops.last() {
383         Some(Ok(CreateOperationResponse { iOperation: Some(op), .. })) => {
384             assert_eq!(Ok(()), key_generations::map_ks_error(perform_sample_sign_operation(op)));
385         }
386         _ => panic!("Operation should have created successfully."),
387     }
388 }
389 
390 /// Try to create forced operations with various contexts -
391 ///   - untrusted_app
392 ///   - system_server
393 ///   - priv_app
394 ///
395 /// `PERMISSION_DENIED` error response is expected.
396 #[test]
keystore2_forced_op_perm_denied_test()397 fn keystore2_forced_op_perm_denied_test() {
398     static TARGET_CTXS: &[&str] =
399         &["u:r:untrusted_app:s0", "u:r:system_server:s0", "u:r:priv_app:s0"];
400     const USER_ID: u32 = 99;
401     const APPLICATION_ID: u32 = 10601;
402 
403     let uid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
404     let gid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
405 
406     for context in TARGET_CTXS.iter() {
407         let forced_op_fn = move || {
408             let alias = format!("ks_app_forced_op_test_key_{}", getuid());
409             let result = key_generations::map_ks_error(create_signing_operation(
410                 ForcedOp(true),
411                 KeyPurpose::SIGN,
412                 Digest::SHA_2_256,
413                 Domain::APP,
414                 -1,
415                 Some(alias),
416             ));
417             assert!(result.is_err());
418             assert_eq!(Error::Rc(ResponseCode::PERMISSION_DENIED), result.unwrap_err());
419         };
420 
421         // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
422         // `--test-threads=1`), and nothing yet done with binder.
423         unsafe {
424             run_as::run_as(context, Uid::from_raw(uid), Gid::from_raw(gid), forced_op_fn);
425         }
426     }
427 }
428 
429 /// Try to create a forced operation with `vold` context.
430 /// Should be able to create forced operation with `vold` context successfully.
431 #[test]
keystore2_forced_op_success_test()432 fn keystore2_forced_op_success_test() {
433     static TARGET_VOLD_CTX: &str = "u:r:vold:s0";
434     const USER_ID: u32 = 99;
435     const APPLICATION_ID: u32 = 10601;
436 
437     let uid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
438     let gid = USER_ID * AID_USER_OFFSET + APPLICATION_ID;
439     let forced_op_fn = move || {
440         let alias = format!("ks_vold_forced_op_key_{}", getuid());
441         create_signing_operation(
442             ForcedOp(true),
443             KeyPurpose::SIGN,
444             Digest::SHA_2_256,
445             Domain::SELINUX,
446             key_generations::SELINUX_VOLD_NAMESPACE,
447             Some(alias),
448         )
449         .expect("Client with vold context failed to create forced operation.");
450     };
451 
452     // Safety: only one thread at this point (enforced by `AndroidTest.xml` setting
453     // `--test-threads=1`), and nothing yet done with binder.
454     unsafe {
455         run_as::run_as(TARGET_VOLD_CTX, Uid::from_raw(uid), Gid::from_raw(gid), forced_op_fn);
456     }
457 }
458 
459 /// Create an operation and try to use this operation handle in multiple threads to perform
460 /// operations. Test should fail to perform an operation with an error response `OPERATION_BUSY`
461 /// when multiple threads try to access the operation handle at same time.
462 #[test]
keystore2_op_fails_operation_busy()463 fn keystore2_op_fails_operation_busy() {
464     let op_response = create_signing_operation(
465         ForcedOp(false),
466         KeyPurpose::SIGN,
467         Digest::SHA_2_256,
468         Domain::APP,
469         -1,
470         Some("op_busy_alias_test_key".to_string()),
471     )
472     .unwrap();
473 
474     let op: binder::Strong<dyn IKeystoreOperation> = op_response.iOperation.unwrap();
475 
476     let th_handle_1 = perform_op_busy_in_thread(op.clone());
477     let th_handle_2 = perform_op_busy_in_thread(op);
478 
479     let result1 = th_handle_1.join().unwrap();
480     let result2 = th_handle_2.join().unwrap();
481 
482     assert!(result1 || result2);
483 }
484 
485 /// Create an operation and use it for performing sign operation. After completing the operation
486 /// try to abort the operation. Test should fail to abort already finalized operation with error
487 /// code `INVALID_OPERATION_HANDLE`.
488 #[test]
keystore2_abort_finalized_op_fail_test()489 fn keystore2_abort_finalized_op_fail_test() {
490     let op_response = create_signing_operation(
491         ForcedOp(false),
492         KeyPurpose::SIGN,
493         Digest::SHA_2_256,
494         Domain::APP,
495         -1,
496         Some("ks_op_abort_fail_test_key".to_string()),
497     )
498     .unwrap();
499 
500     let op: binder::Strong<dyn IKeystoreOperation> = op_response.iOperation.unwrap();
501     perform_sample_sign_operation(&op).unwrap();
502     let result = key_generations::map_ks_error(op.abort());
503     assert!(result.is_err());
504     assert_eq!(Error::Km(ErrorCode::INVALID_OPERATION_HANDLE), result.unwrap_err());
505 }
506 
507 /// Create an operation and use it for performing sign operation. Before finishing the operation
508 /// try to abort the operation. Test should successfully abort the operation. After aborting try to
509 /// use the operation handle, test should fail to use already aborted operation handle with error
510 /// code `INVALID_OPERATION_HANDLE`.
511 #[test]
keystore2_op_abort_success_test()512 fn keystore2_op_abort_success_test() {
513     let op_response = create_signing_operation(
514         ForcedOp(false),
515         KeyPurpose::SIGN,
516         Digest::SHA_2_256,
517         Domain::APP,
518         -1,
519         Some("ks_op_abort_success_key".to_string()),
520     )
521     .unwrap();
522 
523     let op: binder::Strong<dyn IKeystoreOperation> = op_response.iOperation.unwrap();
524     op.update(b"my message").unwrap();
525     let result = key_generations::map_ks_error(op.abort());
526     assert!(result.is_ok());
527 
528     // Try to use the op handle after abort.
529     let result = key_generations::map_ks_error(op.finish(None, None));
530     assert!(result.is_err());
531     assert_eq!(Error::Km(ErrorCode::INVALID_OPERATION_HANDLE), result.unwrap_err());
532 }
533 
534 /// Executes an operation in a thread. Performs an `update` operation repeatedly till the user
535 /// interrupts it or encounters any error other than `OPERATION_BUSY`.
536 /// Return `false` in case of any error other than `OPERATION_BUSY`, otherwise it returns true.
perform_abort_op_busy_in_thread( op: binder::Strong<dyn IKeystoreOperation>, should_exit_clone: Arc<AtomicBool>, ) -> JoinHandle<bool>537 fn perform_abort_op_busy_in_thread(
538     op: binder::Strong<dyn IKeystoreOperation>,
539     should_exit_clone: Arc<AtomicBool>,
540 ) -> JoinHandle<bool> {
541     thread::spawn(move || {
542         loop {
543             if should_exit_clone.load(Ordering::Relaxed) {
544                 // Caller requested to exit the thread.
545                 return true;
546             }
547 
548             match key_generations::map_ks_error(op.update(b"my message")) {
549                 Ok(_) => continue,
550                 Err(Error::Rc(ResponseCode::OPERATION_BUSY)) => continue,
551                 Err(_) => return false,
552             }
553         }
554     })
555 }
556 
557 /// Create an operation and try to use same operation handle in multiple threads to perform
558 /// operations. Test tries to abort the operation and expects `abort` call to fail with the error
559 /// response `OPERATION_BUSY` as multiple threads try to access the same operation handle
560 /// simultaneously. Test tries to simulate `OPERATION_BUSY` error response from `abort` api.
561 #[test]
keystore2_op_abort_fails_with_operation_busy_error_test()562 fn keystore2_op_abort_fails_with_operation_busy_error_test() {
563     loop {
564         let op_response = create_signing_operation(
565             ForcedOp(false),
566             KeyPurpose::SIGN,
567             Digest::SHA_2_256,
568             Domain::APP,
569             -1,
570             Some("op_abort_busy_alias_test_key".to_string()),
571         )
572         .unwrap();
573         let op: binder::Strong<dyn IKeystoreOperation> = op_response.iOperation.unwrap();
574 
575         let should_exit = Arc::new(AtomicBool::new(false));
576 
577         let update_t_handle1 = perform_abort_op_busy_in_thread(op.clone(), should_exit.clone());
578         let update_t_handle2 = perform_abort_op_busy_in_thread(op.clone(), should_exit.clone());
579 
580         // Attempt to abort the operation and anticipate an 'OPERATION_BUSY' error, as multiple
581         // threads are concurrently accessing the same operation handle.
582         let result = match op.abort() {
583             Ok(_) => 0, // Operation successfully aborted.
584             Err(e) => e.service_specific_error(),
585         };
586 
587         // Notify threads to stop performing `update` operation.
588         should_exit.store(true, Ordering::Relaxed);
589 
590         let _update_op_result = update_t_handle1.join().unwrap();
591         let _update_op_result2 = update_t_handle2.join().unwrap();
592 
593         if result == ResponseCode::OPERATION_BUSY.0 {
594             // The abort call failed with an OPERATION_BUSY error, as anticipated, due to multiple
595             // threads competing for access to the same operation handle.
596             return;
597         }
598         assert_eq!(result, 0);
599     }
600 }
601