1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.core;
18 
19 import android.graphics.PixelFormat;
20 
21 import androidx.annotation.IntDef;
22 import androidx.annotation.RestrictTo;
23 
24 import org.jspecify.annotations.NonNull;
25 
26 import java.lang.annotation.Retention;
27 import java.lang.annotation.RetentionPolicy;
28 import java.util.concurrent.Executor;
29 
30 /**
31  * Post-processing effect for images.
32  *
33  * <p>This interface is for post-processing images. The input is an image from the camera, with the
34  * instructions on how to process it; the output is a processed image. CameraX forwards images to
35  * the implementation, and delivers the processed images to back the app.
36  *
37  * <p>Currently, it can only be used with the {@link ImageCapture} by targeting
38  * {@link CameraEffect#IMAGE_CAPTURE}.
39  *
40  * <p>If the implementation fails to process the input, it should throw
41  * {@link ProcessingException}. The error will be caught by CameraX and propagated to the app via
42  * error callbacks such as {@link ImageCapture.OnImageSavedCallback#onError} or
43  * {@link ImageCapture.OnImageCapturedCallback#onError}.
44  *
45  * <p>Code sample:
46  * <pre><code>
47  *  class ImageProcessorImpl implements ImageProcessor {
48  *      Response process(Request request) throws ProcessingException {
49  *          try {
50  *              ImageProxy image = request.getInputImages().get(0);
51  *              ByteBuffer byteBuffer = image.getPlanes()[0];
52  *              // Process the content of byteBuffer and create a Response object.
53  *          } catch(Exception e) {
54  *              throws new ProcessingException(e);
55  *          }
56  *      }
57  *  }
58  * </code></pre>
59  *
60  * @see CameraEffect
61  */
62 public interface ImageProcessor {
63 
64     /**
65      * Accepts original image from CameraX and returns processed image.
66      *
67      * <p>CameraX invokes this method for each new incoming image. It's invoked on the
68      * {@link Executor} provided in {@link CameraEffect}'s constructor. It might be called in
69      * parallel, should the {@link Executor} allow multi-threading. The implementation must block
70      * the current calling thread until the output image is returned.
71      *
72      * <p>The implementation must follow the instruction in the {@link Request} to process the
73      * input image. For example, it must produce an output image with the format following the
74      * JavaDoc of {@link Request#getInputImage()}. Failing to do so might cause the processing to
75      * fail. For example, for {@link ImageCapture}, it will cause the
76      * {@link ImageCapture#takePicture} call to fail.
77      *
78      * <p>The implementation must throw a {@link ProcessingException} if it fails to process the
79      * {@link Request}. CameraX will catch the error and deliver it to the app via error
80      * callbacks. For {@link ImageCapture}, the error callbacks are
81      * {@link ImageCapture.OnImageCapturedCallback#onError} or
82      * {@link ImageCapture.OnImageSavedCallback#onError}.
83      *
84      * @param request a {@link Request} that contains the original image.
85      * @return a {@link Response} that contains the processed image.
86      * @throws ProcessingException if the implementation fails to process the {@link Request}.
87      */
process(@onNull Request request)88     @NonNull Response process(@NonNull Request request) throws ProcessingException;
89 
90     /**
91      * Valid output formats.
92      *
93      * <p>{@link Request#getOutputFormat()} can only return the formats defined by this annotation.
94      */
95     @Retention(RetentionPolicy.SOURCE)
96     @RestrictTo(RestrictTo.Scope.LIBRARY)
97     @IntDef(value = {PixelFormat.RGBA_8888})
98     @interface OutputFormats {
99     }
100 
101     /**
102      * A request for processing one or multiple {@link ImageProxy}.
103      */
104     interface Request {
105 
106         /**
107          * Gets the input images.
108          *
109          * <p>Return a single image captured by the camera. The implementation should check the
110          * format of the image before processing it. For example, checking the value of
111          * {@link ImageProxy#getFormat()}, {@link ImageProxy.PlaneProxy#getRowStride()} and/or
112          * {@link ImageProxy.PlaneProxy#getPixelStride()}.
113          *
114          * <p>Currently, the image format is always {@link PixelFormat#RGBA_8888} with pixel
115          * stride equals to 4 and row stride equals to width * 4.
116          */
getInputImage()117         @NonNull ImageProxy getInputImage();
118 
119         /**
120          * Gets the output image format.
121          *
122          * <p>The return value will one of the values in the table. The implementation must
123          * create the {@link Response} {@link ImageProxy} following the corresponding
124          * instruction, or the processing may fail.
125          *
126          * <table>
127          * <tr>
128          *     <th>Value</th>
129          *     <th>Instruction</th>
130          * </tr>
131          * <tr>
132          *     <td>{@link PixelFormat#RGBA_8888}</td>
133          *     <td>The output image must contain a single plane with a pixel stride of 4 and a
134          *     row stride of width * 4. e.g. each pixel is stored on 4 bytes and each RGBA
135          *     channel is stored with 8 bits of precision. For more details, see the JavaDoc of
136          *     {@code Bitmap.Config#ARGB_8888}.</td>
137          * </tr>
138          * </table>
139          */
140         @OutputFormats
getOutputFormat()141         int getOutputFormat();
142     }
143 
144     /**
145      * A response for returning a processed {@link ImageProxy} to CameraX.
146      */
147     interface Response {
148 
149         /**
150          * Gets the output image returned by the {@link ImageProcessor}.
151          *
152          * <p>{@link ImageProcessor} should implement the {@link ImageProxy} and
153          * {@link ImageProxy.PlaneProxy} interfaces to create the return value. Once
154          * return, CameraX will inject the image back to the processing pipeline.
155          *
156          * <p>The {@link ImageProxy} must follow the instruction in the request, or CameraX may
157          * throw error. For example, the image format must match the description of the
158          * {@link Request#getOutputFormat()}.
159          *
160          * @return the output image.
161          */
getOutputImage()162         @NonNull ImageProxy getOutputImage();
163     }
164 }
165