facade/webcam/WebCamFacade.java

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.googlecode.android_scripting.facade.webcam;

import android.app.Service;
import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.hardware.Camera;
import android.hardware.Camera.Parameters;
import android.hardware.Camera.PreviewCallback;
import android.hardware.Camera.Size;
import android.util.Base64;
import android.view.SurfaceHolder;
import android.view.SurfaceHolder.Callback;
import android.view.SurfaceView;
import android.view.WindowManager;

import com.googlecode.android_scripting.BaseApplication;
import com.googlecode.android_scripting.FutureActivityTaskExecutor;
import com.googlecode.android_scripting.Log;
import com.googlecode.android_scripting.SimpleServer.SimpleServerObserver;
import com.googlecode.android_scripting.SingleThreadExecutor;
import com.googlecode.android_scripting.facade.EventFacade;
import com.googlecode.android_scripting.facade.FacadeManager;
import com.googlecode.android_scripting.future.FutureActivityTask;
import com.googlecode.android_scripting.jsonrpc.RpcReceiver;
import com.googlecode.android_scripting.rpc.Rpc;
import com.googlecode.android_scripting.rpc.RpcDefault;
import com.googlecode.android_scripting.rpc.RpcOptional;
import com.googlecode.android_scripting.rpc.RpcParameter;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;

/**
 * Manages access to camera streaming.
 * <br>
 * <h3>Usage Notes</h3>
 * <br><b>webCamStart</b> and <b>webCamStop</b> are used to start and stop an Mpeg stream on a
 * given port. <b>webcamAdjustQuality</b> is used to ajust the quality of the streaming video.
 * <br><b>cameraStartPreview</b> is used to get access to the camera preview screen. It will
 * generate "preview" events as images become available.
 * <br>The preview has two modes: data or file. If you pass a non-blank, writable file path to
 * the <b>cameraStartPreview</b> it will store jpg images in that folder.
 * It is up to the caller to clean up these files after the fact. If no file element is provided,
 * the event will include the image data as a base64 encoded string.
 * <h3>Event details</h3>
 * <br>The data element of the preview event will be a map, with the following elements defined.
 * <ul>
 * <li><b>format</b> - currently always "jpeg"
 * <li><b>width</b> - image width (in pixels)
 * <li><b>height</b> - image height (in pixels)
 * <li><b>quality</b> - JPEG quality. Number from 1-100
 * <li><b>filename</b> - Name of file where image was saved. Only relevant if filepath defined.
 * <li><b>error</b> - included if there was an IOException saving file, ie, disk full or path write
 * protected.
 * <li><b>encoding</b> - Data encoding. If filepath defined, will be "file" otherwise "base64"
 * <li><b>data</b> - Base64 encoded image data.
 * </ul>
 * <br>Note that "filename", "error" and "data" are mutual exclusive.
 * <br>
 * <br>The webcam and preview modes use the same resources, so you can't use them both at the same
 * time. Stop one mode before starting the other.
 */
public class WebCamFacade extends RpcReceiver {

    private final Service mService;
    private final Executor mJpegCompressionExecutor = new SingleThreadExecutor();
    private final ByteArrayOutputStream mJpegCompressionBuffer = new ByteArrayOutputStream();

    private volatile byte[] mJpegData;

    private CountDownLatch mJpegDataReady;
    private boolean mStreaming;
    private int mPreviewHeight;
    private int mPreviewWidth;
    private int mJpegQuality;

    private MjpegServer mJpegServer;
    private FutureActivityTask<SurfaceHolder> mPreviewTask;
    private Camera mCamera;
    private Parameters mParameters;
    private final EventFacade mEventFacade;
    private boolean mPreview;
    private File mDest;

    private final PreviewCallback mPreviewCallback = new PreviewCallback() {
        @Override
        public void onPreviewFrame(final byte[] data, final Camera camera) {
            mJpegCompressionExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    mJpegData = compressYuvToJpeg(data);
                    mJpegDataReady.countDown();
                    if (mStreaming) {
                        camera.setOneShotPreviewCallback(mPreviewCallback);
                    }
                }
            });
        }
    };

    private final PreviewCallback mPreviewEvent = new PreviewCallback() {
        @Override
        public void onPreviewFrame(final byte[] data, final Camera camera) {
            mJpegCompressionExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    mJpegData = compressYuvToJpeg(data);
                    Map<String, Object> map = new HashMap<String, Object>();
                    map.put("format", "jpeg");
                    map.put("width", mPreviewWidth);
                    map.put("height", mPreviewHeight);
                    map.put("quality", mJpegQuality);
                    if (mDest != null) {
                        try {
                            File dest = File.createTempFile("prv", ".jpg", mDest);
                            OutputStream output = new FileOutputStream(dest);
                            output.write(mJpegData);
                            output.close();
                            map.put("encoding", "file");
                            map.put("filename", dest.toString());
                        } catch (IOException e) {
                            map.put("error", e.toString());
                        }
                    } else {
                        map.put("encoding", "Base64");
                        map.put("data", Base64.encodeToString(mJpegData, Base64.DEFAULT));
                    }
                    mEventFacade.postEvent("preview", map);
                    if (mPreview) {
                        camera.setOneShotPreviewCallback(mPreviewEvent);
                    }
                }
            });
        }
    };

    public WebCamFacade(FacadeManager manager) {
        super(manager);
        mService = manager.getService();
        mJpegDataReady = new CountDownLatch(1);
        mEventFacade = manager.getReceiver(EventFacade.class);
    }

    private byte[] compressYuvToJpeg(final byte[] yuvData) {
        mJpegCompressionBuffer.reset();
        YuvImage yuvImage =
                new YuvImage(yuvData, ImageFormat.NV21, mPreviewWidth, mPreviewHeight, null);
        yuvImage.compressToJpeg(new Rect(0, 0, mPreviewWidth, mPreviewHeight), mJpegQuality,
                mJpegCompressionBuffer);
        return mJpegCompressionBuffer.toByteArray();
    }

    /**
     * Starts an MJPEG stream and returns a Tuple of address and port for the stream.
     * @param resolutionLevel increasing this number provides higher resolution
     * @param jpegQuality a number from 0-100
     * @param port If port is specified, the webcam service will bind to port, otherwise it will
     *             pick any available port.
     * @return a Tuple of address and port for the stream
     * @throws Exception upon failure to open the webcam or start the server
     */
    @Rpc(description = "Starts an MJPEG stream and returns a Tuple of address and port "
            + "for the stream.")
    public InetSocketAddress webcamStart(
            @RpcParameter(name = "resolutionLevel",
                          description = "increasing this number provides higher resolution")
            @RpcDefault("0")
                    Integer resolutionLevel,
            @RpcParameter(name = "jpegQuality", description = "a number from 0-100")
            @RpcDefault("20")
                    Integer jpegQuality,
            @RpcParameter(name = "port",
                          description = "If port is specified, the webcam service will bind to "
                                  + "port, otherwise it will pick any available port.")
            @RpcDefault("0")
                    Integer port)
            throws Exception {
        try {
            openCamera(resolutionLevel, jpegQuality);
            return startServer(port);
        } catch (Exception e) {
            webcamStop();
            throw e;
        }
    }

    private InetSocketAddress startServer(Integer port) {
        mJpegServer = new MjpegServer(new JpegProvider() {
            @Override
            public byte[] getJpeg() {
                try {
                    mJpegDataReady.await();
                } catch (InterruptedException e) {
                    Log.e(e);
                }
                return mJpegData;
            }
        });
        mJpegServer.addObserver(new SimpleServerObserver() {
            @Override
            public void onDisconnect() {
                if (mJpegServer.getNumberOfConnections() == 0 && mStreaming) {
                    stopStream();
                }
            }

            @Override
            public void onConnect() {
                if (!mStreaming) {
                    startStream();
                }
            }
        });
        return mJpegServer.startPublic(port);
    }

    private void stopServer() {
        if (mJpegServer != null) {
            mJpegServer.shutdown();
            mJpegServer = null;
        }
    }

    /**
     * Adjusts the quality of the webcam stream while it is running.
     * @param resolutionLevel increasing this number provides higher resolution
     * @param jpegQuality a number from 0-100
     * @throws Exception if the webcam is not streaming or the camera is unable to open
     */
    @Rpc(description = "Adjusts the quality of the webcam stream while it is running.")
    public void webcamAdjustQuality(
            @RpcParameter(name = "resolutionLevel",
                          description = "increasing this number provides higher resolution")
            @RpcDefault("0")
                    Integer resolutionLevel,
            @RpcParameter(name = "jpegQuality", description = "a number from 0-100")
            @RpcDefault("20")
                    Integer jpegQuality)
            throws Exception {
        if (!mStreaming) {
            throw new IllegalStateException("Webcam not streaming.");
        }
        stopStream();
        releaseCamera();
        openCamera(resolutionLevel, jpegQuality);
        startStream();
    }

    private void openCamera(Integer resolutionLevel, Integer jpegQuality) throws IOException,
            InterruptedException {
        mCamera = Camera.open();
        mParameters = mCamera.getParameters();
        mParameters.setPictureFormat(ImageFormat.JPEG);
        mParameters.setPreviewFormat(ImageFormat.JPEG);
        List<Size> supportedPreviewSizes = mParameters.getSupportedPreviewSizes();
        Collections.sort(supportedPreviewSizes, new Comparator<Size>() {
            @Override
            public int compare(Size o1, Size o2) {
                return o1.width - o2.width;
            }
        });
        Size previewSize =
                supportedPreviewSizes.get(
                        Math.min(resolutionLevel, supportedPreviewSizes.size() - 1));
        mPreviewHeight = previewSize.height;
        mPreviewWidth = previewSize.width;
        mParameters.setPreviewSize(mPreviewWidth, mPreviewHeight);
        mJpegQuality = Math.min(Math.max(jpegQuality, 0), 100);
        mCamera.setParameters(mParameters);
        // TODO(damonkohler): Rotate image based on orientation.
        mPreviewTask = createPreviewTask();
        mCamera.startPreview();
    }

    private void startStream() {
        mStreaming = true;
        mCamera.setOneShotPreviewCallback(mPreviewCallback);
    }

    private void stopStream() {
        mJpegDataReady = new CountDownLatch(1);
        mStreaming = false;
        if (mPreviewTask != null) {
            mPreviewTask.finish();
            mPreviewTask = null;
        }
    }

    private void releaseCamera() {
        if (mCamera != null) {
            mCamera.release();
            mCamera = null;
        }
        mParameters = null;
    }

    /** Stops the webcam stream. */
    @Rpc(description = "Stops the webcam stream.")
    public void webcamStop() {
        stopServer();
        stopStream();
        releaseCamera();
    }

    private FutureActivityTask<SurfaceHolder> createPreviewTask() throws IOException,
            InterruptedException {
        FutureActivityTask<SurfaceHolder> task = new FutureActivityTask<SurfaceHolder>() {
            @Override
            public void onCreate() {
                super.onCreate();
                final SurfaceView view = new SurfaceView(getActivity());
                getActivity().setContentView(view);
                getActivity().getWindow().setSoftInputMode(
                        WindowManager.LayoutParams.SOFT_INPUT_STATE_HIDDEN);
                //view.getHolder().setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS);
                view.getHolder().addCallback(new Callback() {
                    @Override
                    public void surfaceDestroyed(SurfaceHolder holder) {
                    }

                    @Override
                    public void surfaceCreated(SurfaceHolder holder) {
                        setResult(view.getHolder());
                    }

                    @Override
                    public void surfaceChanged(SurfaceHolder holder,
                                               int format,
                                               int width,
                                               int height) {
                    }
                });
            }
        };
        FutureActivityTaskExecutor taskExecutor =
                ((BaseApplication) mService.getApplication()).getTaskExecutor();
        taskExecutor.execute(task);
        mCamera.setPreviewDisplay(task.getResult());
        return task;
    }

    /**
     * Start Preview Mode. Throws 'preview' events.
     * @param resolutionLevel increasing this number provides higher resolution
     * @param jpegQuality a number from 0-100
     * @param filepath the path to store jpeg files
     * @return a Tuple of address and port for the stream
     * @throws InterruptedException if interrupted while trying to open the camera
     */
    @Rpc(description = "Start Preview Mode. Throws 'preview' events.",
            returns = "True if successful")
    public boolean cameraStartPreview(
            @RpcParameter(name = "resolutionLevel",
                          description = "increasing this number provides higher resolution")
            @RpcDefault("0")
                    Integer resolutionLevel,
            @RpcParameter(name = "jpegQuality", description = "a number from 0-100")
            @RpcDefault("20")
                    Integer jpegQuality,
            @RpcParameter(name = "filepath", description = "Path to store jpeg files.")
            @RpcOptional
                    String filepath)
            throws InterruptedException {
        mDest = null;
        if (filepath != null && (filepath.length() > 0)) {
            mDest = new File(filepath);
            if (!mDest.exists()) mDest.mkdirs();
            if (!(mDest.isDirectory() && mDest.canWrite())) {
                return false;
            }
        }

        try {
            openCamera(resolutionLevel, jpegQuality);
        } catch (IOException e) {
            Log.e(e);
            return false;
        }
        startPreview();
        return true;
    }

    /** Stops the preview mode. */
    @Rpc(description = "Stop the preview mode.")
    public void cameraStopPreview() {
        stopPreview();
    }

    private void startPreview() {
        mPreview = true;
        mCamera.setOneShotPreviewCallback(mPreviewEvent);
    }

    private void stopPreview() {
        mPreview = false;
        if (mPreviewTask != null) {
            mPreviewTask.finish();
            mPreviewTask = null;
        }
        releaseCamera();
    }

    @Override
    public void shutdown() {
        mPreview = false;
        webcamStop();
    }
}