From ec39d182075dfbfd97bd62921bb15f104630018e Mon Sep 17 00:00:00 2001 From: Zhijun He Date: Fri, 16 Jan 2015 16:44:40 -0800 Subject: [PATCH] camera3: update reprocessing spec Also bump the HAL version to 3.3 for reprocessing spec update Change-Id: I37eda90970eace23c42ba9106d4db1587df8d1b4 --- include/hardware/camera3.h | 160 ++++++++++++++++++++++++++++++++++++++- include/hardware/camera_common.h | 5 +- 2 files changed, 161 insertions(+), 4 deletions(-) diff --git a/include/hardware/camera3.h b/include/hardware/camera3.h index 2bb3ba1..5fcbf0f 100644 --- a/include/hardware/camera3.h +++ b/include/hardware/camera3.h @@ -21,7 +21,7 @@ #include "camera_common.h" /** - * Camera device HAL 3.2 [ CAMERA_DEVICE_API_VERSION_3_2 ] + * Camera device HAL 3.3 [ CAMERA_DEVICE_API_VERSION_3_3 ] * * This is the current recommended version of the camera device HAL. * @@ -29,9 +29,14 @@ * android.hardware.camera2 API in LIMITED or FULL modes. * * Camera devices that support this version of the HAL must return - * CAMERA_DEVICE_API_VERSION_3_2 in camera_device_t.common.version and in + * CAMERA_DEVICE_API_VERSION_3_3 in camera_device_t.common.version and in * camera_info_t.device_version (from camera_module_t.get_camera_info). * + * CAMERA_DEVICE_API_VERSION_3_3: + * Camera modules that may contain version 3.3 devices must implement at + * least version 2.2 of the camera module interface (as defined by + * camera_module_t.common.module_api_version). + * * CAMERA_DEVICE_API_VERSION_3_2: * Camera modules that may contain version 3.2 devices must implement at * least version 2.2 of the camera module interface (as defined by @@ -54,6 +59,7 @@ * S7. Key Performance Indicator (KPI) glossary * S8. Sample Use Cases * S9. Notes on Controls and Metadata + * S10. Reprocessing flow and controls */ /** @@ -119,6 +125,9 @@ * - change the input buffer return path. The buffer is returned in * process_capture_result instead of process_capture_request. * + * 3.3: Minor revision of expanded-capability HAL: + * + * - OPAQUE and YUV reprocessing API updates. */ /** @@ -1109,6 +1118,56 @@ * as input. * - And a HAL_PIXEL_FORMAT_BLOB (JPEG) output stream. * + * S8.2 ZSL (OPAQUE) reprocessing with CAMERA3_STREAM_INPUT stream. + * + * CAMERA_DEVICE_API_VERSION_3_3: + * When OPAQUE_REPROCESSING capability is supported by the camera device, the INPUT stream + * can be used for application/framework implemented use case like Zero Shutter Lag (ZSL). + * This kind of stream will be used by the framework as follows: + * + * 1. Application/framework configures an opaque (RAW or YUV based) format output stream that is + * used to produce the ZSL output buffers. The stream pixel format will be + * HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED. + * + * 2. Application/framework configures an opaque format input stream that is used to + * send the reprocessing ZSL buffers to the HAL. The stream pixel format will + * also be HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED. + * + * 3. Application/framework configures a YUV/JPEG output stream that is used to receive the + * reprocessed data. The stream pixel format will be YCbCr_420/HAL_PIXEL_FORMAT_BLOB. + * + * 4. Application/framework picks a ZSL buffer from the ZSL output stream when a ZSL capture is + * issued by the application, and sends the data back as an input buffer in a + * reprocessing request, then sends to the HAL for reprocessing. + * + * 5. The HAL sends back the output YUV/JPEG result to framework. + * + * The HAL can select the actual opaque buffer format and configure the ISP pipeline + * appropriately based on the HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED format and + * the gralloc usage flag GRALLOC_USAGE_HW_CAMERA_ZSL. + + * S8.3 YUV reprocessing with CAMERA3_STREAM_INPUT stream. + * + * When YUV reprocessing is supported by the HAL, the INPUT stream + * can be used for the YUV reprocessing use cases like lucky-shot and image fusion. + * This kind of stream will be used by the framework as follows: + * + * 1. Application/framework configures an YCbCr_420 format output stream that is + * used to produce the output buffers. + * + * 2. Application/framework configures an YCbCr_420 format input stream that is used to + * send the reprocessing YUV buffers to the HAL. + * + * 3. Application/framework configures a YUV/JPEG output stream that is used to receive the + * reprocessed data. The stream pixel format will be YCbCr_420/HAL_PIXEL_FORMAT_BLOB. + * + * 4. Application/framework processes the output buffers (could be as simple as picking + * an output buffer directly) from the output stream when a capture is issued, and sends + * the data back as an input buffer in a reprocessing request, then sends to the HAL + * for reprocessing. + * + * 5. The HAL sends back the output YUV/JPEG result to framework. + * */ /** @@ -1137,6 +1196,100 @@ * be included in the 'available modes' tag to represent this operating * mode. */ + +/** + * S10. Reprocessing flow and controls + * + * This section describes the OPAQUE and YUV reprocessing flow and controls. OPAQUE reprocessing + * uses an opaque format that is not directly application-visible, and the application can + * only select some of the output buffers and send back to HAL for reprocessing, while YUV + * reprocessing gives the application opportunity to process the buffers before reprocessing. + * + * S8 gives the stream configurations for the typical reprocessing uses cases, + * this section specifies the buffer flow and controls in more details. + * + * S10.1 OPAQUE (typically for ZSL use case) reprocessing flow and controls + * + * For OPAQUE reprocessing (e.g. ZSL) use case, after the application creates the specific + * output and input streams, runtime buffer flow and controls are specified as below: + * + * 1. Application starts output streaming by sending repeating requests for output + * opaque buffers and preview. The buffers are held by an application + * maintained circular buffer. The requests are based on CAMERA3_TEMPLATE_ZERO_SHUTTER_LAG + * capture template, which should have all necessary settings that guarantee output + * frame rate is not slowed down relative to sensor output frame rate. + * + * 2. When a capture is issued, the application selects one output buffer based + * on application buffer selection logic, e.g. good AE and AF statistics etc. + * Application then creates an reprocess request based on the capture result associated + * with this selected buffer. The selected output buffer is now added to this reprocess + * request as an input buffer, the output buffer of this reprocess request should be + * either JPEG output buffer or YUV output buffer, or both, depending on the application + * choice. + * + * 3. Application then alters the reprocess settings to get best image quality. The HAL must + * support and only support below controls if the HAL support OPAQUE_REPROCESSING capability: + * - android.jpeg.* (if JPEG buffer is included as one of the output) + * - android.noiseReduction.mode (change to HIGH_QUALITY if it is supported) + * - android.edge.mode (change to HIGH_QUALITY if it is supported) + * All other controls must be ignored by the HAL. + * 4. HAL processed the input buffer and return the output buffers in the capture results + * as normal. + * + * S10.2 YUV reprocessing flow and controls + * + * The YUV reprocessing buffer flow is similar as OPAQUE reprocessing, with below difference: + * + * 1. Application may want to have finer granularity control of the intermediate YUV images + * (before reprocessing). For example, application may choose + * - android.noiseReduction.mode == MINIMAL + * to make sure the no YUV domain noise reduction has applied to the output YUV buffers, + * then it can do its own advanced noise reduction on them. For OPAQUE reprocessing case, this + * doesn't matter, as long as the final reprocessed image has the best quality. + * 2. Application may modify the YUV output buffer data. For example, for image fusion use + * case, where multiple output images are merged together to improve the signal-to-noise + * ratio (SNR). The input buffer may be generated from multiple buffers by the application. + * To avoid excessive amount of noise reduction and insufficient amount of edge enhancement + * being applied to the input buffer, the application can hint the HAL how much effective + * exposure time improvement has been done by the application, then the HAL can adjust the + * noise reduction and edge enhancement paramters to get best reprocessed image quality. + * Below tag can be used for this purpose: + * - android.reprocess.effectiveExposureFactor + * The value would be exposure time increase factor applied to the original output image, + * for example, if there are N image merged, the exposure time increase factor would be up + * to sqrt(N). See this tag spec for more details. + * + * S10.3 Reprocessing pipeline characteristics + * + * Reprocessing pipeline has below different characteristics comparing with normal output + * pipeline: + * + * 1. The reprocessing result can be returned ahead of the pending normal output results. But + * the FIFO ordering must be maintained for all reprocessing results. For example, there are + * below requests (A stands for output requests, B stands for reprocessing requests) + * being processed by the HAL: + * A1, A2, A3, A4, B1, A5, B2, A6... + * result of B1 can be returned before A1-A4, but result of B2 must be returned after B1. + * 2. Single input rule: For a given reprocessing request, all output buffers must be from the + * input buffer, rather than sensor output. For example, if a reprocess request include both + * JPEG and preview buffers, all output buffers must be produced from the input buffer + * included by the reprocessing request, rather than sensor. The HAL must not output preview + * buffers from sensor, while output JPEG buffer from the input buffer. + * 3. Input buffer will be from camera output directly (ZSL case) or indirectly(image fusion + * case). For the case where buffer is modified, the size will remain same. The HAL can + * notify CAMERA3_MSG_ERROR_REQUEST if buffer from unknown source is sent. + * 4. Result as reprocessing request: The HAL can expect that a reprocessing request is a copy + * of one of the output results with minor allowed setting changes. The HAL can notify + * CAMERA3_MSG_ERROR_REQUEST if a request from unknown source is issued. + * 5. Output buffers may not be used as inputs across the configure stream boundary, This is + * because an opaque stream like the ZSL output stream may have different actual image size + * inside of the ZSL buffer to save power and bandwidth for smaller resolution JPEG capture. + * The HAL may notify CAMERA3_MSG_ERROR_REQUEST if this case occurs. + * 6. HAL Reprocess requests error reporting during flush should follow the same rule specified + * by flush() method. + * + */ + __BEGIN_DECLS struct camera3_device; @@ -1184,6 +1337,9 @@ typedef enum camera3_stream_type { * quality images (that otherwise would cause a frame rate performance * loss), or to do off-line reprocessing. * + * CAMERA_DEVICE_API_VERSION_3_3: + * The typical use cases are OPAQUE (typically ZSL) and YUV reprocessing, + * see S8.2, S8.3 and S10 for more details. */ CAMERA3_STREAM_INPUT = 1, diff --git a/include/hardware/camera_common.h b/include/hardware/camera_common.h index 47a01c9..7d5dce5 100644 --- a/include/hardware/camera_common.h +++ b/include/hardware/camera_common.h @@ -142,10 +142,11 @@ __BEGIN_DECLS #define CAMERA_DEVICE_API_VERSION_3_0 HARDWARE_DEVICE_API_VERSION(3, 0) #define CAMERA_DEVICE_API_VERSION_3_1 HARDWARE_DEVICE_API_VERSION(3, 1) #define CAMERA_DEVICE_API_VERSION_3_2 HARDWARE_DEVICE_API_VERSION(3, 2) +#define CAMERA_DEVICE_API_VERSION_3_3 HARDWARE_DEVICE_API_VERSION(3, 3) -// Device version 3.2 is current, older HAL camera device versions are not +// Device version 3.3 is current, older HAL camera device versions are not // recommended for new devices. -#define CAMERA_DEVICE_API_VERSION_CURRENT CAMERA_DEVICE_API_VERSION_3_2 +#define CAMERA_DEVICE_API_VERSION_CURRENT CAMERA_DEVICE_API_VERSION_3_3 /** * Defined in /system/media/camera/include/system/camera_metadata.h -- 2.11.0