1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.cts.rs; 18 19 import android.graphics.Bitmap; 20 import android.hardware.camera2.CameraCharacteristics; 21 import android.hardware.camera2.CameraMetadata; 22 import android.hardware.camera2.CaptureResult; 23 import android.hardware.camera2.params.ColorSpaceTransform; 24 import android.hardware.camera2.params.LensShadingMap; 25 import android.renderscript.Allocation; 26 import android.renderscript.Element; 27 import android.renderscript.Float3; 28 import android.renderscript.Float4; 29 import android.renderscript.Int4; 30 import android.renderscript.Matrix3f; 31 import android.renderscript.RenderScript; 32 import android.renderscript.Type; 33 34 import android.hardware.camera2.cts.ScriptC_raw_converter; 35 import android.util.Log; 36 import android.util.Rational; 37 import android.util.SparseIntArray; 38 39 import java.util.Arrays; 40 41 /** 42 * Utility class providing methods for rendering RAW16 images into other colorspaces. 43 */ 44 public class RawConverter { 45 private static final String TAG = "RawConverter"; 46 private static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); 47 48 /** 49 * Matrix to convert from CIE XYZ colorspace to sRGB, Bradford-adapted to D65. 50 */ 51 private static final float[] sXYZtoRGBBradford = new float[] { 52 3.1338561f, -1.6168667f, -0.4906146f, 53 -0.9787684f, 1.9161415f, 0.0334540f, 54 0.0719453f, -0.2289914f, 1.4052427f 55 }; 56 57 /** 58 * Matrix to convert from the ProPhoto RGB colorspace to CIE XYZ colorspace. 59 */ 60 private static final float[] sProPhotoToXYZ = new float[] { 61 0.797779f, 0.135213f, 0.031303f, 62 0.288000f, 0.711900f, 0.000100f, 63 0.000000f, 0.000000f, 0.825105f 64 }; 65 66 /** 67 * Matrix to convert from CIE XYZ colorspace to ProPhoto RGB colorspace. 68 */ 69 private static final float[] sXYZtoProPhoto = new float[] { 70 1.345753f, -0.255603f, -0.051025f, 71 -0.544426f, 1.508096f, 0.020472f, 72 0.000000f, 0.000000f, 1.211968f 73 }; 74 75 /** 76 * Coefficients for a 3rd order polynomial, ordered from highest to lowest power. This 77 * polynomial approximates the default tonemapping curve used for ACR3. 78 */ 79 private static final float[] DEFAULT_ACR3_TONEMAP_CURVE_COEFFS = new float[] { 80 -0.7836f, 0.8469f, 0.943f, 0.0209f 81 }; 82 83 /** 84 * The D50 whitepoint coordinates in CIE XYZ colorspace. 85 */ 86 private static final float[] D50_XYZ = new float[] { 0.9642f, 1, 0.8249f }; 87 88 /** 89 * An array containing the color temperatures for standard reference illuminants. 90 */ 91 private static final SparseIntArray sStandardIlluminants = new SparseIntArray(); 92 private static final int NO_ILLUMINANT = -1; 93 static { sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_CLOUDY_WEATHER, 6504)94 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_CLOUDY_WEATHER, 95 6504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT, 6504)96 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT, 6504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D65, 6504)97 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D65, 6504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FINE_WEATHER, 5003)98 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FINE_WEATHER, 5003); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D50, 5003)99 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D50, 5003); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FLASH, 5503)100 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FLASH, 5503); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D55, 5503)101 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D55, 5503); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_SHADE, 7504)102 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_SHADE, 7504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D75, 7504)103 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D75, 7504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_ISO_STUDIO_TUNGSTEN, 2856)104 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_ISO_STUDIO_TUNGSTEN, 105 2856); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_TUNGSTEN, 2856)106 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_TUNGSTEN, 2856); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_A, 2856)107 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_A, 2856); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAY_WHITE_FLUORESCENT, 4874)108 sStandardIlluminants.append( 109 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAY_WHITE_FLUORESCENT, 4874); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_B, 4874)110 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_B, 4874); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_C, 6774)111 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_C, 6774); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT_FLUORESCENT, 6430)112 sStandardIlluminants.append( 113 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT_FLUORESCENT, 6430); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_COOL_WHITE_FLUORESCENT, 4230)114 sStandardIlluminants.append( 115 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_COOL_WHITE_FLUORESCENT, 4230); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_WHITE_FLUORESCENT, 3450)116 sStandardIlluminants.append( 117 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_WHITE_FLUORESCENT, 3450); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FLUORESCENT, 2940)118 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_FLUORESCENT, 2940); 119 } 120 121 /** 122 * Utility class wrapping Bayer specific DNG metadata. 123 */ 124 static class DngBayerMetadata { 125 public final int referenceIlluminant1; 126 public final int referenceIlluminant2; 127 public final float[] calibrationTransform1; 128 public final float[] calibrationTransform2; 129 public final float[] colorMatrix1; 130 public final float[] colorMatrix2; 131 public final float[] forwardTransform1; 132 public final float[] forwardTransform2; 133 public final Rational[/*3*/] neutralColorPoint; 134 135 /** 136 * Convert a 9x9 {@link ColorSpaceTransform} to a matrix and write the matrix into the 137 * output. 138 * 139 * @param xform a {@link ColorSpaceTransform} to transform. 140 * @param output the 3x3 matrix to overwrite. 141 */ convertColorspaceTransform(ColorSpaceTransform xform, float[] output)142 private static void convertColorspaceTransform(ColorSpaceTransform xform, 143 /*out*/float[] output) { 144 for (int i = 0; i < 3; i++) { 145 for (int j = 0; j < 3; j++) { 146 output[i * 3 + j] = xform.getElement(j, i).floatValue(); 147 } 148 } 149 } 150 151 /** 152 * Constructor to parse static and dynamic metadata into DNG metadata. 153 */ DngBayerMetadata(CameraCharacteristics staticMetadata, CaptureResult dynamicMetadata)154 public DngBayerMetadata(CameraCharacteristics staticMetadata, 155 CaptureResult dynamicMetadata) { 156 referenceIlluminant1 = 157 staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT1); 158 if (staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT2) != null) { 159 referenceIlluminant2 = 160 staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT2); 161 } else { 162 referenceIlluminant2 = referenceIlluminant1; 163 } 164 calibrationTransform1 = new float[9]; 165 calibrationTransform2 = new float[9]; 166 convertColorspaceTransform( 167 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM1), 168 calibrationTransform1); 169 if (staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM2) != null) { 170 convertColorspaceTransform( 171 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM2), 172 calibrationTransform2); 173 } else { 174 convertColorspaceTransform( 175 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM1), 176 calibrationTransform2); 177 } 178 colorMatrix1 = new float[9]; 179 colorMatrix2 = new float[9]; 180 convertColorspaceTransform( 181 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM1), 182 colorMatrix1); 183 if (staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM2) != null) { 184 convertColorspaceTransform( 185 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM2), 186 colorMatrix2); 187 } else { 188 convertColorspaceTransform( 189 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM1), 190 colorMatrix2); 191 } 192 forwardTransform1 = new float[9]; 193 forwardTransform2 = new float[9]; 194 convertColorspaceTransform( 195 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX1), 196 forwardTransform1); 197 if (staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX2) != null) { 198 convertColorspaceTransform( 199 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX2), 200 forwardTransform2); 201 } else { 202 convertColorspaceTransform( 203 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX1), 204 forwardTransform2); 205 } 206 207 neutralColorPoint = dynamicMetadata.get(CaptureResult.SENSOR_NEUTRAL_COLOR_POINT); 208 } 209 } 210 211 /** 212 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 213 * 214 * <p> This function applies the operations roughly outlined in the Adobe DNG specification 215 * using the provided metadata about the image sensor. Sensor data for Android devices is 216 * assumed to be relatively linear, and no extra linearization step is applied here. The 217 * following operations are applied in the given order:</p> 218 * 219 * <ul> 220 * <li> 221 * Black level subtraction - the black levels given in the SENSOR_BLACK_LEVEL_PATTERN 222 * tag are subtracted from the corresponding raw pixels. 223 * </li> 224 * <li> 225 * Rescaling - each raw pixel is scaled by 1/(white level - black level). 226 * </li> 227 * <li> 228 * Lens shading correction - the interpolated gains from the gain map defined in the 229 * STATISTICS_LENS_SHADING_CORRECTION_MAP are applied to each raw pixel. 230 * </li> 231 * <li> 232 * Clipping - each raw pixel is clipped to a range of [0.0, 1.0]. 233 * </li> 234 * <li> 235 * Demosaic - the RGB channels for each pixel are retrieved from the Bayer mosaic 236 * of raw pixels using a simple bilinear-interpolation demosaicing algorithm. 237 * </li> 238 * <li> 239 * Colorspace transform to wide-gamut RGB - each pixel is mapped into a 240 * wide-gamut colorspace (in this case ProPhoto RGB is used) from the sensor 241 * colorspace. 242 * </li> 243 * <li> 244 * Tonemapping - A basic tonemapping curve using the default from ACR3 is applied 245 * (no further exposure compensation is applied here, though this could be improved). 246 * </li> 247 * <li> 248 * Colorspace transform to final RGB - each pixel is mapped into linear sRGB colorspace. 249 * </li> 250 * <li> 251 * Gamma correction - each pixel is gamma corrected using γ=2.2 to map into sRGB 252 * colorspace for viewing. 253 * </li> 254 * <li> 255 * Packing - each pixel is scaled so that each color channel has a range of [0, 255], 256 * and is packed into an Android bitmap. 257 * </li> 258 * </ul> 259 * 260 * <p> Arguments given here are assumed to come from the values for the corresponding 261 * {@link CameraCharacteristics.Key}s defined for the camera that produced this RAW16 buffer. 262 * </p> 263 * @param rs a {@link RenderScript} context to use. 264 * @param inputWidth width of the input RAW16 image in pixels. 265 * @param inputHeight height of the input RAW16 image in pixels. 266 * @param inputStride stride of the input RAW16 image in bytes. 267 * @param rawImageInput a byte array containing a RAW16 image. 268 * @param staticMetadata the {@link CameraCharacteristics} for this RAW capture. 269 * @param dynamicMetadata the {@link CaptureResult} for this RAW capture. 270 * @param outputOffsetX the offset width into the raw image of the left side of the output 271 * rectangle. 272 * @param outputOffsetY the offset height into the raw image of the top side of the output 273 * rectangle. 274 * @param argbOutput a {@link Bitmap} to output the rendered RAW image into. The height and 275 * width of this bitmap along with the output offsets are used to determine 276 * the dimensions and offset of the output rectangle contained in the RAW 277 * image to be rendered. 278 */ convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, int inputStride, byte[] rawImageInput, CameraCharacteristics staticMetadata, CaptureResult dynamicMetadata, int outputOffsetX, int outputOffsetY, Bitmap argbOutput)279 public static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 280 int inputStride, byte[] rawImageInput, CameraCharacteristics staticMetadata, 281 CaptureResult dynamicMetadata, int outputOffsetX, int outputOffsetY, 282 /*out*/Bitmap argbOutput) { 283 int cfa = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); 284 boolean isMono = (cfa == CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_MONO || 285 cfa == CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_NIR); 286 int[] blackLevelPattern = new int[4]; 287 staticMetadata.get(CameraCharacteristics.SENSOR_BLACK_LEVEL_PATTERN). 288 copyTo(blackLevelPattern, /*offset*/0); 289 int whiteLevel = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_WHITE_LEVEL); 290 291 LensShadingMap shadingMap = dynamicMetadata.get( 292 CaptureResult.STATISTICS_LENS_SHADING_CORRECTION_MAP); 293 294 DngBayerMetadata dngBayerMetadata = null; 295 if (!isMono) { 296 dngBayerMetadata = new DngBayerMetadata(staticMetadata, dynamicMetadata); 297 } 298 convertToSRGB(rs, inputWidth, inputHeight, inputStride, cfa, blackLevelPattern, 299 whiteLevel, rawImageInput, dngBayerMetadata, 300 shadingMap, outputOffsetX, outputOffsetY, argbOutput); 301 } 302 303 /** 304 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 305 * 306 * @see #convertToSRGB 307 */ convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, int inputStride, int cfa, int[] blackLevelPattern, int whiteLevel, byte[] rawImageInput, DngBayerMetadata dngBayerMetadata, LensShadingMap lensShadingMap, int outputOffsetX, int outputOffsetY, Bitmap argbOutput)308 private static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 309 int inputStride, int cfa, int[] blackLevelPattern, int whiteLevel, byte[] rawImageInput, 310 DngBayerMetadata dngBayerMetadata, LensShadingMap lensShadingMap, 311 int outputOffsetX, int outputOffsetY, /*out*/Bitmap argbOutput) { 312 313 // Validate arguments 314 if (argbOutput == null || rs == null || rawImageInput == null) { 315 throw new IllegalArgumentException("Null argument to convertToSRGB"); 316 } 317 if (argbOutput.getConfig() != Bitmap.Config.ARGB_8888) { 318 throw new IllegalArgumentException( 319 "Output bitmap passed to convertToSRGB is not ARGB_8888 format"); 320 } 321 if (outputOffsetX < 0 || outputOffsetY < 0) { 322 throw new IllegalArgumentException("Negative offset passed to convertToSRGB"); 323 } 324 if ((inputStride / 2) < inputWidth) { 325 throw new IllegalArgumentException("Stride too small."); 326 } 327 if ((inputStride % 2) != 0) { 328 throw new IllegalArgumentException("Invalid stride for RAW16 format, see graphics.h."); 329 } 330 int outWidth = argbOutput.getWidth(); 331 int outHeight = argbOutput.getHeight(); 332 if (outWidth + outputOffsetX > inputWidth || outHeight + outputOffsetY > inputHeight) { 333 throw new IllegalArgumentException("Raw image with dimensions (w=" + inputWidth + 334 ", h=" + inputHeight + "), cannot converted into sRGB image with dimensions (w=" 335 + outWidth + ", h=" + outHeight + ")."); 336 } 337 if (cfa < 0 || cfa > 5) { 338 throw new IllegalArgumentException("Unsupported cfa pattern " + cfa + " used."); 339 } 340 if (DEBUG) { 341 Log.d(TAG, "Metadata Used:"); 342 Log.d(TAG, "Input width,height: " + inputWidth + "," + inputHeight); 343 Log.d(TAG, "Output offset x,y: " + outputOffsetX + "," + outputOffsetY); 344 Log.d(TAG, "Output width,height: " + outWidth + "," + outHeight); 345 Log.d(TAG, "CFA: " + cfa); 346 Log.d(TAG, "BlackLevelPattern: " + Arrays.toString(blackLevelPattern)); 347 Log.d(TAG, "WhiteLevel: " + whiteLevel); 348 } 349 350 Allocation gainMap = null; 351 if (lensShadingMap != null) { 352 float[] lsm = new float[lensShadingMap.getGainFactorCount()]; 353 lensShadingMap.copyGainFactors(/*inout*/lsm, /*offset*/0); 354 gainMap = createFloat4Allocation(rs, lsm, lensShadingMap.getColumnCount(), 355 lensShadingMap.getRowCount()); 356 } 357 358 float[] sensorToProPhoto = new float[9]; 359 float[] proPhotoToSRGB = new float[9]; 360 if (dngBayerMetadata != null) { 361 float[] normalizedForwardTransform1 = Arrays.copyOf(dngBayerMetadata.forwardTransform1, 362 dngBayerMetadata.forwardTransform1.length); 363 normalizeFM(normalizedForwardTransform1); 364 float[] normalizedForwardTransform2 = Arrays.copyOf(dngBayerMetadata.forwardTransform2, 365 dngBayerMetadata.forwardTransform2.length); 366 normalizeFM(normalizedForwardTransform2); 367 368 float[] normalizedColorMatrix1 = Arrays.copyOf(dngBayerMetadata.colorMatrix1, 369 dngBayerMetadata.colorMatrix1.length); 370 normalizeCM(normalizedColorMatrix1); 371 float[] normalizedColorMatrix2 = Arrays.copyOf(dngBayerMetadata.colorMatrix2, 372 dngBayerMetadata.colorMatrix2.length); 373 normalizeCM(normalizedColorMatrix2); 374 375 if (DEBUG) { 376 Log.d(TAG, "ReferenceIlluminant1: " + dngBayerMetadata.referenceIlluminant1); 377 Log.d(TAG, "ReferenceIlluminant2: " + dngBayerMetadata.referenceIlluminant2); 378 Log.d(TAG, "CalibrationTransform1: " 379 + Arrays.toString(dngBayerMetadata.calibrationTransform1)); 380 Log.d(TAG, "CalibrationTransform2: " 381 + Arrays.toString(dngBayerMetadata.calibrationTransform2)); 382 Log.d(TAG, "ColorMatrix1: " 383 + Arrays.toString(dngBayerMetadata.colorMatrix1)); 384 Log.d(TAG, "ColorMatrix2: " 385 + Arrays.toString(dngBayerMetadata.colorMatrix2)); 386 Log.d(TAG, "ForwardTransform1: " 387 + Arrays.toString(dngBayerMetadata.forwardTransform1)); 388 Log.d(TAG, "ForwardTransform2: " 389 + Arrays.toString(dngBayerMetadata.forwardTransform2)); 390 Log.d(TAG, "NeutralColorPoint: " 391 + Arrays.toString(dngBayerMetadata.neutralColorPoint)); 392 393 Log.d(TAG, "Normalized ForwardTransform1: " 394 + Arrays.toString(normalizedForwardTransform1)); 395 Log.d(TAG, "Normalized ForwardTransform2: " 396 + Arrays.toString(normalizedForwardTransform2)); 397 Log.d(TAG, "Normalized ColorMatrix1: " 398 + Arrays.toString(normalizedColorMatrix1)); 399 Log.d(TAG, "Normalized ColorMatrix2: " 400 + Arrays.toString(normalizedColorMatrix2)); 401 } 402 403 // Calculate full sensor colorspace to sRGB colorspace transform. 404 double interpolationFactor = findDngInterpolationFactor( 405 dngBayerMetadata.referenceIlluminant1, dngBayerMetadata.referenceIlluminant2, 406 dngBayerMetadata.calibrationTransform1, dngBayerMetadata.calibrationTransform2, 407 normalizedColorMatrix1, normalizedColorMatrix2, 408 dngBayerMetadata.neutralColorPoint); 409 if (DEBUG) Log.d(TAG, "Interpolation factor used: " + interpolationFactor); 410 float[] sensorToXYZ = new float[9]; 411 calculateCameraToXYZD50Transform(normalizedForwardTransform1, 412 normalizedForwardTransform2, 413 dngBayerMetadata.calibrationTransform1, dngBayerMetadata.calibrationTransform2, 414 dngBayerMetadata.neutralColorPoint, 415 interpolationFactor, /*out*/sensorToXYZ); 416 if (DEBUG) Log.d(TAG, "CameraToXYZ xform used: " + Arrays.toString(sensorToXYZ)); 417 multiply(sXYZtoProPhoto, sensorToXYZ, /*out*/sensorToProPhoto); 418 if (DEBUG) { 419 Log.d(TAG, "CameraToIntemediate xform used: " + Arrays.toString(sensorToProPhoto)); 420 } 421 multiply(sXYZtoRGBBradford, sProPhotoToXYZ, /*out*/proPhotoToSRGB); 422 } 423 424 Allocation output = Allocation.createFromBitmap(rs, argbOutput); 425 426 // Setup input allocation (16-bit raw pixels) 427 Type.Builder typeBuilder = new Type.Builder(rs, Element.U16(rs)); 428 typeBuilder.setX((inputStride / 2)); 429 typeBuilder.setY(inputHeight); 430 Type inputType = typeBuilder.create(); 431 Allocation input = Allocation.createTyped(rs, inputType); 432 input.copyFromUnchecked(rawImageInput); 433 434 // Setup RS kernel globals 435 ScriptC_raw_converter converterKernel = new ScriptC_raw_converter(rs); 436 converterKernel.set_inputRawBuffer(input); 437 converterKernel.set_whiteLevel(whiteLevel); 438 converterKernel.set_offsetX(outputOffsetX); 439 converterKernel.set_offsetY(outputOffsetY); 440 converterKernel.set_rawHeight(inputHeight); 441 converterKernel.set_rawWidth(inputWidth); 442 converterKernel.set_toneMapCoeffs(new Float4(DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[0], 443 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[1], DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[2], 444 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[3])); 445 converterKernel.set_hasGainMap(gainMap != null); 446 if (gainMap != null) { 447 converterKernel.set_gainMap(gainMap); 448 converterKernel.set_gainMapWidth(lensShadingMap.getColumnCount()); 449 converterKernel.set_gainMapHeight(lensShadingMap.getRowCount()); 450 } 451 452 converterKernel.set_isMonochrome(dngBayerMetadata == null); 453 if (dngBayerMetadata != null) { 454 converterKernel.set_sensorToIntermediate(new Matrix3f(transpose(sensorToProPhoto))); 455 converterKernel.set_intermediateToSRGB(new Matrix3f(transpose(proPhotoToSRGB))); 456 converterKernel.set_neutralPoint( 457 new Float3(dngBayerMetadata.neutralColorPoint[0].floatValue(), 458 dngBayerMetadata.neutralColorPoint[1].floatValue(), 459 dngBayerMetadata.neutralColorPoint[2].floatValue())); 460 } 461 462 converterKernel.set_cfaPattern(cfa); 463 converterKernel.set_blackLevelPattern(new Int4(blackLevelPattern[0], 464 blackLevelPattern[1], blackLevelPattern[2], blackLevelPattern[3])); 465 converterKernel.forEach_convert_RAW_To_ARGB(output); 466 output.copyTo(argbOutput); // Force RS sync with bitmap (does not do an extra copy). 467 } 468 469 /** 470 * Create a float-backed renderscript {@link Allocation} with the given dimensions, containing 471 * the contents of the given float array. 472 * 473 * @param rs a {@link RenderScript} context to use. 474 * @param fArray the float array to copy into the {@link Allocation}. 475 * @param width the width of the {@link Allocation}. 476 * @param height the height of the {@link Allocation}. 477 * @return an {@link Allocation} containing the given floats. 478 */ createFloat4Allocation(RenderScript rs, float[] fArray, int width, int height)479 private static Allocation createFloat4Allocation(RenderScript rs, float[] fArray, 480 int width, int height) { 481 if (fArray.length != width * height * 4) { 482 throw new IllegalArgumentException("Invalid float array of length " + fArray.length + 483 ", must be correct size for Allocation of dimensions " + width + "x" + height); 484 } 485 Type.Builder builder = new Type.Builder(rs, Element.F32_4(rs)); 486 builder.setX(width); 487 builder.setY(height); 488 Allocation fAlloc = Allocation.createTyped(rs, builder.create()); 489 fAlloc.copyFrom(fArray); 490 return fAlloc; 491 } 492 493 /** 494 * Calculate the correlated color temperature (CCT) for a given x,y chromaticity in CIE 1931 x,y 495 * chromaticity space using McCamy's cubic approximation algorithm given in: 496 * 497 * McCamy, Calvin S. (April 1992). 498 * "Correlated color temperature as an explicit function of chromaticity coordinates". 499 * Color Research & Application 17 (2): 142–144 500 * 501 * @param x x chromaticity component. 502 * @param y y chromaticity component. 503 * 504 * @return the CCT associated with this chromaticity coordinate. 505 */ calculateColorTemperature(double x, double y)506 private static double calculateColorTemperature(double x, double y) { 507 double n = (x - 0.332) / (y - 0.1858); 508 return -449 * Math.pow(n, 3) + 3525 * Math.pow(n, 2) - 6823.3 * n + 5520.33; 509 } 510 511 /** 512 * Calculate the x,y chromaticity coordinates in CIE 1931 x,y chromaticity space from the given 513 * CIE XYZ coordinates. 514 * 515 * @param X the CIE XYZ X coordinate. 516 * @param Y the CIE XYZ Y coordinate. 517 * @param Z the CIE XYZ Z coordinate. 518 * 519 * @return the [x, y] chromaticity coordinates as doubles. 520 */ calculateCIExyCoordinates(double X, double Y, double Z)521 private static double[] calculateCIExyCoordinates(double X, double Y, double Z) { 522 double[] ret = new double[] { 0, 0 }; 523 ret[0] = X / (X + Y + Z); 524 ret[1] = Y / (X + Y + Z); 525 return ret; 526 } 527 528 /** 529 * Linearly interpolate between a and b given fraction f. 530 * 531 * @param a first term to interpolate between, a will be returned when f == 0. 532 * @param b second term to interpolate between, b will be returned when f == 1. 533 * @param f the fraction to interpolate by. 534 * 535 * @return interpolated result as double. 536 */ lerp(double a, double b, double f)537 private static double lerp(double a, double b, double f) { 538 return (a * (1.0f - f)) + (b * f); 539 } 540 541 /** 542 * Linearly interpolate between 3x3 matrices a and b given fraction f. 543 * 544 * @param a first 3x3 matrix to interpolate between, a will be returned when f == 0. 545 * @param b second 3x3 matrix to interpolate between, b will be returned when f == 1. 546 * @param f the fraction to interpolate by. 547 * @param result will be set to contain the interpolated matrix. 548 */ lerp(float[] a, float[] b, double f, float[] result)549 private static void lerp(float[] a, float[] b, double f, /*out*/float[] result) { 550 for (int i = 0; i < 9; i++) { 551 result[i] = (float) lerp(a[i], b[i], f); 552 } 553 } 554 555 /** 556 * Find the interpolation factor to use with the RAW matrices given a neutral color point. 557 * 558 * @param referenceIlluminant1 first reference illuminant. 559 * @param referenceIlluminant2 second reference illuminant. 560 * @param calibrationTransform1 calibration matrix corresponding to the first reference 561 * illuminant. 562 * @param calibrationTransform2 calibration matrix corresponding to the second reference 563 * illuminant. 564 * @param colorMatrix1 color matrix corresponding to the first reference illuminant. 565 * @param colorMatrix2 color matrix corresponding to the second reference illuminant. 566 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 567 * 568 * @return the interpolation factor corresponding to the given neutral color point. 569 */ findDngInterpolationFactor(int referenceIlluminant1, int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, float[] colorMatrix1, float[] colorMatrix2, Rational[ ] neutralColorPoint)570 private static double findDngInterpolationFactor(int referenceIlluminant1, 571 int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, 572 float[] colorMatrix1, float[] colorMatrix2, Rational[/*3*/] neutralColorPoint) { 573 574 int colorTemperature1 = sStandardIlluminants.get(referenceIlluminant1, NO_ILLUMINANT); 575 if (colorTemperature1 == NO_ILLUMINANT) { 576 throw new IllegalArgumentException("No such illuminant for reference illuminant 1: " + 577 referenceIlluminant1); 578 } 579 580 int colorTemperature2 = sStandardIlluminants.get(referenceIlluminant2, NO_ILLUMINANT); 581 if (colorTemperature2 == NO_ILLUMINANT) { 582 throw new IllegalArgumentException("No such illuminant for reference illuminant 2: " + 583 referenceIlluminant2); 584 } 585 586 if (DEBUG) Log.d(TAG, "ColorTemperature1: " + colorTemperature1); 587 if (DEBUG) Log.d(TAG, "ColorTemperature2: " + colorTemperature2); 588 589 double interpFactor = 0.5; // Initial guess for interpolation factor 590 double oldInterpFactor = interpFactor; 591 592 double lastDiff = Double.MAX_VALUE; 593 double tolerance = 0.0001; 594 float[] XYZToCamera1 = new float[9]; 595 float[] XYZToCamera2 = new float[9]; 596 multiply(calibrationTransform1, colorMatrix1, /*out*/XYZToCamera1); 597 multiply(calibrationTransform2, colorMatrix2, /*out*/XYZToCamera2); 598 599 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 600 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 601 602 float[] neutralGuess = new float[3]; 603 float[] interpXYZToCamera = new float[9]; 604 float[] interpXYZToCameraInverse = new float[9]; 605 606 607 double lower = Math.min(colorTemperature1, colorTemperature2); 608 double upper = Math.max(colorTemperature1, colorTemperature2); 609 610 if(DEBUG) { 611 Log.d(TAG, "XYZtoCamera1: " + Arrays.toString(XYZToCamera1)); 612 Log.d(TAG, "XYZtoCamera2: " + Arrays.toString(XYZToCamera2)); 613 Log.d(TAG, "Finding interpolation factor, initial guess 0.5..."); 614 } 615 // Iteratively guess xy value, find new CCT, and update interpolation factor. 616 int loopLimit = 30; 617 int count = 0; 618 while (lastDiff > tolerance && loopLimit > 0) { 619 if (DEBUG) Log.d(TAG, "Loop count " + count); 620 lerp(XYZToCamera1, XYZToCamera2, interpFactor, interpXYZToCamera); 621 if (!invert(interpXYZToCamera, /*out*/interpXYZToCameraInverse)) { 622 throw new IllegalArgumentException( 623 "Cannot invert XYZ to Camera matrix, input matrices are invalid."); 624 } 625 626 map(interpXYZToCameraInverse, cameraNeutral, /*out*/neutralGuess); 627 double[] xy = calculateCIExyCoordinates(neutralGuess[0], neutralGuess[1], 628 neutralGuess[2]); 629 630 double colorTemperature = calculateColorTemperature(xy[0], xy[1]); 631 632 if (colorTemperature <= lower) { 633 interpFactor = 1; 634 } else if (colorTemperature >= upper) { 635 interpFactor = 0; 636 } else { 637 double invCT = 1.0 / colorTemperature; 638 interpFactor = (invCT - 1.0 / upper) / ( 1.0 / lower - 1.0 / upper); 639 } 640 641 if (lower == colorTemperature1) { 642 interpFactor = 1.0 - interpFactor; 643 } 644 645 interpFactor = (interpFactor + oldInterpFactor) / 2; 646 lastDiff = Math.abs(oldInterpFactor - interpFactor); 647 oldInterpFactor = interpFactor; 648 loopLimit--; 649 count++; 650 651 if (DEBUG) { 652 Log.d(TAG, "CameraToXYZ chosen: " + Arrays.toString(interpXYZToCameraInverse)); 653 Log.d(TAG, "XYZ neutral color guess: " + Arrays.toString(neutralGuess)); 654 Log.d(TAG, "xy coordinate: " + Arrays.toString(xy)); 655 Log.d(TAG, "xy color temperature: " + colorTemperature); 656 Log.d(TAG, "New interpolation factor: " + interpFactor); 657 } 658 } 659 660 if (loopLimit == 0) { 661 Log.w(TAG, "Could not converge on interpolation factor, using factor " + interpFactor + 662 " with remaining error factor of " + lastDiff); 663 } 664 return interpFactor; 665 } 666 667 /** 668 * Calculate the transform from the raw camera sensor colorspace to CIE XYZ colorspace with a 669 * D50 whitepoint. 670 * 671 * @param forwardTransform1 forward transform matrix corresponding to the first reference 672 * illuminant. 673 * @param forwardTransform2 forward transform matrix corresponding to the second reference 674 * illuminant. 675 * @param calibrationTransform1 calibration transform matrix corresponding to the first 676 * reference illuminant. 677 * @param calibrationTransform2 calibration transform matrix corresponding to the second 678 * reference illuminant. 679 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 680 * @param interpolationFactor the interpolation factor to use for the forward and 681 * calibration transforms. 682 * @param outputTransform set to the full sensor to XYZ colorspace transform. 683 */ calculateCameraToXYZD50Transform(float[] forwardTransform1, float[] forwardTransform2, float[] calibrationTransform1, float[] calibrationTransform2, Rational[ ] neutralColorPoint, double interpolationFactor, float[] outputTransform)684 private static void calculateCameraToXYZD50Transform(float[] forwardTransform1, 685 float[] forwardTransform2, float[] calibrationTransform1, float[] calibrationTransform2, 686 Rational[/*3*/] neutralColorPoint, double interpolationFactor, 687 /*out*/float[] outputTransform) { 688 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 689 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 690 if (DEBUG) Log.d(TAG, "Camera neutral: " + Arrays.toString(cameraNeutral)); 691 692 float[] interpolatedCC = new float[9]; 693 lerp(calibrationTransform1, calibrationTransform2, interpolationFactor, 694 interpolatedCC); 695 float[] inverseInterpolatedCC = new float[9]; 696 if (!invert(interpolatedCC, /*out*/inverseInterpolatedCC)) { 697 throw new IllegalArgumentException( "Cannot invert interpolated calibration transform" + 698 ", input matrices are invalid."); 699 } 700 if (DEBUG) Log.d(TAG, "Inverted interpolated CalibrationTransform: " + 701 Arrays.toString(inverseInterpolatedCC)); 702 703 float[] referenceNeutral = new float[3]; 704 map(inverseInterpolatedCC, cameraNeutral, /*out*/referenceNeutral); 705 if (DEBUG) Log.d(TAG, "Reference neutral: " + Arrays.toString(referenceNeutral)); 706 float maxNeutral = Math.max(Math.max(referenceNeutral[0], referenceNeutral[1]), 707 referenceNeutral[2]); 708 float[] D = new float[] { maxNeutral/referenceNeutral[0], 0, 0, 709 0, maxNeutral/referenceNeutral[1], 0, 710 0, 0, maxNeutral/referenceNeutral[2] }; 711 if (DEBUG) Log.d(TAG, "Reference Neutral Diagonal: " + Arrays.toString(D)); 712 713 float[] intermediate = new float[9]; 714 float[] intermediate2 = new float[9]; 715 716 lerp(forwardTransform1, forwardTransform2, interpolationFactor, /*out*/intermediate); 717 if (DEBUG) Log.d(TAG, "Interpolated ForwardTransform: " + Arrays.toString(intermediate)); 718 719 multiply(D, inverseInterpolatedCC, /*out*/intermediate2); 720 multiply(intermediate, intermediate2, /*out*/outputTransform); 721 } 722 723 /** 724 * Map a 3d column vector using the given matrix. 725 * 726 * @param matrix float array containing 3x3 matrix to map vector by. 727 * @param input 3 dimensional vector to map. 728 * @param output 3 dimensional vector result. 729 */ map(float[] matrix, float[] input, float[] output)730 private static void map(float[] matrix, float[] input, /*out*/float[] output) { 731 output[0] = input[0] * matrix[0] + input[1] * matrix[1] + input[2] * matrix[2]; 732 output[1] = input[0] * matrix[3] + input[1] * matrix[4] + input[2] * matrix[5]; 733 output[2] = input[0] * matrix[6] + input[1] * matrix[7] + input[2] * matrix[8]; 734 } 735 736 /** 737 * Multiply two 3x3 matrices together: A * B 738 * 739 * @param a left matrix. 740 * @param b right matrix. 741 */ multiply(float[] a, float[] b, float[] output)742 private static void multiply(float[] a, float[] b, /*out*/float[] output) { 743 output[0] = a[0] * b[0] + a[1] * b[3] + a[2] * b[6]; 744 output[3] = a[3] * b[0] + a[4] * b[3] + a[5] * b[6]; 745 output[6] = a[6] * b[0] + a[7] * b[3] + a[8] * b[6]; 746 output[1] = a[0] * b[1] + a[1] * b[4] + a[2] * b[7]; 747 output[4] = a[3] * b[1] + a[4] * b[4] + a[5] * b[7]; 748 output[7] = a[6] * b[1] + a[7] * b[4] + a[8] * b[7]; 749 output[2] = a[0] * b[2] + a[1] * b[5] + a[2] * b[8]; 750 output[5] = a[3] * b[2] + a[4] * b[5] + a[5] * b[8]; 751 output[8] = a[6] * b[2] + a[7] * b[5] + a[8] * b[8]; 752 } 753 754 /** 755 * Transpose a 3x3 matrix in-place. 756 * 757 * @param m the matrix to transpose. 758 * @return the transposed matrix. 759 */ transpose( float[ ] m)760 private static float[] transpose(/*inout*/float[/*9*/] m) { 761 float t = m[1]; 762 m[1] = m[3]; 763 m[3] = t; 764 t = m[2]; 765 m[2] = m[6]; 766 m[6] = t; 767 t = m[5]; 768 m[5] = m[7]; 769 m[7] = t; 770 return m; 771 } 772 773 /** 774 * Invert a 3x3 matrix, or return false if the matrix is singular. 775 * 776 * @param m matrix to invert. 777 * @param output set the output to be the inverse of m. 778 */ invert(float[] m, float[] output)779 private static boolean invert(float[] m, /*out*/float[] output) { 780 double a00 = m[0]; 781 double a01 = m[1]; 782 double a02 = m[2]; 783 double a10 = m[3]; 784 double a11 = m[4]; 785 double a12 = m[5]; 786 double a20 = m[6]; 787 double a21 = m[7]; 788 double a22 = m[8]; 789 790 double t00 = a11 * a22 - a21 * a12; 791 double t01 = a21 * a02 - a01 * a22; 792 double t02 = a01 * a12 - a11 * a02; 793 double t10 = a20 * a12 - a10 * a22; 794 double t11 = a00 * a22 - a20 * a02; 795 double t12 = a10 * a02 - a00 * a12; 796 double t20 = a10 * a21 - a20 * a11; 797 double t21 = a20 * a01 - a00 * a21; 798 double t22 = a00 * a11 - a10 * a01; 799 800 double det = a00 * t00 + a01 * t10 + a02 * t20; 801 if (Math.abs(det) < 1e-9) { 802 return false; // Inverse too close to zero, not invertible. 803 } 804 805 output[0] = (float) (t00 / det); 806 output[1] = (float) (t01 / det); 807 output[2] = (float) (t02 / det); 808 output[3] = (float) (t10 / det); 809 output[4] = (float) (t11 / det); 810 output[5] = (float) (t12 / det); 811 output[6] = (float) (t20 / det); 812 output[7] = (float) (t21 / det); 813 output[8] = (float) (t22 / det); 814 return true; 815 } 816 817 /** 818 * Scale each element in a matrix by the given scaling factor. 819 * 820 * @param factor factor to scale by. 821 * @param matrix the float array containing a 3x3 matrix to scale. 822 */ scale(float factor, float[] matrix)823 private static void scale(float factor, /*inout*/float[] matrix) { 824 for (int i = 0; i < 9; i++) { 825 matrix[i] *= factor; 826 } 827 } 828 829 /** 830 * Clamp a value to a given range. 831 * 832 * @param low lower bound to clamp to. 833 * @param high higher bound to clamp to. 834 * @param value the value to clamp. 835 * @return the clamped value. 836 */ clamp(double low, double high, double value)837 private static double clamp(double low, double high, double value) { 838 return Math.max(low, Math.min(high, value)); 839 } 840 841 /** 842 * Return the max float in the array. 843 * 844 * @param array array of floats to search. 845 * @return max float in the array. 846 */ max(float[] array)847 private static float max(float[] array) { 848 float val = array[0]; 849 for (float f : array) { 850 val = (f > val) ? f : val; 851 } 852 return val; 853 } 854 855 /** 856 * Normalize ColorMatrix to eliminate headroom for input space scaled to [0, 1] using 857 * the D50 whitepoint. This maps the D50 whitepoint into the colorspace used by the 858 * ColorMatrix, then uses the resulting whitepoint to renormalize the ColorMatrix so 859 * that the channel values in the resulting whitepoint for this operation are clamped 860 * to the range [0, 1]. 861 * 862 * @param colorMatrix a 3x3 matrix containing a DNG ColorMatrix to be normalized. 863 */ normalizeCM( float[] colorMatrix)864 private static void normalizeCM(/*inout*/float[] colorMatrix) { 865 float[] tmp = new float[3]; 866 map(colorMatrix, D50_XYZ, /*out*/tmp); 867 float maxVal = max(tmp); 868 if (maxVal > 0) { 869 scale(1.0f / maxVal, colorMatrix); 870 } 871 } 872 873 /** 874 * Normalize ForwardMatrix to ensure that sensor whitepoint [1, 1, 1] maps to D50 in CIE XYZ 875 * colorspace. 876 * 877 * @param forwardMatrix a 3x3 matrix containing a DNG ForwardTransform to be normalized. 878 */ normalizeFM( float[] forwardMatrix)879 private static void normalizeFM(/*inout*/float[] forwardMatrix) { 880 float[] tmp = new float[] {1, 1, 1}; 881 float[] xyz = new float[3]; 882 map(forwardMatrix, tmp, /*out*/xyz); 883 884 float[] intermediate = new float[9]; 885 float[] m = new float[] {1.0f / xyz[0], 0, 0, 0, 1.0f / xyz[1], 0, 0, 0, 1.0f / xyz[2]}; 886 887 multiply(m, forwardMatrix, /*out*/ intermediate); 888 float[] m2 = new float[] {D50_XYZ[0], 0, 0, 0, D50_XYZ[1], 0, 0, 0, D50_XYZ[2]}; 889 multiply(m2, intermediate, /*out*/forwardMatrix); 890 } 891 } 892