1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.filterpacks.videoproc; 18 19 import android.filterfw.core.Filter; 20 import android.filterfw.core.FilterContext; 21 import android.filterfw.core.GenerateFieldPort; 22 import android.filterfw.core.GenerateFinalPort; 23 import android.filterfw.core.Frame; 24 import android.filterfw.core.GLFrame; 25 import android.filterfw.core.FrameFormat; 26 import android.filterfw.core.MutableFrameFormat; 27 import android.filterfw.core.Program; 28 import android.filterfw.core.ShaderProgram; 29 import android.filterfw.format.ImageFormat; 30 import android.opengl.GLES20; 31 import android.os.SystemClock; 32 import android.util.Log; 33 34 import java.lang.ArrayIndexOutOfBoundsException; 35 import java.lang.Math; 36 import java.util.Arrays; 37 import java.nio.ByteBuffer; 38 39 /** 40 * @hide 41 */ 42 public class BackDropperFilter extends Filter { 43 /** User-visible parameters */ 44 45 private final int BACKGROUND_STRETCH = 0; 46 private final int BACKGROUND_FIT = 1; 47 private final int BACKGROUND_FILL_CROP = 2; 48 49 @GenerateFieldPort(name = "backgroundFitMode", hasDefault = true) 50 private int mBackgroundFitMode = BACKGROUND_FILL_CROP; 51 @GenerateFieldPort(name = "learningDuration", hasDefault = true) 52 private int mLearningDuration = DEFAULT_LEARNING_DURATION; 53 @GenerateFieldPort(name = "learningVerifyDuration", hasDefault = true) 54 private int mLearningVerifyDuration = DEFAULT_LEARNING_VERIFY_DURATION; 55 @GenerateFieldPort(name = "acceptStddev", hasDefault = true) 56 private float mAcceptStddev = DEFAULT_ACCEPT_STDDEV; 57 @GenerateFieldPort(name = "hierLrgScale", hasDefault = true) 58 private float mHierarchyLrgScale = DEFAULT_HIER_LRG_SCALE; 59 @GenerateFieldPort(name = "hierMidScale", hasDefault = true) 60 private float mHierarchyMidScale = DEFAULT_HIER_MID_SCALE; 61 @GenerateFieldPort(name = "hierSmlScale", hasDefault = true) 62 private float mHierarchySmlScale = DEFAULT_HIER_SML_SCALE; 63 64 // Dimensions of foreground / background mask. Optimum value should take into account only 65 // image contents, NOT dimensions of input video stream. 66 @GenerateFieldPort(name = "maskWidthExp", hasDefault = true) 67 private int mMaskWidthExp = DEFAULT_MASK_WIDTH_EXPONENT; 68 @GenerateFieldPort(name = "maskHeightExp", hasDefault = true) 69 private int mMaskHeightExp = DEFAULT_MASK_HEIGHT_EXPONENT; 70 71 // Levels at which to compute foreground / background decision. Think of them as are deltas 72 // SUBTRACTED from maskWidthExp and maskHeightExp. 73 @GenerateFieldPort(name = "hierLrgExp", hasDefault = true) 74 private int mHierarchyLrgExp = DEFAULT_HIER_LRG_EXPONENT; 75 @GenerateFieldPort(name = "hierMidExp", hasDefault = true) 76 private int mHierarchyMidExp = DEFAULT_HIER_MID_EXPONENT; 77 @GenerateFieldPort(name = "hierSmlExp", hasDefault = true) 78 private int mHierarchySmlExp = DEFAULT_HIER_SML_EXPONENT; 79 80 @GenerateFieldPort(name = "lumScale", hasDefault = true) 81 private float mLumScale = DEFAULT_Y_SCALE_FACTOR; 82 @GenerateFieldPort(name = "chromaScale", hasDefault = true) 83 private float mChromaScale = DEFAULT_UV_SCALE_FACTOR; 84 @GenerateFieldPort(name = "maskBg", hasDefault = true) 85 private float mMaskBg = DEFAULT_MASK_BLEND_BG; 86 @GenerateFieldPort(name = "maskFg", hasDefault = true) 87 private float mMaskFg = DEFAULT_MASK_BLEND_FG; 88 @GenerateFieldPort(name = "exposureChange", hasDefault = true) 89 private float mExposureChange = DEFAULT_EXPOSURE_CHANGE; 90 @GenerateFieldPort(name = "whitebalanceredChange", hasDefault = true) 91 private float mWhiteBalanceRedChange = DEFAULT_WHITE_BALANCE_RED_CHANGE; 92 @GenerateFieldPort(name = "whitebalanceblueChange", hasDefault = true) 93 private float mWhiteBalanceBlueChange = DEFAULT_WHITE_BALANCE_BLUE_CHANGE; 94 @GenerateFieldPort(name = "autowbToggle", hasDefault = true) 95 private int mAutoWBToggle = DEFAULT_WHITE_BALANCE_TOGGLE; 96 97 // TODO: These are not updatable: 98 @GenerateFieldPort(name = "learningAdaptRate", hasDefault = true) 99 private float mAdaptRateLearning = DEFAULT_LEARNING_ADAPT_RATE; 100 @GenerateFieldPort(name = "adaptRateBg", hasDefault = true) 101 private float mAdaptRateBg = DEFAULT_ADAPT_RATE_BG; 102 @GenerateFieldPort(name = "adaptRateFg", hasDefault = true) 103 private float mAdaptRateFg = DEFAULT_ADAPT_RATE_FG; 104 @GenerateFieldPort(name = "maskVerifyRate", hasDefault = true) 105 private float mVerifyRate = DEFAULT_MASK_VERIFY_RATE; 106 @GenerateFieldPort(name = "learningDoneListener", hasDefault = true) 107 private LearningDoneListener mLearningDoneListener = null; 108 109 @GenerateFieldPort(name = "useTheForce", hasDefault = true) 110 private boolean mUseTheForce = false; 111 112 @GenerateFinalPort(name = "provideDebugOutputs", hasDefault = true) 113 private boolean mProvideDebugOutputs = false; 114 115 /** Default algorithm parameter values, for non-shader use */ 116 117 // Frame count for learning bg model 118 private static final int DEFAULT_LEARNING_DURATION = 40; 119 // Frame count for learning verification 120 private static final int DEFAULT_LEARNING_VERIFY_DURATION = 10; 121 // Maximum distance (in standard deviations) for considering a pixel as background 122 private static final float DEFAULT_ACCEPT_STDDEV = 1.1f; 123 // Variance threshold scale factor for large scale of hierarchy 124 private static final float DEFAULT_HIER_LRG_SCALE = 1.5f; 125 // Variance threshold scale factor for medium scale of hierarchy 126 private static final float DEFAULT_HIER_MID_SCALE = 1.0f; 127 // Variance threshold scale factor for small scale of hierarchy 128 private static final float DEFAULT_HIER_SML_SCALE = 0.5f; 129 // Width of foreground / background mask. 130 private static final int DEFAULT_MASK_WIDTH_EXPONENT = 8; 131 // Height of foreground / background mask. 132 private static final int DEFAULT_MASK_HEIGHT_EXPONENT = 8; 133 // Area over which to average for large scale (length in pixels = 2^HIERARCHY_*_EXPONENT) 134 private static final int DEFAULT_HIER_LRG_EXPONENT = 3; 135 // Area over which to average for medium scale 136 private static final int DEFAULT_HIER_MID_EXPONENT = 2; 137 // Area over which to average for small scale 138 private static final int DEFAULT_HIER_SML_EXPONENT = 0; 139 // Scale factor for luminance channel in distance calculations (larger = more significant) 140 private static final float DEFAULT_Y_SCALE_FACTOR = 0.45f; 141 // Scale factor for chroma channels in distance calculations 142 private static final float DEFAULT_UV_SCALE_FACTOR = 1.25f; 143 // Mask value to start blending away from background 144 private static final float DEFAULT_MASK_BLEND_BG = 0.65f; 145 // Mask value to start blending away from foreground 146 private static final float DEFAULT_MASK_BLEND_FG = 0.95f; 147 // Exposure stop number to change the brightness of foreground 148 private static final float DEFAULT_EXPOSURE_CHANGE = 1.0f; 149 // White balance change in Red channel for foreground 150 private static final float DEFAULT_WHITE_BALANCE_RED_CHANGE = 0.0f; 151 // White balance change in Blue channel for foreground 152 private static final float DEFAULT_WHITE_BALANCE_BLUE_CHANGE = 0.0f; 153 // Variable to control automatic white balance effect 154 // 0.f -> Auto WB is off; 1.f-> Auto WB is on 155 private static final int DEFAULT_WHITE_BALANCE_TOGGLE = 0; 156 157 // Default rate at which to learn bg model during learning period 158 private static final float DEFAULT_LEARNING_ADAPT_RATE = 0.2f; 159 // Default rate at which to learn bg model from new background pixels 160 private static final float DEFAULT_ADAPT_RATE_BG = 0.0f; 161 // Default rate at which to learn bg model from new foreground pixels 162 private static final float DEFAULT_ADAPT_RATE_FG = 0.0f; 163 // Default rate at which to verify whether background is stable 164 private static final float DEFAULT_MASK_VERIFY_RATE = 0.25f; 165 // Default rate at which to verify whether background is stable 166 private static final int DEFAULT_LEARNING_DONE_THRESHOLD = 20; 167 168 // Default 3x3 matrix, column major, for fitting background 1:1 169 private static final float[] DEFAULT_BG_FIT_TRANSFORM = new float[] { 170 1.0f, 0.0f, 0.0f, 171 0.0f, 1.0f, 0.0f, 172 0.0f, 0.0f, 1.0f 173 }; 174 175 /** Default algorithm parameter values, for shader use */ 176 177 // Area over which to blur binary mask values (length in pixels = 2^MASK_SMOOTH_EXPONENT) 178 private static final String MASK_SMOOTH_EXPONENT = "2.0"; 179 // Scale value for mapping variance distance to fit nicely to 0-1, 8-bit 180 private static final String DISTANCE_STORAGE_SCALE = "0.6"; 181 // Scale value for mapping variance to fit nicely to 0-1, 8-bit 182 private static final String VARIANCE_STORAGE_SCALE = "5.0"; 183 // Default scale of auto white balance parameters 184 private static final String DEFAULT_AUTO_WB_SCALE = "0.25"; 185 // Minimum variance (0-255 scale) 186 private static final String MIN_VARIANCE = "3.0"; 187 // Column-major array for 4x4 matrix converting RGB to YCbCr, JPEG definition (no pedestal) 188 private static final String RGB_TO_YUV_MATRIX = "0.299, -0.168736, 0.5, 0.000, " + 189 "0.587, -0.331264, -0.418688, 0.000, " + 190 "0.114, 0.5, -0.081312, 0.000, " + 191 "0.000, 0.5, 0.5, 1.000 "; 192 /** Stream names */ 193 194 private static final String[] mInputNames = {"video", 195 "background"}; 196 197 private static final String[] mOutputNames = {"video"}; 198 199 private static final String[] mDebugOutputNames = {"debug1", 200 "debug2"}; 201 202 /** Other private variables */ 203 204 private FrameFormat mOutputFormat; 205 private MutableFrameFormat mMemoryFormat; 206 private MutableFrameFormat mMaskFormat; 207 private MutableFrameFormat mAverageFormat; 208 209 private final boolean mLogVerbose; 210 private static final String TAG = "BackDropperFilter"; 211 212 /** Shader source code */ 213 214 // Shared uniforms and utility functions 215 private static String mSharedUtilShader = 216 "precision mediump float;\n" + 217 "uniform float fg_adapt_rate;\n" + 218 "uniform float bg_adapt_rate;\n" + 219 "const mat4 coeff_yuv = mat4(" + RGB_TO_YUV_MATRIX + ");\n" + 220 "const float dist_scale = " + DISTANCE_STORAGE_SCALE + ";\n" + 221 "const float inv_dist_scale = 1. / dist_scale;\n" + 222 "const float var_scale=" + VARIANCE_STORAGE_SCALE + ";\n" + 223 "const float inv_var_scale = 1. / var_scale;\n" + 224 "const float min_variance = inv_var_scale *" + MIN_VARIANCE + "/ 256.;\n" + 225 "const float auto_wb_scale = " + DEFAULT_AUTO_WB_SCALE + ";\n" + 226 "\n" + 227 // Variance distance in luminance between current pixel and background model 228 "float gauss_dist_y(float y, float mean, float variance) {\n" + 229 " float dist = (y - mean) * (y - mean) / variance;\n" + 230 " return dist;\n" + 231 "}\n" + 232 // Sum of variance distances in chroma between current pixel and background 233 // model 234 "float gauss_dist_uv(vec2 uv, vec2 mean, vec2 variance) {\n" + 235 " vec2 dist = (uv - mean) * (uv - mean) / variance;\n" + 236 " return dist.r + dist.g;\n" + 237 "}\n" + 238 // Select learning rate for pixel based on smoothed decision mask alpha 239 "float local_adapt_rate(float alpha) {\n" + 240 " return mix(bg_adapt_rate, fg_adapt_rate, alpha);\n" + 241 "}\n" + 242 "\n"; 243 244 // Distance calculation shader. Calculates a distance metric between the foreground and the 245 // current background model, in both luminance and in chroma (yuv space). Distance is 246 // measured in variances from the mean background value. For chroma, the distance is the sum 247 // of the two individual color channel distances. The distances are output on the b and alpha 248 // channels, r and g are for debug information. 249 // Inputs: 250 // tex_sampler_0: Mip-map for foreground (live) video frame. 251 // tex_sampler_1: Background mean mask. 252 // tex_sampler_2: Background variance mask. 253 // subsample_level: Level on foreground frame's mip-map. 254 private static final String mBgDistanceShader = 255 "uniform sampler2D tex_sampler_0;\n" + 256 "uniform sampler2D tex_sampler_1;\n" + 257 "uniform sampler2D tex_sampler_2;\n" + 258 "uniform float subsample_level;\n" + 259 "varying vec2 v_texcoord;\n" + 260 "void main() {\n" + 261 " vec4 fg = coeff_yuv * texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 262 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 263 " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 264 "\n" + 265 " float dist_y = gauss_dist_y(fg.r, mean.r, variance.r);\n" + 266 " float dist_uv = gauss_dist_uv(fg.gb, mean.gb, variance.gb);\n" + 267 " gl_FragColor = vec4(0.5*fg.rg, dist_scale*dist_y, dist_scale*dist_uv);\n" + 268 "}\n"; 269 270 // Foreground/background mask decision shader. Decides whether a frame is in the foreground or 271 // the background using a hierarchical threshold on the distance. Binary foreground/background 272 // mask is placed in the alpha channel. The RGB channels contain debug information. 273 private static final String mBgMaskShader = 274 "uniform sampler2D tex_sampler_0;\n" + 275 "uniform float accept_variance;\n" + 276 "uniform vec2 yuv_weights;\n" + 277 "uniform float scale_lrg;\n" + 278 "uniform float scale_mid;\n" + 279 "uniform float scale_sml;\n" + 280 "uniform float exp_lrg;\n" + 281 "uniform float exp_mid;\n" + 282 "uniform float exp_sml;\n" + 283 "varying vec2 v_texcoord;\n" + 284 // Decide whether pixel is foreground or background based on Y and UV 285 // distance and maximum acceptable variance. 286 // yuv_weights.x is smaller than yuv_weights.y to discount the influence of shadow 287 "bool is_fg(vec2 dist_yc, float accept_variance) {\n" + 288 " return ( dot(yuv_weights, dist_yc) >= accept_variance );\n" + 289 "}\n" + 290 "void main() {\n" + 291 " vec4 dist_lrg_sc = texture2D(tex_sampler_0, v_texcoord, exp_lrg);\n" + 292 " vec4 dist_mid_sc = texture2D(tex_sampler_0, v_texcoord, exp_mid);\n" + 293 " vec4 dist_sml_sc = texture2D(tex_sampler_0, v_texcoord, exp_sml);\n" + 294 " vec2 dist_lrg = inv_dist_scale * dist_lrg_sc.ba;\n" + 295 " vec2 dist_mid = inv_dist_scale * dist_mid_sc.ba;\n" + 296 " vec2 dist_sml = inv_dist_scale * dist_sml_sc.ba;\n" + 297 " vec2 norm_dist = 0.75 * dist_sml / accept_variance;\n" + // For debug viz 298 " bool is_fg_lrg = is_fg(dist_lrg, accept_variance * scale_lrg);\n" + 299 " bool is_fg_mid = is_fg_lrg || is_fg(dist_mid, accept_variance * scale_mid);\n" + 300 " float is_fg_sml =\n" + 301 " float(is_fg_mid || is_fg(dist_sml, accept_variance * scale_sml));\n" + 302 " float alpha = 0.5 * is_fg_sml + 0.3 * float(is_fg_mid) + 0.2 * float(is_fg_lrg);\n" + 303 " gl_FragColor = vec4(alpha, norm_dist, is_fg_sml);\n" + 304 "}\n"; 305 306 // Automatic White Balance parameter decision shader 307 // Use the Gray World assumption that in a white balance corrected image, the average of R, G, B 308 // channel will be a common gray value. 309 // To match the white balance of foreground and background, the average of R, G, B channel of 310 // two videos should match. 311 // Inputs: 312 // tex_sampler_0: Mip-map for foreground (live) video frame. 313 // tex_sampler_1: Mip-map for background (playback) video frame. 314 // pyramid_depth: Depth of input frames' mip-maps. 315 private static final String mAutomaticWhiteBalance = 316 "uniform sampler2D tex_sampler_0;\n" + 317 "uniform sampler2D tex_sampler_1;\n" + 318 "uniform float pyramid_depth;\n" + 319 "uniform bool autowb_toggle;\n" + 320 "varying vec2 v_texcoord;\n" + 321 "void main() {\n" + 322 " vec4 mean_video = texture2D(tex_sampler_0, v_texcoord, pyramid_depth);\n"+ 323 " vec4 mean_bg = texture2D(tex_sampler_1, v_texcoord, pyramid_depth);\n" + 324 // If Auto WB is toggled off, the return texture will be a unicolor texture of value 1 325 // If Auto WB is toggled on, the return texture will be a unicolor texture with 326 // adjustment parameters for R and B channels stored in the corresponding channel 327 " float green_normalizer = mean_video.g / mean_bg.g;\n"+ 328 " vec4 adjusted_value = vec4(mean_bg.r / mean_video.r * green_normalizer, 1., \n" + 329 " mean_bg.b / mean_video.b * green_normalizer, 1.) * auto_wb_scale; \n" + 330 " gl_FragColor = autowb_toggle ? adjusted_value : vec4(auto_wb_scale);\n" + 331 "}\n"; 332 333 334 // Background subtraction shader. Uses a mipmap of the binary mask map to blend smoothly between 335 // foreground and background 336 // Inputs: 337 // tex_sampler_0: Foreground (live) video frame. 338 // tex_sampler_1: Background (playback) video frame. 339 // tex_sampler_2: Foreground/background mask. 340 // tex_sampler_3: Auto white-balance factors. 341 private static final String mBgSubtractShader = 342 "uniform mat3 bg_fit_transform;\n" + 343 "uniform float mask_blend_bg;\n" + 344 "uniform float mask_blend_fg;\n" + 345 "uniform float exposure_change;\n" + 346 "uniform float whitebalancered_change;\n" + 347 "uniform float whitebalanceblue_change;\n" + 348 "uniform sampler2D tex_sampler_0;\n" + 349 "uniform sampler2D tex_sampler_1;\n" + 350 "uniform sampler2D tex_sampler_2;\n" + 351 "uniform sampler2D tex_sampler_3;\n" + 352 "varying vec2 v_texcoord;\n" + 353 "void main() {\n" + 354 " vec2 bg_texcoord = (bg_fit_transform * vec3(v_texcoord, 1.)).xy;\n" + 355 " vec4 bg_rgb = texture2D(tex_sampler_1, bg_texcoord);\n" + 356 // The foreground texture is modified by multiplying both manual and auto white balance changes in R and B 357 // channel and multiplying exposure change in all R, G, B channels. 358 " vec4 wb_auto_scale = texture2D(tex_sampler_3, v_texcoord) * exposure_change / auto_wb_scale;\n" + 359 " vec4 wb_manual_scale = vec4(1. + whitebalancered_change, 1., 1. + whitebalanceblue_change, 1.);\n" + 360 " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord);\n" + 361 " vec4 fg_adjusted = fg_rgb * wb_manual_scale * wb_auto_scale;\n"+ 362 " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 363 " " + MASK_SMOOTH_EXPONENT + ");\n" + 364 " float alpha = smoothstep(mask_blend_bg, mask_blend_fg, mask.a);\n" + 365 " gl_FragColor = mix(bg_rgb, fg_adjusted, alpha);\n"; 366 367 // May the Force... Makes the foreground object translucent blue, with a bright 368 // blue-white outline 369 private static final String mBgSubtractForceShader = 370 " vec4 ghost_rgb = (fg_adjusted * 0.7 + vec4(0.3,0.3,0.4,0.))*0.65 + \n" + 371 " 0.35*bg_rgb;\n" + 372 " float glow_start = 0.75 * mask_blend_bg; \n"+ 373 " float glow_max = mask_blend_bg; \n"+ 374 " gl_FragColor = mask.a < glow_start ? bg_rgb : \n" + 375 " mask.a < glow_max ? mix(bg_rgb, vec4(0.9,0.9,1.0,1.0), \n" + 376 " (mask.a - glow_start) / (glow_max - glow_start) ) : \n" + 377 " mask.a < mask_blend_fg ? mix(vec4(0.9,0.9,1.0,1.0), ghost_rgb, \n" + 378 " (mask.a - glow_max) / (mask_blend_fg - glow_max) ) : \n" + 379 " ghost_rgb;\n" + 380 "}\n"; 381 382 // Background model mean update shader. Skews the current model mean toward the most recent pixel 383 // value for a pixel, weighted by the learning rate and by whether the pixel is classified as 384 // foreground or background. 385 // Inputs: 386 // tex_sampler_0: Mip-map for foreground (live) video frame. 387 // tex_sampler_1: Background mean mask. 388 // tex_sampler_2: Foreground/background mask. 389 // subsample_level: Level on foreground frame's mip-map. 390 private static final String mUpdateBgModelMeanShader = 391 "uniform sampler2D tex_sampler_0;\n" + 392 "uniform sampler2D tex_sampler_1;\n" + 393 "uniform sampler2D tex_sampler_2;\n" + 394 "uniform float subsample_level;\n" + 395 "varying vec2 v_texcoord;\n" + 396 "void main() {\n" + 397 " vec4 fg = coeff_yuv * texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 398 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 399 " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 400 " " + MASK_SMOOTH_EXPONENT + ");\n" + 401 "\n" + 402 " float alpha = local_adapt_rate(mask.a);\n" + 403 " vec4 new_mean = mix(mean, fg, alpha);\n" + 404 " gl_FragColor = new_mean;\n" + 405 "}\n"; 406 407 // Background model variance update shader. Skews the current model variance toward the most 408 // recent variance for the pixel, weighted by the learning rate and by whether the pixel is 409 // classified as foreground or background. 410 // Inputs: 411 // tex_sampler_0: Mip-map for foreground (live) video frame. 412 // tex_sampler_1: Background mean mask. 413 // tex_sampler_2: Background variance mask. 414 // tex_sampler_3: Foreground/background mask. 415 // subsample_level: Level on foreground frame's mip-map. 416 // TODO: to improve efficiency, use single mark for mean + variance, then merge this into 417 // mUpdateBgModelMeanShader. 418 private static final String mUpdateBgModelVarianceShader = 419 "uniform sampler2D tex_sampler_0;\n" + 420 "uniform sampler2D tex_sampler_1;\n" + 421 "uniform sampler2D tex_sampler_2;\n" + 422 "uniform sampler2D tex_sampler_3;\n" + 423 "uniform float subsample_level;\n" + 424 "varying vec2 v_texcoord;\n" + 425 "void main() {\n" + 426 " vec4 fg = coeff_yuv * texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 427 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 428 " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 429 " vec4 mask = texture2D(tex_sampler_3, v_texcoord, \n" + 430 " " + MASK_SMOOTH_EXPONENT + ");\n" + 431 "\n" + 432 " float alpha = local_adapt_rate(mask.a);\n" + 433 " vec4 cur_variance = (fg-mean)*(fg-mean);\n" + 434 " vec4 new_variance = mix(variance, cur_variance, alpha);\n" + 435 " new_variance = max(new_variance, vec4(min_variance));\n" + 436 " gl_FragColor = var_scale * new_variance;\n" + 437 "}\n"; 438 439 // Background verification shader. Skews the current background verification mask towards the 440 // most recent frame, weighted by the learning rate. 441 private static final String mMaskVerifyShader = 442 "uniform sampler2D tex_sampler_0;\n" + 443 "uniform sampler2D tex_sampler_1;\n" + 444 "uniform float verify_rate;\n" + 445 "varying vec2 v_texcoord;\n" + 446 "void main() {\n" + 447 " vec4 lastmask = texture2D(tex_sampler_0, v_texcoord);\n" + 448 " vec4 mask = texture2D(tex_sampler_1, v_texcoord);\n" + 449 " float newmask = mix(lastmask.a, mask.a, verify_rate);\n" + 450 " gl_FragColor = vec4(0., 0., 0., newmask);\n" + 451 "}\n"; 452 453 /** Shader program objects */ 454 455 private ShaderProgram mBgDistProgram; 456 private ShaderProgram mBgMaskProgram; 457 private ShaderProgram mBgSubtractProgram; 458 private ShaderProgram mBgUpdateMeanProgram; 459 private ShaderProgram mBgUpdateVarianceProgram; 460 private ShaderProgram mCopyOutProgram; 461 private ShaderProgram mAutomaticWhiteBalanceProgram; 462 private ShaderProgram mMaskVerifyProgram; 463 private ShaderProgram copyShaderProgram; 464 465 /** Background model storage */ 466 467 private boolean mPingPong; 468 private GLFrame mBgMean[]; 469 private GLFrame mBgVariance[]; 470 private GLFrame mMaskVerify[]; 471 private GLFrame mDistance; 472 private GLFrame mAutoWB; 473 private GLFrame mMask; 474 private GLFrame mVideoInput; 475 private GLFrame mBgInput; 476 private GLFrame mMaskAverage; 477 478 /** Overall filter state */ 479 480 private boolean isOpen; 481 private int mFrameCount; 482 private boolean mStartLearning; 483 private boolean mBackgroundFitModeChanged; 484 private float mRelativeAspect; 485 private int mPyramidDepth; 486 private int mSubsampleLevel; 487 488 /** Learning listener object */ 489 490 public interface LearningDoneListener { onLearningDone(BackDropperFilter filter)491 public void onLearningDone(BackDropperFilter filter); 492 } 493 494 /** Public Filter methods */ 495 BackDropperFilter(String name)496 public BackDropperFilter(String name) { 497 super(name); 498 499 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 500 } 501 502 @Override setupPorts()503 public void setupPorts() { 504 // Inputs. 505 // TODO: Target should be GPU, but relaxed for now. 506 FrameFormat imageFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 507 FrameFormat.TARGET_UNSPECIFIED); 508 for (String inputName : mInputNames) { 509 addMaskedInputPort(inputName, imageFormat); 510 } 511 // Normal outputs 512 for (String outputName : mOutputNames) { 513 addOutputBasedOnInput(outputName, "video"); 514 } 515 516 // Debug outputs 517 if (mProvideDebugOutputs) { 518 for (String outputName : mDebugOutputNames) { 519 addOutputBasedOnInput(outputName, "video"); 520 } 521 } 522 } 523 524 @Override getOutputFormat(String portName, FrameFormat inputFormat)525 public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { 526 // Create memory format based on video input. 527 MutableFrameFormat format = inputFormat.mutableCopy(); 528 // Is this a debug output port? If so, leave dimensions unspecified. 529 if (!Arrays.asList(mOutputNames).contains(portName)) { 530 format.setDimensions(FrameFormat.SIZE_UNSPECIFIED, FrameFormat.SIZE_UNSPECIFIED); 531 } 532 return format; 533 } 534 createMemoryFormat(FrameFormat inputFormat)535 private boolean createMemoryFormat(FrameFormat inputFormat) { 536 // We can't resize because that would require re-learning. 537 if (mMemoryFormat != null) { 538 return false; 539 } 540 541 if (inputFormat.getWidth() == FrameFormat.SIZE_UNSPECIFIED || 542 inputFormat.getHeight() == FrameFormat.SIZE_UNSPECIFIED) { 543 throw new RuntimeException("Attempting to process input frame with unknown size"); 544 } 545 546 mMaskFormat = inputFormat.mutableCopy(); 547 int maskWidth = (int)Math.pow(2, mMaskWidthExp); 548 int maskHeight = (int)Math.pow(2, mMaskHeightExp); 549 mMaskFormat.setDimensions(maskWidth, maskHeight); 550 551 mPyramidDepth = Math.max(mMaskWidthExp, mMaskHeightExp); 552 mMemoryFormat = mMaskFormat.mutableCopy(); 553 int widthExp = Math.max(mMaskWidthExp, pyramidLevel(inputFormat.getWidth())); 554 int heightExp = Math.max(mMaskHeightExp, pyramidLevel(inputFormat.getHeight())); 555 mPyramidDepth = Math.max(widthExp, heightExp); 556 int memWidth = Math.max(maskWidth, (int)Math.pow(2, widthExp)); 557 int memHeight = Math.max(maskHeight, (int)Math.pow(2, heightExp)); 558 mMemoryFormat.setDimensions(memWidth, memHeight); 559 mSubsampleLevel = mPyramidDepth - Math.max(mMaskWidthExp, mMaskHeightExp); 560 561 if (mLogVerbose) { 562 Log.v(TAG, "Mask frames size " + maskWidth + " x " + maskHeight); 563 Log.v(TAG, "Pyramid levels " + widthExp + " x " + heightExp); 564 Log.v(TAG, "Memory frames size " + memWidth + " x " + memHeight); 565 } 566 567 mAverageFormat = inputFormat.mutableCopy(); 568 mAverageFormat.setDimensions(1,1); 569 return true; 570 } 571 prepare(FilterContext context)572 public void prepare(FilterContext context){ 573 if (mLogVerbose) Log.v(TAG, "Preparing BackDropperFilter!"); 574 575 mBgMean = new GLFrame[2]; 576 mBgVariance = new GLFrame[2]; 577 mMaskVerify = new GLFrame[2]; 578 copyShaderProgram = ShaderProgram.createIdentity(context); 579 } 580 allocateFrames(FrameFormat inputFormat, FilterContext context)581 private void allocateFrames(FrameFormat inputFormat, FilterContext context) { 582 if (!createMemoryFormat(inputFormat)) { 583 return; // All set. 584 } 585 if (mLogVerbose) Log.v(TAG, "Allocating BackDropperFilter frames"); 586 587 // Create initial background model values 588 int numBytes = mMaskFormat.getSize(); 589 byte[] initialBgMean = new byte[numBytes]; 590 byte[] initialBgVariance = new byte[numBytes]; 591 byte[] initialMaskVerify = new byte[numBytes]; 592 for (int i = 0; i < numBytes; i++) { 593 initialBgMean[i] = (byte)128; 594 initialBgVariance[i] = (byte)10; 595 initialMaskVerify[i] = (byte)0; 596 } 597 598 // Get frames to store background model in 599 for (int i = 0; i < 2; i++) { 600 mBgMean[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 601 mBgMean[i].setData(initialBgMean, 0, numBytes); 602 603 mBgVariance[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 604 mBgVariance[i].setData(initialBgVariance, 0, numBytes); 605 606 mMaskVerify[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 607 mMaskVerify[i].setData(initialMaskVerify, 0, numBytes); 608 } 609 610 // Get frames to store other textures in 611 if (mLogVerbose) Log.v(TAG, "Done allocating texture for Mean and Variance objects!"); 612 613 mDistance = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 614 mMask = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 615 mAutoWB = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 616 mVideoInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 617 mBgInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 618 mMaskAverage = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 619 620 // Create shader programs 621 mBgDistProgram = new ShaderProgram(context, mSharedUtilShader + mBgDistanceShader); 622 mBgDistProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 623 624 mBgMaskProgram = new ShaderProgram(context, mSharedUtilShader + mBgMaskShader); 625 mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 626 float[] yuvWeights = { mLumScale, mChromaScale }; 627 mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 628 mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 629 mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 630 mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 631 mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 632 mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 633 mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 634 635 if (mUseTheForce) { 636 mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + mBgSubtractForceShader); 637 } else { 638 mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + "}\n"); 639 } 640 mBgSubtractProgram.setHostValue("bg_fit_transform", DEFAULT_BG_FIT_TRANSFORM); 641 mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 642 mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 643 mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 644 mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 645 mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 646 647 648 mBgUpdateMeanProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelMeanShader); 649 mBgUpdateMeanProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 650 651 mBgUpdateVarianceProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelVarianceShader); 652 mBgUpdateVarianceProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 653 654 mCopyOutProgram = ShaderProgram.createIdentity(context); 655 656 mAutomaticWhiteBalanceProgram = new ShaderProgram(context, mSharedUtilShader + mAutomaticWhiteBalance); 657 mAutomaticWhiteBalanceProgram.setHostValue("pyramid_depth", (float)mPyramidDepth); 658 mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 659 660 mMaskVerifyProgram = new ShaderProgram(context, mSharedUtilShader + mMaskVerifyShader); 661 mMaskVerifyProgram.setHostValue("verify_rate", mVerifyRate); 662 663 if (mLogVerbose) Log.v(TAG, "Shader width set to " + mMemoryFormat.getWidth()); 664 665 mRelativeAspect = 1.f; 666 667 mFrameCount = 0; 668 mStartLearning = true; 669 } 670 process(FilterContext context)671 public void process(FilterContext context) { 672 // Grab inputs and ready intermediate frames and outputs. 673 Frame video = pullInput("video"); 674 Frame background = pullInput("background"); 675 allocateFrames(video.getFormat(), context); 676 677 // Update learning rate after initial learning period 678 if (mStartLearning) { 679 if (mLogVerbose) Log.v(TAG, "Starting learning"); 680 mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 681 mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 682 mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 683 mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 684 mFrameCount = 0; 685 mStartLearning = false; 686 } 687 688 // Select correct pingpong buffers 689 int inputIndex = mPingPong ? 0 : 1; 690 int outputIndex = mPingPong ? 1 : 0; 691 mPingPong = !mPingPong; 692 693 // Check relative aspect ratios 694 updateBgScaling(video, background, mBackgroundFitModeChanged); 695 mBackgroundFitModeChanged = false; 696 697 // Make copies for input frames to GLFrames 698 699 copyShaderProgram.process(video, mVideoInput); 700 copyShaderProgram.process(background, mBgInput); 701 702 mVideoInput.generateMipMap(); 703 mVideoInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 704 GLES20.GL_LINEAR_MIPMAP_NEAREST); 705 706 mBgInput.generateMipMap(); 707 mBgInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 708 GLES20.GL_LINEAR_MIPMAP_NEAREST); 709 710 // Process shaders 711 Frame[] distInputs = { mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex] }; 712 mBgDistProgram.process(distInputs, mDistance); 713 mDistance.generateMipMap(); 714 mDistance.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 715 GLES20.GL_LINEAR_MIPMAP_NEAREST); 716 717 mBgMaskProgram.process(mDistance, mMask); 718 mMask.generateMipMap(); 719 mMask.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 720 GLES20.GL_LINEAR_MIPMAP_NEAREST); 721 722 Frame[] autoWBInputs = { mVideoInput, mBgInput }; 723 mAutomaticWhiteBalanceProgram.process(autoWBInputs, mAutoWB); 724 725 if (mFrameCount <= mLearningDuration) { 726 // During learning 727 pushOutput("video", video); 728 729 if (mFrameCount == mLearningDuration - mLearningVerifyDuration) { 730 copyShaderProgram.process(mMask, mMaskVerify[outputIndex]); 731 732 mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 733 mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 734 mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 735 mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 736 737 738 } else if (mFrameCount > mLearningDuration - mLearningVerifyDuration) { 739 // In the learning verification stage, compute background masks and a weighted average 740 // with weights grow exponentially with time 741 Frame[] maskVerifyInputs = {mMaskVerify[inputIndex], mMask}; 742 mMaskVerifyProgram.process(maskVerifyInputs, mMaskVerify[outputIndex]); 743 mMaskVerify[outputIndex].generateMipMap(); 744 mMaskVerify[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 745 GLES20.GL_LINEAR_MIPMAP_NEAREST); 746 } 747 748 if (mFrameCount == mLearningDuration) { 749 // In the last verification frame, verify if the verification mask is almost blank 750 // If not, restart learning 751 copyShaderProgram.process(mMaskVerify[outputIndex], mMaskAverage); 752 ByteBuffer mMaskAverageByteBuffer = mMaskAverage.getData(); 753 byte[] mask_average = mMaskAverageByteBuffer.array(); 754 int bi = (int)(mask_average[3] & 0xFF); 755 if (mLogVerbose) Log.v(TAG, String.format("Mask_average is %d", bi)); 756 757 if (bi >= DEFAULT_LEARNING_DONE_THRESHOLD) { 758 mStartLearning = true; // Restart learning 759 } else { 760 if (mLogVerbose) Log.v(TAG, "Learning done"); 761 if (mLearningDoneListener != null) { 762 mLearningDoneListener.onLearningDone(this); 763 } 764 } 765 } 766 } else { 767 Frame output = context.getFrameManager().newFrame(video.getFormat()); 768 Frame[] subtractInputs = { video, background, mMask, mAutoWB }; 769 mBgSubtractProgram.process(subtractInputs, output); 770 pushOutput("video", output); 771 output.release(); 772 } 773 774 // Compute mean and variance of the background 775 if (mFrameCount < mLearningDuration - mLearningVerifyDuration || 776 mAdaptRateBg > 0.0 || mAdaptRateFg > 0.0) { 777 Frame[] meanUpdateInputs = { mVideoInput, mBgMean[inputIndex], mMask }; 778 mBgUpdateMeanProgram.process(meanUpdateInputs, mBgMean[outputIndex]); 779 mBgMean[outputIndex].generateMipMap(); 780 mBgMean[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 781 GLES20.GL_LINEAR_MIPMAP_NEAREST); 782 783 Frame[] varianceUpdateInputs = { 784 mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex], mMask 785 }; 786 mBgUpdateVarianceProgram.process(varianceUpdateInputs, mBgVariance[outputIndex]); 787 mBgVariance[outputIndex].generateMipMap(); 788 mBgVariance[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 789 GLES20.GL_LINEAR_MIPMAP_NEAREST); 790 } 791 792 // Provide debug output to two smaller viewers 793 if (mProvideDebugOutputs) { 794 Frame dbg1 = context.getFrameManager().newFrame(video.getFormat()); 795 mCopyOutProgram.process(video, dbg1); 796 pushOutput("debug1", dbg1); 797 dbg1.release(); 798 799 Frame dbg2 = context.getFrameManager().newFrame(mMemoryFormat); 800 mCopyOutProgram.process(mMask, dbg2); 801 pushOutput("debug2", dbg2); 802 dbg2.release(); 803 } 804 805 mFrameCount++; 806 807 if (mLogVerbose) { 808 if (mFrameCount % 30 == 0) { 809 if (startTime == -1) { 810 context.getGLEnvironment().activate(); 811 GLES20.glFinish(); 812 startTime = SystemClock.elapsedRealtime(); 813 } else { 814 context.getGLEnvironment().activate(); 815 GLES20.glFinish(); 816 long endTime = SystemClock.elapsedRealtime(); 817 Log.v(TAG, "Avg. frame duration: " + String.format("%.2f",(endTime-startTime)/30.) + 818 " ms. Avg. fps: " + String.format("%.2f", 1000./((endTime-startTime)/30.)) ); 819 startTime = endTime; 820 } 821 } 822 } 823 } 824 825 private long startTime = -1; 826 close(FilterContext context)827 public void close(FilterContext context) { 828 if (mMemoryFormat == null) { 829 return; 830 } 831 832 if (mLogVerbose) Log.v(TAG, "Filter Closing!"); 833 for (int i = 0; i < 2; i++) { 834 mBgMean[i].release(); 835 mBgVariance[i].release(); 836 mMaskVerify[i].release(); 837 } 838 mDistance.release(); 839 mMask.release(); 840 mAutoWB.release(); 841 mVideoInput.release(); 842 mBgInput.release(); 843 mMaskAverage.release(); 844 845 mMemoryFormat = null; 846 } 847 848 // Relearn background model relearn()849 synchronized public void relearn() { 850 // Let the processing thread know about learning restart 851 mStartLearning = true; 852 } 853 854 @Override fieldPortValueUpdated(String name, FilterContext context)855 public void fieldPortValueUpdated(String name, FilterContext context) { 856 // TODO: Many of these can be made ProgramPorts! 857 if (name.equals("backgroundFitMode")) { 858 mBackgroundFitModeChanged = true; 859 } else if (name.equals("acceptStddev")) { 860 mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 861 } else if (name.equals("hierLrgScale")) { 862 mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 863 } else if (name.equals("hierMidScale")) { 864 mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 865 } else if (name.equals("hierSmlScale")) { 866 mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 867 } else if (name.equals("hierLrgExp")) { 868 mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 869 } else if (name.equals("hierMidExp")) { 870 mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 871 } else if (name.equals("hierSmlExp")) { 872 mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 873 } else if (name.equals("lumScale") || name.equals("chromaScale")) { 874 float[] yuvWeights = { mLumScale, mChromaScale }; 875 mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 876 } else if (name.equals("maskBg")) { 877 mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 878 } else if (name.equals("maskFg")) { 879 mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 880 } else if (name.equals("exposureChange")) { 881 mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 882 } else if (name.equals("whitebalanceredChange")) { 883 mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 884 } else if (name.equals("whitebalanceblueChange")) { 885 mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 886 } else if (name.equals("autowbToggle")){ 887 mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 888 } 889 } 890 updateBgScaling(Frame video, Frame background, boolean fitModeChanged)891 private void updateBgScaling(Frame video, Frame background, boolean fitModeChanged) { 892 float foregroundAspect = (float)video.getFormat().getWidth() / video.getFormat().getHeight(); 893 float backgroundAspect = (float)background.getFormat().getWidth() / background.getFormat().getHeight(); 894 float currentRelativeAspect = foregroundAspect/backgroundAspect; 895 if (currentRelativeAspect != mRelativeAspect || fitModeChanged) { 896 mRelativeAspect = currentRelativeAspect; 897 float xMin = 0.f, xWidth = 1.f, yMin = 0.f, yWidth = 1.f; 898 switch (mBackgroundFitMode) { 899 case BACKGROUND_STRETCH: 900 // Just map 1:1 901 break; 902 case BACKGROUND_FIT: 903 if (mRelativeAspect > 1.0f) { 904 // Foreground is wider than background, scale down 905 // background in X 906 xMin = 0.5f - 0.5f * mRelativeAspect; 907 xWidth = 1.f * mRelativeAspect; 908 } else { 909 // Foreground is taller than background, scale down 910 // background in Y 911 yMin = 0.5f - 0.5f / mRelativeAspect; 912 yWidth = 1 / mRelativeAspect; 913 } 914 break; 915 case BACKGROUND_FILL_CROP: 916 if (mRelativeAspect > 1.0f) { 917 // Foreground is wider than background, crop 918 // background in Y 919 yMin = 0.5f - 0.5f / mRelativeAspect; 920 yWidth = 1.f / mRelativeAspect; 921 } else { 922 // Foreground is taller than background, crop 923 // background in X 924 xMin = 0.5f - 0.5f * mRelativeAspect; 925 xWidth = mRelativeAspect; 926 } 927 break; 928 } 929 float[] bgTransform = {xWidth, 0.f, 0.f, 930 0.f, yWidth, 0.f, 931 xMin, yMin, 1.f}; 932 mBgSubtractProgram.setHostValue("bg_fit_transform", bgTransform); 933 } 934 } 935 pyramidLevel(int size)936 private int pyramidLevel(int size) { 937 return (int)Math.floor(Math.log10(size) / Math.log10(2)) - 1; 938 } 939 940 } 941