• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_
17 #define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_
18 
19 #include <stdint.h>
20 
21 #include "tensorflow/examples/android/jni/object_tracking/geom.h"
22 #include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
23 #include "tensorflow/examples/android/jni/object_tracking/image.h"
24 #include "tensorflow/examples/android/jni/object_tracking/utils.h"
25 
26 
27 namespace tf_tracking {
28 
GetUV(const uint8_t * const input,Image<uint8_t> * const u,Image<uint8_t> * const v)29 inline void GetUV(const uint8_t* const input, Image<uint8_t>* const u,
30                   Image<uint8_t>* const v) {
31   const uint8_t* pUV = input;
32 
33   for (int row = 0; row < u->GetHeight(); ++row) {
34     uint8_t* u_curr = (*u)[row];
35     uint8_t* v_curr = (*v)[row];
36     for (int col = 0; col < u->GetWidth(); ++col) {
37 #ifdef __APPLE__
38       *u_curr++ = *pUV++;
39       *v_curr++ = *pUV++;
40 #else
41       *v_curr++ = *pUV++;
42       *u_curr++ = *pUV++;
43 #endif
44     }
45   }
46 }
47 
48 // Marks every point within a circle of a given radius on the given boolean
49 // image true.
50 template <typename U>
MarkImage(const int x,const int y,const int radius,Image<U> * const img)51 inline static void MarkImage(const int x, const int y, const int radius,
52                              Image<U>* const img) {
53   SCHECK(img->ValidPixel(x, y), "Marking invalid pixel in image! %d, %d", x, y);
54 
55   // Precomputed for efficiency.
56   const int squared_radius = Square(radius);
57 
58   // Mark every row in the circle.
59   for (int d_y = 0; d_y <= radius; ++d_y) {
60     const int squared_y_dist = Square(d_y);
61 
62     const int min_y = MAX(y - d_y, 0);
63     const int max_y = MIN(y + d_y, img->height_less_one_);
64 
65     // The max d_x of the circle must be strictly greater or equal to
66     // radius - d_y for any positive d_y. Thus, starting from radius - d_y will
67     // reduce the number of iterations required as compared to starting from
68     // either 0 and counting up or radius and counting down.
69     for (int d_x = radius - d_y; d_x <= radius; ++d_x) {
70       // The first time this criteria is met, we know the width of the circle at
71       // this row (without using sqrt).
72       if (squared_y_dist + Square(d_x) >= squared_radius) {
73         const int min_x = MAX(x - d_x, 0);
74         const int max_x = MIN(x + d_x, img->width_less_one_);
75 
76         // Mark both above and below the center row.
77         bool* const top_row_start = (*img)[min_y] + min_x;
78         bool* const bottom_row_start = (*img)[max_y] + min_x;
79 
80         const int x_width = max_x - min_x + 1;
81         memset(top_row_start, true, sizeof(*top_row_start) * x_width);
82         memset(bottom_row_start, true, sizeof(*bottom_row_start) * x_width);
83 
84         // This row is marked, time to move on to the next row.
85         break;
86       }
87     }
88   }
89 }
90 
91 #ifdef __ARM_NEON
92 void CalculateGNeon(
93     const float* const vals_x, const float* const vals_y,
94     const int num_vals, float* const G);
95 #endif
96 
97 // Puts the image gradient matrix about a pixel into the 2x2 float array G.
98 // vals_x should be an array of the window x gradient values, whose indices
99 // can be in any order but are parallel to the vals_y entries.
100 // See http://robots.stanford.edu/cs223b04/algo_tracking.pdf for more details.
CalculateG(const float * const vals_x,const float * const vals_y,const int num_vals,float * const G)101 inline void CalculateG(const float* const vals_x, const float* const vals_y,
102                        const int num_vals, float* const G) {
103 #ifdef __ARM_NEON
104   CalculateGNeon(vals_x, vals_y, num_vals, G);
105   return;
106 #endif
107 
108   // Non-accelerated version.
109   for (int i = 0; i < num_vals; ++i) {
110     G[0] += Square(vals_x[i]);
111     G[1] += vals_x[i] * vals_y[i];
112     G[3] += Square(vals_y[i]);
113   }
114 
115   // The matrix is symmetric, so this is a given.
116   G[2] = G[1];
117 }
118 
CalculateGInt16(const int16_t * const vals_x,const int16_t * const vals_y,const int num_vals,int * const G)119 inline void CalculateGInt16(const int16_t* const vals_x,
120                             const int16_t* const vals_y, const int num_vals,
121                             int* const G) {
122   // Non-accelerated version.
123   for (int i = 0; i < num_vals; ++i) {
124     G[0] += Square(vals_x[i]);
125     G[1] += vals_x[i] * vals_y[i];
126     G[3] += Square(vals_y[i]);
127   }
128 
129   // The matrix is symmetric, so this is a given.
130   G[2] = G[1];
131 }
132 
133 
134 // Puts the image gradient matrix about a pixel into the 2x2 float array G.
135 // Looks up interpolated pixels, then calls above method for implementation.
CalculateG(const int window_radius,const float center_x,const float center_y,const Image<int32_t> & I_x,const Image<int32_t> & I_y,float * const G)136 inline void CalculateG(const int window_radius, const float center_x,
137                        const float center_y, const Image<int32_t>& I_x,
138                        const Image<int32_t>& I_y, float* const G) {
139   SCHECK(I_x.ValidPixel(center_x, center_y), "Problem in calculateG!");
140 
141   // Hardcoded to allow for a max window radius of 5 (9 pixels x 9 pixels).
142   static const int kMaxWindowRadius = 5;
143   SCHECK(window_radius <= kMaxWindowRadius,
144         "Window %d > %d!", window_radius, kMaxWindowRadius);
145 
146   // Diameter of window is 2 * radius + 1 for center pixel.
147   static const int kWindowBufferSize =
148       (kMaxWindowRadius * 2 + 1) * (kMaxWindowRadius * 2 + 1);
149 
150   // Preallocate buffers statically for efficiency.
151   static int16_t vals_x[kWindowBufferSize];
152   static int16_t vals_y[kWindowBufferSize];
153 
154   const int src_left_fixed = RealToFixed1616(center_x - window_radius);
155   const int src_top_fixed = RealToFixed1616(center_y - window_radius);
156 
157   int16_t* vals_x_ptr = vals_x;
158   int16_t* vals_y_ptr = vals_y;
159 
160   const int window_size = 2 * window_radius + 1;
161   for (int y = 0; y < window_size; ++y) {
162     const int fp_y = src_top_fixed + (y << 16);
163 
164     for (int x = 0; x < window_size; ++x) {
165       const int fp_x = src_left_fixed + (x << 16);
166 
167       *vals_x_ptr++ = I_x.GetPixelInterpFixed1616(fp_x, fp_y);
168       *vals_y_ptr++ = I_y.GetPixelInterpFixed1616(fp_x, fp_y);
169     }
170   }
171 
172   int32_t g_temp[] = {0, 0, 0, 0};
173   CalculateGInt16(vals_x, vals_y, window_size * window_size, g_temp);
174 
175   for (int i = 0; i < 4; ++i) {
176     G[i] = g_temp[i];
177   }
178 }
179 
ImageCrossCorrelation(const Image<float> & image1,const Image<float> & image2,const int x_offset,const int y_offset)180 inline float ImageCrossCorrelation(const Image<float>& image1,
181                                    const Image<float>& image2,
182                                    const int x_offset, const int y_offset) {
183   SCHECK(image1.GetWidth() == image2.GetWidth() &&
184          image1.GetHeight() == image2.GetHeight(),
185         "Dimension mismatch! %dx%d vs %dx%d",
186         image1.GetWidth(), image1.GetHeight(),
187         image2.GetWidth(), image2.GetHeight());
188 
189   const int num_pixels = image1.GetWidth() * image1.GetHeight();
190   const float* data1 = image1.data();
191   const float* data2 = image2.data();
192   return ComputeCrossCorrelation(data1, data2, num_pixels);
193 }
194 
195 // Copies an arbitrary region of an image to another (floating point)
196 // image, scaling as it goes using bilinear interpolation.
CopyArea(const Image<uint8_t> & image,const BoundingBox & area_to_copy,Image<float> * const patch_image)197 inline void CopyArea(const Image<uint8_t>& image,
198                      const BoundingBox& area_to_copy,
199                      Image<float>* const patch_image) {
200   VLOG(2) << "Copying from: " << area_to_copy << std::endl;
201 
202   const int patch_width = patch_image->GetWidth();
203   const int patch_height = patch_image->GetHeight();
204 
205   const float x_dist_between_samples = patch_width > 0 ?
206       area_to_copy.GetWidth() / (patch_width - 1) : 0;
207 
208   const float y_dist_between_samples = patch_height > 0 ?
209       area_to_copy.GetHeight() / (patch_height - 1) : 0;
210 
211   for (int y_index = 0; y_index < patch_height; ++y_index) {
212     const float sample_y =
213         y_index * y_dist_between_samples + area_to_copy.top_;
214 
215     for (int x_index = 0; x_index < patch_width; ++x_index) {
216       const float sample_x =
217           x_index * x_dist_between_samples + area_to_copy.left_;
218 
219       if (image.ValidInterpPixel(sample_x, sample_y)) {
220         // TODO(andrewharp): Do area averaging when downsampling.
221         (*patch_image)[y_index][x_index] =
222             image.GetPixelInterp(sample_x, sample_y);
223       } else {
224         (*patch_image)[y_index][x_index] = -1.0f;
225       }
226     }
227   }
228 }
229 
230 
231 // Takes a floating point image and normalizes it in-place.
232 //
233 // First, negative values will be set to the mean of the non-negative pixels
234 // in the image.
235 //
236 // Then, the resulting will be normalized such that it has mean value of 0.0 and
237 // a standard deviation of 1.0.
NormalizeImage(Image<float> * const image)238 inline void NormalizeImage(Image<float>* const image) {
239   const float* const data_ptr = image->data();
240 
241   // Copy only the non-negative values to some temp memory.
242   float running_sum = 0.0f;
243   int num_data_gte_zero = 0;
244   {
245     float* const curr_data = (*image)[0];
246     for (int i = 0; i < image->data_size_; ++i) {
247       if (curr_data[i] >= 0.0f) {
248         running_sum += curr_data[i];
249         ++num_data_gte_zero;
250       } else {
251         curr_data[i] = -1.0f;
252       }
253     }
254   }
255 
256   // If none of the pixels are valid, just set the entire thing to 0.0f.
257   if (num_data_gte_zero == 0) {
258     image->Clear(0.0f);
259     return;
260   }
261 
262   const float corrected_mean = running_sum / num_data_gte_zero;
263 
264   float* curr_data = (*image)[0];
265   for (int i = 0; i < image->data_size_; ++i) {
266     const float curr_val = *curr_data;
267     *curr_data++ = curr_val < 0 ? 0 : curr_val - corrected_mean;
268   }
269 
270   const float std_dev = ComputeStdDev(data_ptr, image->data_size_, 0.0f);
271 
272   if (std_dev > 0.0f) {
273     curr_data = (*image)[0];
274     for (int i = 0; i < image->data_size_; ++i) {
275       *curr_data++ /= std_dev;
276     }
277 
278 #ifdef SANITY_CHECKS
279     LOGV("corrected_mean: %1.2f  std_dev: %1.2f", corrected_mean, std_dev);
280     const float correlation =
281         ComputeCrossCorrelation(image->data(),
282                                 image->data(),
283                                 image->data_size_);
284 
285     if (std::abs(correlation - 1.0f) > EPSILON) {
286       LOG(ERROR) << "Bad image!" << std::endl;
287       LOG(ERROR) << *image << std::endl;
288     }
289 
290     SCHECK(std::abs(correlation - 1.0f) < EPSILON,
291            "Correlation wasn't 1.0f:  %.10f", correlation);
292 #endif
293   }
294 }
295 
296 }  // namespace tf_tracking
297 
298 #endif  // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_UTILS_H_
299