1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "TfLiteMotionPredictor"
18 #include <input/TfLiteMotionPredictor.h>
19
20 #include <fcntl.h>
21 #include <sys/mman.h>
22 #include <unistd.h>
23
24 #include <algorithm>
25 #include <cmath>
26 #include <cstddef>
27 #include <cstdint>
28 #include <memory>
29 #include <span>
30 #include <type_traits>
31 #include <utility>
32
33 #include <android-base/file.h>
34 #include <android-base/logging.h>
35 #include <android-base/mapped_file.h>
36 #define ATRACE_TAG ATRACE_TAG_INPUT
37 #include <cutils/trace.h>
38 #include <log/log.h>
39 #include <utils/Timers.h>
40
41 #include "tensorflow/lite/core/api/error_reporter.h"
42 #include "tensorflow/lite/core/api/op_resolver.h"
43 #include "tensorflow/lite/interpreter.h"
44 #include "tensorflow/lite/kernels/builtin_op_kernels.h"
45 #include "tensorflow/lite/model.h"
46 #include "tensorflow/lite/mutable_op_resolver.h"
47
48 #include "tinyxml2.h"
49
50 namespace android {
51 namespace {
52
53 constexpr char SIGNATURE_KEY[] = "serving_default";
54
55 // Input tensor names.
56 constexpr char INPUT_R[] = "r";
57 constexpr char INPUT_PHI[] = "phi";
58 constexpr char INPUT_PRESSURE[] = "pressure";
59 constexpr char INPUT_TILT[] = "tilt";
60 constexpr char INPUT_ORIENTATION[] = "orientation";
61
62 // Output tensor names.
63 constexpr char OUTPUT_R[] = "r";
64 constexpr char OUTPUT_PHI[] = "phi";
65 constexpr char OUTPUT_PRESSURE[] = "pressure";
66
67 // Ideally, we would just use std::filesystem::exists here, but it requires libc++fs, which causes
68 // build issues in other parts of the system.
69 #if defined(__ANDROID__)
fileExists(const char * filename)70 bool fileExists(const char* filename) {
71 struct stat buffer;
72 return stat(filename, &buffer) == 0;
73 }
74 #endif
75
getModelPath()76 std::string getModelPath() {
77 #if defined(__ANDROID__)
78 static const char* oemModel = "/vendor/etc/motion_predictor_model.tflite";
79 if (fileExists(oemModel)) {
80 return oemModel;
81 }
82 return "/system/etc/motion_predictor_model.tflite";
83 #else
84 return base::GetExecutableDirectory() + "/motion_predictor_model.tflite";
85 #endif
86 }
87
getConfigPath()88 std::string getConfigPath() {
89 // The config file should be alongside the model file.
90 return base::Dirname(getModelPath()) + "/motion_predictor_config.xml";
91 }
92
parseXMLInt64(const tinyxml2::XMLElement & configRoot,const char * elementName)93 int64_t parseXMLInt64(const tinyxml2::XMLElement& configRoot, const char* elementName) {
94 const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName);
95 LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName);
96
97 int64_t value = 0;
98 LOG_ALWAYS_FATAL_IF(element->QueryInt64Text(&value) != tinyxml2::XML_SUCCESS,
99 "Failed to parse %s: %s", elementName, element->GetText());
100 return value;
101 }
102
parseXMLFloat(const tinyxml2::XMLElement & configRoot,const char * elementName)103 float parseXMLFloat(const tinyxml2::XMLElement& configRoot, const char* elementName) {
104 const tinyxml2::XMLElement* element = configRoot.FirstChildElement(elementName);
105 LOG_ALWAYS_FATAL_IF(!element, "Could not find '%s' element", elementName);
106
107 float value = 0;
108 LOG_ALWAYS_FATAL_IF(element->QueryFloatText(&value) != tinyxml2::XML_SUCCESS,
109 "Failed to parse %s: %s", elementName, element->GetText());
110 return value;
111 }
112
113 // A TFLite ErrorReporter that logs to logcat.
114 class LoggingErrorReporter : public tflite::ErrorReporter {
115 public:
Report(const char * format,va_list args)116 int Report(const char* format, va_list args) override {
117 return LOG_PRI_VA(ANDROID_LOG_ERROR, LOG_TAG, format, args);
118 }
119 };
120
121 // Searches a runner for an input tensor.
findInputTensor(const char * name,tflite::SignatureRunner * runner)122 TfLiteTensor* findInputTensor(const char* name, tflite::SignatureRunner* runner) {
123 TfLiteTensor* tensor = runner->input_tensor(name);
124 LOG_ALWAYS_FATAL_IF(!tensor, "Failed to find input tensor '%s'", name);
125 return tensor;
126 }
127
128 // Searches a runner for an output tensor.
findOutputTensor(const char * name,tflite::SignatureRunner * runner)129 const TfLiteTensor* findOutputTensor(const char* name, tflite::SignatureRunner* runner) {
130 const TfLiteTensor* tensor = runner->output_tensor(name);
131 LOG_ALWAYS_FATAL_IF(!tensor, "Failed to find output tensor '%s'", name);
132 return tensor;
133 }
134
135 // Returns the buffer for a tensor of type T.
136 template <typename T>
getTensorBuffer(typename std::conditional<std::is_const<T>::value,const TfLiteTensor *,TfLiteTensor * >::type tensor)137 std::span<T> getTensorBuffer(typename std::conditional<std::is_const<T>::value, const TfLiteTensor*,
138 TfLiteTensor*>::type tensor) {
139 LOG_ALWAYS_FATAL_IF(!tensor);
140
141 const TfLiteType type = tflite::typeToTfLiteType<typename std::remove_cv<T>::type>();
142 LOG_ALWAYS_FATAL_IF(tensor->type != type, "Unexpected type for '%s' tensor: %s (expected %s)",
143 tensor->name, TfLiteTypeGetName(tensor->type), TfLiteTypeGetName(type));
144
145 LOG_ALWAYS_FATAL_IF(!tensor->data.data);
146 return {reinterpret_cast<T*>(tensor->data.data),
147 static_cast<typename std::span<T>::index_type>(tensor->bytes / sizeof(T))};
148 }
149
150 // Verifies that a tensor exists and has an underlying buffer of type T.
151 template <typename T>
checkTensor(const TfLiteTensor * tensor)152 void checkTensor(const TfLiteTensor* tensor) {
153 LOG_ALWAYS_FATAL_IF(!tensor);
154
155 const auto buffer = getTensorBuffer<const T>(tensor);
156 LOG_ALWAYS_FATAL_IF(buffer.empty(), "No buffer for tensor '%s'", tensor->name);
157 }
158
createOpResolver()159 std::unique_ptr<tflite::OpResolver> createOpResolver() {
160 auto resolver = std::make_unique<tflite::MutableOpResolver>();
161 resolver->AddBuiltin(::tflite::BuiltinOperator_CONCATENATION,
162 ::tflite::ops::builtin::Register_CONCATENATION());
163 resolver->AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED,
164 ::tflite::ops::builtin::Register_FULLY_CONNECTED());
165 resolver->AddBuiltin(::tflite::BuiltinOperator_GELU, ::tflite::ops::builtin::Register_GELU());
166 return resolver;
167 }
168
169 } // namespace
170
TfLiteMotionPredictorBuffers(size_t inputLength)171 TfLiteMotionPredictorBuffers::TfLiteMotionPredictorBuffers(size_t inputLength)
172 : mInputR(inputLength, 0),
173 mInputPhi(inputLength, 0),
174 mInputPressure(inputLength, 0),
175 mInputTilt(inputLength, 0),
176 mInputOrientation(inputLength, 0) {
177 LOG_ALWAYS_FATAL_IF(inputLength == 0, "Buffer input size must be greater than 0");
178 }
179
reset()180 void TfLiteMotionPredictorBuffers::reset() {
181 std::fill(mInputR.begin(), mInputR.end(), 0);
182 std::fill(mInputPhi.begin(), mInputPhi.end(), 0);
183 std::fill(mInputPressure.begin(), mInputPressure.end(), 0);
184 std::fill(mInputTilt.begin(), mInputTilt.end(), 0);
185 std::fill(mInputOrientation.begin(), mInputOrientation.end(), 0);
186 mAxisFrom.reset();
187 mAxisTo.reset();
188 }
189
copyTo(TfLiteMotionPredictorModel & model) const190 void TfLiteMotionPredictorBuffers::copyTo(TfLiteMotionPredictorModel& model) const {
191 LOG_ALWAYS_FATAL_IF(mInputR.size() != model.inputLength(),
192 "Buffer length %zu doesn't match model input length %zu", mInputR.size(),
193 model.inputLength());
194 LOG_ALWAYS_FATAL_IF(!isReady(), "Buffers are incomplete");
195
196 std::copy(mInputR.begin(), mInputR.end(), model.inputR().begin());
197 std::copy(mInputPhi.begin(), mInputPhi.end(), model.inputPhi().begin());
198 std::copy(mInputPressure.begin(), mInputPressure.end(), model.inputPressure().begin());
199 std::copy(mInputTilt.begin(), mInputTilt.end(), model.inputTilt().begin());
200 std::copy(mInputOrientation.begin(), mInputOrientation.end(), model.inputOrientation().begin());
201 }
202
pushSample(int64_t timestamp,const TfLiteMotionPredictorSample sample)203 void TfLiteMotionPredictorBuffers::pushSample(int64_t timestamp,
204 const TfLiteMotionPredictorSample sample) {
205 // Convert the sample (x, y) into polar (r, φ) based on a reference axis
206 // from the preceding two points (mAxisFrom/mAxisTo).
207
208 mTimestamp = timestamp;
209
210 if (!mAxisTo) { // First point.
211 mAxisTo = sample;
212 return;
213 }
214
215 // Vector from the last point to the current sample point.
216 const TfLiteMotionPredictorSample::Point v = sample.position - mAxisTo->position;
217
218 const float r = std::hypot(v.x, v.y);
219 float phi = 0;
220 float orientation = 0;
221
222 if (!mAxisFrom && r > 0) { // Second point.
223 // We can only determine the distance from the first point, and not any
224 // angle. However, if the second point forms an axis, the orientation can
225 // be transformed relative to that axis.
226 const float axisPhi = std::atan2(v.y, v.x);
227 // A MotionEvent's orientation is measured clockwise from the vertical
228 // axis, but axisPhi is measured counter-clockwise from the horizontal
229 // axis.
230 orientation = M_PI_2 - sample.orientation - axisPhi;
231 } else {
232 const TfLiteMotionPredictorSample::Point axis = mAxisTo->position - mAxisFrom->position;
233 const float axisPhi = std::atan2(axis.y, axis.x);
234 phi = std::atan2(v.y, v.x) - axisPhi;
235
236 if (std::hypot(axis.x, axis.y) > 0) {
237 // See note above.
238 orientation = M_PI_2 - sample.orientation - axisPhi;
239 }
240 }
241
242 // Update the axis for the next point.
243 if (r > 0) {
244 mAxisFrom = mAxisTo;
245 mAxisTo = sample;
246 }
247
248 // Push the current sample onto the end of the input buffers.
249 mInputR.pushBack(r);
250 mInputPhi.pushBack(phi);
251 mInputPressure.pushBack(sample.pressure);
252 mInputTilt.pushBack(sample.tilt);
253 mInputOrientation.pushBack(orientation);
254 }
255
create()256 std::unique_ptr<TfLiteMotionPredictorModel> TfLiteMotionPredictorModel::create() {
257 const std::string modelPath = getModelPath();
258 android::base::unique_fd fd(open(modelPath.c_str(), O_RDONLY));
259 if (fd == -1) {
260 PLOG(FATAL) << "Could not read model from " << modelPath;
261 }
262
263 const off_t fdSize = lseek(fd, 0, SEEK_END);
264 if (fdSize == -1) {
265 PLOG(FATAL) << "Failed to determine file size";
266 }
267
268 std::unique_ptr<android::base::MappedFile> modelBuffer =
269 android::base::MappedFile::FromFd(fd, /*offset=*/0, fdSize, PROT_READ);
270 if (!modelBuffer) {
271 PLOG(FATAL) << "Failed to mmap model";
272 }
273
274 const std::string configPath = getConfigPath();
275 tinyxml2::XMLDocument configDocument;
276 LOG_ALWAYS_FATAL_IF(configDocument.LoadFile(configPath.c_str()) != tinyxml2::XML_SUCCESS,
277 "Failed to load config file from %s", configPath.c_str());
278
279 // Parse configuration file.
280 const tinyxml2::XMLElement* configRoot = configDocument.FirstChildElement("motion-predictor");
281 LOG_ALWAYS_FATAL_IF(!configRoot);
282 Config config{
283 .predictionInterval = parseXMLInt64(*configRoot, "prediction-interval"),
284 .distanceNoiseFloor = parseXMLFloat(*configRoot, "distance-noise-floor"),
285 };
286
287 return std::unique_ptr<TfLiteMotionPredictorModel>(
288 new TfLiteMotionPredictorModel(std::move(modelBuffer), std::move(config)));
289 }
290
TfLiteMotionPredictorModel(std::unique_ptr<android::base::MappedFile> model,Config config)291 TfLiteMotionPredictorModel::TfLiteMotionPredictorModel(
292 std::unique_ptr<android::base::MappedFile> model, Config config)
293 : mFlatBuffer(std::move(model)), mConfig(std::move(config)) {
294 CHECK(mFlatBuffer);
295 mErrorReporter = std::make_unique<LoggingErrorReporter>();
296 mModel = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(mFlatBuffer->data(),
297 mFlatBuffer->size(),
298 /*extra_verifier=*/nullptr,
299 mErrorReporter.get());
300 LOG_ALWAYS_FATAL_IF(!mModel);
301
302 auto resolver = createOpResolver();
303 tflite::InterpreterBuilder builder(*mModel, *resolver);
304
305 if (builder(&mInterpreter) != kTfLiteOk || !mInterpreter) {
306 LOG_ALWAYS_FATAL("Failed to build interpreter");
307 }
308
309 mRunner = mInterpreter->GetSignatureRunner(SIGNATURE_KEY);
310 LOG_ALWAYS_FATAL_IF(!mRunner, "Failed to find runner for signature '%s'", SIGNATURE_KEY);
311
312 allocateTensors();
313 }
314
~TfLiteMotionPredictorModel()315 TfLiteMotionPredictorModel::~TfLiteMotionPredictorModel() {}
316
allocateTensors()317 void TfLiteMotionPredictorModel::allocateTensors() {
318 if (mRunner->AllocateTensors() != kTfLiteOk) {
319 LOG_ALWAYS_FATAL("Failed to allocate tensors");
320 }
321
322 attachInputTensors();
323 attachOutputTensors();
324
325 checkTensor<float>(mInputR);
326 checkTensor<float>(mInputPhi);
327 checkTensor<float>(mInputPressure);
328 checkTensor<float>(mInputTilt);
329 checkTensor<float>(mInputOrientation);
330 checkTensor<float>(mOutputR);
331 checkTensor<float>(mOutputPhi);
332 checkTensor<float>(mOutputPressure);
333
334 const auto checkInputTensorSize = [this](const TfLiteTensor* tensor) {
335 const size_t size = getTensorBuffer<const float>(tensor).size();
336 LOG_ALWAYS_FATAL_IF(size != inputLength(),
337 "Tensor '%s' length %zu does not match input length %zu", tensor->name,
338 size, inputLength());
339 };
340
341 checkInputTensorSize(mInputR);
342 checkInputTensorSize(mInputPhi);
343 checkInputTensorSize(mInputPressure);
344 checkInputTensorSize(mInputTilt);
345 checkInputTensorSize(mInputOrientation);
346 }
347
attachInputTensors()348 void TfLiteMotionPredictorModel::attachInputTensors() {
349 mInputR = findInputTensor(INPUT_R, mRunner);
350 mInputPhi = findInputTensor(INPUT_PHI, mRunner);
351 mInputPressure = findInputTensor(INPUT_PRESSURE, mRunner);
352 mInputTilt = findInputTensor(INPUT_TILT, mRunner);
353 mInputOrientation = findInputTensor(INPUT_ORIENTATION, mRunner);
354 }
355
attachOutputTensors()356 void TfLiteMotionPredictorModel::attachOutputTensors() {
357 mOutputR = findOutputTensor(OUTPUT_R, mRunner);
358 mOutputPhi = findOutputTensor(OUTPUT_PHI, mRunner);
359 mOutputPressure = findOutputTensor(OUTPUT_PRESSURE, mRunner);
360 }
361
invoke()362 bool TfLiteMotionPredictorModel::invoke() {
363 ATRACE_BEGIN("TfLiteMotionPredictorModel::invoke");
364 TfLiteStatus result = mRunner->Invoke();
365 ATRACE_END();
366
367 if (result != kTfLiteOk) {
368 return false;
369 }
370
371 // Invoke() might reallocate tensors, so they need to be reattached.
372 attachInputTensors();
373 attachOutputTensors();
374
375 if (outputR().size() != outputPhi().size() || outputR().size() != outputPressure().size()) {
376 LOG_ALWAYS_FATAL("Output size mismatch: (r: %zu, phi: %zu, pressure: %zu)",
377 outputR().size(), outputPhi().size(), outputPressure().size());
378 }
379
380 return true;
381 }
382
inputLength() const383 size_t TfLiteMotionPredictorModel::inputLength() const {
384 return getTensorBuffer<const float>(mInputR).size();
385 }
386
outputLength() const387 size_t TfLiteMotionPredictorModel::outputLength() const {
388 return getTensorBuffer<const float>(mOutputR).size();
389 }
390
inputR()391 std::span<float> TfLiteMotionPredictorModel::inputR() {
392 return getTensorBuffer<float>(mInputR);
393 }
394
inputPhi()395 std::span<float> TfLiteMotionPredictorModel::inputPhi() {
396 return getTensorBuffer<float>(mInputPhi);
397 }
398
inputPressure()399 std::span<float> TfLiteMotionPredictorModel::inputPressure() {
400 return getTensorBuffer<float>(mInputPressure);
401 }
402
inputTilt()403 std::span<float> TfLiteMotionPredictorModel::inputTilt() {
404 return getTensorBuffer<float>(mInputTilt);
405 }
406
inputOrientation()407 std::span<float> TfLiteMotionPredictorModel::inputOrientation() {
408 return getTensorBuffer<float>(mInputOrientation);
409 }
410
outputR() const411 std::span<const float> TfLiteMotionPredictorModel::outputR() const {
412 return getTensorBuffer<const float>(mOutputR);
413 }
414
outputPhi() const415 std::span<const float> TfLiteMotionPredictorModel::outputPhi() const {
416 return getTensorBuffer<const float>(mOutputPhi);
417 }
418
outputPressure() const419 std::span<const float> TfLiteMotionPredictorModel::outputPressure() const {
420 return getTensorBuffer<const float>(mOutputPressure);
421 }
422
423 } // namespace android
424