• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "common/softmax.h"
18 
19 #include <limits>
20 
21 #include "common/fastexp.h"
22 #include "util/base/logging.h"
23 
24 namespace libtextclassifier {
25 namespace nlp_core {
26 
ComputeSoftmaxProbability(const std::vector<float> & scores,int label)27 float ComputeSoftmaxProbability(const std::vector<float> &scores, int label) {
28   if ((label < 0) || (label >= scores.size())) {
29     TC_LOG(ERROR) << "label " << label << " outside range "
30                   << "[0, " << scores.size() << ")";
31     return 0.0f;
32   }
33 
34   // Standard softmax formula for label's probability is
35   //
36   //   exp(scores[label]) / sum_i exp(scores[i])
37   //
38   // We compute the mathematically equivalent
39   //
40   //   1 / (1 + sum_{i != label} exp(scores[i] - scores[label]))
41   //
42   // which saves two calls to exp().
43   const float label_score = scores[label];
44   float denominator = 1.0f;  // Contribution of i == label.
45   for (int i = 0; i < scores.size(); ++i) {
46     if (i == label) continue;
47     const float delta_score = scores[i] - label_score;
48 
49     // TODO(salcianu): one can optimize the test below, to avoid any float
50     // operation: extract exponent (via bit mask + shift) and check it's >= 4.
51     if (fabs(delta_score) >= 16.0f) {
52       if (delta_score > 0.0f) {
53         // If delta_score >= 16, the denominator (e^delta_score + other positive
54         // terms) is very big and its inverse can be approximated with 0.
55         return 0.0f;
56       } else {
57         // If delta_score <= -16, then e^delta_score < 1.2e-7.  Even if we have
58         // 1000 such labels i, their sum is < 1.2e-4 (which gets summed with
59         // 1.0f for i == label).  Hence, we can approximate each such label with
60         // 0 and skip the call to VeryFastExp and the update to denominator.
61         continue;
62       }
63     }
64 
65     // At this point, delta_score is in (-16.0, 16.0).  For such values, vfexp
66     // works fine: no under/overflows (we have tests for that in fastexp_test).
67     // Also, even for 1000 labels, denominator will not overflow.
68     denominator += VeryFastExp(delta_score);
69   }
70   return 1.0f / denominator;
71 }
72 
ComputeSoftmax(const std::vector<float> & scores)73 std::vector<float> ComputeSoftmax(const std::vector<float> &scores) {
74   std::vector<float> softmax;
75   std::vector<float> exp_scores;
76   exp_scores.reserve(scores.size());
77   softmax.reserve(scores.size());
78 
79   // Find max value in "scores" vector and rescale to avoid overflows.
80   float max = std::numeric_limits<float>::min();
81   for (const auto &score : scores) {
82     if (score > max) max = score;
83   }
84   float denominator = 0;
85   for (auto &score : scores) {
86     // See comments above in ComputeSoftmaxProbability for the reasoning behind
87     // this approximation.
88     const float exp_score = score - max < -16.0f ? 0 : VeryFastExp(score - max);
89     exp_scores.push_back(exp_score);
90     denominator += exp_score;
91   }
92 
93   for (int i = 0; i < scores.size(); ++i) {
94     softmax.push_back(exp_scores[i] / denominator);
95   }
96   return softmax;
97 }
98 
99 }  // namespace nlp_core
100 }  // namespace libtextclassifier
101