• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Lint as: python2, python3
2# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15# ==============================================================================
16"""Python TFLite metrics helper."""
17from typing import Optional, Text
18import uuid
19
20from tensorflow.lite.python import metrics_interface
21from tensorflow.lite.python.metrics_wrapper import converter_error_data_pb2
22from tensorflow.lite.python.metrics_wrapper import metrics_wrapper
23from tensorflow.python.eager import monitoring
24
25_counter_debugger_creation = monitoring.Counter(
26    '/tensorflow/lite/quantization_debugger/created',
27    'Counter for the number of debugger created.')
28
29_counter_interpreter_creation = monitoring.Counter(
30    '/tensorflow/lite/interpreter/created',
31    'Counter for number of interpreter created in Python.', 'language')
32
33# The following are conversion metrics. Attempt and success are kept separated
34# instead of using a single metric with a label because the converter may
35# raise exceptions if conversion failed. That may lead to cases when we are
36# unable to capture the conversion attempt. Increasing attempt count at the
37# beginning of conversion process and the success count at the end is more
38# suitable in these cases.
39_counter_conversion_attempt = monitoring.Counter(
40    '/tensorflow/lite/convert/attempt',
41    'Counter for number of conversion attempts.')
42
43_counter_conversion_success = monitoring.Counter(
44    '/tensorflow/lite/convert/success',
45    'Counter for number of successful conversions.')
46
47_gauge_conversion_params = monitoring.StringGauge(
48    '/tensorflow/lite/convert/params',
49    'Gauge for keeping conversion parameters.', 'name')
50
51_gauge_conversion_errors = monitoring.StringGauge(
52    '/tensorflow/lite/convert/errors',
53    'Gauge for collecting conversion errors. The value represents the error '
54    'message.', 'component', 'subcomponent', 'op_name', 'error_code')
55
56_gauge_conversion_latency = monitoring.IntGauge(
57    '/tensorflow/lite/convert/latency', 'Conversion latency in ms.')
58
59
60class TFLiteMetrics(metrics_interface.TFLiteMetricsInterface):
61  """TFLite metrics helper for prod (borg) environment.
62
63  Attributes:
64    model_hash: A string containing the hash of the model binary.
65    model_path: A string containing the path of the model for debugging
66      purposes.
67  """
68
69  def __init__(self,
70               model_hash: Optional[Text] = None,
71               model_path: Optional[Text] = None) -> None:
72    del self  # Temporarily removing self until parameter logic is implemented.
73    if model_hash and not model_path or not model_hash and model_path:
74      raise ValueError('Both model metadata(model_hash, model_path) should be '
75                       'given at the same time.')
76    if model_hash:
77      # TODO(b/180400857): Create stub once the service is implemented.
78      pass
79
80  def increase_counter_debugger_creation(self):
81    _counter_debugger_creation.get_cell().increase_by(1)
82
83  def increase_counter_interpreter_creation(self):
84    _counter_interpreter_creation.get_cell('python').increase_by(1)
85
86  def increase_counter_converter_attempt(self):
87    _counter_conversion_attempt.get_cell().increase_by(1)
88
89  def increase_counter_converter_success(self):
90    _counter_conversion_success.get_cell().increase_by(1)
91
92  def set_converter_param(self, name, value):
93    _gauge_conversion_params.get_cell(name).set(value)
94
95  def set_converter_error(
96      self, error_data: converter_error_data_pb2.ConverterErrorData):
97    error_code_str = converter_error_data_pb2.ConverterErrorData.ErrorCode.Name(
98        error_data.error_code)
99    _gauge_conversion_errors.get_cell(
100        error_data.component,
101        error_data.subcomponent,
102        error_data.operator.name,
103        error_code_str,
104    ).set(error_data.error_message)
105
106  def set_converter_latency(self, value):
107    _gauge_conversion_latency.get_cell().set(value)
108
109
110class TFLiteConverterMetrics(TFLiteMetrics):
111  """Similar to TFLiteMetrics but specialized for converter.
112
113  A unique session id will be created for each new TFLiteConverterMetrics.
114  """
115
116  def __init__(self) -> None:
117    super(TFLiteConverterMetrics, self).__init__()
118    session_id = uuid.uuid4().hex
119    self._metrics_exporter = metrics_wrapper.MetricsWrapper(session_id)
120    self._exported = False
121
122  def __del__(self):
123    if not self._exported:
124      self.export_metrics()
125
126  def set_export_required(self):
127    self._exported = False
128
129  def export_metrics(self):
130    self._metrics_exporter.ExportMetrics()
131    self._exported = True
132