• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Contains functions for evaluation and summarization of metrics."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import math
22import time
23
24from tensorflow.python.framework import dtypes
25from tensorflow.python.framework import ops
26from tensorflow.python.ops import array_ops
27from tensorflow.python.ops import init_ops
28from tensorflow.python.ops import math_ops
29from tensorflow.python.ops import state_ops
30from tensorflow.python.ops import variable_scope
31from tensorflow.python.platform import tf_logging as logging
32from tensorflow.python.training import basic_session_run_hooks
33from tensorflow.python.training import monitored_session
34from tensorflow.python.training import session_run_hook
35
36
37def _get_or_create_eval_step():
38  """Gets or creates the eval step `Tensor`.
39
40  Returns:
41    A `Tensor` representing a counter for the evaluation step.
42
43  Raises:
44    ValueError: If multiple `Tensors` have been added to the
45      `tf.GraphKeys.EVAL_STEP` collection.
46  """
47  graph = ops.get_default_graph()
48  eval_steps = graph.get_collection(ops.GraphKeys.EVAL_STEP)
49  if len(eval_steps) == 1:
50    return eval_steps[0]
51  elif len(eval_steps) > 1:
52    raise ValueError('Multiple tensors added to tf.GraphKeys.EVAL_STEP')
53  else:
54    counter = variable_scope.get_variable(
55        'eval_step',
56        shape=[],
57        dtype=dtypes.int64,
58        initializer=init_ops.zeros_initializer(),
59        trainable=False,
60        collections=[ops.GraphKeys.LOCAL_VARIABLES, ops.GraphKeys.EVAL_STEP])
61    return counter
62
63
64def _get_latest_eval_step_value(update_ops):
65  """Gets the eval step `Tensor` value after running `update_ops`.
66
67  Args:
68    update_ops: A list of `Tensors` or a dictionary of names to `Tensors`, which
69      are run before reading the eval step value.
70
71  Returns:
72    A `Tensor` representing the value for the evaluation step.
73  """
74  if isinstance(update_ops, dict):
75    update_ops = list(update_ops.values())
76
77  with ops.control_dependencies(update_ops):
78    return array_ops.identity(_get_or_create_eval_step().read_value())
79
80
81class _MultiStepStopAfterNEvalsHook(session_run_hook.SessionRunHook):
82  """Run hook used by the evaluation routines to run the `eval_ops` N times."""
83
84  def __init__(self, num_evals, steps_per_run=1):
85    """Constructs the run hook.
86
87    Args:
88      num_evals: The number of evaluations to run for. if set to None, will
89        iterate the dataset until all inputs are exhausted.
90      steps_per_run: Number of steps executed per run call.
91    """
92    self._num_evals = num_evals
93    self._evals_completed = None
94    self._steps_per_run_initial_value = steps_per_run
95
96  def _set_evals_completed_tensor(self, updated_eval_step):
97    self._evals_completed = updated_eval_step
98
99  def begin(self):
100    self._steps_per_run_variable = \
101        basic_session_run_hooks.get_or_create_steps_per_run_variable()
102
103  def after_create_session(self, session, coord):
104    # Update number of steps to run in the first run call
105    if self._num_evals is None:
106      steps = self._steps_per_run_initial_value
107    else:
108      steps = min(self._steps_per_run_initial_value, self._num_evals)
109    self._steps_per_run_variable.load(steps, session=session)
110
111  def before_run(self, run_context):
112    return session_run_hook.SessionRunArgs(
113        {'evals_completed': self._evals_completed})
114
115  def after_run(self, run_context, run_values):
116    evals_completed = run_values.results['evals_completed']
117    # Update number of steps to run in the next iteration
118    if self._num_evals is None:
119      steps = self._steps_per_run_initial_value
120    else:
121      steps = min(self._num_evals - evals_completed,
122                  self._steps_per_run_initial_value)
123    self._steps_per_run_variable.load(steps, session=run_context.session)
124
125    if self._num_evals is None:
126      logging.info('Evaluation [%d]', evals_completed)
127    else:
128      logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals)
129    if self._num_evals is not None and evals_completed >= self._num_evals:
130      run_context.request_stop()
131
132
133class _StopAfterNEvalsHook(session_run_hook.SessionRunHook):
134  """Run hook used by the evaluation routines to run the `eval_ops` N times."""
135
136  def __init__(self, num_evals, log_progress=True):
137    """Constructs the run hook.
138
139    Args:
140      num_evals: The number of evaluations to run for. if set to None, will
141        iterate the dataset until all inputs are exhausted.
142      log_progress: Whether to log evaluation progress, defaults to True.
143    """
144    # The number of evals to run for.
145    self._num_evals = num_evals
146    self._evals_completed = None
147    self._log_progress = log_progress
148    # Reduce logging frequency if there are 20 or more evaluations.
149    self._log_frequency = (1 if (num_evals is None or num_evals < 20) else
150                           math.floor(num_evals / 10.))
151
152  def _set_evals_completed_tensor(self, updated_eval_step):
153    self._evals_completed = updated_eval_step
154
155  def before_run(self, run_context):
156    return session_run_hook.SessionRunArgs(
157        {'evals_completed': self._evals_completed})
158
159  def after_run(self, run_context, run_values):
160    evals_completed = run_values.results['evals_completed']
161    if self._log_progress:
162      if self._num_evals is None:
163        logging.info('Evaluation [%d]', evals_completed)
164      else:
165        if ((evals_completed % self._log_frequency) == 0 or
166            (self._num_evals == evals_completed)):
167          logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals)
168    if self._num_evals is not None and evals_completed >= self._num_evals:
169      run_context.request_stop()
170
171
172def _evaluate_once(checkpoint_path,
173                   master='',
174                   scaffold=None,
175                   eval_ops=None,
176                   feed_dict=None,
177                   final_ops=None,
178                   final_ops_feed_dict=None,
179                   hooks=None,
180                   config=None):
181  """Evaluates the model at the given checkpoint path.
182
183  During a single evaluation, the `eval_ops` is run until the session is
184  interrupted or requested to finish. This is typically requested via a
185  `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running
186  the requested number of times.
187
188  Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of
189  `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is
190  evaluated a single time after `eval_ops` has finished running and the fetched
191  values of `final_ops` are returned. If `final_ops` is left as `None`, then
192  `None` is returned.
193
194  One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record
195  summaries after the `eval_ops` have run. If `eval_ops` is `None`, the
196  summaries run immediately after the model checkpoint has been restored.
197
198  Note that `evaluate_once` creates a local variable used to track the number of
199  evaluations run via `tf.contrib.training.get_or_create_eval_step`.
200  Consequently, if a custom local init op is provided via a `scaffold`, the
201  caller should ensure that the local init op also initializes the eval step.
202
203  Args:
204    checkpoint_path: The path to a checkpoint to use for evaluation.
205    master: The BNS address of the TensorFlow master.
206    scaffold: An tf.compat.v1.train.Scaffold instance for initializing variables
207      and restoring variables. Note that `scaffold.init_fn` is used by the
208      function to restore the checkpoint. If you supply a custom init_fn, then
209      it must also take care of restoring the model from its checkpoint.
210    eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to
211      `Tensors`, which is run until the session is requested to stop, commonly
212      done by a `tf.contrib.training.StopAfterNEvalsHook`.
213    feed_dict: The feed dictionary to use when executing the `eval_ops`.
214    final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names
215      to `Tensors`.
216    final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`.
217    hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside
218      the evaluation loop.
219    config: An instance of `tf.compat.v1.ConfigProto` that will be used to
220      configure the `Session`. If left as `None`, the default will be used.
221
222  Returns:
223    The fetched values of `final_ops` or `None` if `final_ops` is `None`.
224  """
225  eval_step = _get_or_create_eval_step()
226
227  # Prepare the run hooks.
228  hooks = list(hooks or [])
229
230  if eval_ops is not None:
231    if any(isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks):
232      steps_per_run_variable = \
233          basic_session_run_hooks.get_or_create_steps_per_run_variable()
234      update_eval_step = state_ops.assign_add(
235          eval_step,
236          math_ops.cast(steps_per_run_variable, dtype=eval_step.dtype),
237          use_locking=True)
238    else:
239      update_eval_step = state_ops.assign_add(eval_step, 1, use_locking=True)
240
241    if isinstance(eval_ops, dict):
242      eval_ops['update_eval_step'] = update_eval_step
243    elif isinstance(eval_ops, (tuple, list)):
244      eval_ops = list(eval_ops) + [update_eval_step]
245    else:
246      eval_ops = [eval_ops, update_eval_step]
247
248    eval_step_value = _get_latest_eval_step_value(eval_ops)
249
250    for h in hooks:
251      if isinstance(h, (_StopAfterNEvalsHook, _MultiStepStopAfterNEvalsHook)):
252        h._set_evals_completed_tensor(eval_step_value)  # pylint: disable=protected-access
253
254  logging.info('Starting evaluation at ' +
255               time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime()))
256  start = time.time()
257  # Prepare the session creator.
258  session_creator = monitored_session.ChiefSessionCreator(
259      scaffold=scaffold,
260      checkpoint_filename_with_path=checkpoint_path,
261      master=master,
262      config=config)
263
264  final_ops_hook = basic_session_run_hooks.FinalOpsHook(final_ops,
265                                                        final_ops_feed_dict)
266  hooks.append(final_ops_hook)
267
268  with monitored_session.MonitoredSession(
269      session_creator=session_creator, hooks=hooks) as session:
270    if eval_ops is not None:
271      while not session.should_stop():
272        session.run(eval_ops, feed_dict)
273  logging.info('Inference Time : {:0.5f}s'.format(time.time() - start))
274
275  logging.info('Finished evaluation at ' +
276               time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()))
277  return final_ops_hook.final_ops_values
278