1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Contains functions for evaluation and summarization of metrics.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import math 22import time 23 24from tensorflow.python.framework import dtypes 25from tensorflow.python.framework import ops 26from tensorflow.python.ops import array_ops 27from tensorflow.python.ops import init_ops 28from tensorflow.python.ops import math_ops 29from tensorflow.python.ops import state_ops 30from tensorflow.python.ops import variable_scope 31from tensorflow.python.platform import tf_logging as logging 32from tensorflow.python.training import basic_session_run_hooks 33from tensorflow.python.training import monitored_session 34from tensorflow.python.training import session_run_hook 35 36 37def _get_or_create_eval_step(): 38 """Gets or creates the eval step `Tensor`. 39 40 Returns: 41 A `Tensor` representing a counter for the evaluation step. 42 43 Raises: 44 ValueError: If multiple `Tensors` have been added to the 45 `tf.GraphKeys.EVAL_STEP` collection. 46 """ 47 graph = ops.get_default_graph() 48 eval_steps = graph.get_collection(ops.GraphKeys.EVAL_STEP) 49 if len(eval_steps) == 1: 50 return eval_steps[0] 51 elif len(eval_steps) > 1: 52 raise ValueError('Multiple tensors added to tf.GraphKeys.EVAL_STEP') 53 else: 54 counter = variable_scope.get_variable( 55 'eval_step', 56 shape=[], 57 dtype=dtypes.int64, 58 initializer=init_ops.zeros_initializer(), 59 trainable=False, 60 collections=[ops.GraphKeys.LOCAL_VARIABLES, ops.GraphKeys.EVAL_STEP]) 61 return counter 62 63 64def _get_latest_eval_step_value(update_ops): 65 """Gets the eval step `Tensor` value after running `update_ops`. 66 67 Args: 68 update_ops: A list of `Tensors` or a dictionary of names to `Tensors`, which 69 are run before reading the eval step value. 70 71 Returns: 72 A `Tensor` representing the value for the evaluation step. 73 """ 74 if isinstance(update_ops, dict): 75 update_ops = list(update_ops.values()) 76 77 with ops.control_dependencies(update_ops): 78 return array_ops.identity(_get_or_create_eval_step().read_value()) 79 80 81class _MultiStepStopAfterNEvalsHook(session_run_hook.SessionRunHook): 82 """Run hook used by the evaluation routines to run the `eval_ops` N times.""" 83 84 def __init__(self, num_evals, steps_per_run=1): 85 """Constructs the run hook. 86 87 Args: 88 num_evals: The number of evaluations to run for. if set to None, will 89 iterate the dataset until all inputs are exhausted. 90 steps_per_run: Number of steps executed per run call. 91 """ 92 self._num_evals = num_evals 93 self._evals_completed = None 94 self._steps_per_run_initial_value = steps_per_run 95 96 def _set_evals_completed_tensor(self, updated_eval_step): 97 self._evals_completed = updated_eval_step 98 99 def begin(self): 100 self._steps_per_run_variable = \ 101 basic_session_run_hooks.get_or_create_steps_per_run_variable() 102 103 def after_create_session(self, session, coord): 104 # Update number of steps to run in the first run call 105 if self._num_evals is None: 106 steps = self._steps_per_run_initial_value 107 else: 108 steps = min(self._steps_per_run_initial_value, self._num_evals) 109 self._steps_per_run_variable.load(steps, session=session) 110 111 def before_run(self, run_context): 112 return session_run_hook.SessionRunArgs( 113 {'evals_completed': self._evals_completed}) 114 115 def after_run(self, run_context, run_values): 116 evals_completed = run_values.results['evals_completed'] 117 # Update number of steps to run in the next iteration 118 if self._num_evals is None: 119 steps = self._steps_per_run_initial_value 120 else: 121 steps = min(self._num_evals - evals_completed, 122 self._steps_per_run_initial_value) 123 self._steps_per_run_variable.load(steps, session=run_context.session) 124 125 if self._num_evals is None: 126 logging.info('Evaluation [%d]', evals_completed) 127 else: 128 logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) 129 if self._num_evals is not None and evals_completed >= self._num_evals: 130 run_context.request_stop() 131 132 133class _StopAfterNEvalsHook(session_run_hook.SessionRunHook): 134 """Run hook used by the evaluation routines to run the `eval_ops` N times.""" 135 136 def __init__(self, num_evals, log_progress=True): 137 """Constructs the run hook. 138 139 Args: 140 num_evals: The number of evaluations to run for. if set to None, will 141 iterate the dataset until all inputs are exhausted. 142 log_progress: Whether to log evaluation progress, defaults to True. 143 """ 144 # The number of evals to run for. 145 self._num_evals = num_evals 146 self._evals_completed = None 147 self._log_progress = log_progress 148 # Reduce logging frequency if there are 20 or more evaluations. 149 self._log_frequency = (1 if (num_evals is None or num_evals < 20) else 150 math.floor(num_evals / 10.)) 151 152 def _set_evals_completed_tensor(self, updated_eval_step): 153 self._evals_completed = updated_eval_step 154 155 def before_run(self, run_context): 156 return session_run_hook.SessionRunArgs( 157 {'evals_completed': self._evals_completed}) 158 159 def after_run(self, run_context, run_values): 160 evals_completed = run_values.results['evals_completed'] 161 if self._log_progress: 162 if self._num_evals is None: 163 logging.info('Evaluation [%d]', evals_completed) 164 else: 165 if ((evals_completed % self._log_frequency) == 0 or 166 (self._num_evals == evals_completed)): 167 logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) 168 if self._num_evals is not None and evals_completed >= self._num_evals: 169 run_context.request_stop() 170 171 172def _evaluate_once(checkpoint_path, 173 master='', 174 scaffold=None, 175 eval_ops=None, 176 feed_dict=None, 177 final_ops=None, 178 final_ops_feed_dict=None, 179 hooks=None, 180 config=None): 181 """Evaluates the model at the given checkpoint path. 182 183 During a single evaluation, the `eval_ops` is run until the session is 184 interrupted or requested to finish. This is typically requested via a 185 `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running 186 the requested number of times. 187 188 Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of 189 `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is 190 evaluated a single time after `eval_ops` has finished running and the fetched 191 values of `final_ops` are returned. If `final_ops` is left as `None`, then 192 `None` is returned. 193 194 One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record 195 summaries after the `eval_ops` have run. If `eval_ops` is `None`, the 196 summaries run immediately after the model checkpoint has been restored. 197 198 Note that `evaluate_once` creates a local variable used to track the number of 199 evaluations run via `tf.contrib.training.get_or_create_eval_step`. 200 Consequently, if a custom local init op is provided via a `scaffold`, the 201 caller should ensure that the local init op also initializes the eval step. 202 203 Args: 204 checkpoint_path: The path to a checkpoint to use for evaluation. 205 master: The BNS address of the TensorFlow master. 206 scaffold: An tf.compat.v1.train.Scaffold instance for initializing variables 207 and restoring variables. Note that `scaffold.init_fn` is used by the 208 function to restore the checkpoint. If you supply a custom init_fn, then 209 it must also take care of restoring the model from its checkpoint. 210 eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to 211 `Tensors`, which is run until the session is requested to stop, commonly 212 done by a `tf.contrib.training.StopAfterNEvalsHook`. 213 feed_dict: The feed dictionary to use when executing the `eval_ops`. 214 final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names 215 to `Tensors`. 216 final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`. 217 hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside 218 the evaluation loop. 219 config: An instance of `tf.compat.v1.ConfigProto` that will be used to 220 configure the `Session`. If left as `None`, the default will be used. 221 222 Returns: 223 The fetched values of `final_ops` or `None` if `final_ops` is `None`. 224 """ 225 eval_step = _get_or_create_eval_step() 226 227 # Prepare the run hooks. 228 hooks = list(hooks or []) 229 230 if eval_ops is not None: 231 if any(isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks): 232 steps_per_run_variable = \ 233 basic_session_run_hooks.get_or_create_steps_per_run_variable() 234 update_eval_step = state_ops.assign_add( 235 eval_step, 236 math_ops.cast(steps_per_run_variable, dtype=eval_step.dtype), 237 use_locking=True) 238 else: 239 update_eval_step = state_ops.assign_add(eval_step, 1, use_locking=True) 240 241 if isinstance(eval_ops, dict): 242 eval_ops['update_eval_step'] = update_eval_step 243 elif isinstance(eval_ops, (tuple, list)): 244 eval_ops = list(eval_ops) + [update_eval_step] 245 else: 246 eval_ops = [eval_ops, update_eval_step] 247 248 eval_step_value = _get_latest_eval_step_value(eval_ops) 249 250 for h in hooks: 251 if isinstance(h, (_StopAfterNEvalsHook, _MultiStepStopAfterNEvalsHook)): 252 h._set_evals_completed_tensor(eval_step_value) # pylint: disable=protected-access 253 254 logging.info('Starting evaluation at ' + 255 time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime())) 256 start = time.time() 257 # Prepare the session creator. 258 session_creator = monitored_session.ChiefSessionCreator( 259 scaffold=scaffold, 260 checkpoint_filename_with_path=checkpoint_path, 261 master=master, 262 config=config) 263 264 final_ops_hook = basic_session_run_hooks.FinalOpsHook(final_ops, 265 final_ops_feed_dict) 266 hooks.append(final_ops_hook) 267 268 with monitored_session.MonitoredSession( 269 session_creator=session_creator, hooks=hooks) as session: 270 if eval_ops is not None: 271 while not session.should_stop(): 272 session.run(eval_ops, feed_dict) 273 logging.info('Inference Time : {:0.5f}s'.format(time.time() - start)) 274 275 logging.info('Finished evaluation at ' + 276 time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime())) 277 return final_ops_hook.final_ops_values 278