• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Benchmarks for static optimizations."""
16from __future__ import absolute_import
17from __future__ import division
18from __future__ import print_function
19
20import time
21
22import numpy as np
23
24from tensorflow.python.client import session
25from tensorflow.python.data.ops import dataset_ops
26from tensorflow.python.framework import ops
27from tensorflow.python.ops import math_ops
28from tensorflow.python.platform import test
29
30
31# TODO(b/119837791): Add eager benchmarks too.
32class OptimizationBenchmark(test.Benchmark):
33  """Benchmarks for static optimizations."""
34
35  def benchmark_map_fusion(self):
36    """Evaluates performance map of fusion."""
37
38    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
39    for chain_length in chain_lengths:
40      self._benchmark_map_fusion(chain_length, False)
41      self._benchmark_map_fusion(chain_length, True)
42
43  def _benchmark_map_fusion(self, chain_length, optimize_dataset):
44    with ops.Graph().as_default():
45      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
46      for _ in range(chain_length):
47        dataset = dataset.map(lambda x: x)
48      if optimize_dataset:
49        options = dataset_ops.Options()
50        options.experimental_optimization.apply_default_optimizations = False
51        options.experimental_optimization.map_fusion = True
52        dataset = dataset.with_options(options)
53
54      iterator = dataset_ops.make_one_shot_iterator(dataset)
55      next_element = iterator.get_next()
56
57      with session.Session() as sess:
58        for _ in range(5):
59          sess.run(next_element.op)
60        deltas = []
61        for _ in range(100):
62          start = time.time()
63          for _ in range(100):
64            sess.run(next_element.op)
65          end = time.time()
66          deltas.append(end - start)
67
68        median_wall_time = np.median(deltas) / 100
69        opt_mark = "opt" if optimize_dataset else "noopt"
70        self.report_benchmark(
71            iters=100,
72            wall_time=median_wall_time,
73            name="map_fusion_{}_chain_length_{}".format(
74                opt_mark, chain_length))
75
76  def benchmark_map_and_filter_fusion(self):
77    """Evaluates performance map of fusion."""
78
79    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
80    for chain_length in chain_lengths:
81      self._benchmark_map_and_filter_fusion(chain_length, False)
82      self._benchmark_map_and_filter_fusion(chain_length, True)
83
84  def _benchmark_map_and_filter_fusion(self, chain_length, optimize_dataset):
85    with ops.Graph().as_default():
86      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
87      for _ in range(chain_length):
88        dataset = dataset.map(lambda x: x + 5).filter(
89            lambda x: math_ops.greater_equal(x - 5, 0))
90      if optimize_dataset:
91        options = dataset_ops.Options()
92        options.experimental_optimization.apply_default_optimizations = False
93        options.experimental_optimization.map_and_filter_fusion = True
94        dataset = dataset.with_options(options)
95      iterator = dataset_ops.make_one_shot_iterator(dataset)
96      next_element = iterator.get_next()
97
98      with session.Session() as sess:
99        for _ in range(10):
100          sess.run(next_element.op)
101        deltas = []
102        for _ in range(100):
103          start = time.time()
104          for _ in range(100):
105            sess.run(next_element.op)
106          end = time.time()
107          deltas.append(end - start)
108
109        median_wall_time = np.median(deltas) / 100
110        opt_mark = "opt" if optimize_dataset else "noopt"
111        self.report_benchmark(
112            iters=100,
113            wall_time=median_wall_time,
114            name="map_and_filter_fusion_{}_chain_length_{}".format(
115                opt_mark, chain_length))
116
117  # This benchmark compares the performance of pipeline with multiple chained
118  # filter with and without filter fusion.
119  def benchmark_filter_fusion(self):
120    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
121    for chain_length in chain_lengths:
122      self._benchmark_filter_fusion(chain_length, False)
123      self._benchmark_filter_fusion(chain_length, True)
124
125  def _benchmark_filter_fusion(self, chain_length, optimize_dataset):
126    with ops.Graph().as_default():
127      dataset = dataset_ops.Dataset.from_tensors(5).repeat(None)
128      for _ in range(chain_length):
129        dataset = dataset.filter(lambda x: math_ops.greater_equal(x - 5, 0))
130      if optimize_dataset:
131        options = dataset_ops.Options()
132        options.experimental_optimization.apply_default_optimizations = False
133        options.experimental_optimization.filter_fusion = True
134        dataset = dataset.with_options(options)
135
136      iterator = dataset_ops.make_one_shot_iterator(dataset)
137      next_element = iterator.get_next()
138
139      with session.Session() as sess:
140        for _ in range(10):
141          sess.run(next_element.op)
142        deltas = []
143        for _ in range(100):
144          start = time.time()
145          for _ in range(100):
146            sess.run(next_element.op)
147          end = time.time()
148          deltas.append(end - start)
149
150        median_wall_time = np.median(deltas) / 100
151        opt_mark = "opt" if optimize_dataset else "no-opt"
152        self.report_benchmark(
153            iters=1000,
154            wall_time=median_wall_time,
155            name="chain_length_{}_{}".format(opt_mark, chain_length))
156
157
158if __name__ == "__main__":
159  test.main()
160